Skip to content
Open
Show file tree
Hide file tree
Changes from 8 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions deploy/helm/charts/platform/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,7 @@ Kubernetes: `>=1.30.0-0`
| dynamo-operator.dynamo.serviceMesh.istio.clientCertificate | string | `""` | Path (in the istio-proxy sidecar's filesystem) to the client TLS certificate used for mTLS. REQUIRED when tlsMode is "MUTUAL"; ignored for other modes. |
| dynamo-operator.dynamo.serviceMesh.istio.privateKey | string | `""` | Path (in the istio-proxy sidecar's filesystem) to the client TLS private key used for mTLS. REQUIRED when tlsMode is "MUTUAL"; ignored for other modes. |
| dynamo-operator.dynamo.serviceMesh.istio.caCertificates | string | `""` | Optional path (in the istio-proxy sidecar's filesystem) to CA certificates used to verify the EPP server certificate. Used only when tlsMode is "MUTUAL". |
| dynamo-operator.dynamo.multiCluster.mcsImplementation | string | `""` | MCS implementation that imports the exports: "mcs" for the SIG Multicluster API or "karmada". For each DGD with a DynamoPoolRelay of the same name, the operator then creates a ServiceExport for `<DGD name>-pool-relay` and for each frontend component's Service. Needs the multicluster.x-k8s.io/v1alpha1 ServiceExport API in this cluster. Empty exports nothing; clearing it leaves earlier exports in place until their DGDs are deleted. |
| dynamo-operator.dynamo.metrics.prometheusEndpoint | string | `""` | Endpoint that services can use to retrieve metrics. If set, dynamo operator will automatically inject the PROMETHEUS_ENDPOINT environment variable into services it manages. Users can override the value of the PROMETHEUS_ENDPOINT environment variable by modifying the corresponding deployment's environment variables |
| dynamo-operator.dynamo.mpiRun.secretName | string | `"mpi-run-ssh-secret"` | Name of the secret containing the SSH key for MPI Run |
| dynamo-operator.webhook.certificateSecret.name | string | `"webhook-server-cert"` | Name of the Kubernetes secret containing webhook TLS certificates. The secret must contain three keys: tls.crt (server certificate), tls.key (server private key), and ca.crt (Certificate Authority certificate). |
Expand Down
16 changes: 16 additions & 0 deletions deploy/helm/charts/platform/components/operator/files/role.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -76,6 +76,12 @@ rules:
- patch
- update
- watch
- apiGroups:
- about.k8s.io
resources:
- clusterproperties
verbs:
- get
- apiGroups:
- apiextensions.k8s.io
resources:
Expand Down Expand Up @@ -255,6 +261,16 @@ rules:
- patch
- update
- watch
- apiGroups:
- multicluster.x-k8s.io
resources:
- serviceexports
verbs:
- create
- delete
- get
- list
- watch
- apiGroups:
- networking.istio.io
resources:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -182,6 +182,11 @@ data:
{{- end }}
{{- end }}
{{- end }}
{{- $mcsImplementation := (.Values.dynamo.multiCluster | default dict).mcsImplementation | default "" }}
{{- if $mcsImplementation }}
multiCluster:
mcsImplementation: {{ $mcsImplementation | quote }}
{{- end }}
{{- if not .Values.namespaceRestriction.enabled }}
rbac:
plannerClusterRoleName: {{ include "dynamo-operator.fullname" . }}-planner
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
# SPDX-License-Identifier: Apache-2.0

suite: multi-cluster operator configuration
templates:
- charts/dynamo-operator/templates/operator-config.yaml
tests:
- it: exports nothing by default
asserts:
- notMatchRegex:
path: data["config.yaml"]
pattern: "multiCluster:"

- it: passes the MCS implementation to the operator
set:
dynamo-operator.dynamo.multiCluster.mcsImplementation: karmada
asserts:
- matchRegex:
path: data["config.yaml"]
pattern: 'multiCluster:\n\s+mcsImplementation: "karmada"'
12 changes: 12 additions & 0 deletions deploy/helm/charts/platform/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -247,6 +247,18 @@ dynamo-operator:
# when tlsMode is "MUTUAL".
caCertificates: ""

# Exports of Global Router pools through the Kubernetes Multi-Cluster
# Services (MCS) API, for the Global Discovery Service's Kubernetes stores.
multiCluster:
# -- MCS implementation that imports the exports: "mcs" for the SIG
# Multicluster API or "karmada". For each DGD with a DynamoPoolRelay of
# the same name, the operator then creates a ServiceExport for
# `<DGD name>-pool-relay` and for each frontend component's Service.
# Needs the multicluster.x-k8s.io/v1alpha1 ServiceExport API in this
# cluster. Empty exports nothing; clearing it leaves earlier exports in
# place until their DGDs are deleted.
mcsImplementation: ""

# Metrics configuration
metrics:
# -- Endpoint that services can use to retrieve metrics. If set, dynamo operator will automatically inject the PROMETHEUS_ENDPOINT environment variable into services it manages. Users can override the value of the PROMETHEUS_ENDPOINT environment variable by modifying the corresponding deployment's environment variables
Expand Down
29 changes: 29 additions & 0 deletions deploy/operator/api/config/v1alpha1/types.go
Original file line number Diff line number Diff line change
Expand Up @@ -62,6 +62,9 @@ type OperatorConfiguration struct {
// Discovery backend configuration
Discovery DiscoveryConfiguration `json:"discovery"`

// MultiCluster configures exporting Global Router pools to other clusters.
MultiCluster MultiClusterConfiguration `json:"multiCluster,omitempty"`

// GPU discovery configuration
GPU GPUConfiguration `json:"gpu"`

Expand Down Expand Up @@ -397,6 +400,32 @@ type DiscoveryConfiguration struct {
Backend DiscoveryBackend `json:"backend"`
}

// MultiClusterConfiguration configures how the operator exports Global Router
// pools through the Kubernetes Multi-Cluster Services (MCS) API, for the Global
// Discovery Service's Kubernetes stores. The operator exports a pool only when
// MCSImplementation is set and a DynamoPoolRelay with its DGD's name exists.
type MultiClusterConfiguration struct {
// MCSImplementation is the MCS implementation that imports the exports. Empty,
// the default, exports nothing: Global Router pools are then discovered without
// Kubernetes. Clearing it leaves earlier exports in place until their DGDs are
// deleted.
MCSImplementation MCSImplementation `json:"mcsImplementation,omitempty"`
}

// MCSImplementation identifies an implementation of the Multi-Cluster Services API.
type MCSImplementation string

const (
// MCSImplementationNone exports nothing.
MCSImplementationNone MCSImplementation = ""
// MCSImplementationStandard is an implementation of the SIG Multicluster API that
// imports Services into ServiceImports and EndpointSlices with the standard labels.
MCSImplementationStandard MCSImplementation = "mcs"
// MCSImplementationKarmada is Karmada, which reads the same ServiceExport in
// member clusters but names imported EndpointSlices after the source cluster.
MCSImplementationKarmada MCSImplementation = "karmada"
)

// DiscoveryBackend is the type for the discovery backend.
type DiscoveryBackend string

Expand Down
16 changes: 16 additions & 0 deletions deploy/operator/api/config/v1alpha1/zz_generated.deepcopy.go

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

13 changes: 13 additions & 0 deletions deploy/operator/api/config/validation/validation.go
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ func ValidateOperatorConfiguration(config *configv1alpha1.OperatorConfiguration)
validateMPI(&config.MPI, field.NewPath("mpi")),
validateInfrastructure(&config.Infrastructure, field.NewPath("infrastructure")),
validateDiscovery(&config.Discovery, field.NewPath("discovery")),
validateMultiCluster(&config.MultiCluster, field.NewPath("multiCluster")),
validateRBAC(config),
validateOrchestrators(&config.Orchestrators, field.NewPath("orchestrators")),
validateIngress(&config.Ingress, field.NewPath("ingress")),
Expand Down Expand Up @@ -209,6 +210,18 @@ func validateDiscovery(discovery *configv1alpha1.DiscoveryConfiguration, fldPath
return allErrs
}

func validateMultiCluster(multiCluster *configv1alpha1.MultiClusterConfiguration, fldPath *field.Path) field.ErrorList {
switch multiCluster.MCSImplementation {
case configv1alpha1.MCSImplementationNone, configv1alpha1.MCSImplementationStandard, configv1alpha1.MCSImplementationKarmada:
return nil
}
return field.ErrorList{field.NotSupported(
fldPath.Child("mcsImplementation"),
multiCluster.MCSImplementation,
[]string{"", "mcs", "karmada"},
)}
}

// validateRBAC is mode-aware: validates RBAC fields based on namespace mode.
func validateRBAC(config *configv1alpha1.OperatorConfiguration) field.ErrorList {
allErrs := field.ErrorList{}
Expand Down
23 changes: 23 additions & 0 deletions deploy/operator/api/config/validation/validation_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,29 @@ func TestValidateOperatorConfiguration_InvalidDiscoveryBackend(t *testing.T) {
}
}

func TestValidateOperatorConfiguration_MCSImplementation(t *testing.T) {
for _, tc := range []struct {
name string
implementation configv1alpha1.MCSImplementation
wantErrors int
}{
{name: "none", implementation: configv1alpha1.MCSImplementationNone},
{name: "standard MCS", implementation: configv1alpha1.MCSImplementationStandard},
{name: "Karmada", implementation: configv1alpha1.MCSImplementationKarmada},
{name: "unknown", implementation: "gke", wantErrors: 1},
} {
t.Run(tc.name, func(t *testing.T) {
t.Log("Validate a configuration that exports Global Router pools through this implementation")
cfg := validConfig()
cfg.MultiCluster.MCSImplementation = tc.implementation
errs := ValidateOperatorConfiguration(cfg)
if len(errs) != tc.wantErrors {
t.Errorf("expected %d errors, got %d: %v", tc.wantErrors, len(errs), errs)
}
})
}
}

func TestValidateOperatorConfiguration_ClusterWideMissingPlannerRole(t *testing.T) {
cfg := validConfig()
cfg.RBAC.PlannerClusterRoleName = ""
Expand Down
Loading
Loading