diff --git a/deploy/helm/charts/platform/README.md b/deploy/helm/charts/platform/README.md index d26dcf732710..0be5c7190010 100644 --- a/deploy/helm/charts/platform/README.md +++ b/deploy/helm/charts/platform/README.md @@ -779,6 +779,7 @@ Kubernetes: `>=1.30.0-0` | dynamo-operator.dynamo.serviceMesh.istio.clientCertificate | string | `""` | Path (in the istio-proxy sidecar's filesystem) to the client TLS certificate used for mTLS. REQUIRED when tlsMode is "MUTUAL"; ignored for other modes. | | dynamo-operator.dynamo.serviceMesh.istio.privateKey | string | `""` | Path (in the istio-proxy sidecar's filesystem) to the client TLS private key used for mTLS. REQUIRED when tlsMode is "MUTUAL"; ignored for other modes. | | dynamo-operator.dynamo.serviceMesh.istio.caCertificates | string | `""` | Optional path (in the istio-proxy sidecar's filesystem) to CA certificates used to verify the EPP server certificate. Used only when tlsMode is "MUTUAL". | +| dynamo-operator.dynamo.multiCluster.mcsImplementation | string | `""` | MCS implementation that imports the exports: "mcs" for the SIG Multicluster API or "karmada". For each DGD with a DynamoPoolRelay of the same name, the operator then creates a ServiceExport for `-pool-relay` and for each frontend component's Service. Needs the multicluster.x-k8s.io/v1alpha1 ServiceExport API in this cluster. Empty exports nothing; clearing it leaves earlier exports in place until their DynamoPoolRelays are deleted. | | dynamo-operator.dynamo.metrics.prometheusEndpoint | string | `""` | Endpoint that services can use to retrieve metrics. If set, dynamo operator will automatically inject the PROMETHEUS_ENDPOINT environment variable into services it manages. Users can override the value of the PROMETHEUS_ENDPOINT environment variable by modifying the corresponding deployment's environment variables | | dynamo-operator.dynamo.mpiRun.secretName | string | `"mpi-run-ssh-secret"` | Name of the secret containing the SSH key for MPI Run | | dynamo-operator.webhook.certificateSecret.name | string | `"webhook-server-cert"` | Name of the Kubernetes secret containing webhook TLS certificates. The secret must contain three keys: tls.crt (server certificate), tls.key (server private key), and ca.crt (Certificate Authority certificate). | diff --git a/deploy/helm/charts/platform/components/operator/files/role.yaml b/deploy/helm/charts/platform/components/operator/files/role.yaml index 97258d1d9b3f..d1f194dac769 100644 --- a/deploy/helm/charts/platform/components/operator/files/role.yaml +++ b/deploy/helm/charts/platform/components/operator/files/role.yaml @@ -77,6 +77,14 @@ rules: - patch - update - watch +- apiGroups: + - about.k8s.io + resources: + - clusterproperties + verbs: + - get + - list + - watch - apiGroups: - apiextensions.k8s.io resources: @@ -256,6 +264,17 @@ rules: - patch - update - watch +- apiGroups: + - multicluster.x-k8s.io + resources: + - serviceexports + verbs: + - create + - delete + - get + - list + - patch + - watch - apiGroups: - networking.istio.io resources: diff --git a/deploy/helm/charts/platform/components/operator/templates/manager-rbac.yaml b/deploy/helm/charts/platform/components/operator/templates/manager-rbac.yaml index 162adc60ef4b..f538241b0a43 100644 --- a/deploy/helm/charts/platform/components/operator/templates/manager-rbac.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/manager-rbac.yaml @@ -314,3 +314,42 @@ subjects: - kind: ServiceAccount name: '{{ include "dynamo-operator.fullname" . }}-controller-manager' namespace: '{{ .Release.Namespace }}' +--- +# Helm keeps ClusterProperty access separate because a namespace-restricted Role cannot grant it. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-clusterproperty-reader + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: dynamo-operator + app.kubernetes.io/part-of: dynamo-operator + {{- include "dynamo-operator.labels" . | nindent 4 }} +rules: +- apiGroups: + - about.k8s.io + resources: + - clusterproperties + verbs: + - get + - list + - watch +--- +# Helm binds the separate cluster-scoped ClusterProperty grant to the controller ServiceAccount. +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-clusterproperty-reader-binding + labels: + app.kubernetes.io/component: rbac + app.kubernetes.io/created-by: dynamo-operator + app.kubernetes.io/part-of: dynamo-operator + {{- include "dynamo-operator.labels" . | nindent 4 }} +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: {{ include "dynamo-operator.fullname" . }}-{{ .Release.Namespace }}-clusterproperty-reader +subjects: +- kind: ServiceAccount + name: '{{ include "dynamo-operator.fullname" . }}-controller-manager' + namespace: '{{ .Release.Namespace }}' diff --git a/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml b/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml index fd8470b34862..b3cd0ad06797 100644 --- a/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml +++ b/deploy/helm/charts/platform/components/operator/templates/operator-config.yaml @@ -182,6 +182,11 @@ data: {{- end }} {{- end }} {{- end }} + {{- $mcsImplementation := (.Values.dynamo.multiCluster | default dict).mcsImplementation | default "" }} + {{- if $mcsImplementation }} + multiCluster: + mcsImplementation: {{ $mcsImplementation | quote }} + {{- end }} {{- if not .Values.namespaceRestriction.enabled }} rbac: plannerClusterRoleName: {{ include "dynamo-operator.fullname" . }}-planner diff --git a/deploy/helm/charts/platform/tests/multi_cluster_config_test.yaml b/deploy/helm/charts/platform/tests/multi_cluster_config_test.yaml new file mode 100644 index 000000000000..afa6d286a15f --- /dev/null +++ b/deploy/helm/charts/platform/tests/multi_cluster_config_test.yaml @@ -0,0 +1,20 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +suite: multi-cluster operator configuration +templates: + - charts/dynamo-operator/templates/operator-config.yaml +tests: + - it: exports nothing by default + asserts: + - notMatchRegex: + path: data["config.yaml"] + pattern: "multiCluster:" + + - it: passes the MCS implementation to the operator + set: + dynamo-operator.dynamo.multiCluster.mcsImplementation: karmada + asserts: + - matchRegex: + path: data["config.yaml"] + pattern: 'multiCluster:\n\s+mcsImplementation: "karmada"' diff --git a/deploy/helm/charts/platform/values.yaml b/deploy/helm/charts/platform/values.yaml index 71c17fbcbae4..59e9c2685ebe 100644 --- a/deploy/helm/charts/platform/values.yaml +++ b/deploy/helm/charts/platform/values.yaml @@ -282,6 +282,18 @@ dynamo-operator: # when tlsMode is "MUTUAL". caCertificates: "" + # Exports of Global Router pools through the Kubernetes Multi-Cluster + # Services (MCS) API, for the Global Discovery Service's Kubernetes stores. + multiCluster: + # -- MCS implementation that imports the exports: "mcs" for the SIG + # Multicluster API or "karmada". For each DGD with a DynamoPoolRelay of + # the same name, the operator then creates a ServiceExport for + # `-pool-relay` and for each frontend component's Service. + # Needs the multicluster.x-k8s.io/v1alpha1 ServiceExport API in this + # cluster. Empty exports nothing; clearing it leaves earlier exports in + # place until their DynamoPoolRelays are deleted. + mcsImplementation: "" + # Metrics configuration metrics: # -- Endpoint that services can use to retrieve metrics. If set, dynamo operator will automatically inject the PROMETHEUS_ENDPOINT environment variable into services it manages. Users can override the value of the PROMETHEUS_ENDPOINT environment variable by modifying the corresponding deployment's environment variables diff --git a/deploy/operator/api/config/v1alpha1/types.go b/deploy/operator/api/config/v1alpha1/types.go index 01d310e3016d..0539290cf8cf 100644 --- a/deploy/operator/api/config/v1alpha1/types.go +++ b/deploy/operator/api/config/v1alpha1/types.go @@ -62,6 +62,9 @@ type OperatorConfiguration struct { // Discovery backend configuration Discovery DiscoveryConfiguration `json:"discovery"` + // MultiCluster configures exporting Global Router pools to other clusters. + MultiCluster MultiClusterConfiguration `json:"multiCluster,omitempty"` + // GPU discovery configuration GPU GPUConfiguration `json:"gpu"` @@ -397,6 +400,32 @@ type DiscoveryConfiguration struct { Backend DiscoveryBackend `json:"backend"` } +// MultiClusterConfiguration configures how the operator exports Global Router +// pools through the Kubernetes Multi-Cluster Services (MCS) API, for the Global +// Discovery Service's Kubernetes stores. The operator exports a pool only when +// MCSImplementation is set and a DynamoPoolRelay with its DGD's name exists. +type MultiClusterConfiguration struct { + // MCSImplementation is the MCS implementation that imports the exports. Empty, + // the default, exports nothing: Global Router pools are then discovered without + // Kubernetes. Clearing it leaves earlier exports in place until their + // DynamoPoolRelays are deleted. + MCSImplementation MCSImplementation `json:"mcsImplementation,omitempty"` +} + +// MCSImplementation identifies an implementation of the Multi-Cluster Services API. +type MCSImplementation string + +const ( + // MCSImplementationNone exports nothing. + MCSImplementationNone MCSImplementation = "" + // MCSImplementationStandard is an implementation of the SIG Multicluster API that + // imports Services into ServiceImports and EndpointSlices with the standard labels. + MCSImplementationStandard MCSImplementation = "mcs" + // MCSImplementationKarmada is Karmada, which reads the same ServiceExport in + // member clusters but names imported EndpointSlices after the source cluster. + MCSImplementationKarmada MCSImplementation = "karmada" +) + // DiscoveryBackend is the type for the discovery backend. type DiscoveryBackend string diff --git a/deploy/operator/api/config/v1alpha1/zz_generated.deepcopy.go b/deploy/operator/api/config/v1alpha1/zz_generated.deepcopy.go index 952d37192b26..a72edb97db00 100644 --- a/deploy/operator/api/config/v1alpha1/zz_generated.deepcopy.go +++ b/deploy/operator/api/config/v1alpha1/zz_generated.deepcopy.go @@ -323,6 +323,21 @@ func (in *MetricsServer) DeepCopy() *MetricsServer { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *MultiClusterConfiguration) DeepCopyInto(out *MultiClusterConfiguration) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MultiClusterConfiguration. +func (in *MultiClusterConfiguration) DeepCopy() *MultiClusterConfiguration { + if in == nil { + return nil + } + out := new(MultiClusterConfiguration) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *NamespaceConfiguration) DeepCopyInto(out *NamespaceConfiguration) { *out = *in @@ -372,6 +387,7 @@ func (in *OperatorConfiguration) DeepCopyInto(out *OperatorConfiguration) { out.MPI = in.MPI in.Checkpoint.DeepCopyInto(&out.Checkpoint) out.Discovery = in.Discovery + out.MultiCluster = in.MultiCluster in.GPU.DeepCopyInto(&out.GPU) out.Logging = in.Logging out.Security = in.Security diff --git a/deploy/operator/api/config/validation/validation.go b/deploy/operator/api/config/validation/validation.go index e251edd690bb..4f3cd8a44bdb 100644 --- a/deploy/operator/api/config/validation/validation.go +++ b/deploy/operator/api/config/validation/validation.go @@ -39,6 +39,7 @@ func ValidateOperatorConfiguration(config *configv1alpha1.OperatorConfiguration) validateMPI(&config.MPI, field.NewPath("mpi")), validateInfrastructure(&config.Infrastructure, field.NewPath("infrastructure")), validateDiscovery(&config.Discovery, field.NewPath("discovery")), + validateMultiCluster(&config.MultiCluster, field.NewPath("multiCluster")), validateRBAC(config), validateOrchestrators(&config.Orchestrators, field.NewPath("orchestrators")), validateIngress(&config.Ingress, field.NewPath("ingress")), @@ -209,6 +210,18 @@ func validateDiscovery(discovery *configv1alpha1.DiscoveryConfiguration, fldPath return allErrs } +func validateMultiCluster(multiCluster *configv1alpha1.MultiClusterConfiguration, fldPath *field.Path) field.ErrorList { + switch multiCluster.MCSImplementation { + case configv1alpha1.MCSImplementationNone, configv1alpha1.MCSImplementationStandard, configv1alpha1.MCSImplementationKarmada: + return nil + } + return field.ErrorList{field.NotSupported( + fldPath.Child("mcsImplementation"), + multiCluster.MCSImplementation, + []string{"", "mcs", "karmada"}, + )} +} + // validateRBAC is mode-aware: validates RBAC fields based on namespace mode. func validateRBAC(config *configv1alpha1.OperatorConfiguration) field.ErrorList { allErrs := field.ErrorList{} diff --git a/deploy/operator/api/config/validation/validation_test.go b/deploy/operator/api/config/validation/validation_test.go index e713c8a68dc3..f9d75025e9d3 100644 --- a/deploy/operator/api/config/validation/validation_test.go +++ b/deploy/operator/api/config/validation/validation_test.go @@ -86,6 +86,29 @@ func TestValidateOperatorConfiguration_InvalidDiscoveryBackend(t *testing.T) { } } +func TestValidateOperatorConfiguration_MCSImplementation(t *testing.T) { + for _, tc := range []struct { + name string + implementation configv1alpha1.MCSImplementation + wantErrors int + }{ + {name: "none", implementation: configv1alpha1.MCSImplementationNone}, + {name: "standard MCS", implementation: configv1alpha1.MCSImplementationStandard}, + {name: "Karmada", implementation: configv1alpha1.MCSImplementationKarmada}, + {name: "unknown", implementation: "gke", wantErrors: 1}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Validate a configuration that exports Global Router pools through this implementation") + cfg := validConfig() + cfg.MultiCluster.MCSImplementation = tc.implementation + errs := ValidateOperatorConfiguration(cfg) + if len(errs) != tc.wantErrors { + t.Errorf("expected %d errors, got %d: %v", tc.wantErrors, len(errs), errs) + } + }) + } +} + func TestValidateOperatorConfiguration_ClusterWideMissingPlannerRole(t *testing.T) { cfg := validConfig() cfg.RBAC.PlannerClusterRoleName = "" diff --git a/deploy/operator/api/v1alpha1/dynamopoolrelay_types.go b/deploy/operator/api/v1alpha1/dynamopoolrelay_types.go index 4b9e1a258878..92ef38439886 100644 --- a/deploy/operator/api/v1alpha1/dynamopoolrelay_types.go +++ b/deploy/operator/api/v1alpha1/dynamopoolrelay_types.go @@ -54,6 +54,10 @@ type DynamoPoolRelayStatus struct { // Its reason is ListenerAvailable, DeploymentProgressing, DGDNotFound, // DGDTerminating, UnsupportedDiscovery, UnsupportedSourceScope, NoWorkers, // InvalidConfiguration, ReconcileFailed, or ResourceOwnershipConflict. + // With multi-cluster export configured, Conditions also includes Exported: True + // with reason Exported once the pool's Services are exported, or False with + // reason Withdrawn, ServiceNotOwned, InvalidServiceName, ServiceNameTooLong, + // NoFrontend, ClusterIDMismatch, or ExportFailed. // +optional // +listType=map // +listMapKey=type diff --git a/deploy/operator/config/crd/bases/nvidia.com_dynamopoolrelays.yaml b/deploy/operator/config/crd/bases/nvidia.com_dynamopoolrelays.yaml index cf7a7134402a..0560f79b4361 100644 --- a/deploy/operator/config/crd/bases/nvidia.com_dynamopoolrelays.yaml +++ b/deploy/operator/config/crd/bases/nvidia.com_dynamopoolrelays.yaml @@ -8211,6 +8211,10 @@ spec: Its reason is ListenerAvailable, DeploymentProgressing, DGDNotFound, DGDTerminating, UnsupportedDiscovery, UnsupportedSourceScope, NoWorkers, InvalidConfiguration, ReconcileFailed, or ResourceOwnershipConflict. + With multi-cluster export configured, Conditions also includes Exported: True + with reason Exported once the pool's Services are exported, or False with + reason Withdrawn, ServiceNotOwned, InvalidServiceName, ServiceNameTooLong, + NoFrontend, ClusterIDMismatch, or ExportFailed. items: description: Condition contains details for one aspect of the current state of this API Resource. properties: diff --git a/deploy/operator/internal/controller/poolrelay/controller.go b/deploy/operator/internal/controller/poolrelay/controller.go index 31b59b4c4135..0c30eb32000c 100644 --- a/deploy/operator/internal/controller/poolrelay/controller.go +++ b/deploy/operator/internal/controller/poolrelay/controller.go @@ -23,6 +23,8 @@ import ( commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/multicluster" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -57,14 +59,18 @@ type reconciler struct { client.Client scheme *runtime.Scheme defaultBackend configv1alpha1.DiscoveryBackend + // exports configures the pool's ServiceExports, which the controller maintains + // when the multiClusterExport gate is on, and is nil otherwise. + exports *exportSettings } -// Setup registers the one-DGD Relay controller and its DGD dependency watch. +// Setup registers the one-DGD Relay controller and its DGD dependency watch, and, +// when the multiClusterExport gate is on, the watches of the pool's exports. func Setup(mgr ctrl.Manager, config *configv1alpha1.OperatorConfiguration, runtimeConfig *commoncontroller.RuntimeConfig) error { r := &reconciler{Client: mgr.GetClient(), scheme: mgr.GetScheme(), defaultBackend: config.Discovery.Backend} - return ctrl.NewControllerManagedBy(mgr). + controllerBuilder := ctrl.NewControllerManagedBy(mgr). For(&v1alpha1.DynamoPoolRelay{}). - WithEventFilter(commoncontroller.EphemeralDeploymentEventFilter(config, runtimeConfig)). + WithEventFilter(eventFilter(config, runtimeConfig)). Watches(&v1beta1.DynamoGraphDeployment{}, handler.EnqueueRequestsFromMapFunc(func(_ context.Context, object client.Object) []ctrl.Request { key := types.NamespacedName{Namespace: object.GetNamespace(), Name: object.GetName()} return []ctrl.Request{{NamespacedName: key}} @@ -74,8 +80,14 @@ func Setup(mgr ctrl.Manager, config *configv1alpha1.OperatorConfiguration, runti Owns(&corev1.ConfigMap{}). Owns(&corev1.ServiceAccount{}). Owns(&rbacv1.Role{}). - Owns(&rbacv1.RoleBinding{}). - Complete(r) + Owns(&rbacv1.RoleBinding{}) + if runtimeConfig.Gate.Enabled(features.MultiClusterExport) { + var err error + if controllerBuilder, err = r.watchExports(mgr, controllerBuilder, config, runtimeConfig); err != nil { + return err + } + } + return controllerBuilder.Complete(r) } // +kubebuilder:rbac:groups=nvidia.com,resources=dynamopoolrelays,verbs=get;list;watch @@ -88,7 +100,8 @@ func Setup(mgr ctrl.Manager, config *configv1alpha1.OperatorConfiguration, runti // +kubebuilder:rbac:groups=core,resources=configmaps;serviceaccounts,verbs=get;list;watch;create;update // +kubebuilder:rbac:groups=rbac.authorization.k8s.io,resources=roles;rolebindings,verbs=get;list;watch;create;update -// Reconcile keeps one Relay Deployment and Service in step with its named DGD. +// Reconcile keeps one Relay Deployment and Service in step with its named DGD, and +// the pool's exports with them. func (r *reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl.Result, error) { var relay v1alpha1.DynamoPoolRelay if err := r.Get(ctx, request.NamespacedName, &relay); err != nil { @@ -139,22 +152,45 @@ func (r *reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. if err != nil { return r.withdraw(ctx, &relay, "InvalidConfiguration", err.Error()) } + result, err := r.publish(ctx, &relay, resources, desiredDeployment) + if errors.Is(err, errOwnershipConflict) { + // publish withdrew the Relay, and with it the pool's exports. + return result, err + } + // The pool's exports follow the DGD whether or not the Relay's own resources + // applied: a failure of one does not hold back the other. + if err := errors.Join(err, r.reconcileExports(ctx, &relay, &dgd)); err != nil { + // An error requeues the Relay with backoff, which replaces the result. + return ctrl.Result{}, err + } + return result, nil +} + +// publish applies the Relay's resources and Deployment, and reports the +// Deployment's progress in the Relay's status. A resource that another owner +// controls withdraws the Relay, with an error that wraps errOwnershipConflict. +func (r *reconciler) publish( + ctx context.Context, + relay *v1alpha1.DynamoPoolRelay, + resources []client.Object, + desiredDeployment *appsv1.Deployment, +) (ctrl.Result, error) { for _, resource := range resources { - if err := r.apply(ctx, &relay, resource); err != nil { + if err := r.apply(ctx, relay, resource); err != nil { if errors.Is(err, errOwnershipConflict) { - _, withdrawErr := r.withdraw(ctx, &relay, "ResourceOwnershipConflict", err.Error()) + _, withdrawErr := r.withdraw(ctx, relay, "ResourceOwnershipConflict", err.Error()) return ctrl.Result{}, errors.Join(err, withdrawErr) } - return ctrl.Result{}, errors.Join(err, r.reportApplyFailure(ctx, &relay, err)) + return ctrl.Result{}, errors.Join(err, r.reportApplyFailure(ctx, relay, err)) } } - deployment, err := r.applyDeployment(ctx, &relay, desiredDeployment) + deployment, err := r.applyDeployment(ctx, relay, desiredDeployment) if err != nil { if errors.Is(err, errOwnershipConflict) { - _, withdrawErr := r.withdraw(ctx, &relay, "ResourceOwnershipConflict", err.Error()) + _, withdrawErr := r.withdraw(ctx, relay, "ResourceOwnershipConflict", err.Error()) return ctrl.Result{}, errors.Join(err, withdrawErr) } - return ctrl.Result{}, errors.Join(err, r.reportApplyFailure(ctx, &relay, err)) + return ctrl.Result{}, errors.Join(err, r.reportApplyFailure(ctx, relay, err)) } // Deployment status is listener availability, never model or KV readiness. @@ -167,7 +203,7 @@ func (r *reconciler) Reconcile(ctx context.Context, request ctrl.Request) (ctrl. if ready { reason, message = "ListenerAvailable", "All requested Relay listeners are available" } - if err := r.setStatus(ctx, &relay, ready, reason, message, serviceName(relay.Name), available, true); err != nil { + if err := r.setStatus(ctx, relay, ready, reason, message, multicluster.PoolRelayServiceName(relay.Name), available, true); err != nil { return ctrl.Result{}, err } if !ready { @@ -237,7 +273,7 @@ func supportsPodDiscovery(dgd *v1beta1.DynamoGraphDeployment, defaultBackend con // withdraw removes the publication endpoint before reporting an unresolved source. func (r *reconciler) withdraw(ctx context.Context, relay *v1alpha1.DynamoPoolRelay, reason, message string) (ctrl.Result, error) { - name := serviceName(relay.Name) + name := multicluster.PoolRelayServiceName(relay.Name) metadata := metav1.ObjectMeta{Name: name, Namespace: relay.Namespace} var cleanupErrors []error for _, object := range []client.Object{ @@ -260,6 +296,10 @@ func (r *reconciler) withdraw(ctx context.Context, relay *v1alpha1.DynamoPoolRel cleanupErrors = append(cleanupErrors, err) } } + // The pool's exports go with the Relay's Service, whatever the reason. + if err := r.withdrawExports(ctx, relay); err != nil { + cleanupErrors = append(cleanupErrors, err) + } if err := r.setStatus(ctx, relay, false, reason, message, "", 0, len(cleanupErrors) == 0); err != nil { cleanupErrors = append(cleanupErrors, err) } @@ -450,10 +490,8 @@ func sourceJSON(namespaces []string, connectionRevision string) string { return string(encoded) } -func serviceName(dgdName string) string { return dgdName + "-pool-relay" } - func render(relay *v1alpha1.DynamoPoolRelay, namespaces []string) ([]client.Object, *appsv1.Deployment, error) { - name := serviceName(relay.Name) + name := multicluster.PoolRelayServiceName(relay.Name) if problems := validation.IsDNS1035Label(name); len(problems) != 0 { return nil, nil, fmt.Errorf("DGD name %q cannot produce a valid -pool-relay Service name %q: %s", relay.Name, name, strings.Join(problems, "; ")) } diff --git a/deploy/operator/internal/controller/poolrelay/controller_envtest_test.go b/deploy/operator/internal/controller/poolrelay/controller_envtest_test.go index 213dfaaa99ad..af10ccebf586 100644 --- a/deploy/operator/internal/controller/poolrelay/controller_envtest_test.go +++ b/deploy/operator/internal/controller/poolrelay/controller_envtest_test.go @@ -18,6 +18,7 @@ import ( v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/multicluster" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -65,7 +66,7 @@ func TestDeploymentAPIDefaultsDoNotCauseRolloutLoop(t *testing.T) { Spec: relaySpec("east"), } r := &reconciler{Client: apiClient, scheme: scheme, defaultBackend: configv1alpha1.DiscoveryBackendKubernetes} - key := types.NamespacedName{Namespace: relay.Namespace, Name: serviceName(relay.Name)} + key := types.NamespacedName{Namespace: relay.Namespace, Name: multicluster.PoolRelayServiceName(relay.Name)} var firstResourceVersion string for attempt := range 2 { _, wanted, err := render(relay, []string{"green-runtime"}) @@ -230,7 +231,7 @@ func TestControllerPublishesAndWithdrawsOneDGDService(t *testing.T) { if err := apiClient.Create(ctx, &relay); err != nil { t.Fatal(err) } - serviceKey := types.NamespacedName{Namespace: key.Namespace, Name: serviceName(key.Name)} + serviceKey := types.NamespacedName{Namespace: key.Namespace, Name: multicluster.PoolRelayServiceName(key.Name)} deadline := time.Now().Add(15 * time.Second) for time.Now().Before(deadline) { var service corev1.Service diff --git a/deploy/operator/internal/controller/poolrelay/controller_test.go b/deploy/operator/internal/controller/poolrelay/controller_test.go index f8fe0f0ac3ca..5aebfd9fbb5a 100644 --- a/deploy/operator/internal/controller/poolrelay/controller_test.go +++ b/deploy/operator/internal/controller/poolrelay/controller_test.go @@ -17,6 +17,7 @@ import ( v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/multicluster" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" rbacv1 "k8s.io/api/rbac/v1" @@ -710,7 +711,7 @@ func TestDiscoveryConversionErrorDoesNotWithdrawPublishedRelay(t *testing.T) { {ComponentName: "decode", ComponentType: v1beta1.ComponentTypeWorker}, }}, } - childKey := types.NamespacedName{Name: serviceName(key.Name), Namespace: key.Namespace} + childKey := types.NamespacedName{Name: multicluster.PoolRelayServiceName(key.Name), Namespace: key.Namespace} service := &corev1.Service{Name: childKey.Name, Namespace: childKey.Namespace} deployment := &appsv1.Deployment{Name: childKey.Name, Namespace: childKey.Namespace} for _, obj := range []client.Object{service, deployment} { @@ -794,7 +795,7 @@ func TestApplyFailureWithdrawsStaleReadiness(t *testing.T) { {ComponentName: "decode", ComponentType: v1beta1.ComponentTypeWorker}, }}, } - childKey := types.NamespacedName{Namespace: key.Namespace, Name: serviceName(key.Name)} + childKey := types.NamespacedName{Namespace: key.Namespace, Name: multicluster.PoolRelayServiceName(key.Name)} denyDeployment := false apiClient := fake.NewClientBuilder().WithScheme(scheme). WithStatusSubresource(relay, dgd, &appsv1.Deployment{}). @@ -865,7 +866,7 @@ func TestWithdrawUsesVerifiedUIDAndPreservesGenerationOnFailure(t *testing.T) { Spec: relaySpec("east"), Status: v1alpha1.DynamoPoolRelayStatus{ObservedGeneration: 2}, } - childKey := types.NamespacedName{Name: serviceName(relay.Name), Namespace: relay.Namespace} + childKey := types.NamespacedName{Name: multicluster.PoolRelayServiceName(relay.Name), Namespace: relay.Namespace} service := &corev1.Service{Name: childKey.Name, Namespace: childKey.Namespace, UID: "old-service"} if err := controllerutil.SetControllerReference(relay, service, scheme); err != nil { t.Fatal(err) @@ -905,7 +906,7 @@ func TestPrerequisiteOwnershipConflictWithdrawsRelay(t *testing.T) { } } key := types.NamespacedName{Namespace: "inference", Name: "green"} - childKey := types.NamespacedName{Namespace: key.Namespace, Name: serviceName(key.Name)} + childKey := types.NamespacedName{Namespace: key.Namespace, Name: multicluster.PoolRelayServiceName(key.Name)} relay := &v1alpha1.DynamoPoolRelay{ Name: key.Name, Namespace: key.Namespace, UID: "relay-uid", Spec: relaySpec("east"), @@ -1020,7 +1021,7 @@ func TestWithdrawCleansOwnedDeploymentAfterForeignService(t *testing.T) { ObjectMeta: relayMeta, Spec: relaySpec("east"), } - key := types.NamespacedName{Name: serviceName(relay.Name), Namespace: relay.Namespace} + key := types.NamespacedName{Name: multicluster.PoolRelayServiceName(relay.Name), Namespace: relay.Namespace} resourceMeta := metav1.ObjectMeta{Name: key.Name, Namespace: key.Namespace} foreignService := &corev1.Service{ObjectMeta: resourceMeta} deployment := &appsv1.Deployment{ObjectMeta: resourceMeta} diff --git a/deploy/operator/internal/controller/poolrelay/exports.go b/deploy/operator/internal/controller/poolrelay/exports.go new file mode 100644 index 000000000000..5a25bdc6bac0 --- /dev/null +++ b/deploy/operator/internal/controller/poolrelay/exports.go @@ -0,0 +1,730 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package poolrelay + +import ( + "context" + "errors" + "fmt" + "maps" + "slices" + "strings" + "unicode/utf8" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/multicluster" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" +) + +// The pool's exports: when the multiClusterExport gate is on, the Relay +// controller exports its DGD's PoolRelay and frontend Services to the other +// clusters of the ClusterSet through the configured Multi-Cluster Services (MCS) +// implementation. The Relay owns the exports, which follow its own decision: they +// are reconciled while it publishes its Service for the DGD, and withdrawn +// whenever it withdraws that Service. + +// +kubebuilder:rbac:groups=multicluster.x-k8s.io,resources=serviceexports,verbs=get;list;watch;create;patch;delete +// +kubebuilder:rbac:groups=about.k8s.io,resources=clusterproperties,verbs=get;list;watch +// +kubebuilder:rbac:groups=nvidia.com,resources=dynamocomponentdeployments,verbs=get;list;watch + +// exportFieldManager is the field manager of the controller's writes to +// ServiceExports, as of its other writes. +const exportFieldManager = "dynamo-pool-relay" + +// The Relay's Exported condition reports on the pool's exports: True once the +// Services that the pool can export are exported, False with reason Withdrawn +// while the Relay is withdrawn, and otherwise False with the reason of the pool's +// first problem. Apart from a failure, the problems are the ones that only the +// operator can see; the MCS implementation reports on each export in the export's +// own status. +// +// The controller records a Warning event for each problem when the condition +// changes, not on every reconcile. A problem's event names the DGD or +// ClusterProperty that it is about, at the version read, so that a problem with +// new details is not merged into the old one: the events API counts an event +// with the same object, reason, action and related object as a repeat, whatever +// its note. Normal events record each export that the controller creates, +// repairs or deletes, and name that export. +const ( + conditionExported = "Exported" + + reasonExported = "Exported" + reasonWithdrawn = "Withdrawn" + reasonServiceNotOwned = "ServiceNotOwned" + reasonInvalidServiceName = "InvalidServiceName" + reasonServiceNameTooLong = "ServiceNameTooLong" + reasonNoFrontend = "NoFrontend" + reasonClusterIDMismatch = "ClusterIDMismatch" + reasonExportFailed = "ExportFailed" +) + +// Limits of the API server: an event's note, and a condition's message. +const ( + maxEventNote = 1024 + maxConditionMessage = 32768 +) + +// relayGroupKind is the kind that controls a pool's exports. +var relayGroupKind = v1alpha1.GroupVersion.WithKind("DynamoPoolRelay").GroupKind() + +// dcdGroupKind is the kind that controls a frontend's Service on the DCD path; a +// DGD controls each of its DCDs. +var dcdGroupKind = v1beta1.GroupVersion.WithKind("DynamoComponentDeployment").GroupKind() + +// exportSettings configures the pool's exports, which the controller maintains +// when the multiClusterExport gate is on. +type exportSettings struct { + // cache reads unstructured objects, ServiceExports and ClusterProperties, from + // the informers that the controller's watches start. The manager's client + // reads unstructured objects from the API server. + cache client.Reader + recorder events.EventRecorder + config *configv1alpha1.OperatorConfiguration + runtimeConfig *commoncontroller.RuntimeConfig + // clusterPropertyWatched is true when the controller watches the About API's + // ClusterProperties: the MCS implementation labels imports with the cluster's + // ID in its ClusterSet, and the cluster serves the API. Otherwise the check of + // a pool's cluster ID is skipped. + clusterPropertyWatched bool +} + +// eventFilter passes the events of the namespaces that the operator manages, and +// those of the cluster-scoped ClusterProperty, whose mapper selects the Relays of +// those namespaces. +func eventFilter( + config *configv1alpha1.OperatorConfiguration, + runtimeConfig *commoncontroller.RuntimeConfig, +) predicate.Predicate { + return predicate.Or( + commoncontroller.EphemeralDeploymentEventFilter(config, runtimeConfig), + predicate.NewPredicateFuncs(func(object client.Object) bool { + return object.GetObjectKind().GroupVersionKind().GroupKind() == multicluster.ClusterPropertyGVK.GroupKind() + }), + ) +} + +// watchExports turns on the pool's exports, and adds the watches that they need: +// the ServiceExports, the Services that a pool may export and, with a standard +// implementation in a cluster that serves the About API, the ClusterProperty that +// holds the cluster's ID in its ClusterSet. The Relay's DGD is watched already. +func (r *reconciler) watchExports( + mgr ctrl.Manager, + controllerBuilder *builder.Builder, + config *configv1alpha1.OperatorConfiguration, + runtimeConfig *commoncontroller.RuntimeConfig, +) (*builder.Builder, error) { + r.exports = &exportSettings{ + cache: mgr.GetCache(), + recorder: mgr.GetEventRecorder("dynamo-pool-relay"), + config: config, + runtimeConfig: runtimeConfig, + } + controllerBuilder = controllerBuilder.Watches( + multicluster.NewServiceExport(runtimeConfig.Gate.ServiceExportVersion), + handler.EnqueueRequestsFromMapFunc(r.mapServiceExportToRelays), + builder.WithPredicates(serviceExportChanged), + ).Watches( + &corev1.Service{}, + handler.EnqueueRequestsFromMapFunc(r.mapServiceToRelays), + builder.WithPredicates(serviceControllerChanged), + ) + if !multicluster.UsesClusterSetID(config.MultiCluster.MCSImplementation) { + return controllerBuilder, nil + } + served, err := clusterPropertyServed(mgr.GetRESTMapper()) + if err != nil { + return controllerBuilder, err + } + if !served { + mgr.GetLogger().Info("The cluster does not serve the About API's ClusterProperty, " + + "so the clusterId of DynamoPoolRelays is not checked; to check it, install the About API and restart the operator") + return controllerBuilder, nil + } + r.exports.clusterPropertyWatched = true + return controllerBuilder.Watches( + multicluster.NewClusterProperty(), + handler.EnqueueRequestsFromMapFunc(r.mapClusterPropertyToRelays), + builder.WithPredicates(clusterIDPropertyEvents), + ), nil +} + +// exportProblem is a problem of the pool's exports that only the operator can +// see, or a failure to reconcile them. related is the object that it is about, +// if any. +type exportProblem struct { + reason string + message string + related runtime.Object +} + +// reconcileExports exports the Services of the pool that the Relay publishes for +// dgd, withdraws the Relay's other exports, and reports the outcome in the +// Relay's Exported condition. It decides from what exists now, not from the +// events that led there, so a DGD that is deleted and created again under the +// Relay's name, quickly or not, ends up with the new DGD's exports. +func (r *reconciler) reconcileExports( + ctx context.Context, + relay *v1alpha1.DynamoPoolRelay, + dgd *v1beta1.DynamoGraphDeployment, +) error { + if r.exports == nil { + return nil + } + exported, missing, problems, err := r.convergeExports(ctx, relay, dgd) + message := fmt.Sprintf("Exported %s through MCS implementation %s", + serviceList(exported), r.exports.config.MultiCluster.MCSImplementation) + if len(missing) > 0 { + message += fmt.Sprintf("; %s does not exist yet", serviceList(missing)) + } + exportedCondition := metav1.Condition{ + Type: conditionExported, Status: metav1.ConditionTrue, Reason: reasonExported, Message: message, + } + return errors.Join(err, r.setExported(ctx, relay, exportedCondition, problems, err)) +} + +// withdrawExports withdraws every export that the Relay controls, as it withdraws +// the Relay's Service, and reports that in the Relay's Exported condition. +func (r *reconciler) withdrawExports(ctx context.Context, relay *v1alpha1.DynamoPoolRelay) error { + if r.exports == nil { + return nil + } + err := r.withdrawAllExports(ctx, relay) + withdrawnCondition := metav1.Condition{ + Type: conditionExported, Status: metav1.ConditionFalse, Reason: reasonWithdrawn, + Message: "The Relay withdrew its Service, and with it the pool's exports", + } + return errors.Join(err, r.setExported(ctx, relay, withdrawnCondition, nil, err)) +} + +// setExported sets the Relay's Exported condition to condition or, with problems +// or err, to False with the first one's reason and every one's message. When the +// condition changes, it writes the Relay's status and then records a Warning +// event for each problem, so the events follow the condition, not each reconcile. +func (r *reconciler) setExported( + ctx context.Context, + relay *v1alpha1.DynamoPoolRelay, + condition metav1.Condition, + problems []exportProblem, + err error, +) error { + if err != nil { + problems = append([]exportProblem{{ + reason: reasonExportFailed, + message: fmt.Sprintf("Failed to reconcile the pool's ServiceExports: %v", err), + }}, problems...) + } + if len(problems) > 0 { + messages := make([]string, 0, len(problems)) + for _, problem := range problems { + messages = append(messages, problem.message) + } + condition.Status = metav1.ConditionFalse + condition.Reason = problems[0].reason + condition.Message = strings.Join(messages, "; ") + } + condition.Message = truncate(condition.Message, maxConditionMessage) + condition.ObservedGeneration = relay.Generation + + previous := meta.FindStatusCondition(relay.Status.Conditions, conditionExported) + changed := previous == nil || previous.Status != condition.Status || + previous.Reason != condition.Reason || previous.Message != condition.Message + if !changed && previous.ObservedGeneration == condition.ObservedGeneration { + return nil + } + meta.SetStatusCondition(&relay.Status.Conditions, condition) + if err := r.Status().Update(ctx, relay); err != nil { + return fmt.Errorf("failed to update the Exported condition: %w", err) + } + if changed { + for _, problem := range problems { + r.exports.recorder.Eventf(relay, problem.related, corev1.EventTypeWarning, problem.reason, "Export", + "%s", truncate(problem.message, maxEventNote)) + } + } + return nil +} + +// unexportedService is a pool Service that the implementation cannot export. +type unexportedService struct { + name string + problem multicluster.NameProblem + detail string +} + +// convergeExports exports each Service of the pool that the implementation can +// export and that the pool controls, and withdraws the Relay's other exports. It +// returns the names exported, those whose Service does not exist yet, the pool's +// problems, and every failed read and write: one failure does not hold back the +// others. +func (r *reconciler) convergeExports( + ctx context.Context, + relay *v1alpha1.DynamoPoolRelay, + dgd *v1beta1.DynamoGraphDeployment, +) (exported, missing []string, problems []exportProblem, err error) { + implementation := r.exports.config.MultiCluster.MCSImplementation + existing, err := r.listExports(ctx, relay) + if err != nil { + return nil, nil, nil, err + } + + // The pool exports each of its Services that the implementation can export and + // that the pool controls: a missing export is created, and one that the Relay + // controls is repaired. A user's export, or one being deleted, stays as it is. + // A Service that does not exist, or that the pool does not control, is not + // exported, so a Service of the same name that someone else created is never + // exported in the pool's name. + var errs []error + var unexported []unexportedService + var notControlled []string + keep := map[string]bool{} + for _, name := range multicluster.ServiceNames(dgd) { + if problem, detail := multicluster.CheckServiceName(implementation, name); problem != multicluster.NameExportable { + unexported = append(unexported, unexportedService{name: name, problem: problem, detail: detail}) + continue + } + found, controlled, err := r.poolControlsService(ctx, relay, dgd, name) + switch { + case err != nil: + // Keep the export until the Service can be read again. + keep[name] = true + errs = append(errs, err) + continue + case !found: + missing = append(missing, name) + continue + case !controlled: + notControlled = append(notControlled, name) + continue + } + keep[name] = true + current, found := existing[name] + switch { + case !found: + err = r.createExport(ctx, relay, dgd, name) + case current.ControlledBy(relay.UID) && !current.Deleting: + err = r.repairExport(ctx, relay, dgd, current) + } + if err != nil { + errs = append(errs, err) + continue + } + exported = append(exported, name) + } + + // Withdraw the Relay's exports of Services that the pool does not export, such + // as a renamed or removed frontend's, or one that someone else's Service took + // over. + for _, name := range slices.Sorted(maps.Keys(existing)) { + if keep[name] { + continue + } + if err := r.withdrawExport(ctx, relay, existing[name]); err != nil { + errs = append(errs, err) + } + } + problems, err = r.exportProblems(ctx, relay, dgd, unexported, notControlled) + return exported, missing, problems, errors.Join(append(errs, err)...) +} + +// poolControlsService reads the Service named name in the Relay's namespace, and +// reports whether it exists and whether the pool controls it: the Relay controls +// its PoolRelay Service, and the DGD, or on the DCD path one of its DCDs, a +// frontend's. +func (r *reconciler) poolControlsService( + ctx context.Context, + relay *v1alpha1.DynamoPoolRelay, + dgd *v1beta1.DynamoGraphDeployment, + name string, +) (found, controlled bool, err error) { + service := &corev1.Service{} + if err := r.Get(ctx, client.ObjectKey{Namespace: relay.Namespace, Name: name}, service); err != nil { + if apierrors.IsNotFound(err) { + return false, false, nil + } + return false, false, fmt.Errorf("failed to read Service %s: %w", name, err) + } + controller := metav1.GetControllerOf(service) + switch { + case controller == nil: + return true, false, nil + case controller.UID == relay.UID || controller.UID == dgd.UID: + return true, true, nil + case schema.FromAPIVersionAndKind(controller.APIVersion, controller.Kind).GroupKind() != dcdGroupKind: + return true, false, nil + } + dcd := &v1beta1.DynamoComponentDeployment{} + if err := r.Get(ctx, client.ObjectKey{Namespace: relay.Namespace, Name: controller.Name}, dcd); err != nil { + if apierrors.IsNotFound(err) { + return true, false, nil + } + return true, false, fmt.Errorf("failed to read DynamoComponentDeployment %s: %w", controller.Name, err) + } + return true, dcd.UID == controller.UID && metav1.IsControlledBy(dcd, dgd), nil +} + +func (r *reconciler) withdrawAllExports(ctx context.Context, relay *v1alpha1.DynamoPoolRelay) error { + existing, err := r.listExports(ctx, relay) + if err != nil { + return err + } + var errs []error + for _, name := range slices.Sorted(maps.Keys(existing)) { + if err := r.withdrawExport(ctx, relay, existing[name]); err != nil { + errs = append(errs, err) + } + } + return errors.Join(errs...) +} + +// listExports returns the ServiceExports in the Relay's namespace by name. +func (r *reconciler) listExports(ctx context.Context, relay *v1alpha1.DynamoPoolRelay) (map[string]multicluster.Export, error) { + return multicluster.ListExports(ctx, r.exports.cache, r.exports.runtimeConfig.Gate.ServiceExportVersion, relay.Namespace) +} + +// desiredExport returns the Relay's ServiceExport of the Service named name, with +// only the fields that the controller writes: the DGD label, and the Relay as its +// controller, so that the export is deleted with the Relay. +func (r *reconciler) desiredExport( + relay *v1alpha1.DynamoPoolRelay, + dgd *v1beta1.DynamoGraphDeployment, + name string, +) (*unstructured.Unstructured, error) { + serviceExport := multicluster.RenderServiceExport( + r.exports.runtimeConfig.Gate.ServiceExportVersion, relay.Namespace, name, dgd.Name) + if err := ctrl.SetControllerReference(relay, serviceExport, r.scheme); err != nil { + return nil, fmt.Errorf("failed to set the owner of ServiceExport %s: %w", name, err) + } + return serviceExport, nil +} + +// createExport creates the Relay's ServiceExport of the Service named name. A +// create fails on any existing export, so the Relay never takes over a user's +// export of the same name that the cache has not seen yet. +func (r *reconciler) createExport( + ctx context.Context, + relay *v1alpha1.DynamoPoolRelay, + dgd *v1beta1.DynamoGraphDeployment, + name string, +) error { + serviceExport, err := r.desiredExport(relay, dgd, name) + if err != nil { + return err + } + if err := r.Create(ctx, serviceExport, client.FieldOwner(exportFieldManager)); err != nil { + // The cache has not seen an export created moments ago. Its watch event + // requeues the Relay. + if apierrors.IsAlreadyExists(err) { + return nil + } + return fmt.Errorf("failed to create ServiceExport %s: %w", name, err) + } + r.exports.recorder.Eventf(relay, serviceExport, corev1.EventTypeNormal, "CreateServiceExport", "Create", + "Exported Service %s through MCS implementation %s", name, r.exports.config.MultiCluster.MCSImplementation) + return nil +} + +// repairExport restores the fields that the controller writes on an export the +// Relay controls, when they differ from desiredExport's, for example after +// someone removed the DGD label. Server-side apply writes only those fields, so +// the export keeps its UID, the MCS implementation's status, its finalizers, and +// other labels and annotations. +func (r *reconciler) repairExport( + ctx context.Context, + relay *v1alpha1.DynamoPoolRelay, + dgd *v1beta1.DynamoGraphDeployment, + current multicluster.Export, +) error { + desired, err := r.desiredExport(relay, dgd, current.Name) + if err != nil { + return err + } + if hasDesiredFields(current, desired) { + return nil + } + + // The UID and resourceVersion tie the apply to the export whose controller was + // just checked. Once it has changed, been replaced or been deleted, the API + // server refuses the apply with a conflict instead of writing to an export + // that the Relay may no longer control, and the change's watch event requeues + // the Relay. Forcing ownership takes back a field that someone else changed: + // without it, that field would also be refused as a conflict. + desired.SetUID(current.UID) + desired.SetResourceVersion(current.ResourceVersion) + err = r.Apply(ctx, client.ApplyConfigurationFromUnstructured(desired), + client.FieldOwner(exportFieldManager), client.ForceOwnership) + if apierrors.IsConflict(err) { + return nil + } + if err != nil { + return fmt.Errorf("failed to repair ServiceExport %s: %w", current.Name, err) + } + r.exports.recorder.Eventf(relay, desired, corev1.EventTypeNormal, "RepairServiceExport", "Apply", + "Restored the DGD label and controller reference of the export of Service %s", current.Name) + return nil +} + +// hasDesiredFields reports whether export has the labels and the controller +// reference of desired. Other labels do not matter. +func hasDesiredFields(export multicluster.Export, desired *unstructured.Unstructured) bool { + for key, value := range desired.GetLabels() { + if export.Labels[key] != value { + return false + } + } + return equality.Semantic.DeepEqual(export.Controller, metav1.GetControllerOf(desired)) +} + +// withdrawExport deletes export if the Relay controls it and it is not already +// being deleted. +func (r *reconciler) withdrawExport(ctx context.Context, relay *v1alpha1.DynamoPoolRelay, export multicluster.Export) error { + if !export.ControlledBy(relay.UID) || export.Deleting { + return nil + } + + // The UID and resourceVersion preconditions delete exactly the export just + // read, at the version whose controller was checked. A change to the export + // since, such as someone removing or transferring the controller reference, + // which keeps the UID, makes the API server refuse the delete with a conflict. + // That is retried from the change's watch event, which requeues the Relay to + // check the controller again. + serviceExport := multicluster.NewServiceExport(r.exports.runtimeConfig.Gate.ServiceExportVersion) + serviceExport.SetNamespace(relay.Namespace) + serviceExport.SetName(export.Name) + if err := r.Delete(ctx, serviceExport, client.Preconditions{ + UID: &export.UID, + ResourceVersion: &export.ResourceVersion, + }); err != nil { + if apierrors.IsNotFound(err) || apierrors.IsConflict(err) { + return nil + } + return fmt.Errorf("failed to delete ServiceExport %s: %w", export.Name, err) + } + // The event names the export deleted. + serviceExport.SetUID(export.UID) + serviceExport.SetResourceVersion(export.ResourceVersion) + r.exports.recorder.Eventf(relay, serviceExport, corev1.EventTypeNormal, "DeleteServiceExport", "Delete", + "Withdrew the export of Service %s", export.Name) + return nil +} + +// exportProblems returns the problems of the pool that only the operator can +// see: a Service that the pool does not control, a Service that the +// implementation cannot export, a DGD without a frontend, and a cluster ID that +// differs from the cluster's ID in its ClusterSet. A failed read of that ID is an +// error. +func (r *reconciler) exportProblems( + ctx context.Context, + relay *v1alpha1.DynamoPoolRelay, + dgd *v1beta1.DynamoGraphDeployment, + unexported []unexportedService, + notControlled []string, +) ([]exportProblem, error) { + var problems []exportProblem + if len(notControlled) > 0 { + problems = append(problems, exportProblem{reason: reasonServiceNotOwned, related: dgd, message: fmt.Sprintf( + "Neither the Relay nor DGD %s controls %s, so it is not exported", dgd.Name, serviceList(notControlled))}) + } + if names, detail := unexportedNames(unexported, multicluster.NameInvalid); len(names) > 0 { + problems = append(problems, exportProblem{reason: reasonInvalidServiceName, related: dgd, message: fmt.Sprintf( + "No Service can be named %s, so it is not exported: %s", strings.Join(names, ", "), detail)}) + } + if names, detail := unexportedNames(unexported, multicluster.NameTooLong); len(names) > 0 { + problems = append(problems, exportProblem{reason: reasonServiceNameTooLong, related: dgd, message: fmt.Sprintf( + "%s, so %s is not exported; shorten the DGD or component name", detail, strings.Join(names, ", "))}) + } + if len(multicluster.FrontendServiceNames(dgd)) == 0 { + problems = append(problems, exportProblem{reason: reasonNoFrontend, related: dgd, message: fmt.Sprintf( + "DGD %s has no frontend component, so the Global Router has nowhere to send the pool's requests", dgd.Name)}) + } + + // A standard implementation labels the pool's imports with the cluster's ID in + // its ClusterSet, and the Global Discovery Service joins the pool's + // registration to them only under the same cluster ID. + if !r.exports.clusterPropertyWatched { + return problems, nil + } + property, err := multicluster.ReadClusterIDProperty(ctx, r.exports.cache) + if err != nil || property == nil { + return problems, err + } + if clusterID := multicluster.ClusterPropertyValue(property); clusterID != "" && clusterID != relay.Spec.ClusterID { + problems = append(problems, exportProblem{reason: reasonClusterIDMismatch, related: property, message: fmt.Sprintf( + "clusterId %q differs from the ID %q that ClusterProperty %s gives this cluster in its ClusterSet; "+ + "the Global Discovery Service lists the pool only when they match", + relay.Spec.ClusterID, clusterID, multicluster.ClusterIDProperty)}) + } + return problems, nil +} + +// serviceList names the Services named names in a message. +func serviceList(names []string) string { + switch len(names) { + case 0: + return "no Service" + case 1: + return "Service " + names[0] + } + return "Services " + strings.Join(names, ", ") +} + +// truncate shortens s to at most limit bytes without splitting a character. +func truncate(s string, limit int) string { + if len(s) <= limit { + return s + } + end := limit + for end > 0 && !utf8.RuneStart(s[end]) { + end-- + } + return s[:end] +} + +// unexportedNames returns the names of the services with problem, and the first +// one's detail. +func unexportedNames(services []unexportedService, problem multicluster.NameProblem) ([]string, string) { + var names []string + detail := "" + for _, service := range services { + if service.problem != problem { + continue + } + if len(names) == 0 { + detail = service.detail + } + names = append(names, service.name) + } + return names, detail +} + +// mapServiceExportToRelays requeues the Relay that controls the export, and each +// Relay whose DGD exports a Service of the export's name: such a Relay takes over +// when a user's export of that Service is deleted. +func (r *reconciler) mapServiceExportToRelays(ctx context.Context, export client.Object) []ctrl.Request { + var requests []ctrl.Request + if controller := metav1.GetControllerOf(export); controller != nil && + schema.FromAPIVersionAndKind(controller.APIVersion, controller.Kind).GroupKind() == relayGroupKind { + requests = append(requests, ctrl.Request{Namespace: export.GetNamespace(), Name: controller.Name}) + } + for _, request := range r.relaysExportingService(ctx, export.GetNamespace(), export.GetName()) { + if !slices.Contains(requests, request) { + requests = append(requests, request) + } + } + return requests +} + +// mapServiceToRelays requeues each Relay whose DGD exports a Service of the +// Service's name: the pool exports a Service only while it exists and the pool +// controls it. +func (r *reconciler) mapServiceToRelays(ctx context.Context, service client.Object) []ctrl.Request { + return r.relaysExportingService(ctx, service.GetNamespace(), service.GetName()) +} + +// relaysExportingService returns a request for each Relay in namespace whose DGD +// exports a Service named name. +func (r *reconciler) relaysExportingService(ctx context.Context, namespace, name string) []ctrl.Request { + relays := &v1alpha1.DynamoPoolRelayList{} + if err := r.List(ctx, relays, client.InNamespace(namespace)); err != nil { + log.FromContext(ctx).Error(err, "Failed to list DynamoPoolRelays for a Service or ServiceExport event") + return nil + } + var requests []ctrl.Request + for i := range relays.Items { + dgd := &v1beta1.DynamoGraphDeployment{} + if err := r.Get(ctx, client.ObjectKeyFromObject(&relays.Items[i]), dgd); err != nil { + continue + } + if slices.Contains(multicluster.ServiceNames(dgd), name) { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(&relays.Items[i])}) + } + } + return requests +} + +// mapClusterPropertyToRelays requeues every Relay in a namespace that the +// operator manages: each pool's cluster ID is compared with the cluster's ID in +// its ClusterSet. +func (r *reconciler) mapClusterPropertyToRelays(ctx context.Context, _ client.Object) []ctrl.Request { + relays := &v1alpha1.DynamoPoolRelayList{} + if err := r.List(ctx, relays); err != nil { + log.FromContext(ctx).Error(err, "Failed to list DynamoPoolRelays for ClusterProperty event") + return nil + } + requests := make([]ctrl.Request, 0, len(relays.Items)) + for i := range relays.Items { + relay := &relays.Items[i] + if commoncontroller.NamespaceAllowed(r.exports.config, r.exports.runtimeConfig, relay, relay.Namespace) { + requests = append(requests, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(relay)}) + } + } + return requests +} + +// serviceExportChanged passes any update of a ServiceExport with a controller, +// before or after the update: the Relay that controls the export repairs it, and +// one that stopped controlling it leaves it alone. This also covers a change that +// made a repair or a withdrawal conflict. The creation and deletion of any export +// pass too. +var serviceExportChanged = predicate.Funcs{UpdateFunc: func(update event.UpdateEvent) bool { + return metav1.GetControllerOf(update.ObjectOld) != nil || metav1.GetControllerOf(update.ObjectNew) != nil +}} + +// serviceControllerChanged passes the creation and deletion of a Service, and an +// update that changes its controller: whether the pool exports a Service depends +// only on whether it exists and who controls it. +var serviceControllerChanged = predicate.Funcs{UpdateFunc: func(update event.UpdateEvent) bool { + return !equality.Semantic.DeepEqual(metav1.GetControllerOf(update.ObjectOld), metav1.GetControllerOf(update.ObjectNew)) +}} + +// clusterIDPropertyEvents passes the events of the ClusterProperty that holds the +// cluster's ID in its ClusterSet: its creation, its deletion and a change of its +// value. +var clusterIDPropertyEvents = predicate.Funcs{ + CreateFunc: func(e event.CreateEvent) bool { return e.Object.GetName() == multicluster.ClusterIDProperty }, + DeleteFunc: func(e event.DeleteEvent) bool { return e.Object.GetName() == multicluster.ClusterIDProperty }, + UpdateFunc: clusterIDPropertyChanged, + GenericFunc: func(event.GenericEvent) bool { return false }, +} + +// clusterIDPropertyChanged passes the updates of the ClusterProperty that holds +// the cluster's ID in its ClusterSet that change its value. +func clusterIDPropertyChanged(update event.UpdateEvent) bool { + oldProperty, oldOK := update.ObjectOld.(*unstructured.Unstructured) + newProperty, newOK := update.ObjectNew.(*unstructured.Unstructured) + return oldOK && newOK && newProperty.GetName() == multicluster.ClusterIDProperty && + multicluster.ClusterPropertyValue(oldProperty) != multicluster.ClusterPropertyValue(newProperty) +} + +// clusterPropertyServed reports whether the cluster serves the About API's +// ClusterProperty, which the controller then watches. +func clusterPropertyServed(mapper meta.RESTMapper) (bool, error) { + _, err := mapper.RESTMapping(multicluster.ClusterPropertyGVK.GroupKind(), multicluster.ClusterPropertyGVK.Version) + if meta.IsNoMatchError(err) { + return false, nil + } + if err != nil { + return false, fmt.Errorf("discover the ClusterProperty API for the pool's exports: %w", err) + } + return true, nil +} diff --git a/deploy/operator/internal/controller/poolrelay/exports_envtest_test.go b/deploy/operator/internal/controller/poolrelay/exports_envtest_test.go new file mode 100644 index 000000000000..19bcc201bb00 --- /dev/null +++ b/deploy/operator/internal/controller/poolrelay/exports_envtest_test.go @@ -0,0 +1,688 @@ +//go:build !clustertest + +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package poolrelay + +import ( + "context" + "fmt" + "maps" + "os" + "path/filepath" + "slices" + "strings" + "sync" + "sync/atomic" + "testing" + "time" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/multicluster" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + eventsv1 "k8s.io/api/events/v1" + apiextensionsv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/rest" + "k8s.io/client-go/util/retry" + "k8s.io/utils/ptr" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/cache" + "sigs.k8s.io/controller-runtime/pkg/client" + ctrlconfig "sigs.k8s.io/controller-runtime/pkg/config" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/envtest" + "sigs.k8s.io/controller-runtime/pkg/metrics/server" + "sigs.k8s.io/yaml" +) + +// testCRD returns a CustomResourceDefinition that serves kind in group at +// version only and keeps any field: a stand-in for the MCS and About APIs, whose +// CRDs the operator does not ship. +func testCRD(group, kind, plural, version string, scope apiextensionsv1.ResourceScope) *apiextensionsv1.CustomResourceDefinition { + return &apiextensionsv1.CustomResourceDefinition{ + Name: plural + "." + group, + Spec: apiextensionsv1.CustomResourceDefinitionSpec{ + Group: group, + Names: apiextensionsv1.CustomResourceDefinitionNames{ + Kind: kind, ListKind: kind + "List", Plural: plural, Singular: strings.ToLower(kind), + }, + Scope: scope, + Versions: []apiextensionsv1.CustomResourceDefinitionVersion{{ + Name: version, Served: true, Storage: true, + Schema: &apiextensionsv1.CustomResourceValidation{OpenAPIV3Schema: &apiextensionsv1.JSONSchemaProps{ + Type: "object", XPreserveUnknownFields: new(true), + }}, + Subresources: &apiextensionsv1.CustomResourceSubresources{ + Status: &apiextensionsv1.CustomResourceSubresourceStatus{}, + }, + }}, + }, + } +} + +// startExportAPIServer starts an API server with the operator's CRDs and +// stand-ins for the ServiceExport API, served at serviceExportVersion only, and +// for the About API's ClusterProperty. It runs no garbage collector. +func startExportAPIServer(t *testing.T, serviceExportVersion string) *rest.Config { + t.Helper() + // about.k8s.io is a protected API group, so its CRD needs the approval + // annotation that the About API's own CRD has. + clusterProperties := testCRD(multicluster.ClusterPropertyGVK.Group, multicluster.ClusterPropertyGVK.Kind, + "clusterproperties", multicluster.ClusterPropertyGVK.Version, apiextensionsv1.ClusterScoped) + clusterProperties.Annotations = map[string]string{ + apiextensionsv1.KubeAPIApprovedAnnotation: "https://github.com/kubernetes/enhancements/pull/3084", + } + environment := &envtest.Environment{ + CRDDirectoryPaths: []string{filepath.Join("..", "..", "..", "config", "crd", "bases")}, + CRDs: []*apiextensionsv1.CustomResourceDefinition{ + testCRD(multicluster.ServiceExportGroupKind.Group, multicluster.ServiceExportGroupKind.Kind, + "serviceexports", serviceExportVersion, apiextensionsv1.NamespaceScoped), + clusterProperties, + }, + ErrorIfCRDPathMissing: true, + } + config, err := environment.Start() + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { + if err := environment.Stop(); err != nil { + t.Error(err) + } + }) + return config +} + +// relayControllerOptions configures the Relay controller that a test starts. +type relayControllerOptions struct { + // version is the multicluster.x-k8s.io version that the gate resolved: the + // one at which the API server serves ServiceExports. + version string + // restricted is the namespace that a namespace-restricted operator manages, + // or empty for an operator that manages every namespace. + restricted string + // wrap, when set, wraps the manager's client. + wrap func(client.Client) client.Client +} + +// startRelayController starts a manager that runs the Relay controller with the +// pool's exports on, for standard MCS, and returns a function that stops the +// manager and waits for it. +func startRelayController(t *testing.T, config *rest.Config, options relayControllerOptions) func() { + t.Helper() + managerOptions := ctrl.Options{ + Scheme: newExportTestScheme(t), + Metrics: server.Options{BindAddress: "0"}, + // A test can start managers one after another, with the same controllers. + Controller: ctrlconfig.Controller{SkipNameValidation: new(true)}, + NewClient: func(restConfig *rest.Config, clientOptions client.Options) (client.Client, error) { + c, err := client.New(restConfig, clientOptions) + if err != nil || options.wrap == nil { + return c, err + } + return options.wrap(c), nil + }, + } + if options.restricted != "" { + // As cmd/main.go configures a namespace-restricted operator's cache. + managerOptions.Cache.DefaultNamespaces = map[string]cache.Config{options.restricted: {}} + } + manager, err := ctrl.NewManager(config, managerOptions) + if err != nil { + t.Fatal(err) + } + if err := Setup(manager, &configv1alpha1.OperatorConfiguration{ + Discovery: configv1alpha1.DiscoveryConfiguration{Backend: configv1alpha1.DiscoveryBackendKubernetes}, + MultiCluster: configv1alpha1.MultiClusterConfiguration{MCSImplementation: configv1alpha1.MCSImplementationStandard}, + Namespace: configv1alpha1.NamespaceConfiguration{Restricted: options.restricted}, + }, &commoncontroller.RuntimeConfig{ + Gate: features.Gates{MultiClusterExport: true, ServiceExportVersion: options.version}, + }); err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + done := make(chan error, 1) + go func() { done <- manager.Start(ctx) }() + var once sync.Once + stop := func() { + once.Do(func() { + cancel() + if err := <-done; err != nil { + t.Error(err) + } + }) + } + t.Cleanup(stop) + return stop +} + +// eventually polls check until it reports done, and fails the test with what +// after 30 seconds. +func eventually(t *testing.T, what string, check func() (bool, error)) { + t.Helper() + deadline := time.Now().Add(30 * time.Second) + for { + done, err := check() + if err != nil { + t.Fatalf("%s: %v", what, err) + } + if done { + return + } + if time.Now().After(deadline) { + t.Fatalf("timed out waiting for %s", what) + } + time.Sleep(100 * time.Millisecond) + } +} + +// readSampleRelay returns the checked-in DynamoPoolRelay sample. +func readSampleRelay(t *testing.T) *v1alpha1.DynamoPoolRelay { + t.Helper() + sample, err := os.ReadFile(filepath.Join("..", "..", "..", "samples", "dynamo-pool-relay.yaml")) + if err != nil { + t.Fatal(err) + } + relay := &v1alpha1.DynamoPoolRelay{} + if err := yaml.UnmarshalStrict(sample, relay); err != nil { + t.Fatal(err) + } + return relay +} + +// createNamespace creates the namespace called name. +func createNamespace(t *testing.T, c client.Client, name string) { + t.Helper() + namespace := &corev1.Namespace{} + namespace.Name = name + if err := c.Create(context.Background(), namespace); err != nil { + t.Fatal(err) + } +} + +// createDGD creates the DGD of the Relay's name with a worker and a frontend +// component named frontend, and the frontend's Service, controlled by the DGD as +// the DGD controller creates it, and returns the DGD. +func createDGD(t *testing.T, c client.Client, relay *v1alpha1.DynamoPoolRelay, frontend string) *v1beta1.DynamoGraphDeployment { + t.Helper() + dgd := &v1beta1.DynamoGraphDeployment{ + Name: relay.Name, Namespace: relay.Namespace, + Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{ + {ComponentName: "decode", ComponentType: v1beta1.ComponentTypeWorker}, + {ComponentName: frontend, ComponentType: v1beta1.ComponentTypeFrontend}, + }}, + } + if err := c.Create(context.Background(), dgd); err != nil { + t.Fatal(err) + } + for _, name := range multicluster.FrontendServiceNames(dgd) { + service := &corev1.Service{ + Name: name, Namespace: dgd.Namespace, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{Name: "http", Port: 8000}}}, + } + if err := controllerutil.SetControllerReference(dgd, service, c.Scheme()); err != nil { + t.Fatal(err) + } + if err := c.Create(context.Background(), service); err != nil { + t.Fatal(err) + } + } + return dgd +} + +// exportVersions returns the resourceVersion of each ServiceExport in namespace, +// by name. +func exportVersions(t *testing.T, c client.Client, namespace string) map[string]string { + t.Helper() + versions := map[string]string{} + for name, export := range exportsByName(t, c, namespace) { + versions[name] = export.GetResourceVersion() + } + return versions +} + +// reportTerminalFailure writes the status that the DGD controller writes when a +// DGD's workloads cannot be reconciled at all, with message. +func reportTerminalFailure(t *testing.T, c client.Client, dgd *v1beta1.DynamoGraphDeployment, message string) { + t.Helper() + if err := c.Get(context.Background(), client.ObjectKeyFromObject(dgd), dgd); err != nil { + t.Fatal(err) + } + dgd.Status.State = v1beta1.DGDStateFailed + meta.SetStatusCondition(&dgd.Status.Conditions, metav1.Condition{ + Type: "Ready", Status: metav1.ConditionFalse, Reason: "UnsupportedWorkloadProvider", Message: message, + }) + if err := c.Status().Update(context.Background(), dgd); err != nil { + t.Fatal(err) + } +} + +// ownedByRelay reports whether relay controls export and blocks its own deletion +// on it, so that the garbage collector deletes the export with the Relay. +func ownedByRelay(export *unstructured.Unstructured, relay *v1alpha1.DynamoPoolRelay) bool { + controller := metav1.GetControllerOf(export) + return controller != nil && controller.UID == relay.UID && ptr.Deref(controller.BlockOwnerDeletion, false) +} + +// exportsByName returns the ServiceExports in namespace by name. +func exportsByName(t *testing.T, c client.Client, namespace string) map[string]*unstructured.Unstructured { + t.Helper() + exports := map[string]*unstructured.Unstructured{} + items := listServiceExports(t, c, exportTestVersion, namespace) + for i := range items { + exports[items[i].GetName()] = &items[i] + } + return exports +} + +// waitForExports waits until the ServiceExports in the Relay's namespace are +// exactly names, none being deleted, and each owned by relay. +func waitForExports(t *testing.T, c client.Client, relay *v1alpha1.DynamoPoolRelay, names ...string) { + t.Helper() + eventually(t, "the exports "+strings.Join(names, ", "), func() (bool, error) { + exports := exportsByName(t, c, relay.Namespace) + if len(exports) != len(names) { + return false, nil + } + for name, export := range exports { + if !slices.Contains(names, name) || export.GetDeletionTimestamp() != nil || !ownedByRelay(export, relay) { + return false, nil + } + } + return true, nil + }) +} + +// waitForRelayService waits until the controller publishes the Relay's Service, +// or withdraws it. +func waitForRelayService(t *testing.T, c client.Client, relay *v1alpha1.DynamoPoolRelay, published bool) { + t.Helper() + key := types.NamespacedName{Namespace: relay.Namespace, Name: multicluster.PoolRelayServiceName(relay.Name)} + eventually(t, fmt.Sprintf("the Relay's Service to be published: %v", published), func() (bool, error) { + err := c.Get(context.Background(), key, &corev1.Service{}) + if apierrors.IsNotFound(err) { + return !published, nil + } + return published, err + }) +} + +// waitForDeletion waits until obj is gone. +func waitForDeletion(t *testing.T, c client.Client, obj client.Object) { + t.Helper() + eventually(t, obj.GetName()+" to be deleted", func() (bool, error) { + err := c.Get(context.Background(), client.ObjectKeyFromObject(obj), obj) + if apierrors.IsNotFound(err) { + return true, nil + } + return false, err + }) +} + +// markRelayListenersAvailable reports the Relay's Deployment as rolled out, as +// the Deployment controller would if envtest ran it, and waits until the Relay +// is Ready. The controller requeues a Relay that is not Ready every 15 seconds; +// a Ready Relay is requeued only by watch events. +func markRelayListenersAvailable(t *testing.T, c client.Client, relay *v1alpha1.DynamoPoolRelay) { + t.Helper() + ctx := context.Background() + key := types.NamespacedName{Namespace: relay.Namespace, Name: multicluster.PoolRelayServiceName(relay.Name)} + eventually(t, "the Relay's Deployment", func() (bool, error) { + err := c.Get(ctx, key, &appsv1.Deployment{}) + if apierrors.IsNotFound(err) { + return false, nil + } + return err == nil, err + }) + replicas := ptr.Deref(relay.Spec.Replicas, 2) + if err := retry.RetryOnConflict(retry.DefaultRetry, func() error { + deployment := &appsv1.Deployment{} + if err := c.Get(ctx, key, deployment); err != nil { + return err + } + deployment.Status = appsv1.DeploymentStatus{ + ObservedGeneration: deployment.Generation, + Replicas: replicas, + UpdatedReplicas: replicas, + ReadyReplicas: replicas, + AvailableReplicas: replicas, + } + return c.Status().Update(ctx, deployment) + }); err != nil { + t.Fatal(err) + } + eventually(t, "the Relay to be Ready", func() (bool, error) { + current := &v1alpha1.DynamoPoolRelay{} + if err := c.Get(ctx, client.ObjectKeyFromObject(relay), current); err != nil { + return false, err + } + return meta.IsStatusConditionTrue(current.Status.Conditions, "Ready"), nil + }) +} + +//nolint:gocyclo // This test follows one Relay through the replacement of its DGD against the API server. +func TestRelayControllerFollowsAReplacedDGD(t *testing.T) { + if os.Getenv("KUBEBUILDER_ASSETS") == "" { + t.Skip("KUBEBUILDER_ASSETS is needed for the controller API test") + } + ctx := context.Background() + config := startExportAPIServer(t, exportTestVersion) + apiClient, err := client.New(config, client.Options{Scheme: newExportTestScheme(t)}) + if err != nil { + t.Fatal(err) + } + stop := startRelayController(t, config, relayControllerOptions{version: exportTestVersion}) + + t.Log("Create a DGD with a frontend component, and its DynamoPoolRelay from the checked-in sample") + relay := readSampleRelay(t) + createNamespace(t, apiClient, relay.Namespace) + first := createDGD(t, apiClient, relay, "Frontend") + if err := apiClient.Create(ctx, relay); err != nil { + t.Fatal(err) + } + relayUID := relay.UID + poolRelayExport := multicluster.PoolRelayServiceName(relay.Name) + waitForExports(t, apiClient, relay, poolRelayExport, relay.Name+"-frontend") + waitForRelayService(t, apiClient, relay, true) + + t.Log("An MCS implementation holds the PoolRelay export with a finalizer, so its deletion will stay pending") + held := exportsByName(t, apiClient, relay.Namespace)[poolRelayExport] + heldUID := held.GetUID() + held.SetFinalizers([]string{"example.com/mcs-hold"}) + if err := apiClient.Update(ctx, held); err != nil { + t.Fatal(err) + } + + t.Log("Delete the DGD: the Relay's Service and frontend export are withdrawn, and the PoolRelay export's deletion is pending") + if err := apiClient.Delete(ctx, first); err != nil { + t.Fatal(err) + } + eventually(t, "the exports to be withdrawn", func() (bool, error) { + exports := exportsByName(t, apiClient, relay.Namespace) + pending, found := exports[poolRelayExport] + return len(exports) == 1 && found && pending.GetDeletionTimestamp() != nil, nil + }) + waitForRelayService(t, apiClient, relay, false) + + t.Log("Create a new DGD of the Relay's name with another frontend: its export is created while the old PoolRelay export is pending") + second := createDGD(t, apiClient, relay, "Web") + if second.UID == first.UID { + t.Fatal("the new DGD has the old DGD's UID") + } + eventually(t, "the new frontend's export next to the pending one", func() (bool, error) { + exports := exportsByName(t, apiClient, relay.Namespace) + web, webFound := exports[relay.Name+"-web"] + pending, pendingFound := exports[poolRelayExport] + return len(exports) == 2 && webFound && ownedByRelay(web, relay) && + pendingFound && pending.GetUID() == heldUID && pending.GetDeletionTimestamp() != nil, nil + }) + waitForRelayService(t, apiClient, relay, true) + + t.Log("The MCS implementation lets go of the old export: a new PoolRelay export is created with no other change") + held = exportsByName(t, apiClient, relay.Namespace)[poolRelayExport] + held.SetFinalizers(nil) + if err := apiClient.Update(ctx, held); err != nil { + t.Fatal(err) + } + waitForExports(t, apiClient, relay, poolRelayExport, relay.Name+"-web") + if uid := exportsByName(t, apiClient, relay.Namespace)[poolRelayExport].GetUID(); uid == heldUID { + t.Fatal("the PoolRelay export was not created again") + } + if err := apiClient.Get(ctx, client.ObjectKeyFromObject(relay), relay); err != nil { + t.Fatal(err) + } + if relay.UID != relayUID { + t.Fatal("the DynamoPoolRelay was created again") + } + + t.Log("Status updates of the DGD, such as a terminal workload failure, write nothing to the exports") + before := exportVersions(t, apiClient, relay.Namespace) + for attempt := range 3 { + reportTerminalFailure(t, apiClient, second, fmt.Sprintf("Grove is disabled (report %d)", attempt)) + } + time.Sleep(2 * time.Second) + if after := exportVersions(t, apiClient, relay.Namespace); !maps.Equal(before, after) { + t.Fatalf("DGD status updates changed the exports: %v -> %v", before, after) + } + + t.Log("Only the frontend's Service changes, so only the Service watch requeues the Relay, whose listeners are " + + "available and which therefore polls nothing: deleting the Service withdraws its export, and creating it " + + "again exports it again") + markRelayListenersAvailable(t, apiClient, relay) + frontendService := &corev1.Service{} + frontendKey := client.ObjectKey{Namespace: relay.Namespace, Name: relay.Name + "-web"} + if err := apiClient.Get(ctx, frontendKey, frontendService); err != nil { + t.Fatal(err) + } + if err := apiClient.Delete(ctx, frontendService); err != nil { + t.Fatal(err) + } + waitForExports(t, apiClient, relay, poolRelayExport) + recreated := &corev1.Service{ + Name: frontendKey.Name, Namespace: frontendKey.Namespace, + Spec: corev1.ServiceSpec{Ports: []corev1.ServicePort{{Name: "http", Port: 8000}}}, + } + if err := controllerutil.SetControllerReference(second, recreated, apiClient.Scheme()); err != nil { + t.Fatal(err) + } + if err := apiClient.Create(ctx, recreated); err != nil { + t.Fatal(err) + } + waitForExports(t, apiClient, relay, poolRelayExport, relay.Name+"-web") + + t.Log("With the controller stopped, replace the DGD at once, so that no cache sees it missing") + stop() + if err := apiClient.Delete(ctx, second); err != nil { + t.Fatal(err) + } + waitForDeletion(t, apiClient, second) + createDGD(t, apiClient, relay, "Api") + startRelayController(t, config, relayControllerOptions{version: exportTestVersion}) + waitForExports(t, apiClient, relay, poolRelayExport, relay.Name+"-api") + waitForRelayService(t, apiClient, relay, true) +} + +// failingCreates refuses the first ServiceExport creates, as an API server that +// is briefly unavailable does, and counts the attempts. +type failingCreates struct { + client.Client + remaining atomic.Int32 + attempts atomic.Int32 +} + +func (c *failingCreates) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + if obj.GetObjectKind().GroupVersionKind().GroupKind() == multicluster.ServiceExportGroupKind { + c.attempts.Add(1) + if c.remaining.Add(-1) >= 0 { + return apierrors.NewServiceUnavailable("injected failure") + } + } + return c.Client.Create(ctx, obj, opts...) +} + +func TestRelayControllerRetriesFailedExports(t *testing.T) { + if os.Getenv("KUBEBUILDER_ASSETS") == "" { + t.Skip("KUBEBUILDER_ASSETS is needed for the controller API test") + } + ctx := context.Background() + config := startExportAPIServer(t, exportTestVersion) + apiClient, err := client.New(config, client.Options{Scheme: newExportTestScheme(t)}) + if err != nil { + t.Fatal(err) + } + failing := &failingCreates{} + startRelayController(t, config, relayControllerOptions{ + version: exportTestVersion, + wrap: func(c client.Client) client.Client { + failing.Client = c + return failing + }, + }) + + t.Log("Create a pool whose DGD reports a terminal workload failure, and wait until its Relay is Ready") + relay := readSampleRelay(t) + createNamespace(t, apiClient, relay.Namespace) + dgd := createDGD(t, apiClient, relay, "Frontend") + reportTerminalFailure(t, apiClient, dgd, "Grove is disabled") + if err := apiClient.Create(ctx, relay); err != nil { + t.Fatal(err) + } + frontendExport := relay.Name + "-frontend" + waitForExports(t, apiClient, relay, multicluster.PoolRelayServiceName(relay.Name), frontendExport) + markRelayListenersAvailable(t, apiClient, relay) + + t.Log("The API server refuses the next two ServiceExport creates, and a user deletes the frontend export") + failing.attempts.Store(0) + failing.remaining.Store(2) + deleted := exportsByName(t, apiClient, relay.Namespace)[frontendExport] + if err := apiClient.Delete(ctx, deleted); err != nil { + t.Fatal(err) + } + + t.Log("Only the retries of the failed reconciles requeue the Ready Relay: the export is created again, with no other change") + waitForExports(t, apiClient, relay, multicluster.PoolRelayServiceName(relay.Name), frontendExport) + if exportsByName(t, apiClient, relay.Namespace)[frontendExport].GetUID() == deleted.GetUID() { + t.Fatal("the frontend export was not created again") + } + if attempts := failing.attempts.Load(); attempts <= 2 { + t.Fatalf("the export was created after %d attempts, without retrying the 2 refused ones", attempts) + } +} + +func TestRelayControllerWorksAtTheResolvedVersion(t *testing.T) { + if os.Getenv("KUBEBUILDER_ASSETS") == "" { + t.Skip("KUBEBUILDER_ASSETS is needed for the controller API test") + } + ctx := context.Background() + // A version other than the one the other tests' clusters serve. + const version = "v1beta1" + config := startExportAPIServer(t, version) + apiClient, err := client.New(config, client.Options{Scheme: newExportTestScheme(t)}) + if err != nil { + t.Fatal(err) + } + startRelayController(t, config, relayControllerOptions{version: version}) + + t.Log("Create a pool in a cluster that serves ServiceExports only at " + version) + relay := readSampleRelay(t) + createNamespace(t, apiClient, relay.Namespace) + createDGD(t, apiClient, relay, "Frontend") + if err := apiClient.Create(ctx, relay); err != nil { + t.Fatal(err) + } + + t.Log("The controller watches and writes ServiceExports at that version, so it exports the pool") + poolRelayExport := func() *unstructured.Unstructured { + items := listServiceExports(t, apiClient, version, relay.Namespace) + for i := range items { + if items[i].GetName() == multicluster.PoolRelayServiceName(relay.Name) && ownedByRelay(&items[i], relay) { + return &items[i] + } + } + return nil + } + eventually(t, "the PoolRelay export at "+version, func() (bool, error) { + return poolRelayExport() != nil, nil + }) + markRelayListenersAvailable(t, apiClient, relay) + + t.Log("Delete the export: only the watch at " + version + " requeues the Ready Relay, which exports its Service again") + deleted := poolRelayExport() + if err := apiClient.Delete(ctx, deleted); err != nil { + t.Fatal(err) + } + eventually(t, "a new PoolRelay export", func() (bool, error) { + export := poolRelayExport() + return export != nil && export.GetUID() != deleted.GetUID(), nil + }) +} + +func TestNamespaceRestrictedRelayControllerFollowsTheClusterID(t *testing.T) { + if os.Getenv("KUBEBUILDER_ASSETS") == "" { + t.Skip("KUBEBUILDER_ASSETS is needed for the controller API test") + } + ctx := context.Background() + config := startExportAPIServer(t, exportTestVersion) + scheme := newExportTestScheme(t) + if err := eventsv1.AddToScheme(scheme); err != nil { + t.Fatal(err) + } + apiClient, err := client.New(config, client.Options{Scheme: scheme}) + if err != nil { + t.Fatal(err) + } + relay := readSampleRelay(t) + startRelayController(t, config, relayControllerOptions{version: exportTestVersion, restricted: relay.Namespace}) + + t.Log("Create a pool in the namespace that a namespace-restricted operator manages, in a cluster with no ID in its ClusterSet") + createNamespace(t, apiClient, relay.Namespace) + createDGD(t, apiClient, relay, "Frontend") + if err := apiClient.Create(ctx, relay); err != nil { + t.Fatal(err) + } + waitForExports(t, apiClient, relay, multicluster.PoolRelayServiceName(relay.Name), relay.Name+"-frontend") + + t.Log("Each export's creation is an event of its own, though both events regard the Relay at one version") + eventually(t, "a CreateServiceExport event for each export", func() (bool, error) { + notes := strings.Join(relayEventNotes(t, apiClient, relay, "CreateServiceExport"), "\n") + return strings.Contains(notes, "Service "+multicluster.PoolRelayServiceName(relay.Name)+" ") && + strings.Contains(notes, "Service "+relay.Name+"-frontend "), nil + }) + markRelayListenersAvailable(t, apiClient, relay) + + t.Log("Give the cluster an ID in its ClusterSet that differs from the Relay's clusterId") + property := newClusterIDProperty(relay.Spec.ClusterID + "-east") + if err := apiClient.Create(ctx, property); err != nil { + t.Fatal(err) + } + mismatchWith := func(clusterID string) func() (bool, error) { + return func() (bool, error) { + for _, note := range relayEventNotes(t, apiClient, relay, reasonClusterIDMismatch) { + if strings.Contains(note, fmt.Sprintf("ID %q", clusterID)) { + return true, nil + } + } + return false, nil + } + } + + t.Log("The cluster-scoped property's event reaches the restricted controller, which reports the mismatch on the Relay") + eventually(t, "a ClusterIDMismatch event on the Relay", mismatchWith(relay.Spec.ClusterID+"-east")) + + t.Log("Change the cluster's ID: the new mismatch is an event of its own, not a repeat of the old one") + property.Object["spec"] = map[string]any{"value": relay.Spec.ClusterID + "-west"} + if err := apiClient.Update(ctx, property); err != nil { + t.Fatal(err) + } + eventually(t, "a ClusterIDMismatch event with the new ID", mismatchWith(relay.Spec.ClusterID+"-west")) +} + +// relayEventNotes returns the notes of the events with reason that regard relay. +func relayEventNotes(t *testing.T, c client.Client, relay *v1alpha1.DynamoPoolRelay, reason string) []string { + t.Helper() + recorded := &eventsv1.EventList{} + if err := c.List(context.Background(), recorded, client.InNamespace(relay.Namespace)); err != nil { + t.Fatal(err) + } + var notes []string + for i := range recorded.Items { + if recorded.Items[i].Reason == reason && recorded.Items[i].Regarding.Name == relay.Name { + notes = append(notes, recorded.Items[i].Note) + } + } + return notes +} diff --git a/deploy/operator/internal/controller/poolrelay/exports_test.go b/deploy/operator/internal/controller/poolrelay/exports_test.go new file mode 100644 index 000000000000..86a37ba02718 --- /dev/null +++ b/deploy/operator/internal/controller/poolrelay/exports_test.go @@ -0,0 +1,1695 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package poolrelay + +import ( + "context" + "errors" + "net/http" + "slices" + "strings" + "testing" + "time" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + v1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1alpha1" + v1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commoncontroller "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/multicluster" + "github.com/onsi/gomega" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + rbacv1 "k8s.io/api/rbac/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/uuid" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/events" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/event" + "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +// exportTestVersion is the multicluster.x-k8s.io version that the tests' clusters +// serve. +const exportTestVersion = "v1alpha1" + +// newExportTestScheme returns a scheme with the kinds that the controller reads +// and writes. +func newExportTestScheme(t *testing.T) *runtime.Scheme { + t.Helper() + scheme := runtime.NewScheme() + for _, add := range []func(*runtime.Scheme) error{ + v1alpha1.AddToScheme, v1beta1.AddToScheme, appsv1.AddToScheme, corev1.AddToScheme, rbacv1.AddToScheme, + } { + if err := add(scheme); err != nil { + t.Fatalf("add to scheme: %v", err) + } + } + return scheme +} + +// newExportTestClient returns a fake client that serves the scheme's kinds, the +// ServiceExport kind at version only, and the ClusterProperty kind, which the +// scheme does not register. +func newExportTestClient( + scheme *runtime.Scheme, + version string, + funcs interceptor.Funcs, + objects ...client.Object, +) client.WithWatch { + mapper := meta.NewDefaultRESTMapper(nil) + for gvk := range scheme.AllKnownTypes() { + mapper.Add(gvk, meta.RESTScopeNamespace) + } + mapper.Add(multicluster.ServiceExportGroupKind.WithVersion(version), meta.RESTScopeNamespace) + mapper.Add(multicluster.ClusterPropertyGVK, meta.RESTScopeRoot) + return fake.NewClientBuilder(). + WithScheme(scheme). + WithRESTMapper(mapper). + WithStatusSubresource(&v1alpha1.DynamoPoolRelay{}, &v1beta1.DynamoGraphDeployment{}, &appsv1.Deployment{}). + WithInterceptorFuncs(funcs). + WithObjects(objects...). + Build() +} + +// newExportTestDGD returns a DGD named name in namespace inference that a Relay +// can publish: a decode worker, and a frontend component for each of frontends. +func newExportTestDGD(name string, frontends ...string) *v1beta1.DynamoGraphDeployment { + dgd := &v1beta1.DynamoGraphDeployment{ + Name: name, Namespace: "inference", UID: uuid.NewUUID(), + Spec: v1beta1.DynamoGraphDeploymentSpec{Components: []v1beta1.DynamoComponentDeploymentSharedSpec{ + {ComponentName: "decode", ComponentType: v1beta1.ComponentTypeWorker}, + }}, + Status: v1beta1.DynamoGraphDeploymentStatus{Components: map[string]v1beta1.ComponentReplicaStatus{ + "decode": {RuntimeNamespace: "inference-" + name + "-hash"}, + }}, + } + for _, frontend := range frontends { + dgd.Spec.Components = append(dgd.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: frontend, ComponentType: v1beta1.ComponentTypeFrontend, + }) + } + return dgd +} + +// newExportTestRelay returns the Relay that makes dgd a pool of cluster east. +func newExportTestRelay(dgd *v1beta1.DynamoGraphDeployment) *v1alpha1.DynamoPoolRelay { + return &v1alpha1.DynamoPoolRelay{ + Name: dgd.Name, Namespace: dgd.Namespace, UID: uuid.NewUUID(), + Spec: relaySpec("east"), + } +} + +// ownedExport returns the export at version of the Service named name that relay +// controls, labeled with the DGD of the Relay's name. +func ownedExport( + t *testing.T, + scheme *runtime.Scheme, + version string, + relay *v1alpha1.DynamoPoolRelay, + name string, +) *unstructured.Unstructured { + t.Helper() + export := multicluster.RenderServiceExport(version, relay.Namespace, name, relay.Name) + if err := controllerutil.SetControllerReference(relay, export, scheme); err != nil { + t.Fatalf("set the owner of ServiceExport %s: %v", name, err) + } + return export +} + +// poolServices returns the Services that the operator creates for the pool: the +// PoolRelay Service, which relay controls, and a Service for each frontend, which +// dgd controls. It leaves out the Services named in skip. +func poolServices( + t *testing.T, + scheme *runtime.Scheme, + relay *v1alpha1.DynamoPoolRelay, + dgd *v1beta1.DynamoGraphDeployment, + skip ...string, +) []client.Object { + t.Helper() + var services []client.Object + for _, name := range multicluster.ServiceNames(dgd) { + if slices.Contains(skip, name) { + continue + } + var controller metav1.Object = dgd + if name == multicluster.PoolRelayServiceName(dgd.Name) { + controller = relay + } + service := &corev1.Service{Name: name, Namespace: dgd.Namespace} + if err := controllerutil.SetControllerReference(controller, service, scheme); err != nil { + t.Fatalf("set the controller of Service %s: %v", name, err) + } + services = append(services, service) + } + return services +} + +// listServiceExports returns the ServiceExports in namespace, read at version. +func listServiceExports(t *testing.T, c client.Client, version, namespace string) []unstructured.Unstructured { + t.Helper() + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(multicluster.ServiceExportGroupKind.WithVersion(version).GroupVersion().WithKind("ServiceExportList")) + if err := c.List(context.Background(), list, client.InNamespace(namespace)); err != nil { + t.Fatalf("list ServiceExports: %v", err) + } + return list.Items +} + +// exportNames returns the names of exports. +func exportNames(exports []unstructured.Unstructured) []string { + var names []string + for i := range exports { + names = append(names, exports[i].GetName()) + } + return names +} + +// newExportTestReconciler returns the Relay controller with the pool's exports on +// for implementation, reading and writing exports at version through c. +func newExportTestReconciler( + c client.WithWatch, + recorder events.EventRecorder, + implementation configv1alpha1.MCSImplementation, + version string, +) *reconciler { + return &reconciler{ + Client: c, + scheme: c.Scheme(), + defaultBackend: configv1alpha1.DiscoveryBackendKubernetes, + exports: &exportSettings{ + cache: c, + recorder: recorder, + config: &configv1alpha1.OperatorConfiguration{ + MultiCluster: configv1alpha1.MultiClusterConfiguration{MCSImplementation: implementation}, + }, + runtimeConfig: &commoncontroller.RuntimeConfig{ + Gate: features.Gates{MultiClusterExport: true, ServiceExportVersion: version}, + }, + }, + } +} + +// drainEvents returns the events recorder holds. +func drainEvents(recorder *events.FakeRecorder) []string { + var recorded []string + for len(recorder.Events) > 0 { + recorded = append(recorded, <-recorder.Events) + } + return recorded +} + +// eventReasons returns the reason of each recorded event, in order. +func eventReasons(recorded []string) []string { + var reasons []string + for _, recordedEvent := range recorded { + if fields := strings.Fields(recordedEvent); len(fields) >= 2 { + reasons = append(reasons, fields[1]) + } + } + return reasons +} + +// countReasons returns how many of the recorded events have reason. +func countReasons(recorded []string, reason string) int { + count := 0 + for _, recordedReason := range eventReasons(recorded) { + if recordedReason == reason { + count++ + } + } + return count +} + +func TestReconcileExports(t *testing.T) { + // -pool-relay is 56 characters, one more than Karmada imports; + // -frontend is 54. + longName := strings.Repeat("a", 45) + + for _, tc := range []struct { + name string + dgdName string + frontends []string + implementation configv1alpha1.MCSImplementation + ownedBefore []string + foreign []string + // ownedByAnother are exports that another Relay controls. + ownedByAnother []string + // elsewhere are a user's exports in namespace other. + elsewhere []string + // foreignServices are Services of the pool's names that no one controls, + // missingServices do not exist, and dcdServices are controlled by a DCD + // that the DGD controls. + foreignServices []string + missingServices []string + dcdServices []string + // otherServices are controlled by another DGD, and otherDCDServices by a + // DCD of another DGD. + otherServices []string + otherDCDServices []string + wantOwned []string + // wantReasons are the reasons of the events of the first reconcile. The + // second changes nothing, so it records none. + wantReasons []string + // wantCondition is the reason of the Relay's Exported condition. + wantCondition string + }{ + { + name: "a pool exports its PoolRelay and frontend Services", + dgdName: "green", frontends: []string{"Frontend"}, + implementation: configv1alpha1.MCSImplementationKarmada, + wantOwned: []string{"green-pool-relay", "green-frontend"}, + wantReasons: []string{"CreateServiceExport", "CreateServiceExport"}, + wantCondition: reasonExported, + }, + { + name: "each frontend component's Service is exported under its own name", + dgdName: "green", frontends: []string{"Frontend", "Web.v2"}, + implementation: configv1alpha1.MCSImplementationStandard, + wantOwned: []string{"green-pool-relay", "green-frontend", "green-web-v2"}, + wantReasons: []string{"CreateServiceExport", "CreateServiceExport", "CreateServiceExport"}, + wantCondition: reasonExported, + }, + { + name: "a renamed frontend's export moves to its new Service", + dgdName: "green", frontends: []string{"Web"}, + implementation: configv1alpha1.MCSImplementationKarmada, + ownedBefore: []string{"green-pool-relay", "green-frontend"}, + wantOwned: []string{"green-pool-relay", "green-web"}, + wantReasons: []string{"CreateServiceExport", "DeleteServiceExport"}, + wantCondition: reasonExported, + }, + { + name: "a pool without a frontend exports its PoolRelay Service and warns", + dgdName: "green", + implementation: configv1alpha1.MCSImplementationStandard, + wantOwned: []string{"green-pool-relay"}, + wantReasons: []string{"CreateServiceExport", reasonNoFrontend}, + wantCondition: reasonNoFrontend, + }, + { + // A Service name has at most 63 characters, and the operator does not + // shorten the names of the frontends' Services. + name: "a frontend Service name that no Service can have is not exported", + dgdName: "green", frontends: []string{"Frontend", strings.Repeat("w", 60)}, + implementation: configv1alpha1.MCSImplementationKarmada, + wantOwned: []string{"green-pool-relay", "green-frontend"}, + wantReasons: []string{"CreateServiceExport", "CreateServiceExport", reasonInvalidServiceName}, + wantCondition: reasonInvalidServiceName, + }, + { + name: "an export a user created is neither adopted nor deleted", + dgdName: "green", frontends: []string{"Frontend"}, + implementation: configv1alpha1.MCSImplementationKarmada, + foreign: []string{"green-frontend"}, + wantOwned: []string{"green-pool-relay"}, + wantReasons: []string{"CreateServiceExport"}, + wantCondition: reasonExported, + }, + { + name: "another Relay's export of a Service the pool no longer has is left alone", + dgdName: "green", frontends: []string{"Frontend"}, + implementation: configv1alpha1.MCSImplementationStandard, + ownedByAnother: []string{"green-old"}, + wantOwned: []string{"green-pool-relay", "green-frontend"}, + wantReasons: []string{"CreateServiceExport", "CreateServiceExport"}, + wantCondition: reasonExported, + }, + { + name: "Karmada skips a Service name it cannot import", + dgdName: longName, frontends: []string{"Frontend"}, + implementation: configv1alpha1.MCSImplementationKarmada, + wantOwned: []string{longName + "-frontend"}, + wantReasons: []string{"CreateServiceExport", reasonServiceNameTooLong}, + wantCondition: reasonServiceNameTooLong, + }, + { + name: "standard MCS exports the same long names", + dgdName: longName, frontends: []string{"Frontend"}, + implementation: configv1alpha1.MCSImplementationStandard, + wantOwned: []string{longName + "-pool-relay", longName + "-frontend"}, + wantReasons: []string{"CreateServiceExport", "CreateServiceExport"}, + wantCondition: reasonExported, + }, + { + name: "Karmada withdraws the Relay's export of a Service name it cannot import", + dgdName: longName, frontends: []string{"Frontend"}, + implementation: configv1alpha1.MCSImplementationKarmada, + ownedBefore: []string{longName + "-pool-relay", longName + "-frontend"}, + wantOwned: []string{longName + "-frontend"}, + wantReasons: []string{"DeleteServiceExport", reasonServiceNameTooLong}, + wantCondition: reasonServiceNameTooLong, + }, + { + name: "a Service that the pool does not control is not exported", + dgdName: "green", frontends: []string{"Frontend"}, + implementation: configv1alpha1.MCSImplementationStandard, + foreignServices: []string{"green-frontend"}, + wantOwned: []string{"green-pool-relay"}, + wantReasons: []string{"CreateServiceExport", reasonServiceNotOwned}, + wantCondition: reasonServiceNotOwned, + }, + { + name: "the Relay withdraws its export of a Service that the pool no longer controls", + dgdName: "green", frontends: []string{"Frontend"}, + implementation: configv1alpha1.MCSImplementationKarmada, + ownedBefore: []string{"green-pool-relay", "green-frontend"}, + foreignServices: []string{"green-frontend"}, + wantOwned: []string{"green-pool-relay"}, + wantReasons: []string{"DeleteServiceExport", reasonServiceNotOwned}, + wantCondition: reasonServiceNotOwned, + }, + { + name: "a Service that another DGD controls is not exported", + dgdName: "green", frontends: []string{"Frontend"}, + implementation: configv1alpha1.MCSImplementationStandard, + otherServices: []string{"green-frontend"}, + wantOwned: []string{"green-pool-relay"}, + wantReasons: []string{"CreateServiceExport", reasonServiceNotOwned}, + wantCondition: reasonServiceNotOwned, + }, + { + name: "a Service that a DCD of another DGD controls is not exported", + dgdName: "green", frontends: []string{"Frontend"}, + implementation: configv1alpha1.MCSImplementationStandard, + otherDCDServices: []string{"green-frontend"}, + wantOwned: []string{"green-pool-relay"}, + wantReasons: []string{"CreateServiceExport", reasonServiceNotOwned}, + wantCondition: reasonServiceNotOwned, + }, + { + name: "a Service that does not exist yet is not exported", + dgdName: "green", frontends: []string{"Frontend"}, + implementation: configv1alpha1.MCSImplementationStandard, + missingServices: []string{"green-frontend"}, + wantOwned: []string{"green-pool-relay"}, + wantReasons: []string{"CreateServiceExport"}, + wantCondition: reasonExported, + }, + { + name: "a frontend Service that a DCD of the DGD controls is exported", + dgdName: "green", frontends: []string{"Frontend"}, + implementation: configv1alpha1.MCSImplementationKarmada, + dcdServices: []string{"green-frontend"}, + wantOwned: []string{"green-pool-relay", "green-frontend"}, + wantReasons: []string{"CreateServiceExport", "CreateServiceExport"}, + wantCondition: reasonExported, + }, + { + name: "exports of the same names in another namespace do not count", + dgdName: "green", frontends: []string{"Frontend"}, + implementation: configv1alpha1.MCSImplementationStandard, + elsewhere: []string{"green-pool-relay", "green-frontend"}, + wantOwned: []string{"green-pool-relay", "green-frontend"}, + wantReasons: []string{"CreateServiceExport", "CreateServiceExport"}, + wantCondition: reasonExported, + }, + } { + t.Run(tc.name, func(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create the Relay, its DGD, and the exports that exist before reconciling") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD(tc.dgdName, tc.frontends...) + relay := newExportTestRelay(dgd) + objects := []client.Object{relay, dgd} + skip := slices.Concat(tc.foreignServices, tc.missingServices, tc.dcdServices, tc.otherServices, tc.otherDCDServices) + objects = append(objects, poolServices(t, scheme, relay, dgd, skip...)...) + for _, name := range tc.foreignServices { + objects = append(objects, &corev1.Service{Name: name, Namespace: dgd.Namespace}) + } + other := newExportTestDGD("other") + for _, name := range tc.otherServices { + service := &corev1.Service{Name: name, Namespace: dgd.Namespace} + g.Expect(controllerutil.SetControllerReference(other, service, scheme)).To(gomega.Succeed()) + objects = append(objects, service) + } + for _, name := range slices.Concat(tc.dcdServices, tc.otherDCDServices) { + parent := dgd + if slices.Contains(tc.otherDCDServices, name) { + parent = other + } + dcd := &v1beta1.DynamoComponentDeployment{Name: name, Namespace: dgd.Namespace, UID: uuid.NewUUID()} + service := &corev1.Service{Name: name, Namespace: dgd.Namespace} + g.Expect(controllerutil.SetControllerReference(parent, dcd, scheme)).To(gomega.Succeed()) + g.Expect(controllerutil.SetControllerReference(dcd, service, scheme)).To(gomega.Succeed()) + objects = append(objects, dcd, service) + } + for _, name := range tc.ownedBefore { + objects = append(objects, ownedExport(t, scheme, exportTestVersion, relay, name)) + } + for _, name := range tc.foreign { + objects = append(objects, multicluster.RenderServiceExport(exportTestVersion, dgd.Namespace, name, dgd.Name)) + } + another := newExportTestRelay(newExportTestDGD("another")) + for _, name := range tc.ownedByAnother { + objects = append(objects, ownedExport(t, scheme, exportTestVersion, another, name)) + } + for _, name := range tc.elsewhere { + objects = append(objects, multicluster.RenderServiceExport(exportTestVersion, "other", name, dgd.Name)) + } + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, objects...) + + t.Log("Reconcile the pool's exports, as the Relay controller does when it publishes the Relay") + recorder := events.NewFakeRecorder(20) + r := newExportTestReconciler(fakeClient, recorder, tc.implementation, exportTestVersion) + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.Succeed()) + first := eventReasons(drainEvents(recorder)) + g.Expect(first).To(gomega.ConsistOf(tc.wantReasons)) + + t.Log("Reconcile again: every export is in place and the condition is unchanged, so nothing is recorded") + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.Succeed()) + g.Expect(drainEvents(recorder)).To(gomega.BeEmpty()) + + t.Log("Check the Relay's Exported condition") + stored := &v1alpha1.DynamoPoolRelay{} + g.Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(relay), stored)).To(gomega.Succeed()) + condition := meta.FindStatusCondition(stored.Status.Conditions, conditionExported) + g.Expect(condition).NotTo(gomega.BeNil()) + g.Expect(condition.Reason).To(gomega.Equal(tc.wantCondition)) + wantStatus := metav1.ConditionFalse + if tc.wantCondition == reasonExported { + wantStatus = metav1.ConditionTrue + } + g.Expect(condition.Status).To(gomega.Equal(wantStatus)) + + t.Log("Check which exports the Relay controls, and that other exports are untouched") + var owned, others []string + for i, export := range listServiceExports(t, fakeClient, exportTestVersion, dgd.Namespace) { + if !metav1.IsControlledBy(&export, relay) { + others = append(others, export.GetName()) + continue + } + owned = append(owned, export.GetName()) + g.Expect(export.GetLabels()).To(gomega.HaveKeyWithValue(commonconsts.KubeLabelDynamoGraphDeploymentName, dgd.Name), i) + } + g.Expect(owned).To(gomega.ConsistOf(tc.wantOwned)) + g.Expect(others).To(gomega.ConsistOf(append(append([]string{}, tc.foreign...), tc.ownedByAnother...))) + var elsewhere []string + for i, export := range listServiceExports(t, fakeClient, exportTestVersion, "other") { + elsewhere = append(elsewhere, export.GetName()) + g.Expect(metav1.GetControllerOf(&export)).To(gomega.BeNil(), i) + } + g.Expect(elsewhere).To(gomega.ConsistOf(tc.elsewhere)) + }) + } +} + +func TestWithdrawExports(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create a Relay's exports, one of them already being deleted, a user's export and another Relay's") + scheme := newExportTestScheme(t) + relay := newExportTestRelay(newExportTestDGD("green")) + terminating := ownedExport(t, scheme, exportTestVersion, relay, "green-old") + terminating.SetFinalizers([]string{"example.com/hold"}) + terminating.SetDeletionTimestamp(&metav1.Time{Time: time.Now()}) + another := newExportTestRelay(newExportTestDGD("blue")) + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, relay, + ownedExport(t, scheme, exportTestVersion, relay, "green-pool-relay"), + ownedExport(t, scheme, exportTestVersion, relay, "green-frontend"), + terminating, + multicluster.RenderServiceExport(exportTestVersion, relay.Namespace, "green-web", relay.Name), + ownedExport(t, scheme, exportTestVersion, another, "blue-frontend")) + + t.Log("Withdraw the Relay's exports, as the Relay controller does when it withdraws the Relay") + recorder := events.NewFakeRecorder(10) + r := newExportTestReconciler(fakeClient, recorder, configv1alpha1.MCSImplementationStandard, exportTestVersion) + g.Expect(r.withdrawExports(ctx, relay)).To(gomega.Succeed()) + g.Expect(eventReasons(drainEvents(recorder))).To(gomega.ConsistOf("DeleteServiceExport", "DeleteServiceExport")) + g.Expect(exportNames(listServiceExports(t, fakeClient, exportTestVersion, relay.Namespace))). + To(gomega.ConsistOf("green-old", "green-web", "blue-frontend")) + + t.Log("The Relay's Exported condition says that the pool's exports are withdrawn") + stored := &v1alpha1.DynamoPoolRelay{} + g.Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(relay), stored)).To(gomega.Succeed()) + condition := meta.FindStatusCondition(stored.Status.Conditions, conditionExported) + g.Expect(condition).NotTo(gomega.BeNil()) + g.Expect(condition.Status).To(gomega.Equal(metav1.ConditionFalse)) + g.Expect(condition.Reason).To(gomega.Equal(reasonWithdrawn)) +} + +func TestRelayControllerMaintainsThePoolsExports(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create a DGD that a Relay can publish, and its Relay") + scheme := newExportTestScheme(t) + first := newExportTestDGD("green", "Frontend") + relay := newExportTestRelay(first) + poolRelayService := multicluster.PoolRelayServiceName(relay.Name) + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, + append([]client.Object{relay, first}, poolServices(t, scheme, relay, first, poolRelayService)...)...) + r := newExportTestReconciler(fakeClient, events.NewFakeRecorder(50), configv1alpha1.MCSImplementationKarmada, exportTestVersion) + request := ctrl.Request{NamespacedName: client.ObjectKeyFromObject(relay)} + serviceKey := client.ObjectKey{Namespace: relay.Namespace, Name: poolRelayService} + + t.Log("Reconcile: the Relay's Service is published, and the pool's exports are created") + _, err := r.Reconcile(ctx, request) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(fakeClient.Get(ctx, serviceKey, &corev1.Service{})).To(gomega.Succeed()) + g.Expect(exportNames(listServiceExports(t, fakeClient, exportTestVersion, relay.Namespace))). + To(gomega.ConsistOf("green-pool-relay", "green-frontend")) + + t.Log("Delete the DGD: the Relay's Service and the pool's exports are withdrawn, and the Relay stays") + g.Expect(fakeClient.Delete(ctx, first)).To(gomega.Succeed()) + _, err = r.Reconcile(ctx, request) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(apierrors.IsNotFound(fakeClient.Get(ctx, serviceKey, &corev1.Service{}))).To(gomega.BeTrue()) + g.Expect(listServiceExports(t, fakeClient, exportTestVersion, relay.Namespace)).To(gomega.BeEmpty()) + + t.Log("Create a new DGD of the Relay's name with another frontend and its Service: the Relay publishes it, and its exports") + second := newExportTestDGD("green", "Web") + g.Expect(fakeClient.Create(ctx, second)).To(gomega.Succeed()) + for _, service := range poolServices(t, scheme, relay, second, poolRelayService) { + g.Expect(fakeClient.Create(ctx, service)).To(gomega.Succeed()) + } + _, err = r.Reconcile(ctx, request) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(fakeClient.Get(ctx, serviceKey, &corev1.Service{})).To(gomega.Succeed()) + g.Expect(exportNames(listServiceExports(t, fakeClient, exportTestVersion, relay.Namespace))). + To(gomega.ConsistOf("green-pool-relay", "green-web")) + + t.Log("Leave the DGD without workers: the Relay withdraws, and the pool's exports with it") + g.Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(second), second)).To(gomega.Succeed()) + second.Spec.Components = second.Spec.Components[1:] + g.Expect(fakeClient.Update(ctx, second)).To(gomega.Succeed()) + _, err = r.Reconcile(ctx, request) + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(apierrors.IsNotFound(fakeClient.Get(ctx, serviceKey, &corev1.Service{}))).To(gomega.BeTrue()) + g.Expect(listServiceExports(t, fakeClient, exportTestVersion, relay.Namespace)).To(gomega.BeEmpty()) +} + +func TestPoolExportsDoNotWaitForTheRelaysDeployment(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create a publishable Relay while the API server refuses its Deployment") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Frontend") + relay := newExportTestRelay(dgd) + refused := apierrors.NewServiceUnavailable("the API server is restarting") + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if _, ok := obj.(*appsv1.Deployment); ok { + return refused + } + return c.Create(ctx, obj, opts...) + }, + }, append([]client.Object{relay, dgd}, poolServices(t, scheme, relay, dgd, multicluster.PoolRelayServiceName(relay.Name))...)...) + r := newExportTestReconciler(fakeClient, events.NewFakeRecorder(20), configv1alpha1.MCSImplementationKarmada, exportTestVersion) + + t.Log("Reconcile: the Deployment's failure is returned, and the pool's exports are created anyway") + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(relay)}) + g.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("restarting"))) + g.Expect(exportNames(listServiceExports(t, fakeClient, exportTestVersion, relay.Namespace))). + To(gomega.ConsistOf("green-pool-relay", "green-frontend")) +} + +func TestRelayControllerReturnsAFailedExportWrite(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create a publishable Relay while the API server refuses ServiceExports") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Frontend") + relay := newExportTestRelay(dgd) + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if obj.GetObjectKind().GroupVersionKind().GroupKind() == multicluster.ServiceExportGroupKind { + return apierrors.NewServiceUnavailable("the API server is restarting") + } + return c.Create(ctx, obj, opts...) + }, + }, append([]client.Object{relay, dgd}, poolServices(t, scheme, relay, dgd, multicluster.PoolRelayServiceName(relay.Name))...)...) + recorder := events.NewFakeRecorder(20) + r := newExportTestReconciler(fakeClient, recorder, configv1alpha1.MCSImplementationKarmada, exportTestVersion) + + t.Log("Reconcile: the Relay is published, and the failed export is returned for controller-runtime to retry with backoff") + result, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(relay)}) + g.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("restarting"))) + g.Expect(result).To(gomega.Equal(ctrl.Result{})) + key := client.ObjectKey{Namespace: relay.Namespace, Name: multicluster.PoolRelayServiceName(relay.Name)} + g.Expect(fakeClient.Get(ctx, key, &appsv1.Deployment{})).To(gomega.Succeed()) + g.Expect(countReasons(drainEvents(recorder), reasonExportFailed)).To(gomega.Equal(1)) +} + +func TestOwnershipConflictWithdrawsThePoolsExports(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create a published pool's exports, and a user's Service that takes the Relay Service's name") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Frontend") + relay := newExportTestRelay(dgd) + foreign := &corev1.Service{Name: multicluster.PoolRelayServiceName(relay.Name), Namespace: relay.Namespace} + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, relay, dgd, foreign, + ownedExport(t, scheme, exportTestVersion, relay, "green-pool-relay"), + ownedExport(t, scheme, exportTestVersion, relay, "green-frontend")) + r := newExportTestReconciler(fakeClient, events.NewFakeRecorder(20), configv1alpha1.MCSImplementationKarmada, exportTestVersion) + + t.Log("Reconcile: the conflict withdraws the Relay, and the pool's exports with it") + _, err := r.Reconcile(ctx, ctrl.Request{NamespacedName: client.ObjectKeyFromObject(relay)}) + g.Expect(errors.Is(err, errOwnershipConflict)).To(gomega.BeTrue(), "%v", err) + g.Expect(listServiceExports(t, fakeClient, exportTestVersion, relay.Namespace)).To(gomega.BeEmpty()) +} + +func TestReconcileExportsUsesTheResolvedVersion(t *testing.T) { + g := gomega.NewGomegaWithT(t) + const version = "v1beta1" + + t.Log("Serve ServiceExports at a version other than the tests' default, with an export to withdraw") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Web") + relay := newExportTestRelay(dgd) + fakeClient := newExportTestClient(scheme, version, interceptor.Funcs{}, append([]client.Object{ + dgd, relay, ownedExport(t, scheme, version, relay, "green-frontend"), + }, poolServices(t, scheme, relay, dgd)...)...) + + t.Log("Reconcile with the gates resolved to that version: exports are listed, created and withdrawn at it") + recorder := events.NewFakeRecorder(10) + r := newExportTestReconciler(fakeClient, recorder, configv1alpha1.MCSImplementationKarmada, version) + g.Expect(r.reconcileExports(context.Background(), relay, dgd)).To(gomega.Succeed()) + g.Expect(recorder.Events).To(gomega.HaveLen(3)) + exports := listServiceExports(t, fakeClient, version, dgd.Namespace) + for i := range exports { + g.Expect(exports[i].GetAPIVersion()).To(gomega.Equal("multicluster.x-k8s.io/" + version)) + } + g.Expect(exportNames(exports)).To(gomega.ConsistOf("green-pool-relay", "green-web")) +} + +func TestReconcileExportsToleratesAStaleCache(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + t.Log("The cache has not yet seen exports that the API server already has") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Frontend") + relay := newExportTestRelay(dgd) + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + return apierrors.NewAlreadyExists( + schema.GroupResource{Group: multicluster.ServiceExportGroupKind.Group, Resource: "serviceexports"}, obj.GetName()) + }, + }, append([]client.Object{dgd, relay}, poolServices(t, scheme, relay, dgd)...)...) + + t.Log("Reconcile: AlreadyExists is no error, and nothing is announced") + recorder := events.NewFakeRecorder(10) + r := newExportTestReconciler(fakeClient, recorder, configv1alpha1.MCSImplementationKarmada, exportTestVersion) + g.Expect(r.reconcileExports(context.Background(), relay, dgd)).To(gomega.Succeed()) + g.Expect(recorder.Events).To(gomega.BeEmpty()) +} + +func TestPoolExportsRetryFailedWrites(t *testing.T) { + unavailable := apierrors.NewServiceUnavailable("the API server is restarting") + for _, tc := range []struct { + name string + // failOnce fails the first write of the case's kind. + failOnce func(failed *bool) interceptor.Funcs + // withdraw withdraws the exports instead of reconciling them. + withdraw bool + // wantAfterFailure are the exports after the failed write: the failure + // does not hold back the other export's write. + wantAfterFailure []string + want []string + }{ + { + name: "a failed create", + failOnce: func(failed *bool) interceptor.Funcs { + return interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if !*failed { + *failed = true + return unavailable + } + return c.Create(ctx, obj, opts...) + }, + } + }, + wantAfterFailure: []string{"green-frontend"}, + want: []string{"green-pool-relay", "green-frontend"}, + }, + { + name: "a failed delete", + failOnce: func(failed *bool) interceptor.Funcs { + return interceptor.Funcs{ + Delete: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.DeleteOption) error { + if !*failed { + *failed = true + return unavailable + } + return c.Delete(ctx, obj, opts...) + }, + } + }, + withdraw: true, + wantAfterFailure: []string{"green-frontend"}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create a Relay whose exports are to be created or withdrawn, and fail the first such write") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Frontend") + relay := newExportTestRelay(dgd) + objects := append([]client.Object{relay, dgd}, poolServices(t, scheme, relay, dgd)...) + if tc.withdraw { + objects = append(objects, + ownedExport(t, scheme, exportTestVersion, relay, "green-pool-relay"), + ownedExport(t, scheme, exportTestVersion, relay, "green-frontend")) + } + failed := false + fakeClient := newExportTestClient(scheme, exportTestVersion, tc.failOnce(&failed), objects...) + recorder := events.NewFakeRecorder(10) + r := newExportTestReconciler(fakeClient, recorder, configv1alpha1.MCSImplementationKarmada, exportTestVersion) + + t.Log("The failure is returned, for controller-runtime to retry with backoff, and is announced") + var err error + if tc.withdraw { + err = r.withdrawExports(ctx, relay) + } else { + err = r.reconcileExports(ctx, relay, dgd) + } + g.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("restarting"))) + g.Expect(drainEvents(recorder)).To(gomega.ContainElement(gomega.HavePrefix(corev1.EventTypeWarning + " " + reasonExportFailed))) + g.Expect(exportNames(listServiceExports(t, fakeClient, exportTestVersion, relay.Namespace))).To(gomega.ConsistOf(tc.wantAfterFailure)) + + t.Log("The retry runs once the API server answers: the exports converge with no other change") + if tc.withdraw { + err = r.withdrawExports(ctx, relay) + } else { + err = r.reconcileExports(ctx, relay, dgd) + } + g.Expect(err).To(gomega.Succeed()) + g.Expect(exportNames(listServiceExports(t, fakeClient, exportTestVersion, relay.Namespace))).To(gomega.ConsistOf(tc.want)) + }) + } +} + +func TestPoolExportsAreRepaired(t *testing.T) { + for _, tc := range []struct { + name string + drift func(export *unstructured.Unstructured) + }{ + { + name: "no DGD label", + drift: func(export *unstructured.Unstructured) { + labels := export.GetLabels() + delete(labels, commonconsts.KubeLabelDynamoGraphDeploymentName) + export.SetLabels(labels) + }, + }, + { + name: "a DGD label naming another DGD", + drift: func(export *unstructured.Unstructured) { + labels := export.GetLabels() + labels[commonconsts.KubeLabelDynamoGraphDeploymentName] = "blue" + export.SetLabels(labels) + }, + }, + { + name: "a controller reference that no longer blocks the Relay's deletion", + drift: func(export *unstructured.Unstructured) { + references := export.GetOwnerReferences() + references[0].BlockOwnerDeletion = new(false) + export.SetOwnerReferences(references) + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create a pool whose frontend export, which the Relay controls, has " + tc.name + + ", and carries the MCS implementation's status and others' finalizer, label and annotation") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Frontend") + relay := newExportTestRelay(dgd) + intact := ownedExport(t, scheme, exportTestVersion, relay, "green-pool-relay") + changed := ownedExport(t, scheme, exportTestVersion, relay, "green-frontend") + changed.Object["status"] = map[string]any{"conditions": []any{ + map[string]any{"type": "Ready", "status": "True", "reason": "Exported", "message": ""}, + }} + changed.SetUID("export-frontend") + changed.SetFinalizers([]string{"example.com/hold"}) + changed.SetAnnotations(map[string]string{"example.com/note": "kept"}) + changed.SetLabels(map[string]string{commonconsts.KubeLabelDynamoGraphDeploymentName: dgd.Name, "team": "router"}) + tc.drift(changed) + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, append([]client.Object{ + dgd, relay, intact.DeepCopy(), changed.DeepCopy(), + }, poolServices(t, scheme, relay, dgd)...)...) + + t.Log("Reconcile: the export is repaired, and the repair is recorded") + recorder := events.NewFakeRecorder(10) + r := newExportTestReconciler(fakeClient, recorder, configv1alpha1.MCSImplementationKarmada, exportTestVersion) + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.Succeed()) + g.Expect(eventReasons(drainEvents(recorder))).To(gomega.ConsistOf("RepairServiceExport")) + + t.Log("Check that it is the same export, with the controller's fields restored and everything else kept") + repaired := multicluster.NewServiceExport(exportTestVersion) + g.Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(changed), repaired)).To(gomega.Succeed()) + g.Expect(repaired.GetUID()).To(gomega.Equal(changed.GetUID())) + g.Expect(repaired.GetLabels()).To(gomega.Equal(map[string]string{ + commonconsts.KubeLabelDynamoGraphDeploymentName: dgd.Name, "team": "router", + })) + g.Expect(metav1.GetControllerOf(repaired)).To(gomega.Equal(metav1.GetControllerOf(intact))) + g.Expect(repaired.GetAnnotations()).To(gomega.Equal(changed.GetAnnotations())) + g.Expect(repaired.GetFinalizers()).To(gomega.Equal(changed.GetFinalizers())) + g.Expect(repaired.Object["status"]).To(gomega.Equal(changed.Object["status"])) + + t.Log("Reconcile again: the export has the controller's fields, so nothing is applied or recorded") + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.Succeed()) + g.Expect(drainEvents(recorder)).To(gomega.BeEmpty()) + }) + } +} + +func TestPoolExportsBeingDeletedAreLeftAlone(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create a pool whose frontend export, which the Relay controls, has no DGD label and is being deleted") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Frontend") + relay := newExportTestRelay(dgd) + terminating := ownedExport(t, scheme, exportTestVersion, relay, "green-frontend") + terminating.SetLabels(nil) + terminating.SetFinalizers([]string{"example.com/hold"}) + terminating.SetDeletionTimestamp(&metav1.Time{Time: time.Now()}) + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, append([]client.Object{ + dgd, relay, ownedExport(t, scheme, exportTestVersion, relay, "green-pool-relay"), terminating.DeepCopy(), + }, poolServices(t, scheme, relay, dgd)...)...) + + t.Log("Reconcile: while its deletion is pending, the export is neither repaired nor created again") + recorder := events.NewFakeRecorder(10) + r := newExportTestReconciler(fakeClient, recorder, configv1alpha1.MCSImplementationStandard, exportTestVersion) + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.Succeed()) + g.Expect(drainEvents(recorder)).To(gomega.BeEmpty()) + current := multicluster.NewServiceExport(exportTestVersion) + g.Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(terminating), current)).To(gomega.Succeed()) + g.Expect(current.GetLabels()).To(gomega.BeEmpty()) + g.Expect(current.GetDeletionTimestamp()).NotTo(gomega.BeNil()) +} + +// staleCacheSetUp returns a pool's Relay and DGD; a cache that holds both of the +// Relay's exports as the API server held them when the cache read them, with the +// frontend export's DGD label removed; the API server, which holds the same +// exports for now; and the controller, which reads exports from the cache and +// writes them to the API server. +func staleCacheSetUp(t *testing.T) ( + *v1alpha1.DynamoPoolRelay, *v1beta1.DynamoGraphDeployment, client.WithWatch, *reconciler, *events.FakeRecorder, *runtime.Scheme, +) { + t.Helper() + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Frontend") + relay := newExportTestRelay(dgd) + relayExport := ownedExport(t, scheme, exportTestVersion, relay, "green-pool-relay") + relayExport.SetUID("export-relay") + frontendExport := ownedExport(t, scheme, exportTestVersion, relay, "green-frontend") + frontendExport.SetUID("export-frontend") + frontendExport.SetLabels(nil) + cache := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, relayExport.DeepCopy(), frontendExport.DeepCopy()) + apiServer := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, append([]client.Object{ + relay, dgd, relayExport.DeepCopy(), frontendExport.DeepCopy(), + }, poolServices(t, scheme, relay, dgd)...)...) + recorder := events.NewFakeRecorder(10) + r := newExportTestReconciler(apiServer, recorder, configv1alpha1.MCSImplementationKarmada, exportTestVersion) + r.exports.cache = cache + return relay, dgd, apiServer, r, recorder, scheme +} + +func TestPoolExportsAreRepairedOnlyAsTheyWereRead(t *testing.T) { + frontendKey := client.ObjectKey{Namespace: "inference", Name: "green-frontend"} + + t.Run("an export that a user took over stays theirs", func(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + relay, dgd, apiServer, r, recorder, _ := staleCacheSetUp(t) + + t.Log("After the cache read the export, a user removes its controller reference") + export := multicluster.NewServiceExport(exportTestVersion) + g.Expect(apiServer.Get(ctx, frontendKey, export)).To(gomega.Succeed()) + export.SetOwnerReferences(nil) + g.Expect(apiServer.Update(ctx, export)).To(gomega.Succeed()) + + t.Log("Reconcile: the apply names the version the cache read, so the API server refuses it, which is no error") + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.Succeed()) + g.Expect(drainEvents(recorder)).To(gomega.BeEmpty()) + + t.Log("Check that the Relay did not take the export back") + g.Expect(apiServer.Get(ctx, frontendKey, export)).To(gomega.Succeed()) + g.Expect(export.GetOwnerReferences()).To(gomega.BeEmpty()) + g.Expect(export.GetLabels()).To(gomega.BeEmpty()) + }) + + t.Run("a deleted export is not created again by a repair", func(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + relay, dgd, apiServer, r, recorder, _ := staleCacheSetUp(t) + + t.Log("After the cache read the export, it is deleted") + export := multicluster.NewServiceExport(exportTestVersion) + export.SetNamespace(frontendKey.Namespace) + export.SetName(frontendKey.Name) + g.Expect(apiServer.Delete(ctx, export)).To(gomega.Succeed()) + + t.Log("Reconcile: the apply names the UID the cache read, so the API server refuses it instead of creating the export") + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.Succeed()) + g.Expect(drainEvents(recorder)).To(gomega.BeEmpty()) + err := apiServer.Get(ctx, frontendKey, multicluster.NewServiceExport(exportTestVersion)) + g.Expect(apierrors.IsNotFound(err)).To(gomega.BeTrue(), "%v", err) + }) +} + +func TestPoolExportsAreWithdrawnOnlyAsTheyWereRead(t *testing.T) { + for _, tc := range []struct { + name string + // change changes the controller reference of an export, but not its UID. + change func(t *testing.T, scheme *runtime.Scheme, export *unstructured.Unstructured) + }{ + { + name: "a user removes the controller reference", + change: func(_ *testing.T, _ *runtime.Scheme, export *unstructured.Unstructured) { + export.SetOwnerReferences(nil) + }, + }, + { + name: "another Relay takes the export over", + change: func(t *testing.T, scheme *runtime.Scheme, export *unstructured.Unstructured) { + export.SetOwnerReferences(nil) + if err := controllerutil.SetControllerReference(newExportTestRelay(newExportTestDGD("blue")), export, scheme); err != nil { + t.Fatalf("set the owner of ServiceExport %s: %v", export.GetName(), err) + } + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("The cache lists both exports of a Relay that is being withdrawn") + relay, _, apiServer, r, recorder, scheme := staleCacheSetUp(t) + + t.Log("Before the DELETE, " + tc.name + " in the API server, and the export keeps its UID") + changed := multicluster.NewServiceExport(exportTestVersion) + g.Expect(apiServer.Get(ctx, client.ObjectKey{Namespace: "inference", Name: "green-frontend"}, changed)).To(gomega.Succeed()) + tc.change(t, scheme, changed) + g.Expect(apiServer.Update(ctx, changed)).To(gomega.Succeed()) + g.Expect(changed.GetUID()).To(gomega.BeEquivalentTo("export-frontend")) + + t.Log("Withdraw from the cache: the stale delete is refused, which is no error, and the Relay's other export is withdrawn") + g.Expect(r.withdrawExports(ctx, relay)).To(gomega.Succeed()) + g.Expect(drainEvents(recorder)).To(gomega.ConsistOf( + corev1.EventTypeNormal + " DeleteServiceExport Withdrew the export of Service green-pool-relay")) + g.Expect(exportNames(listServiceExports(t, apiServer, exportTestVersion, relay.Namespace))).To(gomega.ConsistOf("green-frontend")) + + t.Log("Withdraw once the cache has the change: the Relay no longer controls the export, so it stays") + r.exports.cache = apiServer + g.Expect(r.withdrawExports(ctx, relay)).To(gomega.Succeed()) + g.Expect(drainEvents(recorder)).To(gomega.BeEmpty()) + remaining := listServiceExports(t, apiServer, exportTestVersion, relay.Namespace) + g.Expect(remaining).To(gomega.HaveLen(1)) + g.Expect(remaining[0].GetUID()).To(gomega.BeEquivalentTo("export-frontend")) + g.Expect(remaining[0].GetOwnerReferences()).To(gomega.Equal(changed.GetOwnerReferences())) + }) + } + + t.Run("an export that is already gone is no error", func(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("The cache lists both exports of a Relay that is being withdrawn") + relay, _, apiServer, r, recorder, _ := staleCacheSetUp(t) + + t.Log("Before the DELETE, the frontend export is deleted in the API server") + gone := multicluster.NewServiceExport(exportTestVersion) + gone.SetNamespace(relay.Namespace) + gone.SetName("green-frontend") + g.Expect(apiServer.Delete(ctx, gone)).To(gomega.Succeed()) + + t.Log("Withdraw from the cache: the missing export is no error, and the Relay's other export is withdrawn") + g.Expect(r.withdrawExports(ctx, relay)).To(gomega.Succeed()) + g.Expect(drainEvents(recorder)).To(gomega.ConsistOf( + corev1.EventTypeNormal + " DeleteServiceExport Withdrew the export of Service green-pool-relay")) + g.Expect(listServiceExports(t, apiServer, exportTestVersion, relay.Namespace)).To(gomega.BeEmpty()) + }) +} + +// newClusterIDProperty returns the ClusterProperty that holds the cluster's ID in +// its ClusterSet, with value. +func newClusterIDProperty(value string) *unstructured.Unstructured { + property := multicluster.NewClusterProperty() + property.SetName(multicluster.ClusterIDProperty) + property.Object["spec"] = map[string]any{"value": value} + return property +} + +func TestPoolExportsCheckTheClusterID(t *testing.T) { + for _, tc := range []struct { + name string + // watched says whether the controller watches ClusterProperties, as it does + // with a standard implementation in a cluster that serves them. + watched bool + // property is the cluster's ID in its ClusterSet, or empty without the + // property. + property string + // emptyProperty adds the property with no value. + emptyProperty bool + readFails bool + wantMismatch bool + }{ + {name: "a Relay with the cluster's ID", watched: true, property: "east"}, + {name: "a Relay with another cluster ID", watched: true, property: "west", wantMismatch: true}, + {name: "a cluster without the property", watched: true}, + {name: "a property without a value gives no cluster ID", watched: true, emptyProperty: true}, + {name: "without the watch the check is skipped", property: "west"}, + {name: "a failed read fails the reconcile instead of skipping the check", watched: true, readFails: true}, + } { + t.Run(tc.name, func(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + t.Log("Create a pool of cluster east, and the cluster's ID property if the case has one") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Frontend") + relay := newExportTestRelay(dgd) + objects := []client.Object{dgd, relay} + if tc.property != "" || tc.emptyProperty { + objects = append(objects, newClusterIDProperty(tc.property)) + } + funcs := interceptor.Funcs{} + if tc.readFails { + funcs.Get = func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if obj.GetObjectKind().GroupVersionKind() == multicluster.ClusterPropertyGVK { + return apierrors.NewForbidden(schema.GroupResource{Group: "about.k8s.io", Resource: "clusterproperties"}, key.Name, errors.New("denied")) + } + return c.Get(ctx, key, obj, opts...) + } + } + fakeClient := newExportTestClient(scheme, exportTestVersion, funcs, objects...) + recorder := events.NewFakeRecorder(10) + r := newExportTestReconciler(fakeClient, recorder, configv1alpha1.MCSImplementationStandard, exportTestVersion) + r.exports.clusterPropertyWatched = tc.watched + + t.Log("Reconcile the pool's exports, and look for the mismatch warning") + err := r.reconcileExports(context.Background(), relay, dgd) + recorded := drainEvents(recorder) + if tc.readFails { + g.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("denied"))) + g.Expect(countReasons(recorded, reasonExportFailed)).To(gomega.Equal(1)) + return + } + g.Expect(err).NotTo(gomega.HaveOccurred()) + if !tc.wantMismatch { + g.Expect(countReasons(recorded, reasonClusterIDMismatch)).To(gomega.BeZero()) + return + } + g.Expect(recorded).To(gomega.ContainElement(gomega.SatisfyAll( + gomega.HavePrefix(corev1.EventTypeWarning+" "+reasonClusterIDMismatch), + gomega.ContainSubstring(`clusterId "east"`), + gomega.ContainSubstring(`the ID "west"`), + ))) + }) + } +} + +func TestPoolsFollowTheClusterIDProperty(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create two pools of cluster east in a cluster whose ID property says east, read through the watch's cache") + scheme := newExportTestScheme(t) + green := newExportTestDGD("green", "Frontend") + blue := newExportTestDGD("blue", "Frontend") + greenRelay, blueRelay := newExportTestRelay(green), newExportTestRelay(blue) + property := newClusterIDProperty("east") + objects := []client.Object{green, blue, greenRelay, blueRelay, property} + objects = append(objects, poolServices(t, scheme, greenRelay, green)...) + objects = append(objects, poolServices(t, scheme, blueRelay, blue)...) + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, objects...) + recorder := events.NewFakeRecorder(20) + r := newExportTestReconciler(fakeClient, recorder, configv1alpha1.MCSImplementationStandard, exportTestVersion) + r.exports.clusterPropertyWatched = true + pools := []ctrl.Request{ + {NamespacedName: client.ObjectKeyFromObject(greenRelay)}, + {NamespacedName: client.ObjectKeyFromObject(blueRelay)}, + } + + for _, step := range []struct { + description string + // relayClusterID, when set, is the clusterId that users give both Relays, + // whose own watch requeues them. + relayClusterID string + // propertyValue, when set, is the property's new value. + propertyValue string + // deleteProperty deletes the property, and createProperty, when set, + // creates it again with that value. + deleteProperty bool + createProperty string + // wantMismatches is how many Relays record a mismatch. + wantMismatches int + }{ + {description: "Reconcile both pools: their IDs match the property"}, + { + description: "The cluster is renamed west: users correct both Relays first, and their watch requeues them while the cache still says east", + relayClusterID: "west", wantMismatches: 2, + }, + { + description: "The cache sees the property's new value: its event requeues both Relays, which no longer report a mismatch", + propertyValue: "west", + }, + {description: "The value changes again: both Relays report the mismatch", propertyValue: "south", wantMismatches: 2}, + {description: "The property is deleted: its event requeues both Relays, which skip the check", deleteProperty: true}, + { + description: "The property is created again with another value: its event requeues both Relays, which report the mismatch", + createProperty: "north", wantMismatches: 2, + }, + } { + t.Log(step.description) + requests := pools + switch { + case step.relayClusterID != "": + for _, relay := range []*v1alpha1.DynamoPoolRelay{greenRelay, blueRelay} { + g.Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(relay), relay)).To(gomega.Succeed()) + relay.Spec.ClusterID = step.relayClusterID + g.Expect(fakeClient.Update(ctx, relay)).To(gomega.Succeed()) + } + case step.propertyValue != "": + g.Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(property), property)).To(gomega.Succeed()) + before := property.DeepCopy() + property.Object["spec"] = map[string]any{"value": step.propertyValue} + g.Expect(fakeClient.Update(ctx, property)).To(gomega.Succeed()) + g.Expect(clusterIDPropertyEvents.Update(event.UpdateEvent{ObjectOld: before, ObjectNew: property})).To(gomega.BeTrue()) + requests = r.mapClusterPropertyToRelays(ctx, property) + case step.deleteProperty: + g.Expect(fakeClient.Delete(ctx, property)).To(gomega.Succeed()) + g.Expect(clusterIDPropertyEvents.Delete(event.DeleteEvent{Object: property})).To(gomega.BeTrue()) + requests = r.mapClusterPropertyToRelays(ctx, property) + case step.createProperty != "": + created := newClusterIDProperty(step.createProperty) + g.Expect(fakeClient.Create(ctx, created)).To(gomega.Succeed()) + g.Expect(clusterIDPropertyEvents.Create(event.CreateEvent{Object: created})).To(gomega.BeTrue()) + requests = r.mapClusterPropertyToRelays(ctx, created) + } + for _, request := range requests { + relay := &v1alpha1.DynamoPoolRelay{} + g.Expect(fakeClient.Get(ctx, request.NamespacedName, relay)).To(gomega.Succeed()) + dgd := &v1beta1.DynamoGraphDeployment{} + g.Expect(fakeClient.Get(ctx, request.NamespacedName, dgd)).To(gomega.Succeed()) + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.Succeed()) + } + g.Expect(countReasons(drainEvents(recorder), reasonClusterIDMismatch)).To(gomega.Equal(step.wantMismatches)) + } +} + +func TestPoolExportsConvergePastAFailedWrite(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create a pool whose frontend was renamed from Frontend to Web, in a cluster whose ID differs from the " + + "Relay's, while a quota refuses every new export") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Web") + relay := newExportTestRelay(dgd) + quota := apierrors.NewForbidden(schema.GroupResource{Group: multicluster.ServiceExportGroupKind.Group, Resource: "serviceexports"}, + "green-web", errors.New("exceeded quota")) + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{ + Create: func(ctx context.Context, c client.WithWatch, obj client.Object, opts ...client.CreateOption) error { + if obj.GetObjectKind().GroupVersionKind().GroupKind() == multicluster.ServiceExportGroupKind { + return quota + } + return c.Create(ctx, obj, opts...) + }, + }, append([]client.Object{ + relay, dgd, newClusterIDProperty("west"), + ownedExport(t, scheme, exportTestVersion, relay, "green-pool-relay"), + ownedExport(t, scheme, exportTestVersion, relay, "green-frontend"), + }, poolServices(t, scheme, relay, dgd)...)...) + recorder := events.NewFakeRecorder(10) + r := newExportTestReconciler(fakeClient, recorder, configv1alpha1.MCSImplementationStandard, exportTestVersion) + r.exports.clusterPropertyWatched = true + + t.Log("Reconcile: the refused create is returned, but the stale export is withdrawn, which frees the quota, " + + "and the mismatch is still reported") + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.MatchError(gomega.ContainSubstring("exceeded quota"))) + g.Expect(exportNames(listServiceExports(t, fakeClient, exportTestVersion, relay.Namespace))).To(gomega.ConsistOf("green-pool-relay")) + g.Expect(eventReasons(drainEvents(recorder))).To(gomega.ConsistOf("DeleteServiceExport", reasonExportFailed, reasonClusterIDMismatch)) + condition := meta.FindStatusCondition(relay.Status.Conditions, conditionExported) + g.Expect(condition).NotTo(gomega.BeNil()) + g.Expect(condition.Reason).To(gomega.Equal(reasonExportFailed)) + g.Expect(condition.Message).To(gomega.And(gomega.ContainSubstring("exceeded quota"), gomega.ContainSubstring(`clusterId "east"`))) +} + +func TestPoolExportProblemsAreReportedPastAFailedWithdrawal(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create a pool whose frontend was renamed from Frontend to Web, in a cluster whose ID differs from the " + + "Relay's, while the API server refuses deletes") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Web") + relay := newExportTestRelay(dgd) + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{ + Delete: func(context.Context, client.WithWatch, client.Object, ...client.DeleteOption) error { + return apierrors.NewServiceUnavailable("the API server is restarting") + }, + }, append([]client.Object{ + relay, dgd, newClusterIDProperty("west"), + ownedExport(t, scheme, exportTestVersion, relay, "green-frontend"), + }, poolServices(t, scheme, relay, dgd)...)...) + recorder := events.NewFakeRecorder(10) + r := newExportTestReconciler(fakeClient, recorder, configv1alpha1.MCSImplementationStandard, exportTestVersion) + r.exports.clusterPropertyWatched = true + + t.Log("Reconcile: the refused delete is returned, the new exports are created, and the mismatch is still reported") + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.MatchError(gomega.ContainSubstring("restarting"))) + g.Expect(exportNames(listServiceExports(t, fakeClient, exportTestVersion, relay.Namespace))). + To(gomega.ConsistOf("green-pool-relay", "green-web", "green-frontend")) + g.Expect(eventReasons(drainEvents(recorder))).To(gomega.ConsistOf( + "CreateServiceExport", "CreateServiceExport", reasonExportFailed, reasonClusterIDMismatch)) +} + +func TestPoolExportsSurviveAFailedServiceRead(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create a pool with both exports while reading its Services fails") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Frontend") + relay := newExportTestRelay(dgd) + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{ + Get: func(ctx context.Context, c client.WithWatch, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + if _, ok := obj.(*corev1.Service); ok { + return apierrors.NewServiceUnavailable("the API server is restarting") + } + return c.Get(ctx, key, obj, opts...) + }, + }, append([]client.Object{ + relay, dgd, + ownedExport(t, scheme, exportTestVersion, relay, "green-pool-relay"), + ownedExport(t, scheme, exportTestVersion, relay, "green-frontend"), + }, poolServices(t, scheme, relay, dgd)...)...) + r := newExportTestReconciler(fakeClient, events.NewFakeRecorder(10), configv1alpha1.MCSImplementationStandard, exportTestVersion) + + t.Log("Reconcile: the failure is returned, and the exports stay until the Services can be read again") + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.MatchError(gomega.ContainSubstring("restarting"))) + g.Expect(exportNames(listServiceExports(t, fakeClient, exportTestVersion, relay.Namespace))). + To(gomega.ConsistOf("green-pool-relay", "green-frontend")) +} + +func TestPoolExportWarningsFollowTheExportedCondition(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create a pool without a frontend") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green") + relay := newExportTestRelay(dgd) + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, + append([]client.Object{relay, dgd}, poolServices(t, scheme, relay, dgd)...)...) + recorder := events.NewFakeRecorder(10) + r := newExportTestReconciler(fakeClient, recorder, configv1alpha1.MCSImplementationStandard, exportTestVersion) + + t.Log("Reconcile: the condition reports the problem, and its Warning is recorded once") + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.Succeed()) + g.Expect(eventReasons(drainEvents(recorder))).To(gomega.ConsistOf("CreateServiceExport", reasonNoFrontend)) + + t.Log("A DGD status write changes only the DGD's resourceVersion, so the condition is unchanged and nothing is recorded") + g.Expect(fakeClient.Get(ctx, client.ObjectKeyFromObject(dgd), dgd)).To(gomega.Succeed()) + dgd.Status.State = "pending" + g.Expect(fakeClient.Status().Update(ctx, dgd)).To(gomega.Succeed()) + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.Succeed()) + g.Expect(drainEvents(recorder)).To(gomega.BeEmpty()) + + t.Log("A frontend is added: the condition turns True, and only the new export is recorded") + dgd.Spec.Components = append(dgd.Spec.Components, v1beta1.DynamoComponentDeploymentSharedSpec{ + ComponentName: "Frontend", ComponentType: v1beta1.ComponentTypeFrontend, + }) + g.Expect(fakeClient.Update(ctx, dgd)).To(gomega.Succeed()) + for _, service := range poolServices(t, scheme, relay, dgd, multicluster.PoolRelayServiceName(dgd.Name)) { + g.Expect(fakeClient.Create(ctx, service)).To(gomega.Succeed()) + } + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.Succeed()) + g.Expect(eventReasons(drainEvents(recorder))).To(gomega.ConsistOf("CreateServiceExport")) + condition := meta.FindStatusCondition(relay.Status.Conditions, conditionExported) + g.Expect(condition).NotTo(gomega.BeNil()) + g.Expect(condition.Status).To(gomega.Equal(metav1.ConditionTrue)) + g.Expect(condition.Message).To(gomega.Equal("Exported Services green-pool-relay, green-frontend through MCS implementation mcs")) +} + +func TestMapServiceToRelays(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create two pools in namespace inference, one with a frontend named Frontend") + scheme := newExportTestScheme(t) + green := newExportTestDGD("green", "Frontend") + blue := newExportTestDGD("blue") + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, + green, blue, newExportTestRelay(green), newExportTestRelay(blue)) + r := newExportTestReconciler(fakeClient, events.NewFakeRecorder(1), configv1alpha1.MCSImplementationStandard, exportTestVersion) + + t.Log("A Service requeues the Relays whose DGD exports a Service of its name") + g.Expect(r.mapServiceToRelays(ctx, &corev1.Service{Name: "green-frontend", Namespace: "inference"})). + To(gomega.ConsistOf(ctrl.Request{Namespace: "inference", Name: "green"})) + g.Expect(r.mapServiceToRelays(ctx, &corev1.Service{Name: "blue-pool-relay", Namespace: "inference"})). + To(gomega.ConsistOf(ctrl.Request{Namespace: "inference", Name: "blue"})) + g.Expect(r.mapServiceToRelays(ctx, &corev1.Service{Name: "blue-frontend", Namespace: "inference"})).To(gomega.BeEmpty()) + g.Expect(r.mapServiceToRelays(ctx, &corev1.Service{Name: "green-frontend", Namespace: "other"})).To(gomega.BeEmpty()) + + t.Log("A Service's creation and deletion count, and an update only when it changes the Service's controller") + scheme = newExportTestScheme(t) + service := &corev1.Service{Name: "green-frontend", Namespace: "inference"} + g.Expect(serviceControllerChanged.Create(event.CreateEvent{Object: service})).To(gomega.BeTrue()) + g.Expect(serviceControllerChanged.Delete(event.DeleteEvent{Object: service})).To(gomega.BeTrue()) + relabeled := service.DeepCopy() + relabeled.Labels = map[string]string{"team": "router"} + g.Expect(serviceControllerChanged.Update(event.UpdateEvent{ObjectOld: service, ObjectNew: relabeled})).To(gomega.BeFalse()) + controlled := service.DeepCopy() + g.Expect(controllerutil.SetControllerReference(green, controlled, scheme)).To(gomega.Succeed()) + g.Expect(serviceControllerChanged.Update(event.UpdateEvent{ObjectOld: service, ObjectNew: controlled})).To(gomega.BeTrue()) + g.Expect(serviceControllerChanged.Update(event.UpdateEvent{ObjectOld: controlled, ObjectNew: service})).To(gomega.BeTrue()) +} + +func TestTruncateKeepsWholeCharacters(t *testing.T) { + g := gomega.NewGomegaWithT(t) + g.Expect(truncate("short", 10)).To(gomega.Equal("short")) + g.Expect(truncate("ab€cd", 4)).To(gomega.Equal("ab")) + g.Expect(truncate("ab€cd", 5)).To(gomega.Equal("ab€")) + g.Expect(truncate(strings.Repeat("x", 2000), maxEventNote)).To(gomega.HaveLen(maxEventNote)) +} + +func TestMapServiceExportToRelays(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create pools green and one with a dot in its name, a DGD without a Relay, a pool in another namespace, " + + "and a pool there whose frontend Service has the name of green's PoolRelay Service") + scheme := newExportTestScheme(t) + green := newExportTestDGD("green", "Frontend", "Web") + dotted := newExportTestDGD("green3-0.6b", "Frontend") + unpooled := newExportTestDGD("blue", "Frontend") + elsewhere := newExportTestDGD("green", "Frontend") + elsewhere.Namespace = "other" + sameServiceName := newExportTestDGD("green-pool", "Relay") + sameServiceName.Namespace = "other" + greenRelay := newExportTestRelay(green) + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, + green, dotted, unpooled, elsewhere, sameServiceName, + greenRelay, newExportTestRelay(dotted), newExportTestRelay(elsewhere), newExportTestRelay(sameServiceName)) + r := newExportTestReconciler(fakeClient, events.NewFakeRecorder(1), configv1alpha1.MCSImplementationStandard, exportTestVersion) + + t.Log("Create an export that a Deployment named blue controls") + controlledByDeployment := multicluster.RenderServiceExport(exportTestVersion, "inference", "blue-cache", "blue") + if err := controllerutil.SetControllerReference( + &appsv1.Deployment{Name: "blue", Namespace: "inference", UID: uuid.NewUUID()}, controlledByDeployment, scheme, + ); err != nil { + t.Fatal(err) + } + + request := func(dgd *v1beta1.DynamoGraphDeployment) ctrl.Request { + return ctrl.Request{NamespacedName: client.ObjectKeyFromObject(dgd)} + } + for _, tc := range []struct { + name string + export *unstructured.Unstructured + want []ctrl.Request + }{ + { + name: "a user's export of a PoolRelay Service", + export: multicluster.RenderServiceExport(exportTestVersion, "inference", "green-pool-relay", "green"), + want: []ctrl.Request{request(green)}, + }, + { + name: "a frontend Service under its component's name", + export: multicluster.RenderServiceExport(exportTestVersion, "inference", "green-web", "green"), + want: []ctrl.Request{request(green)}, + }, + { + name: "the frontend Service of a DGD with a dot in its name", + export: multicluster.RenderServiceExport(exportTestVersion, "inference", "green3-0-6b-frontend", "green3-0.6b"), + want: []ctrl.Request{request(dotted)}, + }, + { + name: "an export the Relay controls for a frontend its DGD no longer has", + export: ownedExport(t, scheme, exportTestVersion, greenRelay, "green-old"), + want: []ctrl.Request{request(green)}, + }, + { + name: "a Service of a DGD without a Relay", + export: multicluster.RenderServiceExport(exportTestVersion, "inference", "blue-frontend", "blue"), + }, + { + name: "another Service", + export: multicluster.RenderServiceExport(exportTestVersion, "inference", "green", "green"), + }, + { + name: "an export that another kind controls", + export: controlledByDeployment, + }, + } { + t.Log("Map an event for " + tc.name + " in namespace inference to the Relays it concerns") + g.Expect(r.mapServiceExportToRelays(ctx, tc.export)).To(gomega.ConsistOf(tc.want), tc.name) + } +} + +func TestMapClusterPropertyToRelays(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + t.Log("Create Relays in two namespaces and one in an ephemeral namespace") + scheme := newExportTestScheme(t) + green := newExportTestRelay(newExportTestDGD("green")) + blue := newExportTestRelay(newExportTestDGD("blue")) + blue.Namespace = "team-b" + ephemeral := newExportTestRelay(newExportTestDGD("red")) + ephemeral.Namespace = "inference-ephemeral" + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, green, blue, ephemeral) + r := newExportTestReconciler(fakeClient, events.NewFakeRecorder(1), configv1alpha1.MCSImplementationStandard, exportTestVersion) + + t.Log("An event of the cluster ID property requeues every Relay that the operator manages") + g.Expect(r.mapClusterPropertyToRelays(context.Background(), newClusterIDProperty("east"))).To(gomega.ConsistOf( + ctrl.Request{NamespacedName: client.ObjectKeyFromObject(green)}, + ctrl.Request{NamespacedName: client.ObjectKeyFromObject(blue)}, + )) +} + +func TestPoolExportWatchPredicates(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + t.Log("An update of a user's export does not count; its creation and deletion do") + export := multicluster.RenderServiceExport(exportTestVersion, "inference", "green-frontend", "green") + statusChanged := export.DeepCopy() + statusChanged.Object["status"] = map[string]any{"conditions": []any{map[string]any{"type": "Ready", "status": "True"}}} + g.Expect(serviceExportChanged.Update(event.UpdateEvent{ObjectOld: export, ObjectNew: statusChanged})).To(gomega.BeFalse()) + g.Expect(serviceExportChanged.Create(event.CreateEvent{Object: export})).To(gomega.BeTrue()) + g.Expect(serviceExportChanged.Delete(event.DeleteEvent{Object: export})).To(gomega.BeTrue()) + + t.Log("Any update of an export with a controller counts, also one that adds or removes the controller reference") + relay := newExportTestRelay(newExportTestDGD("green")) + owned := ownedExport(t, newExportTestScheme(t), exportTestVersion, relay, "green-frontend") + unlabeled := owned.DeepCopy() + unlabeled.SetLabels(nil) + released := owned.DeepCopy() + released.SetOwnerReferences(nil) + g.Expect(serviceExportChanged.Update(event.UpdateEvent{ObjectOld: owned, ObjectNew: unlabeled})).To(gomega.BeTrue()) + g.Expect(serviceExportChanged.Update(event.UpdateEvent{ObjectOld: owned, ObjectNew: released})).To(gomega.BeTrue()) + g.Expect(serviceExportChanged.Update(event.UpdateEvent{ObjectOld: released, ObjectNew: owned})).To(gomega.BeTrue()) + + t.Log("A namespace-restricted operator still sees the cluster-scoped ClusterProperty, and filters other objects by namespace") + filter := eventFilter( + &configv1alpha1.OperatorConfiguration{Namespace: configv1alpha1.NamespaceConfiguration{Restricted: "inference"}}, + &commoncontroller.RuntimeConfig{}, + ) + g.Expect(filter.Create(event.CreateEvent{Object: newClusterIDProperty("east")})).To(gomega.BeTrue()) + g.Expect(filter.Create(event.CreateEvent{Object: export})).To(gomega.BeTrue()) + elsewhere := multicluster.RenderServiceExport(exportTestVersion, "other", "green-frontend", "green") + g.Expect(filter.Create(event.CreateEvent{Object: elsewhere})).To(gomega.BeFalse()) +} + +func TestClusterIDPropertyEvents(t *testing.T) { + g := gomega.NewGomegaWithT(t) + property := newClusterIDProperty("east") + other := newClusterIDProperty("east") + other.SetName("clusterset.k8s.io") + + t.Log("The creation and deletion of the cluster ID property count, another property's do not") + g.Expect(clusterIDPropertyEvents.Create(event.CreateEvent{Object: property})).To(gomega.BeTrue()) + g.Expect(clusterIDPropertyEvents.Delete(event.DeleteEvent{Object: property})).To(gomega.BeTrue()) + g.Expect(clusterIDPropertyEvents.Create(event.CreateEvent{Object: other})).To(gomega.BeFalse()) + g.Expect(clusterIDPropertyEvents.Delete(event.DeleteEvent{Object: other})).To(gomega.BeFalse()) + + t.Log("An update counts when it changes the cluster ID property's value") + relabeled := property.DeepCopy() + relabeled.SetLabels(map[string]string{"team": "router"}) + otherChanged := other.DeepCopy() + otherChanged.Object["spec"] = map[string]any{"value": "west"} + g.Expect(clusterIDPropertyEvents.Update(event.UpdateEvent{ObjectOld: property, ObjectNew: newClusterIDProperty("west")})).To(gomega.BeTrue()) + g.Expect(clusterIDPropertyEvents.Update(event.UpdateEvent{ObjectOld: property, ObjectNew: relabeled})).To(gomega.BeFalse()) + g.Expect(clusterIDPropertyEvents.Update(event.UpdateEvent{ObjectOld: other, ObjectNew: otherChanged})).To(gomega.BeFalse()) +} + +// failingRESTMapper fails every REST mapping, as when discovery is down. +type failingRESTMapper struct { + meta.RESTMapper +} + +func (failingRESTMapper) RESTMapping(schema.GroupKind, ...string) (*meta.RESTMapping, error) { + return nil, errors.New("discovery is down") +} + +func TestClusterPropertyServed(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + t.Log("A cluster that serves the About API gets the watch") + served := meta.NewDefaultRESTMapper(nil) + served.Add(multicluster.ClusterPropertyGVK, meta.RESTScopeRoot) + g.Expect(clusterPropertyServed(served)).To(gomega.BeTrue()) + + t.Log("Without the API, the controller starts without the watch, and so skips the check") + g.Expect(clusterPropertyServed(meta.NewDefaultRESTMapper(nil))).To(gomega.BeFalse()) + + t.Log("A failed discovery fails the start") + _, err := clusterPropertyServed(failingRESTMapper{RESTMapper: meta.NewDefaultRESTMapper(nil)}) + g.Expect(err).To(gomega.MatchError(gomega.ContainSubstring("discovery is down"))) +} + +func TestWatchExportsChecksTheClusterIDOnlyWhereItCan(t *testing.T) { + aboutAPI := meta.NewDefaultRESTMapper(nil) + aboutAPI.Add(multicluster.ClusterPropertyGVK, meta.RESTScopeRoot) + for _, tc := range []struct { + name string + implementation configv1alpha1.MCSImplementation + mapper meta.RESTMapper + wantWatched bool + wantError string + }{ + { + name: "standard MCS in a cluster that serves the About API checks the cluster ID", + implementation: configv1alpha1.MCSImplementationStandard, mapper: aboutAPI, wantWatched: true, + }, + { + name: "Karmada names its member clusters itself, so it does not check", + implementation: configv1alpha1.MCSImplementationKarmada, mapper: aboutAPI, + }, + { + name: "without the About API, the controller starts without the check", + implementation: configv1alpha1.MCSImplementationStandard, mapper: meta.NewDefaultRESTMapper(nil), + }, + { + name: "a failed discovery fails the start", + implementation: configv1alpha1.MCSImplementationStandard, + mapper: failingRESTMapper{RESTMapper: meta.NewDefaultRESTMapper(nil)}, + wantError: "discovery is down", + }, + } { + t.Run(tc.name, func(t *testing.T) { + g := gomega.NewGomegaWithT(t) + + t.Log("Create a manager whose REST mapper serves the case's APIs; the test never starts it") + scheme := newExportTestScheme(t) + manager, err := ctrl.NewManager(&rest.Config{Host: "https://127.0.0.1:1"}, ctrl.Options{ + Scheme: scheme, + Metrics: server.Options{BindAddress: "0"}, + MapperProvider: func(*rest.Config, *http.Client) (meta.RESTMapper, error) { + return tc.mapper, nil + }, + }) + g.Expect(err).NotTo(gomega.HaveOccurred()) + + t.Log("Add the watches of the pool's exports, as Setup does when the gate is on") + r := &reconciler{Client: manager.GetClient(), scheme: scheme} + _, err = r.watchExports(manager, ctrl.NewControllerManagedBy(manager).For(&v1alpha1.DynamoPoolRelay{}), + &configv1alpha1.OperatorConfiguration{ + MultiCluster: configv1alpha1.MultiClusterConfiguration{MCSImplementation: tc.implementation}, + }, + &commoncontroller.RuntimeConfig{ + Gate: features.Gates{MultiClusterExport: true, ServiceExportVersion: exportTestVersion}, + }) + if tc.wantError != "" { + g.Expect(err).To(gomega.MatchError(gomega.ContainSubstring(tc.wantError))) + return + } + g.Expect(err).NotTo(gomega.HaveOccurred()) + g.Expect(r.exports.clusterPropertyWatched).To(gomega.Equal(tc.wantWatched)) + }) + } +} + +// relatedRecorder records each event's reason with the kind and name of the +// event's related object. +type relatedRecorder struct { + recorded []string +} + +func (r *relatedRecorder) Eventf(_, related runtime.Object, _, reason, _, _ string, _ ...any) { + name := "none" + switch object := related.(type) { + case *unstructured.Unstructured: + name = object.GetKind() + "/" + object.GetName() + case *v1beta1.DynamoGraphDeployment: + name = "DynamoGraphDeployment/" + object.Name + } + r.recorded = append(r.recorded, reason+" "+name) +} + +func TestPoolExportEventsNameTheirObjects(t *testing.T) { + g := gomega.NewGomegaWithT(t) + ctx := context.Background() + + t.Log("Create a pool with a frontend whose Service no Service can be named, an export to repair and one to " + + "withdraw, in a cluster whose ID in its ClusterSet differs from the Relay's") + scheme := newExportTestScheme(t) + dgd := newExportTestDGD("green", "Frontend", strings.Repeat("w", 60)) + relay := newExportTestRelay(dgd) + unlabeled := ownedExport(t, scheme, exportTestVersion, relay, "green-frontend") + unlabeled.SetLabels(nil) + fakeClient := newExportTestClient(scheme, exportTestVersion, interceptor.Funcs{}, append([]client.Object{ + relay, dgd, unlabeled, ownedExport(t, scheme, exportTestVersion, relay, "green-old"), newClusterIDProperty("west"), + }, poolServices(t, scheme, relay, dgd)...)...) + recorder := &relatedRecorder{} + r := newExportTestReconciler(fakeClient, recorder, configv1alpha1.MCSImplementationStandard, exportTestVersion) + r.exports.clusterPropertyWatched = true + + t.Log("Reconcile: each event names the export it reports, or the DGD or property that its problem is about") + g.Expect(r.reconcileExports(ctx, relay, dgd)).To(gomega.Succeed()) + g.Expect(recorder.recorded).To(gomega.ConsistOf( + "CreateServiceExport ServiceExport/green-pool-relay", + "RepairServiceExport ServiceExport/green-frontend", + "DeleteServiceExport ServiceExport/green-old", + "InvalidServiceName DynamoGraphDeployment/green", + "ClusterIDMismatch ClusterProperty/"+multicluster.ClusterIDProperty, + )) + + t.Log("Remove the frontends: the withdrawal names its export, and the missing frontend the DGD") + recorder.recorded = nil + g.Expect(r.reconcileExports(ctx, relay, newExportTestDGD("green"))).To(gomega.Succeed()) + g.Expect(recorder.recorded).To(gomega.ConsistOf( + "DeleteServiceExport ServiceExport/green-frontend", + "NoFrontend DynamoGraphDeployment/green", + "ClusterIDMismatch ClusterProperty/"+multicluster.ClusterIDProperty, + )) + + t.Log("With Karmada, a Service name too long to import names the DGD too") + recorder.recorded = nil + r.exports.config.MultiCluster.MCSImplementation = configv1alpha1.MCSImplementationKarmada + long := newExportTestDGD(strings.Repeat("a", 45), "Frontend") + g.Expect(r.reconcileExports(ctx, relay, long)).To(gomega.Succeed()) + g.Expect(recorder.recorded).To(gomega.ContainElement("ServiceNameTooLong DynamoGraphDeployment/" + long.Name)) +} diff --git a/deploy/operator/internal/features/gates.go b/deploy/operator/internal/features/gates.go index feca0117b253..21538ddc5717 100644 --- a/deploy/operator/internal/features/gates.go +++ b/deploy/operator/internal/features/gates.go @@ -148,6 +148,7 @@ var allNames = [...]Name{ VolcanoScheduler, DRA, Istio, + MultiClusterExport, GPUDiscovery, } @@ -156,18 +157,25 @@ type Gate interface { Enabled(Name) bool } -// Gates is the complete set of operator feature gates. +// Gates is the complete set of operator feature gates, with the API version +// that a gate resolved where its consumers depend on it. type Gates struct { - Checkpoint bool `json:"checkpoint"` - Grove bool `json:"grove"` - LPX bool `json:"lpx"` - LWS bool `json:"lws"` - DisaggregatedSet bool `json:"disaggregatedSet"` - KaiScheduler bool `json:"kaiScheduler"` - VolcanoScheduler bool `json:"volcanoScheduler"` - DRA bool `json:"dra"` - Istio bool `json:"istio"` - GPUDiscovery bool `json:"gpuDiscovery"` + Checkpoint bool `json:"checkpoint"` + Grove bool `json:"grove"` + LPX bool `json:"lpx"` + LWS bool `json:"lws"` + DisaggregatedSet bool `json:"disaggregatedSet"` + KaiScheduler bool `json:"kaiScheduler"` + VolcanoScheduler bool `json:"volcanoScheduler"` + DRA bool `json:"dra"` + Istio bool `json:"istio"` + MultiClusterExport bool `json:"multiClusterExport"` + GPUDiscovery bool `json:"gpuDiscovery"` + + // ServiceExportVersion is not a gate. It is the multicluster.x-k8s.io + // version that the operator reads and writes ServiceExports in, set + // whenever MultiClusterExport is enabled. + ServiceExportVersion string `json:"-"` } // Defaults returns the default feature gates. @@ -284,6 +292,15 @@ func New( } } + // Export Global Router pools only through a configured MCS implementation whose + // ServiceExport API this cluster serves, at the version it serves. + if config.MultiCluster.MCSImplementation != configv1alpha1.MCSImplementationNone { + if gates.ServiceExportVersion, err = resolveMultiClusterExport(ctx, mgr.GetConfig()); err != nil { + return Gates{}, err + } + gates.MultiClusterExport = true + } + logValues := make([]any, 0, 2*len(allNames)) for _, name := range allNames { logValues = append(logValues, string(name), gates.Enabled(name)) diff --git a/deploy/operator/internal/features/gates_test.go b/deploy/operator/internal/features/gates_test.go index 7db902449817..8f9cf63f5cd2 100644 --- a/deploy/operator/internal/features/gates_test.go +++ b/deploy/operator/internal/features/gates_test.go @@ -56,16 +56,17 @@ func TestGateRegistryIsComplete(t *testing.T) { func allEnabledGates() Gates { return Gates{ - Checkpoint: true, - Grove: true, - LPX: true, - LWS: true, - DisaggregatedSet: true, - KaiScheduler: true, - VolcanoScheduler: true, - DRA: true, - Istio: true, - GPUDiscovery: true, + Checkpoint: true, + Grove: true, + LPX: true, + LWS: true, + DisaggregatedSet: true, + KaiScheduler: true, + VolcanoScheduler: true, + DRA: true, + Istio: true, + MultiClusterExport: true, + GPUDiscovery: true, } } diff --git a/deploy/operator/internal/features/multicluster.go b/deploy/operator/internal/features/multicluster.go new file mode 100644 index 000000000000..27cdc4c3ed1d --- /dev/null +++ b/deploy/operator/internal/features/multicluster.go @@ -0,0 +1,44 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package features + +import ( + "context" + "errors" + + "k8s.io/client-go/rest" +) + +// MultiClusterExport exports the PoolRelay and frontend Services of each DGD +// that is a Global Router pool through the Multi-Cluster Services (MCS) API. +// +// Owner: @atchernych +// Experimental since: v1.7.0 +// Beta since: N/A +// GA since: N/A +// Configuration: multiCluster.mcsImplementation, and a DynamoPoolRelay named +// after each pool's DGD +// Auto-detection: N/A; API availability is verified when an implementation is configured +// Requires: an MCS implementation, such as Karmada, serving +// multicluster.x-k8s.io/v1alpha1 ServiceExport resources +// Default: false +const MultiClusterExport Name = "multiClusterExport" + +// resolveMultiClusterExport verifies API availability for a configured MCS +// implementation and returns the multicluster.x-k8s.io version at which the +// cluster serves ServiceExports. A nil config retains the native +// discovery-unavailable error. +func resolveMultiClusterExport(ctx context.Context, config *rest.Config) (string, error) { + // Every supported implementation reads the SIG Multicluster ServiceExport. + const version = "v1alpha1" + available, err := detectAPIAvailability(ctx, config, "multicluster.x-k8s.io", version, "serviceexports") + if err != nil { + return "", err + } + if !available { + return "", errors.New("multiCluster.mcsImplementation is set but the multicluster.x-k8s.io/v1alpha1 " + + "ServiceExport API was not detected in the cluster") + } + return version, nil +} diff --git a/deploy/operator/internal/features/multicluster_test.go b/deploy/operator/internal/features/multicluster_test.go new file mode 100644 index 000000000000..47240af05ff1 --- /dev/null +++ b/deploy/operator/internal/features/multicluster_test.go @@ -0,0 +1,137 @@ +// SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +// SPDX-License-Identifier: Apache-2.0 + +package features + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "strings" + "testing" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/rest" + ctrl "sigs.k8s.io/controller-runtime" + metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server" +) + +func TestNewResolvesMultiClusterExport(t *testing.T) { + t.Log("Serve discovery for a cluster whose only API group is multicluster.x-k8s.io, with ServiceExports at v1alpha1") + mcs := metav1.GroupVersionForDiscovery{GroupVersion: "multicluster.x-k8s.io/v1alpha1", Version: "v1alpha1"} + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + var response any + switch r.URL.Path { + case "/api": + response = metav1.APIVersions{Kind: "APIVersions", Versions: []string{"v1"}} + case "/apis": + response = metav1.APIGroupList{ + Kind: "APIGroupList", APIVersion: "v1", + Groups: []metav1.APIGroup{{ + Name: "multicluster.x-k8s.io", Versions: []metav1.GroupVersionForDiscovery{mcs}, PreferredVersion: mcs, + }}, + } + case "/apis/multicluster.x-k8s.io/v1alpha1": + response = metav1.APIResourceList{ + Kind: "APIResourceList", APIVersion: "v1", + GroupVersion: mcs.GroupVersion, + APIResources: []metav1.APIResource{{Name: "serviceexports", Namespaced: true, Kind: "ServiceExport"}}, + } + default: + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(response); err != nil { + t.Errorf("write discovery response: %v", err) + } + })) + defer server.Close() + manager, err := ctrl.NewManager(&rest.Config{Host: server.URL}, ctrl.Options{ + Metrics: metricsserver.Options{BindAddress: "0"}, + }) + if err != nil { + t.Fatal(err) + } + + for _, tc := range []struct { + implementation configv1alpha1.MCSImplementation + wantVersion string + }{ + {implementation: configv1alpha1.MCSImplementationNone}, + {implementation: configv1alpha1.MCSImplementationStandard, wantVersion: "v1alpha1"}, + {implementation: configv1alpha1.MCSImplementationKarmada, wantVersion: "v1alpha1"}, + } { + t.Logf("Resolve the gates with mcsImplementation %q: a configured implementation turns the gate on", tc.implementation) + gates, err := New(t.Context(), manager, &configv1alpha1.OperatorConfiguration{ + MultiCluster: configv1alpha1.MultiClusterConfiguration{MCSImplementation: tc.implementation}, + }) + if err != nil { + t.Fatalf("New() with %q: %v", tc.implementation, err) + } + wantGate := tc.implementation != configv1alpha1.MCSImplementationNone + if gates.MultiClusterExport != wantGate || gates.ServiceExportVersion != tc.wantVersion { + t.Errorf("New() with %q: multiClusterExport %v at %q, want %v at %q", + tc.implementation, gates.MultiClusterExport, gates.ServiceExportVersion, wantGate, tc.wantVersion) + } + } +} + +func TestResolveMultiClusterExport(t *testing.T) { + const ( + mcsPath = "/apis/multicluster.x-k8s.io/v1alpha1" + missingAPI = "multiCluster.mcsImplementation is set but the multicluster.x-k8s.io/v1alpha1 ServiceExport API was not detected in the cluster" + ) + tests := []struct { + name string + status int + resource string + wantErr string + }{ + {name: "ServiceExport API present", status: http.StatusOK, resource: "serviceexports"}, + {name: "group version absent", status: http.StatusNotFound, wantErr: missingAPI}, + {name: "only ServiceImport served", status: http.StatusOK, resource: "serviceimports", wantErr: missingAPI}, + {name: "discovery forbidden", status: http.StatusForbidden, wantErr: "discover multicluster.x-k8s.io/v1alpha1 API resources: Forbidden"}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("Serve the multicluster.x-k8s.io/v1alpha1 discovery path") + server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + if r.URL.Path != mcsPath { + t.Errorf("unexpected discovery path %q", r.URL.Path) + http.NotFound(w, r) + return + } + w.Header().Set("Content-Type", "application/json") + w.WriteHeader(tt.status) + var response any = metav1.Status{Kind: "Status", APIVersion: "v1", Status: metav1.StatusFailure, Code: int32(tt.status), Message: http.StatusText(tt.status)} + if tt.status == http.StatusOK { + response = metav1.APIResourceList{ + Kind: "APIResourceList", + APIVersion: "v1", + GroupVersion: strings.TrimPrefix(r.URL.Path, "/apis/"), + APIResources: []metav1.APIResource{{Name: tt.resource}}, + } + } + if err := json.NewEncoder(w).Encode(response); err != nil { + t.Errorf("write discovery response: %v", err) + } + })) + defer server.Close() + + t.Log("Require the ServiceExport resource, return its version, and preserve discovery failures") + version, err := resolveMultiClusterExport(t.Context(), &rest.Config{Host: server.URL}) + wantVersion := "" + if tt.wantErr == "" { + wantVersion = "v1alpha1" + } + if version != wantVersion || (err == nil) != (tt.wantErr == "") { + t.Fatalf("resolveMultiClusterExport() = %q, %v; want %q, error=%q", version, err, wantVersion, tt.wantErr) + } + if err != nil && err.Error() != tt.wantErr { + t.Errorf("resolveMultiClusterExport() error = %q, want %q", err, tt.wantErr) + } + }) + } +} diff --git a/deploy/operator/internal/multicluster/serviceexport.go b/deploy/operator/internal/multicluster/serviceexport.go new file mode 100644 index 000000000000..e422412dc14e --- /dev/null +++ b/deploy/operator/internal/multicluster/serviceexport.go @@ -0,0 +1,251 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +// Package multicluster renders the ServiceExports that announce a Global Router +// pool to other clusters through the Kubernetes Multi-Cluster Services (MCS) API, +// lists them, and reads the cluster's ID in its ClusterSet. +package multicluster + +import ( + "context" + "fmt" + "slices" + "strings" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/validation" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// ServiceExportGroupKind is the SIG Multicluster ServiceExport. Every supported +// implementation reads it in the cluster that runs the Service, Karmada included: +// its ServiceExport controller watches this kind in member clusters. Karmada +// watches a member only while its control plane has propagated at least one +// ServiceExport there, so keep one in place in each member. That setup is outside +// the operator. The operator reads and writes it at the multicluster.x-k8s.io +// version that the multiClusterExport feature gate resolved. +var ServiceExportGroupKind = schema.GroupKind{ + Group: "multicluster.x-k8s.io", + Kind: "ServiceExport", +} + +// ClusterPropertyGVK is the About API's ClusterProperty (KEP-2149), which holds +// facts about the cluster for its ClusterSet. +var ClusterPropertyGVK = schema.GroupVersionKind{ + Group: "about.k8s.io", + Version: "v1alpha1", + Kind: "ClusterProperty", +} + +const ( + // ClusterIDProperty names the ClusterProperty that holds the cluster's ID in its + // ClusterSet. A standard MCS implementation labels the EndpointSlices it imports + // with that ID, and the Global Discovery Service takes a pool's cluster from the + // label. + ClusterIDProperty = "cluster.clusterset.k8s.io" + + // poolRelayServiceSuffix names a pool's PoolRelay Service, as the Global + // Discovery Service expects by default. + poolRelayServiceSuffix = "-pool-relay" + + // maxKarmadaServiceNameLength is the longest Service name Karmada imports. + // Karmada labels each collected slice kubernetes.io/service-name: + // derived-, and a label value has at most 63 characters. + maxKarmadaServiceNameLength = 55 +) + +// PoolRelayServiceName returns the name of the PoolRelay Service of the DGD named +// dgdName, -pool-relay, as the Global Discovery Service expects by +// default. The operator creates this Service for a DynamoPoolRelay of the same +// name. +func PoolRelayServiceName(dgdName string) string { + return dgdName + poolRelayServiceSuffix +} + +// FrontendServiceNames returns the names of the Services the operator creates for +// the DGD's frontend components, in component order: -, +// lowercased, with dots replaced by dashes. +func FrontendServiceNames(dgd *v1beta1.DynamoGraphDeployment) []string { + var names []string + for i := range dgd.Spec.Components { + component := &dgd.Spec.Components[i] + if component.ComponentType != v1beta1.ComponentTypeFrontend { + continue + } + name := dynamo.NormalizeKubeResourceName(dynamo.GetDCDResourceName(dgd, component.ComponentName, "")) + if !slices.Contains(names, name) { + names = append(names, name) + } + } + return names +} + +// ServiceNames returns the Services the operator exports for the DGD as a Global +// Router pool: its PoolRelay Service, then its frontend Services. +func ServiceNames(dgd *v1beta1.DynamoGraphDeployment) []string { + names := []string{PoolRelayServiceName(dgd.Name)} + for _, name := range FrontendServiceNames(dgd) { + if !slices.Contains(names, name) { + names = append(names, name) + } + } + return names +} + +// MaxServiceNameLength returns the longest Service name that implementation can +// export with the cluster it came from. Standard implementations label imported +// EndpointSlices with their source cluster, so any Service name works. +func MaxServiceNameLength(implementation configv1alpha1.MCSImplementation) int { + if implementation == configv1alpha1.MCSImplementationKarmada { + return maxKarmadaServiceNameLength + } + return validation.DNS1035LabelMaxLength +} + +// NameProblem is why a Service name cannot be exported. +type NameProblem int + +const ( + // NameExportable means that the implementation can export a Service of the name. + NameExportable NameProblem = iota + // NameInvalid means that no Service can have the name. + NameInvalid + // NameTooLong means that the implementation imports no Service name that long. + NameTooLong +) + +// CheckServiceName returns whether implementation can export a Service named name +// with the cluster it came from, and describes the problem when it cannot. +func CheckServiceName(implementation configv1alpha1.MCSImplementation, name string) (NameProblem, string) { + if problems := validation.IsDNS1035Label(name); len(problems) > 0 { + return NameInvalid, strings.Join(problems, "; ") + } + if maxLength := MaxServiceNameLength(implementation); len(name) > maxLength { + return NameTooLong, fmt.Sprintf("%s imports no Service name longer than %d characters", implementation, maxLength) + } + return NameExportable, "" +} + +// UsesClusterSetID reports whether implementation labels the EndpointSlices it +// imports with the cluster's ID in its ClusterSet, the ClusterIDProperty. Karmada +// names its member clusters itself. +func UsesClusterSetID(implementation configv1alpha1.MCSImplementation) bool { + return implementation == configv1alpha1.MCSImplementationStandard +} + +// NewServiceExport returns an empty ServiceExport at version, to watch exports or +// to name one. version must be a multicluster.x-k8s.io version that the cluster +// serves. +func NewServiceExport(version string) *unstructured.Unstructured { + serviceExport := &unstructured.Unstructured{} + serviceExport.SetGroupVersionKind(ServiceExportGroupKind.WithVersion(version)) + return serviceExport +} + +// RenderServiceExport returns the ServiceExport at version for the Service named +// name in namespace, labeled with the DGD that exports it. It sets no owner +// reference. version must be a multicluster.x-k8s.io version that the cluster +// serves. +func RenderServiceExport(version, namespace, name, dgdName string) *unstructured.Unstructured { + serviceExport := NewServiceExport(version) + serviceExport.SetNamespace(namespace) + serviceExport.SetName(name) + serviceExport.SetLabels(map[string]string{ + commonconsts.KubeLabelDynamoGraphDeploymentName: dgdName, + }) + return serviceExport +} + +// Export is a ServiceExport as the operator sees it. +type Export struct { + Name string + UID types.UID + // ResourceVersion is the version of the export that was read. A write + // conditioned on it fails once the export has changed. + ResourceVersion string + // Labels are the export's labels, and Controller its controller reference, + // nil without one: the operator writes both on the exports of a pool. + Labels map[string]string + Controller *metav1.OwnerReference + // Deleting is true once the export's deletion has started. + Deleting bool +} + +// ControlledBy reports whether the object with uid controls the export. +func (e Export) ControlledBy(uid types.UID) bool { + return e.Controller != nil && e.Controller.UID == uid +} + +// ListExports returns the ServiceExports in namespace by name, read at version. +// reader should serve unstructured objects from a cache: the manager's client +// reads them from the API server. +func ListExports( + ctx context.Context, + reader client.Reader, + version, namespace string, +) (map[string]Export, error) { + list := &unstructured.UnstructuredList{} + list.SetGroupVersionKind(schema.GroupVersionKind{ + Group: ServiceExportGroupKind.Group, Version: version, Kind: ServiceExportGroupKind.Kind + "List", + }) + if err := reader.List(ctx, list, client.InNamespace(namespace)); err != nil { + return nil, fmt.Errorf("failed to list ServiceExports: %w", err) + } + exports := make(map[string]Export, len(list.Items)) + for i := range list.Items { + item := &list.Items[i] + export := Export{ + Name: item.GetName(), + UID: item.GetUID(), + ResourceVersion: item.GetResourceVersion(), + Labels: item.GetLabels(), + Controller: metav1.GetControllerOf(item), + Deleting: item.GetDeletionTimestamp() != nil, + } + exports[export.Name] = export + } + return exports, nil +} + +// NewClusterProperty returns an empty ClusterProperty, to watch properties or to +// read one. +func NewClusterProperty() *unstructured.Unstructured { + property := &unstructured.Unstructured{} + property.SetGroupVersionKind(ClusterPropertyGVK) + return property +} + +// ClusterPropertyValue returns the value of property, or "" without one. +func ClusterPropertyValue(property *unstructured.Unstructured) string { + value, _, _ := unstructured.NestedString(property.Object, "spec", "value") + return value +} + +// ReadClusterIDProperty returns the About API's ClusterProperty named +// ClusterIDProperty, whose value is the cluster's ID in its ClusterSet, or nil +// when the cluster has no such property or does not serve the About API. Any +// other failure is an error. reader should be the manager's cache, which a watch +// of ClusterProperties fills; without permission to list and watch them, that +// watch never syncs, and the controller does not start. +func ReadClusterIDProperty(ctx context.Context, reader client.Reader) (*unstructured.Unstructured, error) { + property := NewClusterProperty() + err := reader.Get(ctx, client.ObjectKey{Name: ClusterIDProperty}, property) + if apierrors.IsNotFound(err) || meta.IsNoMatchError(err) { + return nil, nil + } + if err != nil { + return nil, fmt.Errorf("failed to read ClusterProperty %s: %w", ClusterIDProperty, err) + } + return property, nil +} diff --git a/deploy/operator/internal/multicluster/serviceexport_test.go b/deploy/operator/internal/multicluster/serviceexport_test.go new file mode 100644 index 000000000000..a2c0b8791865 --- /dev/null +++ b/deploy/operator/internal/multicluster/serviceexport_test.go @@ -0,0 +1,250 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + */ + +package multicluster + +import ( + "context" + "errors" + "slices" + "strings" + "testing" + "time" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + "k8s.io/apimachinery/pkg/api/equality" + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + "sigs.k8s.io/controller-runtime/pkg/client/interceptor" +) + +func TestMaxServiceNameLength(t *testing.T) { + for _, tc := range []struct { + implementation configv1alpha1.MCSImplementation + want int + }{ + {implementation: configv1alpha1.MCSImplementationStandard, want: 63}, + {implementation: configv1alpha1.MCSImplementationKarmada, want: 55}, + } { + t.Run(string(tc.implementation), func(t *testing.T) { + t.Log("Look up the longest Service name the implementation exports with its cluster") + if got := MaxServiceNameLength(tc.implementation); got != tc.want { + t.Errorf("MaxServiceNameLength(%q) = %d, want %d", tc.implementation, got, tc.want) + } + }) + } +} + +func TestServiceNames(t *testing.T) { + component := func(name string, componentType v1beta1.ComponentType) v1beta1.DynamoComponentDeploymentSharedSpec { + return v1beta1.DynamoComponentDeploymentSharedSpec{ComponentName: name, ComponentType: componentType} + } + for _, tc := range []struct { + name string + dgdName string + components []v1beta1.DynamoComponentDeploymentSharedSpec + wantFrontends []string + }{ + { + name: "a frontend named Frontend gets the Service the Global Discovery Service expects by default", + dgdName: "qwen", + components: []v1beta1.DynamoComponentDeploymentSharedSpec{ + component("Frontend", v1beta1.ComponentTypeFrontend), + component("VllmDecodeWorker", v1beta1.ComponentTypeDecode), + }, + wantFrontends: []string{"qwen-frontend"}, + }, + { + name: "every frontend keeps its component's name, lowercased, with dots replaced", + dgdName: "qwen3-0.6b", + components: []v1beta1.DynamoComponentDeploymentSharedSpec{ + component("Web.v2", v1beta1.ComponentTypeFrontend), + component("Frontend", v1beta1.ComponentTypeFrontend), + }, + wantFrontends: []string{"qwen3-0-6b-web-v2", "qwen3-0-6b-frontend"}, + }, + { + name: "frontends whose Services share a name are listed once", + dgdName: "qwen", + components: []v1beta1.DynamoComponentDeploymentSharedSpec{ + component("Frontend", v1beta1.ComponentTypeFrontend), + component("frontend", v1beta1.ComponentTypeFrontend), + }, + wantFrontends: []string{"qwen-frontend"}, + }, + { + name: "a DGD without a frontend has only its PoolRelay Service", + dgdName: "qwen", + components: []v1beta1.DynamoComponentDeploymentSharedSpec{component("Worker", v1beta1.ComponentTypeWorker)}, + }, + } { + t.Run(tc.name, func(t *testing.T) { + dgd := &v1beta1.DynamoGraphDeployment{Name: tc.dgdName, Spec: v1beta1.DynamoGraphDeploymentSpec{Components: tc.components}} + + t.Log("Name the frontend Services as the operator names its own") + if got := FrontendServiceNames(dgd); !slices.Equal(got, tc.wantFrontends) { + t.Errorf("FrontendServiceNames = %v, want %v", got, tc.wantFrontends) + } + + t.Log("The pool's Services are its PoolRelay Service, then its frontend Services") + want := append([]string{tc.dgdName + "-pool-relay"}, tc.wantFrontends...) + if got := ServiceNames(dgd); !slices.Equal(got, want) { + t.Errorf("ServiceNames = %v, want %v", got, want) + } + }) + } +} + +func TestRenderServiceExport(t *testing.T) { + t.Log("Render the export of pool qwen's PoolRelay Service at the version the gate resolved") + export := RenderServiceExport("v1alpha1", "dynamo", "qwen-pool-relay", "qwen") + if export.GetAPIVersion() != "multicluster.x-k8s.io/v1alpha1" || export.GetKind() != "ServiceExport" { + t.Errorf("export type = %s %s, want multicluster.x-k8s.io/v1alpha1 ServiceExport", export.GetAPIVersion(), export.GetKind()) + } + if export.GetNamespace() != "dynamo" || export.GetName() != "qwen-pool-relay" { + t.Errorf("export = %s/%s, want dynamo/qwen-pool-relay", export.GetNamespace(), export.GetName()) + } + if got := export.GetLabels()[commonconsts.KubeLabelDynamoGraphDeploymentName]; got != "qwen" { + t.Errorf("DGD label = %q, want qwen", got) + } +} + +func TestCheckServiceName(t *testing.T) { + for _, tc := range []struct { + name string + implementation configv1alpha1.MCSImplementation + service string + want NameProblem + }{ + {name: "a Service name", implementation: configv1alpha1.MCSImplementationKarmada, service: "qwen-pool-relay", want: NameExportable}, + {name: "a dot", implementation: configv1alpha1.MCSImplementationStandard, service: "qwen3-0.6b-pool-relay", want: NameInvalid}, + {name: "a leading digit", implementation: configv1alpha1.MCSImplementationStandard, service: "3b-frontend", want: NameInvalid}, + {name: "longer than Karmada imports", implementation: configv1alpha1.MCSImplementationKarmada, service: strings.Repeat("a", 56), want: NameTooLong}, + {name: "as long as a Service name gets", implementation: configv1alpha1.MCSImplementationStandard, service: strings.Repeat("a", 63), want: NameExportable}, + {name: "longer than any Service name", implementation: configv1alpha1.MCSImplementationStandard, service: strings.Repeat("a", 64), want: NameInvalid}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Check whether the implementation can export a Service of the name") + got, detail := CheckServiceName(tc.implementation, tc.service) + if got != tc.want || (got == NameExportable) != (detail == "") { + t.Errorf("CheckServiceName(%q) = %v, %q, want %v", tc.service, got, detail, tc.want) + } + }) + } +} + +func TestListExports(t *testing.T) { + t.Log("Store an export controlled by a DynamoPoolRelay that is being deleted, and one a user created") + owned := RenderServiceExport("v1alpha1", "dynamo", "qwen-pool-relay", "qwen") + owned.SetUID("export-1") + owned.SetOwnerReferences([]metav1.OwnerReference{ + {APIVersion: "nvidia.com/v1alpha1", Kind: "DynamoPoolRelay", Name: "qwen", UID: "relay-1", Controller: new(true)}, + }) + owned.SetFinalizers([]string{"example.com/hold"}) + owned.SetDeletionTimestamp(&metav1.Time{Time: time.Now()}) + foreign := RenderServiceExport("v1alpha1", "dynamo", "qwen-frontend", "qwen") + foreign.SetUID("export-2") + elsewhere := RenderServiceExport("v1alpha1", "other", "qwen-frontend", "qwen") + mapper := meta.NewDefaultRESTMapper(nil) + mapper.Add(ServiceExportGroupKind.WithVersion("v1alpha1"), meta.RESTScopeNamespace) + reader := fake.NewClientBuilder(). + WithScheme(runtime.NewScheme()). + WithRESTMapper(mapper). + WithObjects(owned, foreign, elsewhere). + Build() + + t.Log("List the exports in namespace dynamo") + got, err := ListExports(context.Background(), reader, "v1alpha1", "dynamo") + if err != nil { + t.Fatalf("ListExports: %v", err) + } + // The fake client gives the objects it starts with resourceVersion 999. + labels := map[string]string{commonconsts.KubeLabelDynamoGraphDeploymentName: "qwen"} + want := map[string]Export{ + "qwen-pool-relay": { + Name: "qwen-pool-relay", UID: "export-1", ResourceVersion: "999", Labels: labels, + Controller: &metav1.OwnerReference{ + APIVersion: "nvidia.com/v1alpha1", Kind: "DynamoPoolRelay", Name: "qwen", UID: "relay-1", Controller: new(true), + }, + Deleting: true, + }, + "qwen-frontend": {Name: "qwen-frontend", UID: "export-2", ResourceVersion: "999", Labels: labels}, + } + if !equality.Semantic.DeepEqual(got, want) { + t.Errorf("ListExports = %+v, want %+v", got, want) + } +} + +func TestReadClusterID(t *testing.T) { + property := &unstructured.Unstructured{Object: map[string]any{"spec": map[string]any{"value": "west"}}} + property.SetGroupVersionKind(ClusterPropertyGVK) + property.SetName(ClusterIDProperty) + otherProperty := property.DeepCopy() + otherProperty.SetName("clusterset.k8s.io") + failure := errors.New("connection refused") + + for _, tc := range []struct { + name string + aboutAPI bool + objects []client.Object + getError error + want string + wantError bool + }{ + {name: "the cluster's ID in its ClusterSet", aboutAPI: true, objects: []client.Object{property, otherProperty}, want: "west"}, + {name: "no cluster ID property", aboutAPI: true, objects: []client.Object{otherProperty}}, + { + // A cache cannot match a kind that the cluster does not serve. + name: "no About API", + getError: &meta.NoKindMatchError{GroupKind: ClusterPropertyGVK.GroupKind(), SearchedVersions: []string{ClusterPropertyGVK.Version}}, + }, + { + name: "no permission is an error, not a missing ID", aboutAPI: true, wantError: true, + getError: apierrors.NewForbidden(schema.GroupResource{Group: "about.k8s.io", Resource: "clusterproperties"}, ClusterIDProperty, failure), + }, + {name: "another failure", aboutAPI: true, getError: failure, wantError: true}, + } { + t.Run(tc.name, func(t *testing.T) { + t.Log("Serve the About API, or not, and fail reads as the case needs") + mapper := meta.NewDefaultRESTMapper(nil) + if tc.aboutAPI { + mapper.Add(ClusterPropertyGVK, meta.RESTScopeRoot) + } + funcs := interceptor.Funcs{} + if tc.getError != nil { + funcs.Get = func(context.Context, client.WithWatch, client.ObjectKey, client.Object, ...client.GetOption) error { + return tc.getError + } + } + reader := fake.NewClientBuilder(). + WithScheme(runtime.NewScheme()). + WithRESTMapper(mapper). + WithObjects(tc.objects...). + WithInterceptorFuncs(funcs). + Build() + + t.Log("Read the property that holds the cluster's ID in its ClusterSet") + property, err := ReadClusterIDProperty(context.Background(), reader) + got := "" + if property != nil { + got = ClusterPropertyValue(property) + } + if (err != nil) != tc.wantError || got != tc.want { + t.Errorf("ReadClusterIDProperty = %q, %v, want %q with error %v", got, err, tc.want, tc.wantError) + } + if tc.wantError && !strings.Contains(err.Error(), ClusterIDProperty) { + t.Errorf("error %q does not name the property", err) + } + }) + } +} diff --git a/docs/fern/pages/kubernetes/kv-aware-routing/pool-relay.md b/docs/fern/pages/kubernetes/kv-aware-routing/pool-relay.md index f79ef37f4434..4fa7872a916f 100644 --- a/docs/fern/pages/kubernetes/kv-aware-routing/pool-relay.md +++ b/docs/fern/pages/kubernetes/kv-aware-routing/pool-relay.md @@ -382,6 +382,57 @@ and allow long-lived server streams. See the [gRPC contract](https://github.com/ai-dynamo/dynamo/blob/main/lib/llm/src/pool_relay/docs/grpc-contract.md) for message sizes, reconnect behavior, and error reasons. +### Export to Other Clusters with MCS + +When the operator is configured with a Multi-Cluster Services implementation +(`dynamo-operator.dynamo.multiCluster.mcsImplementation`, `mcs` or `karmada`), a `DynamoPoolRelay` +also makes its DGD a Global Router pool. The operator creates a `multicluster.x-k8s.io/v1alpha1` +`ServiceExport` in the DGD's namespace for the PoolRelay Service, `-pool-relay`, and for the +Service of each frontend component, `-` in lowercase with dots replaced by +dashes. The Global Discovery Service reads the imported EndpointSlices of those Services to list the +pool. It finds the frontends by a Service name suffix, `-frontend` by default, which matches a +frontend component named `Frontend`. The operator does not know that suffix, so a frontend component +with another name is exported but not found until the suffix matches it. + +The operator exports a Service only while it exists and the pool controls it: the relay controls the +PoolRelay Service, and the DGD, or one of the DGD's `DynamoComponentDeployment`s, controls each +frontend's Service. A Service that does not exist yet is exported once it does, and a Service of such +a name that someone else created is not exported. + +The `DynamoPoolRelay` owns these exports, so deleting it deletes them. They follow the relay's own +Service: the operator withdraws them whenever it withdraws that Service, for example when the DGD is +deleted or has no workers, as the relay's `Ready` condition reports, and exports a DGD created again +under the relay's name. Renaming or removing a frontend component withdraws that component's export. +While the relay is an export's controller, the operator restores the export's DGD label and +controller reference if someone changes them. The operator leaves alone any `ServiceExport` that the +relay does not control: one that a user created, or one whose controller reference someone removed +or replaced. Such an export stays when the pool is withdrawn, and is not deleted with the relay; once +it is deleted, the operator creates its own. Without a configured implementation, the operator +creates no exports. Clearing the implementation leaves the exports that it created in place, no longer +maintained, until their `DynamoPoolRelay`s are deleted. + +Set `clusterId` to the name that the MCS implementation gives this cluster, because the Global +Discovery Service matches the pool's registration to its imports by that name: with standard MCS, +the cluster's ID in its ClusterSet, which the About API's `ClusterProperty` +`cluster.clusterset.k8s.io` holds; with Karmada, this cluster's member cluster name. + +The MCS implementation reports on each export in the export's status, with the conditions `Valid`, +`Ready` and `Conflict`; Karmada writes none. The operator reports on the pool's exports in the +relay's `Exported` condition: `True` once the Services that the pool can export are exported, +`False` with reason `Withdrawn` while the relay is withdrawn, and otherwise `False` with the reason of +the pool's first problem below; its message lists every problem. When the condition changes, the +operator also records a Warning event for each problem. `kubectl describe dynamopoolrelay ` +shows both. + +| Reason | Meaning | +| --- | --- | +| `ServiceNotOwned` | A Service of a name that the pool needs exists, but neither the relay nor the DGD, or one of its `DynamoComponentDeployment`s, controls it, so it is not exported. | +| `InvalidServiceName` | No Service can have a name that the pool needs, so that Service is not exported. For example, a frontend component's Service name, `-`, is longer than 63 characters. | +| `ServiceNameTooLong` | With `karmada`, a Service name is longer than 55 characters, which Karmada cannot import, so that Service is not exported. | +| `NoFrontend` | The DGD has no frontend component, so the Global Router has nowhere to send the pool's requests. | +| `ClusterIDMismatch` | With `mcs`, the relay's `clusterId` differs from the cluster's ID in its ClusterSet. The Global Discovery Service lists the pool only when they match. The operator watches the property, so creating, changing or deleting it checks every pool again. It skips the check when the cluster has no such property, or did not serve the About API when the operator started. | +| `ExportFailed` | Listing, creating, repairing or deleting an export, reading one of the pool's Services, or reading the cluster's ID in its ClusterSet failed. The operator retries with backoff. | + ## Optional mTLS Sidecar Mutual TLS (mTLS) is optional and implemented outside Relay. Relay has no built-in TLS configuration, diff --git a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md index c3e1aa22891b..0e5f4ca6bd77 100644 --- a/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md +++ b/docs/fern/pages/reference/kubernetes-api/additional-resources/api-reference-k8s.md @@ -811,7 +811,7 @@ _Appears in:_ | `observedGeneration` _integer_ | ObservedGeneration is the Relay resource generation last reconciled by the operator. | | Optional: \{\}
| | `endpoint` _string_ | Endpoint is the cluster-local `-pool-relay..svc:5561` address.
It is empty after withdrawal or a failed reconciliation. | | Optional: \{\}
| | `availableReplicas` _integer_ | AvailableReplicas is the number of Relay Pods the Deployment reports available. | | Optional: \{\}
| -| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#condition-v1-meta) array_ | Conditions includes Ready for listener availability, not model or KV readiness.
Its reason is ListenerAvailable, DeploymentProgressing, DGDNotFound,
DGDTerminating, UnsupportedDiscovery, UnsupportedSourceScope, NoWorkers,
InvalidConfiguration, ReconcileFailed, or ResourceOwnershipConflict. | | Optional: \{\}
| +| `conditions` _[Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#condition-v1-meta) array_ | Conditions includes Ready for listener availability, not model or KV readiness.
Its reason is ListenerAvailable, DeploymentProgressing, DGDNotFound,
DGDTerminating, UnsupportedDiscovery, UnsupportedSourceScope, NoWorkers,
InvalidConfiguration, ReconcileFailed, or ResourceOwnershipConflict.
With multi-cluster export configured, Conditions also includes Exported: True
with reason Exported once the pool's Services are exported, or False with
reason Withdrawn, ServiceNotOwned, InvalidServiceName, ServiceNameTooLong,
NoFrontend, ClusterIDMismatch, or ExportFailed. | | Optional: \{\}
| #### EPPConfig @@ -4366,6 +4366,24 @@ _Appears in:_ | `format` _string_ | Format is the log format (e.g., "json", "text") | json | | +#### MCSImplementation + +_Underlying type:_ _string_ + +MCSImplementation identifies an implementation of the Multi-Cluster Services API. + + + +_Appears in:_ +- [MultiClusterConfiguration](#multiclusterconfiguration) + +| Field | Description | +| --- | --- | +| `` | MCSImplementationNone exports nothing.
| +| `mcs` | MCSImplementationStandard is an implementation of the SIG Multicluster API that
imports Services into ServiceImports and EndpointSlices with the standard labels.
| +| `karmada` | MCSImplementationKarmada is Karmada, which reads the same ServiceExport in
member clusters but names imported EndpointSlices after the source cluster.
| + + #### MPIConfiguration @@ -4401,6 +4419,25 @@ _Appears in:_ | `secure` _boolean_ | Secure enables secure serving for the metrics endpoint.
nil = default to true (secure by default). | | | +#### MultiClusterConfiguration + + + +MultiClusterConfiguration configures how the operator exports Global Router +pools through the Kubernetes Multi-Cluster Services (MCS) API, for the Global +Discovery Service's Kubernetes stores. The operator exports a pool only when +MCSImplementation is set and a DynamoPoolRelay with its DGD's name exists. + + + +_Appears in:_ +- [OperatorConfiguration](#operatorconfiguration) + +| Field | Description | Default | Validation | +| --- | --- | --- | --- | +| `mcsImplementation` _[MCSImplementation](#mcsimplementation)_ | MCSImplementation is the MCS implementation that imports the exports. Empty,
the default, exports nothing: Global Router pools are then discovered without
Kubernetes. Clearing it leaves earlier exports in place until their
DynamoPoolRelays are deleted. | | | + + #### NamespaceConfiguration @@ -4461,6 +4498,7 @@ OperatorConfiguration is the Schema for the operator configuration. | `mpi` _[MPIConfiguration](#mpiconfiguration)_ | MPI SSH secret configuration | | | | `checkpoint` _[CheckpointConfiguration](#checkpointconfiguration)_ | Checkpoint/restore configuration | | | | `discovery` _[DiscoveryConfiguration](#discoveryconfiguration)_ | Discovery backend configuration | | | +| `multiCluster` _[MultiClusterConfiguration](#multiclusterconfiguration)_ | MultiCluster configures exporting Global Router pools to other clusters. | | | | `gpu` _[GPUConfiguration](#gpuconfiguration)_ | GPU discovery configuration | | | | `logging` _[LoggingConfiguration](#loggingconfiguration)_ | Logging configuration | | | | `security` _[SecurityConfiguration](#securityconfiguration)_ | HTTP/2 and TLS settings | | | diff --git a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx index 1310861d005d..9ecbaa7179a7 100644 --- a/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx +++ b/docs/fern/pages/reference/kubernetes-api/full-api-reference.mdx @@ -10,13 +10,13 @@ subtitle: Full Kubernetes CRD + operator-config API reference, generated from th > [!WARNING] > **Auto-generated from source.** This page is regenerated from the Dynamo operator CRDs by `docs/fern/scripts/gen_kubernetes_api.py` and covers **every** type across all three API packages: the deprecated `nvidia.com/v1alpha1` surface, the supported `nvidia.com/v1beta1` surface, and the operator's own `operator.config.dynamo.nvidia.com/v1alpha1` configuration. The trimmed [DGD](dynamo-graph-deployment.mdx), [DGDR](dynamo-graph-deployment-request.mdx), and [DCD](dynamo-component-deployment.mdx) references cover only user-facing `v1beta1` fields. To edit the surface, change the Go types under `deploy/operator/api/` and let CI regenerate this page. -Dynamo publishes 4 Kubernetes API packages with 219 typed CRD and config sections. +Dynamo publishes 4 Kubernetes API packages with 221 typed CRD and config sections. 74 types 76 types 40 types -29 types +31 types
@@ -1378,7 +1378,7 @@ AvailableReplicas is the number of Relay Pods the Deployment reports available. -Conditions includes Ready for listener availability, not model or KV readiness. Its reason is ListenerAvailable, DeploymentProgressing, DGDNotFound, DGDTerminating, UnsupportedDiscovery, UnsupportedSourceScope, NoWorkers, InvalidConfiguration, ReconcileFailed, or ResourceOwnershipConflict. +Conditions includes Ready for listener availability, not model or KV readiness. Its reason is ListenerAvailable, DeploymentProgressing, DGDNotFound, DGDTerminating, UnsupportedDiscovery, UnsupportedSourceScope, NoWorkers, InvalidConfiguration, ReconcileFailed, or ResourceOwnershipConflict. With multi-cluster export configured, Conditions also includes Exported: True with reason Exported once the pool's Services are exported, or False with reason Withdrawn, ServiceNotOwned, InvalidServiceName, ServiceNameTooLong, NoFrontend, ClusterIDMismatch, or ExportFailed. See [Condition](https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.28/#condition-v1-meta). **Validation:** Optional: \{\} @@ -5989,6 +5989,21 @@ Format is the log format (e.g., "json", "text") + +MCSImplementation identifies an implementation of the Multi-Cluster Services API. + + +**Kind:** `enum` +**Underlying type:** `string` +**Appears in:** [MultiClusterConfiguration](#multiclusterconfiguration) + +**Allowed values** + +- `` MCSImplementationNone exports nothing. +- mcs MCSImplementationStandard is an implementation of the SIG Multicluster API that imports Services into ServiceImports and EndpointSlices with the standard labels. +- karmada MCSImplementationKarmada is Karmada, which reads the same ServiceExport in member clusters but names imported EndpointSlices after the source cluster. + + MPIConfiguration holds MPI SSH secret settings. @@ -6025,6 +6040,22 @@ Secure enables secure serving for the metrics endpoint. nil = default to true (s + +MultiClusterConfiguration configures how the operator exports Global Router +pools through the Kubernetes Multi-Cluster Services (MCS) API, for the Global +Discovery Service's Kubernetes stores. The operator exports a pool only when +MCSImplementation is set and a DynamoPoolRelay with its DGD's name exists. + + +**Kind:** `type` +**Appears in:** [OperatorConfiguration](#operatorconfiguration) + + +MCSImplementation is the MCS implementation that imports the exports. Empty, the default, exports nothing: Global Router pools are then discovered without Kubernetes. Clearing it leaves earlier exports in place until their DynamoPoolRelays are deleted. +See [MCSImplementation](#mcsimplementation). + + + NamespaceConfiguration determines operator namespace mode. @@ -6134,6 +6165,11 @@ Discovery backend configuration See [DiscoveryConfiguration](#discoveryconfiguration). + +MultiCluster configures exporting Global Router pools to other clusters. +See [MultiClusterConfiguration](#multiclusterconfiguration). + + GPU discovery configuration See [GPUConfiguration](#gpuconfiguration). diff --git a/docs/fern/scripts/tests/test_gen_kubernetes_api.py b/docs/fern/scripts/tests/test_gen_kubernetes_api.py index 2b480cf15ee6..4dd10f7ba772 100644 --- a/docs/fern/scripts/tests/test_gen_kubernetes_api.py +++ b/docs/fern/scripts/tests/test_gen_kubernetes_api.py @@ -52,7 +52,7 @@ "nvidia.com/v1alpha1": 74, "nvidia.com/v1beta1": 76, "nvidia.com/v1beta2": 40, - "operator.config.dynamo.nvidia.com/v1alpha1": 29, + "operator.config.dynamo.nvidia.com/v1alpha1": 31, } EXPECTED_OPERATOR_DEFAULT_SECTIONS = ( "Pod Specification Defaults",