diff --git a/deploy/operator/internal/checkpoint/resolve.go b/deploy/operator/internal/checkpoint/resolve.go index 4e0a968a3f7d..baf2d576cfd3 100644 --- a/deploy/operator/internal/checkpoint/resolve.go +++ b/deploy/operator/internal/checkpoint/resolve.go @@ -69,7 +69,7 @@ func checkpointArtifactVersion(ckpt *nvidiacomv1alpha1.DynamoCheckpoint) string func ResolveCheckpointForService( ctx context.Context, - c client.Client, + c client.Reader, namespace string, config *nvidiacomv1alpha1.ServiceCheckpointConfig, gate features.Gate, diff --git a/deploy/operator/internal/checkpoint/resource.go b/deploy/operator/internal/checkpoint/resource.go index eb7db1bf25e7..570d690de326 100644 --- a/deploy/operator/internal/checkpoint/resource.go +++ b/deploy/operator/internal/checkpoint/resource.go @@ -59,7 +59,7 @@ func CheckpointID(ckpt *nvidiacomv1alpha1.DynamoCheckpoint) (string, error) { func FindCheckpointByCheckpointID( ctx context.Context, - c client.Client, + c client.Reader, namespace string, checkpointID string, excludeName string, @@ -125,7 +125,7 @@ func FindCheckpointByCheckpointID( func FindCheckpointByIdentityHash( ctx context.Context, - c client.Client, + c client.Reader, namespace string, hash string, excludeName string, diff --git a/deploy/operator/internal/controller/dcd_checkpoint_storage_reconciler.go b/deploy/operator/internal/controller/dcd_checkpoint_storage_reconciler.go new file mode 100644 index 000000000000..3051e7600f8d --- /dev/null +++ b/deploy/operator/internal/controller/dcd_checkpoint_storage_reconciler.go @@ -0,0 +1,72 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "fmt" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/checkpoint" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// dcdCheckpointStorageReconciler owns the checkpoint storage resources needed +// by a DynamoComponentDeployment. Keeping this convergence step outside the +// workload renderer makes rendering read-only and prevents multinode rendering +// from repeating the same write for both leader and worker templates. +type dcdCheckpointStorageReconciler struct { + client client.Client + storage configv1alpha1.CheckpointStorageConfiguration + gate features.Gate +} + +func newDCDCheckpointStorageReconciler( + kubeClient client.Client, + storage configv1alpha1.CheckpointStorageConfiguration, + gate features.Gate, +) *dcdCheckpointStorageReconciler { + return &dcdCheckpointStorageReconciler{ + client: kubeClient, + storage: storage, + gate: gate, + } +} + +func (r *dcdCheckpointStorageReconciler) Reconcile( + ctx context.Context, + dcd *nvidiacomv1beta1.DynamoComponentDeployment, +) error { + if dcd == nil { + return fmt.Errorf("dynamo component deployment is required") + } + if r.gate == nil { + return fmt.Errorf("feature gate is required") + } + if !r.gate.Enabled(features.Checkpoint) { + return nil + } + if dynamo.GetCheckpoint(&dcd.Spec.DynamoComponentDeploymentSharedSpec) == nil { + return nil + } + + return checkpoint.EnsureStoragePVC(ctx, r.client, dcd.Namespace, r.storage) +} diff --git a/deploy/operator/internal/controller/dcd_checkpoint_storage_reconciler_test.go b/deploy/operator/internal/controller/dcd_checkpoint_storage_reconciler_test.go new file mode 100644 index 000000000000..1048348b38b1 --- /dev/null +++ b/deploy/operator/internal/controller/dcd_checkpoint_storage_reconciler_test.go @@ -0,0 +1,102 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "testing" + + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + snapshotprotocol "github.com/ai-dynamo/dynamo/deploy/snapshot/protocol" + "github.com/stretchr/testify/require" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client/fake" +) + +func TestDCDCheckpointStorageReconciler(t *testing.T) { + testScheme := runtime.NewScheme() + require.NoError(t, corev1.AddToScheme(testScheme)) + + storage := configv1alpha1.CheckpointStorageConfiguration{ + Type: snapshotprotocol.StorageTypePVC, + PVC: configv1alpha1.CheckpointPVCConfig{ + PVCName: "checkpoint-storage", + BasePath: "/checkpoints", + Create: true, + Size: "1Gi", + AccessMode: string(corev1.ReadWriteMany), + }, + } + + tests := []struct { + name string + gate features.Gates + checkpointEnabled bool + wantPVC bool + }{ + { + name: "creates storage for a checkpoint-enabled component", + gate: features.Gates{Checkpoint: true}, + checkpointEnabled: true, + wantPVC: true, + }, + { + name: "does nothing when the checkpoint feature is disabled", + checkpointEnabled: true, + }, + { + name: "does nothing when the component has no enabled checkpoint", + gate: features.Gates{Checkpoint: true}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + t.Log("Reconcile namespace checkpoint storage independently from workload rendering") + kubeClient := fake.NewClientBuilder().WithScheme(testScheme).Build() + dcd := &nvidiacomv1beta1.DynamoComponentDeployment{ + ObjectMeta: metav1.ObjectMeta{Name: "worker", Namespace: "default"}, + } + if tt.checkpointEnabled { + dcd.Spec.Experimental = &nvidiacomv1beta1.ExperimentalSpec{ + Checkpoint: &nvidiacomv1beta1.ComponentCheckpointConfig{Enabled: true}, + } + } + + reconciler := newDCDCheckpointStorageReconciler(kubeClient, storage, tt.gate) + require.NoError(t, reconciler.Reconcile(context.Background(), dcd)) + + pvc := &corev1.PersistentVolumeClaim{} + err := kubeClient.Get(context.Background(), types.NamespacedName{ + Name: storage.PVC.PVCName, + Namespace: dcd.Namespace, + }, pvc) + if tt.wantPVC { + require.NoError(t, err) + return + } + require.True(t, k8serrors.IsNotFound(err), "expected no PVC, got %v", err) + }) + } +} diff --git a/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go b/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go index f71ef43a6d44..a767fb740904 100644 --- a/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamocomponentdeployment_controller.go @@ -34,20 +34,17 @@ import ( "emperror.dev/errors" configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/checkpoint" "github.com/ai-dynamo/dynamo/deploy/operator/internal/common" commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" commonController "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" - "github.com/ai-dynamo/dynamo/deploy/operator/internal/gms" "github.com/ai-dynamo/dynamo/deploy/operator/internal/observability" networkingv1beta1 "istio.io/client-go/pkg/apis/networking/v1beta1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/selection" - "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/tools/record" ctrl "sigs.k8s.io/controller-runtime" @@ -184,6 +181,15 @@ func (r *DynamoComponentDeploymentReconciler) Reconcile(ctx context.Context, req } } + checkpointStorageReconciler := newDCDCheckpointStorageReconciler( + r.Client, + r.Config.Checkpoint.Storage, + r.RuntimeConfig.Gate, + ) + if err = checkpointStorageReconciler.Reconcile(ctx, dynamoComponentDeployment); err != nil { + return ctrl.Result{}, fmt.Errorf("failed to reconcile checkpoint storage: %w", err) + } + // Create the appropriate workload resource based on deployment type var componentReconcileResult ComponentReconcileResult if r.RuntimeConfig.Gate.Enabled(features.LWS) && dynamoComponentDeployment.IsMultinode() { @@ -476,25 +482,6 @@ func IsLeaderWorkerSetReady(leaderWorkerSet *leaderworkersetv1.LeaderWorkerSet) return false } -func (r *DynamoComponentDeploymentReconciler) generateLeaderPodTemplateSpec(ctx context.Context, opt generateResourceOption, labels map[string]string) (*corev1.PodTemplateSpec, error) { - leaderPodTemplateSpec, err := r.generatePodTemplateSpec(ctx, opt, dynamo.RoleLeader) - if err != nil { - return nil, errors.Wrap(err, "failed to generate leader pod template") - } - - maps.Copy(leaderPodTemplateSpec.ObjectMeta.Labels, labels) - leaderPodTemplateSpec.ObjectMeta.Labels["role"] = "leader" - delete(leaderPodTemplateSpec.ObjectMeta.Labels, commonconsts.KubeLabelDynamoSelector) - - err = checkMainContainer(&leaderPodTemplateSpec.Spec) - - if err != nil { - return nil, errors.Wrap(err, "generateLeaderPodTemplateSpec: failed to check main container") - } - - return leaderPodTemplateSpec, nil -} - func checkMainContainer(spec *corev1.PodSpec) error { if len(spec.Containers) == 0 { @@ -526,31 +513,17 @@ func checkMainContainer(spec *corev1.PodSpec) error { return nil } -func (r *DynamoComponentDeploymentReconciler) generateWorkerPodTemplateSpec(ctx context.Context, opt generateResourceOption, labels map[string]string) (*corev1.PodTemplateSpec, error) { - workerPodTemplateSpec, err := r.generatePodTemplateSpec(ctx, opt, dynamo.RoleWorker) - if err != nil { - return nil, errors.Wrap(err, "failed to generate worker pod template") - } - - maps.Copy(workerPodTemplateSpec.ObjectMeta.Labels, labels) - workerPodTemplateSpec.ObjectMeta.Labels["role"] = "worker" - delete(workerPodTemplateSpec.ObjectMeta.Labels, commonconsts.KubeLabelDynamoSelector) - - err = checkMainContainer(&workerPodTemplateSpec.Spec) - - if err != nil { - return nil, errors.Wrap(err, "generateWorkerPodTemplateSpec: failed to check LWS worker main container") - } - - return workerPodTemplateSpec, nil -} - // generateLeaderWorkerSet creates a single LeaderWorkerSet resource from the DynamoComponentDeployment // with Spec.Replicas set to the desired replica count, allowing LWS to natively manage scaling. func (r *DynamoComponentDeploymentReconciler) generateLeaderWorkerSet(ctx context.Context, opt generateResourceOption) (*leaderworkersetv1.LeaderWorkerSet, bool, error) { logs := log.FromContext(ctx) logs.Info("Generating LeaderWorkerSet") + leaderPodTemplateSpec, workerPodTemplateSpec, err := r.workloadRenderer().renderMultinodePodTemplateSpecs(ctx, opt.dynamoComponentDeployment) + if err != nil { + return nil, false, errors.Wrap(err, "generateLeaderWorkerSet: failed to render multinode pod templates") + } + kubeName := leaderWorkerSetName(opt.dynamoComponentDeployment) kubeNs := opt.dynamoComponentDeployment.Namespace labels := dynamo.GetDCDKubeLabels(opt.dynamoComponentDeployment) @@ -558,10 +531,6 @@ func (r *DynamoComponentDeploymentReconciler) generateLeaderWorkerSet(ctx contex if labels == nil { labels = make(map[string]string) } - podLabels, err := r.getDCDWorkloadPodLabels(ctx, opt.dynamoComponentDeployment) - if err != nil { - return nil, false, err - } leaderWorkerSet := &leaderworkersetv1.LeaderWorkerSet{ ObjectMeta: metav1.ObjectMeta{ @@ -571,24 +540,6 @@ func (r *DynamoComponentDeploymentReconciler) generateLeaderWorkerSet(ctx contex }, } - leaderPodLabels := make(map[string]string) - for k, v := range podLabels { - leaderPodLabels[k] = v - } - leaderPodTemplateSpec, err := r.generateLeaderPodTemplateSpec(ctx, opt, leaderPodLabels) - if err != nil { - return nil, false, errors.Wrap(err, "generateLeaderWorkerSet: failed to generate leader pod template") - } - - workerPodLabels := make(map[string]string) - for k, v := range podLabels { - workerPodLabels[k] = v - } - workerPodTemplateSpec, err := r.generateWorkerPodTemplateSpec(ctx, opt, workerPodLabels) - if err != nil { - return nil, false, errors.Wrap(err, "generateLeaderWorkerSet: failed to generate worker pod template") - } - desiredReplicas := int32(1) if opt.dynamoComponentDeployment.Spec.Replicas != nil { desiredReplicas = *opt.dynamoComponentDeployment.Spec.Replicas @@ -608,30 +559,6 @@ func (r *DynamoComponentDeploymentReconciler) generateLeaderWorkerSet(ctx contex return leaderWorkerSet, false, nil } -// getDCDWorkloadPodLabels keeps LWS pod labels aligned with the workload -// component type used by Deployment and Service rendering. -func (r *DynamoComponentDeploymentReconciler) getDCDWorkloadPodLabels( - ctx context.Context, - dcd *nvidiacomv1beta1.DynamoComponentDeployment, -) (map[string]string, error) { - labels := dynamo.GetDCDKubeLabels(dcd) - componentType, err := r.getDCDWorkloadComponentType(ctx, dcd) - if err != nil { - return nil, err - } - if componentType == "" { - return labels, nil - } - labels[commonconsts.KubeLabelDynamoComponentType] = componentType - specType := string(dcd.Spec.ComponentType) - if componentType == commonconsts.ComponentTypeWorker && - (specType == commonconsts.ComponentTypePrefill || specType == commonconsts.ComponentTypeDecode) && - labels[commonconsts.KubeLabelDynamoSubComponentType] == "" { - labels[commonconsts.KubeLabelDynamoSubComponentType] = specType - } - return labels, nil -} - // leaderWorkerSetName keeps the native LWS at -0 so it can adopt // legacy replicas and avoid colliding with the operator ClusterIP service. func leaderWorkerSetName(dcd *nvidiacomv1beta1.DynamoComponentDeployment) string { @@ -864,7 +791,7 @@ func (r *DynamoComponentDeploymentReconciler) generateDeployment(ctx context.Con } // nolint: gosimple - podTemplateSpec, err := r.generatePodTemplateSpec(ctx, opt, dynamo.RoleMain) + podTemplateSpec, err := r.workloadRenderer().generatePodTemplateSpec(ctx, opt.dynamoComponentDeployment, dynamo.RoleMain) if err != nil { return } @@ -917,191 +844,8 @@ type generateResourceOption struct { dynamoComponentDeployment *nvidiacomv1beta1.DynamoComponentDeployment } -func (r *DynamoComponentDeploymentReconciler) generatePodTemplateSpec(ctx context.Context, opt generateResourceOption, role dynamo.Role) (*corev1.PodTemplateSpec, error) { - dcd := opt.dynamoComponentDeployment - component := &dcd.Spec.DynamoComponentDeploymentSharedSpec - componentType, err := r.getDCDWorkloadComponentType(ctx, dcd) - if err != nil { - return nil, err - } - podLabels := dynamo.GetDCDKubeLabels(dcd) - podAnnotations := dynamo.GetDCDKubeAnnotations(dcd) - kubeName := dcd.Name - - // Convert user-provided metrics annotation into controller-managed label - // By default (no annotation), metrics are enabled - if podAnnotations[commonconsts.KubeAnnotationEnableMetrics] == commonconsts.KubeLabelValueFalse { - // Explicitly disabled, don't add the label - } else { - // Any other value (including empty) enables metrics - podLabels[commonconsts.KubeLabelMetricsEnabled] = commonconsts.KubeLabelValueTrue - } - - if parentName := dcd.GetLabels()[commonconsts.KubeLabelDynamoGraphDeploymentName]; parentName != "" { - podLabels[commonconsts.KubeLabelDynamoGraphDeploymentName] = parentName - } else if parentName := dcd.GetParentGraphDeploymentName(); parentName != "" { - podLabels[commonconsts.KubeLabelDynamoGraphDeploymentName] = parentName - } - if componentType != "" { - podLabels[commonconsts.KubeLabelDynamoComponentType] = componentType - } - if componentName := dynamo.GetDCDComponentName(dcd); componentName != "" { - podLabels[commonconsts.KubeLabelDynamoComponent] = componentName - } - if dynamoNamespace := dynamo.GetDCDDynamoNamespace(dcd); dynamoNamespace != "" { - podLabels[commonconsts.KubeLabelDynamoNamespace] = dynamoNamespace - } - if workerHash := dcd.GetLabels()[commonconsts.KubeLabelDynamoWorkerHash]; workerHash != "" { - podLabels[commonconsts.KubeLabelDynamoWorkerHash] = workerHash - } - - // Resolve checkpoint for this component - var checkpointInfo *checkpoint.CheckpointInfo - if checkpointConfig := dynamo.GetCheckpoint(component); r.RuntimeConfig.Gate.Enabled(features.Checkpoint) && checkpointConfig != nil { - info, err := checkpoint.ResolveCheckpointForService(ctx, r.Client, dcd.Namespace, dynamo.ToAlphaCheckpointConfig(checkpointConfig), r.RuntimeConfig.Gate) - if err != nil { - return nil, errors.Wrap(err, "failed to resolve checkpoint") - } - if dynamo.IsIntraPodFailoverEnabled(&opt.dynamoComponentDeployment.Spec.DynamoComponentDeploymentSharedSpec) { - info.RestoreTargetContainers = dynamo.IntraPodFailoverEngineContainerNames() - } - if err := gms.OverlayClients(&info.GPUMemoryService, info.CheckpointName, info.Exists, dynamo.GetGPUMemoryService(component)); err != nil { - return nil, errors.Wrap(err, "failed to apply checkpoint gpuMemoryService config") - } - checkpointInfo = info - if err := checkpoint.EnsureStoragePVC(ctx, r.Client, opt.dynamoComponentDeployment.Namespace, r.Config.Checkpoint.Storage); err != nil { - return nil, errors.Wrap(err, "failed to ensure checkpoint storage PVC") - } - } - - podSpec, err := dynamo.GenerateBasePodSpecForController( - dcd, - r.DockerSecretRetriever, - r.Config, - role, - commonconsts.MultinodeDeploymentTypeLWS, - checkpointInfo, - dynamo.GenerateBasePodSpecForControllerOptions{ - WorkloadComponentType: nvidiacomv1beta1.ComponentType(componentType), - }, - ) - if err != nil { - return nil, errors.Wrap(err, "failed to generate base pod spec") - } - if r.RuntimeConfig.Gate.Enabled(features.Checkpoint) { - if checkpointInfo == nil || - string(checkpointInfo.StartupPolicy) == string(nvidiacomv1beta1.CheckpointStartupPolicyWaitForCheckpoint) { - if err := checkpoint.InjectCheckpointIntoPodSpecWithStorageConfig( - ctx, - r.Client, - dcd.Namespace, - podSpec, - checkpointInfo, - r.Config.Checkpoint.Storage, - r.Config.Checkpoint.EffectiveSeccompProfile(), - ); err != nil { - return nil, errors.Wrap(err, "failed to inject checkpoint config") - } - } - // Immediate mode keeps owner pod templates stable when checkpoint - // readiness changes. The pod-create webhook performs restore shaping - // only for newly-created Pods after the checkpoint is Ready. - } - - // Ensure we have at least one container (the main container should be there from GenerateBasePodSpec) - if len(podSpec.Containers) == 0 { - return nil, errors.New("no containers found in base pod spec") - } - - podLabels[commonconsts.KubeLabelDynamoSelector] = kubeName - - // Add discovery labels to pod template for Pod-based daemon filtering - if commonController.IsK8sDiscoveryEnabled(r.Config.Discovery.Backend, podAnnotations) { - podLabels[commonconsts.KubeLabelDynamoDiscoveryBackend] = "kubernetes" - podLabels[commonconsts.KubeLabelDynamoDiscoveryEnabled] = commonconsts.KubeLabelValueTrue - } - - // Restore labels are operator-controlled state. Immediate mode stamps a - // stable candidate annotation and defers restore mutation to Pod CREATE; all - // other modes can shape the owner template once the checkpoint is ready. - if checkpointInfo != nil && - (checkpointInfo.StartupPolicy == "" || - string(checkpointInfo.StartupPolicy) == string(nvidiacomv1beta1.CheckpointStartupPolicyImmediate)) { - if err := checkpoint.ApplyRestoreCandidateMetadata(podLabels, podAnnotations, checkpointInfo); err != nil { - return nil, errors.Wrap(err, "failed to apply checkpoint candidate metadata") - } - } else if err := checkpoint.ApplyRestorePodMetadataWithStorageConfig(podLabels, podAnnotations, checkpointInfo, r.Config.Checkpoint.Storage); err != nil { - return nil, errors.Wrap(err, "failed to apply checkpoint metadata") - } - - if podSpec.ServiceAccountName == "" { - serviceAccounts := &corev1.ServiceAccountList{} - err = r.List(ctx, serviceAccounts, client.InNamespace(dcd.Namespace), client.MatchingLabels{ - commonconsts.KubeLabelDynamoComponentPod: commonconsts.KubeLabelValueTrue, - }) - if err != nil { - return nil, errors.Wrapf(err, "failed to list service accounts in namespace %s", dcd.Namespace) - } - if len(serviceAccounts.Items) > 0 { - podSpec.ServiceAccountName = serviceAccounts.Items[0].Name - } else { - podSpec.ServiceAccountName = DefaultServiceAccountName - } - } - - return &corev1.PodTemplateSpec{ - ObjectMeta: metav1.ObjectMeta{ - Labels: podLabels, - Annotations: podAnnotations, - }, - Spec: *podSpec, - }, nil -} - func (r *DynamoComponentDeploymentReconciler) generateService(ctx context.Context, opt generateResourceOption) (*corev1.Service, bool, error) { - dcd := opt.dynamoComponentDeployment - - deleteStub := &corev1.Service{ - ObjectMeta: metav1.ObjectMeta{ - Name: dynamo.NormalizeKubeResourceName(dcd.Name), - Namespace: dcd.Namespace, - }, - } - - annotations := dynamo.GetDCDKubeAnnotations(dcd) - isK8sDiscovery := commonController.IsK8sDiscoveryEnabled(r.Config.Discovery.Backend, annotations) - - if !(isK8sDiscovery || dcd.IsFrontendComponent()) { - return deleteStub, true, nil - } - - dynamoNamespace := dynamo.GetDCDDynamoNamespace(dcd) - if dynamoNamespace == "" { - return nil, false, fmt.Errorf("expected DynamoComponentDeployment %s to have a dynamoNamespace", dcd.Name) - } - - componentType, err := r.getDCDWorkloadComponentType(ctx, dcd) - if err != nil { - return nil, false, err - } - - svc, err := dynamo.GenerateComponentService(dynamo.ComponentServiceParams{ - ServiceName: dcd.Name, - Namespace: dcd.Namespace, - ComponentType: componentType, - DynamoNamespace: dynamoNamespace, - ComponentName: dynamo.GetDCDComponentName(dcd), - Labels: dynamo.GetDCDKubeLabels(dcd), - Annotations: annotations, - IsK8sDiscovery: isK8sDiscovery, - }) - if err != nil { - return nil, false, err - } - if dcd.IsMultinode() { - svc.Spec.Selector["role"] = "leader" - } - return svc, false, nil + return r.workloadRenderer().generateService(ctx, opt.dynamoComponentDeployment) } // getDCDWorkloadComponentType returns the component type that should be @@ -1112,77 +856,7 @@ func (r *DynamoComponentDeploymentReconciler) getDCDWorkloadComponentType( ctx context.Context, dcd *nvidiacomv1beta1.DynamoComponentDeployment, ) (string, error) { - componentType := dynamo.GetDCDWorkloadComponentType(dcd) - if componentType == commonconsts.ComponentTypeWorker || !dynamo.IsWorkerComponent(componentType) { - return componentType, nil - } - if dcd == nil { - return componentType, nil - } - - // Decode/prefill are the current v1beta1 workload-facing worker types. If - // this DCD generation is already serving with legacy v1alpha1 worker - // selectors, keep rendering pod labels/env and service selectors as - // "worker" so a no-op upgrade keeps matching already-running pods. - if hasLegacyWorkerSelector(dcd.GetLabels(), componentType) { - return commonconsts.ComponentTypeWorker, nil - } - - legacy, err := r.hasExistingLegacyWorkerSelector(ctx, dcd, componentType) - if err != nil { - return "", err - } - if legacy { - return commonconsts.ComponentTypeWorker, nil - } - - return componentType, nil -} - -func (r *DynamoComponentDeploymentReconciler) hasExistingLegacyWorkerSelector( - ctx context.Context, - dcd *nvidiacomv1beta1.DynamoComponentDeployment, - componentType string, -) (bool, error) { - if dcd == nil || r == nil || r.Client == nil { - return false, nil - } - - deployment := &appsv1.Deployment{} - if err := r.Get(ctx, types.NamespacedName{Name: dcd.Name, Namespace: dcd.Namespace}, deployment); err == nil { - if hasLegacyWorkerSelector(deployment.Spec.Template.Labels, componentType) { - return true, nil - } - } else if !k8serrors.IsNotFound(err) { - return false, fmt.Errorf("failed to get deployment %s/%s: %w", dcd.Namespace, dcd.Name, err) - } - - if r.RuntimeConfig.Gate.Enabled(features.LWS) { - // Check the adopted "-0" LWS to keep alpha-era worker labels stable. - lwsName := leaderWorkerSetName(dcd) - leaderWorkerSet := &leaderworkersetv1.LeaderWorkerSet{} - if err := r.Get(ctx, types.NamespacedName{Name: lwsName, Namespace: dcd.Namespace}, leaderWorkerSet); err == nil { - template := leaderWorkerSet.Spec.LeaderWorkerTemplate - if template.LeaderTemplate != nil && hasLegacyWorkerSelector(template.LeaderTemplate.Labels, componentType) { - return true, nil - } - if hasLegacyWorkerSelector(template.WorkerTemplate.Labels, componentType) { - return true, nil - } - } else if !k8serrors.IsNotFound(err) { - return false, fmt.Errorf("failed to get leaderworkerset %s/%s: %w", dcd.Namespace, lwsName, err) - } - } - - serviceName := dynamo.NormalizeKubeResourceName(dcd.Name) - service := &corev1.Service{} - if err := r.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: dcd.Namespace}, service); err == nil { - return hasLegacyWorkerSelector(service.Spec.Selector, componentType), nil - } else if !k8serrors.IsNotFound(err) { - return false, fmt.Errorf("failed to get service %s/%s: %w", dcd.Namespace, serviceName, err) - } - - return false, nil + return r.workloadRenderer().getDCDWorkloadComponentType(ctx, dcd) } func hasLegacyWorkerSelector(labels map[string]string, componentType string) bool { diff --git a/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go b/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go index 9de64d2b6ec4..771f6251a723 100644 --- a/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go @@ -633,6 +633,7 @@ func TestDynamoComponentDeploymentReconciler_generateService_DottedDeleteStub(t } func TestDynamoComponentDeploymentReconciler_LWSNameDoesNotCollideWithComponentService(t *testing.T) { + t.Log("Build a multinode DCD and the dependencies shared by reconciliation and rendering") s := scheme.Scheme require.NoError(t, v1alpha1.AddToScheme(s)) require.NoError(t, corev1.AddToScheme(s)) @@ -694,6 +695,15 @@ func TestDynamoComponentDeploymentReconciler_LWSNameDoesNotCollideWithComponentS }, } + t.Log("Render the reusable component Service and multinode pod templates directly") + renderer := newDCDWorkloadRenderer(r.Client, r.Config, r.RuntimeConfig, r.DockerSecretRetriever) + renderedService, renderedServiceToDelete, err := renderer.generateService(context.Background(), dcd) + require.NoError(t, err) + require.False(t, renderedServiceToDelete) + renderedLeaderTemplate, renderedWorkerTemplate, err := renderer.renderMultinodePodTemplateSpecs(context.Background(), dcd) + require.NoError(t, err) + + t.Log("Compose the same resources through the DCD controller") service, toDelete, err := r.generateService(context.Background(), generateResourceOption{dynamoComponentDeployment: dcd}) require.NoError(t, err) require.False(t, toDelete) @@ -702,6 +712,10 @@ func TestDynamoComponentDeploymentReconciler_LWSNameDoesNotCollideWithComponentS require.NoError(t, err) require.False(t, toDelete) + t.Log("Verify controller composition preserves the reusable renderer output") + require.Equal(t, renderedService, service) + require.Equal(t, renderedLeaderTemplate, lws.Spec.LeaderWorkerTemplate.LeaderTemplate) + require.Equal(t, *renderedWorkerTemplate, lws.Spec.LeaderWorkerTemplate.WorkerTemplate) require.Equal(t, "vllm-disagg-decode-4e5bb2af", service.Name) require.Equal(t, "vllm-disagg-decode-4e5bb2af-0", lws.Name) require.NotEqual(t, service.Name, lws.Name) @@ -774,9 +788,9 @@ func TestDynamoComponentDeploymentReconciler_LegacyAlphaWorkloadComponentType(t }, } - podTemplate, err := r.generatePodTemplateSpec( + podTemplate, err := r.workloadRenderer().generatePodTemplateSpec( context.Background(), - generateResourceOption{dynamoComponentDeployment: dcd}, + dcd, dynamo.RoleMain, ) require.NoError(t, err) @@ -949,9 +963,9 @@ func TestDynamoComponentDeploymentReconciler_BetaPrefillWorkloadComponentType(t }, } - podTemplate, err := r.generatePodTemplateSpec( + podTemplate, err := r.workloadRenderer().generatePodTemplateSpec( context.Background(), - generateResourceOption{dynamoComponentDeployment: dcd}, + dcd, dynamo.RoleMain, ) require.NoError(t, err) @@ -1764,9 +1778,9 @@ func TestDynamoComponentDeploymentReconciler_generatePodTemplateSpec_RestoreLabe } r := makeReconciler(dcd, ckpt) - podTemplateSpec, err := r.generatePodTemplateSpec( + podTemplateSpec, err := r.workloadRenderer().generatePodTemplateSpec( context.Background(), - generateResourceOption{dynamoComponentDeployment: dcd}, + dcd, dynamo.RoleMain, ) if err != nil { @@ -1824,9 +1838,9 @@ func TestDynamoComponentDeploymentReconciler_generatePodTemplateSpec_RestoreLabe r := makeReconciler(dcd, ckpt) r.RuntimeConfig = &controller_common.RuntimeConfig{Gate: features.Gates{Checkpoint: true, GMSSnapshot: true}} - podTemplateSpec, err := r.generatePodTemplateSpec( + podTemplateSpec, err := r.workloadRenderer().generatePodTemplateSpec( context.Background(), - generateResourceOption{dynamoComponentDeployment: dcd}, + dcd, dynamo.RoleMain, ) if err != nil { @@ -1903,9 +1917,9 @@ func TestDynamoComponentDeploymentReconciler_generatePodTemplateSpec_RestoreLabe } r := makeReconciler(dcd, ckpt) - _, err = r.generatePodTemplateSpec( + _, err = r.workloadRenderer().generatePodTemplateSpec( context.Background(), - generateResourceOption{dynamoComponentDeployment: dcd}, + dcd, dynamo.RoleMain, ) require.Error(t, err) @@ -1945,9 +1959,9 @@ func TestDynamoComponentDeploymentReconciler_generatePodTemplateSpec_RestoreLabe r := makeReconciler(dcd, ckpt) r.RuntimeConfig = &controller_common.RuntimeConfig{Gate: features.Gates{Checkpoint: true, GMSSnapshot: true}} - podTemplateSpec, err := r.generatePodTemplateSpec( + podTemplateSpec, err := r.workloadRenderer().generatePodTemplateSpec( context.Background(), - generateResourceOption{dynamoComponentDeployment: dcd}, + dcd, dynamo.RoleMain, ) if err != nil { @@ -1990,9 +2004,9 @@ func TestDynamoComponentDeploymentReconciler_generatePodTemplateSpec_RestoreLabe } r := makeReconciler(dcd, ckpt) - podTemplateSpec, err := r.generatePodTemplateSpec( + podTemplateSpec, err := r.workloadRenderer().generatePodTemplateSpec( context.Background(), - generateResourceOption{dynamoComponentDeployment: dcd}, + dcd, dynamo.RoleMain, ) if err != nil { @@ -2051,9 +2065,9 @@ func TestDynamoComponentDeploymentReconciler_generatePodTemplateSpec_RestoreLabe } r := makeReconciler(dcd, ckpt) - podTemplateSpec, err := r.generatePodTemplateSpec( + podTemplateSpec, err := r.workloadRenderer().generatePodTemplateSpec( context.Background(), - generateResourceOption{dynamoComponentDeployment: dcd}, + dcd, dynamo.RoleMain, ) if err != nil { @@ -2094,9 +2108,9 @@ func TestDynamoComponentDeploymentReconciler_generatePodTemplateSpec_RestoreLabe } r := makeReconciler(dcd, ckpt) - podTemplateSpec, err := r.generatePodTemplateSpec( + podTemplateSpec, err := r.workloadRenderer().generatePodTemplateSpec( context.Background(), - generateResourceOption{dynamoComponentDeployment: dcd}, + dcd, dynamo.RoleMain, ) if err != nil { @@ -3468,9 +3482,9 @@ func TestGenerateWorkerPodTemplateSpecDoesNotRequireGPUResource(t *testing.T) { }, } - got, err := reconciler.generateWorkerPodTemplateSpec( + got, err := reconciler.workloadRenderer().generateWorkerPodTemplateSpec( context.Background(), - generateResourceOption{dynamoComponentDeployment: dcd}, + dcd, map[string]string{"app": "demo"}, ) require.NoError(t, err) diff --git a/deploy/operator/internal/controller/dynamocomponentdeployment_renderer.go b/deploy/operator/internal/controller/dynamocomponentdeployment_renderer.go new file mode 100644 index 000000000000..7a62fd1b2e30 --- /dev/null +++ b/deploy/operator/internal/controller/dynamocomponentdeployment_renderer.go @@ -0,0 +1,438 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. + * SPDX-License-Identifier: Apache-2.0 + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package controller + +import ( + "context" + "fmt" + "maps" + + "emperror.dev/errors" + configv1alpha1 "github.com/ai-dynamo/dynamo/deploy/operator/api/config/v1alpha1" + nvidiacomv1beta1 "github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/checkpoint" + commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts" + commonController "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/dynamo" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/features" + "github.com/ai-dynamo/dynamo/deploy/operator/internal/gms" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" + leaderworkersetv1 "sigs.k8s.io/lws/api/leaderworkerset/v1" +) + +const dcdWorkloadRoleLabel = "role" + +// dcdWorkloadRenderer contains the dependencies required to render the +// workload-facing resources of a DynamoComponentDeployment. It deliberately +// does not own reconciliation, watches, finalizers, or status. +// +// Keeping this concrete and package-private establishes a reusable rendering +// boundary without committing to a public provider framework. Composite +// workload programs can reuse this unit without constructing a +// DynamoComponentDeploymentReconciler. +type dcdWorkloadRenderer struct { + reader client.Reader + config *configv1alpha1.OperatorConfiguration + runtimeConfig *commonController.RuntimeConfig + dockerSecretRetriever dockerSecretRetriever +} + +func newDCDWorkloadRenderer( + reader client.Reader, + config *configv1alpha1.OperatorConfiguration, + runtimeConfig *commonController.RuntimeConfig, + dockerSecretRetriever dockerSecretRetriever, +) *dcdWorkloadRenderer { + return &dcdWorkloadRenderer{ + reader: reader, + config: config, + runtimeConfig: runtimeConfig, + dockerSecretRetriever: dockerSecretRetriever, + } +} + +func (r *DynamoComponentDeploymentReconciler) workloadRenderer() *dcdWorkloadRenderer { + return newDCDWorkloadRenderer(r.Client, r.Config, r.RuntimeConfig, r.DockerSecretRetriever) +} + +// renderMultinodePodTemplateSpecs renders the leader and worker pod templates +// shared by LWS and composite multinode workload resources. The caller remains +// responsible for composing those templates into its provider-native object. +func (r *dcdWorkloadRenderer) renderMultinodePodTemplateSpecs( + ctx context.Context, + dcd *nvidiacomv1beta1.DynamoComponentDeployment, +) (*corev1.PodTemplateSpec, *corev1.PodTemplateSpec, error) { + podLabels, err := r.getDCDWorkloadPodLabels(ctx, dcd) + if err != nil { + return nil, nil, err + } + + leaderLabels := make(map[string]string, len(podLabels)) + maps.Copy(leaderLabels, podLabels) + leaderPodTemplateSpec, err := r.generateLeaderPodTemplateSpec(ctx, dcd, leaderLabels) + if err != nil { + return nil, nil, err + } + + workerLabels := make(map[string]string, len(podLabels)) + maps.Copy(workerLabels, podLabels) + workerPodTemplateSpec, err := r.generateWorkerPodTemplateSpec(ctx, dcd, workerLabels) + if err != nil { + return nil, nil, err + } + + return leaderPodTemplateSpec, workerPodTemplateSpec, nil +} + +func (r *dcdWorkloadRenderer) generateLeaderPodTemplateSpec( + ctx context.Context, + dcd *nvidiacomv1beta1.DynamoComponentDeployment, + labels map[string]string, +) (*corev1.PodTemplateSpec, error) { + leaderPodTemplateSpec, err := r.generatePodTemplateSpec(ctx, dcd, dynamo.RoleLeader) + if err != nil { + return nil, errors.Wrap(err, "failed to generate leader pod template") + } + + maps.Copy(leaderPodTemplateSpec.ObjectMeta.Labels, labels) + leaderPodTemplateSpec.ObjectMeta.Labels[dcdWorkloadRoleLabel] = string(dynamo.RoleLeader) + delete(leaderPodTemplateSpec.ObjectMeta.Labels, commonconsts.KubeLabelDynamoSelector) + + if err := checkMainContainer(&leaderPodTemplateSpec.Spec); err != nil { + return nil, errors.Wrap(err, "generateLeaderPodTemplateSpec: failed to check main container") + } + + return leaderPodTemplateSpec, nil +} + +func (r *dcdWorkloadRenderer) generateWorkerPodTemplateSpec( + ctx context.Context, + dcd *nvidiacomv1beta1.DynamoComponentDeployment, + labels map[string]string, +) (*corev1.PodTemplateSpec, error) { + workerPodTemplateSpec, err := r.generatePodTemplateSpec(ctx, dcd, dynamo.RoleWorker) + if err != nil { + return nil, errors.Wrap(err, "failed to generate worker pod template") + } + + maps.Copy(workerPodTemplateSpec.ObjectMeta.Labels, labels) + workerPodTemplateSpec.ObjectMeta.Labels[dcdWorkloadRoleLabel] = string(dynamo.RoleWorker) + delete(workerPodTemplateSpec.ObjectMeta.Labels, commonconsts.KubeLabelDynamoSelector) + + if err := checkMainContainer(&workerPodTemplateSpec.Spec); err != nil { + return nil, errors.Wrap(err, "generateWorkerPodTemplateSpec: failed to check LWS worker main container") + } + + return workerPodTemplateSpec, nil +} + +func (r *dcdWorkloadRenderer) generatePodTemplateSpec( + ctx context.Context, + dcd *nvidiacomv1beta1.DynamoComponentDeployment, + role dynamo.Role, +) (*corev1.PodTemplateSpec, error) { + component := &dcd.Spec.DynamoComponentDeploymentSharedSpec + componentType, err := r.getDCDWorkloadComponentType(ctx, dcd) + if err != nil { + return nil, err + } + podLabels := dynamo.GetDCDKubeLabels(dcd) + podAnnotations := dynamo.GetDCDKubeAnnotations(dcd) + kubeName := dcd.Name + + // Convert user-provided metrics annotation into controller-managed label. + // By default (no annotation), metrics are enabled. + if podAnnotations[commonconsts.KubeAnnotationEnableMetrics] != commonconsts.KubeLabelValueFalse { + podLabels[commonconsts.KubeLabelMetricsEnabled] = commonconsts.KubeLabelValueTrue + } + + if parentName := dcd.GetLabels()[commonconsts.KubeLabelDynamoGraphDeploymentName]; parentName != "" { + podLabels[commonconsts.KubeLabelDynamoGraphDeploymentName] = parentName + } else if parentName := dcd.GetParentGraphDeploymentName(); parentName != "" { + podLabels[commonconsts.KubeLabelDynamoGraphDeploymentName] = parentName + } + if componentType != "" { + podLabels[commonconsts.KubeLabelDynamoComponentType] = componentType + } + if componentName := dynamo.GetDCDComponentName(dcd); componentName != "" { + podLabels[commonconsts.KubeLabelDynamoComponent] = componentName + } + if dynamoNamespace := dynamo.GetDCDDynamoNamespace(dcd); dynamoNamespace != "" { + podLabels[commonconsts.KubeLabelDynamoNamespace] = dynamoNamespace + } + if workerHash := dcd.GetLabels()[commonconsts.KubeLabelDynamoWorkerHash]; workerHash != "" { + podLabels[commonconsts.KubeLabelDynamoWorkerHash] = workerHash + } + + var checkpointInfo *checkpoint.CheckpointInfo + if checkpointConfig := dynamo.GetCheckpoint(component); r.runtimeConfig.Gate.Enabled(features.Checkpoint) && checkpointConfig != nil { + info, err := checkpoint.ResolveCheckpointForService( + ctx, + r.reader, + dcd.Namespace, + dynamo.ToAlphaCheckpointConfig(checkpointConfig), + r.runtimeConfig.Gate, + ) + if err != nil { + return nil, errors.Wrap(err, "failed to resolve checkpoint") + } + if dynamo.IsIntraPodFailoverEnabled(&dcd.Spec.DynamoComponentDeploymentSharedSpec) { + info.RestoreTargetContainers = dynamo.IntraPodFailoverEngineContainerNames() + } + if err := gms.OverlayClients( + &info.GPUMemoryService, + info.CheckpointName, + info.Exists, + dynamo.GetGPUMemoryService(component), + ); err != nil { + return nil, errors.Wrap(err, "failed to apply checkpoint gpuMemoryService config") + } + checkpointInfo = info + } + + podSpec, err := dynamo.GenerateBasePodSpecForController( + dcd, + r.dockerSecretRetriever, + r.config, + role, + commonconsts.MultinodeDeploymentTypeLWS, + checkpointInfo, + dynamo.GenerateBasePodSpecForControllerOptions{ + WorkloadComponentType: nvidiacomv1beta1.ComponentType(componentType), + }, + ) + if err != nil { + return nil, errors.Wrap(err, "failed to generate base pod spec") + } + if r.runtimeConfig.Gate.Enabled(features.Checkpoint) { + if checkpointInfo == nil || + string(checkpointInfo.StartupPolicy) == string(nvidiacomv1beta1.CheckpointStartupPolicyWaitForCheckpoint) { + if err := checkpoint.InjectCheckpointIntoPodSpecWithStorageConfig( + ctx, + r.reader, + dcd.Namespace, + podSpec, + checkpointInfo, + r.config.Checkpoint.Storage, + r.config.Checkpoint.EffectiveSeccompProfile(), + ); err != nil { + return nil, errors.Wrap(err, "failed to inject checkpoint config") + } + } + } + + if len(podSpec.Containers) == 0 { + return nil, errors.New("no containers found in base pod spec") + } + + podLabels[commonconsts.KubeLabelDynamoSelector] = kubeName + + if commonController.IsK8sDiscoveryEnabled(r.config.Discovery.Backend, podAnnotations) { + podLabels[commonconsts.KubeLabelDynamoDiscoveryBackend] = "kubernetes" + podLabels[commonconsts.KubeLabelDynamoDiscoveryEnabled] = commonconsts.KubeLabelValueTrue + } + + if checkpointInfo != nil && + (checkpointInfo.StartupPolicy == "" || + string(checkpointInfo.StartupPolicy) == string(nvidiacomv1beta1.CheckpointStartupPolicyImmediate)) { + if err := checkpoint.ApplyRestoreCandidateMetadata(podLabels, podAnnotations, checkpointInfo); err != nil { + return nil, errors.Wrap(err, "failed to apply checkpoint candidate metadata") + } + } else if err := checkpoint.ApplyRestorePodMetadataWithStorageConfig( + podLabels, + podAnnotations, + checkpointInfo, + r.config.Checkpoint.Storage, + ); err != nil { + return nil, errors.Wrap(err, "failed to apply checkpoint metadata") + } + + if podSpec.ServiceAccountName == "" { + serviceAccounts := &corev1.ServiceAccountList{} + err = r.reader.List(ctx, serviceAccounts, client.InNamespace(dcd.Namespace), client.MatchingLabels{ + commonconsts.KubeLabelDynamoComponentPod: commonconsts.KubeLabelValueTrue, + }) + if err != nil { + return nil, errors.Wrapf(err, "failed to list service accounts in namespace %s", dcd.Namespace) + } + if len(serviceAccounts.Items) > 0 { + podSpec.ServiceAccountName = serviceAccounts.Items[0].Name + } else { + podSpec.ServiceAccountName = DefaultServiceAccountName + } + } + + return &corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: podLabels, + Annotations: podAnnotations, + }, + Spec: *podSpec, + }, nil +} + +func (r *dcdWorkloadRenderer) generateService( + ctx context.Context, + dcd *nvidiacomv1beta1.DynamoComponentDeployment, +) (*corev1.Service, bool, error) { + deleteStub := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: dynamo.NormalizeKubeResourceName(dcd.Name), + Namespace: dcd.Namespace, + }, + } + + annotations := dynamo.GetDCDKubeAnnotations(dcd) + isK8sDiscovery := commonController.IsK8sDiscoveryEnabled(r.config.Discovery.Backend, annotations) + + if !(isK8sDiscovery || dcd.IsFrontendComponent()) { + return deleteStub, true, nil + } + + dynamoNamespace := dynamo.GetDCDDynamoNamespace(dcd) + if dynamoNamespace == "" { + return nil, false, fmt.Errorf("expected DynamoComponentDeployment %s to have a dynamoNamespace", dcd.Name) + } + + componentType, err := r.getDCDWorkloadComponentType(ctx, dcd) + if err != nil { + return nil, false, err + } + + svc, err := dynamo.GenerateComponentService(dynamo.ComponentServiceParams{ + ServiceName: dcd.Name, + Namespace: dcd.Namespace, + ComponentType: componentType, + DynamoNamespace: dynamoNamespace, + ComponentName: dynamo.GetDCDComponentName(dcd), + Labels: dynamo.GetDCDKubeLabels(dcd), + Annotations: annotations, + IsK8sDiscovery: isK8sDiscovery, + }) + if err != nil { + return nil, false, err + } + if dcd.IsMultinode() { + svc.Spec.Selector[dcdWorkloadRoleLabel] = string(dynamo.RoleLeader) + } + return svc, false, nil +} + +func (r *dcdWorkloadRenderer) getDCDWorkloadPodLabels( + ctx context.Context, + dcd *nvidiacomv1beta1.DynamoComponentDeployment, +) (map[string]string, error) { + labels := dynamo.GetDCDKubeLabels(dcd) + componentType, err := r.getDCDWorkloadComponentType(ctx, dcd) + if err != nil { + return nil, err + } + if componentType == "" { + return labels, nil + } + labels[commonconsts.KubeLabelDynamoComponentType] = componentType + specType := string(dcd.Spec.ComponentType) + if componentType == commonconsts.ComponentTypeWorker && + (specType == commonconsts.ComponentTypePrefill || specType == commonconsts.ComponentTypeDecode) && + labels[commonconsts.KubeLabelDynamoSubComponentType] == "" { + labels[commonconsts.KubeLabelDynamoSubComponentType] = specType + } + return labels, nil +} + +// getDCDWorkloadComponentType returns the component type that should be +// rendered into pod metadata, env, and service selectors for this DCD. It keeps +// legacy-compatible worker generations as "worker" even when the v1beta1 DCD +// spec is represented as a more specific prefill/decode worker component. +func (r *dcdWorkloadRenderer) getDCDWorkloadComponentType( + ctx context.Context, + dcd *nvidiacomv1beta1.DynamoComponentDeployment, +) (string, error) { + if dcd == nil { + return "", nil + } + + componentType := dynamo.GetDCDWorkloadComponentType(dcd) + if componentType == commonconsts.ComponentTypeWorker || !dynamo.IsWorkerComponent(componentType) { + return componentType, nil + } + + if hasLegacyWorkerSelector(dcd.GetLabels(), componentType) { + return commonconsts.ComponentTypeWorker, nil + } + + legacy, err := r.hasExistingLegacyWorkerSelector(ctx, dcd, componentType) + if err != nil { + return "", err + } + if legacy { + return commonconsts.ComponentTypeWorker, nil + } + + return componentType, nil +} + +func (r *dcdWorkloadRenderer) hasExistingLegacyWorkerSelector( + ctx context.Context, + dcd *nvidiacomv1beta1.DynamoComponentDeployment, + componentType string, +) (bool, error) { + if dcd == nil || r == nil || r.reader == nil { + return false, nil + } + + deployment := &appsv1.Deployment{} + if err := r.reader.Get(ctx, types.NamespacedName{Name: dcd.Name, Namespace: dcd.Namespace}, deployment); err == nil { + if hasLegacyWorkerSelector(deployment.Spec.Template.Labels, componentType) { + return true, nil + } + } else if !k8serrors.IsNotFound(err) { + return false, fmt.Errorf("failed to get deployment %s/%s: %w", dcd.Namespace, dcd.Name, err) + } + + if r.runtimeConfig.Gate.Enabled(features.LWS) { + lwsName := leaderWorkerSetName(dcd) + leaderWorkerSet := &leaderworkersetv1.LeaderWorkerSet{} + if err := r.reader.Get(ctx, types.NamespacedName{Name: lwsName, Namespace: dcd.Namespace}, leaderWorkerSet); err == nil { + template := leaderWorkerSet.Spec.LeaderWorkerTemplate + if template.LeaderTemplate != nil && hasLegacyWorkerSelector(template.LeaderTemplate.Labels, componentType) { + return true, nil + } + if hasLegacyWorkerSelector(template.WorkerTemplate.Labels, componentType) { + return true, nil + } + } else if !k8serrors.IsNotFound(err) { + return false, fmt.Errorf("failed to get leaderworkerset %s/%s: %w", dcd.Namespace, lwsName, err) + } + } + + serviceName := dynamo.NormalizeKubeResourceName(dcd.Name) + service := &corev1.Service{} + if err := r.reader.Get(ctx, types.NamespacedName{Name: serviceName, Namespace: dcd.Namespace}, service); err == nil { + return hasLegacyWorkerSelector(service.Spec.Selector, componentType), nil + } else if !k8serrors.IsNotFound(err) { + return false, fmt.Errorf("failed to get service %s/%s: %w", dcd.Namespace, serviceName, err) + } + + return false, nil +}