diff --git a/deploy/operator/internal/consts/consts.go b/deploy/operator/internal/consts/consts.go index db134cb8ebda..df3b89871f50 100644 --- a/deploy/operator/internal/consts/consts.go +++ b/deploy/operator/internal/consts/consts.go @@ -237,23 +237,15 @@ const ( ResourceStateUnknown = "unknown" // Worker hash rolling-update annotations are controller-owned annotations on - // DynamoGraphDeployment. They record the active worker generation and must not - // be treated as user-configurable inputs. During a managed rolling update, - // these annotations remain on the previously serving worker generation until - // the new generation is fully ready and old workers have drained. + // DynamoGraphDeployment, not on worker DCDs. During a managed rolling update, + // they remain on the previously serving generation until the new generation + // is fully ready and old workers have drained. // - // The compatibility contract is intentionally additive: existing annotation - // and label keys keep their old meaning. AnnotationCurrentWorkerHash stores - // the v1alpha1-compatible worker hash so a downgrade can still understand the - // active generation. AnnotationCurrentWorkerHashV2 stores the v2 worker hash - // for the same active generation. A worker DCD whose - // KubeLabelDynamoWorkerHash value matches either annotation is current. While - // v1 compatibility is required, generated worker DCDs use the v1 hash as the - // label value. If a worker change is visible only to v2, the controller - // removes the v1 annotation and rolls to a v2-labeled DCD because the v1 hash - // can no longer prove pod-template compatibility. A future v2-only release - // can start using the v2 value with the same label key and keep accepting the - // v1 annotation until the next v2 generation change drains old workers. + // Existing 1.2 DGDs keep both annotations until a worker change completes. + // AnnotationCurrentWorkerHash stores their active v1 hash and + // AnnotationCurrentWorkerHashV2 the v2 hash for the same worker spec. Fresh + // DGDs and completed v2 generations omit AnnotationCurrentWorkerHash and use + // AnnotationCurrentWorkerHashV2 as the active DCD generation hash. // AnnotationCurrentWorkerHash stores the active v1alpha1-compatible worker // generation hash. diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller.go b/deploy/operator/internal/controller/dynamographdeployment_controller.go index 3a28963b8177..0f783616a96c 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller.go @@ -239,7 +239,7 @@ func (r *DynamoGraphDeploymentReconciler) Reconcile(ctx context.Context, req ctr message = Message(err.Error()) return ctrl.Result{}, err } - r.setCurrentWorkerHashes(dynamoDeployment, hashes) + r.setCurrentWorkerHashes(dynamoDeployment, workerHashesForCompletedGeneration(hashes.v2, hashes)) if updateErr := r.Update(ctx, dynamoDeployment); updateErr != nil { logger.Error(updateErr, "Failed to initialize worker hash for unsupported pathway") reason = reasonFailedToInitializeWorkerHash diff --git a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go index 543d3a65a599..2b7eefeddb56 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_controller_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_controller_test.go @@ -2106,11 +2106,11 @@ func TestDynamoGraphDeploymentReconciler_checkpointWorkerHashForComponentUsesAct }, }, }) + reconciler.setCurrentWorkerHashes(dgd, workerGenerationHashes{v1: "oldhash"}) desired, err := reconciler.desiredWorkerHashes(dgd) if err != nil { t.Fatalf("desiredWorkerHashes() error = %v", err) } - reconciler.setCurrentWorkerHashes(dgd, workerGenerationHashes{v1: "oldhash"}) workerHash, err := reconciler.checkpointWorkerHashForComponent(dgd, "worker") if err != nil { @@ -3995,6 +3995,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { tests := []struct { name string dgdSpec v1alpha1.DynamoGraphDeploymentSpec + dgdAnnotations map[string]string existingDCDs []client.Object wantReconcileResult ReconcileResult }{ @@ -4205,6 +4206,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }, }, }, + dgdAnnotations: map[string]string{commonconsts.AnnotationCurrentWorkerHashV2: "1b69c0d3"}, existingDCDs: []client.Object{ betaDCD(t, &v1alpha1.DynamoComponentDeployment{ ObjectMeta: metav1.ObjectMeta{ @@ -4237,7 +4239,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }), betaDCD(t, &v1alpha1.DynamoComponentDeployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-dgd-decode-e1f2a6fe", + Name: "test-dgd-decode-1b69c0d3", Namespace: "default", }, Spec: v1alpha1.DynamoComponentDeploymentSpec{ @@ -4245,6 +4247,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{ ServiceName: "decode", Replicas: ptr.To(int32(2)), + Labels: map[string]string{commonconsts.KubeLabelDynamoWorkerHash: "1b69c0d3"}, }, }, Status: v1alpha1.DynamoComponentDeploymentStatus{ @@ -4256,7 +4259,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }, Service: &v1alpha1.ServiceReplicaStatus{ ComponentKind: v1alpha1.ComponentKindDeployment, - ComponentNames: []string{"test-dgd-decode-e1f2a6fe-deployment"}, + ComponentNames: []string{"test-dgd-decode-1b69c0d3-deployment"}, Replicas: 2, UpdatedReplicas: 2, ReadyReplicas: ptr.To(int32(2)), @@ -4266,7 +4269,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }), betaDCD(t, &v1alpha1.DynamoComponentDeployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-dgd-prefill-e1f2a6fe", + Name: "test-dgd-prefill-1b69c0d3", Namespace: "default", }, Spec: v1alpha1.DynamoComponentDeploymentSpec{ @@ -4274,6 +4277,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{ ServiceName: "prefill", Replicas: ptr.To(int32(3)), + Labels: map[string]string{commonconsts.KubeLabelDynamoWorkerHash: "1b69c0d3"}, }, }, Status: v1alpha1.DynamoComponentDeploymentStatus{ @@ -4285,7 +4289,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }, Service: &v1alpha1.ServiceReplicaStatus{ ComponentKind: v1alpha1.ComponentKindDeployment, - ComponentNames: []string{"test-dgd-prefill-e1f2a6fe-deployment"}, + ComponentNames: []string{"test-dgd-prefill-1b69c0d3-deployment"}, Replicas: 3, UpdatedReplicas: 3, ReadyReplicas: ptr.To(int32(3)), @@ -4309,7 +4313,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }, "decode": { ComponentKind: v1beta1.ComponentKindDeployment, - ComponentNames: []string{"test-dgd-decode-e1f2a6fe-deployment"}, + ComponentNames: []string{"test-dgd-decode-1b69c0d3-deployment"}, Replicas: 2, UpdatedReplicas: 2, ReadyReplicas: ptr.To(int32(2)), @@ -4317,7 +4321,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }, "prefill": { ComponentKind: v1beta1.ComponentKindDeployment, - ComponentNames: []string{"test-dgd-prefill-e1f2a6fe-deployment"}, + ComponentNames: []string{"test-dgd-prefill-1b69c0d3-deployment"}, Replicas: 3, UpdatedReplicas: 3, ReadyReplicas: ptr.To(int32(3)), @@ -4351,6 +4355,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }, }, }, + dgdAnnotations: map[string]string{commonconsts.AnnotationCurrentWorkerHashV2: "1b69c0d3"}, existingDCDs: []client.Object{ betaDCD(t, &v1alpha1.DynamoComponentDeployment{ ObjectMeta: metav1.ObjectMeta{ @@ -4383,7 +4388,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }), betaDCD(t, &v1alpha1.DynamoComponentDeployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-dgd-decode-e1f2a6fe", + Name: "test-dgd-decode-1b69c0d3", Namespace: "default", }, Spec: v1alpha1.DynamoComponentDeploymentSpec{ @@ -4391,6 +4396,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{ ServiceName: "decode", Replicas: ptr.To(int32(2)), + Labels: map[string]string{commonconsts.KubeLabelDynamoWorkerHash: "1b69c0d3"}, }, }, Status: v1alpha1.DynamoComponentDeploymentStatus{ @@ -4402,7 +4408,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }, Service: &v1alpha1.ServiceReplicaStatus{ ComponentKind: v1alpha1.ComponentKindDeployment, - ComponentNames: []string{"test-dgd-decode-e1f2a6fe-deployment"}, + ComponentNames: []string{"test-dgd-decode-1b69c0d3-deployment"}, Replicas: 2, UpdatedReplicas: 1, ReadyReplicas: ptr.To(int32(1)), @@ -4412,7 +4418,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }), betaDCD(t, &v1alpha1.DynamoComponentDeployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-dgd-prefill-e1f2a6fe", + Name: "test-dgd-prefill-1b69c0d3", Namespace: "default", }, Spec: v1alpha1.DynamoComponentDeploymentSpec{ @@ -4420,6 +4426,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{ ServiceName: "prefill", Replicas: ptr.To(int32(3)), + Labels: map[string]string{commonconsts.KubeLabelDynamoWorkerHash: "1b69c0d3"}, }, }, Status: v1alpha1.DynamoComponentDeploymentStatus{ @@ -4431,7 +4438,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }, Service: &v1alpha1.ServiceReplicaStatus{ ComponentKind: v1alpha1.ComponentKindDeployment, - ComponentNames: []string{"test-dgd-prefill-e1f2a6fe-deployment"}, + ComponentNames: []string{"test-dgd-prefill-1b69c0d3-deployment"}, Replicas: 3, UpdatedReplicas: 3, ReadyReplicas: ptr.To(int32(3)), @@ -4443,7 +4450,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { wantReconcileResult: ReconcileResult{ State: v1beta1.DGDStatePending, Reason: "some_resources_are_not_ready", - Message: "Resources not ready: test-dgd-decode-e1f2a6fe: Component deployment not ready - Available condition not true", + Message: "Resources not ready: test-dgd-decode-1b69c0d3: Component deployment not ready - Available condition not true", ComponentStatus: map[string]v1beta1.ComponentReplicaStatus{ "frontend": { ComponentKind: v1beta1.ComponentKindDeployment, @@ -4455,7 +4462,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }, "decode": { ComponentKind: v1beta1.ComponentKindDeployment, - ComponentNames: []string{"test-dgd-decode-e1f2a6fe-deployment"}, + ComponentNames: []string{"test-dgd-decode-1b69c0d3-deployment"}, Replicas: 2, UpdatedReplicas: 1, ReadyReplicas: ptr.To(int32(1)), @@ -4463,7 +4470,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }, "prefill": { ComponentKind: v1beta1.ComponentKindDeployment, - ComponentNames: []string{"test-dgd-prefill-e1f2a6fe-deployment"}, + ComponentNames: []string{"test-dgd-prefill-1b69c0d3-deployment"}, Replicas: 3, UpdatedReplicas: 3, ReadyReplicas: ptr.To(int32(3)), @@ -4491,6 +4498,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }, }, }, + dgdAnnotations: map[string]string{commonconsts.AnnotationCurrentWorkerHashV2: "cabcd5c9"}, existingDCDs: []client.Object{ betaDCD(t, &v1alpha1.DynamoComponentDeployment{ ObjectMeta: metav1.ObjectMeta{ @@ -4523,7 +4531,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }), betaDCD(t, &v1alpha1.DynamoComponentDeployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-dgd-decode-5f3d46ba", + Name: "test-dgd-decode-cabcd5c9", Namespace: "default", }, Spec: v1alpha1.DynamoComponentDeploymentSpec{ @@ -4531,6 +4539,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{ ServiceName: "decode", Replicas: ptr.To(int32(2)), + Labels: map[string]string{commonconsts.KubeLabelDynamoWorkerHash: "cabcd5c9"}, }, }, Status: v1alpha1.DynamoComponentDeploymentStatus{ @@ -4542,7 +4551,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }, Service: &v1alpha1.ServiceReplicaStatus{ ComponentKind: v1alpha1.ComponentKindDeployment, - ComponentNames: []string{"test-dgd-decode-5f3d46ba-deployment"}, + ComponentNames: []string{"test-dgd-decode-cabcd5c9-deployment"}, Replicas: 2, UpdatedReplicas: 1, ReadyReplicas: ptr.To(int32(1)), @@ -4554,7 +4563,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { wantReconcileResult: ReconcileResult{ State: v1beta1.DGDStatePending, Reason: "some_resources_are_not_ready", - Message: "Resources not ready: test-dgd-decode-5f3d46ba: Component deployment not ready - Available condition not true; test-dgd-frontend: Component deployment not ready - Available condition not true", + Message: "Resources not ready: test-dgd-decode-cabcd5c9: Component deployment not ready - Available condition not true; test-dgd-frontend: Component deployment not ready - Available condition not true", ComponentStatus: map[string]v1beta1.ComponentReplicaStatus{ "frontend": { ComponentKind: v1beta1.ComponentKindDeployment, @@ -4566,7 +4575,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { }, "decode": { ComponentKind: v1beta1.ComponentKindDeployment, - ComponentNames: []string{"test-dgd-decode-5f3d46ba-deployment"}, + ComponentNames: []string{"test-dgd-decode-cabcd5c9-deployment"}, Replicas: 2, UpdatedReplicas: 1, ReadyReplicas: ptr.To(int32(1)), @@ -4587,8 +4596,9 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) { dgd := betaDGD(t, &v1alpha1.DynamoGraphDeployment{ ObjectMeta: metav1.ObjectMeta{ - Name: "test-dgd", - Namespace: "default", + Name: "test-dgd", + Namespace: "default", + Annotations: tt.dgdAnnotations, }, Spec: tt.dgdSpec, }) diff --git a/deploy/operator/internal/controller/dynamographdeployment_rollingupdate.go b/deploy/operator/internal/controller/dynamographdeployment_rollingupdate.go index 400b6c9c321a..b455638b9f8a 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_rollingupdate.go +++ b/deploy/operator/internal/controller/dynamographdeployment_rollingupdate.go @@ -55,16 +55,27 @@ func (h workerGenerationHashes) contains(hash string) bool { func (r *DynamoGraphDeploymentReconciler) desiredWorkerHashes( dgd *nvidiacomv1beta1.DynamoGraphDeployment, ) (workerGenerationHashes, error) { - v1Hash, err := dynamo.ComputeLegacyAlphaDGDWorkersSpecHash(dgd) - if err != nil { - return workerGenerationHashes{}, fmt.Errorf("failed to compute v1 worker hash: %w", err) - } - v2Hash, err := dynamo.ComputeDGDWorkersSpecHash(dgd) if err != nil { return workerGenerationHashes{}, fmt.Errorf("failed to compute v2 worker hash: %w", err) } + current := r.currentWorkerHashes(dgd) + v1Hash := v2Hash + if current.v2 != "" { + v1Hash = current.v1 + } + if current.v2 == "" && current.v1 != "" && current.v1 != v2Hash { + // Keep v1 only when its value proves a pre-dual generation or is the explicit sentinel. + legacyHash, err := dynamo.ComputeLegacyAlphaDGDWorkersSpecHash(dgd) + if err != nil { + return workerGenerationHashes{}, fmt.Errorf("failed to compute v1 worker hash: %w", err) + } + if current.v1 == consts.LegacyWorkerHash || current.v1 == legacyHash { + v1Hash = legacyHash + } + } + return workerGenerationHashes{v1: v1Hash, v2: v2Hash}, nil } @@ -107,7 +118,7 @@ func workerHashForDCDGeneration(current, desired workerGenerationHashes) string } func workerHashesForCompletedGeneration(newWorkerHash string, desired workerGenerationHashes) workerGenerationHashes { - if newWorkerHash == desired.v2 && desired.v2 != desired.v1 { + if newWorkerHash == desired.v2 { return workerGenerationHashes{v2: desired.v2} } return desired @@ -141,7 +152,7 @@ func (r *DynamoGraphDeploymentReconciler) shouldTriggerRollingUpdate( } // initializeWorkerHashIfNeeded establishes the DGD's active worker generation. -// New DGDs store the current v1 and v2 worker hashes immediately. DGDs created before +// New DGDs store only the canonical v2 worker hash. DGDs created before // managed rolling updates may already have worker DCDs without a hash label; in // that case we label those DCDs with the legacy sentinel and let the normal // rolling update path migrate from that sentinel to the desired compatibility hash. @@ -191,12 +202,12 @@ func (r *DynamoGraphDeploymentReconciler) initializeWorkerHashIfNeeded( return nil } - // Normal first deploy — set the actual computed compatibility hashes + // Normal first deploy — set the canonical v2 hash. hashes, err := r.desiredWorkerHashes(dgd) if err != nil { return err } - r.setCurrentWorkerHashes(dgd, hashes) + r.setCurrentWorkerHashes(dgd, workerHashesForCompletedGeneration(hashes.v2, hashes)) if err := r.Update(ctx, dgd); err != nil { return fmt.Errorf("failed to initialize worker hash: %w", err) @@ -208,9 +219,7 @@ func (r *DynamoGraphDeploymentReconciler) initializeWorkerHashIfNeeded( } // migrateCurrentWorkerHashIfNeeded fills in additive v2 worker-hash state while -// the v1 hash still represents the active worker generation. If v2 changes -// without a v1 change, v1 compatibility no longer proves current pod contents, -// so the v1 annotation is removed before rolling to a v2-labeled DCD. +// the v1 hash still represents the active worker generation. func (r *DynamoGraphDeploymentReconciler) migrateCurrentWorkerHashIfNeeded( ctx context.Context, dgd *nvidiacomv1beta1.DynamoGraphDeployment, @@ -233,13 +242,10 @@ func (r *DynamoGraphDeploymentReconciler) migrateCurrentWorkerHashIfNeeded( var next workerGenerationHashes var eventMessage string switch { - case current.v1 == desired.v1 && current.v2 == "": + case current.v1 == desired.v1 && current.v2 == "" && current.v1 != desired.v2: next = current next.v2 = desired.v2 eventMessage = "Recorded compatible v1 and v2 worker hash annotations without rolling workers" - case current.v1 == desired.v1 && current.v2 != desired.v2: - next = workerGenerationHashes{v2: current.v2} - eventMessage = "Removed v1 worker hash annotation before rolling a v2-only worker change" default: return nil } @@ -260,10 +266,9 @@ func (r *DynamoGraphDeploymentReconciler) migrateCurrentWorkerHashIfNeeded( } // activeWorkerHashForDCDGeneration returns the hash used for generated worker -// DCD names and worker-hash labels in this reconcile. While v1 compatibility is -// required, new DCDs continue to use the v1 hash. If the active generation is -// already v2-labeled, preserve that value so future v2-only transitions do not -// create a v1-labeled replacement. +// DCD names and worker-hash labels in this reconcile. Existing bridge generations +// keep their v1 identity until a worker change selects v2. Already v2-labeled +// generations preserve that value. func (r *DynamoGraphDeploymentReconciler) activeWorkerHashForDCDGeneration( dgd *nvidiacomv1beta1.DynamoGraphDeployment, desired workerGenerationHashes, @@ -330,10 +335,8 @@ func (r *DynamoGraphDeploymentReconciler) supportsManagedRollingUpdate( return !r.isGrovePathway(dgd) && !dgd.HasAnyMultinodeComponent() } -// getCurrentWorkerHash returns the active worker generation stored on the DGD. -// During a rolling update this is the previous serving hash; it is not advanced -// to the desired hash until the new generation is ready and the old generation -// has drained. Empty means this DGD has not initialized rolling-update state. +// getCurrentWorkerHash returns the v1 worker generation stored on the DGD. +// It is empty after the DGD has converged to a v2-only generation. func (r *DynamoGraphDeploymentReconciler) getCurrentWorkerHash( dgd *nvidiacomv1beta1.DynamoGraphDeployment, ) string { diff --git a/deploy/operator/internal/controller/dynamographdeployment_rollingupdate_test.go b/deploy/operator/internal/controller/dynamographdeployment_rollingupdate_test.go index 705629220f29..9f1a5635c8b4 100644 --- a/deploy/operator/internal/controller/dynamographdeployment_rollingupdate_test.go +++ b/deploy/operator/internal/controller/dynamographdeployment_rollingupdate_test.go @@ -234,13 +234,16 @@ func TestShouldTriggerRollingUpdate_IgnoresReplicaChanges(t *testing.T) { assert.False(t, trigger) } -func TestInitializeWorkerHashIfNeeded_FirstDeploy(t *testing.T) { +func TestCanonicalWorkerHashLifecycle_FirstDeploySpecChangeAndCompletion(t *testing.T) { dgd := createTestDGD("test-dgd", map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ "worker": { ComponentType: consts.ComponentTypeWorker, Envs: []corev1.EnvVar{ {Name: "FOO", Value: "bar"}, }, + Resources: &nvidiacomv1alpha1.Resources{ + Requests: &nvidiacomv1alpha1.ResourceItem{CPU: "1"}, + }, }, }) @@ -252,16 +255,38 @@ func TestInitializeWorkerHashIfNeeded_FirstDeploy(t *testing.T) { err := r.initializeWorkerHashIfNeeded(ctx, dgd) require.NoError(t, err) - // Verify the hash was set - hash := r.getCurrentWorkerHash(dgd) + // Verify only the v2 hash was set. + hash := r.getCurrentWorkerHashV2(dgd) assert.NotEmpty(t, hash, "Hash should be set after initialization") + assert.NotContains(t, dgd.Annotations, consts.AnnotationCurrentWorkerHash) - // Verify both compatibility hashes are correct. - expectedV1Hash, err := dynamo.ComputeLegacyAlphaDGDWorkersSpecHash(dgd) - require.NoError(t, err) + // Fresh deployments store one canonical v2 hash. expectedV2Hash := betaDGDWorkersSpecHash(t, dgd) - assert.Equal(t, expectedV1Hash, hash, "v1 hash should remain the downgrade-compatible current hash") - assert.Equal(t, expectedV2Hash, dgd.Annotations[consts.AnnotationCurrentWorkerHashV2]) + assert.Equal(t, expectedV2Hash, hash) + + // A later worker change rolls directly from one canonical v2 generation to the next. + dgd.Spec.Components[0].PodTemplate.Spec.Containers[0].Env = append( + dgd.Spec.Components[0].PodTemplate.Spec.Containers[0].Env, + corev1.EnvVar{Name: "NEW_WORKER_SETTING", Value: "true"}, + ) + newV2Hash := betaDGDWorkersSpecHash(t, dgd) + newLegacyHash, err := dynamo.ComputeLegacyAlphaDGDWorkersSpecHash(dgd) + require.NoError(t, err) + require.NotEqual(t, expectedV2Hash, newV2Hash) + require.NotEqual(t, newLegacyHash, newV2Hash) + + trigger, err := r.shouldTriggerRollingUpdate(dgd) + require.NoError(t, err) + require.True(t, trigger) + + rollingCtx, err := r.buildRollingUpdateContext(ctx, dgd) + require.NoError(t, err) + require.Equal(t, newV2Hash, rollingCtx.NewWorkerHash) + require.NotEqual(t, newLegacyHash, rollingCtx.NewWorkerHash) + + require.NoError(t, r.completeRollingUpdate(ctx, dgd, newV2Hash)) + require.NotContains(t, dgd.Annotations, consts.AnnotationCurrentWorkerHash) + require.Equal(t, newV2Hash, dgd.Annotations[consts.AnnotationCurrentWorkerHashV2]) } func TestInitializeWorkerHashIfNeeded_AlreadyInitialized(t *testing.T) { @@ -392,7 +417,7 @@ func TestLegacyAlphaHashCompatibility_NoOpUpgradeUsesExistingWorkerGeneration(t require.NotEqual(t, "qwen-vllmdecodeworker-"+v2Hash, dcds["VllmDecodeWorker"].Name) } -func TestLegacyAlphaHashCompatibility_WorkerSpecChangeUsesNewV1Generation(t *testing.T) { +func TestLegacyAlphaHashCompatibility_WorkerSpecChangeUsesNewV2Generation(t *testing.T) { dgd := createTestDGD("test-dgd", map[string]*nvidiacomv1alpha1.DynamoComponentDeploymentSharedSpec{ "worker": { ComponentType: consts.ComponentTypeWorker, @@ -434,8 +459,8 @@ func TestLegacyAlphaHashCompatibility_WorkerSpecChangeUsesNewV1Generation(t *tes rollingCtx, err := r.buildRollingUpdateContext(context.Background(), dgd) require.NoError(t, err) - require.Equal(t, newLegacyHash, rollingCtx.NewWorkerHash) - require.NotEqual(t, newV2Hash, rollingCtx.NewWorkerHash) + require.Equal(t, newV2Hash, rollingCtx.NewWorkerHash) + require.NotEqual(t, newLegacyHash, rollingCtx.NewWorkerHash) } func TestLegacyAlphaHashCompatibility_V2OnlyChangeUsesNewV2Generation(t *testing.T) { @@ -464,7 +489,7 @@ func TestLegacyAlphaHashCompatibility_V2OnlyChangeUsesNewV2Generation(t *testing require.NotEqual(t, v2Hash, newV2Hash) require.NoError(t, r.migrateCurrentWorkerHashIfNeeded(context.Background(), dgd)) - require.Empty(t, dgd.Annotations[consts.AnnotationCurrentWorkerHash]) + require.Equal(t, legacyHash, dgd.Annotations[consts.AnnotationCurrentWorkerHash]) require.Equal(t, v2Hash, dgd.Annotations[consts.AnnotationCurrentWorkerHashV2]) trigger, err := r.shouldTriggerRollingUpdate(dgd) @@ -517,7 +542,7 @@ func TestUnsupportedPathwayMigratesV1OnlyAndKeepsV2OnlyGeneration(t *testing.T) require.NotEqual(t, v2Hash, newV2Hash) require.NoError(t, r.migrateCurrentWorkerHashIfNeeded(context.Background(), dgd)) - require.Empty(t, dgd.Annotations[consts.AnnotationCurrentWorkerHash]) + require.Equal(t, legacyHash, dgd.Annotations[consts.AnnotationCurrentWorkerHash]) require.Equal(t, v2Hash, dgd.Annotations[consts.AnnotationCurrentWorkerHashV2]) desired, err := r.desiredWorkerHashes(dgd) @@ -1500,8 +1525,7 @@ func TestGetExistingRestartAnnotationsDCD(t *testing.T) { ComponentType: consts.ComponentTypeWorker, }, }) - // Annotation hash can differ from computed hash — function uses active compatibility hash. - computedHash := legacyDGDWorkersSpecHash(t, dgd) + computedHash := betaDGDWorkersSpecHash(t, dgd) dgd.Annotations = map[string]string{ consts.AnnotationCurrentWorkerHash: "oldhash", } @@ -1836,6 +1860,14 @@ func TestInitializeWorkerHashIfNeeded_LegacyDCDsMigration(t *testing.T) { require.NoError(t, err) assert.Equal(t, consts.LegacyWorkerHash, updatedDCD.Labels[consts.KubeLabelDynamoWorkerHash], "Legacy DCD should have worker hash label backfilled") + + desired, err := r.desiredWorkerHashes(dgd) + require.NoError(t, err) + require.NoError(t, r.completeRollingUpdate(ctx, dgd, desired.v1)) + + trigger, err := r.shouldTriggerRollingUpdate(dgd) + require.NoError(t, err) + require.False(t, trigger) } func TestInitializeWorkerHashIfNeeded_LegacyMultipleWorkers(t *testing.T) { @@ -3226,7 +3258,7 @@ func TestReconcileRollingUpdate_StaleAnnotationRequiresAllNewWorkersReady(t *tes dgd.Status.RollingUpdate = &nvidiacomv1beta1.RollingUpdateStatus{ Phase: nvidiacomv1beta1.RollingUpdatePhaseCompleted, } - newHash := legacyDGDWorkersSpecHash(t, dgd) + newHash := betaDGDWorkersSpecHash(t, dgd) require.NotEqual(t, testOldWorkerHash, newHash) newPrefillDCD := betaDCD(t, &nvidiacomv1alpha1.DynamoComponentDeployment{ @@ -3268,7 +3300,7 @@ func TestReconcileRollingUpdate_StaleAnnotationUpdatesAfterAllNewWorkersReady(t dgd.Status.RollingUpdate = &nvidiacomv1beta1.RollingUpdateStatus{ Phase: nvidiacomv1beta1.RollingUpdatePhaseCompleted, } - newHash := legacyDGDWorkersSpecHash(t, dgd) + newHash := betaDGDWorkersSpecHash(t, dgd) require.NotEqual(t, testOldWorkerHash, newHash) makeReadyDCD := func(componentName, componentType string) *nvidiacomv1beta1.DynamoComponentDeployment { @@ -3305,7 +3337,8 @@ func TestReconcileRollingUpdate_StaleAnnotationUpdatesAfterAllNewWorkersReady(t err := r.reconcileRollingUpdate(context.Background(), dgd) require.NoError(t, err) - assert.Equal(t, newHash, dgd.Annotations[consts.AnnotationCurrentWorkerHash]) + assert.NotContains(t, dgd.Annotations, consts.AnnotationCurrentWorkerHash) + assert.Equal(t, newHash, dgd.Annotations[consts.AnnotationCurrentWorkerHashV2]) assert.Equal(t, nvidiacomv1beta1.RollingUpdatePhaseCompleted, dgd.Status.RollingUpdate.Phase) } @@ -3653,7 +3686,7 @@ func TestBuildRollingUpdateContext(t *testing.T) { } // Compute the actual new DCD label hash from the DGD spec. - newHash := legacyDGDWorkersSpecHash(t, dgd) + newHash := betaDGDWorkersSpecHash(t, dgd) require.NotEqual(t, testOldWorkerHash, newHash, "test setup: computed hash must differ from old hash") // Collect all mock objects @@ -3719,7 +3752,7 @@ func TestBuildRollingUpdateContext_NoNewDCDExists(t *testing.T) { }, }) - newHash := legacyDGDWorkersSpecHash(t, dgd) + newHash := betaDGDWorkersSpecHash(t, dgd) assert.NotEqual(t, testOldWorkerHash, newHash, "test setup: computed hash must differ from old hash") r := createTestReconcilerWithStatus(dgd, withObjects(oldDCD)) @@ -3745,7 +3778,7 @@ func TestBuildRollingUpdateContext_ListOldDCDsError(t *testing.T) { consts.AnnotationCurrentWorkerHash: testOldWorkerHash, } - assert.NotEqual(t, testOldWorkerHash, legacyDGDWorkersSpecHash(t, dgd), + assert.NotEqual(t, testOldWorkerHash, betaDGDWorkersSpecHash(t, dgd), "test setup: computed hash must differ so we proceed past the early-return") injectedErr := errors.New("simulated apiserver list failure")