Skip to content

refactor(operator): lazily converge worker hashes to v2 #11529

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
sttts merged 10 commits into from
Jul 13, 2026
Merged

refactor(operator): lazily converge worker hashes to v2 #11529

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
d8ff9e8
operator/worker-hash: converge generations lazily to v2
sttts Jul 10, 2026
ac994a3
fixup! operator/worker-hash: converge generations lazily to v2
sttts Jul 10, 2026
4600c1d
fixup! operator/worker-hash: converge generations lazily to v2
sttts Jul 10, 2026
add4bf1
fixup! operator/worker-hash: converge generations lazily to v2
sttts Jul 10, 2026
1fa4bb8
fixup! operator/worker-hash: converge generations lazily to v2
sttts Jul 10, 2026
564845b
fixup! operator/worker-hash: converge generations lazily to v2
sttts Jul 10, 2026
7a4222f
fixup! operator/worker-hash: converge generations lazily to v2
sttts Jul 10, 2026
4b3916a
fixup! operator/worker-hash: converge generations lazily to v2
sttts Jul 10, 2026
34281ac
fixup! operator/worker-hash: converge generations lazily to v2
sttts Jul 10, 2026
678ff5c
fixup! operator/worker-hash: converge generations lazily to v2
sttts Jul 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 8 additions & 16 deletions deploy/operator/internal/consts/consts.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -237,23 +237,15 @@ const (
ResourceStateUnknown = "unknown"

// Worker hash rolling-update annotations are controller-owned annotations on
// DynamoGraphDeployment. They record the active worker generation and must not
// be treated as user-configurable inputs. During a managed rolling update,
// these annotations remain on the previously serving worker generation until
// the new generation is fully ready and old workers have drained.
// DynamoGraphDeployment, not on worker DCDs. During a managed rolling update,
// they remain on the previously serving generation until the new generation
// is fully ready and old workers have drained.
//
// The compatibility contract is intentionally additive: existing annotation
// and label keys keep their old meaning. AnnotationCurrentWorkerHash stores
// the v1alpha1-compatible worker hash so a downgrade can still understand the
// active generation. AnnotationCurrentWorkerHashV2 stores the v2 worker hash
// for the same active generation. A worker DCD whose
// KubeLabelDynamoWorkerHash value matches either annotation is current. While
// v1 compatibility is required, generated worker DCDs use the v1 hash as the
// label value. If a worker change is visible only to v2, the controller
// removes the v1 annotation and rolls to a v2-labeled DCD because the v1 hash
// can no longer prove pod-template compatibility. A future v2-only release
// can start using the v2 value with the same label key and keep accepting the
// v1 annotation until the next v2 generation change drains old workers.
// Existing 1.2 DGDs keep both annotations until a worker change completes.
// AnnotationCurrentWorkerHash stores their active v1 hash and
// AnnotationCurrentWorkerHashV2 the v2 hash for the same worker spec. Fresh
// DGDs and completed v2 generations omit AnnotationCurrentWorkerHash and use
// AnnotationCurrentWorkerHashV2 as the active DCD generation hash.

// AnnotationCurrentWorkerHash stores the active v1alpha1-compatible worker
// generation hash.
Expand Down
2 changes: 1 addition & 1 deletion deploy/operator/internal/controller/dynamographdeployment_controller.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -239,7 +239,7 @@ func (r *DynamoGraphDeploymentReconciler) Reconcile(ctx context.Context, req ctr
message = Message(err.Error())
return ctrl.Result{}, err
}
r.setCurrentWorkerHashes(dynamoDeployment, hashes)
r.setCurrentWorkerHashes(dynamoDeployment, workerHashesForCompletedGeneration(hashes.v2, hashes))
if updateErr := r.Update(ctx, dynamoDeployment); updateErr != nil {
logger.Error(updateErr, "Failed to initialize worker hash for unsupported pathway")
reason = reasonFailedToInitializeWorkerHash
Expand Down
50 changes: 30 additions & 20 deletions deploy/operator/internal/controller/dynamographdeployment_controller_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2106,11 +2106,11 @@ func TestDynamoGraphDeploymentReconciler_checkpointWorkerHashForComponentUsesAct
},
},
})
reconciler.setCurrentWorkerHashes(dgd, workerGenerationHashes{v1: "oldhash"})
desired, err := reconciler.desiredWorkerHashes(dgd)
if err != nil {
t.Fatalf("desiredWorkerHashes() error = %v", err)
}
reconciler.setCurrentWorkerHashes(dgd, workerGenerationHashes{v1: "oldhash"})

workerHash, err := reconciler.checkpointWorkerHashForComponent(dgd, "worker")
if err != nil {
Expand Down Expand Up @@ -3995,6 +3995,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
tests := []struct {
name string
dgdSpec v1alpha1.DynamoGraphDeploymentSpec
dgdAnnotations map[string]string
existingDCDs []client.Object
wantReconcileResult ReconcileResult
}{
Expand Down Expand Up @@ -4205,6 +4206,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
},
},
},
dgdAnnotations: map[string]string{commonconsts.AnnotationCurrentWorkerHashV2: "1b69c0d3"},
existingDCDs: []client.Object{
betaDCD(t, &v1alpha1.DynamoComponentDeployment{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -4237,14 +4239,15 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
}),
betaDCD(t, &v1alpha1.DynamoComponentDeployment{
ObjectMeta: metav1.ObjectMeta{
Name: "test-dgd-decode-e1f2a6fe",
Name: "test-dgd-decode-1b69c0d3",
Comment thread
sttts marked this conversation as resolved.
Namespace: "default",
},
Spec: v1alpha1.DynamoComponentDeploymentSpec{
BackendFramework: "vllm",
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
ServiceName: "decode",
Replicas: ptr.To(int32(2)),
Labels: map[string]string{commonconsts.KubeLabelDynamoWorkerHash: "1b69c0d3"},
},
},
Status: v1alpha1.DynamoComponentDeploymentStatus{
Expand All @@ -4256,7 +4259,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
},
Service: &v1alpha1.ServiceReplicaStatus{
ComponentKind: v1alpha1.ComponentKindDeployment,
ComponentNames: []string{"test-dgd-decode-e1f2a6fe-deployment"},
ComponentNames: []string{"test-dgd-decode-1b69c0d3-deployment"},
Replicas: 2,
UpdatedReplicas: 2,
ReadyReplicas: ptr.To(int32(2)),
Expand All @@ -4266,14 +4269,15 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
}),
betaDCD(t, &v1alpha1.DynamoComponentDeployment{
ObjectMeta: metav1.ObjectMeta{
Name: "test-dgd-prefill-e1f2a6fe",
Name: "test-dgd-prefill-1b69c0d3",
Namespace: "default",
},
Spec: v1alpha1.DynamoComponentDeploymentSpec{
BackendFramework: "vllm",
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
ServiceName: "prefill",
Replicas: ptr.To(int32(3)),
Labels: map[string]string{commonconsts.KubeLabelDynamoWorkerHash: "1b69c0d3"},
},
},
Status: v1alpha1.DynamoComponentDeploymentStatus{
Expand All @@ -4285,7 +4289,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
},
Service: &v1alpha1.ServiceReplicaStatus{
ComponentKind: v1alpha1.ComponentKindDeployment,
ComponentNames: []string{"test-dgd-prefill-e1f2a6fe-deployment"},
ComponentNames: []string{"test-dgd-prefill-1b69c0d3-deployment"},
Replicas: 3,
UpdatedReplicas: 3,
ReadyReplicas: ptr.To(int32(3)),
Expand All @@ -4309,15 +4313,15 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
},
"decode": {
ComponentKind: v1beta1.ComponentKindDeployment,
ComponentNames: []string{"test-dgd-decode-e1f2a6fe-deployment"},
ComponentNames: []string{"test-dgd-decode-1b69c0d3-deployment"},
Replicas: 2,
UpdatedReplicas: 2,
ReadyReplicas: ptr.To(int32(2)),
AvailableReplicas: ptr.To(int32(2)),
},
"prefill": {
ComponentKind: v1beta1.ComponentKindDeployment,
ComponentNames: []string{"test-dgd-prefill-e1f2a6fe-deployment"},
ComponentNames: []string{"test-dgd-prefill-1b69c0d3-deployment"},
Replicas: 3,
UpdatedReplicas: 3,
ReadyReplicas: ptr.To(int32(3)),
Expand Down Expand Up @@ -4351,6 +4355,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
},
},
},
dgdAnnotations: map[string]string{commonconsts.AnnotationCurrentWorkerHashV2: "1b69c0d3"},
existingDCDs: []client.Object{
betaDCD(t, &v1alpha1.DynamoComponentDeployment{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -4383,14 +4388,15 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
}),
betaDCD(t, &v1alpha1.DynamoComponentDeployment{
ObjectMeta: metav1.ObjectMeta{
Name: "test-dgd-decode-e1f2a6fe",
Name: "test-dgd-decode-1b69c0d3",
Namespace: "default",
},
Spec: v1alpha1.DynamoComponentDeploymentSpec{
BackendFramework: "vllm",
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
ServiceName: "decode",
Replicas: ptr.To(int32(2)),
Labels: map[string]string{commonconsts.KubeLabelDynamoWorkerHash: "1b69c0d3"},
},
},
Status: v1alpha1.DynamoComponentDeploymentStatus{
Expand All @@ -4402,7 +4408,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
},
Service: &v1alpha1.ServiceReplicaStatus{
ComponentKind: v1alpha1.ComponentKindDeployment,
ComponentNames: []string{"test-dgd-decode-e1f2a6fe-deployment"},
ComponentNames: []string{"test-dgd-decode-1b69c0d3-deployment"},
Replicas: 2,
UpdatedReplicas: 1,
ReadyReplicas: ptr.To(int32(1)),
Expand All @@ -4412,14 +4418,15 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
}),
betaDCD(t, &v1alpha1.DynamoComponentDeployment{
ObjectMeta: metav1.ObjectMeta{
Name: "test-dgd-prefill-e1f2a6fe",
Name: "test-dgd-prefill-1b69c0d3",
Namespace: "default",
},
Spec: v1alpha1.DynamoComponentDeploymentSpec{
BackendFramework: "vllm",
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
ServiceName: "prefill",
Replicas: ptr.To(int32(3)),
Labels: map[string]string{commonconsts.KubeLabelDynamoWorkerHash: "1b69c0d3"},
},
},
Status: v1alpha1.DynamoComponentDeploymentStatus{
Expand All @@ -4431,7 +4438,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
},
Service: &v1alpha1.ServiceReplicaStatus{
ComponentKind: v1alpha1.ComponentKindDeployment,
ComponentNames: []string{"test-dgd-prefill-e1f2a6fe-deployment"},
ComponentNames: []string{"test-dgd-prefill-1b69c0d3-deployment"},
Replicas: 3,
UpdatedReplicas: 3,
ReadyReplicas: ptr.To(int32(3)),
Expand All @@ -4443,7 +4450,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
wantReconcileResult: ReconcileResult{
State: v1beta1.DGDStatePending,
Reason: "some_resources_are_not_ready",
Message: "Resources not ready: test-dgd-decode-e1f2a6fe: Component deployment not ready - Available condition not true",
Message: "Resources not ready: test-dgd-decode-1b69c0d3: Component deployment not ready - Available condition not true",
ComponentStatus: map[string]v1beta1.ComponentReplicaStatus{
"frontend": {
ComponentKind: v1beta1.ComponentKindDeployment,
Expand All @@ -4455,15 +4462,15 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
},
"decode": {
ComponentKind: v1beta1.ComponentKindDeployment,
ComponentNames: []string{"test-dgd-decode-e1f2a6fe-deployment"},
ComponentNames: []string{"test-dgd-decode-1b69c0d3-deployment"},
Replicas: 2,
UpdatedReplicas: 1,
ReadyReplicas: ptr.To(int32(1)),
AvailableReplicas: ptr.To(int32(0)),
},
"prefill": {
ComponentKind: v1beta1.ComponentKindDeployment,
ComponentNames: []string{"test-dgd-prefill-e1f2a6fe-deployment"},
ComponentNames: []string{"test-dgd-prefill-1b69c0d3-deployment"},
Replicas: 3,
UpdatedReplicas: 3,
ReadyReplicas: ptr.To(int32(3)),
Expand Down Expand Up @@ -4491,6 +4498,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
},
},
},
dgdAnnotations: map[string]string{commonconsts.AnnotationCurrentWorkerHashV2: "cabcd5c9"},
existingDCDs: []client.Object{
betaDCD(t, &v1alpha1.DynamoComponentDeployment{
ObjectMeta: metav1.ObjectMeta{
Expand Down Expand Up @@ -4523,14 +4531,15 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
}),
betaDCD(t, &v1alpha1.DynamoComponentDeployment{
ObjectMeta: metav1.ObjectMeta{
Name: "test-dgd-decode-5f3d46ba",
Name: "test-dgd-decode-cabcd5c9",
Namespace: "default",
},
Spec: v1alpha1.DynamoComponentDeploymentSpec{
BackendFramework: "vllm",
DynamoComponentDeploymentSharedSpec: v1alpha1.DynamoComponentDeploymentSharedSpec{
ServiceName: "decode",
Replicas: ptr.To(int32(2)),
Labels: map[string]string{commonconsts.KubeLabelDynamoWorkerHash: "cabcd5c9"},
},
},
Status: v1alpha1.DynamoComponentDeploymentStatus{
Expand All @@ -4542,7 +4551,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
},
Service: &v1alpha1.ServiceReplicaStatus{
ComponentKind: v1alpha1.ComponentKindDeployment,
ComponentNames: []string{"test-dgd-decode-5f3d46ba-deployment"},
ComponentNames: []string{"test-dgd-decode-cabcd5c9-deployment"},
Replicas: 2,
UpdatedReplicas: 1,
ReadyReplicas: ptr.To(int32(1)),
Expand All @@ -4554,7 +4563,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
wantReconcileResult: ReconcileResult{
State: v1beta1.DGDStatePending,
Reason: "some_resources_are_not_ready",
Message: "Resources not ready: test-dgd-decode-5f3d46ba: Component deployment not ready - Available condition not true; test-dgd-frontend: Component deployment not ready - Available condition not true",
Message: "Resources not ready: test-dgd-decode-cabcd5c9: Component deployment not ready - Available condition not true; test-dgd-frontend: Component deployment not ready - Available condition not true",
ComponentStatus: map[string]v1beta1.ComponentReplicaStatus{
"frontend": {
ComponentKind: v1beta1.ComponentKindDeployment,
Expand All @@ -4566,7 +4575,7 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {
},
"decode": {
ComponentKind: v1beta1.ComponentKindDeployment,
ComponentNames: []string{"test-dgd-decode-5f3d46ba-deployment"},
ComponentNames: []string{"test-dgd-decode-cabcd5c9-deployment"},
Replicas: 2,
UpdatedReplicas: 1,
ReadyReplicas: ptr.To(int32(1)),
Expand All @@ -4587,8 +4596,9 @@ func Test_reconcileDynamoComponentsDeployments(t *testing.T) {

dgd := betaDGD(t, &v1alpha1.DynamoGraphDeployment{
ObjectMeta: metav1.ObjectMeta{
Name: "test-dgd",
Namespace: "default",
Name: "test-dgd",
Namespace: "default",
Annotations: tt.dgdAnnotations,
},
Spec: tt.dgdSpec,
})
Expand Down
Loading
Loading