feat(operator): enable worker canary health checks

deploy/operator/internal/controller/dynamocomponentdeployment_controller_test.go

@@ -1241,7 +1241,7 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 										},
 									},
 									MainContainer: &corev1.Container{
-										Image: "test-image:latest",
+										Image: "test-image:1.5.0",
 										Command: []string{
 											"some",
 											"dynamo",
@@ -1333,14 +1333,14 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 									},
 									{
 										Name:    commonconsts.MainContainerName,
-										Image:   "test-image:latest",
+										Image:   "test-image:1.5.0",
 										Command: []string{"/bin/sh", "-c"},
 										Args:    []string{"ray start --head --port=6379 && some dynamo command --tensor-parallel-size 4 --pipeline-parallel-size 1 --distributed-executor-backend ray"},
 										Env: []corev1.EnvVar{
 											{Name: commonconsts.DynamoComponentEnvVar, Value: commonconsts.ComponentTypeWorker},
 											{Name: commonconsts.DynamoDiscoveryBackendEnvVar, Value: "kubernetes"},
 											{Name: "DYN_FORWARDPASS_METRIC_PORT", Value: "20380"},
-											{Name: "DYN_HEALTH_CHECK_ENABLED", Value: "false"},
+											{Name: "DYN_HEALTH_CHECK_ENABLED", Value: "true"},
 											{Name: commonconsts.DynamoNamespaceEnvVar, Value: "default-test-lws-deploy"},
 											{Name: "DYN_PARENT_DGD_K8S_NAME", Value: "test-lws-deploy"},
 											{Name: "DYN_PARENT_DGD_K8S_NAMESPACE", Value: "default"},
@@ -1474,14 +1474,14 @@ func TestDynamoComponentDeploymentReconciler_generateLeaderWorkerSet(t *testing.
 									},
 									{
 										Name:    commonconsts.MainContainerName,
-										Image:   "test-image:latest",
+										Image:   "test-image:1.5.0",
 										Command: []string{"/bin/sh", "-c"},
 										Args:    []string{"ray start --address=$(LWS_LEADER_ADDRESS):6379 --block"},
 										Env: []corev1.EnvVar{
 											{Name: commonconsts.DynamoComponentEnvVar, Value: commonconsts.ComponentTypeWorker},
 											{Name: commonconsts.DynamoDiscoveryBackendEnvVar, Value: "kubernetes"},
 											{Name: "DYN_FORWARDPASS_METRIC_PORT", Value: "20380"},
-											{Name: "DYN_HEALTH_CHECK_ENABLED", Value: "false"},
+											{Name: "DYN_HEALTH_CHECK_ENABLED", Value: "true"},
 											{Name: commonconsts.DynamoNamespaceEnvVar, Value: "default-test-lws-deploy"},
 											{Name: "DYN_PARENT_DGD_K8S_NAME", Value: "test-lws-deploy"},
 											{Name: "DYN_PARENT_DGD_K8S_NAMESPACE", Value: "default"},

deploy/operator/internal/controller/upgrade_test.go

@@ -100,7 +100,7 @@ func TestLegacyWorkerIdentityUpgradeDoesNotTriggerRollout(t *testing.T) {
 						ExtraPodSpec: &v1alpha1.ExtraPodSpec{
 							MainContainer: &corev1.Container{
 								Name:    commonconsts.MainContainerName,
-								Image:   "test-image:latest",
+								Image:   "test-image:1.4.0",
 								Command: []string{"python3"},
 								Args:    []string{"-m", "dynamo.vllm"},
 								Resources: corev1.ResourceRequirements{
@@ -144,7 +144,7 @@ spec:
           sizeLimit: 8Gi
       containers:
       - name: main
-        image: test-image:latest
+        image: test-image:1.4.0
         command:
         - python3
         args:
@@ -295,7 +295,7 @@ spec:
 						ExtraPodSpec: &v1alpha1.ExtraPodSpec{
 							MainContainer: &corev1.Container{
 								Name:    commonconsts.MainContainerName,
-								Image:   "test-image:latest",
+								Image:   "test-image:1.4.0",
 								Command: []string{"python3"},
 								Args:    []string{"-m", "dynamo.vllm"},
 								Resources: corev1.ResourceRequirements{
@@ -339,7 +339,7 @@ spec:
             sizeLimit: 8Gi
         containers:
         - name: main
-          image: test-image:latest
+          image: test-image:1.4.0
           command:
           - python3
           args:
@@ -446,7 +446,7 @@ spec:
             sizeLimit: 8Gi
         containers:
         - name: main
-          image: test-image:latest
+          image: test-image:1.4.0
           command:
           - python3
           args:
@@ -869,8 +869,8 @@ spec:
 			oldPodLabels := tt.childPodLabels(t, oldChild)
 			newPodLabels := tt.childPodLabels(t, newChild)
 
-			t.Log("compare old and new child specs; a change here would trigger a rollout")
-			require.Equal(t, specHash(t, oldChild), specHash(t, newChild), "upgrade should not change the child spec hash")
+			t.Log("compare old and new child specs; an operator-only upgrade must not trigger a rollout")
+			require.Equal(t, specHash(t, oldChild), specHash(t, newChild), "upgrade should preserve the child spec hash for a pinned older runtime")
 
 			t.Log("assert worker pod labels keep the legacy worker identity")
 			for site, subComponentType := range tt.expectedWorkerSites {

deploy/operator/internal/dynamo/component_common.go

@@ -10,6 +10,7 @@ import (
 	"github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1"
 	commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts"
 	controller_common "github.com/ai-dynamo/dynamo/deploy/operator/internal/controller_common"
+	"github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/utils/ptr"
 )
@@ -67,6 +68,7 @@ type ComponentContext struct {
 	Discovery                      DiscoveryContext
 	EPPConfig                      *v1beta1.EPPConfig
 	WorkerHashSuffix               string
+	RuntimeVersion                 *runtimeversion.Version
 }
 
 func (b *BaseComponentDefaults) GetBaseContainer(context ComponentContext) (corev1.Container, error) {

deploy/operator/internal/dynamo/component_worker.go

@@ -7,9 +7,11 @@ package dynamo
 
 import (
 	"fmt"
+	"strconv"
 
 	"github.com/ai-dynamo/dynamo/deploy/operator/api/v1beta1"
 	commonconsts "github.com/ai-dynamo/dynamo/deploy/operator/internal/consts"
+	runtimefeatures "github.com/ai-dynamo/dynamo/deploy/operator/internal/features/runtime"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
@@ -25,6 +27,7 @@ func NewWorkerDefaults() *WorkerDefaults {
 
 func (w *WorkerDefaults) GetBaseContainer(context ComponentContext) (corev1.Container, error) {
 	container := w.getCommonContainer(context)
+	canaryHealthChecksEnabled := runtimefeatures.CanaryHealthChecks.Enabled(context.RuntimeVersion)
 
 	// Add system port
 	container.Ports = []corev1.ContainerPort{
@@ -49,7 +52,7 @@ func (w *WorkerDefaults) GetBaseContainer(context ComponentContext) (corev1.Cont
 		},
 		PeriodSeconds:    5,
 		TimeoutSeconds:   4, // TimeoutSeconds should be < PeriodSeconds
-		FailureThreshold: 1, // Note this default FailureThreshold is 3, with 1 a single failure will restart Pod
+		FailureThreshold: 1, // A single failed liveness check restarts the Pod.
 	}
 
 	// ReadinessProbe in Dynamo worker context doesn't determine that the worker is ready to receive traffic
@@ -94,7 +97,7 @@ func (w *WorkerDefaults) GetBaseContainer(context ComponentContext) (corev1.Cont
 		},
 		{
 			Name:  "DYN_HEALTH_CHECK_ENABLED",
-			Value: "false",
+			Value: strconv.FormatBool(canaryHealthChecksEnabled),
 		},
 		{
 			Name:  "NIXL_TELEMETRY_ENABLE",

deploy/operator/internal/dynamo/component_worker_test.go

@@ -0,0 +1,60 @@
+/*
+ * SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+package dynamo
+
+import (
+	"testing"
+
+	"github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion"
+	"github.com/stretchr/testify/require"
+)
+
+func TestWorkerDefaultsCanaryHealthCheckVersionGate(t *testing.T) {
+	tests := []struct {
+		name           string
+		runtimeVersion *runtimeversion.Version
+		wantEnabled    string
+	}{
+		{
+			name:        "unknown legacy runtime",
+			wantEnabled: "false",
+		},
+		{
+			name:           "older runtime",
+			runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 4, Patch: 9},
+			wantEnabled:    "false",
+		},
+		{
+			name:           "minimum supported runtime",
+			runtimeVersion: &runtimeversion.Version{Major: 1, Minor: 5, Patch: 0},
+			wantEnabled:    "true",
+		},
+		{
+			name:           "newer runtime",
+			runtimeVersion: &runtimeversion.Version{Major: 2, Minor: 0, Patch: 0},
+			wantEnabled:    "true",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			container, err := NewWorkerDefaults().GetBaseContainer(ComponentContext{
+				RuntimeVersion: tt.runtimeVersion,
+			})
+			require.NoError(t, err)
+			require.NotNil(t, container.LivenessProbe)
+			require.EqualValues(t, 1, container.LivenessProbe.FailureThreshold)
+
+			for _, env := range container.Env {
+				if env.Name == "DYN_HEALTH_CHECK_ENABLED" {
+					require.Equal(t, tt.wantEnabled, env.Value)
+					return
+				}
+			}
+			t.Fatal("DYN_HEALTH_CHECK_ENABLED was not rendered")
+		})
+	}
+}

deploy/operator/internal/dynamo/graph.go

@@ -39,6 +39,7 @@ import (
 	"github.com/ai-dynamo/dynamo/deploy/operator/internal/dra"
 	"github.com/ai-dynamo/dynamo/deploy/operator/internal/features"
 	gms "github.com/ai-dynamo/dynamo/deploy/operator/internal/gms"
+	"github.com/ai-dynamo/dynamo/deploy/operator/internal/runtimeversion"
 	grovev1alpha1 "github.com/ai-dynamo/grove/operator/api/core/v1alpha1"
 	"github.com/imdario/mergo"
 	"google.golang.org/protobuf/types/known/wrapperspb"
@@ -1659,7 +1660,10 @@ func GenerateBasePodSpec(
 ) (*corev1.PodSpec, error) {
 	// Start with base container generated per component type
 	annotations := GetPodTemplateAnnotations(component)
-	componentContext := generateComponentContext(component, parentGraphDeploymentName, namespace, numberOfNodes, NewDiscoveryContext(operatorConfig.Discovery.Backend, annotations))
+	componentContext, err := generateComponentContext(component, parentGraphDeploymentName, namespace, numberOfNodes, NewDiscoveryContext(operatorConfig.Discovery.Backend, annotations))
+	if err != nil {
+		return nil, err
+	}
 	componentDefaults := ComponentDefaultsFactory(string(component.ComponentType))
 	container, err := componentDefaults.GetBaseContainer(componentContext)
 	if err != nil {
@@ -2035,14 +2039,25 @@ func setMetricsLabels(labels map[string]string, dynamoGraphDeployment *v1beta1.D
 	labels[commonconsts.KubeLabelMetricsEnabled] = commonconsts.KubeLabelValueTrue
 }
 
-func generateComponentContext(component *v1beta1.DynamoComponentDeploymentSharedSpec, parentGraphDeploymentName string, namespace string, numberOfNodes int32, discovery DiscoveryContext) ComponentContext {
+func generateComponentContext(component *v1beta1.DynamoComponentDeploymentSharedSpec, parentGraphDeploymentName string, namespace string, numberOfNodes int32, discovery DiscoveryContext) (ComponentContext, error) {
 	dynamoNamespace := v1beta1.ComputeDynamoNamespace(component.GlobalDynamoNamespace, namespace, parentGraphDeploymentName)
 	var workerHashSuffix string
 	labels := GetPodTemplateLabels(component)
 	if workerHash := labels[commonconsts.KubeLabelDynamoWorkerHash]; IsWorkerComponent(string(component.ComponentType)) && workerHash != "" {
 		workerHashSuffix = workerHash
 	}
 
+	var image string
+	if main := GetMainContainer(component); main != nil {
+		image = main.Image
+	}
+	var resolvedRuntimeVersion *runtimeversion.Version
+	if version, err := runtimeversion.Resolve(image, component.RuntimeVersionOverride); err == nil {
+		resolvedRuntimeVersion = &version
+	} else if component.RuntimeVersionOverride != "" {
+		return ComponentContext{}, fmt.Errorf("resolve runtime version override: %w", err)
+	}
+
 	componentContext := ComponentContext{
 		numberOfNodes:                  numberOfNodes,
 		ComponentType:                  string(component.ComponentType),
@@ -2052,8 +2067,9 @@ func generateComponentContext(component *v1beta1.DynamoComponentDeploymentShared
 		DynamoNamespace:                dynamoNamespace,
 		EPPConfig:                      component.EPPConfig,
 		WorkerHashSuffix:               workerHashSuffix,
+		RuntimeVersion:                 resolvedRuntimeVersion,
 	}
-	return componentContext
+	return componentContext, nil
 }
 
 // GeneratePodSpecForComponent creates a PodSpec for Grove deployments (simplified wrapper)