[Feature] Support recreate pods for RayCluster using RayClusterSpec.upgradeStrategy

docs/reference/api.md

@@ -318,6 +318,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
+| `upgradeStrategy` _[RayClusterUpgradeStrategy](#rayclusterupgradestrategy)_ | UpgradeStrategy defines the scaling policy used when upgrading the RayCluster |  |  |
 | `authOptions` _[AuthOptions](#authoptions)_ | AuthOptions specifies the authentication options for the RayCluster. |  |  |
 | `suspend` _boolean_ | Suspend indicates whether a RayCluster should be suspended.<br />A suspended RayCluster will have head pods and worker pods deleted. |  |  |
 | `managedBy` _string_ | ManagedBy is an optional configuration for the controller or entity that manages a RayCluster.<br />The value must be either 'ray.io/kuberay-operator' or 'kueue.x-k8s.io/multikueue'.<br />The kuberay-operator reconciles a RayCluster which doesn't have this field at all or<br />the field value is the reserved string 'ray.io/kuberay-operator',<br />but delegates reconciling the RayCluster with 'kueue.x-k8s.io/multikueue' to the Kueue.<br />The field is immutable. |  |  |
@@ -330,6 +331,40 @@ _Appears in:_
 | `workerGroupSpecs` _[WorkerGroupSpec](#workergroupspec) array_ | WorkerGroupSpecs are the specs for the worker pods |  |  |
 
 
+#### RayClusterUpgradeStrategy
+
+
+
+
+
+
+
+_Appears in:_
+- [RayClusterSpec](#rayclusterspec)
+
+| Field | Description | Default | Validation |
+| --- | --- | --- | --- |
+| `type` _[RayClusterUpgradeType](#rayclusterupgradetype)_ | Type represents the strategy used when upgrading the RayCluster Pods. Currently supports `Recreate` and `None`. |  | Enum: [Recreate None] <br /> |
+
+
+#### RayClusterUpgradeType
+
+_Underlying type:_ _string_
+
+
+
+_Validation:_
+- Enum: [Recreate None]
+
+_Appears in:_
+- [RayClusterUpgradeStrategy](#rayclusterupgradestrategy)
+
+| Field | Description |
+| --- | --- |
+| `Recreate` | During upgrade, Recreate strategy will delete all existing pods before creating new ones<br /> |
+| `None` | No new pod will be created while the strategy is set to None<br /> |
+
+
 #### RayCronJob
 
 
@@ -482,7 +517,7 @@ _Appears in:_
 
 | Field | Description | Default | Validation |
 | --- | --- | --- | --- |
-| `type` _[RayServiceUpgradeType](#rayserviceupgradetype)_ | Type represents the strategy used when upgrading the RayService. Currently supports `NewCluster` and `None`. |  |  |
+| `type` _[RayServiceUpgradeType](#rayserviceupgradetype)_ | Type represents the strategy used when upgrading the RayService. Currently supports `NewCluster`, `NewClusterWithIncrementalUpgrade` and `None`. |  |  |
 | `clusterUpgradeOptions` _[ClusterUpgradeOptions](#clusterupgradeoptions)_ | ClusterUpgradeOptions defines the behavior of a NewClusterWithIncrementalUpgrade type.<br />RayServiceIncrementalUpgrade feature gate must be enabled to set ClusterUpgradeOptions. |  |  |
 
 

helm-chart/kuberay-operator/crds/ray.io_raycronjobs.yaml

@@ -4614,6 +4614,14 @@ spec:
                         type: string
                       suspend:
                         type: boolean
+                      upgradeStrategy:
+                        properties:
+                          type:
+                            enum:
+                            - Recreate
+                            - None
+                            type: string
+                        type: object
                       workerGroupSpecs:
                         items:
                           properties:

ray-operator/apis/ray/v1/raycluster_types.go

@@ -11,6 +11,9 @@ import (
 
 // RayClusterSpec defines the desired state of RayCluster
 type RayClusterSpec struct {
+	// UpgradeStrategy defines the scaling policy used when upgrading the RayCluster
+	// +optional
+	UpgradeStrategy *RayClusterUpgradeStrategy `json:"upgradeStrategy,omitempty"`
 	// AuthOptions specifies the authentication options for the RayCluster.
 	// +optional
 	AuthOptions *AuthOptions `json:"authOptions,omitempty"`
@@ -49,6 +52,22 @@ type RayClusterSpec struct {
 	WorkerGroupSpecs []WorkerGroupSpec `json:"workerGroupSpecs,omitempty"`
 }
 
+// +kubebuilder:validation:Enum=Recreate;None
+type RayClusterUpgradeType string
+
+const (
+	// During upgrade, Recreate strategy will delete all existing pods before creating new ones
+	RayClusterRecreate RayClusterUpgradeType = "Recreate"
+	// No new pod will be created while the strategy is set to None
+	RayClusterUpgradeNone RayClusterUpgradeType = "None"
+)
+
+type RayClusterUpgradeStrategy struct {
+	// Type represents the strategy used when upgrading the RayCluster Pods. Currently supports `Recreate` and `None`.
+	// +optional
+	Type *RayClusterUpgradeType `json:"type,omitempty"`
+}
+
 // AuthMode describes the authentication mode for the Ray cluster.
 type AuthMode string
 

ray-operator/apis/ray/v1/rayservice_types.go

@@ -24,11 +24,11 @@ type RayServiceUpgradeType string
 const (
 	// During upgrade, NewClusterWithIncrementalUpgrade strategy will create an upgraded cluster to gradually scale
 	// and migrate traffic to using Gateway API.
-	NewClusterWithIncrementalUpgrade RayServiceUpgradeType = "NewClusterWithIncrementalUpgrade"
+	RayServiceNewClusterWithIncrementalUpgrade RayServiceUpgradeType = "NewClusterWithIncrementalUpgrade"
 	// During upgrade, NewCluster strategy will create new upgraded cluster and switch to it when it becomes ready
-	NewCluster RayServiceUpgradeType = "NewCluster"
+	RayServiceNewCluster RayServiceUpgradeType = "NewCluster"
 	// No new cluster will be created while the strategy is set to None
-	None RayServiceUpgradeType = "None"
+	RayServiceUpgradeNone RayServiceUpgradeType = "None"
 )
 
 // These statuses should match Ray Serve's application statuses
@@ -75,7 +75,7 @@ type ClusterUpgradeOptions struct {
 }
 
 type RayServiceUpgradeStrategy struct {
-	// Type represents the strategy used when upgrading the RayService. Currently supports `NewCluster` and `None`.
+	// Type represents the strategy used when upgrading the RayService. Currently supports `NewCluster`, `NewClusterWithIncrementalUpgrade` and `None`.
 	// +optional
 	Type *RayServiceUpgradeType `json:"type,omitempty"`
 	// ClusterUpgradeOptions defines the behavior of a NewClusterWithIncrementalUpgrade type.

ray-operator/config/crd/bases/ray.io_raycronjobs.yaml

@@ -4614,6 +4614,14 @@ spec:
                         type: string
                       suspend:
                         type: boolean
+                      upgradeStrategy:
+                        properties:
+                          type:
+                            enum:
+                            - Recreate
+                            - None
+                            type: string
+                        type: object
                       workerGroupSpecs:
                         items:
                           properties:

ray-operator/config/samples/ray-cluster.sample.yaml

@@ -6,6 +6,21 @@ kind: RayCluster
 metadata:
   name: raycluster-kuberay
 spec:
+  # UpgradeStrategy defines the pod recreation behavior when RayCluster configuration changes
+  # Available since: KubeRay 1.6.0
+  # Type of upgrade strategy to use:
+  # - "None" (default):
+  #   * Configuration changes do not trigger pod recreation
+  #   * Existing pods continue running with old configuration
+  #   * New pods (from scaling) will use new configuration
+  #   * Use this for maximum stability
+  # - "Recreate":
+  #   * All pods are deleted and recreated when configuration changes
+  #   * Ensures all pods run with the latest configuration
+  #   * Useful for GitOps system like ArgoCD
+  #   * May cause temporary service disruption during recreation
+  # upgradeStrategy:
+  #   type: None
   rayVersion: '2.52.0' # should match the Ray version in the image of the containers
   # Ray head pod template
   headGroupSpec: