Skip to content
28 changes: 21 additions & 7 deletions ray-operator/controllers/ray/common/job.go
Original file line number Diff line number Diff line change
Expand Up @@ -125,14 +125,28 @@ func BuildJobSubmitCommand(rayJobInstance *rayv1.RayJob, submissionMode rayv1.Jo
jobFollowCommand := []string{"ray", "job", "logs", "--address", address, "--follow", jobId}

if submissionMode == rayv1.SidecarMode {
rayVersion := ""
if rayJobInstance.Spec.RayClusterSpec != nil {
rayVersion = rayJobInstance.Spec.RayClusterSpec.RayVersion
}

// Wait until Ray Dashboard GCS is healthy before proceeding.
// Use the same Ray Dashboard GCS health check command as the readiness probe
rayDashboardGCSHealthCommand := fmt.Sprintf(
utils.BaseWgetHealthCommand,
utils.DefaultReadinessProbeFailureThreshold,
port,
utils.RayDashboardGCSHealthPath,
)
var rayDashboardGCSHealthCommand string
if supportsUnifiedHealthCheck(rayVersion) {
rayDashboardGCSHealthCommand = fmt.Sprintf(
utils.BasePythonHealthCommand,
port,
utils.RayDashboardGCSHealthPath,
utils.DefaultReadinessProbeFailureThreshold,
)
} else {
rayDashboardGCSHealthCommand = fmt.Sprintf(
utils.BaseWgetHealthCommand,
utils.DefaultReadinessProbeFailureThreshold,
port,
utils.RayDashboardGCSHealthPath,
)
}

waitLoop := []string{
"until", rayDashboardGCSHealthCommand, ">/dev/null", "2>&1", ";",
Expand Down
67 changes: 67 additions & 0 deletions ray-operator/controllers/ray/common/job_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,73 @@ func TestBuildJobSubmitCommandWithSidecarMode(t *testing.T) {
assert.Equal(t, expected, command)
}

func TestBuildJobSubmitCommandWithSidecarModeVersionSwitch(t *testing.T) {
tests := []struct {
name string
rayVersion string
usePythonHealth bool
}{
{
name: "uses python health command for ray >= 2.53",
rayVersion: "2.53.0",
usePythonHealth: true,
},
{
name: "uses wget health command for ray < 2.53",
rayVersion: "2.52.0",
usePythonHealth: false,
},
{
name: "uses wget health command when rayVersion is invalid",
rayVersion: "invalid-version",
usePythonHealth: false,
},
}

for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
testRayJob := rayJobTemplate()
testRayJob.Spec.RayClusterSpec.RayVersion = tt.rayVersion
if tt.rayVersion == "invalid-version" {
// Avoid metadata-json version parsing failure; this test only checks health command selection.
testRayJob.Spec.Metadata = nil
}
testRayJob.Spec.RayClusterSpec.HeadGroupSpec.Template.Spec.Containers = []corev1.Container{
{
Ports: []corev1.ContainerPort{
{
Name: utils.DashboardPortName,
ContainerPort: utils.DefaultDashboardPort,
},
},
},
}

command, err := BuildJobSubmitCommand(testRayJob, rayv1.SidecarMode)
require.NoError(t, err)
require.GreaterOrEqual(t, len(command), 2)
assert.Equal(t, "until", command[0])

expected := fmt.Sprintf(
utils.BaseWgetHealthCommand,
utils.DefaultReadinessProbeFailureThreshold,
utils.DefaultDashboardPort,
utils.RayDashboardGCSHealthPath,
)
if tt.usePythonHealth {
expected = fmt.Sprintf(
utils.BasePythonHealthCommand,
utils.DefaultDashboardPort,
utils.RayDashboardGCSHealthPath,
utils.DefaultReadinessProbeFailureThreshold,
)
}

assert.Equal(t, expected, command[1])
Comment thread
400Ping marked this conversation as resolved.
Outdated
})
}
}

func TestBuildJobSubmitCommandWithK8sJobModeAndYAML(t *testing.T) {
rayJobWithYAML := &rayv1.RayJob{
Spec: rayv1.RayJobSpec{
Expand Down
9 changes: 7 additions & 2 deletions ray-operator/controllers/ray/utils/constant.go
Original file line number Diff line number Diff line change
Expand Up @@ -244,8 +244,13 @@ const (
RayAgentRayletHealthPath = "api/local_raylet_healthz"
RayDashboardGCSHealthPath = "api/gcs_healthz"
RayServeProxyHealthPath = "-/healthz"
BaseWgetHealthCommand = "wget --tries 1 -T %d -q -O- http://localhost:%d/%s | grep success"
RayNodeHealthPath = "/api/healthz"
// BaseWgetHealthCommand checks a single health URL; args: timeout_sec, port, path (no leading slash).
// This is used for Ray versions that rely on exec probes and assume common CLI tools exist in the image.
BaseWgetHealthCommand = `wget -q -T %d -O- http://localhost:%d/%s | grep -q success`
Comment thread
cursor[bot] marked this conversation as resolved.
Outdated
// BasePythonHealthCommand checks a single health URL; args: port, path (no leading slash), timeout_sec.
// This is used when wget is not available (e.g. slim Ray images).
BasePythonHealthCommand = `python3 -c "import urllib.request; r=urllib.request.urlopen('http://localhost:%d/%s', timeout=%d); exit(0 if b'success' in r.read() else 1)"`
Comment thread
400Ping marked this conversation as resolved.
Outdated
RayNodeHealthPath = "/api/healthz"

// Finalizers for RayJob
RayJobStopJobFinalizer = "ray.io/rayjob-finalizer"
Expand Down
Loading