From 254a4afcf0c201f02eafad4ea6f6b0f50d02d34b Mon Sep 17 00:00:00 2001 From: Rob Bell Date: Mon, 3 Aug 2026 15:22:54 +0100 Subject: [PATCH 1/3] fix: remove race conditions in progression tracking E2E test The "metrics endpoint unreachable" test had two race conditions: 1. The Consistently block checking running status could race with job completion and annotation synthesis between status check and assertion 2. The final assertion expected no annotation, but raced with the reconciler's fallback that synthesizes 100% on completion Test was passing for 8 months probably because the race condition never got hit - the check that no annotation was added was done quickly, probably before the final reconcile happened. Fix: Remove flaky assertions about annotation state. Focus on what matters - the controller handles metrics errors gracefully and the TrainJob completes successfully Signed-off-by: Rob Bell --- test/e2e/rhai/progression_e2e_test.go | 32 +-------------------------- 1 file changed, 1 insertion(+), 31 deletions(-) diff --git a/test/e2e/rhai/progression_e2e_test.go b/test/e2e/rhai/progression_e2e_test.go index 3bf240ad45..6dd9be14c2 100644 --- a/test/e2e/rhai/progression_e2e_test.go +++ b/test/e2e/rhai/progression_e2e_test.go @@ -533,32 +533,8 @@ var _ = ginkgo.Describe("RHAI Progression Tracking E2E Tests", func() { g.Expect(pod.Status.Phase).To(gomega.Equal(corev1.PodRunning)) }, timeout, interval).Should(gomega.Succeed()) - ginkgo.By("Verifying controller continues to reconcile despite connection errors") + ginkgo.By("Check controller still reconciles the TrainJob by ensuring it gets marked complete") // Controller should log errors but continue running - // TrainJob should not have trainerStatus annotation since metrics are unreachable during running phase - gomega.Consistently(func(g gomega.Gomega) { - gotTrainJob := &trainer.TrainJob{} - g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trainJob), gotTrainJob)).Should(gomega.Succeed()) - - // Only check while job is still running (not completed/failed) - isRunning := true - for _, cond := range gotTrainJob.Status.Conditions { - if (cond.Type == trainer.TrainJobComplete || cond.Type == trainer.TrainJobFailed) && - cond.Status == metav1.ConditionTrue { - isRunning = false - break - } - } - - // Annotation should not be created while running if metrics are unreachable - // (It will be synthesized after completion, which is checked later) - if isRunning { - _, exists := gotTrainJob.Annotations[constants.AnnotationTrainerStatus] - g.Expect(exists).Should(gomega.BeFalse(), "trainerStatus should not be created during running when metrics are unreachable") - } - }, 10*time.Second, interval).Should(gomega.Succeed()) - - ginkgo.By("Waiting for TrainJob to complete despite metrics errors") gomega.Eventually(func(g gomega.Gomega) { gotTrainJob := &trainer.TrainJob{} g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trainJob), gotTrainJob)).Should(gomega.Succeed()) @@ -571,12 +547,6 @@ var _ = ginkgo.Describe("RHAI Progression Tracking E2E Tests", func() { } g.Expect(completed).Should(gomega.BeTrue(), "TrainJob should complete even without metrics") }, timeout, interval).Should(gomega.Succeed()) - - ginkgo.By("Verifying no trainerStatus annotation is created when metrics were never reachable") - gotTrainJob := &trainer.TrainJob{} - gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trainJob), gotTrainJob)).Should(gomega.Succeed()) - _, exists := gotTrainJob.Annotations[constants.AnnotationTrainerStatus] - gomega.Expect(exists).Should(gomega.BeFalse(), "trainerStatus should not be synthesized when metrics were never reachable") }) }) From 513ee956a05bb36a485d13ab371bda27681aef75 Mon Sep 17 00:00:00 2001 From: kapil27 Date: Wed, 5 Aug 2026 21:22:10 +0530 Subject: [PATCH 2/3] fix: point trainer controller image at odh-3.5.0 Konflux release-push publishes quay.io/opendatahub/trainer:odh-3.5.0, but params.env referenced the non-existent odh-3.5 stream tag, causing operator RHOAI e2e ImagePullBackOff (RHOAIENG-80313). Leave TH image tags on odh-3.5 where those Quay tags already exist. Co-authored-by: Cursor --- manifests/rhoai/params.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/rhoai/params.env b/manifests/rhoai/params.env index f2f4a2adc7..bdd9290a25 100644 --- a/manifests/rhoai/params.env +++ b/manifests/rhoai/params.env @@ -1,4 +1,4 @@ -odh-kubeflow-trainer-controller-image=quay.io/opendatahub/trainer:odh-3.5 +odh-kubeflow-trainer-controller-image=quay.io/opendatahub/trainer:odh-3.5.0 odh-th-torch-cuda-py312-image=quay.io/opendatahub/odh-th-torch-cuda-py312:odh-3.5 odh-th-torch-rocm-py312-image=quay.io/opendatahub/odh-th-torch-rocm-py312:odh-3.5 odh-th-torch-cpu-py312-image=quay.io/opendatahub/odh-th-torch-cpu-py312:odh-3.5 From 762b000bdc68477e22af351047446888cd5586ce Mon Sep 17 00:00:00 2001 From: kapil27 Date: Wed, 5 Aug 2026 22:20:36 +0530 Subject: [PATCH 3/3] chore: add missing newline at end of params.env Satisfy pre-commit end-of-file-fixer on the RHOAI 3.5 cherry-pick. Co-authored-by: Cursor --- manifests/rhoai/params.env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/manifests/rhoai/params.env b/manifests/rhoai/params.env index bdd9290a25..5fc89c5f5f 100644 --- a/manifests/rhoai/params.env +++ b/manifests/rhoai/params.env @@ -7,4 +7,4 @@ odh-training-universal-workbench-image-rocm-3-4=registry.redhat.io/rhoai/odh-th0 odh-training-universal-workbench-image-cpu-3-4=registry.redhat.io/rhoai/odh-th06-cpu-torch210-py312-rhel9@sha256:4b762d722faa80d7c6641a631d891f0ddf048950a4f8b6e5b93010d65134687d odh-training-universal-workbench-image-cuda-3-5=quay.io/opendatahub/odh-th-torch-cuda-py312:odh-3.5 odh-training-universal-workbench-image-rocm-3-5=quay.io/opendatahub/odh-th-torch-rocm-py312:odh-3.5 -odh-training-universal-workbench-image-cpu-3-5=quay.io/opendatahub/odh-th-torch-cpu-py312:odh-3.5 \ No newline at end of file +odh-training-universal-workbench-image-cpu-3-5=quay.io/opendatahub/odh-th-torch-cpu-py312:odh-3.5