Skip to content

[RHOAI 3.5] fix: trainer image tag odh-3.5.0 + progression e2e races #141

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
kapil27 merged 3 commits into from
Aug 6, 2026
Merged

[RHOAI 3.5] fix: trainer image tag odh-3.5.0 + progression e2e races #141

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
254a4af
fix: remove race conditions in progression tracking E2E test
robert-bell Aug 3, 2026
513ee95
fix: point trainer controller image at odh-3.5.0
kapil27 Aug 5, 2026
762b000
chore: add missing newline at end of params.env
kapil27 Aug 5, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions manifests/rhoai/params.env
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
odh-kubeflow-trainer-controller-image=quay.io/opendatahub/trainer:odh-3.5
odh-kubeflow-trainer-controller-image=quay.io/opendatahub/trainer:odh-3.5.0
odh-th-torch-cuda-py312-image=quay.io/opendatahub/odh-th-torch-cuda-py312:odh-3.5
odh-th-torch-rocm-py312-image=quay.io/opendatahub/odh-th-torch-rocm-py312:odh-3.5
odh-th-torch-cpu-py312-image=quay.io/opendatahub/odh-th-torch-cpu-py312:odh-3.5
Expand All @@ -7,4 +7,4 @@ odh-training-universal-workbench-image-rocm-3-4=registry.redhat.io/rhoai/odh-th0
odh-training-universal-workbench-image-cpu-3-4=registry.redhat.io/rhoai/odh-th06-cpu-torch210-py312-rhel9@sha256:4b762d722faa80d7c6641a631d891f0ddf048950a4f8b6e5b93010d65134687d
odh-training-universal-workbench-image-cuda-3-5=quay.io/opendatahub/odh-th-torch-cuda-py312:odh-3.5
odh-training-universal-workbench-image-rocm-3-5=quay.io/opendatahub/odh-th-torch-rocm-py312:odh-3.5
odh-training-universal-workbench-image-cpu-3-5=quay.io/opendatahub/odh-th-torch-cpu-py312:odh-3.5
odh-training-universal-workbench-image-cpu-3-5=quay.io/opendatahub/odh-th-torch-cpu-py312:odh-3.5
32 changes: 1 addition & 31 deletions test/e2e/rhai/progression_e2e_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -533,32 +533,8 @@ var _ = ginkgo.Describe("RHAI Progression Tracking E2E Tests", func() {
g.Expect(pod.Status.Phase).To(gomega.Equal(corev1.PodRunning))
}, timeout, interval).Should(gomega.Succeed())

ginkgo.By("Verifying controller continues to reconcile despite connection errors")
ginkgo.By("Check controller still reconciles the TrainJob by ensuring it gets marked complete")
// Controller should log errors but continue running
// TrainJob should not have trainerStatus annotation since metrics are unreachable during running phase
gomega.Consistently(func(g gomega.Gomega) {
gotTrainJob := &trainer.TrainJob{}
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trainJob), gotTrainJob)).Should(gomega.Succeed())

// Only check while job is still running (not completed/failed)
isRunning := true
for _, cond := range gotTrainJob.Status.Conditions {
if (cond.Type == trainer.TrainJobComplete || cond.Type == trainer.TrainJobFailed) &&
cond.Status == metav1.ConditionTrue {
isRunning = false
break
}
}

// Annotation should not be created while running if metrics are unreachable
// (It will be synthesized after completion, which is checked later)
if isRunning {
_, exists := gotTrainJob.Annotations[constants.AnnotationTrainerStatus]
g.Expect(exists).Should(gomega.BeFalse(), "trainerStatus should not be created during running when metrics are unreachable")
}
}, 10*time.Second, interval).Should(gomega.Succeed())

ginkgo.By("Waiting for TrainJob to complete despite metrics errors")
gomega.Eventually(func(g gomega.Gomega) {
gotTrainJob := &trainer.TrainJob{}
g.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trainJob), gotTrainJob)).Should(gomega.Succeed())
Expand All @@ -571,12 +547,6 @@ var _ = ginkgo.Describe("RHAI Progression Tracking E2E Tests", func() {
}
g.Expect(completed).Should(gomega.BeTrue(), "TrainJob should complete even without metrics")
}, timeout, interval).Should(gomega.Succeed())

ginkgo.By("Verifying no trainerStatus annotation is created when metrics were never reachable")
gotTrainJob := &trainer.TrainJob{}
gomega.Expect(k8sClient.Get(ctx, client.ObjectKeyFromObject(trainJob), gotTrainJob)).Should(gomega.Succeed())
_, exists := gotTrainJob.Annotations[constants.AnnotationTrainerStatus]
gomega.Expect(exists).Should(gomega.BeFalse(), "trainerStatus should not be synthesized when metrics were never reachable")
})
})

Expand Down
Loading