diff --git a/pkg/monitor/pod.go b/pkg/monitor/pod.go index b9c89826aeb5..85214638b103 100644 --- a/pkg/monitor/pod.go +++ b/pkg/monitor/pod.go @@ -313,6 +313,18 @@ func startPodMonitoring(ctx context.Context, m Recorder, client kubernetes.Inter } return conditions }, + // inform when a pod gets reassigned to a new node + func(pod, oldPod *corev1.Pod) []monitorapi.Condition { + var conditions []monitorapi.Condition + if len(oldPod.Spec.NodeName) > 0 && pod.Spec.NodeName != oldPod.Spec.NodeName { + conditions = append(conditions, monitorapi.Condition{ + Level: monitorapi.Error, + Locator: locatePod(pod), + Message: fmt.Sprintf("invariant violation, pod once assigned to a node must stay on it. The pod previously scheduled to %s, has just been assigned to a new node %s", oldPod.Spec.NodeName, pod.Spec.NodeName), + }) + } + return conditions + }, } podDeleteFns := []func(pod *corev1.Pod) []monitorapi.Condition{ // check for transitions to being deleted diff --git a/pkg/synthetictests/apiserver.go b/pkg/synthetictests/apiserver.go index 3089076e453a..e33c76905ee6 100644 --- a/pkg/synthetictests/apiserver.go +++ b/pkg/synthetictests/apiserver.go @@ -72,3 +72,25 @@ func testAllIngressAvailability(events monitorapi.Intervals, jobRunDuration time return ret } + +func testPodNodeNameIsImmutable(events monitorapi.Intervals) []*junitapi.JUnitTestCase { + const testName = "[sig-api-machinery] the pod.spec.nodeName field is immutable, once set cannot be changed" + + failures := []string{} + for _, event := range events { + if strings.Contains(event.Message, "pod once assigned to a node must stay on it") { + failures = append(failures, fmt.Sprintf("%v %v", event.Locator, event.Message)) + } + } + if len(failures) == 0 { + return []*junitapi.JUnitTestCase{{Name: testName}} + } + + return []*junitapi.JUnitTestCase{{ + Name: testName, + SystemOut: strings.Join(failures, "\n"), + FailureOutput: &junitapi.FailureOutput{ + Output: fmt.Sprintf("Please report in https://bugzilla.redhat.com/show_bug.cgi?id=2042657\n\n%d pods had their immutable field (spec.nodeName) changed\n\n%v", len(failures), strings.Join(failures, "\n")), + }, + }} +} diff --git a/pkg/synthetictests/event_junits.go b/pkg/synthetictests/event_junits.go index 0aa8ef212a5d..77643a928e34 100644 --- a/pkg/synthetictests/event_junits.go +++ b/pkg/synthetictests/event_junits.go @@ -32,6 +32,7 @@ func StableSystemEventInvariants(events monitorapi.Intervals, duration time.Dura tests = append(tests, testErrImagePullGeneric(events)...) tests = append(tests, testAlerts(events, kubeClientConfig)...) tests = append(tests, testOperatorOSUpdateStaged(events)...) + tests = append(tests, testPodNodeNameIsImmutable(events)...) return tests } @@ -56,6 +57,8 @@ func SystemUpgradeEventInvariants(events monitorapi.Intervals, duration time.Dur tests = append(tests, testErrImagePullGeneric(events)...) tests = append(tests, testAlerts(events, kubeClientConfig)...) tests = append(tests, testOperatorOSUpdateStaged(events)...) + tests = append(tests, testPodNodeNameIsImmutable(events)...) + return tests }