adds a synthetic test for checking if the pod.spec.nodeName field is immutable

pkg/monitor/pod.go

@@ -313,6 +313,18 @@ func startPodMonitoring(ctx context.Context, m Recorder, client kubernetes.Inter
 			}
 			return conditions
 		},
+		// inform when a pod gets reassigned to a new node
+		func(pod, oldPod *corev1.Pod) []monitorapi.Condition {
+			var conditions []monitorapi.Condition
+			if len(oldPod.Spec.NodeName) > 0 && pod.Spec.NodeName != oldPod.Spec.NodeName {
+				conditions = append(conditions, monitorapi.Condition{
+					Level:   monitorapi.Error,
+					Locator: locatePod(pod),
+					Message: fmt.Sprintf("invariant violation, pod once assigned to a node must stay on it. The pod previously scheduled to %s, has just been assigned to a new node %s", oldPod.Spec.NodeName, pod.Spec.NodeName),
+				})
+			}
+			return conditions
+		},
 	}
 	podDeleteFns := []func(pod *corev1.Pod) []monitorapi.Condition{
 		// check for transitions to being deleted

pkg/synthetictests/apiserver.go

@@ -72,3 +72,25 @@ func testAllIngressAvailability(events monitorapi.Intervals, jobRunDuration time
 
 	return ret
 }
+
+func testPodNodeNameIsImmutable(events monitorapi.Intervals) []*junitapi.JUnitTestCase {
+	const testName = "[sig-api-machinery] the pod.spec.nodeName field is immutable, once set cannot be changed"
+
+	failures := []string{}
+	for _, event := range events {
+		if strings.Contains(event.Message, "pod once assigned to a node must stay on it") {
+			failures = append(failures, fmt.Sprintf("%v %v", event.Locator, event.Message))
+		}
+	}
+	if len(failures) == 0 {
+		return []*junitapi.JUnitTestCase{{Name: testName}}
+	}
+
+	return []*junitapi.JUnitTestCase{{
+		Name:      testName,
+		SystemOut: strings.Join(failures, "\n"),
+		FailureOutput: &junitapi.FailureOutput{
+			Output: fmt.Sprintf("Please report in https://bugzilla.redhat.com/show_bug.cgi?id=2042657\n\n%d pods had their immutable field (spec.nodeName) changed\n\n%v", len(failures), strings.Join(failures, "\n")),
+		},
+	}}
+}

pkg/synthetictests/event_junits.go

@@ -32,6 +32,7 @@ func StableSystemEventInvariants(events monitorapi.Intervals, duration time.Dura
 	tests = append(tests, testErrImagePullGeneric(events)...)
 	tests = append(tests, testAlerts(events, kubeClientConfig)...)
 	tests = append(tests, testOperatorOSUpdateStaged(events)...)
+	tests = append(tests, testPodNodeNameIsImmutable(events)...)
 
 	return tests
 }
@@ -56,6 +57,8 @@ func SystemUpgradeEventInvariants(events monitorapi.Intervals, duration time.Dur
 	tests = append(tests, testErrImagePullGeneric(events)...)
 	tests = append(tests, testAlerts(events, kubeClientConfig)...)
 	tests = append(tests, testOperatorOSUpdateStaged(events)...)
+	tests = append(tests, testPodNodeNameIsImmutable(events)...)
+
 	return tests
 }