Skip to content

OCPBUGS-65970: Fix MachineConfigNode test in two-node fencing clusters #30532

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
isabella-janssen wants to merge 1 commit into
base: main
Choose a base branch
from
+64 −29
Open

OCPBUGS-65970: Fix MachineConfigNode test in two-node fencing clusters #30532

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 24 additions & 7 deletions test/extended/machine_config/helpers.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -703,17 +703,34 @@ func WaitForNodeCurrentConfig(oc *exutil.CLI, nodeName string, config string) {
}, 5*time.Minute, 10*time.Second).Should(o.BeTrue(), "Timed out waiting for node '%v' to have a current config version of '%v'.", nodeName, config)
}

// `GetUpdatingNodeSNO` returns the SNO node when the `master` MCP of the cluster starts updating
func GetUpdatingNodeSNO(oc *exutil.CLI, mcpName string) corev1.Node {
// `GetUpdatingNode` returns the updating node, determined by the node targetting a new desired
// config, when the corresponding MCP starts updating
func GetUpdatingNode(oc *exutil.CLI, mcpName, originalConfigVersion string) corev1.Node {
// Wait for the MCP to start updating
o.Expect(WaitForMCPConditionStatus(oc, mcpName, mcfgv1.MachineConfigPoolUpdating, corev1.ConditionTrue, 3*time.Minute, 2*time.Second)).NotTo(o.HaveOccurred(), "Waiting for 'Updating' status change failed.")

// SNO only has one node, so when the MCP is updating, the node is also updating
node, nodeErr := GetNodesByRole(oc, mcpName)
o.Expect(nodeErr).NotTo(o.HaveOccurred(), "Error getting nodes from %v MCP.", mcpName)
o.Expect(node).ShouldNot(o.BeEmpty(), "No nodes found for %v MCP.", mcpName)
// Get first updating node & return it
var updatingNode corev1.Node
o.Eventually(func() bool {
framework.Logf("Trying to get updating node in '%v' MCP.", mcpName)

// Get nodes in MCP
nodes, nodeErr := GetNodesByRole(oc, mcpName)
o.Expect(nodeErr).NotTo(o.HaveOccurred(), "Error getting nodes from %v MCP.", mcpName)
o.Expect(nodes).ShouldNot(o.BeEmpty(), "No nodes found for %v MCP.", mcpName)

// Loop through nodes to see which is targetting a new desired config version
for _, node := range nodes {
if node.Annotations[desiredConfigAnnotationKey] != originalConfigVersion {
updatingNode = node
return true
}
}

return false
}, 30*time.Second, 1*time.Second).Should(o.BeTrue())

return node[0]
return updatingNode
}

// `WaitForMCPConditionStatus` waits up to the desired timeout for the desired MCP condition to match the desired status (ex. wait until "Updating" is "True")
Expand Down
62 changes: 40 additions & 22 deletions test/extended/machine_config/machine_config_node.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ import (
"fmt"
"os/exec"
"path/filepath"
"slices"
"time"

mcfgv1 "github.com/openshift/api/machineconfiguration/v1"
Expand Down Expand Up @@ -77,10 +78,19 @@ var _ = g.Describe("[sig-mco][OCPFeatureGate:MachineConfigNodes]", func() {
// regression tests handle the different conditions list.
SkipWhenFeatureGateEnabled(oc.AdminConfigClient(), "ImageModeStatusReporting")

if IsSingleNode(oc) {
ValidateMCNConditionTransitionsOnRebootlessUpdateSNO(oc, nodeDisruptionFixture, nodeDisruptionEmptyFixture, masterMCFixture)
} else {
ValidateMCNConditionTransitionsOnRebootlessUpdate(oc, nodeDisruptionFixture, nodeDisruptionEmptyFixture, customMCFixture, infraMCPFixture)
// Create client set for test
clientSet, clientErr := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
o.Expect(clientErr).NotTo(o.HaveOccurred(), "Error creating client set for test.")

// Get MCPs to test for cluster
poolNames := GetRolesToTest(oc, clientSet)
framework.Logf("Validating MCN properties for node(s) in pool(s) '%v'.", poolNames)

// When the cluster has machines in the "worker" MCP, use a custom MCP to test the update
if slices.Contains(poolNames, worker) {
ValidateMCNConditionTransitionsOnRebootlessUpdate(oc, clientSet, nodeDisruptionFixture, nodeDisruptionEmptyFixture, customMCFixture, infraMCPFixture)
} else { // When there are no machines in the "worker" MCP, test the update by applying a MC targeting the "master" MCP
ValidateMCNConditionTransitionsOnRebootlessUpdateMaster(oc, clientSet, nodeDisruptionFixture, nodeDisruptionEmptyFixture, masterMCFixture)
}
})

Expand Down Expand Up @@ -168,17 +178,17 @@ func ValidateMCNPropertiesCustomMCP(oc *exutil.CLI, fixture string) {
o.Expect(mcnErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Error validating MCN properties node in custom pool '%v'.", custom))
}

// `ValidateMCNConditionTransitions` checks that Conditions properly update on a node update
// Note that a custom MCP is created for this test to limit the number of upgrading nodes &
// decrease cleanup time.
func ValidateMCNConditionTransitionsOnRebootlessUpdate(oc *exutil.CLI, nodeDisruptionFixture string, nodeDisruptionEmptyFixture string, mcFixture string, mcpFixture string) {
// `ValidateMCNConditionTransitionsOnRebootlessUpdate` checks that the `Conditions` in an MCN
// properly update on a node update in a custom MCP. The steps of this function are:
// 1. Apply a node disruption policy
// 2. Create a custom MCP with one node
// 3. Apply a MC
// 4. Validate the MCN conditions transition as expected throughout the update
// 5. Clean up the test resources
func ValidateMCNConditionTransitionsOnRebootlessUpdate(oc *exutil.CLI, clientSet *machineconfigclient.Clientset, nodeDisruptionFixture string, nodeDisruptionEmptyFixture string, mcFixture string, mcpFixture string) {
poolName := custom
mcName := fmt.Sprintf("90-%v-testfile", poolName)

// Create client set for test
clientSet, clientErr := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
o.Expect(clientErr).NotTo(o.HaveOccurred(), "Error creating client set for test.")

// Grab a random worker node
workerNode := GetRandomNode(oc, worker)
o.Expect(workerNode.Name).NotTo(o.Equal(""), "Could not get a worker node.")
Expand Down Expand Up @@ -218,15 +228,22 @@ func ValidateMCNConditionTransitionsOnRebootlessUpdate(oc *exutil.CLI, nodeDisru
o.Expect(ConfirmUpdatedMCNStatus(clientSet, updatingNodeName)).Should(o.BeTrue(), "Error, all conditions must be 'False' when Updated=True.")
}

// `ValidateMCNConditionTransitionsSNO` checks that Conditions properly update on a node update
// in Single Node Openshift
func ValidateMCNConditionTransitionsOnRebootlessUpdateSNO(oc *exutil.CLI, nodeDisruptionFixture string, nodeDisruptionEmptyFixture string, mcFixture string) {
// `ValidateMCNConditionTransitionsOnRebootlessUpdateMaster` checks that the `Conditions` in an MCN
// properly update on a node update in the master MCP. The steps of this function are:
// 1. Apply a node disruption policy
// 2. Apply a MC
// 3. Get the updating node
// 4. Validate the MCN conditions transition as expected throughout the update
// 5. Clean up the test resources
func ValidateMCNConditionTransitionsOnRebootlessUpdateMaster(oc *exutil.CLI, clientSet *machineconfigclient.Clientset, nodeDisruptionFixture string, nodeDisruptionEmptyFixture string, mcFixture string) {
poolName := master
mcName := fmt.Sprintf("90-%v-testfile", poolName)

// Create client set for test
clientSet, clientErr := machineconfigclient.NewForConfig(oc.KubeFramework().ClientConfig())
o.Expect(clientErr).NotTo(o.HaveOccurred(), "Error creating client set for test.")
// Get the starting config version & machine count
mcp, mcpErr := clientSet.MachineconfigurationV1().MachineConfigPools().Get(context.TODO(), poolName, metav1.GetOptions{})
o.Expect(mcpErr).NotTo(o.HaveOccurred(), fmt.Sprintf("Could not get MCP '%v'; %v", poolName, mcpErr))
startingConfigVersion := mcp.Spec.Configuration.Name
machineCount := mcp.Status.MachineCount

// Remove node disruption policy on test completion or failure
defer func() {
Expand All @@ -247,16 +264,17 @@ func ValidateMCNConditionTransitionsOnRebootlessUpdateSNO(oc *exutil.CLI, nodeDi

// Wait for master MCP to be ready
time.Sleep(15 * time.Second) //wait to not catch the updated state before the deleted mc triggers an update
framework.Logf("Waiting for %v MCP to be updated with %v ready machines.", poolName, 1)
WaitForMCPToBeReady(oc, clientSet, poolName, 1)
framework.Logf("Waiting for %v MCP to be updated with %v ready machines.", poolName, machineCount)
WaitForMCPToBeReady(oc, clientSet, poolName, machineCount)
}()

// Apply MC targeting worker node
// Apply MC targeting master MCP
mcErr := oc.Run("apply").Args("-f", mcFixture).Execute()
o.Expect(mcErr).NotTo(o.HaveOccurred(), "Could not apply MachineConfig.")

// Get the updating node
updatingNode := GetUpdatingNodeSNO(oc, poolName)
updatingNode := GetUpdatingNode(oc, poolName, startingConfigVersion)
o.Expect(updatingNode).NotTo(o.BeNil(), "Could not get updating node.")
framework.Logf("Node '%v' is updating.", updatingNode.Name)

// Validate transition through conditions for MCN
Expand Down