Merge pull request openshift#330 from openshift/dependabot/go_modules…

…/github.com/onsi/gomega-1.27.6

Bump github.com/onsi/gomega from 1.27.4 to 1.27.6

go.mod

@@ -8,7 +8,8 @@ require (
 	github.com/go-logr/logr v1.2.4
 	github.com/mitchellh/go-homedir v1.1.0
 	github.com/onsi/ginkgo v1.16.5
-	github.com/onsi/gomega v1.27.4
+	github.com/onsi/gomega v1.27.6
+	github.com/openshift/api v0.0.0-20230120195050-6ba31fa438f2
 	github.com/openshift/library-go v0.0.0-20230228181805-0899dfdba7d2
 	github.com/openshift/machine-config-operator v0.0.1-0.20200913004441-7eba765c69c9
 	github.com/pborman/uuid v1.2.1
@@ -70,7 +71,6 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/nbutton23/zxcvbn-go v0.0.0-20210217022336-fa2cb2858354 // indirect
 	github.com/nxadm/tail v1.4.8 // indirect
-	github.com/openshift/api v0.0.0-20230120195050-6ba31fa438f2 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	github.com/prometheus/common v0.41.0 // indirect
 	github.com/prometheus/procfs v0.9.0 // indirect

go.sum

@@ -262,8 +262,8 @@ github.com/go-openapi/validate v0.18.0/go.mod h1:Uh4HdOzKt19xGIGm1qHf/ofbX1YQ4Y+
 github.com/go-openapi/validate v0.19.2/go.mod h1:1tRCw7m3jtI8eNWEEliiAqUIcBztB2KDnRCRMUi7GTA=
 github.com/go-openapi/validate v0.19.5/go.mod h1:8DJv2CVJQ6kGNpFW6eV9N3JviE1C85nY1c2z52x1Gk4=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
-github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I=
 github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
+github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
 github.com/go-toolsmith/astcast v1.0.0/go.mod h1:mt2OdQTeAQcY4DQgPSArJjHCcOwlX+Wl/kwN+LbLGQ4=
 github.com/go-toolsmith/astcopy v1.0.0/go.mod h1:vrgyG+5Bxrnz4MZWPF+pI4R8h3qKRjjyvV/DSez4WVQ=
 github.com/go-toolsmith/astequal v0.0.0-20180903214952-dcb477bfacd6/go.mod h1:H+xSiq0+LtiDC11+h1G32h7Of5O3CYFJ99GVbS5lDKY=
@@ -525,14 +525,14 @@ github.com/onsi/ginkgo v1.11.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+
 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk=
 github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE=
 github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU=
-github.com/onsi/ginkgo/v2 v2.9.1 h1:zie5Ly042PD3bsCvsSOPvRnFwyo3rKe64TJlD6nu0mk=
+github.com/onsi/ginkgo/v2 v2.9.2 h1:BA2GMJOtfGAfagzYtrAlufIP0lq6QERkFmHLMLPwFSU=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.4.2/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.27.4 h1:Z2AnStgsdSayCMDiCU42qIz+HLqEPcgiOCXjAU/w+8E=
-github.com/onsi/gomega v1.27.4/go.mod h1:riYq/GJKh8hhoM01HN6Vmuy93AarCXCBGpvFDK3q3fQ=
+github.com/onsi/gomega v1.27.6 h1:ENqfyGeS5AX/rlXDd/ETokDz93u0YufY1Pgxuy/PvWE=
+github.com/onsi/gomega v1.27.6/go.mod h1:PIQNjfQwkP3aQAH7lf7j87O/5FiNr+ZR8+ipb+qQlhg=
 github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=

pkg/controller/fileintegrity/config_defaults.go

@@ -45,6 +45,7 @@ CONTENT_EX = sha512+ftype+p+u+g+n+acl+selinux+xattrs
 !/hostroot/etc/docker/certs.d
 !/hostroot/etc/selinux/targeted
 !/hostroot/etc/openvswitch/conf.db
+!/hostroot/etc/kubernetes/cni/net.d
 !/hostroot/etc/kubernetes/cni/net.d/*
 !/hostroot/etc/machine-config-daemon/currentconfig$
 !/hostroot/etc/pki/ca-trust/extracted/java/cacerts$

pkg/controller/status/status_controller.go

@@ -2,9 +2,10 @@ package status
 
 import (
 	"context"
+	"time"
+
 	"github.com/openshift/file-integrity-operator/pkg/apis/fileintegrity/v1alpha1"
 	"github.com/openshift/file-integrity-operator/pkg/controller/metrics"
-	"time"
 
 	"github.com/go-logr/logr"
 
@@ -171,7 +172,25 @@ func (r *StatusReconciler) mapActiveStatus(integrity *v1alpha1.FileIntegrity) (v
 		return v1alpha1.PhaseError, err
 	}
 
+	nodeList := corev1.NodeList{}
+	if err := r.client.List(context.TODO(), &nodeList, &client.ListOptions{}); err != nil {
+		return v1alpha1.PhaseError, err
+	}
+	nodeNameList := make(map[string]bool)
+	for _, node := range nodeList.Items {
+		nodeNameList[node.Name] = true
+	}
+
 	for _, nodeStatus := range nodeStatusList.Items {
+		// Check if the node is still there, and remove the node status if it's not.
+		// This is to handle the case where the node is deleted, but the node status is not.
+		if _, ok := nodeNameList[nodeStatus.Name]; !ok {
+			// If the node is not there, and the node status is success, we can just delete it.
+			if err := r.client.Delete(context.TODO(), &nodeStatus); err != nil && nodeStatus.LastResult.Condition == v1alpha1.NodeConditionSucceeded {
+				return v1alpha1.PhaseError, err
+			}
+			continue
+		}
 		if nodeStatus.LastResult.Condition == v1alpha1.NodeConditionErrored {
 			return v1alpha1.PhaseError, nil
 		}

tests/e2e/e2e_test.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/openshift/file-integrity-operator/pkg/apis/fileintegrity/v1alpha1"
 	fileintegrity2 "github.com/openshift/file-integrity-operator/pkg/controller/fileintegrity"
-
 	framework "github.com/openshift/file-integrity-operator/tests/framework"
 
 	"k8s.io/apimachinery/pkg/types"
@@ -918,3 +917,43 @@ func TestFileIntegrityAcceptsExpectedChange(t *testing.T) {
 	t.Log("Asserting that the FileIntegrity check is in a SUCCESS state after expected changes")
 	assertNodesConditionIsSuccess(t, f, testName, namespace, 5*time.Second, 10*time.Minute)
 }
+
+// This checks test for adding new node and remove a existing node to the cluster and making sure
+// the all the nodestatuses are in a success state, and the old nodestatus is removed for the removed node.
+func TestFileIntegrityNodeScaling(t *testing.T) {
+	f, testctx, namespace := setupTest(t)
+	testName := testIntegrityNamePrefix + "-nodescale"
+	setupFileIntegrity(t, f, testctx, testName, namespace)
+	defer testctx.Cleanup()
+	defer func() {
+		if err := cleanNodes(f, namespace); err != nil {
+			t.Fatal(err)
+		}
+		if err := resetBundleTestMetrics(f, namespace); err != nil {
+			t.Fatal(err)
+		}
+	}()
+	defer logContainerOutput(t, f, namespace, testName)
+	// wait to go active.
+	err := waitForScanStatus(t, f, namespace, testName, v1alpha1.PhaseActive)
+	if err != nil {
+		t.Errorf("Timeout waiting for scan status")
+	}
+
+	t.Log("Asserting that the FileIntegrity check is in a SUCCESS state after deploying it")
+	assertNodesConditionIsSuccess(t, f, testName, namespace, 2*time.Second, 5*time.Minute)
+
+	t.Log("Adding a new worker node to the cluster through the machineset")
+	scaledUpMachineSetName, newNodeName := scaleUpWorkerMachineSet(t, f, 2*time.Second, 10*time.Minute)
+	if newNodeName == "" || scaledUpMachineSetName == "" {
+		t.Fatal("Failed to scale up worker machineset")
+	}
+	assertSingleNodeConditionIsSuccess(t, f, testName, namespace, newNodeName, 2*time.Second, 5*time.Minute)
+
+	t.Log("Scale down the worker machineset")
+	removedNodeName := scaleDownWorkerMachineSet(t, f, scaledUpMachineSetName, 2*time.Second, 10*time.Minute)
+	if removedNodeName == "" {
+		t.Fatal("Failed to scale down worker machineset")
+	}
+	assertNodeStatusForRemovedNode(t, f, testName, namespace, removedNodeName, 2*time.Second, 5*time.Minute)
+}

tests/e2e/helpers.go

@@ -3,6 +3,7 @@ package e2e
 import (
 	"bufio"
 	"bytes"
+	"context"
 	goctx "context"
 	"encoding/json"
 	"fmt"
@@ -16,6 +17,7 @@ import (
 	"testing"
 	"time"
 
+	machinev1 "github.com/openshift/api/machine/v1beta1"
 	"github.com/openshift/file-integrity-operator/pkg/apis/fileintegrity/v1alpha1"
 	"github.com/openshift/file-integrity-operator/pkg/controller/metrics"
 	"github.com/pborman/uuid"
@@ -66,6 +68,7 @@ const (
 	metricsTestCRBName      = "fio-metrics-client"
 	metricsTestSAName       = "default"
 	metricsTestTokenName    = "metrics-token"
+	machineSetNamespace     = "openshift-machine-api"
 	compressionFileCmd      = "for i in `seq 1 10000`; do mktemp \"/hostroot/etc/addedbytest$i.XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX\"; done || true"
 )
 
@@ -115,6 +118,7 @@ CONTENT_EX = sha512+ftype+p+u+g+n+acl+selinux+xattrs
 !/hostroot/etc/docker/certs.d
 !/hostroot/etc/selinux/targeted
 !/hostroot/etc/openvswitch/conf.db
+!/hostroot/etc/kubernetes/cni/net.d
 !/hostroot/etc/kubernetes/cni/net.d/*
 !/hostroot/etc/machine-config-daemon/currentconfig$
 !/hostroot/etc/pki/ca-trust/extracted/java/cacerts$
@@ -1005,6 +1009,159 @@ func assertNodeOKStatusEvents(t *testing.T, f *framework.Framework, namespace st
 	}
 }
 
+func scaleUpWorkerMachineSet(t *testing.T, f *framework.Framework, interval, timeout time.Duration) (string, string) {
+	// Add a new worker node to the cluster through the machineset
+	// Get the machineset
+	machineSets := &machinev1.MachineSetList{}
+	err := f.Client.List(context.TODO(), machineSets, &client.ListOptions{
+		Namespace: machineSetNamespace})
+	if err != nil {
+		t.Error(err)
+	}
+	if len(machineSets.Items) == 0 {
+		t.Error("No machinesets found")
+	}
+	machineSetName := ""
+	for _, ms := range machineSets.Items {
+		if ms.Spec.Replicas != nil && *ms.Spec.Replicas > 0 {
+			t.Logf("Found machineset %s with %d replicas", ms.Name, *ms.Spec.Replicas)
+			machineSetName = ms.Name
+			break
+		}
+	}
+
+	// Add one more replica to one of the machinesets
+	machineSet := &machinev1.MachineSet{}
+	err = f.Client.Get(context.TODO(), types.NamespacedName{Name: machineSetName, Namespace: machineSetNamespace}, machineSet)
+	if err != nil {
+		t.Error(err)
+	}
+	t.Logf("Scaling up machineset %s", machineSetName)
+
+	replicas := *machineSet.Spec.Replicas + 1
+	machineSet.Spec.Replicas = &replicas
+	err = f.Client.Update(context.TODO(), machineSet)
+	if err != nil {
+		t.Error(err)
+	}
+	t.Logf("Waiting for scaling up machineset %s", machineSetName)
+	provisionningMachineName := ""
+	err = wait.Poll(interval, timeout, func() (bool, error) {
+		err = f.Client.Get(context.TODO(), types.NamespacedName{Name: machineSetName, Namespace: machineSetNamespace}, machineSet)
+		if err != nil {
+			t.Error(err)
+		}
+		// get name of the new machine
+		if provisionningMachineName == "" {
+			machines := &machinev1.MachineList{}
+			err = f.Client.List(context.TODO(), machines, &client.ListOptions{
+				Namespace: machineSetNamespace})
+			if err != nil {
+				t.Error(err)
+			}
+			for _, machine := range machines.Items {
+				if *machine.Status.Phase == "Provisioning" {
+					provisionningMachineName = machine.Name
+					break
+				}
+			}
+		}
+		if &machineSet.Status.Replicas == &machineSet.Status.ReadyReplicas {
+			t.Logf("Machineset %s scaled up", machineSetName)
+			return true, nil
+		}
+		t.Logf("Waiting for machineset %s to scale up, current ready replicas: %d of %d", machineSetName, &machineSet.Status.ReadyReplicas, &machineSet.Status.Replicas)
+		return false, nil
+	})
+	if err != nil {
+		t.Error(err)
+	}
+
+	// get the new node name
+	newNodeName := ""
+	machine := &machinev1.Machine{}
+	err = f.Client.Get(context.TODO(), types.NamespacedName{Name: provisionningMachineName, Namespace: machineSetNamespace}, machine)
+	if err != nil {
+		t.Error(err)
+	}
+	newNodeName = machine.Status.NodeRef.Name
+	t.Logf("New node name is %s", newNodeName)
+
+	return machineSetName, newNodeName
+}
+
+func scaleDownWorkerMachineSet(t *testing.T, f *framework.Framework, machineSetName string, interval, timeout time.Duration) string {
+	// Remove the worker node from the cluster through the machineset
+	// Get the machineset
+	machineSet := &machinev1.MachineSet{}
+	err := f.Client.Get(context.TODO(), types.NamespacedName{Name: machineSetName, Namespace: machineSetNamespace}, machineSet)
+	if err != nil {
+		t.Error(err)
+	}
+
+	// Remove one replica from the machineset
+	t.Logf("Scaling down machineset %s", machineSetName)
+	replicas := *machineSet.Spec.Replicas - 1
+	machineSet.Spec.Replicas = &replicas
+	err = f.Client.Update(context.TODO(), machineSet)
+	if err != nil {
+		t.Error(err)
+	}
+
+	deletedNodeName := ""
+
+	t.Logf("Waiting for scaling down machineset %s", machineSetName)
+	err = wait.Poll(interval, timeout, func() (bool, error) {
+		err = f.Client.Get(context.TODO(), types.NamespacedName{Name: machineSetName, Namespace: machineSetNamespace}, machineSet)
+		if err != nil {
+			t.Error(err)
+		}
+		if &machineSet.Status.Replicas == &machineSet.Status.ReadyReplicas {
+			t.Logf("Machineset %s scaled down", machineSetName)
+			return true, nil
+		}
+		t.Logf("Waiting for machineset %s to scale down, current ready replicas: %d of %d", machineSetName, &machineSet.Status.ReadyReplicas, &machineSet.Status.Replicas)
+		if deletedNodeName == "" {
+			// Get the node that was deleted
+			machineList := &machinev1.MachineList{}
+			err = f.Client.List(context.TODO(), machineList, &client.ListOptions{
+				Namespace: machineSetNamespace})
+			if err != nil {
+				t.Error(err)
+			}
+			if len(machineList.Items) == 0 {
+				t.Error("No machines found")
+			}
+			for _, machine := range machineList.Items {
+				if machine.DeletionTimestamp != nil {
+					deletedNodeName = machine.Status.NodeRef.Name
+					t.Logf("Found deleted node %s", deletedNodeName)
+					break
+				}
+			}
+		}
+		return false, nil
+	})
+	if err != nil {
+		t.Error(err)
+	}
+	return deletedNodeName
+}
+
+func assertNodeStatusForRemovedNode(t *testing.T, f *framework.Framework, integrityName, namespace, deletedNodeName string, interval, timeout time.Duration) {
+	nodestatus := &v1alpha1.FileIntegrityNodeStatus{}
+	err := f.Client.Get(goctx.TODO(), types.NamespacedName{Name: integrityName + "-" + deletedNodeName, Namespace: namespace}, nodestatus)
+	if err != nil {
+		if kerr.IsNotFound(err) {
+			t.Logf("Node status for node %s not found, as expected", deletedNodeName)
+		} else {
+			t.Errorf("error getting node status for node %s: %v", deletedNodeName, err)
+		}
+	} else {
+		t.Errorf("Node status for node %s found, but should not have been", deletedNodeName)
+	}
+}
+
 func assertNodesConditionIsSuccess(t *testing.T, f *framework.Framework, integrityName, namespace string, interval, timeout time.Duration) {
 	var lastErr error
 	type nodeStatus struct {