diff --git a/test/extended/security/fips.go b/test/extended/security/fips.go new file mode 100644 index 000000000000..ea98b5c3dd74 --- /dev/null +++ b/test/extended/security/fips.go @@ -0,0 +1,87 @@ +package security + +import ( + "context" + "fmt" + "strconv" + "strings" + + g "github.com/onsi/ginkgo" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + clientcorev1 "k8s.io/client-go/kubernetes/typed/core/v1" + "sigs.k8s.io/yaml" +) + +const ( + installConfigName = "cluster-config-v1" + fipsFile = "/proc/sys/crypto/fips_enabled" +) + +// installConfig The subset of openshift-install's InstallConfig we parse for this test +type installConfig struct { + FIPS bool `json:"fips,omitempty"` +} + +func installConfigFromCluster(client clientcorev1.ConfigMapsGetter) (*installConfig, error) { + cm, err := client.ConfigMaps("kube-system").Get(context.Background(), installConfigName, metav1.GetOptions{}) + if err != nil { + return nil, err + } + data, ok := cm.Data["install-config"] + if !ok { + return nil, fmt.Errorf("No install-config found in kube-system/%s", installConfigName) + } + config := &installConfig{} + if err := yaml.Unmarshal([]byte(data), config); err != nil { + return nil, err + } + return config, nil +} + +func validateFIPSOnNode(oc *exutil.CLI, fipsExpected bool, node *corev1.Node) error { + command := []string{"cat", fipsFile} + out, err := exutil.ExecCommandOnMachineConfigDaemon(oc.AdminKubeClient(), oc, node, command) + if err != nil { + return err + } + nodeFips, err := strconv.ParseBool(strings.TrimSuffix(string(out), "\n")) + if err != nil { + return fmt.Errorf("Error parsing %s on node %s: %v", fipsFile, node.Name, err) + } + if nodeFips != fipsExpected { + return fmt.Errorf("Expected FIPS state %v, found %v", fipsExpected, nodeFips) + } + return nil +} + +var _ = g.Describe("[sig-arch] [Conformance] FIPS", func() { + defer g.GinkgoRecover() + oc := exutil.NewCLI("fips") + + g.It("TestFIPS", func() { + clusterAdminKubeClientset := oc.AdminKubeClient() + installConfig, err := installConfigFromCluster(clusterAdminKubeClientset.CoreV1()) + o.Expect(err).NotTo(o.HaveOccurred()) + + // fetch one control plane and one worker, and validate FIPS state on it + masterNodes, err := clusterAdminKubeClientset.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{ + LabelSelector: "node-role.kubernetes.io/master", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + masterNode := &masterNodes.Items[0] + err = validateFIPSOnNode(oc, installConfig.FIPS, masterNode) + o.Expect(err).NotTo(o.HaveOccurred()) + workerNodes, err := clusterAdminKubeClientset.CoreV1().Nodes().List(context.Background(), metav1.ListOptions{ + LabelSelector: "node-role.kubernetes.io/worker", + }) + o.Expect(err).NotTo(o.HaveOccurred()) + if len(workerNodes.Items) > 0 { + workerNode := &workerNodes.Items[0] + err = validateFIPSOnNode(oc, installConfig.FIPS, workerNode) + o.Expect(err).NotTo(o.HaveOccurred()) + } + }) +}) diff --git a/test/extended/topology_manager/utils.go b/test/extended/topology_manager/utils.go index fc99a23270d9..b44f329b5fe4 100644 --- a/test/extended/topology_manager/utils.go +++ b/test/extended/topology_manager/utils.go @@ -15,7 +15,6 @@ import ( exutil "github.com/openshift/origin/test/extended/util" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/fields" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" @@ -40,9 +39,6 @@ const ( // no default for sriovNetworkNamespace: use the e2e test framework default defaultSriovNetwork = "sriov-network" defaultIPFamily = "v4" - - namespaceMachineConfigOperator = "openshift-machine-config-operator" - containerMachineConfigDaemon = "machine-config-daemon" ) const ( @@ -118,23 +114,6 @@ func getNodeByRole(c clientset.Interface, role string) ([]corev1.Node, error) { return nodes.Items, nil } -func getMachineConfigDaemonByNode(c clientset.Interface, node *corev1.Node) (*corev1.Pod, error) { - listOptions := metav1.ListOptions{ - FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": node.Name}).String(), - LabelSelector: labels.SelectorFromSet(labels.Set{"k8s-app": "machine-config-daemon"}).String(), - } - - mcds, err := c.CoreV1().Pods(namespaceMachineConfigOperator).List(context.Background(), listOptions) - if err != nil { - return nil, err - } - - if len(mcds.Items) < 1 { - return nil, fmt.Errorf("failed to get machine-config-daemon pod for the node %q", node.Name) - } - return &mcds.Items[0], nil -} - const ( sysFSNumaNodePath = "/sys/devices/system/node" ) @@ -196,27 +175,10 @@ func makeAllowedCpuListEnv(out string) string { return fmt.Sprintf("CPULIST_ALLOWED=%s\n", strings.TrimSpace(pair[1])) } -// execCommandOnMachineConfigDaemon returns the output of the command execution on the machine-config-daemon pod that runs on the specified node -func execCommandOnMachineConfigDaemon(c clientset.Interface, oc *exutil.CLI, node *corev1.Node, command []string) (string, error) { - mcd, err := getMachineConfigDaemonByNode(c, node) - if err != nil { - return "", err - } - - initialArgs := []string{ - "-n", namespaceMachineConfigOperator, - "-c", containerMachineConfigDaemon, - "--request-timeout", "30", - mcd.Name, - } - args := append(initialArgs, command...) - return oc.AsAdmin().Run("rsh").Args(args...).Output() -} - // getKubeletConfig returns KubeletConfiguration loaded from the node /etc/kubernetes/kubelet.conf func getKubeletConfig(c clientset.Interface, oc *exutil.CLI, node *corev1.Node) (*kubeletconfigv1beta1.KubeletConfiguration, error) { command := []string{"cat", path.Join("/rootfs", filePathKubeletConfig)} - kubeletData, err := execCommandOnMachineConfigDaemon(c, oc, node, command) + kubeletData, err := exutil.ExecCommandOnMachineConfigDaemon(c, oc, node, command) if err != nil { return nil, err } @@ -249,7 +211,7 @@ func parseSysfsNodeOnline(data string) (int, error) { func getNumaNodeCountFromNode(c clientset.Interface, oc *exutil.CLI, node *corev1.Node) (int, error) { command := []string{"cat", "/sys/devices/system/node/online"} - out, err := execCommandOnMachineConfigDaemon(c, oc, node, command) + out, err := exutil.ExecCommandOnMachineConfigDaemon(c, oc, node, command) if err != nil { return 0, err } diff --git a/test/extended/util/annotate/generated/zz_generated.annotations.go b/test/extended/util/annotate/generated/zz_generated.annotations.go index 5164adf5e901..9d66eea87487 100644 --- a/test/extended/util/annotate/generated/zz_generated.annotations.go +++ b/test/extended/util/annotate/generated/zz_generated.annotations.go @@ -791,6 +791,8 @@ var annotations = map[string]string{ "[Top Level] [sig-arch] Managed cluster should should expose cluster services outside the cluster": "should expose cluster services outside the cluster [Suite:openshift/conformance/parallel]", + "[Top Level] [sig-arch] [Conformance] FIPS TestFIPS": "TestFIPS [Suite:openshift/conformance/parallel/minimal]", + "[Top Level] [sig-arch] ocp payload should be based on existing source [Serial] olm version should contain the source commit id": "[Serial] olm version should contain the source commit id [Suite:openshift/conformance/serial]", "[Top Level] [sig-arch][Early] Managed cluster should start all core operators": "start all core operators [Suite:openshift/conformance/parallel]", diff --git a/test/extended/util/pods.go b/test/extended/util/pods.go index bd0fa08c8d09..f4c383411cd3 100644 --- a/test/extended/util/pods.go +++ b/test/extended/util/pods.go @@ -2,18 +2,28 @@ package util import ( "context" + "fmt" "strings" "time" + corev1 "k8s.io/api/core/v1" v1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/fields" + "k8s.io/apimachinery/pkg/labels" kutilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" + clientset "k8s.io/client-go/kubernetes" e2e "k8s.io/kubernetes/test/e2e/framework" "k8s.io/kubernetes/test/e2e/framework/pod" ) +const ( + namespaceMachineConfigOperator = "openshift-machine-config-operator" + containerMachineConfigDaemon = "machine-config-daemon" +) + // WaitForNoPodsAvailable waits until there are no pods in the // given namespace func WaitForNoPodsAvailable(oc *CLI) error { @@ -65,3 +75,38 @@ func CreateUbiExecPodOrFail(client kubernetes.Interface, ns, generateName string } }) } + +// getMachineConfigDaemonByNode finds the privileged daemonset from the Machine Config Operator +func getMachineConfigDaemonByNode(c clientset.Interface, node *corev1.Node) (*corev1.Pod, error) { + listOptions := metav1.ListOptions{ + FieldSelector: fields.SelectorFromSet(fields.Set{"spec.nodeName": node.Name}).String(), + LabelSelector: labels.SelectorFromSet(labels.Set{"k8s-app": "machine-config-daemon"}).String(), + } + + mcds, err := c.CoreV1().Pods(namespaceMachineConfigOperator).List(context.Background(), listOptions) + if err != nil { + return nil, err + } + + if len(mcds.Items) < 1 { + return nil, fmt.Errorf("failed to get machine-config-daemon pod for the node %q", node.Name) + } + return &mcds.Items[0], nil +} + +// ExecCommandOnMachineConfigDaemon returns the output of the command execution on the machine-config-daemon pod that runs on the specified node +func ExecCommandOnMachineConfigDaemon(c clientset.Interface, oc *CLI, node *corev1.Node, command []string) (string, error) { + mcd, err := getMachineConfigDaemonByNode(c, node) + if err != nil { + return "", err + } + + initialArgs := []string{ + "-n", namespaceMachineConfigOperator, + "-c", containerMachineConfigDaemon, + "--request-timeout", "30", + mcd.Name, + } + args := append(initialArgs, command...) + return oc.AsAdmin().Run("rsh").Args(args...).Output() +}