diff --git a/test/extended/storage/csi_readonly_rootfs.go b/test/extended/storage/csi_readonly_rootfs.go new file mode 100644 index 000000000000..1b203ced89dc --- /dev/null +++ b/test/extended/storage/csi_readonly_rootfs.go @@ -0,0 +1,449 @@ +package storage + +import ( + "context" + "fmt" + "strings" + + g "github.com/onsi/ginkgo/v2" + o "github.com/onsi/gomega" + exutil "github.com/openshift/origin/test/extended/util" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + e2e "k8s.io/kubernetes/test/e2e/framework" +) + +// csiResourceCheck defines a check for CSI controller or node resources +type csiResourceCheck struct { + ResourceType ResourceType + Namespace string + Name string + Platform string +} + +// ResourceType defines the type of Kubernetes resource +type ResourceType string + +const ( + Deployment ResourceType = "Deployment" + DaemonSet ResourceType = "DaemonSet" +) + +var _ = g.Describe("[sig-storage][OCPFeature:CSIReadOnlyRootFilesystem][Jira:"Storage"] CSI Driver ReadOnly Root Filesystem", func() { + defer g.GinkgoRecover() + var ( + oc = exutil.NewCLI("csi-readonly-rootfs") + currentPlatform = e2e.TestContext.Provider + ) + + g.BeforeEach(func() { + isMicroShift, err := exutil.IsMicroShiftCluster(oc.AdminKubeClient()) + o.Expect(err).NotTo(o.HaveOccurred()) + if isMicroShift { + g.Skip("CSI ReadOnlyRootFilesystem tests are not supported on MicroShift") + } + + // Check to see if we have Storage enabled + isStorageEnabled, err := exutil.IsCapabilityEnabled(oc, configv1.ClusterVersionCapabilityStorage) + if err != nil || !isStorageEnabled { + g.Skip("skipping, this test is only expected to work with storage enabled clusters") + } + }) + + g.It("should verify CSI controller containers have readOnlyRootFilesystem set to true", func() { + controllerResources := []csiResourceCheck{ + // AWS EBS + { + ResourceType: Deployment, + Namespace: CSINamespace, + Name: "aws-ebs-csi-driver-controller", + Platform: "aws", + }, + // AWS EFS + { + ResourceType: Deployment, + Namespace: CSINamespace, + Name: "aws-efs-csi-driver-controller", + Platform: "aws", + }, + // Azure Disk + { + ResourceType: Deployment, + Namespace: CSINamespace, + Name: "azure-disk-csi-driver-controller", + Platform: "azure", + }, + // Azure File + { + ResourceType: Deployment, + Namespace: CSINamespace, + Name: "azure-file-csi-driver-controller", + Platform: "azure", + }, + // GCP PD + { + ResourceType: Deployment, + Namespace: CSINamespace, + Name: "gcp-pd-csi-driver-controller", + Platform: "gcp", + }, + // GCP Filestore + { + ResourceType: Deployment, + Namespace: CSINamespace, + Name: "gcp-filestore-csi-driver-controller", + Platform: "gcp", + }, + // vSphere + { + ResourceType: Deployment, + Namespace: CSINamespace, + Name: "vmware-vsphere-csi-driver-controller", + Platform: "vsphere", + }, + // IBM VPC Block + { + ResourceType: Deployment, + Namespace: CSINamespace, + Name: "ibm-vpc-block-csi-controller", + Platform: "ibmcloud", + }, + // OpenStack Cinder + { + ResourceType: Deployment, + Namespace: CSINamespace, + Name: "openstack-cinder-csi-driver-controller", + Platform: "openstack", + }, + // OpenStack Manila + { + ResourceType: Deployment, + Namespace: ManilaCSINamespace, + Name: "openstack-manila-csi-controllerplugin", + Platform: "openstack", + }, + // SMB + { + ResourceType: Deployment, + Namespace: CSINamespace, + Name: "smb-csi-driver-controller", + Platform: "all", + }, + } + + runReadOnlyRootFsChecks(oc, controllerResources, currentPlatform) + }) + + g.It("should verify CSI node containers have readOnlyRootFilesystem set to true", func() { + nodeResources := []csiResourceCheck{ + // AWS EBS + { + ResourceType: DaemonSet, + Namespace: CSINamespace, + Name: "aws-ebs-csi-driver-node", + Platform: "aws", + }, + // AWS EFS + { + ResourceType: DaemonSet, + Namespace: CSINamespace, + Name: "aws-efs-csi-driver-node", + Platform: "aws", + }, + // Azure Disk + { + ResourceType: DaemonSet, + Namespace: CSINamespace, + Name: "azure-disk-csi-driver-node", + Platform: "azure", + }, + // Azure File + { + ResourceType: DaemonSet, + Namespace: CSINamespace, + Name: "azure-file-csi-driver-node", + Platform: "azure", + }, + // GCP PD + { + ResourceType: DaemonSet, + Namespace: CSINamespace, + Name: "gcp-pd-csi-driver-node", + Platform: "gcp", + }, + // GCP Filestore + { + ResourceType: DaemonSet, + Namespace: CSINamespace, + Name: "gcp-filestore-csi-driver-node", + Platform: "gcp", + }, + // vSphere + { + ResourceType: DaemonSet, + Namespace: CSINamespace, + Name: "vmware-vsphere-csi-driver-node", + Platform: "vsphere", + }, + // IBM VPC Block + { + ResourceType: DaemonSet, + Namespace: CSINamespace, + Name: "ibm-vpc-block-csi-node", + Platform: "ibmcloud", + }, + // OpenStack Cinder + { + ResourceType: DaemonSet, + Namespace: CSINamespace, + Name: "openstack-cinder-csi-driver-node", + Platform: "openstack", + }, + // OpenStack Manila + { + ResourceType: DaemonSet, + Namespace: ManilaCSINamespace, + Name: "openstack-manila-csi-nodeplugin", + Platform: "openstack", + }, + // SMB + { + ResourceType: DaemonSet, + Namespace: CSINamespace, + Name: "smb-csi-driver-node", + Platform: "all", + }, + } + + runReadOnlyRootFsChecks(oc, nodeResources, currentPlatform) + }) + +// runReadOnlyRootFsChecks verifies that all containers in the resource have readOnlyRootFilesystem set +func runReadOnlyRootFsChecks(oc *exutil.CLI, resources []csiResourceCheck, currentPlatform string) { + results := []string{} + hasFail := false + + for _, resource := range resources { + // Skip if platform doesn't match + if resource.Platform != currentPlatform && resource.Platform != "all" { + results = append(results, fmt.Sprintf("[SKIP] %s %s/%s (platform mismatch: %s)", resource.ResourceType, resource.Namespace, resource.Name, resource.Platform)) + continue + } + + resourceName := fmt.Sprintf("%s %s/%s", resource.ResourceType, resource.Namespace, resource.Name) + + var podSpec *corev1.PodSpec + var found bool + + switch resource.ResourceType { + case Deployment: + deployment, err := oc.AdminKubeClient().AppsV1().Deployments(resource.Namespace).Get(context.TODO(), resource.Name, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + results = append(results, fmt.Sprintf("[SKIP] %s not found", resourceName)) + continue + } + g.Fail(fmt.Sprintf("Error fetching %s: %v", resourceName, err)) + } + podSpec = &deployment.Spec.Template.Spec + found = true + + case DaemonSet: + daemonset, err := oc.AdminKubeClient().AppsV1().DaemonSets(resource.Namespace).Get(context.TODO(), resource.Name, metav1.GetOptions{}) + if err != nil { + if errors.IsNotFound(err) { + results = append(results, fmt.Sprintf("[SKIP] %s not found", resourceName)) + continue + } + g.Fail(fmt.Sprintf("Error fetching %s: %v", resourceName, err)) + } + podSpec = &daemonset.Spec.Template.Spec + found = true + + default: + g.Fail(fmt.Sprintf("Unsupported resource type: %s", resource.ResourceType)) + } + + if !found { + continue + } + + // Check all containers and init containers + containersWithoutReadOnlyRootFs := []string{} + allContainers := append([]corev1.Container{}, podSpec.Containers...) + allContainers = append(allContainers, podSpec.InitContainers...) + + for _, container := range allContainers { + if container.SecurityContext == nil || container.SecurityContext.ReadOnlyRootFilesystem == nil || !*container.SecurityContext.ReadOnlyRootFilesystem { + containersWithoutReadOnlyRootFs = append(containersWithoutReadOnlyRootFs, container.Name) + } + } + + if len(containersWithoutReadOnlyRootFs) > 0 { + results = append(results, fmt.Sprintf("[FAIL] %s has containers without readOnlyRootFilesystem: %s", resourceName, strings.Join(containersWithoutReadOnlyRootFs, ", "))) + hasFail = true + } else { + results = append(results, fmt.Sprintf("[PASS] %s (all %d containers have readOnlyRootFilesystem: true)", resourceName, len(allContainers))) + } + } + + if hasFail { + summary := strings.Join(results, "\n") + g.Fail(fmt.Sprintf("Some CSI resources have containers without readOnlyRootFilesystem:\n\n%s\n", summary)) + } else { + e2e.Logf("All checked CSI resources have readOnlyRootFilesystem set correctly:\n%s", strings.Join(results, "\n")) + } +} + +// runPodReadinessChecks verifies that pods for the given resources are running and ready +func runPodReadinessChecks(oc *exutil.CLI, resources []csiResourceCheck, currentPlatform string) { + results := []string{} + hasFail := false + + for _, resource := range resources { + // Skip if platform doesn't match + if resource.Platform != "" && resource.Platform != currentPlatform && resource.Platform != "all" { + results = append(results, fmt.Sprintf("[SKIP] %s %s/%s (platform mismatch: %s)", resource.ResourceType, resource.Namespace, resource.Name, resource.Platform)) + continue + } + + resourceName := fmt.Sprintf("%s %s/%s", resource.ResourceType, resource.Namespace, resource.Name) + + var isReady bool + var readyReplicas, desiredReplicas int32 + var err error + + switch resource.ResourceType { + case Deployment: + isReady, readyReplicas, desiredReplicas, err = checkDeploymentReady(oc, resource.Namespace, resource.Name) + case DaemonSet: + isReady, readyReplicas, desiredReplicas, err = checkDaemonSetReady(oc, resource.Namespace, resource.Name) + default: + g.Fail(fmt.Sprintf("Unsupported resource type: %s", resource.ResourceType)) + } + + if err != nil { + if errors.IsNotFound(err) { + results = append(results, fmt.Sprintf("[SKIP] %s not found", resourceName)) + continue + } + g.Fail(fmt.Sprintf("Error checking readiness of %s: %v", resourceName, err)) + } + + if !isReady { + results = append(results, fmt.Sprintf("[FAIL] %s is not ready (ready: %d/%d)", resourceName, readyReplicas, desiredReplicas)) + hasFail = true + } else { + results = append(results, fmt.Sprintf("[PASS] %s is ready (%d/%d pods ready)", resourceName, readyReplicas, desiredReplicas)) + } + } + + if hasFail { + summary := strings.Join(results, "\n") + g.Fail(fmt.Sprintf("Some CSI resources are not ready:\n\n%s\n", summary)) + } else { + e2e.Logf("All checked CSI resources are ready and functioning:\n%s", strings.Join(results, "\n")) + } +} + +// checkDeploymentReady checks if a Deployment is ready +func checkDeploymentReady(oc *exutil.CLI, namespace, name string) (bool, int32, int32, error) { + deployment, err := oc.AdminKubeClient().AppsV1().Deployments(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return false, 0, 0, err + } + + desiredReplicas := int32(1) + if deployment.Spec.Replicas != nil { + desiredReplicas = *deployment.Spec.Replicas + } + + readyReplicas := deployment.Status.ReadyReplicas + isReady := readyReplicas == desiredReplicas && deployment.Status.UpdatedReplicas == desiredReplicas + + return isReady, readyReplicas, desiredReplicas, nil +} + +// checkDaemonSetReady checks if a DaemonSet is ready +func checkDaemonSetReady(oc *exutil.CLI, namespace, name string) (bool, int32, int32, error) { + daemonset, err := oc.AdminKubeClient().AppsV1().DaemonSets(namespace).Get(context.TODO(), name, metav1.GetOptions{}) + if err != nil { + return false, 0, 0, err + } + + desiredReplicas := daemonset.Status.DesiredNumberScheduled + readyReplicas := daemonset.Status.NumberReady + isReady := readyReplicas == desiredReplicas && daemonset.Status.NumberUnavailable == 0 + + return isReady, readyReplicas, desiredReplicas, nil +} + +// verifyPodsFunctioningCorrectly performs deeper validation that CSI driver pods are working +func verifyPodsFunctioningCorrectly(oc *exutil.CLI, resource csiResourceCheck) error { + ctx := context.TODO() + + // Get pods for the resource + var labelSelector string + switch resource.ResourceType { + case Deployment: + deployment, err := oc.AdminKubeClient().AppsV1().Deployments(resource.Namespace).Get(ctx, resource.Name, metav1.GetOptions{}) + if err != nil { + return err + } + labelSelector = metav1.FormatLabelSelector(deployment.Spec.Selector) + case DaemonSet: + daemonset, err := oc.AdminKubeClient().AppsV1().DaemonSets(resource.Namespace).Get(ctx, resource.Name, metav1.GetOptions{}) + if err != nil { + return err + } + labelSelector = metav1.FormatLabelSelector(daemonset.Spec.Selector) + } + + pods, err := oc.AdminKubeClient().CoreV1().Pods(resource.Namespace).List(ctx, metav1.ListOptions{ + LabelSelector: labelSelector, + }) + if err != nil { + return err + } + + if len(pods.Items) == 0 { + return fmt.Errorf("no pods found for resource %s/%s", resource.Namespace, resource.Name) + } + + // Check that all pods are running + for _, pod := range pods.Items { + if pod.Status.Phase != corev1.PodRunning { + return fmt.Errorf("pod %s is in phase %s, expected Running", pod.Name, pod.Status.Phase) + } + + // Check all containers are ready + for _, containerStatus := range pod.Status.ContainerStatuses { + if !containerStatus.Ready { + return fmt.Errorf("container %s in pod %s is not ready", containerStatus.Name, pod.Name) + } + } + } + + return nil +} + +// getResourcePodSpec retrieves the PodSpec from a Deployment or DaemonSet +func getResourcePodSpec(oc *exutil.CLI, resource csiResourceCheck) (*corev1.PodSpec, error) { + switch resource.ResourceType { + case Deployment: + deployment, err := oc.AdminKubeClient().AppsV1().Deployments(resource.Namespace).Get(context.TODO(), resource.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return &deployment.Spec.Template.Spec, nil + case DaemonSet: + daemonset, err := oc.AdminKubeClient().AppsV1().DaemonSets(resource.Namespace).Get(context.TODO(), resource.Name, metav1.GetOptions{}) + if err != nil { + return nil, err + } + return &daemonset.Spec.Template.Spec, nil + default: + return nil, fmt.Errorf("unsupported resource type: %s", resource.ResourceType) + } +}