K8SPSMDB-1262: Fix demand-backup-fs on openshift

e2e-tests/demand-backup-fs/conf/nfs-server.yaml → e2e-tests/demand-backup-fs/conf/nfs-server.yml

@@ -1,3 +1,21 @@
+---
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: nfs-server
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: system:openshift:scc:privileged
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: system:openshift:scc:privileged
+subjects:
+- kind: ServiceAccount
+  name: nfs-server
+---
 kind: PersistentVolumeClaim
 apiVersion: v1
 metadata:
@@ -8,7 +26,7 @@ spec:
     - ReadWriteOnce
   resources:
     requests:
-      storage: 50Gi
+      storage: 5Gi
 ---
 apiVersion: apps/v1
 kind: Deployment
@@ -24,9 +42,10 @@ spec:
       labels:
         app: nfs-server
     spec:
+      serviceAccountName: nfs-server
       containers:
       - name: nfs-server
-        image: k8s.gcr.io/volume-nfs:0.8
+        image: egegunes/nfs-server:0.9.0
         ports:
         - name: nfs
           containerPort: 2049

e2e-tests/demand-backup-fs/conf/rbac-oc.yml

@@ -0,0 +1,45 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: psmdb-pods
+---
+apiVersion: security.openshift.io/v1
+kind: SecurityContextConstraints
+metadata:
+  name: custom-psmdb-scc
+runAsUser:
+  type: RunAsAny
+seLinuxContext:
+  type: RunAsAny
+fsGroup:
+  type: RunAsAny
+supplementalGroups:
+  type: RunAsAny
+volumes:
+- "*"
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  name: use-custom-psmdb-scc
+rules:
+- apiGroups:
+  - security.openshift.io
+  resources:
+  - securitycontextconstraints
+  resourceNames:
+  - custom-psmdb-scc
+  verbs:
+  - use
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: custom-psmdb-scc
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: use-custom-psmdb-scc
+subjects:
+- kind: ServiceAccount
+  name: psmdb-pods

e2e-tests/demand-backup-fs/conf/some-name.yaml → e2e-tests/demand-backup-fs/conf/some-name-oc.yml

@@ -22,6 +22,9 @@ spec:
     volumeMounts:
     - mountPath: /mnt/nfs/
       name: backup-nfs
+    containerSecurityContext:
+      runAsNonRoot: true
+      runAsUser: 1001210000
   image: perconalab/percona-server-mongodb-operator:main-mongod8.0
   imagePullPolicy: Always
   pmm:
@@ -30,6 +33,7 @@ spec:
     serverHost: monitoring-service
   replsets:
   - name: rs0
+    size: 3
     expose:
       enabled: false
       type: ClusterIP
@@ -47,52 +51,21 @@ spec:
       nfs:
         server: "nfs-service.storage.svc.cluster.local"
         path: "/psmdb-some-name-rs0"
-    size: 3
     volumeSpec:
       persistentVolumeClaim:
         resources:
           requests:
             storage: 3Gi
+    serviceAccountName: psmdb-pods
+    containerSecurityContext:
+      runAsNonRoot: true
+      runAsUser: 1001210000
+    podSecurityContext:
+      runAsNonRoot: true
+      runAsUser: 1001210000
+      fsGroup: 1001210000
   secrets:
     users: some-users
-  sharding:
-    configsvrReplSet:
-      affinity:
-        antiAffinityTopologyKey: none
-      expose:
-        enabled: false
-        type: ClusterIP
-      podDisruptionBudget:
-        maxUnavailable: 1
-      resources:
-        limits:
-          cpu: 300m
-          memory: 0.5G
-        requests:
-          cpu: 300m
-          memory: 0.5G
-      size: 3
-      volumeSpec:
-        persistentVolumeClaim:
-          resources:
-            requests:
-              storage: 3Gi
-    enabled: false
-    mongos:
-      affinity:
-        antiAffinityTopologyKey: none
-      expose:
-        type: ClusterIP
-      podDisruptionBudget:
-        maxUnavailable: 1
-      resources:
-        limits:
-          cpu: 300m
-          memory: 0.5G
-        requests:
-          cpu: 300m
-          memory: 0.5G
-      size: 3
   updateStrategy: SmartUpdate
   upgradeOptions:
     apply: disabled

e2e-tests/demand-backup-fs/conf/some-name.yml

@@ -0,0 +1,63 @@
+apiVersion: psmdb.percona.com/v1
+kind: PerconaServerMongoDB
+metadata:
+  finalizers:
+  - percona.com/delete-psmdb-pods-in-order
+  name: some-name
+spec:
+  backup:
+    enabled: true
+    image: perconalab/percona-server-mongodb-operator:main-backup
+    pitr:
+      compressionLevel: 6
+      compressionType: gzip
+      enabled: true
+      oplogOnly: false
+      oplogSpanMin: 1
+    storages:
+      nfs:
+        filesystem:
+          path: /mnt/nfs/
+        type: filesystem
+    volumeMounts:
+    - mountPath: /mnt/nfs/
+      name: backup-nfs
+  image: perconalab/percona-server-mongodb-operator:main-mongod8.0
+  imagePullPolicy: Always
+  pmm:
+    enabled: false
+    image: perconalab/pmm-client:dev-latest
+    serverHost: monitoring-service
+  replsets:
+  - name: rs0
+    size: 3
+    expose:
+      enabled: false
+      type: ClusterIP
+    podDisruptionBudget:
+      maxUnavailable: 1
+    resources:
+      limits:
+        cpu: 300m
+        memory: 0.5G
+      requests:
+        cpu: 300m
+        memory: 0.5G
+    sidecarVolumes:
+    - name: backup-nfs
+      nfs:
+        server: "nfs-service.storage.svc.cluster.local"
+        path: "/psmdb-some-name-rs0"
+    volumeSpec:
+      persistentVolumeClaim:
+        resources:
+          requests:
+            storage: 3Gi
+  secrets:
+    users: some-users
+  updateStrategy: SmartUpdate
+  upgradeOptions:
+    apply: disabled
+    schedule: 0 2 * * *
+    setFCV: false
+    versionServiceEndpoint: https://check-dev.percona.com

e2e-tests/demand-backup-fs/run

@@ -6,14 +6,11 @@ test_dir=$(realpath $(dirname $0))
 . ${test_dir}/../functions
 set_debug
 
-if [[ "${OPENSHIFT}" ]]; then
-	echo "This test is not supported on OpenShift due to nfs privileged. See K8SPSMDB-1262"
-	exit 0
-fi
-
 deploy_nfs_server() {
+	local uid=$1
+
 	kubectl_bin create namespace storage
-	kubectl_bin apply -n storage -f ${test_dir}/conf/nfs-server.yaml
+	kubectl_bin apply -n storage -f ${test_dir}/conf/nfs-server.yml
 	sleep 5 # wait for NFS server pod to be created
 
 	local nfsPod=$(kubectl_bin get pod -n storage -l app=nfs-server -o jsonpath={.items[].metadata.name})
@@ -23,7 +20,7 @@ deploy_nfs_server() {
 	done
 
 	kubectl_bin exec -n storage ${nfsPod} -- mkdir /exports/psmdb-${cluster}-rs0
-	kubectl_bin exec -n storage ${nfsPod} -- chown 1001:1001 /exports/psmdb-${cluster}-rs0
+	kubectl_bin exec -n storage ${nfsPod} -- chown ${uid}:${uid} /exports/psmdb-${cluster}-rs0
 }
 
 run_recovery_check() {
@@ -60,21 +57,35 @@ if [[ $GKE != 1 ]]; then
 	kubectl_bin annotate storageclass ${sc} storageclass.kubernetes.io/is-default-class=true
 fi
 
+uid=1001
+if [[ "${OPENSHIFT}" ]]; then
+	uid=1001210000
+fi
+
 log "deploying NFS server"
-deploy_nfs_server
+deploy_nfs_server ${uid}
 
 log 'creating secrets and start client'
 kubectl_bin apply \
 	-f "${conf_dir}/secrets.yml" \
 	-f "${conf_dir}/client.yml"
 
+if [[ "${OPENSHIFT}" ]]; then
+	log "creating custom security context constraints to allow NFS volumes"
+	kubectl_bin apply -f ${test_dir}/conf/rbac-oc.yml
+fi
+
 log "creating PSMDB cluster ${cluster}"
 if [[ $GKE != 1 ]]; then
+	cluster_yaml=${test_dir}/conf/${cluster}.yml
+	if [[ "${OPENSHIFT}" ]]; then
+		cluster_yaml=${test_dir}/conf/${cluster}-oc.yml
+	fi
+
 	nfs_ip=$(kubectl_bin -n storage get svc nfs-service -o jsonpath={.spec.clusterIP})
-	sed "s/nfs-service.storage.svc.cluster.local/${nfs_ip}/g" ${test_dir}/conf/${cluster}.yaml \
-		| kubectl_bin apply -f -
+	sed "s/nfs-service.storage.svc.cluster.local/${nfs_ip}/g" ${cluster_yaml} | kubectl_bin apply -f -
 else
-	apply_cluster ${test_dir}/conf/${cluster}.yaml
+	apply_cluster ${test_dir}/conf/${cluster}.yml
 fi
 
 log 'wait for all 3 pods to start'