OCPBUGS-18131: Block upgrade to 4.13 via admin ack

pkg/operator/starter.go

@@ -35,12 +35,13 @@ const (
 	operandName                       = "vmware-vsphere-csi-driver"
 	secretName                        = "vmware-vsphere-cloud-credentials"
 	envVMWareVsphereDriverSyncerImage = "VMWARE_VSPHERE_SYNCER_IMAGE"
+	managedConfigNamespace            = "openshift-config-managed"
 )
 
 func RunOperator(ctx context.Context, controllerConfig *controllercmd.ControllerContext) error {
 	// Create core clientset and informers
 	kubeClient := kubeclient.NewForConfigOrDie(rest.AddUserAgent(controllerConfig.KubeConfig, operatorName))
-	kubeInformersForNamespaces := v1helpers.NewKubeInformersForNamespaces(kubeClient, utils.DefaultNamespace, cloudConfigNamespace, "")
+	kubeInformersForNamespaces := v1helpers.NewKubeInformersForNamespaces(kubeClient, utils.DefaultNamespace, cloudConfigNamespace, managedConfigNamespace, "")
 	secretInformer := kubeInformersForNamespaces.InformersFor(utils.DefaultNamespace).Core().V1().Secrets()
 	configMapInformer := kubeInformersForNamespaces.InformersFor(utils.DefaultNamespace).Core().V1().ConfigMaps()
 	nodeInformer := kubeInformersForNamespaces.InformersFor("").Core().V1().Nodes()

pkg/operator/testlib/testlib.go

@@ -12,7 +12,6 @@ import (
 	operatorinformers "github.com/openshift/client-go/operator/informers/externalversions"
 	"github.com/openshift/library-go/pkg/operator/v1helpers"
 	"github.com/openshift/vmware-vsphere-csi-driver-operator/pkg/operator/utils"
-	"github.com/openshift/vmware-vsphere-csi-driver-operator/pkg/operator/vspherecontroller/checks"
 	"gopkg.in/gcfg.v1"
 	v1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
@@ -27,10 +26,12 @@ import (
 var f embed.FS
 
 const (
-	cloudConfigNamespace = "openshift-config"
-	infraGlobalName      = "cluster"
-	secretName           = "vmware-vsphere-cloud-credentials"
-	defaultNamespace     = "openshift-cluster-csi-drivers"
+	cloudConfigNamespace   = "openshift-config"
+	infraGlobalName        = "cluster"
+	storageOperatorName    = "cluster"
+	secretName             = "vmware-vsphere-cloud-credentials"
+	defaultNamespace       = "openshift-cluster-csi-drivers"
+	managedConfigNamespace = "openshift-config-managed"
 )
 
 // fakeInstance is a fake CSI driver instance that also fullfils the OperatorClient interface
@@ -72,7 +73,7 @@ func StartFakeInformer(clients *utils.APIClient, stopCh <-chan struct{}) {
 func NewFakeClients(coreObjects []runtime.Object, operatorObject *FakeDriverInstance, configObject runtime.Object) *utils.APIClient {
 	dynamicClient := dynamicfake.NewSimpleDynamicClient(runtime.NewScheme())
 	kubeClient := fakecore.NewSimpleClientset(coreObjects...)
-	kubeInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient, defaultNamespace, cloudConfigNamespace, "")
+	kubeInformers := v1helpers.NewKubeInformersForNamespaces(kubeClient, defaultNamespace, cloudConfigNamespace, managedConfigNamespace, "")
 	nodeInformer := kubeInformers.InformersFor("").Core().V1().Nodes()
 	secretInformer := kubeInformers.InformersFor(defaultNamespace).Core().V1().Secrets()
 
@@ -132,8 +133,9 @@ func AddInitialObjects(objects []runtime.Object, clients *utils.APIClient) error
 	for _, obj := range objects {
 		switch obj.(type) {
 		case *v1.ConfigMap:
-			configMapInformer := clients.KubeInformers.InformersFor(cloudConfigNamespace).Core().V1().ConfigMaps().Informer()
-			configMapInformer.GetStore().Add(obj)
+			configMap := obj.(*v1.ConfigMap)
+			configMapInformer := clients.KubeInformers.InformersFor(configMap.Namespace).Core().V1().ConfigMaps().Informer()
+			configMapInformer.GetStore().Add(configMap)
 		case *v1.Secret:
 			secretInformer := clients.SecretInformer.Informer()
 			secretInformer.GetStore().Add(obj)
@@ -149,6 +151,12 @@ func AddInitialObjects(objects []runtime.Object, clients *utils.APIClient) error
 		case *opv1.ClusterCSIDriver:
 			clusterCSIDriverInformer := clients.ClusterCSIDriverInformer.Informer()
 			clusterCSIDriverInformer.GetStore().Add(obj)
+		case *opv1.Storage:
+			storageInformer := clients.OCPOperatorInformers.Operator().V1().Storages().Informer()
+			storageInformer.GetStore().Add(obj)
+		case *v1.PersistentVolume:
+			pvInformer := clients.KubeInformers.InformersFor("").Core().V1().PersistentVolumes().Informer()
+			pvInformer.GetStore().Add(obj)
 		default:
 			return fmt.Errorf("Unknown initalObject type: %+v", obj)
 		}
@@ -219,6 +227,38 @@ func GetCSIDriver(withOCPAnnotation bool) *storagev1.CSIDriver {
 	return driver
 }
 
+func GetIntreePV(pvName string) *v1.PersistentVolume {
+	return &v1.PersistentVolume{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: pvName,
+		},
+		Spec: v1.PersistentVolumeSpec{
+			PersistentVolumeSource: v1.PersistentVolumeSource{
+				VsphereVolume: &v1.VsphereVirtualDiskVolumeSource{
+					VolumePath: "foobar/baz.vmdk",
+				},
+			},
+		},
+	}
+}
+
+func GetNodeWithInlinePV(nodeName string, hasIntreePV bool) *v1.Node {
+	node := &v1.Node{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: nodeName,
+		},
+		Spec:   v1.NodeSpec{},
+		Status: v1.NodeStatus{},
+	}
+
+	if hasIntreePV {
+		node.Status.VolumesInUse = []v1.UniqueVolumeName{
+			"kubernetes.io/vsphere-volume/foobar",
+		}
+	}
+	return node
+}
+
 func GetCSINode() *storagev1.CSINode {
 	return &storagev1.CSINode{
 		ObjectMeta: metav1.ObjectMeta{
@@ -283,6 +323,26 @@ datacenters = "DC0"
 	}
 }
 
+func GetAdminGateConfigMap(withAckKey bool) *v1.ConfigMap {
+	cMap := &v1.ConfigMap{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "admin-gates",
+			Namespace: managedConfigNamespace,
+		},
+		Data: map[string]string{},
+	}
+
+	if withAckKey {
+		cMap.Data["ack-4.12-kube-1.26-api-removals-in-4.13"] = `
+			Kubernetes 1.26 and therefore OpenShift
+			4.13 remove several APIs which require admin consideration. Please see the knowledge
+			article https://access.redhat.com/articles/6958394 for details and instructions.`
+		cMap.Data["ack-4.13-kube-127-vsphere-migration-in-4.14"] = "remove this to upgrade"
+	}
+
+	return cMap
+}
+
 func GetSecret() *v1.Secret {
 	return &v1.Secret{
 		ObjectMeta: metav1.ObjectMeta{
@@ -295,10 +355,3 @@ func GetSecret() *v1.Secret {
 		},
 	}
 }
-
-func GetTestClusterResult(statusType checks.CheckStatusType) checks.ClusterCheckResult {
-	return checks.ClusterCheckResult{
-		CheckError:  fmt.Errorf("some error"),
-		CheckStatus: statusType,
-	}
-}

pkg/operator/vspherecontroller/checks/api_interface.go

@@ -14,6 +14,7 @@ type KubeAPIInterface interface {
 	ListNodes() ([]*v1.Node, error)
 	GetCSIDriver(name string) (*storagev1.CSIDriver, error)
 	ListCSINodes() ([]*storagev1.CSINode, error)
+	ListPersistentVolumes() ([]*v1.PersistentVolume, error)
 	GetStorageClass(name string) (*storagev1.StorageClass, error)
 	GetInfrastructure() *ocpv1.Infrastructure
 }
@@ -24,6 +25,7 @@ type KubeAPIInterfaceImpl struct {
 	CSINodeLister      storagelister.CSINodeLister
 	CSIDriverLister    storagelister.CSIDriverLister
 	StorageClassLister storagelister.StorageClassLister
+	PvLister           corelister.PersistentVolumeLister
 }
 
 func (k *KubeAPIInterfaceImpl) ListNodes() ([]*v1.Node, error) {
@@ -45,3 +47,7 @@ func (k *KubeAPIInterfaceImpl) GetStorageClass(name string) (*storagev1.StorageC
 func (k *KubeAPIInterfaceImpl) GetInfrastructure() *ocpv1.Infrastructure {
 	return k.Infrastructure
 }
+
+func (k *KubeAPIInterfaceImpl) ListPersistentVolumes() ([]*v1.PersistentVolume, error) {
+	return k.PvLister.List(labels.Everything())
+}

pkg/operator/vspherecontroller/checks/check_error.go

@@ -2,12 +2,15 @@ package checks
 
 import (
 	"fmt"
+	"strings"
 
 	"github.com/openshift/vmware-vsphere-csi-driver-operator/pkg/operator/utils"
+	corev1 "k8s.io/api/core/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/klog/v2"
 )
 
+// CheckStatusType stores exact error that was observed during performing various checks
 type CheckStatusType string
 
 const (
@@ -18,27 +21,20 @@ const (
 	CheckStatusDeprecatedVCenter       CheckStatusType = "check_deprecated_vcenter"
 	CheckStatusDeprecatedHWVersion     CheckStatusType = "check_deprecated_hw_version"
 	CheckStatusDeprecatedESXIVersion   CheckStatusType = "check_deprecated_esxi_version"
+	CheckStatusBuggyMigrationPlatform  CheckStatusType = "buggy_vsphere_migration_version"
 	CheckStatusVcenterAPIError         CheckStatusType = "vcenter_api_error"
 	CheckStatusGenericError            CheckStatusType = "generic_error"
 )
 
-type ClusterCheckStatus string
-
-const (
-	ClusterCheckAllGood                   ClusterCheckStatus = "pass"
-	ClusterCheckBlockUpgradeDriverInstall ClusterCheckStatus = "installation_blocked"
-	ClusterCheckBlockUpgrade              ClusterCheckStatus = "upgrades_blocked"
-	ClusterCheckUpgradeStateUnknown       ClusterCheckStatus = "upgrades_unknown"
-	ClusterCheckDegrade                   ClusterCheckStatus = "degraded"
-)
-
+// CheckAction stores what a single failing check would do.
 type CheckAction int
 
 // Ordered by severity, Pass must be 0 (for struct initialization).
 const (
 	CheckActionPass                      = iota
+	CheckActionRequiresAdminAck          // blocks upgrades via admin-ack
 	CheckActionBlockUpgrade              // Only block upgrade
-	CheckActionBlockUpgradeDriverInstall // Block voth upgrade and driver install
+	CheckActionBlockUpgradeDriverInstall // Block both upgrade and driver install
 	CheckActionBlockUpgradeOrDegrade     // Degrade if the driver is installed, block upgrade otherwise
 	CheckActionDegrade
 )
@@ -55,11 +51,30 @@ func ActionToString(a CheckAction) string {
 		return "BlockUpgradeOrDegrade"
 	case CheckActionDegrade:
 		return "Degrade"
+	case CheckActionRequiresAdminAck:
+		return "UpgradeRequiresAdminAck"
 	default:
 		return "Unknown"
 	}
 }
 
+// ClusterCheckStatus stores what is the status of overall cluster after checking everything and then applying
+// additional logic which may not be included in checks themselves.
+type ClusterCheckStatus string
+
+const (
+	ClusterCheckAllGood                    ClusterCheckStatus = "pass"
+	ClusterCheckBlockUpgradeDriverInstall  ClusterCheckStatus = "installation_blocked"
+	ClusterCheckBlockUpgrade               ClusterCheckStatus = "upgrades_blocked"
+	ClusterCheckUpgradeStateUnknown        ClusterCheckStatus = "upgrades_unknown"
+	ClusterCheckUpgradesBlockedViaAdminAck ClusterCheckStatus = "upgrades_blocked_via_admin_ack"
+	ClusterCheckDegrade                    ClusterCheckStatus = "degraded"
+)
+
+const (
+	inTreePluginName = "kubernetes.io/vsphere-volume"
+)
+
 type ClusterCheckResult struct {
 	CheckError  error
 	CheckStatus CheckStatusType
@@ -98,6 +113,16 @@ func makeDeprecatedEnvironmentError(statusType CheckStatusType, reason error) Cl
 	return checkResult
 }
 
+func makeBuggyEnvironmentError(statusType CheckStatusType, reason error) ClusterCheckResult {
+	checkResult := ClusterCheckResult{
+		CheckStatus: statusType,
+		CheckError:  reason,
+		Action:      CheckActionRequiresAdminAck,
+		Reason:      reason.Error(),
+	}
+	return checkResult
+}
+
 func MakeGenericVCenterAPIError(reason error) ClusterCheckResult {
 	return ClusterCheckResult{
 		CheckStatus: CheckStatusVcenterAPIError,
@@ -125,6 +150,8 @@ func MakeClusterUnupgradeableError(checkStatus CheckStatusType, reason error) Cl
 	}
 }
 
+// CheckClusterStatus uses results from all the checks we ran and applies additional logic to determine
+// overall CheckClusterStatus
 func CheckClusterStatus(result ClusterCheckResult, apiDependencies KubeAPIInterface) (ClusterCheckStatus, ClusterCheckResult) {
 	switch result.Action {
 	case CheckActionDegrade:
@@ -162,11 +189,62 @@ func CheckClusterStatus(result ClusterCheckResult, apiDependencies KubeAPIInterf
 
 	case CheckActionBlockUpgrade:
 		return ClusterCheckBlockUpgrade, result
-
+	case CheckActionRequiresAdminAck:
+		return checkForIntreePluginUse(result, apiDependencies)
 	case CheckActionBlockUpgradeDriverInstall:
 		return ClusterCheckBlockUpgradeDriverInstall, result
-
 	default:
 		return ClusterCheckAllGood, result
 	}
 }
+
+// returns false is migration is enabled in the cluster.
+// returns true if migration is not enabled in the cluster and cluster is using
+// in-tree vSphere volumes.
+func checkForIntreePluginUse(result ClusterCheckResult, apiDependencies KubeAPIInterface) (ClusterCheckStatus, ClusterCheckResult) {
+	allGoodCheckResult := MakeClusterCheckResultPass()
+
+	pvs, err := apiDependencies.ListPersistentVolumes()
+	if err != nil {
+		reason := fmt.Errorf("vsphere csi driver installed failed with %s, unable to list pvs: %v", result.Reason, err)
+		return ClusterCheckDegrade, MakeClusterDegradedError(CheckStatusOpenshiftAPIError, reason)
+	}
+
+	usingvSphereVolumes := false
+
+	for _, pv := range pvs {
+		if pv.Spec.VsphereVolume != nil {
+			klog.V(2).Infof("found vSphere in-tree persistent volumes in the cluster")
+			usingvSphereVolumes = true
+			break
+		}
+	}
+
+	if usingvSphereVolumes {
+		return ClusterCheckUpgradesBlockedViaAdminAck, allGoodCheckResult
+	}
+
+	nodes, err := apiDependencies.ListNodes()
+	if err != nil {
+		reason := fmt.Errorf("csi driver installed failed with %s, unable to list nodes: %v", result.Reason, err)
+		return ClusterCheckDegrade, MakeClusterDegradedError(CheckStatusOpenshiftAPIError, reason)
+	}
+
+	if checkInlineIntreeVolumeUse(nodes) {
+		klog.V(2).Infof("found vSphere in-line persistent volumes in the cluster")
+		return ClusterCheckUpgradesBlockedViaAdminAck, allGoodCheckResult
+	}
+	return ClusterCheckAllGood, allGoodCheckResult
+}
+
+func checkInlineIntreeVolumeUse(nodes []*corev1.Node) bool {
+	for _, node := range nodes {
+		volumesInUse := node.Status.VolumesInUse
+		for _, volume := range volumesInUse {
+			if strings.HasPrefix(string(volume), inTreePluginName) {
+				return true
+			}
+		}
+	}
+	return false
+}