[release-4.3] Bug 1799447: Swiftless OpenStack

manifests/00-crd.yaml

@@ -135,6 +135,10 @@ spec:
                     value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/'
                   type: object
               type: object
+            rolloutStrategy:
+              description: rolloutStrategy defines rollout strategy for the image
+                registry deployment.
+              type: string
             routes:
               description: Routes defines additional external facing routes which
                 should be created for the registry.

pkg/apis/imageregistry/v1/types.go

@@ -15,6 +15,9 @@ const (
 	// ImageRegistryName is the name of the image-registry workload resource (deployment)
 	ImageRegistryName = "image-registry"
 
+	// PVCImageRegistryName is the default name of the claim provisioned for PVC backend
+	PVCImageRegistryName = "image-registry-storage"
+
 	// ImageRegistryResourceName is the name of the image registry config instance
 	ImageRegistryResourceName = "cluster"
 
@@ -155,6 +158,10 @@ type ImageRegistrySpec struct {
 	// Tolerations defines the tolerations for the registry pod.
 	// +optional
 	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+	// rolloutStrategy defines rollout strategy for the image registry
+	// deployment.
+	// +optional
+	RolloutStrategy string `json:"rolloutStrategy,omitempty"`
 }
 
 type ImageRegistryStatus struct {

pkg/operator/bootstrap.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/klog"
 
@@ -14,6 +15,14 @@ import (
 	regopset "github.com/openshift/cluster-image-registry-operator/pkg/generated/clientset/versioned/typed/imageregistry/v1"
 	"github.com/openshift/cluster-image-registry-operator/pkg/parameters"
 	"github.com/openshift/cluster-image-registry-operator/pkg/storage"
+	"github.com/openshift/cluster-image-registry-operator/pkg/storage/pvc"
+	"github.com/openshift/cluster-image-registry-operator/pkg/storage/swift"
+	"github.com/openshift/cluster-image-registry-operator/pkg/storage/util"
+
+	configapiv1 "github.com/openshift/api/config/v1"
+
+	appsapi "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 )
 
 // randomSecretSize is the number of random bytes to generate
@@ -60,6 +69,32 @@ func (c *Controller) Bootstrap() error {
 		mgmtState = operatorapi.Removed
 	}
 
+	infra, err := util.GetInfrastructure(c.listers)
+	if err != nil {
+		return err
+	}
+
+	rolloutStrategy := appsapi.RollingUpdateDeploymentStrategyType
+
+	// If Swift service is not available for OpenStack, we have to start using
+	// Cinder with RWO PVC backend. It means that we need to create an RWO claim
+	// and set the rollout strategy to Recreate.
+	switch infra.Status.PlatformStatus.Type {
+	case configapiv1.OpenStackPlatformType:
+		isSwiftEnabled, err := swift.IsSwiftEnabled(c.listers)
+		if err != nil {
+			return err
+		}
+		if !isSwiftEnabled {
+			err = c.createPVC(corev1.ReadWriteOnce)
+			if err != nil {
+				return err
+			}
+
+			rolloutStrategy = appsapi.RecreateDeploymentStrategyType
+		}
+	}
+
 	cr = &imageregistryv1.Config{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:       imageregistryv1.ImageRegistryResourceName,
@@ -72,6 +107,7 @@ func (c *Controller) Bootstrap() error {
 			Storage:         imageregistryv1.ImageRegistryConfigStorage{},
 			Replicas:        1,
 			HTTPSecret:      fmt.Sprintf("%x", string(secretBytes[:])),
+			RolloutStrategy: string(rolloutStrategy),
 		},
 		Status: imageregistryv1.ImageRegistryStatus{},
 	}
@@ -92,3 +128,46 @@ func (c *Controller) Bootstrap() error {
 
 	return nil
 }
+
+func (c *Controller) createPVC(accessMode corev1.PersistentVolumeAccessMode) error {
+	claimName := imageregistryv1.PVCImageRegistryName
+
+	// Check that the claim does not exist before creating it
+	claim, err := c.clients.Core.PersistentVolumeClaims(imageregistryv1.ImageRegistryOperatorNamespace).Get(claimName, metav1.GetOptions{})
+	if err != nil {
+		if !errors.IsNotFound(err) {
+			return err
+		}
+
+		// "standard" is the default StorageClass name, that was provisioned by the cloud provider
+		storageClassName := "standard"
+
+		claim = &corev1.PersistentVolumeClaim{
+			ObjectMeta: metav1.ObjectMeta{
+				Name:      claimName,
+				Namespace: imageregistryv1.ImageRegistryOperatorNamespace,
+				Annotations: map[string]string{
+					pvc.PVCOwnerAnnotation: "true",
+				},
+			},
+			Spec: corev1.PersistentVolumeClaimSpec{
+				AccessModes: []corev1.PersistentVolumeAccessMode{
+					accessMode,
+				},
+				Resources: corev1.ResourceRequirements{
+					Requests: corev1.ResourceList{
+						corev1.ResourceStorage: resource.MustParse("100Gi"),
+					},
+				},
+				StorageClassName: &storageClassName,
+			},
+		}
+
+		_, err = c.clients.Core.PersistentVolumeClaims(imageregistryv1.ImageRegistryOperatorNamespace).Create(claim)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}

pkg/resource/deployment.go

@@ -80,7 +80,13 @@ func (gd *generatorDeployment) expected() (runtime.Object, error) {
 	if podTemplateSpec.Annotations == nil {
 		podTemplateSpec.Annotations = map[string]string{}
 	}
+
 	podTemplateSpec.Annotations[parameters.ChecksumOperatorDepsAnnotation] = depsChecksum
+	// Strategy defaults to RollingUpdate
+	strategy := gd.cr.Spec.RolloutStrategy
+	if strategy == "" {
+		strategy = string(appsapi.RollingUpdateDeploymentStrategyType)
+	}
 
 	deploy := &appsapi.Deployment{
 		ObjectMeta: metav1.ObjectMeta{
@@ -97,6 +103,9 @@ func (gd *generatorDeployment) expected() (runtime.Object, error) {
 				MatchLabels: gd.params.Deployment.Labels,
 			},
 			Template: podTemplateSpec,
+			Strategy: appsapi.DeploymentStrategy{
+				Type: appsapi.DeploymentStrategyType(strategy),
+			},
 		},
 	}
 

pkg/storage/pvc/pvc.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"reflect"
 
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -21,7 +22,7 @@ import (
 const (
 	rootDirectory      = "/registry"
 	randomSecretSize   = 32
-	pvcOwnerAnnotation = "imageregistry.openshift.io"
+	PVCOwnerAnnotation = "imageregistry.openshift.io"
 )
 
 type driver struct {
@@ -110,13 +111,37 @@ func (d *driver) checkPVC(cr *imageregistryv1.Config, claim *corev1.PersistentVo
 		}
 	}
 
+	// Check what access modes are available.
+
+	// We allow using RWO PV backend, but it has some limitations:
+	// 1. Image registry rollout strategy must be set to Recreate (default is RollingUpdate).
+	// 2. It's not possible to use more than 1 replica of the image registry.
+
+	// RWX backends are accepted with no additional conditions.
+	rwoModeEnabled := false
+
 	for _, claimMode := range claim.Spec.AccessModes {
 		if claimMode == corev1.ReadWriteMany {
 			return nil
 		}
+		if claimMode == corev1.ReadWriteOnce {
+			rwoModeEnabled = true
+		}
+	}
+
+	if rwoModeEnabled {
+		if cr.Spec.Replicas > 1 {
+			return fmt.Errorf("cannot use %s access mode with more than one replica of the image registry", corev1.ReadWriteOnce)
+		}
+
+		if cr.Spec.RolloutStrategy != string(appsv1.RecreateDeploymentStrategyType) {
+			return fmt.Errorf("cannot use %s access mode with %s rollout strategy", corev1.ReadWriteOnce, cr.Spec.RolloutStrategy)
+		}
+
+		return nil
 	}
 
-	return fmt.Errorf("PVC %s does not contain the necessary access mode (%s)", d.Config.Claim, corev1.ReadWriteMany)
+	return fmt.Errorf("PVC %s does not contain the necessary access modes: %s or %s", d.Config.Claim, corev1.ReadWriteMany, corev1.ReadWriteOnce)
 }
 
 func (d *driver) createPVC(cr *imageregistryv1.Config) (*corev1.PersistentVolumeClaim, error) {
@@ -125,7 +150,7 @@ func (d *driver) createPVC(cr *imageregistryv1.Config) (*corev1.PersistentVolume
 			Name:      d.Config.Claim,
 			Namespace: d.Namespace,
 			Annotations: map[string]string{
-				pvcOwnerAnnotation: "true",
+				PVCOwnerAnnotation: "true",
 			},
 		},
 		Spec: corev1.PersistentVolumeClaimSpec{
@@ -151,7 +176,7 @@ func (d *driver) CreateStorage(cr *imageregistryv1.Config) error {
 	)
 
 	if len(d.Config.Claim) == 0 {
-		d.Config.Claim = fmt.Sprintf("%s-storage", imageregistryv1.ImageRegistryName)
+		d.Config.Claim = imageregistryv1.PVCImageRegistryName
 
 		// If there is no name and there is no PVC, then we will create a PVC.
 		// If PVC is there and it was created by us, then just start using it again.
@@ -206,7 +231,7 @@ func (d *driver) RemoveStorage(cr *imageregistryv1.Config) (retriable bool, err
 }
 
 func pvcIsCreatedByOperator(claim *corev1.PersistentVolumeClaim) (exist bool) {
-	_, exist = claim.Annotations[pvcOwnerAnnotation]
+	_, exist = claim.Annotations[PVCOwnerAnnotation]
 	return
 }
 

pkg/storage/storage.go

@@ -154,7 +154,17 @@ func GetPlatformStorage(listers *regopclient.Listers) (imageregistryv1.ImageRegi
 	case configapiv1.GCPPlatformType:
 		cfg.GCS = &imageregistryv1.ImageRegistryConfigStorageGCS{}
 	case configapiv1.OpenStackPlatformType:
-		cfg.Swift = &imageregistryv1.ImageRegistryConfigStorageSwift{}
+		isSwiftEnabled, err := swift.IsSwiftEnabled(listers)
+		if err != nil {
+			return imageregistryv1.ImageRegistryConfigStorage{}, err
+		}
+		if !isSwiftEnabled {
+			cfg.PVC = &imageregistryv1.ImageRegistryConfigStoragePVC{
+				Claim: imageregistryv1.PVCImageRegistryName,
+			}
+		} else {
+			cfg.Swift = &imageregistryv1.ImageRegistryConfigStorageSwift{}
+		}
 
 	// Unknown platforms or LibVirt: we configure image registry using
 	// EmptyDir storage.

pkg/storage/swift/swift.go

@@ -48,6 +48,20 @@ func replaceEmpty(a string, b string) string {
 	return a
 }
 
+// IsSwiftEnabled checks if Swift service is available for OpenStack platform
+func IsSwiftEnabled(listers *regopclient.Listers) (bool, error) {
+	driver := NewDriver(&imageregistryv1.ImageRegistryConfigStorageSwift{}, listers)
+	_, err := driver.getSwiftClient()
+	if err != nil {
+		// ErrEndpointNotFound means that Swift is not available
+		if _, ok := err.(*gophercloud.ErrEndpointNotFound); ok {
+			return false, nil
+		}
+		return false, err
+	}
+	return true, nil
+}
+
 // GetConfig reads credentials
 func GetConfig(listers *regopclient.Listers) (*Swift, error) {
 	cfg := &Swift{}
@@ -125,7 +139,7 @@ func GetConfig(listers *regopclient.Listers) (*Swift, error) {
 }
 
 // getSwiftClient returns a client that allows to interact with the OpenStack Swift service
-func (d *driver) getSwiftClient(cr *imageregistryv1.Config) (*gophercloud.ServiceClient, error) {
+func (d *driver) getSwiftClient() (*gophercloud.ServiceClient, error) {
 	cfg, err := GetConfig(d.Listers)
 	if err != nil {
 		return nil, err
@@ -312,7 +326,7 @@ func (d *driver) containerExists(client *gophercloud.ServiceClient, containerNam
 }
 
 func (d *driver) StorageExists(cr *imageregistryv1.Config) (bool, error) {
-	client, err := d.getSwiftClient(cr)
+	client, err := d.getSwiftClient()
 	if err != nil {
 		util.UpdateCondition(cr, imageregistryv1.StorageExists, operatorapi.ConditionUnknown, "Could not connect to registry storage", err.Error())
 		return false, err
@@ -342,7 +356,7 @@ func (d *driver) StorageChanged(cr *imageregistryv1.Config) bool {
 }
 
 func (d *driver) CreateStorage(cr *imageregistryv1.Config) error {
-	client, err := d.getSwiftClient(cr)
+	client, err := d.getSwiftClient()
 	if err != nil {
 		util.UpdateCondition(cr, imageregistryv1.StorageExists, operatorapi.ConditionFalse, err.Error(), err.Error())
 		return err
@@ -420,7 +434,7 @@ func (d *driver) RemoveStorage(cr *imageregistryv1.Config) (bool, error) {
 		return false, nil
 	}
 
-	client, err := d.getSwiftClient(cr)
+	client, err := d.getSwiftClient()
 	if err != nil {
 		return false, err
 	}

pkg/storage/swift/swift_test.go

@@ -5,6 +5,7 @@ import (
 	"net/http"
 	"testing"
 
+	"github.com/gophercloud/gophercloud"
 	th "github.com/gophercloud/gophercloud/testhelper"
 
 	corev1 "k8s.io/api/core/v1"
@@ -605,3 +606,33 @@ func TestSwiftEndpointTypeObjectStore(t *testing.T) {
 	th.AssertNoErr(t, err)
 	th.AssertEquals(t, true, res)
 }
+
+func TestSwiftIsNotAvailable(t *testing.T) {
+	th.SetupHTTP()
+	defer th.TeardownHTTP()
+	// Swift endpoint is not registered
+	handleAuthentication(t, "INVALID")
+
+	th.Mux.HandleFunc("/"+container, func(w http.ResponseWriter, r *http.Request) {
+		th.TestMethod(t, r, "HEAD")
+		th.TestHeader(t, r, "Accept", "application/json")
+		w.Header().Set("Accept-Ranges", "bytes")
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.Header().Set("Date", "Wed, 17 Aug 2016 19:25:43 GMT")
+		w.Header().Set("X-Container-Bytes-Used", "100")
+		w.Header().Set("X-Container-Object-Count", "4")
+		w.Header().Set("X-Container-Read", "test")
+		w.Header().Set("X-Container-Write", "test2,user4")
+		w.Header().Set("X-Timestamp", "1471298837.95721")
+		w.Header().Set("X-Trans-Id", "tx554ed59667a64c61866f1-0057b4ba37")
+		w.Header().Set("X-Storage-Policy", "test_policy")
+		w.WriteHeader(http.StatusNoContent)
+	})
+
+	d, _ := mockConfig(false, th.Endpoint()+"v3", MockUPISecretNamespaceLister{})
+
+	_, err := d.getSwiftClient()
+	// if Swift endpoint is not registered, getSwiftClient should return ErrEndpointNotFound
+	_, ok := err.(*gophercloud.ErrEndpointNotFound)
+	th.AssertEquals(t, true, ok)
+}