diff --git a/manifests/00-crd.yaml b/manifests/00-crd.yaml index 43228a5278..d42d68f05a 100644 --- a/manifests/00-crd.yaml +++ b/manifests/00-crd.yaml @@ -135,6 +135,10 @@ spec: value. More info: https://kubernetes.io/docs/concepts/configuration/manage-compute-resources-container/' type: object type: object + rolloutStrategy: + description: rolloutStrategy defines rollout strategy for the image + registry deployment. + type: string routes: description: Routes defines additional external facing routes which should be created for the registry. diff --git a/pkg/apis/imageregistry/v1/types.go b/pkg/apis/imageregistry/v1/types.go index 8819c900ed..72b22373c1 100644 --- a/pkg/apis/imageregistry/v1/types.go +++ b/pkg/apis/imageregistry/v1/types.go @@ -15,6 +15,9 @@ const ( // ImageRegistryName is the name of the image-registry workload resource (deployment) ImageRegistryName = "image-registry" + // PVCImageRegistryName is the default name of the claim provisioned for PVC backend + PVCImageRegistryName = "image-registry-storage" + // ImageRegistryResourceName is the name of the image registry config instance ImageRegistryResourceName = "cluster" @@ -155,6 +158,10 @@ type ImageRegistrySpec struct { // Tolerations defines the tolerations for the registry pod. // +optional Tolerations []corev1.Toleration `json:"tolerations,omitempty"` + // rolloutStrategy defines rollout strategy for the image registry + // deployment. + // +optional + RolloutStrategy string `json:"rolloutStrategy,omitempty"` } type ImageRegistryStatus struct { diff --git a/pkg/operator/bootstrap.go b/pkg/operator/bootstrap.go index 26a78fd984..7e94012f82 100644 --- a/pkg/operator/bootstrap.go +++ b/pkg/operator/bootstrap.go @@ -5,6 +5,7 @@ import ( "fmt" "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/klog" @@ -14,6 +15,14 @@ import ( regopset "github.com/openshift/cluster-image-registry-operator/pkg/generated/clientset/versioned/typed/imageregistry/v1" "github.com/openshift/cluster-image-registry-operator/pkg/parameters" "github.com/openshift/cluster-image-registry-operator/pkg/storage" + "github.com/openshift/cluster-image-registry-operator/pkg/storage/pvc" + "github.com/openshift/cluster-image-registry-operator/pkg/storage/swift" + "github.com/openshift/cluster-image-registry-operator/pkg/storage/util" + + configapiv1 "github.com/openshift/api/config/v1" + + appsapi "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" ) // randomSecretSize is the number of random bytes to generate @@ -60,6 +69,32 @@ func (c *Controller) Bootstrap() error { mgmtState = operatorapi.Removed } + infra, err := util.GetInfrastructure(c.listers) + if err != nil { + return err + } + + rolloutStrategy := appsapi.RollingUpdateDeploymentStrategyType + + // If Swift service is not available for OpenStack, we have to start using + // Cinder with RWO PVC backend. It means that we need to create an RWO claim + // and set the rollout strategy to Recreate. + switch infra.Status.PlatformStatus.Type { + case configapiv1.OpenStackPlatformType: + isSwiftEnabled, err := swift.IsSwiftEnabled(c.listers) + if err != nil { + return err + } + if !isSwiftEnabled { + err = c.createPVC(corev1.ReadWriteOnce) + if err != nil { + return err + } + + rolloutStrategy = appsapi.RecreateDeploymentStrategyType + } + } + cr = &imageregistryv1.Config{ ObjectMeta: metav1.ObjectMeta{ Name: imageregistryv1.ImageRegistryResourceName, @@ -72,6 +107,7 @@ func (c *Controller) Bootstrap() error { Storage: imageregistryv1.ImageRegistryConfigStorage{}, Replicas: 1, HTTPSecret: fmt.Sprintf("%x", string(secretBytes[:])), + RolloutStrategy: string(rolloutStrategy), }, Status: imageregistryv1.ImageRegistryStatus{}, } @@ -92,3 +128,46 @@ func (c *Controller) Bootstrap() error { return nil } + +func (c *Controller) createPVC(accessMode corev1.PersistentVolumeAccessMode) error { + claimName := imageregistryv1.PVCImageRegistryName + + // Check that the claim does not exist before creating it + claim, err := c.clients.Core.PersistentVolumeClaims(imageregistryv1.ImageRegistryOperatorNamespace).Get(claimName, metav1.GetOptions{}) + if err != nil { + if !errors.IsNotFound(err) { + return err + } + + // "standard" is the default StorageClass name, that was provisioned by the cloud provider + storageClassName := "standard" + + claim = &corev1.PersistentVolumeClaim{ + ObjectMeta: metav1.ObjectMeta{ + Name: claimName, + Namespace: imageregistryv1.ImageRegistryOperatorNamespace, + Annotations: map[string]string{ + pvc.PVCOwnerAnnotation: "true", + }, + }, + Spec: corev1.PersistentVolumeClaimSpec{ + AccessModes: []corev1.PersistentVolumeAccessMode{ + accessMode, + }, + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceStorage: resource.MustParse("100Gi"), + }, + }, + StorageClassName: &storageClassName, + }, + } + + _, err = c.clients.Core.PersistentVolumeClaims(imageregistryv1.ImageRegistryOperatorNamespace).Create(claim) + if err != nil { + return err + } + } + + return nil +} diff --git a/pkg/resource/deployment.go b/pkg/resource/deployment.go index 8884f83792..9fd964670e 100644 --- a/pkg/resource/deployment.go +++ b/pkg/resource/deployment.go @@ -80,7 +80,13 @@ func (gd *generatorDeployment) expected() (runtime.Object, error) { if podTemplateSpec.Annotations == nil { podTemplateSpec.Annotations = map[string]string{} } + podTemplateSpec.Annotations[parameters.ChecksumOperatorDepsAnnotation] = depsChecksum + // Strategy defaults to RollingUpdate + strategy := gd.cr.Spec.RolloutStrategy + if strategy == "" { + strategy = string(appsapi.RollingUpdateDeploymentStrategyType) + } deploy := &appsapi.Deployment{ ObjectMeta: metav1.ObjectMeta{ @@ -97,6 +103,9 @@ func (gd *generatorDeployment) expected() (runtime.Object, error) { MatchLabels: gd.params.Deployment.Labels, }, Template: podTemplateSpec, + Strategy: appsapi.DeploymentStrategy{ + Type: appsapi.DeploymentStrategyType(strategy), + }, }, } diff --git a/pkg/storage/pvc/pvc.go b/pkg/storage/pvc/pvc.go index 5ec56a456d..eccd39f32f 100644 --- a/pkg/storage/pvc/pvc.go +++ b/pkg/storage/pvc/pvc.go @@ -4,6 +4,7 @@ import ( "fmt" "reflect" + appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -21,7 +22,7 @@ import ( const ( rootDirectory = "/registry" randomSecretSize = 32 - pvcOwnerAnnotation = "imageregistry.openshift.io" + PVCOwnerAnnotation = "imageregistry.openshift.io" ) type driver struct { @@ -110,13 +111,37 @@ func (d *driver) checkPVC(cr *imageregistryv1.Config, claim *corev1.PersistentVo } } + // Check what access modes are available. + + // We allow using RWO PV backend, but it has some limitations: + // 1. Image registry rollout strategy must be set to Recreate (default is RollingUpdate). + // 2. It's not possible to use more than 1 replica of the image registry. + + // RWX backends are accepted with no additional conditions. + rwoModeEnabled := false + for _, claimMode := range claim.Spec.AccessModes { if claimMode == corev1.ReadWriteMany { return nil } + if claimMode == corev1.ReadWriteOnce { + rwoModeEnabled = true + } + } + + if rwoModeEnabled { + if cr.Spec.Replicas > 1 { + return fmt.Errorf("cannot use %s access mode with more than one replica of the image registry", corev1.ReadWriteOnce) + } + + if cr.Spec.RolloutStrategy != string(appsv1.RecreateDeploymentStrategyType) { + return fmt.Errorf("cannot use %s access mode with %s rollout strategy", corev1.ReadWriteOnce, cr.Spec.RolloutStrategy) + } + + return nil } - return fmt.Errorf("PVC %s does not contain the necessary access mode (%s)", d.Config.Claim, corev1.ReadWriteMany) + return fmt.Errorf("PVC %s does not contain the necessary access modes: %s or %s", d.Config.Claim, corev1.ReadWriteMany, corev1.ReadWriteOnce) } func (d *driver) createPVC(cr *imageregistryv1.Config) (*corev1.PersistentVolumeClaim, error) { @@ -125,7 +150,7 @@ func (d *driver) createPVC(cr *imageregistryv1.Config) (*corev1.PersistentVolume Name: d.Config.Claim, Namespace: d.Namespace, Annotations: map[string]string{ - pvcOwnerAnnotation: "true", + PVCOwnerAnnotation: "true", }, }, Spec: corev1.PersistentVolumeClaimSpec{ @@ -151,7 +176,7 @@ func (d *driver) CreateStorage(cr *imageregistryv1.Config) error { ) if len(d.Config.Claim) == 0 { - d.Config.Claim = fmt.Sprintf("%s-storage", imageregistryv1.ImageRegistryName) + d.Config.Claim = imageregistryv1.PVCImageRegistryName // If there is no name and there is no PVC, then we will create a PVC. // If PVC is there and it was created by us, then just start using it again. @@ -206,7 +231,7 @@ func (d *driver) RemoveStorage(cr *imageregistryv1.Config) (retriable bool, err } func pvcIsCreatedByOperator(claim *corev1.PersistentVolumeClaim) (exist bool) { - _, exist = claim.Annotations[pvcOwnerAnnotation] + _, exist = claim.Annotations[PVCOwnerAnnotation] return } diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index 02a14e3613..fdc5351e03 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -154,7 +154,17 @@ func GetPlatformStorage(listers *regopclient.Listers) (imageregistryv1.ImageRegi case configapiv1.GCPPlatformType: cfg.GCS = &imageregistryv1.ImageRegistryConfigStorageGCS{} case configapiv1.OpenStackPlatformType: - cfg.Swift = &imageregistryv1.ImageRegistryConfigStorageSwift{} + isSwiftEnabled, err := swift.IsSwiftEnabled(listers) + if err != nil { + return imageregistryv1.ImageRegistryConfigStorage{}, err + } + if !isSwiftEnabled { + cfg.PVC = &imageregistryv1.ImageRegistryConfigStoragePVC{ + Claim: imageregistryv1.PVCImageRegistryName, + } + } else { + cfg.Swift = &imageregistryv1.ImageRegistryConfigStorageSwift{} + } // Unknown platforms or LibVirt: we configure image registry using // EmptyDir storage. diff --git a/pkg/storage/swift/swift.go b/pkg/storage/swift/swift.go index fb8045fed3..032ed022a0 100644 --- a/pkg/storage/swift/swift.go +++ b/pkg/storage/swift/swift.go @@ -48,6 +48,20 @@ func replaceEmpty(a string, b string) string { return a } +// IsSwiftEnabled checks if Swift service is available for OpenStack platform +func IsSwiftEnabled(listers *regopclient.Listers) (bool, error) { + driver := NewDriver(&imageregistryv1.ImageRegistryConfigStorageSwift{}, listers) + _, err := driver.getSwiftClient() + if err != nil { + // ErrEndpointNotFound means that Swift is not available + if _, ok := err.(*gophercloud.ErrEndpointNotFound); ok { + return false, nil + } + return false, err + } + return true, nil +} + // GetConfig reads credentials func GetConfig(listers *regopclient.Listers) (*Swift, error) { cfg := &Swift{} @@ -125,7 +139,7 @@ func GetConfig(listers *regopclient.Listers) (*Swift, error) { } // getSwiftClient returns a client that allows to interact with the OpenStack Swift service -func (d *driver) getSwiftClient(cr *imageregistryv1.Config) (*gophercloud.ServiceClient, error) { +func (d *driver) getSwiftClient() (*gophercloud.ServiceClient, error) { cfg, err := GetConfig(d.Listers) if err != nil { return nil, err @@ -312,7 +326,7 @@ func (d *driver) containerExists(client *gophercloud.ServiceClient, containerNam } func (d *driver) StorageExists(cr *imageregistryv1.Config) (bool, error) { - client, err := d.getSwiftClient(cr) + client, err := d.getSwiftClient() if err != nil { util.UpdateCondition(cr, imageregistryv1.StorageExists, operatorapi.ConditionUnknown, "Could not connect to registry storage", err.Error()) return false, err @@ -342,7 +356,7 @@ func (d *driver) StorageChanged(cr *imageregistryv1.Config) bool { } func (d *driver) CreateStorage(cr *imageregistryv1.Config) error { - client, err := d.getSwiftClient(cr) + client, err := d.getSwiftClient() if err != nil { util.UpdateCondition(cr, imageregistryv1.StorageExists, operatorapi.ConditionFalse, err.Error(), err.Error()) return err @@ -420,7 +434,7 @@ func (d *driver) RemoveStorage(cr *imageregistryv1.Config) (bool, error) { return false, nil } - client, err := d.getSwiftClient(cr) + client, err := d.getSwiftClient() if err != nil { return false, err } diff --git a/pkg/storage/swift/swift_test.go b/pkg/storage/swift/swift_test.go index 08f586272f..534dd007e8 100644 --- a/pkg/storage/swift/swift_test.go +++ b/pkg/storage/swift/swift_test.go @@ -5,6 +5,7 @@ import ( "net/http" "testing" + "github.com/gophercloud/gophercloud" th "github.com/gophercloud/gophercloud/testhelper" corev1 "k8s.io/api/core/v1" @@ -605,3 +606,33 @@ func TestSwiftEndpointTypeObjectStore(t *testing.T) { th.AssertNoErr(t, err) th.AssertEquals(t, true, res) } + +func TestSwiftIsNotAvailable(t *testing.T) { + th.SetupHTTP() + defer th.TeardownHTTP() + // Swift endpoint is not registered + handleAuthentication(t, "INVALID") + + th.Mux.HandleFunc("/"+container, func(w http.ResponseWriter, r *http.Request) { + th.TestMethod(t, r, "HEAD") + th.TestHeader(t, r, "Accept", "application/json") + w.Header().Set("Accept-Ranges", "bytes") + w.Header().Set("Content-Type", "application/json; charset=utf-8") + w.Header().Set("Date", "Wed, 17 Aug 2016 19:25:43 GMT") + w.Header().Set("X-Container-Bytes-Used", "100") + w.Header().Set("X-Container-Object-Count", "4") + w.Header().Set("X-Container-Read", "test") + w.Header().Set("X-Container-Write", "test2,user4") + w.Header().Set("X-Timestamp", "1471298837.95721") + w.Header().Set("X-Trans-Id", "tx554ed59667a64c61866f1-0057b4ba37") + w.Header().Set("X-Storage-Policy", "test_policy") + w.WriteHeader(http.StatusNoContent) + }) + + d, _ := mockConfig(false, th.Endpoint()+"v3", MockUPISecretNamespaceLister{}) + + _, err := d.getSwiftClient() + // if Swift endpoint is not registered, getSwiftClient should return ErrEndpointNotFound + _, ok := err.(*gophercloud.ErrEndpointNotFound) + th.AssertEquals(t, true, ok) +} diff --git a/test/e2e/configuration_test.go b/test/e2e/configuration_test.go index d1ebb3b27e..7b7e217e33 100644 --- a/test/e2e/configuration_test.go +++ b/test/e2e/configuration_test.go @@ -21,6 +21,7 @@ import ( configapiv1 "github.com/openshift/api/config/v1" operatorapiv1 "github.com/openshift/api/operator/v1" + appsapi "k8s.io/api/apps/v1" imageregistryapiv1 "github.com/openshift/cluster-image-registry-operator/pkg/apis/imageregistry/v1" "github.com/openshift/cluster-image-registry-operator/test/framework" @@ -89,6 +90,61 @@ func TestPodResourceConfiguration(t *testing.T) { } } +func TestRolloutStrategyConfiguration(t *testing.T) { + client := framework.MustNewClientset(t, nil) + + defer framework.MustRemoveImageRegistry(t, client) + + cr := &imageregistryapiv1.Config{ + ObjectMeta: metav1.ObjectMeta{ + Name: imageregistryapiv1.ImageRegistryResourceName, + }, + Spec: imageregistryapiv1.ImageRegistrySpec{ + ManagementState: operatorapiv1.Managed, + Storage: imageregistryapiv1.ImageRegistryConfigStorage{ + EmptyDir: &imageregistryapiv1.ImageRegistryConfigStorageEmptyDir{}, + }, + Replicas: 1, + Resources: &corev1.ResourceRequirements{ + Limits: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("512Mi"), + }, + }, + RolloutStrategy: string(appsapi.RecreateDeploymentStrategyType), + NodeSelector: map[string]string{ + "node-role.kubernetes.io/master": "", + }, + Tolerations: []corev1.Toleration{ + { + Key: "node-role.kubernetes.io/master", + Operator: "Exists", + Effect: "NoSchedule", + }, + }, + }, + } + framework.MustDeployImageRegistry(t, client, cr) + framework.MustEnsureImageRegistryIsAvailable(t, client) + framework.MustEnsureClusterOperatorStatusIsNormal(t, client) + + deployments, err := client.Deployments(imageregistryapiv1.ImageRegistryOperatorNamespace).List(metav1.ListOptions{}) + if err != nil { + t.Fatal(err) + } + if len(deployments.Items) == 0 { + t.Errorf("no deployments found in registry namespace") + } + + registryDeployment, err := client.Deployments(imageregistryapiv1.ImageRegistryOperatorNamespace).Get(imageregistryapiv1.ImageRegistryName, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + + if registryDeployment.Spec.Strategy.Type != appsapi.RecreateDeploymentStrategyType { + t.Errorf("expected %v deployment strategy", appsapi.RecreateDeploymentStrategyType) + } +} + func TestPodTolerationsConfiguration(t *testing.T) { client := framework.MustNewClientset(t, nil) diff --git a/test/e2e/pvc_test.go b/test/e2e/pvc_test.go index 7369f918ac..bbdd6d10c8 100644 --- a/test/e2e/pvc_test.go +++ b/test/e2e/pvc_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + appsapi "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/resource" @@ -121,7 +122,7 @@ func createPVWithStorageClass(t *testing.T) error { return nil } -func createPVC(t *testing.T, name string) error { +func createPVC(t *testing.T, name string, accessMode corev1.PersistentVolumeAccessMode) error { client := framework.MustNewClientset(t, nil) claim := &corev1.PersistentVolumeClaim{ @@ -131,7 +132,7 @@ func createPVC(t *testing.T, name string) error { }, Spec: corev1.PersistentVolumeClaimSpec{ AccessModes: []corev1.PersistentVolumeAccessMode{ - corev1.ReadWriteMany, + accessMode, }, Resources: corev1.ResourceRequirements{ Requests: corev1.ResourceList{ @@ -198,7 +199,8 @@ func TestDefaultPVC(t *testing.T) { checkTestResult(t, client) } -func TestCustomPVC(t *testing.T) { +func TestCustomRWXPVC(t *testing.T) { + claimName := "test-custom-rwx-pvc" client := framework.MustNewClientset(t, nil) defer testDefer(t, client) @@ -211,7 +213,7 @@ func TestCustomPVC(t *testing.T) { t.Fatal(err) } - if err := createPVC(t, "test-custom-pvc"); err != nil { + if err := createPVC(t, claimName, corev1.ReadWriteMany); err != nil { t.Fatal(err) } @@ -227,7 +229,7 @@ func TestCustomPVC(t *testing.T) { ManagementState: operatorapi.Managed, Storage: imageregistryv1.ImageRegistryConfigStorage{ PVC: &imageregistryv1.ImageRegistryConfigStoragePVC{ - Claim: "test-custom-pvc", + Claim: claimName, }, }, Replicas: 1, @@ -236,3 +238,44 @@ func TestCustomPVC(t *testing.T) { checkTestResult(t, client) } + +func TestCustomRWOPVC(t *testing.T) { + claimName := "test-custom-rwo-pvc" + client := framework.MustNewClientset(t, nil) + + defer testDefer(t, client) + + if err := createPV(t, ""); err != nil { + t.Fatal(err) + } + + if err := createPVWithStorageClass(t); err != nil { + t.Fatal(err) + } + + if err := createPVC(t, claimName, corev1.ReadWriteOnce); err != nil { + t.Fatal(err) + } + + framework.MustDeployImageRegistry(t, client, &imageregistryv1.Config{ + TypeMeta: metav1.TypeMeta{ + APIVersion: imageregistryv1.SchemeGroupVersion.String(), + Kind: "Config", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: imageregistryv1.ImageRegistryResourceName, + }, + Spec: imageregistryv1.ImageRegistrySpec{ + ManagementState: operatorapi.Managed, + Storage: imageregistryv1.ImageRegistryConfigStorage{ + PVC: &imageregistryv1.ImageRegistryConfigStoragePVC{ + Claim: claimName, + }, + }, + Replicas: 1, + RolloutStrategy: string(appsapi.RecreateDeploymentStrategyType), + }, + }) + + checkTestResult(t, client) +}