diff --git a/Gopkg.lock b/Gopkg.lock index 8ef54696b..a3f1e84bc 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -504,13 +504,14 @@ version = "kubernetes-1.11.1" [[projects]] - digest = "1:e9232c127196055e966ae56877363f82fb494dd8c7fda0112b477e1339082d05" + digest = "1:df623efa281121be190731b5fba78d24f5ceb4fb9345c4ee45a385f8645b7d2c" name = "k8s.io/client-go" packages = [ "discovery", "discovery/cached", "discovery/fake", "dynamic", + "dynamic/fake", "informers", "informers/admissionregistration", "informers/admissionregistration/v1alpha1", @@ -761,6 +762,7 @@ "k8s.io/client-go/discovery", "k8s.io/client-go/discovery/cached", "k8s.io/client-go/dynamic", + "k8s.io/client-go/dynamic/fake", "k8s.io/client-go/informers", "k8s.io/client-go/kubernetes", "k8s.io/client-go/kubernetes/scheme", diff --git a/README.md b/README.md index 0c7b627ea..126d6f453 100644 --- a/README.md +++ b/README.md @@ -43,3 +43,15 @@ podman run --rm -ti \ 1. Use CVO `render` to render all the manifests from release-payload to a directory. [here](#using-cvo-to-render-the-release-payload-locally) 2. Create the operators from the manifests by using `oc create -f `. + +## Running CVO tests + +```sh +# Run all unit tests +go test ./... + +# Run integration tests against a cluster (creates content in a given namespace) +# Requires the CVO CRD to be installed. +export KUBECONFIG= +TEST_INTEGRATION=1 go test ./... -test.run=^TestIntegration +``` \ No newline at end of file diff --git a/cmd/start.go b/cmd/start.go index 5332a50c2..15495a7d7 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -2,6 +2,7 @@ package main import ( "flag" + "fmt" "math/rand" "net/http" "os" @@ -16,12 +17,10 @@ import ( "github.com/openshift/cluster-version-operator/pkg/version" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/spf13/cobra" - "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" apiext "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" - apiextinformers "k8s.io/apiextensions-apiserver/pkg/client/informers/externalversions" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - kubeinformers "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" "k8s.io/client-go/tools/clientcmd" @@ -47,6 +46,11 @@ var ( } startOpts struct { + // name is provided for testing only to allow multiple CVO's to be running at once + name string + // namespace is provided for testing only + namespace string + kubeconfig string nodeName string listenAddr string @@ -78,6 +82,16 @@ func runStartCmd(cmd *cobra.Command, args []string) { startOpts.nodeName = name } + // exposed for end-to-end testing only + startOpts.name = os.Getenv("CVO_NAME") + if len(startOpts.name) == 0 { + startOpts.name = componentName + } + startOpts.namespace = os.Getenv("CVO_NAMESPACE") + if len(startOpts.name) == 0 { + startOpts.namespace = componentNamespace + } + if rootOpts.releaseImage == "" { glog.Fatalf("missing --release-image flag, it is required") } @@ -99,14 +113,13 @@ func runStartCmd(cmd *cobra.Command, args []string) { stopCh := make(chan struct{}) run := func(stop <-chan struct{}) { - ctx := createControllerContext(cb, stopCh) + ctx := createControllerContext(cb, startOpts.name, stopCh) if err := startControllers(ctx); err != nil { glog.Fatalf("error starting controllers: %v", err) } + ctx.CVInformerFactory.Start(ctx.Stop) ctx.InformerFactory.Start(ctx.Stop) - ctx.KubeInformerFactory.Start(ctx.Stop) - ctx.APIExtInformerFactory.Start(ctx.Stop) close(ctx.InformersStarted) select {} @@ -209,9 +222,8 @@ func newClientBuilder(kubeconfig string) (*clientBuilder, error) { type controllerContext struct { ClientBuilder *clientBuilder - InformerFactory informers.SharedInformerFactory - KubeInformerFactory kubeinformers.SharedInformerFactory - APIExtInformerFactory apiextinformers.SharedInformerFactory + CVInformerFactory informers.SharedInformerFactory + InformerFactory informers.SharedInformerFactory Stop <-chan struct{} @@ -220,23 +232,21 @@ type controllerContext struct { ResyncPeriod func() time.Duration } -func createControllerContext(cb *clientBuilder, stop <-chan struct{}) *controllerContext { +func createControllerContext(cb *clientBuilder, name string, stop <-chan struct{}) *controllerContext { client := cb.ClientOrDie("shared-informer") - kubeClient := cb.KubeClientOrDie("kube-shared-informer") - apiExtClient := cb.APIExtClientOrDie("apiext-shared-informer") + cvInformer := informers.NewFilteredSharedInformerFactory(client, resyncPeriod()(), "", func(opts *metav1.ListOptions) { + opts.FieldSelector = fmt.Sprintf("metadata.name=%s", name) + }) sharedInformers := informers.NewSharedInformerFactory(client, resyncPeriod()()) - kubeSharedInformer := kubeinformers.NewSharedInformerFactory(kubeClient, resyncPeriod()()) - apiExtSharedInformer := apiextinformers.NewSharedInformerFactory(apiExtClient, resyncPeriod()()) return &controllerContext{ - ClientBuilder: cb, - InformerFactory: sharedInformers, - KubeInformerFactory: kubeSharedInformer, - APIExtInformerFactory: apiExtSharedInformer, - Stop: stop, - InformersStarted: make(chan struct{}), - ResyncPeriod: resyncPeriod(), + ClientBuilder: cb, + CVInformerFactory: cvInformer, + InformerFactory: sharedInformers, + Stop: stop, + InformersStarted: make(chan struct{}), + ResyncPeriod: resyncPeriod(), } } @@ -248,22 +258,23 @@ func startControllers(ctx *controllerContext) error { go cvo.New( startOpts.nodeName, - componentNamespace, componentName, + startOpts.namespace, startOpts.name, rootOpts.releaseImage, overrideDirectory, ctx.ResyncPeriod(), - ctx.InformerFactory.Config().V1().ClusterVersions(), + ctx.CVInformerFactory.Config().V1().ClusterVersions(), ctx.InformerFactory.Config().V1().ClusterOperators(), ctx.ClientBuilder.RestConfig(), ctx.ClientBuilder.ClientOrDie(componentName), ctx.ClientBuilder.KubeClientOrDie(componentName), ctx.ClientBuilder.APIExtClientOrDie(componentName), + true, ).Run(2, ctx.Stop) if startOpts.enableAutoUpdate { go autoupdate.New( componentNamespace, componentName, - ctx.InformerFactory.Config().V1().ClusterVersions(), + ctx.CVInformerFactory.Config().V1().ClusterVersions(), ctx.InformerFactory.Config().V1().ClusterOperators(), ctx.ClientBuilder.ClientOrDie(componentName), ctx.ClientBuilder.KubeClientOrDie(componentName), diff --git a/hack/test-integration.sh b/hack/test-integration.sh new file mode 100755 index 000000000..c83b2d0d3 --- /dev/null +++ b/hack/test-integration.sh @@ -0,0 +1,8 @@ +#!/usr/bin/env bash + +set -euo pipefail + +base=$( dirname "${BASH_SOURCE[0]}") + +go run "${base}/test-prerequisites.go" +TEST_INTEGRATION=1 go test ./... -test.run=^TestIntegration -args -alsologtostderr -v=5 \ No newline at end of file diff --git a/hack/test-prerequisites.go b/hack/test-prerequisites.go new file mode 100644 index 000000000..80a8746ac --- /dev/null +++ b/hack/test-prerequisites.go @@ -0,0 +1,55 @@ +package main + +import ( + "io/ioutil" + "log" + "time" + + "github.com/ghodss/yaml" + v1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiext "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/tools/clientcmd" +) + +// main installs the CV CRD to a cluster for integration testing. +func main() { + log.SetFlags(0) + kcfg := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}) + cfg, err := kcfg.ClientConfig() + if err != nil { + log.Fatalf("cannot load config: %v", err) + } + + client := apiext.NewForConfigOrDie(cfg) + for _, path := range []string{ + "install/0000_00_cluster-version-operator_01_clusterversion.crd.yaml", + "install/0000_00_cluster-version-operator_01_clusteroperator.crd.yaml", + } { + var name string + err := wait.PollImmediate(time.Second, 30*time.Second, func() (bool, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + log.Fatalf("Unable to read %s: %v", path, err) + } + var crd v1beta1.CustomResourceDefinition + if err := yaml.Unmarshal(data, &crd); err != nil { + log.Fatalf("Unable to parse CRD %s: %v", path, err) + } + name = crd.Name + _, err = client.Apiextensions().CustomResourceDefinitions().Create(&crd) + if errors.IsAlreadyExists(err) { + return true, nil + } + if err != nil { + return false, err + } + log.Printf("Installed %s CRD", crd.Name) + return true, nil + }) + if err != nil { + log.Fatalf("Could not install %s CRD: %v", name, err) + } + } +} diff --git a/pkg/cvo/cvo.go b/pkg/cvo/cvo.go index a721e79c3..72e66ac41 100644 --- a/pkg/cvo/cvo.go +++ b/pkg/cvo/cvo.go @@ -6,6 +6,8 @@ import ( "sync" "time" + "github.com/openshift/cluster-version-operator/lib/resourcemerge" + "github.com/blang/semver" "github.com/golang/glog" "github.com/google/uuid" @@ -28,7 +30,6 @@ import ( configinformersv1 "github.com/openshift/client-go/config/informers/externalversions/config/v1" configlistersv1 "github.com/openshift/client-go/config/listers/config/v1" "github.com/openshift/cluster-version-operator/lib/resourceapply" - "github.com/openshift/cluster-version-operator/lib/resourcemerge" "github.com/openshift/cluster-version-operator/lib/validation" ) @@ -85,9 +86,6 @@ type Operator struct { apiExtClient apiextclientset.Interface eventRecorder record.EventRecorder - // updatePayloadHandler allows unit tests to inject arbitrary payload errors - updatePayloadHandler func(config *configv1.ClusterVersion, payload *updatePayload) error - // minimumUpdateCheckInterval is the minimum duration to check for updates from // the upstream. minimumUpdateCheckInterval time.Duration @@ -112,9 +110,13 @@ type Operator struct { statusLock sync.Mutex availableUpdates *availableUpdates + configSync ConfigSyncWorker + // statusInterval is how often the configSync worker is allowed to retrigger + // the main sync status loop. + statusInterval time.Duration + // lastAtLock guards access to controller memory about the sync loop lastAtLock sync.Mutex - lastSyncAt time.Time lastResourceVersion int64 } @@ -131,6 +133,7 @@ func New( client clientset.Interface, kubeClient kubernetes.Interface, apiExtClient apiextclientset.Interface, + enableMetrics bool, ) *Operator { eventBroadcaster := record.NewBroadcaster() eventBroadcaster.StartLogging(glog.Infof) @@ -142,14 +145,10 @@ func New( name: name, releaseImage: releaseImage, + statusInterval: 15 * time.Second, minimumUpdateCheckInterval: minimumInterval, payloadDir: overridePayloadDir, defaultUpstreamServer: "https://api.openshift.com/api/upgrades_info/v1/graph", - syncBackoff: wait.Backoff{ - Duration: time.Second * 10, - Factor: 1.3, - Steps: 3, - }, restConfig: restConfig, client: client, @@ -161,7 +160,16 @@ func New( availableUpdatesQueue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "availableupdates"), } - optr.updatePayloadHandler = optr.syncUpdatePayload + optr.configSync = NewSyncWorker( + optr.defaultPayloadRetriever(), + optr.defaultResourceBuilder(), + minimumInterval, + wait.Backoff{ + Duration: time.Second * 10, + Factor: 1.3, + Steps: 3, + }, + ) cvInformer.Informer().AddEventHandler(optr.eventHandler()) clusterOperatorInformer.Informer().AddEventHandler(optr.eventHandler()) @@ -172,11 +180,13 @@ func New( optr.cvLister = cvInformer.Lister() optr.cvListerSynced = cvInformer.Informer().HasSynced - if err := optr.registerMetrics(); err != nil { - panic(err) + if enableMetrics { + if err := optr.registerMetrics(); err != nil { + panic(err) + } } - if meta, _, err := loadUpdatePayloadMetadata(optr.baseDirectory(), releaseImage); err != nil { + if meta, _, err := loadUpdatePayloadMetadata(optr.defaultPayloadDir(), releaseImage); err != nil { glog.Warningf("The local payload is invalid - no current version can be determined from disk: %v", err) } else { // XXX: set this to the cincinnati version in preference @@ -190,7 +200,7 @@ func New( return optr } -// Run runs the cluster version operator. +// Run runs the cluster version operator until stopCh is completed. Workers is ignored for now. func (optr *Operator) Run(workers int, stopCh <-chan struct{}) { defer utilruntime.HandleCrash() defer optr.queue.ShutDown() @@ -208,6 +218,10 @@ func (optr *Operator) Run(workers int, stopCh <-chan struct{}) { // trigger the first cluster version reconcile always optr.queue.Add(optr.queueKey()) + // start the config sync loop, and have it notify the queue when new status is detected + go runThrottledStatusNotifier(stopCh, optr.statusInterval, 2, optr.configSync.StatusCh(), func() { optr.queue.Add(optr.queueKey()) }) + go optr.configSync.Start(stopCh) + go wait.Until(func() { optr.worker(optr.queue, optr.sync) }, time.Second, stopCh) go wait.Until(func() { optr.worker(optr.availableUpdatesQueue, optr.availableUpdatesSync) }, time.Second, stopCh) @@ -218,6 +232,8 @@ func (optr *Operator) queueKey() string { return fmt.Sprintf("%s/%s", optr.namespace, optr.name) } +// eventHandler queues an update for the cluster version on any change to the given object. +// Callers should use this with a scoped informer. func (optr *Operator) eventHandler() cache.ResourceEventHandler { workQueueKey := optr.queueKey() return cache.ResourceEventHandlerFuncs{ @@ -272,6 +288,13 @@ func handleErr(queue workqueue.RateLimitingInterface, err error, key interface{} queue.Forget(key) } +// sync ensures: +// +// 1. A ClusterVersion object exists +// 2. The ClusterVersion object has the appropriate status for the state of the cluster +// 3. The configSync object is kept up to date maintaining the user's desired version +// +// It returns an error if it could not update the cluster version object. func (optr *Operator) sync(key string) error { startTime := time.Now() glog.V(4).Infof("Started syncing cluster version %q (%v)", key, startTime) @@ -290,57 +313,35 @@ func (optr *Operator) sync(key string) error { return nil } - glog.V(3).Infof("ClusterVersion: %#v", original) - - // when we're up to date, limit how frequently we check the payload - availableAndUpdated := original.Status.Generation == original.Generation && - resourcemerge.IsOperatorStatusConditionTrue(original.Status.Conditions, configv1.OperatorAvailable) - hasRecentlySynced := availableAndUpdated && optr.hasRecentlySynced() - if hasRecentlySynced { - glog.V(4).Infof("Cluster version has been recently synced and no new changes detected") - return nil - } - - optr.setLastSyncAt(time.Time{}) - - // read the payload - payload, err := optr.loadUpdatePayload(original) - if err != nil { - // the payload is invalid, try and update the status to indicate that - if sErr := optr.syncPayloadFailingStatus(original, err); sErr != nil { - glog.V(2).Infof("Unable to write status when payload was invalid: %v", sErr) - } - return err - } - - update := configv1.Update{ - Version: payload.ReleaseVersion, - Payload: payload.ReleaseImage, - } + // ensure that the object we do have is valid + errs := validation.ValidateClusterVersion(original) + // for fields that have meaning that are incomplete, clear them + // prevents us from loading clearly malformed payloads + config := validation.ClearInvalidFields(original, errs) - // if the current payload is already live, we are reconciling, not updating, - // and we won't set the progressing status. - if availableAndUpdated && payload.ManifestHash == original.Status.VersionHash { - glog.V(2).Infof("Reconciling cluster to version %s and image %s (hash=%s)", update.Version, update.Payload, payload.ManifestHash) + // identify the desired next version + desired, ok := findUpdateFromConfig(config) + if ok { + glog.V(4).Infof("Desired version from spec is %#v", desired) } else { - glog.V(2).Infof("Updating the cluster to version %s and image %s (hash=%s)", update.Version, update.Payload, payload.ManifestHash) - if err := optr.syncProgressingStatus(original, update); err != nil { - return err - } + desired = optr.currentVersion() + glog.V(4).Infof("Desired version from operator is %#v", desired) } - if err := optr.updatePayloadHandler(original, payload); err != nil { - if applyErr := optr.syncUpdateFailingStatus(original, err); applyErr != nil { - glog.V(2).Infof("Unable to write status when sync error occurred: %v", applyErr) - } - return err + // handle the case of a misconfigured CVO by doing nothing + if len(desired.Payload) == 0 { + return optr.syncStatus(original, config, &SyncWorkerStatus{ + Failure: fmt.Errorf("No configured operator version, unable to update cluster"), + }, errs) } - glog.V(2).Infof("Payload for cluster version %s synced", update.Version) + // inform the config sync loop about our desired state + reconciling := resourcemerge.IsOperatorStatusConditionTrue(config.Status.Conditions, configv1.OperatorAvailable) && + resourcemerge.IsOperatorStatusConditionFalse(config.Status.Conditions, configv1.OperatorProgressing) + status := optr.configSync.Update(desired, config.Spec.Overrides, reconciling) - // update the status to indicate we have synced - optr.setLastSyncAt(time.Now()) - return optr.syncAvailableStatus(original, update, payload.ManifestHash) + // write cluster version status + return optr.syncStatus(original, config, status, errs) } // availableUpdatesSync is triggered on cluster version change (and periodic requeues) to @@ -366,24 +367,6 @@ func (optr *Operator) availableUpdatesSync(key string) error { return optr.syncAvailableUpdates(config) } -// hasRecentlySynced returns true if the most recent sync was newer than the -// minimum check interval. -func (optr *Operator) hasRecentlySynced() bool { - if optr.minimumUpdateCheckInterval == 0 { - return false - } - optr.lastAtLock.Lock() - defer optr.lastAtLock.Unlock() - return optr.lastSyncAt.After(time.Now().Add(-optr.minimumUpdateCheckInterval)) -} - -// setLastSyncAt sets the time the operator was last synced at. -func (optr *Operator) setLastSyncAt(t time.Time) { - optr.lastAtLock.Lock() - defer optr.lastAtLock.Unlock() - optr.lastSyncAt = t -} - // isOlderThanLastUpdate returns true if the cluster version is older than // the last update we saw. func (optr *Operator) isOlderThanLastUpdate(config *configv1.ClusterVersion) bool { @@ -418,18 +401,7 @@ func (optr *Operator) getOrCreateClusterVersion() (*configv1.ClusterVersion, boo if optr.isOlderThanLastUpdate(obj) { return nil, true, nil } - - // ensure that the object we do have is valid - errs := validation.ValidateClusterVersion(obj) - changed, err := optr.syncInitialObjectStatus(obj, errs) - if err != nil { - return nil, false, err - } - - // for fields that have meaning that are incomplete, clear them - // prevents us from loading clearly malformed payloads - obj = validation.ClearInvalidFields(obj, errs) - return obj, changed, nil + return obj, false, nil } if !apierrors.IsNotFound(err) { @@ -462,34 +434,6 @@ func (optr *Operator) getOrCreateClusterVersion() (*configv1.ClusterVersion, boo return actual, true, err } -// versionString returns a string describing the current version. -func (optr *Operator) currentVersionString(config *configv1.ClusterVersion) string { - if len(config.Status.History) > 0 { - last := config.Status.History[0] - if s := last.Version; len(s) > 0 { - return s - } - if s := last.Payload; len(s) > 0 { - return s - } - } - if s := optr.releaseVersion; len(s) > 0 { - return s - } - if s := optr.releaseImage; len(s) > 0 { - return s - } - return "" -} - -// desiredVersion returns the update that is currently targeted by the config. -func (optr *Operator) desiredVersion(config *configv1.ClusterVersion) configv1.Update { - if update := config.Spec.DesiredUpdate; update != nil { - return *update - } - return optr.currentVersion() -} - // versionString returns a string describing the desired version. func versionString(update configv1.Update) string { if len(update.Version) > 0 { diff --git a/pkg/cvo/cvo_integration_test.go b/pkg/cvo/cvo_integration_test.go new file mode 100644 index 000000000..033ab5dbc --- /dev/null +++ b/pkg/cvo/cvo_integration_test.go @@ -0,0 +1,712 @@ +package cvo + +import ( + "encoding/json" + "fmt" + "io/ioutil" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/diff" + + v1 "k8s.io/api/core/v1" + apiext "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + randutil "k8s.io/apimachinery/pkg/util/rand" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/tools/clientcmd" + + configv1 "github.com/openshift/api/config/v1" + clientset "github.com/openshift/client-go/config/clientset/versioned" + informers "github.com/openshift/client-go/config/informers/externalversions" + "github.com/openshift/cluster-version-operator/lib/resourcemerge" +) + +var ( + version_0_0_1 = map[string]interface{}{ + "release-manifests": map[string]interface{}{ + "image-references": ` + { + "kind": "ImageStream", + "apiVersion": "image.openshift.io/v1", + "metadata": { + "name": "0.0.1" + } + } + `, + // this manifest should not have ReleaseImage replaced because it is part of the user facing payload + "config2.json": ` + { + "kind": "ConfigMap", + "apiVersion": "v1", + "metadata": { + "name": "config2", + "namespace": "$(NAMESPACE)" + }, + "data": { + "version": "0.0.1", + "releaseImage": "{{.ReleaseImage}}" + } + } + `, + }, + "manifests": map[string]interface{}{ + // this manifest is part of the innate payload and should have ReleaseImage replaced + "config1.json": ` + { + "kind": "ConfigMap", + "apiVersion": "v1", + "metadata": { + "name": "config1", + "namespace": "$(NAMESPACE)" + }, + "data": { + "version": "0.0.1", + "releaseImage": "{{.ReleaseImage}}" + } + } + `, + }, + } + version_0_0_2 = map[string]interface{}{ + "release-manifests": map[string]interface{}{ + "image-references": ` + { + "kind": "ImageStream", + "apiVersion": "image.openshift.io/v1", + "metadata": { + "name": "0.0.2" + } + } + `, + "config2.json": ` + { + "kind": "ConfigMap", + "apiVersion": "v1", + "metadata": { + "name": "config2", + "namespace": "$(NAMESPACE)" + }, + "data": { + "version": "0.0.2", + "releaseImage": "{{.ReleaseImage}}" + } + } + `, + }, + "manifests": map[string]interface{}{ + "config1.json": ` + { + "kind": "ConfigMap", + "apiVersion": "v1", + "metadata": { + "name": "config1", + "namespace": "$(NAMESPACE)" + }, + "data": { + "version": "0.0.2", + "releaseImage": "{{.ReleaseImage}}" + } + } + `, + }, + } + version_0_0_2_failing = map[string]interface{}{ + "release-manifests": map[string]interface{}{ + "image-references": ` + { + "kind": "ImageStream", + "apiVersion": "image.openshift.io/v1", + "metadata": { + "name": "0.0.2" + } + } + `, + // has invalid label value, API server will reject + "config2.json": ` + { + "kind": "ConfigMap", + "apiVersion": "v1", + "metadata": { + "name": "config2", + "namespace": "$(NAMESPACE)", + "labels": {"": ""} + }, + "data": { + "version": "0.0.2", + "releaseImage": "{{.ReleaseImage}}" + } + } + `, + }, + "manifests": map[string]interface{}{ + "config1.json": ` + { + "kind": "ConfigMap", + "apiVersion": "v1", + "metadata": { + "name": "config1", + "namespace": "$(NAMESPACE)" + }, + "data": { + "version": "0.0.2", + "releaseImage": "{{.ReleaseImage}}" + } + } + `, + }, + } +) + +func TestIntegrationCVO_initializeAndUpgrade(t *testing.T) { + if os.Getenv("TEST_INTEGRATION") != "1" { + t.Skipf("Integration tests are disabled unless TEST_INTEGRATION=1") + } + t.Parallel() + + kcfg := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}) + cfg, err := kcfg.ClientConfig() + if err != nil { + t.Fatalf("cannot load config: %v", err) + } + + kc := kubernetes.NewForConfigOrDie(cfg) + client := clientset.NewForConfigOrDie(cfg) + apiExtClient := apiext.NewForConfigOrDie(cfg) + + ns := fmt.Sprintf("e2e-cvo-%s", randutil.String(4)) + + if _, err := kc.Core().Namespaces().Create(&v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns, + }, + }); err != nil { + t.Fatal(err) + } + defer func() { + if err := client.Config().ClusterVersions().Delete(ns, nil); err != nil { + t.Logf("failed to delete cluster version %s: %v", ns, err) + } + if err := kc.Core().Namespaces().Delete(ns, nil); err != nil { + t.Logf("failed to delete namespace %s: %v", ns, err) + } + }() + + cvInformer := informers.NewFilteredSharedInformerFactory(client, 1*time.Minute, "", func(opts *metav1.ListOptions) { + opts.FieldSelector = fmt.Sprintf("metadata.name=%s", ns) + }) + sharedInformers := informers.NewSharedInformerFactory(client, 1*time.Minute) + + dir, err := ioutil.TempDir("", "cvo-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + if err := createContent(filepath.Join(dir, "0.0.1"), version_0_0_1, map[string]string{"NAMESPACE": ns}); err != nil { + t.Fatal(err) + } + if err := createContent(filepath.Join(dir, "0.0.2"), version_0_0_2, map[string]string{"NAMESPACE": ns}); err != nil { + t.Fatal(err) + } + payloadImage1 := "arbitrary/release:image" + payloadImage2 := "arbitrary/release:image-2" + retriever := &mapPayloadRetriever{map[string]string{ + payloadImage1: filepath.Join(dir, "0.0.1"), + payloadImage2: filepath.Join(dir, "0.0.2"), + }} + + optr := New( + "", ns, ns, payloadImage1, + filepath.Join(dir, "ignored"), + 5*time.Second, + cvInformer.Config().V1().ClusterVersions(), + sharedInformers.Config().V1().ClusterOperators(), + cfg, + client, kc, apiExtClient, + false, + ) + + worker := optr.configSync.(*SyncWorker) + worker.retriever = retriever + + stopCh := make(chan struct{}) + defer close(stopCh) + go cvInformer.Start(stopCh) + go sharedInformers.Start(stopCh) + go optr.Run(1, stopCh) + + t.Logf("wait until we observe the cluster version become available") + lastCV, err := waitForAvailableUpdate(t, client, ns, false, "0.0.1") + if err != nil { + t.Logf("latest version:\n%s", printCV(lastCV)) + t.Fatalf("cluster version never became available: %v", err) + } + + status := optr.configSync.(*SyncWorker).Status() + + t.Logf("verify the available cluster version's status matches our expectations") + t.Logf("Cluster version:\n%s", printCV(lastCV)) + verifyClusterVersionStatus(t, lastCV, configv1.Update{Payload: payloadImage1, Version: "0.0.1"}, 1) + verifyReleasePayload(t, kc, ns, "0.0.1", payloadImage1) + + t.Logf("wait for the next resync and verify that status didn't change") + if err := wait.Poll(time.Second, 30*time.Second, func() (bool, error) { + updated := optr.configSync.(*SyncWorker).Status() + if updated.Completed > status.Completed { + return true, nil + } + return false, nil + }); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + cv, err := client.Config().ClusterVersions().Get(ns, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(cv.Status, lastCV.Status) { + t.Fatalf("unexpected: %s", diff.ObjectReflectDiff(lastCV.Status, cv.Status)) + } + verifyReleasePayload(t, kc, ns, "0.0.1", payloadImage1) + + t.Logf("trigger an update to a new version") + cv, err = client.Config().ClusterVersions().Patch(ns, types.MergePatchType, []byte(fmt.Sprintf(`{"spec":{"desiredUpdate":{"payload":"%s"}}}`, payloadImage2))) + if err != nil { + t.Fatal(err) + } + if cv.Spec.DesiredUpdate == nil { + t.Fatalf("cluster desired version was not preserved: %s", printCV(cv)) + } + + t.Logf("wait for the new version to be available") + lastCV, err = waitForAvailableUpdate(t, client, ns, false, "0.0.1", "0.0.2") + if err != nil { + t.Logf("latest version:\n%s", printCV(lastCV)) + t.Fatalf("cluster version never reached available at 0.0.2: %v", err) + } + t.Logf("Upgraded version:\n%s", printCV(lastCV)) + verifyClusterVersionStatus(t, lastCV, configv1.Update{Payload: payloadImage2, Version: "0.0.2"}, 2) + verifyReleasePayload(t, kc, ns, "0.0.2", payloadImage2) + + t.Logf("delete an object so that the next resync will recover it") + if err := kc.CoreV1().ConfigMaps(ns).Delete("config1", nil); err != nil { + t.Fatalf("couldn't delete CVO managed object: %v", err) + } + + status = optr.configSync.(*SyncWorker).Status() + + t.Logf("wait for the next resync and verify that status didn't change") + if err := wait.Poll(time.Second, 30*time.Second, func() (bool, error) { + updated := optr.configSync.(*SyncWorker).Status() + if updated.Completed > status.Completed { + return true, nil + } + return false, nil + }); err != nil { + t.Fatal(err) + } + time.Sleep(time.Second) + cv, err = client.Config().ClusterVersions().Get(ns, metav1.GetOptions{}) + if err != nil { + t.Fatal(err) + } + if !reflect.DeepEqual(cv.Status, lastCV.Status) { + t.Fatalf("unexpected: %s", diff.ObjectReflectDiff(lastCV.Status, cv.Status)) + } + + // should have recreated our deleted object + verifyReleasePayload(t, kc, ns, "0.0.2", payloadImage2) +} + +func TestIntegrationCVO_initializeAndHandleError(t *testing.T) { + if os.Getenv("TEST_INTEGRATION") != "1" { + t.Skipf("Integration tests are disabled unless TEST_INTEGRATION=1") + } + t.Parallel() + + kcfg := clientcmd.NewNonInteractiveDeferredLoadingClientConfig(clientcmd.NewDefaultClientConfigLoadingRules(), &clientcmd.ConfigOverrides{}) + cfg, err := kcfg.ClientConfig() + if err != nil { + t.Fatalf("cannot load config: %v", err) + } + + kc := kubernetes.NewForConfigOrDie(cfg) + client := clientset.NewForConfigOrDie(cfg) + apiExtClient := apiext.NewForConfigOrDie(cfg) + + ns := fmt.Sprintf("e2e-cvo-%s", randutil.String(4)) + + if _, err := kc.Core().Namespaces().Create(&v1.Namespace{ + ObjectMeta: metav1.ObjectMeta{ + Name: ns, + }, + }); err != nil { + t.Fatal(err) + } + defer func() { + if err := client.Config().ClusterVersions().Delete(ns, nil); err != nil { + t.Logf("failed to delete cluster version %s: %v", ns, err) + } + if err := kc.Core().Namespaces().Delete(ns, nil); err != nil { + t.Logf("failed to delete namespace %s: %v", ns, err) + } + }() + + cvInformer := informers.NewFilteredSharedInformerFactory(client, 1*time.Minute, "", func(opts *metav1.ListOptions) { + opts.FieldSelector = fmt.Sprintf("metadata.name=%s", ns) + }) + sharedInformers := informers.NewSharedInformerFactory(client, 1*time.Minute) + + dir, err := ioutil.TempDir("", "cvo-test") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(dir) + if err := createContent(filepath.Join(dir, "0.0.1"), version_0_0_1, map[string]string{"NAMESPACE": ns}); err != nil { + t.Fatal(err) + } + if err := createContent(filepath.Join(dir, "0.0.2"), version_0_0_2_failing, map[string]string{"NAMESPACE": ns}); err != nil { + t.Fatal(err) + } + payloadImage1 := "arbitrary/release:image" + payloadImage2 := "arbitrary/release:image-2-failing" + retriever := &mapPayloadRetriever{map[string]string{ + payloadImage1: filepath.Join(dir, "0.0.1"), + payloadImage2: filepath.Join(dir, "0.0.2"), + }} + + optr := New( + "", ns, ns, payloadImage1, + filepath.Join(dir, "ignored"), + 10*time.Second, + cvInformer.Config().V1().ClusterVersions(), + sharedInformers.Config().V1().ClusterOperators(), + cfg, + client, kc, apiExtClient, + false, + ) + + worker := optr.configSync.(*SyncWorker) + worker.retriever = retriever + + stopCh := make(chan struct{}) + defer close(stopCh) + go cvInformer.Start(stopCh) + go sharedInformers.Start(stopCh) + go optr.Run(1, stopCh) + + t.Logf("wait until we observe the cluster version become available") + lastCV, err := waitForAvailableUpdate(t, client, ns, false, "0.0.1") + if err != nil { + t.Logf("latest version:\n%s", printCV(lastCV)) + t.Fatalf("cluster version never became available: %v", err) + } + + t.Logf("verify the available cluster version's status matches our expectations") + t.Logf("Cluster version:\n%s", printCV(lastCV)) + verifyClusterVersionStatus(t, lastCV, configv1.Update{Payload: payloadImage1, Version: "0.0.1"}, 1) + verifyReleasePayload(t, kc, ns, "0.0.1", payloadImage1) + + t.Logf("trigger an update to a new version that should fail") + cv, err := client.Config().ClusterVersions().Patch(ns, types.MergePatchType, []byte(fmt.Sprintf(`{"spec":{"desiredUpdate":{"payload":"%s"}}}`, payloadImage2))) + if err != nil { + t.Fatal(err) + } + if cv.Spec.DesiredUpdate == nil { + t.Fatalf("cluster desired version was not preserved: %s", printCV(cv)) + } + + t.Logf("wait for operator to report failure") + lastCV, err = waitUntilUpgradeFails( + t, client, ns, + "UpdatePayloadResourceInvalid", + fmt.Sprintf( + `Could not update configmap "%s/config2" (v1, 2 of 2): the object is invalid, possibly due to local cluster configuration`, + ns, + ), + "Unable to apply 0.0.2: some cluster configuration is invalid", + "0.0.1", "0.0.2", + ) + if err != nil { + t.Logf("latest version:\n%s", printCV(lastCV)) + t.Fatalf("cluster version didn't report failure: %v", err) + } + + t.Logf("ensure that one config map was updated and the other was not") + verifyReleasePayloadConfigMap1(t, kc, ns, "0.0.2", payloadImage2) + verifyReleasePayloadConfigMap2(t, kc, ns, "0.0.1", payloadImage1) + + t.Logf("switch back to 0.0.1 and verify it succeeds") + cv, err = client.Config().ClusterVersions().Patch(ns, types.MergePatchType, []byte(`{"spec":{"desiredUpdate":{"payload":"", "version":"0.0.1"}}}`)) + if err != nil { + t.Fatal(err) + } + if cv.Spec.DesiredUpdate == nil { + t.Fatalf("cluster desired version was not preserved: %s", printCV(cv)) + } + lastCV, err = waitForAvailableUpdate(t, client, ns, true, "0.0.2", "0.0.1") + if err != nil { + t.Logf("latest version:\n%s", printCV(lastCV)) + t.Fatalf("cluster version never reverted to 0.0.1: %v", err) + } + verifyClusterVersionStatus(t, lastCV, configv1.Update{Payload: payloadImage1, Version: "0.0.1"}, 3) + verifyReleasePayload(t, kc, ns, "0.0.1", payloadImage1) +} + +// waitForAvailableUpdates checks invariants during an upgrade process. versions is a list of the expected versions that +// should be seen during update, with the last version being the one we wait to see. +func waitForAvailableUpdate(t *testing.T, client clientset.Interface, ns string, allowIncrementalFailure bool, versions ...string) (*configv1.ClusterVersion, error) { + var lastCV *configv1.ClusterVersion + return lastCV, wait.PollImmediate(200*time.Millisecond, 60*time.Second, func() (bool, error) { + cv, err := client.Config().ClusterVersions().Get(ns, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + lastCV = cv + + if !allowIncrementalFailure { + if failing := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorFailing); failing != nil && failing.Status == configv1.ConditionTrue { + return false, fmt.Errorf("operator listed as failing (%s): %s", failing.Reason, failing.Message) + } + } + + // just wait until the operator is available + if len(versions) == 0 { + available := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorAvailable) + return available != nil && available.Status == configv1.ConditionTrue, nil + } + + if len(versions) == 1 { + if available := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorAvailable); available == nil || available.Status == configv1.ConditionFalse { + if progressing := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorProgressing); available != nil && (progressing == nil || progressing.Status != configv1.ConditionTrue) { + return false, fmt.Errorf("initializing operator should have progressing if available is false: %#v", progressing) + } + return false, nil + } + if len(cv.Status.History) == 0 { + return false, fmt.Errorf("initializing operator should have history after available goes true") + } + if cv.Status.History[0].Version != versions[len(versions)-1] { + return false, fmt.Errorf("initializing operator should report the target version in history once available") + } + if cv.Status.History[0].State != configv1.CompletedUpdate { + return false, fmt.Errorf("initializing operator should report history completed %#v", cv.Status.History[0]) + } + if progressing := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorProgressing); progressing == nil || progressing.Status == configv1.ConditionTrue { + return false, fmt.Errorf("initializing operator should never be available and still progressing or lacking the condition: %#v", progressing) + } + return true, nil + } + + if available := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorAvailable); available == nil || available.Status == configv1.ConditionFalse { + return false, fmt.Errorf("upgrading operator should remain available: %#v", available) + } + if !stringInSlice(versions, cv.Status.Desired.Version) { + return false, fmt.Errorf("upgrading operator status reported desired version %s which is not in the allowed set %v", cv.Status.Desired.Version, versions) + } + if len(cv.Status.History) == 0 { + return false, fmt.Errorf("upgrading operator should have at least once history entry") + } + if !stringInSlice(versions, cv.Status.History[0].Version) { + return false, fmt.Errorf("upgrading operator should have a valid history[0] version %s: %v", cv.Status.Desired.Version, versions) + } + + if cv.Status.History[0].Version != versions[len(versions)-1] { + return false, nil + } + + if failing := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorFailing); failing != nil && failing.Status == configv1.ConditionTrue { + return false, fmt.Errorf("operator listed as failing (%s): %s", failing.Reason, failing.Message) + } + + progressing := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorProgressing) + if cv.Status.History[0].State != configv1.CompletedUpdate { + if progressing == nil || progressing.Status != configv1.ConditionTrue { + return false, fmt.Errorf("upgrading operator should have progressing true: %#v", progressing) + } + return false, nil + } + + if progressing == nil || progressing.Status != configv1.ConditionFalse { + return false, fmt.Errorf("upgraded operator should have progressing condition false: %#v", progressing) + } + return true, nil + }) +} + +// waitUntilUpgradeFails checks invariants during an upgrade process. versions is a list of the expected versions that +// should be seen during update, with the last version being the one we wait to see. +func waitUntilUpgradeFails(t *testing.T, client clientset.Interface, ns string, failingReason, failingMessage, progressingMessage string, versions ...string) (*configv1.ClusterVersion, error) { + var lastCV *configv1.ClusterVersion + return lastCV, wait.PollImmediate(200*time.Millisecond, 60*time.Second, func() (bool, error) { + cv, err := client.Config().ClusterVersions().Get(ns, metav1.GetOptions{}) + if errors.IsNotFound(err) { + return false, nil + } + if err != nil { + return false, err + } + lastCV = cv + + if c := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorAvailable); c == nil || c.Status != configv1.ConditionTrue { + return false, fmt.Errorf("operator should remain available: %#v", c) + } + + // just wait until the operator is failing + if len(versions) == 0 { + c := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorFailing) + return c != nil && c.Status == configv1.ConditionTrue, nil + } + + // TODO: add a test for initializing to an error state + // if len(versions) == 1 { + // if available := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorAvailable); available == nil || available.Status == configv1.ConditionFalse { + // if progressing := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorProgressing); available != nil && (progressing == nil || progressing.Status != configv1.ConditionTrue) { + // return false, fmt.Errorf("initializing operator should have progressing if available is false: %#v", progressing) + // } + // return false, nil + // } + // if len(cv.Status.History) == 0 { + // return false, fmt.Errorf("initializing operator should have history after available goes true") + // } + // if cv.Status.History[0].Version != versions[len(versions)-1] { + // return false, fmt.Errorf("initializing operator should report the target version in history once available") + // } + // if cv.Status.History[0].State != configv1.CompletedUpdate { + // return false, fmt.Errorf("initializing operator should report history completed %#v", cv.Status.History[0]) + // } + // if progressing := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorProgressing); progressing == nil || progressing.Status == configv1.ConditionTrue { + // return false, fmt.Errorf("initializing operator should never be available and still progressing or lacking the condition: %#v", progressing) + // } + // return true, nil + // } + if len(versions) == 1 { + return false, fmt.Errorf("unimplemented") + } + + if !stringInSlice(versions, cv.Status.Desired.Version) { + return false, fmt.Errorf("upgrading operator status reported desired version %s which is not in the allowed set %v", cv.Status.Desired.Version, versions) + } + if len(cv.Status.History) == 0 { + return false, fmt.Errorf("upgrading operator should have at least once history entry") + } + if !stringInSlice(versions, cv.Status.History[0].Version) { + return false, fmt.Errorf("upgrading operator should have a valid history[0] version %s: %v", cv.Status.Desired.Version, versions) + } + + if cv.Status.History[0].Version != versions[len(versions)-1] { + return false, nil + } + if cv.Status.History[0].State == configv1.CompletedUpdate { + return false, fmt.Errorf("upgrading operator to failed payload should remain partial: %#v", cv.Status.History) + } + + failing := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorFailing) + if failing == nil || failing.Status != configv1.ConditionTrue { + return false, nil + } + progressing := resourcemerge.FindOperatorStatusCondition(cv.Status.Conditions, configv1.OperatorProgressing) + if progressing == nil || progressing.Status != configv1.ConditionTrue { + return false, fmt.Errorf("upgrading operator to failed payload should have progressing true: %#v", progressing) + } + if !strings.Contains(failing.Message, failingMessage) { + return false, fmt.Errorf("failure message mismatch: %s", failing.Message) + } + if failing.Reason != failingReason { + return false, fmt.Errorf("failure reason mismatch: %s", failing.Reason) + } + if progressing.Reason != failing.Reason { + return false, fmt.Errorf("failure reason and progressing reason don't match: %s", progressing.Reason) + } + if !strings.Contains(progressing.Message, progressingMessage) { + return false, fmt.Errorf("progressing message mismatch: %s", progressing.Message) + } + + return true, nil + }) +} + +func stringInSlice(slice []string, s string) bool { + for _, item := range slice { + if s == item { + return true + } + } + return false +} + +func verifyClusterVersionStatus(t *testing.T, cv *configv1.ClusterVersion, expectedUpdate configv1.Update, expectHistory int) { + t.Helper() + if cv.Status.Desired != expectedUpdate { + t.Fatalf("unexpected: %#v", cv.Status.Desired) + } + if len(cv.Status.History) != expectHistory { + t.Fatalf("unexpected: %#v", cv.Status.History) + } + actual := cv.Status.History[0] + if actual.StartedTime.Time.IsZero() || actual.CompletionTime == nil || actual.CompletionTime.Time.IsZero() || actual.CompletionTime.Time.Before(actual.StartedTime.Time) { + t.Fatalf("unexpected: %s -> %s", actual.StartedTime, actual.CompletionTime) + } + expect := configv1.UpdateHistory{ + State: configv1.CompletedUpdate, + Version: expectedUpdate.Version, + Payload: expectedUpdate.Payload, + StartedTime: actual.StartedTime, + CompletionTime: actual.CompletionTime, + } + if !reflect.DeepEqual(expect, actual) { + t.Fatalf("unexpected history: %s", diff.ObjectReflectDiff(expect, actual)) + } + if len(cv.Status.VersionHash) == 0 { + t.Fatalf("unexpected version hash: %#v", cv.Status.VersionHash) + } + if cv.Status.Generation != cv.Generation { + t.Fatalf("unexpected generation: %#v", cv.Status.Generation) + } +} + +func verifyReleasePayload(t *testing.T, kc kubernetes.Interface, ns, version, payload string) { + t.Helper() + verifyReleasePayloadConfigMap1(t, kc, ns, version, payload) + verifyReleasePayloadConfigMap2(t, kc, ns, version, payload) +} + +func verifyReleasePayloadConfigMap1(t *testing.T, kc kubernetes.Interface, ns, version, payload string) { + t.Helper() + cm, err := kc.CoreV1().ConfigMaps(ns).Get("config1", metav1.GetOptions{}) + if err != nil { + t.Fatalf("unable to find cm/config1 in ns %s: %v", ns, err) + } + if cm.Data["version"] != version || cm.Data["releaseImage"] != payload { + t.Fatalf("unexpected cm/config1 contents: %#v", cm.Data) + } +} + +func verifyReleasePayloadConfigMap2(t *testing.T, kc kubernetes.Interface, ns, version, payload string) { + t.Helper() + cm, err := kc.CoreV1().ConfigMaps(ns).Get("config2", metav1.GetOptions{}) + if err != nil { + t.Fatalf("unable to find cm/config2 in ns %s: %v", ns, err) + } + if cm.Data["version"] != version || cm.Data["releaseImage"] != "{{.ReleaseImage}}" { + t.Fatalf("unexpected cm/config2 contents: %#v", cm.Data) + } +} + +func printCV(cv *configv1.ClusterVersion) string { + data, err := json.MarshalIndent(cv, "", " ") + if err != nil { + return fmt.Sprintf("", err) + } + return string(data) +} diff --git a/pkg/cvo/cvo_scenarios_test.go b/pkg/cvo/cvo_scenarios_test.go new file mode 100644 index 000000000..dd8c4b54d --- /dev/null +++ b/pkg/cvo/cvo_scenarios_test.go @@ -0,0 +1,708 @@ +package cvo + +import ( + "fmt" + "reflect" + "strconv" + "testing" + "time" + + "k8s.io/apimachinery/pkg/util/diff" + "k8s.io/apimachinery/pkg/util/wait" + + "github.com/davecgh/go-spew/spew" + "github.com/google/uuid" + + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + dynamicfake "k8s.io/client-go/dynamic/fake" + clientgotesting "k8s.io/client-go/testing" + "k8s.io/client-go/util/workqueue" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/client-go/config/clientset/versioned/fake" + "github.com/openshift/cluster-version-operator/lib" +) + +func setupCVOTest() (*Operator, map[string]runtime.Object, *fake.Clientset, *dynamicfake.FakeDynamicClient, func()) { + client := &fake.Clientset{} + client.AddReactor("*", "*", func(action clientgotesting.Action) (handled bool, ret runtime.Object, err error) { + return false, nil, fmt.Errorf("unexpected client action: %#v", action) + }) + cvs := make(map[string]runtime.Object) + client.AddReactor("*", "clusterversions", func(action clientgotesting.Action) (handled bool, ret runtime.Object, err error) { + switch a := action.(type) { + case clientgotesting.GetAction: + obj, ok := cvs[a.GetName()] + if !ok { + return true, nil, errors.NewNotFound(schema.GroupResource{Resource: "clusterversions"}, a.GetName()) + } + return true, obj.DeepCopyObject(), nil + case clientgotesting.CreateAction: + obj := a.GetObject().DeepCopyObject() + m := obj.(metav1.Object) + cvs[m.GetName()] = obj + return true, obj, nil + case clientgotesting.UpdateAction: + obj := a.GetObject().DeepCopyObject().(*configv1.ClusterVersion) + existing := cvs[obj.Name].DeepCopyObject().(*configv1.ClusterVersion) + rv, _ := strconv.Atoi(existing.ResourceVersion) + nextRV := strconv.Itoa(rv + 1) + if a.GetSubresource() == "status" { + existing.Status = obj.Status + } else { + existing.Spec = obj.Spec + existing.ObjectMeta = obj.ObjectMeta + } + existing.ResourceVersion = nextRV + cvs[existing.Name] = existing + return true, existing, nil + } + return false, nil, fmt.Errorf("unexpected client action: %#v", action) + }) + + o := &Operator{ + namespace: "test", + name: "version", + queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "cvo-loop-test"), + client: client, + cvLister: &clientCVLister{client: client}, + } + + dynamicScheme := runtime.NewScheme() + //dynamicScheme.AddKnownTypeWithName(schema.GroupVersionKind{Group: "test.cvo.io", Version: "v1", Kind: "TestA"}, &unstructured.Unstructured{}) + dynamicScheme.AddKnownTypeWithName(schema.GroupVersionKind{Group: "test.cvo.io", Version: "v1", Kind: "TestB"}, &unstructured.Unstructured{}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(dynamicScheme) + + worker := NewSyncWorker( + &fakeDirectoryRetriever{Path: "testdata/payloadtest"}, + &testResourceBuilder{client: dynamicClient}, + time.Second/2, + wait.Backoff{ + Steps: 1, + }, + ) + o.configSync = worker + + return o, cvs, client, dynamicClient, func() { o.queue.ShutDown() } +} + +func TestCVO_StartupAndSync(t *testing.T) { + o, cvs, client, _, shutdownFn := setupCVOTest() + stopCh := make(chan struct{}) + defer close(stopCh) + defer shutdownFn() + worker := o.configSync.(*SyncWorker) + go worker.Start(stopCh) + + // Step 1: Verify the CVO creates the initial Cluster Version object + // + client.ClearActions() + err := o.sync(o.queueKey()) + if err != nil { + t.Fatal(err) + } + actions := client.Actions() + if len(actions) != 3 { + t.Fatalf("%s", spew.Sdump(actions)) + } + // read from lister + expectGet(t, actions[0], "clusterversions", "", "version") + // read before create + expectGet(t, actions[1], "clusterversions", "", "version") + // create initial version + actual := cvs["version"].(*configv1.ClusterVersion) + expectCreate(t, actions[2], "clusterversions", "", &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "version", + }, + Spec: configv1.ClusterVersionSpec{ + ClusterID: actual.Spec.ClusterID, + Channel: "fast", + }, + }) + verifyAllStatus(t, worker.StatusCh()) + + // Step 2: Ensure the CVO reports a status error if it has nothing to sync + // + client.ClearActions() + err = o.sync(o.queueKey()) + if err != nil { + t.Fatal(err) + } + actions = client.Actions() + if len(actions) != 2 { + t.Fatalf("%s", spew.Sdump(actions)) + } + expectGet(t, actions[0], "clusterversions", "", "version") + actual = cvs["version"].(*configv1.ClusterVersion) + expectUpdateStatus(t, actions[1], "clusterversions", "", &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "version", + }, + Spec: configv1.ClusterVersionSpec{ + ClusterID: actual.Spec.ClusterID, + Channel: "fast", + }, + Status: configv1.ClusterVersionStatus{ + History: []configv1.UpdateHistory{ + // empty because the operator release image is not set, so we have no input + }, + Conditions: []configv1.ClusterOperatorStatusCondition{ + {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, + // report back to the user that we don't have enough info to proceed + {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Message: "No configured operator version, unable to update cluster"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Unable to apply : an error occurred"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, + }, + }, + }) + verifyAllStatus(t, worker.StatusCh()) + + // Step 3: Given an operator payload, begin synchronizing + // + o.releaseImage = "payload/image:1" + o.releaseVersion = "4.0.1" + desired := configv1.Update{Version: "4.0.1", Payload: "payload/image:1"} + // + client.ClearActions() + err = o.sync(o.queueKey()) + if err != nil { + t.Fatal(err) + } + actions = client.Actions() + if len(actions) != 2 { + t.Fatalf("%s", spew.Sdump(actions)) + } + expectGet(t, actions[0], "clusterversions", "", "version") + actual = cvs["version"].(*configv1.ClusterVersion) + expectUpdateStatus(t, actions[1], "clusterversions", "", &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "version", + }, + Spec: configv1.ClusterVersionSpec{ + ClusterID: actual.Spec.ClusterID, + Channel: "fast", + }, + Status: configv1.ClusterVersionStatus{ + Desired: desired, + History: []configv1.UpdateHistory{ + {State: configv1.PartialUpdate, Payload: "payload/image:1", Version: "4.0.1", StartedTime: defaultStartedTime}, + }, + Conditions: []configv1.ClusterOperatorStatusCondition{ + {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, + // cleared failing status and set progressing + {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Working towards 4.0.1"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, + }, + }, + }) + verifyAllStatus(t, worker.StatusCh(), + SyncWorkerStatus{ + Step: "RetrievePayload", + // the desired version is briefly incorrect (user provided) until we retrieve the payload + Actual: configv1.Update{Version: "4.0.1", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Fraction: float32(1) / 3, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Fraction: float32(2) / 3, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Reconciling: true, + Completed: 1, + Fraction: 1, + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + ) + + // Step 4: Now that sync is complete, verify status is updated to represent payload contents + // + client.ClearActions() + err = o.sync(o.queueKey()) + if err != nil { + t.Fatal(err) + } + actions = client.Actions() + if len(actions) != 2 { + t.Fatalf("%s", spew.Sdump(actions)) + } + expectGet(t, actions[0], "clusterversions", "", "version") + // update the status to indicate we are synced, available, and report versions + actual = cvs["version"].(*configv1.ClusterVersion) + expectUpdateStatus(t, actions[1], "clusterversions", "", &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "version", + }, + Spec: configv1.ClusterVersionSpec{ + ClusterID: actual.Spec.ClusterID, + Channel: "fast", + }, + Status: configv1.ClusterVersionStatus{ + // Prefers the payload version over the operator's version (although in general they will remain in sync) + Desired: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + VersionHash: "6GC9TkkG9PA=", + History: []configv1.UpdateHistory{ + // Because payload and operator had mismatched versions, we get two entries (which shouldn't happen unless there is a bug in the CVO) + {State: configv1.CompletedUpdate, Payload: "payload/image:1", Version: "1.0.0-abc", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, + {State: configv1.PartialUpdate, Payload: "payload/image:1", Version: "4.0.1", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, + }, + Conditions: []configv1.ClusterOperatorStatusCondition{ + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 1.0.0-abc"}, + {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: "Cluster version is 1.0.0-abc"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, + }, + }, + }) + + // Step 5: Wait for the SyncWorker to trigger a reconcile (500ms after the first) + // + verifyAllStatus(t, worker.StatusCh(), + SyncWorkerStatus{ + Reconciling: true, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Reconciling: true, + Fraction: float32(1) / 3, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Reconciling: true, + Fraction: float32(2) / 3, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Reconciling: true, + Completed: 2, + Fraction: 1, + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + ) + + // Step 6: After a reconciliation, there should be no status change because the state is the same + // + client.ClearActions() + err = o.sync(o.queueKey()) + if err != nil { + t.Fatal(err) + } + actions = client.Actions() + if len(actions) != 1 { + t.Fatalf("%s", spew.Sdump(actions)) + } + expectGet(t, actions[0], "clusterversions", "", "version") +} + +func TestCVO_RestartAndReconcile(t *testing.T) { + o, cvs, client, _, shutdownFn := setupCVOTest() + stopCh := make(chan struct{}) + defer close(stopCh) + defer shutdownFn() + worker := o.configSync.(*SyncWorker) + + // Setup: a successful sync from a previous run, and the operator at the same payload as before + // + o.releaseImage = "payload/image:1" + o.releaseVersion = "1.0.0-abc" + desired := configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"} + uid, _ := uuid.NewRandom() + clusterUID := configv1.ClusterID(uid.String()) + cvs["version"] = &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "version", + ResourceVersion: "1", + }, + Spec: configv1.ClusterVersionSpec{ + ClusterID: clusterUID, + Channel: "fast", + }, + Status: configv1.ClusterVersionStatus{ + // Prefers the payload version over the operator's version (although in general they will remain in sync) + Desired: desired, + VersionHash: "6GC9TkkG9PA=", + History: []configv1.UpdateHistory{ + // TODO: this is wrong, should be single partial entry + {State: configv1.CompletedUpdate, Payload: "payload/image:1", Version: "1.0.0-abc", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, + {State: configv1.PartialUpdate, Payload: "payload/image:1", Version: "4.0.1", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, + {State: configv1.PartialUpdate, StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, + {State: configv1.PartialUpdate, StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, + }, + Conditions: []configv1.ClusterOperatorStatusCondition{ + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 1.0.0-abc"}, + {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: "Cluster version is 1.0.0-abc"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, + }, + }, + } + + // Step 1: The sync loop starts and triggers a sync, but does not update status + // + client.ClearActions() + err := o.sync(o.queueKey()) + if err != nil { + t.Fatal(err) + } + actions := client.Actions() + if len(actions) != 1 { + t.Fatalf("%s", spew.Sdump(actions)) + } + expectGet(t, actions[0], "clusterversions", "", "version") + + // check the worker status is initially set to reconciling + if status := worker.Status(); !status.Reconciling || status.Completed != 0 { + t.Fatalf("The worker should be reconciling from the beginning: %#v", status) + } + + // Step 2: Start the sync worker and verify the sequence of events, and then verify + // the status does not change + // + go worker.Start(stopCh) + // + verifyAllStatus(t, worker.StatusCh(), + SyncWorkerStatus{ + Reconciling: true, + Step: "RetrievePayload", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Reconciling: true, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Reconciling: true, + Fraction: float32(1) / 3, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Reconciling: true, + Fraction: float32(2) / 3, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Reconciling: true, + Completed: 1, + Fraction: 1, + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + ) + client.ClearActions() + err = o.sync(o.queueKey()) + if err != nil { + t.Fatal(err) + } + actions = client.Actions() + if len(actions) != 1 { + t.Fatalf("%s", spew.Sdump(actions)) + } + expectGet(t, actions[0], "clusterversions", "", "version") + + // Step 3: Wait until the next resync is triggered, and then verify that status does + // not change + // + verifyAllStatus(t, worker.StatusCh(), + // note that the payload is not retrieved a second time + SyncWorkerStatus{ + Reconciling: true, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Reconciling: true, + Fraction: float32(1) / 3, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Reconciling: true, + Fraction: float32(2) / 3, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Reconciling: true, + Completed: 2, + Fraction: 1, + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + ) + client.ClearActions() + err = o.sync(o.queueKey()) + if err != nil { + t.Fatal(err) + } + actions = client.Actions() + if len(actions) != 1 { + t.Fatalf("%s", spew.Sdump(actions)) + } + expectGet(t, actions[0], "clusterversions", "", "version") +} + +func TestCVO_ErrorDuringReconcile(t *testing.T) { + o, cvs, client, _, shutdownFn := setupCVOTest() + stopCh := make(chan struct{}) + defer close(stopCh) + defer shutdownFn() + worker := o.configSync.(*SyncWorker) + b := newBlockingResourceBuilder() + worker.builder = b + + // Setup: a successful sync from a previous run, and the operator at the same payload as before + // + o.releaseImage = "payload/image:1" + o.releaseVersion = "1.0.0-abc" + desired := configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"} + uid, _ := uuid.NewRandom() + clusterUID := configv1.ClusterID(uid.String()) + cvs["version"] = &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "version", + ResourceVersion: "1", + }, + Spec: configv1.ClusterVersionSpec{ + ClusterID: clusterUID, + Channel: "fast", + }, + Status: configv1.ClusterVersionStatus{ + // Prefers the payload version over the operator's version (although in general they will remain in sync) + Desired: desired, + VersionHash: "6GC9TkkG9PA=", + History: []configv1.UpdateHistory{ + {State: configv1.CompletedUpdate, Payload: "payload/image:1", Version: "1.0.0-abc", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, + }, + Conditions: []configv1.ClusterOperatorStatusCondition{ + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 1.0.0-abc"}, + {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: "Cluster version is 1.0.0-abc"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, + }, + }, + } + + // Step 1: The sync loop starts and triggers a sync, but does not update status + // + client.ClearActions() + err := o.sync(o.queueKey()) + if err != nil { + t.Fatal(err) + } + actions := client.Actions() + if len(actions) != 1 { + t.Fatalf("%s", spew.Sdump(actions)) + } + expectGet(t, actions[0], "clusterversions", "", "version") + + // check the worker status is initially set to reconciling + if status := worker.Status(); !status.Reconciling || status.Completed != 0 { + t.Fatalf("The worker should be reconciling from the beginning: %#v", status) + } + + // Step 2: Start the sync worker and verify the sequence of events + // + go worker.Start(stopCh) + // + verifyAllStatus(t, worker.StatusCh(), + SyncWorkerStatus{ + Reconciling: true, + Step: "RetrievePayload", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + SyncWorkerStatus{ + Reconciling: true, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + ) + // verify we haven't observed any other events + verifyAllStatus(t, worker.StatusCh()) + + // Step 3: Simulate a sync being triggered while we are partway through our first + // reconcile sync and verify status is not updated + // + client.ClearActions() + err = o.sync(o.queueKey()) + if err != nil { + t.Fatal(err) + } + actions = client.Actions() + if len(actions) != 1 { + t.Fatalf("%s", spew.Sdump(actions)) + } + expectGet(t, actions[0], "clusterversions", "", "version") + + // Step 4: Unblock the first item from being applied + // + b.Send(nil) + // + // verify we observe the remaining changes in the first sync + verifyAllStatus(t, worker.StatusCh(), + SyncWorkerStatus{ + Reconciling: true, + Fraction: float32(1) / 3, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + ) + verifyAllStatus(t, worker.StatusCh()) + + // Step 5: Unblock the first item from being applied + // + b.Send(nil) + // + // verify we observe the remaining changes in the first sync + verifyAllStatus(t, worker.StatusCh(), + SyncWorkerStatus{ + Reconciling: true, + Fraction: float32(2) / 3, + Step: "ApplyResources", + VersionHash: "6GC9TkkG9PA=", + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + ) + verifyAllStatus(t, worker.StatusCh()) + + // Step 6: Send an error, then verify it shows up in status + // + b.Send(fmt.Errorf("unable to proceed")) + // + verifyAllStatus(t, worker.StatusCh(), + SyncWorkerStatus{ + Reconciling: true, + Step: "ApplyResources", + Fraction: float32(2) / 3, + VersionHash: "6GC9TkkG9PA=", + Failure: &updateError{ + cause: fmt.Errorf("unable to proceed"), + Reason: "UpdatePayloadFailed", + Message: "Could not update test \"file-yml\" (v1, 3 of 3)", + }, + Actual: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + }, + ) + client.ClearActions() + err = o.sync(o.queueKey()) + if err != nil { + t.Fatal(err) + } + actions = client.Actions() + if len(actions) != 2 { + t.Fatalf("%s", spew.Sdump(actions)) + } + expectGet(t, actions[0], "clusterversions", "", "version") + expectUpdateStatus(t, actions[1], "clusterversions", "", &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "version", + ResourceVersion: "1", + }, + Spec: configv1.ClusterVersionSpec{ + ClusterID: clusterUID, + Channel: "fast", + }, + Status: configv1.ClusterVersionStatus{ + // Prefers the payload version over the operator's version (although in general they will remain in sync) + Desired: configv1.Update{Version: "1.0.0-abc", Payload: "payload/image:1"}, + VersionHash: "6GC9TkkG9PA=", + History: []configv1.UpdateHistory{ + // Because payload and operator had mismatched versions, we get two entries (which shouldn't happen unless there is a bug in the CVO) + {State: configv1.CompletedUpdate, Payload: "payload/image:1", Version: "1.0.0-abc", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, + }, + Conditions: []configv1.ClusterOperatorStatusCondition{ + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 1.0.0-abc"}, + {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Reason: "UpdatePayloadFailed", Message: "Could not update test \"file-yml\" (v1, 3 of 3)"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Reason: "UpdatePayloadFailed", Message: "Error while reconciling 1.0.0-abc: the update could not be applied"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, + }, + }, + }) +} + +func verifyAllStatus(t *testing.T, ch <-chan SyncWorkerStatus, items ...SyncWorkerStatus) { + t.Helper() + if len(items) == 0 { + if len(ch) > 0 { + t.Fatalf("expected status to empty, got %#v", <-ch) + } + return + } + for i, expect := range items { + actual, ok := <-ch + if !ok { + t.Fatalf("channel closed after reading only %d items", i) + } + if !reflect.DeepEqual(expect, actual) { + t.Fatalf("unexpected status item %d: %s", i, diff.ObjectReflectDiff(expect, actual)) + } + } +} + +func waitFor(t *testing.T, fn func() bool) { + t.Helper() + err := wait.PollImmediate(100*time.Millisecond, 1*time.Second, func() (bool, error) { + return fn(), nil + }) + if err == wait.ErrWaitTimeout { + t.Fatalf("Worker condition was not reached within timeout") + } + if err != nil { + t.Fatal(err) + } +} + +// blockingResourceBuilder controls how quickly Apply() is executed and allows error +// injection. +type blockingResourceBuilder struct { + ch chan error +} + +func newBlockingResourceBuilder() *blockingResourceBuilder { + return &blockingResourceBuilder{ + ch: make(chan error), + } +} + +func (b *blockingResourceBuilder) Send(err error) { + b.ch <- err +} + +func (b *blockingResourceBuilder) Apply(m *lib.Manifest) error { + return <-b.ch +} diff --git a/pkg/cvo/cvo_test.go b/pkg/cvo/cvo_test.go index b6a57e5bb..17430a6ae 100644 --- a/pkg/cvo/cvo_test.go +++ b/pkg/cvo/cvo_test.go @@ -1,6 +1,7 @@ package cvo import ( + "context" "fmt" "io/ioutil" "net/http" @@ -8,15 +9,23 @@ import ( "os" "path/filepath" "reflect" + "regexp" "strconv" "testing" "time" - "k8s.io/apimachinery/pkg/runtime" - + "github.com/davecgh/go-spew/spew" + "github.com/golang/glog" + "github.com/google/uuid" + configv1 "github.com/openshift/api/config/v1" + clientset "github.com/openshift/client-go/config/clientset/versioned" + "github.com/openshift/client-go/config/clientset/versioned/fake" + apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" + apiextclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/diff" @@ -25,14 +34,13 @@ import ( "k8s.io/client-go/rest" ktesting "k8s.io/client-go/testing" "k8s.io/client-go/util/workqueue" +) - "github.com/golang/glog" - "github.com/google/uuid" - configv1 "github.com/openshift/api/config/v1" - clientset "github.com/openshift/client-go/config/clientset/versioned" - "github.com/openshift/client-go/config/clientset/versioned/fake" - apiextv1beta1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1beta1" - apiextclientv1 "k8s.io/apiextensions-apiserver/pkg/client/clientset/clientset/typed/apiextensions/v1beta1" +var ( + // defaultStartedTime is a shorthand for verifying a start time is set + defaultStartedTime = metav1.Time{Time: time.Unix(1, 0)} + // defaultCompletionTime is a shorthand for verifying a completion time is set + defaultCompletionTime = metav1.Time{Time: time.Unix(2, 0)} ) type clientCVLister struct { @@ -190,43 +198,45 @@ func TestOperator_sync(t *testing.T) { `, }, } - contentWithoutManifests := map[string]interface{}{ - "release-manifests": map[string]interface{}{ - "image-references": ` - { - "kind": "ImageStream", - "apiVersion": "image.openshift.io/v1", - "metadata": { - "name": "0.0.1-abc" - } - } - `, - }, - } - content_4_0_1 := map[string]interface{}{ - "manifests": map[string]interface{}{}, - "release-manifests": map[string]interface{}{ - "image-references": ` - { - "kind": "ImageStream", - "apiVersion": "image.openshift.io/v1", - "metadata": { - "name": "4.0.1" - } - } - `, - }, - } + // contentWithoutManifests := map[string]interface{}{ + // "release-manifests": map[string]interface{}{ + // "image-references": ` + // { + // "kind": "ImageStream", + // "apiVersion": "image.openshift.io/v1", + // "metadata": { + // "name": "0.0.1-abc" + // } + // } + // `, + // }, + // } + // content_4_0_1 := map[string]interface{}{ + // "manifests": map[string]interface{}{}, + // "release-manifests": map[string]interface{}{ + // "image-references": ` + // { + // "kind": "ImageStream", + // "apiVersion": "image.openshift.io/v1", + // "metadata": { + // "name": "4.0.1" + // } + // } + // `, + // }, + // } tests := []struct { name string key string content map[string]interface{} + syncStatus *SyncWorkerStatus optr Operator init func(optr *Operator) want bool wantErr func(*testing.T, error) wantActions func(*testing.T, *Operator) + wantSync []configv1.Update }{ { name: "create version and status", @@ -257,8 +267,16 @@ func TestOperator_sync(t *testing.T) { }, }, { - name: "progressing and previously failed", - content: content1, + name: "progressing and previously failed, not reconciling", + syncStatus: &SyncWorkerStatus{ + Step: "Moving", + Reconciling: false, + Actual: configv1.Update{Version: "0.0.1-abc", Payload: "payload/image:v4.0.1"}, + Failure: &updateError{ + Reason: "UpdatePayloadIntegrity", + Message: "unable to apply object", + }, + }, optr: Operator{ releaseVersion: "4.0.1", releaseImage: "payload/image:v4.0.1", @@ -274,7 +292,7 @@ func TestOperator_sync(t *testing.T) { }, Status: configv1.ClusterVersionStatus{ History: []configv1.UpdateHistory{ - {Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: metav1.Time{Time: time.Unix(1, 0)}}, + {Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime}, }, Desired: configv1.Update{Version: "4.0.1", Payload: "payload/image:v4.0.1"}, VersionHash: "", @@ -291,7 +309,7 @@ func TestOperator_sync(t *testing.T) { wantActions: func(t *testing.T, optr *Operator) { f := optr.client.(*fake.Clientset) act := f.Actions() - if len(act) != 3 { + if len(act) != 2 { t.Fatalf("unknown actions: %d %#v", len(act), act) } expectGet(t, act[0], "clusterversions", "", "default") @@ -304,8 +322,8 @@ func TestOperator_sync(t *testing.T) { }, Status: configv1.ClusterVersionStatus{ History: []configv1.UpdateHistory{ - {State: configv1.PartialUpdate, Version: "0.0.1-abc", Payload: "payload/image:v4.0.1", StartedTime: metav1.Time{Time: time.Unix(1, 0)}}, - {Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: metav1.Time{Time: time.Unix(1, 0)}, CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}}, + {State: configv1.PartialUpdate, Version: "0.0.1-abc", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime}, + {Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, }, Desired: configv1.Update{Version: "0.0.1-abc", Payload: "payload/image:v4.0.1"}, VersionHash: "", @@ -317,39 +335,76 @@ func TestOperator_sync(t *testing.T) { }, }, }) - expectUpdateStatus(t, act[2], "clusterversions", "", &configv1.ClusterVersion{ + }, + }, + { + name: "progressing and previously failed, reconciling", + content: content1, + optr: Operator{ + releaseVersion: "4.0.1", + releaseImage: "payload/image:v4.0.1", + namespace: "test", + name: "default", + configSync: &fakeSyncRecorder{ + Returns: &SyncWorkerStatus{ + Step: "Moving", + Reconciling: true, + Actual: configv1.Update{Version: "0.0.1-abc", Payload: "payload/image:v4.0.1"}, + Failure: &updateError{ + Reason: "UpdatePayloadIntegrity", + Message: "unable to apply object", + }, + }, + }, + client: fakeClientsetWithUpdates( + &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + Spec: configv1.ClusterVersionSpec{ + Channel: "fast", + }, + Status: configv1.ClusterVersionStatus{ + History: []configv1.UpdateHistory{ + {Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime}, + }, + Desired: configv1.Update{Version: "4.0.1", Payload: "payload/image:v4.0.1"}, + VersionHash: "", + Conditions: []configv1.ClusterOperatorStatusCondition{ + {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Reason: "UpdatePayloadIntegrity", Message: "unable to apply object"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Working towards 4.0.1"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, + }, + }, + }, + ), + }, + wantActions: func(t *testing.T, optr *Operator) { + f := optr.client.(*fake.Clientset) + act := f.Actions() + if len(act) != 2 { + t.Fatalf("unknown actions: %d %#v", len(act), act) + } + expectGet(t, act[0], "clusterversions", "", "default") + expectUpdateStatus(t, act[1], "clusterversions", "", &configv1.ClusterVersion{ ObjectMeta: metav1.ObjectMeta{ - Name: "default", - ResourceVersion: "1", + Name: "default", }, Spec: configv1.ClusterVersionSpec{ Channel: "fast", }, Status: configv1.ClusterVersionStatus{ History: []configv1.UpdateHistory{ - { - State: configv1.CompletedUpdate, - Version: "0.0.1-abc", - Payload: "payload/image:v4.0.1", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, - }, - { - Version: "4.0.1", - Payload: "payload/image:v4.0.1", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, - }, + {State: configv1.PartialUpdate, Version: "0.0.1-abc", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime}, + {Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, }, - Desired: configv1.Update{ - Version: "0.0.1-abc", - Payload: "payload/image:v4.0.1", - }, - VersionHash: "y_Kc5IQiIyU=", + Desired: configv1.Update{Version: "0.0.1-abc", Payload: "payload/image:v4.0.1"}, + VersionHash: "", Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 0.0.1-abc"}, - {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: "Cluster version is 0.0.1-abc"}, + {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Reason: "UpdatePayloadIntegrity", Message: "unable to apply object"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Reason: "UpdatePayloadIntegrity", Message: "Error while reconciling 0.0.1-abc: the contents of the update are invalid"}, {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, }, }, @@ -357,17 +412,26 @@ func TestOperator_sync(t *testing.T) { }, }, { - name: "progressing and encounters error during payload sync", + name: "progressing and previously failed, reconciling and multiple completions", content: content1, optr: Operator{ releaseVersion: "4.0.1", releaseImage: "payload/image:v4.0.1", namespace: "test", name: "default", - updatePayloadHandler: func(config *configv1.ClusterVersion, payload *updatePayload) error { - return fmt.Errorf("injected error") + configSync: &fakeSyncRecorder{ + Returns: &SyncWorkerStatus{ + Step: "Moving", + Reconciling: true, + Completed: 2, + Actual: configv1.Update{Version: "0.0.1-abc", Payload: "payload/image:v4.0.1"}, + Failure: &updateError{ + Reason: "UpdatePayloadIntegrity", + Message: "unable to apply object", + }, + }, }, - client: fake.NewSimpleClientset( + client: fakeClientsetWithUpdates( &configv1.ClusterVersion{ ObjectMeta: metav1.ObjectMeta{ Name: "default", @@ -377,32 +441,28 @@ func TestOperator_sync(t *testing.T) { }, Status: configv1.ClusterVersionStatus{ History: []configv1.UpdateHistory{ - {Version: "4.0.1", Payload: "payload/image:v4.0.1"}, + {Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime}, }, + Desired: configv1.Update{Version: "4.0.1", Payload: "payload/image:v4.0.1"}, VersionHash: "", Conditions: []configv1.ClusterOperatorStatusCondition{ {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Message: "unable to apply object"}, + {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Reason: "UpdatePayloadIntegrity", Message: "unable to apply object"}, {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Working towards 4.0.1"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, }, }, }, ), }, - wantErr: func(t *testing.T, err error) { - if err == nil || err.Error() != "injected error" { - t.Fatalf("unexpected error: %v", err) - } - }, wantActions: func(t *testing.T, optr *Operator) { f := optr.client.(*fake.Clientset) act := f.Actions() - if len(act) != 5 { + if len(act) != 2 { t.Fatalf("unknown actions: %d %#v", len(act), act) } expectGet(t, act[0], "clusterversions", "", "default") - expectGet(t, act[1], "clusterversions", "", "default") - expectUpdateStatus(t, act[2], "clusterversions", "", &configv1.ClusterVersion{ + expectUpdateStatus(t, act[1], "clusterversions", "", &configv1.ClusterVersion{ ObjectMeta: metav1.ObjectMeta{ Name: "default", }, @@ -411,18 +471,69 @@ func TestOperator_sync(t *testing.T) { }, Status: configv1.ClusterVersionStatus{ History: []configv1.UpdateHistory{ - {Version: "4.0.1", Payload: "payload/image:v4.0.1"}, + {State: configv1.CompletedUpdate, Version: "0.0.1-abc", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, + {Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, }, + Desired: configv1.Update{Version: "0.0.1-abc", Payload: "payload/image:v4.0.1"}, VersionHash: "", Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Message: "unable to apply object"}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Unable to apply 4.0.1: unable to apply object"}, + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 0.0.1-abc"}, + {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Reason: "UpdatePayloadIntegrity", Message: "unable to apply object"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Reason: "UpdatePayloadIntegrity", Message: "Error while reconciling 0.0.1-abc: the contents of the update are invalid"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, }, }, }) - expectGet(t, act[3], "clusterversions", "", "default") - expectUpdateStatus(t, act[4], "clusterversions", "", &configv1.ClusterVersion{ + }, + }, + { + name: "progressing and encounters error during payload sync", + content: content1, + optr: Operator{ + releaseVersion: "4.0.1", + releaseImage: "payload/image:v4.0.1", + namespace: "test", + name: "default", + configSync: &fakeSyncRecorder{ + Returns: &SyncWorkerStatus{ + Step: "Moving", + Actual: configv1.Update{Version: "0.0.1-abc", Payload: "payload/image:v4.0.1"}, + Failure: fmt.Errorf("injected error"), + VersionHash: "foo", + }, + }, + client: fake.NewSimpleClientset( + &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + }, + Spec: configv1.ClusterVersionSpec{ + Channel: "fast", + }, + Status: configv1.ClusterVersionStatus{ + History: []configv1.UpdateHistory{ + {Version: "4.0.1", Payload: "payload/image:v4.0.1"}, + }, + VersionHash: "", + Conditions: []configv1.ClusterOperatorStatusCondition{ + {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Message: "unable to apply object"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Working towards 4.0.1"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, + }, + }, + }, + ), + }, + wantActions: func(t *testing.T, optr *Operator) { + f := optr.client.(*fake.Clientset) + act := f.Actions() + if len(act) != 2 { + t.Fatalf("unknown actions: %d %#v", len(act), act) + } + expectGet(t, act[0], "clusterversions", "", "default") + // syncing config status + expectUpdateStatus(t, act[1], "clusterversions", "", &configv1.ClusterVersion{ ObjectMeta: metav1.ObjectMeta{ Name: "default", }, @@ -430,22 +541,28 @@ func TestOperator_sync(t *testing.T) { Channel: "fast", }, Status: configv1.ClusterVersionStatus{ + Desired: configv1.Update{Version: "0.0.1-abc", Payload: "payload/image:v4.0.1"}, History: []configv1.UpdateHistory{ - {Version: "0.0.1-abc", Payload: "payload/image:v4.0.1"}, + {State: configv1.PartialUpdate, Version: "0.0.1-abc", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime}, + {Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: metav1.Time{Time: time.Unix(0, 0)}, CompletionTime: &defaultCompletionTime}, }, - VersionHash: "y_Kc5IQiIyU=", + VersionHash: "foo", Conditions: []configv1.ClusterOperatorStatusCondition{ {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Message: "injected error"}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Unable to apply 4.0.1: injected error"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Unable to apply 0.0.1-abc: an error occurred"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, }, }, }) }, }, { - name: "invalid payload reports payload error", - content: contentWithoutManifests, + name: "invalid payload reports payload error", + syncStatus: &SyncWorkerStatus{ + Failure: os.ErrNotExist, + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, + }, optr: Operator{ releaseVersion: "4.0.1", releaseImage: "payload/image:v4.0.1", @@ -462,20 +579,14 @@ func TestOperator_sync(t *testing.T) { }, ), }, - wantErr: func(t *testing.T, err error) { - if err == nil || !os.IsNotExist(err) { - t.Fatalf("unexpected error: %v", err) - } - }, wantActions: func(t *testing.T, optr *Operator) { f := optr.client.(*fake.Clientset) act := f.Actions() - if len(act) != 5 { + if len(act) != 2 { t.Fatalf("unknown actions: %d %#v", len(act), act) } expectGet(t, act[0], "clusterversions", "", "default") - expectGet(t, act[1], "clusterversions", "", "default") - expectUpdateStatus(t, act[2], "clusterversions", "", &configv1.ClusterVersion{ + expectUpdateStatus(t, act[1], "clusterversions", "", &configv1.ClusterVersion{ ObjectMeta: metav1.ObjectMeta{ Name: "default", }, @@ -483,42 +594,29 @@ func TestOperator_sync(t *testing.T) { Channel: "fast", }, Status: configv1.ClusterVersionStatus{ + Desired: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, History: []configv1.UpdateHistory{ - {Version: "4.0.1", Payload: "payload/image:v4.0.1"}, + {State: configv1.PartialUpdate, Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime}, }, VersionHash: "", Conditions: []configv1.ClusterOperatorStatusCondition{ {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Message: "unable to apply object"}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Unable to apply 4.0.1: unable to apply object"}, - }, - }, - }) - expectGet(t, act[3], "clusterversions", "", "default") - expectUpdateStatus(t, act[4], "clusterversions", "", &configv1.ClusterVersion{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default", - }, - Spec: configv1.ClusterVersionSpec{ - Channel: "fast", - }, - Status: configv1.ClusterVersionStatus{ - History: []configv1.UpdateHistory{ - {Version: "0.0.1-abc", Payload: "payload/image:v4.0.1"}, - }, - VersionHash: "y_Kc5IQiIyU=", - Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 0.0.1-abc"}, - {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: "Cluster version is 0.0.1-abc"}, + {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Message: "file does not exist"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Unable to apply 4.0.1: an error occurred"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, }, }, }) }, }, { - name: "invalid payload while progressing reports payload error and preserves progressing", - content: contentWithoutManifests, + name: "invalid payload while progressing preserves progressing order and partial history", + syncStatus: &SyncWorkerStatus{ + Step: "Working", + Fraction: 0.6, + Failure: os.ErrNotExist, + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, + }, optr: Operator{ releaseVersion: "4.0.1", releaseImage: "payload/image:v4.0.1", @@ -534,6 +632,7 @@ func TestOperator_sync(t *testing.T) { }, Status: configv1.ClusterVersionStatus{ History: []configv1.UpdateHistory{ + // this is a partial history struct, which we will fill out {Version: "4.0.1", Payload: "payload/image:v4.0.1"}, }, VersionHash: "", @@ -544,20 +643,14 @@ func TestOperator_sync(t *testing.T) { }, ), }, - wantErr: func(t *testing.T, err error) { - if err == nil || !os.IsNotExist(err) { - t.Fatalf("unexpected error: %v", err) - } - }, wantActions: func(t *testing.T, optr *Operator) { f := optr.client.(*fake.Clientset) act := f.Actions() - if len(act) != 5 { + if len(act) != 2 { t.Fatalf("unknown actions: %d %#v", len(act), act) } expectGet(t, act[0], "clusterversions", "", "default") - expectGet(t, act[1], "clusterversions", "", "default") - expectUpdateStatus(t, act[2], "clusterversions", "", &configv1.ClusterVersion{ + expectUpdateStatus(t, act[1], "clusterversions", "", &configv1.ClusterVersion{ ObjectMeta: metav1.ObjectMeta{ Name: "default", }, @@ -565,42 +658,28 @@ func TestOperator_sync(t *testing.T) { Channel: "fast", }, Status: configv1.ClusterVersionStatus{ + Desired: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, History: []configv1.UpdateHistory{ - {Version: "4.0.1", Payload: "payload/image:v4.0.1"}, + // we populate state, but not startedTime + {State: configv1.PartialUpdate, Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: metav1.Time{time.Unix(0, 0)}}, }, VersionHash: "", Conditions: []configv1.ClusterOperatorStatusCondition{ + // the order of progressing in the conditions array is preserved + {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Unable to apply 4.0.1: an error occurred"}, {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Message: "unable to apply object"}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Unable to apply 4.0.1: unable to apply object"}, - }, - }, - }) - expectGet(t, act[3], "clusterversions", "", "default") - expectUpdateStatus(t, act[4], "clusterversions", "", &configv1.ClusterVersion{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default", - }, - Spec: configv1.ClusterVersionSpec{ - Channel: "fast", - }, - Status: configv1.ClusterVersionStatus{ - History: []configv1.UpdateHistory{ - {Version: "0.0.1-abc", Payload: "payload/image:v4.0.1"}, - }, - VersionHash: "y_Kc5IQiIyU=", - Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 0.0.1-abc"}, - {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: "Cluster version is 0.0.1-abc"}, + {Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Message: "file does not exist"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, }, }, }) }, }, { - name: "set initial status conditions", - content: content1, + name: "set initial status conditions", + syncStatus: &SyncWorkerStatus{ + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: ""}, + }, optr: Operator{ releaseImage: "payload/image:v4.0.1", namespace: "test", @@ -639,15 +718,15 @@ func TestOperator_sync(t *testing.T) { State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", Version: "", // we don't know our payload yet and releaseVersion is unset - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, + StartedTime: defaultStartedTime, }, }, - Desired: configv1.Update{Payload: "payload/image:v4.0.1"}, + Desired: configv1.Update{Version: "", Payload: "payload/image:v4.0.1"}, VersionHash: "", Conditions: []configv1.ClusterOperatorStatusCondition{ {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Initializing, will work towards payload/image:v4.0.1"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Working towards payload/image:v4.0.1"}, {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, }, }, @@ -655,8 +734,10 @@ func TestOperator_sync(t *testing.T) { }, }, { - name: "record a new version entry if the controller is restarted with a new image", - content: content1, + name: "record a new version entry if the controller is restarted with a new image", + syncStatus: &SyncWorkerStatus{ + Actual: configv1.Update{Payload: "payload/image:v4.0.2", Version: "4.0.2"}, + }, optr: Operator{ releaseImage: "payload/image:v4.0.2", releaseVersion: "4.0.2", @@ -678,7 +759,7 @@ func TestOperator_sync(t *testing.T) { State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", Version: "", // we didn't know our payload before - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, + StartedTime: defaultStartedTime, }, }, Desired: configv1.Update{Payload: "payload/image:v4.0.1"}, @@ -714,14 +795,14 @@ func TestOperator_sync(t *testing.T) { State: configv1.PartialUpdate, Payload: "payload/image:v4.0.2", Version: "4.0.2", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, + StartedTime: defaultStartedTime, }, { State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", Version: "", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, + StartedTime: defaultStartedTime, + CompletionTime: &defaultCompletionTime, }, }, Desired: configv1.Update{Payload: "payload/image:v4.0.2", Version: "4.0.2"}, @@ -729,7 +810,7 @@ func TestOperator_sync(t *testing.T) { Conditions: []configv1.ClusterOperatorStatusCondition{ {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Initializing, will work towards payload/image:v4.0.1"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Working towards 4.0.2"}, {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, }, }, @@ -737,8 +818,16 @@ func TestOperator_sync(t *testing.T) { }, }, { - name: "when user cancels desired update, clear status desired", - content: content1, + name: "when user cancels desired update, clear status desired", + syncStatus: &SyncWorkerStatus{ + // TODO: we can't actually react to spec changes in a single sync round + // because the sync worker updates desired state and cancels under the + // lock, so the sync worker loop will never report the status of the + // update unless we add some sort of delay - which might make clearing status + // slightly more useful to the user (instead of two status updates you get + // one). + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, + }, optr: Operator{ releaseImage: "payload/image:v4.0.1", releaseVersion: "4.0.1", @@ -760,14 +849,14 @@ func TestOperator_sync(t *testing.T) { State: configv1.PartialUpdate, Payload: "payload/image:v4.0.2", Version: "4.0.2", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, + StartedTime: defaultStartedTime, }, { State: configv1.CompletedUpdate, Payload: "payload/image:v4.0.1", Version: "4.0.1", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, + StartedTime: defaultStartedTime, + CompletionTime: &defaultCompletionTime, }, }, Desired: configv1.Update{Payload: "payload/image:v4.0.2"}, @@ -803,21 +892,21 @@ func TestOperator_sync(t *testing.T) { State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", Version: "4.0.1", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, + StartedTime: defaultStartedTime, }, { State: configv1.PartialUpdate, Payload: "payload/image:v4.0.2", Version: "4.0.2", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, + StartedTime: defaultStartedTime, + CompletionTime: &defaultCompletionTime, }, { State: configv1.CompletedUpdate, Payload: "payload/image:v4.0.1", Version: "4.0.1", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, + StartedTime: defaultStartedTime, + CompletionTime: &defaultCompletionTime, }, }, Desired: configv1.Update{ @@ -829,7 +918,7 @@ func TestOperator_sync(t *testing.T) { {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, // we don't reset the message here until the payload is loaded - {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Working towards 4.0.2"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Working towards 4.0.1"}, {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, }, }, @@ -837,8 +926,10 @@ func TestOperator_sync(t *testing.T) { }, }, { - name: "after desired update is cancelled, go to reconciling", - content: content_4_0_1, + name: "after desired update is cancelled, go to reconciling", + syncStatus: &SyncWorkerStatus{ + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, + }, optr: Operator{ releaseImage: "payload/image:v4.0.1", releaseVersion: "4.0.1", @@ -860,21 +951,21 @@ func TestOperator_sync(t *testing.T) { State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", Version: "4.0.1", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, + StartedTime: defaultStartedTime, }, { State: configv1.PartialUpdate, Payload: "payload/image:v4.0.2", Version: "4.0.2", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, + StartedTime: defaultStartedTime, + CompletionTime: &defaultCompletionTime, }, { State: configv1.CompletedUpdate, Payload: "payload/image:v4.0.1", Version: "4.0.1", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, + StartedTime: defaultStartedTime, + CompletionTime: &defaultCompletionTime, }, }, Desired: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, @@ -892,7 +983,7 @@ func TestOperator_sync(t *testing.T) { wantActions: func(t *testing.T, optr *Operator) { f := optr.client.(*fake.Clientset) act := f.Actions() - if len(act) != 3 { + if len(act) != 2 { t.Fatalf("unknown actions: %d %#v", len(act), act) } expectGet(t, act[0], "clusterversions", "", "default") @@ -911,21 +1002,21 @@ func TestOperator_sync(t *testing.T) { State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", Version: "4.0.1", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, + StartedTime: defaultStartedTime, }, { State: configv1.PartialUpdate, Payload: "payload/image:v4.0.2", Version: "4.0.2", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, + StartedTime: defaultStartedTime, + CompletionTime: &defaultCompletionTime, }, { State: configv1.CompletedUpdate, Payload: "payload/image:v4.0.1", Version: "4.0.1", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, + StartedTime: defaultStartedTime, + CompletionTime: &defaultCompletionTime, }, }, Desired: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, @@ -939,57 +1030,14 @@ func TestOperator_sync(t *testing.T) { }, }, }) - expectUpdateStatus(t, act[2], "clusterversions", "", &configv1.ClusterVersion{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default", - ResourceVersion: "2", - }, - Spec: configv1.ClusterVersionSpec{ - Upstream: configv1.URL("http://localhost:8080/graph"), - Channel: "fast", - }, - Status: configv1.ClusterVersionStatus{ - History: []configv1.UpdateHistory{ - { - State: configv1.CompletedUpdate, - Payload: "payload/image:v4.0.1", - Version: "4.0.1", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, - }, - { - State: configv1.PartialUpdate, - Payload: "payload/image:v4.0.2", - Version: "4.0.2", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, - }, - { - State: configv1.CompletedUpdate, - Payload: "payload/image:v4.0.1", - Version: "4.0.1", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, - }, - }, - Desired: configv1.Update{ - Version: "4.0.1", - Payload: "payload/image:v4.0.1", - }, - VersionHash: "y_Kc5IQiIyU=", - Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 4.0.1"}, - {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: "Cluster version is 4.0.1"}, - {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, - }, - }, - }) }, }, { - name: "after initial status is set, set reconciling and hash and correct version number", - content: content1, + name: "after initial status is set, set hash and correct version number", + syncStatus: &SyncWorkerStatus{ + VersionHash: "xyz", + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, + }, optr: Operator{ releaseImage: "payload/image:v4.0.1", namespace: "test", @@ -1009,7 +1057,7 @@ func TestOperator_sync(t *testing.T) { { State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, + StartedTime: defaultStartedTime, }, }, Desired: configv1.Update{Payload: "payload/image:v4.0.1"}, @@ -1026,7 +1074,7 @@ func TestOperator_sync(t *testing.T) { wantActions: func(t *testing.T, optr *Operator) { f := optr.client.(*fake.Clientset) act := f.Actions() - if len(act) != 3 { + if len(act) != 2 { t.Fatalf("unknown actions: %d %#v", len(act), act) } expectGet(t, act[0], "clusterversions", "", "default") @@ -1042,10 +1090,10 @@ func TestOperator_sync(t *testing.T) { }, Status: configv1.ClusterVersionStatus{ History: []configv1.UpdateHistory{ - {State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", Version: "0.0.1-abc", StartedTime: metav1.Time{Time: time.Unix(1, 0)}}, + {State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", Version: "0.0.1-abc", StartedTime: defaultStartedTime}, }, Desired: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, - VersionHash: "", + VersionHash: "xyz", Conditions: []configv1.ClusterOperatorStatusCondition{ {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, @@ -1054,44 +1102,21 @@ func TestOperator_sync(t *testing.T) { }, }, }) - expectUpdateStatus(t, act[2], "clusterversions", "", &configv1.ClusterVersion{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default", - ResourceVersion: "2", - }, - Spec: configv1.ClusterVersionSpec{ - Upstream: configv1.URL("http://localhost:8080/graph"), - Channel: "fast", - }, - Status: configv1.ClusterVersionStatus{ - History: []configv1.UpdateHistory{ - {State: configv1.CompletedUpdate, Payload: "payload/image:v4.0.1", Version: "0.0.1-abc", StartedTime: metav1.Time{Time: time.Unix(1, 0)}, CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}}, - }, - Desired: configv1.Update{ - Version: "0.0.1-abc", - Payload: "payload/image:v4.0.1", - }, - VersionHash: "y_Kc5IQiIyU=", - Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 0.0.1-abc"}, - {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: "Cluster version is 0.0.1-abc"}, - {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, - }, - }, - }) }, }, { - name: "version is live and was recently synced, do nothing", - content: content1, + name: "version is live and was recently synced, do nothing", + syncStatus: &SyncWorkerStatus{ + Reconciling: true, + Completed: 1, + VersionHash: "xyz", + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, + }, optr: Operator{ - minimumUpdateCheckInterval: 1 * time.Minute, - lastSyncAt: time.Now(), - releaseImage: "payload/image:v4.0.1", - releaseVersion: "0.0.1-abc", - namespace: "test", - name: "default", + releaseImage: "payload/image:v4.0.1", + releaseVersion: "0.0.1-abc", + namespace: "test", + name: "default", client: fakeClientsetWithUpdates( &configv1.ClusterVersion{ ObjectMeta: metav1.ObjectMeta{ @@ -1109,16 +1134,17 @@ func TestOperator_sync(t *testing.T) { State: configv1.CompletedUpdate, Payload: "payload/image:v4.0.1", Version: "0.0.1-abc", - CompletionTime: &metav1.Time{Time: time.Unix(1, 0)}, + CompletionTime: &defaultStartedTime, }, }, Desired: configv1.Update{ Version: "0.0.1-abc", Payload: "payload/image:v4.0.1", }, - Generation: 2, + VersionHash: "xyz", + Generation: 2, Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue}, + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 0.0.1-abc"}, {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: "Cluster version is 0.0.1-abc"}, {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, @@ -1131,20 +1157,22 @@ func TestOperator_sync(t *testing.T) { f := optr.client.(*fake.Clientset) act := f.Actions() if len(act) != 1 { - t.Fatalf("unknown actions: %d %#v", len(act), act) + t.Fatalf("unexpected actions %d: %s", len(act), spew.Sdump(act)) } expectGet(t, act[0], "clusterversions", "", "default") }, }, { - name: "new available updates, version is live and was recently synced, sync", - content: content1, + name: "new available updates, version is live and was recently synced, sync", + syncStatus: &SyncWorkerStatus{ + Reconciling: true, + Completed: 1, + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, + }, optr: Operator{ - minimumUpdateCheckInterval: 1 * time.Minute, - lastSyncAt: time.Now(), - releaseImage: "payload/image:v4.0.1", - namespace: "test", - name: "default", + releaseImage: "payload/image:v4.0.1", + namespace: "test", + name: "default", availableUpdates: &availableUpdates{ Upstream: "http://localhost:8080/graph", Channel: "fast", @@ -1201,14 +1229,12 @@ func TestOperator_sync(t *testing.T) { {Version: "4.0.3", Payload: "test/image:2"}, }, History: []configv1.UpdateHistory{ - {State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", StartedTime: metav1.Time{Time: time.Unix(1, 0)}}, - }, - Desired: configv1.Update{ - Payload: "payload/image:v4.0.1", + {State: configv1.CompletedUpdate, Version: "0.0.1-abc", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, }, + Desired: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, Generation: 2, Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 0.0.1-abc"}, {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: "Cluster version is 0.0.1-abc"}, {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, {Type: configv1.RetrievedUpdates, Status: configv1.ConditionTrue}, @@ -1218,15 +1244,17 @@ func TestOperator_sync(t *testing.T) { }, }, { - name: "new available updates for the default upstream URL, client has no upstream", - content: content1, + name: "new available updates for the default upstream URL, client has no upstream", + syncStatus: &SyncWorkerStatus{ + Reconciling: true, + Completed: 1, + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, + }, optr: Operator{ - minimumUpdateCheckInterval: 1 * time.Minute, - lastSyncAt: time.Now(), - releaseImage: "payload/image:v4.0.1", - namespace: "test", - name: "default", - defaultUpstreamServer: "http://localhost:8080/graph", + releaseImage: "payload/image:v4.0.1", + namespace: "test", + name: "default", + defaultUpstreamServer: "http://localhost:8080/graph", availableUpdates: &availableUpdates{ Upstream: "", Channel: "fast", @@ -1283,14 +1311,12 @@ func TestOperator_sync(t *testing.T) { {Version: "4.0.3", Payload: "test/image:2"}, }, History: []configv1.UpdateHistory{ - {State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", StartedTime: metav1.Time{Time: time.Unix(1, 0)}}, - }, - Desired: configv1.Update{ - Payload: "payload/image:v4.0.1", + {State: configv1.CompletedUpdate, Version: "0.0.1-abc", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, }, + Desired: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, Generation: 2, Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 0.0.1-abc"}, {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: "Cluster version is 0.0.1-abc"}, {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, {Type: configv1.RetrievedUpdates, Status: configv1.ConditionTrue}, @@ -1300,14 +1326,16 @@ func TestOperator_sync(t *testing.T) { }, }, { - name: "new available updates but for a different channel", - content: content1, + name: "new available updates but for a different channel", + syncStatus: &SyncWorkerStatus{ + Reconciling: true, + Completed: 1, + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, + }, optr: Operator{ - minimumUpdateCheckInterval: 1 * time.Minute, - lastSyncAt: time.Now(), - releaseImage: "payload/image:v4.0.1", - namespace: "test", - name: "default", + releaseImage: "payload/image:v4.0.1", + namespace: "test", + name: "default", availableUpdates: &availableUpdates{ Upstream: "http://localhost:8080/graph", Channel: "fast", @@ -1360,14 +1388,12 @@ func TestOperator_sync(t *testing.T) { }, Status: configv1.ClusterVersionStatus{ History: []configv1.UpdateHistory{ - {State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", StartedTime: metav1.Time{Time: time.Unix(1, 0)}}, - }, - Desired: configv1.Update{ - Payload: "payload/image:v4.0.1", + {State: configv1.CompletedUpdate, Version: "0.0.1-abc", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, }, + Desired: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, Generation: 2, Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 0.0.1-abc"}, {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: "Cluster version is 0.0.1-abc"}, {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, @@ -1377,8 +1403,73 @@ func TestOperator_sync(t *testing.T) { }, }, { - name: "user requested a version that isn't in the updates or history", - content: content1, + name: "user requested a version, sync loop hasn't started", + syncStatus: &SyncWorkerStatus{ + Reconciling: true, + Completed: 1, + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, + }, + optr: Operator{ + releaseImage: "payload/image:v4.0.1", + namespace: "test", + name: "default", + client: fakeClientsetWithUpdates(&configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Generation: 2, + }, + Spec: configv1.ClusterVersionSpec{ + ClusterID: configv1.ClusterID(id), + DesiredUpdate: &configv1.Update{ + Payload: "payload/image:v4.0.2", + }, + }, + }), + }, + wantSync: []configv1.Update{ + {Payload: "payload/image:v4.0.2", Version: ""}, + }, + wantActions: func(t *testing.T, optr *Operator) { + f := optr.client.(*fake.Clientset) + act := f.Actions() + if len(act) != 2 { + t.Fatalf("unknown actions: %d %#v", len(act), act) + } + expectGet(t, act[0], "clusterversions", "", "default") + expectUpdateStatus(t, act[1], "clusterversions", "", &configv1.ClusterVersion{ + ObjectMeta: metav1.ObjectMeta{ + Name: "default", + Generation: 2, + }, + Spec: configv1.ClusterVersionSpec{ + ClusterID: configv1.ClusterID(id), + DesiredUpdate: &configv1.Update{ + Payload: "payload/image:v4.0.2", + }, + }, + Status: configv1.ClusterVersionStatus{ + History: []configv1.UpdateHistory{ + {State: configv1.CompletedUpdate, Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, + }, + Desired: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, + Generation: 2, + Conditions: []configv1.ClusterOperatorStatusCondition{ + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 4.0.1"}, + {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Message: "Cluster version is 4.0.1"}, + {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, + }, + }, + }) + }, + }, + { + name: "user requested a version that isn't in the updates or history", + syncStatus: &SyncWorkerStatus{ + Reconciling: true, + Completed: 1, + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, + }, optr: Operator{ releaseImage: "payload/image:v4.0.1", namespace: "test", @@ -1418,36 +1509,39 @@ func TestOperator_sync(t *testing.T) { Spec: configv1.ClusterVersionSpec{ ClusterID: configv1.ClusterID(id), Upstream: configv1.URL("http://localhost:8080/graph"), - DesiredUpdate: &configv1.Update{ - Version: "4.0.4", - }, + // The object passed to status update is the one with desired update cleared + // DesiredUpdate: &configv1.Update{ + // Version: "4.0.4", + // }, }, Status: configv1.ClusterVersionStatus{ History: []configv1.UpdateHistory{ - {State: configv1.PartialUpdate, Version: "4.0.4", StartedTime: metav1.Time{Time: time.Unix(1, 0)}}, - {State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", StartedTime: metav1.Time{Time: time.Unix(1, 0)}, CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}}, - }, - Desired: configv1.Update{ - Version: "4.0.4", + {State: configv1.CompletedUpdate, Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, }, + Desired: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, AvailableUpdates: []configv1.Update{ {Version: "4.0.2", Payload: "test/image:1"}, {Version: "4.0.3", Payload: "test/image:2"}, }, + Generation: 2, Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, + {Type: ClusterVersionInvalid, Status: configv1.ConditionTrue, Reason: "InvalidClusterVersion", Message: "The cluster version is invalid: spec.desiredUpdate.version: Invalid value: \"4.0.4\": when payload is empty the update must be a previous version or an available update"}, + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 4.0.1"}, {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Reason: "InvalidClusterVersion", Message: "Stopped at payload/image:v4.0.1: the cluster version is invalid"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Reason: "InvalidClusterVersion", Message: "Stopped at 4.0.1: the cluster version is invalid"}, {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, - {Type: ClusterVersionInvalid, Status: configv1.ConditionTrue, Reason: "InvalidClusterVersion", Message: "The cluster version is invalid: spec.desiredUpdate.version: Invalid value: \"4.0.4\": when payload is empty the update must be a previous version or an available update"}, }, }, }) }, }, { - name: "user requested a version has duplicates", - content: content1, + name: "user requested a version has duplicates", + syncStatus: &SyncWorkerStatus{ + Reconciling: true, + Completed: 1, + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, + }, optr: Operator{ releaseImage: "payload/image:v4.0.1", namespace: "test", @@ -1488,37 +1582,41 @@ func TestOperator_sync(t *testing.T) { Spec: configv1.ClusterVersionSpec{ ClusterID: configv1.ClusterID(id), Upstream: configv1.URL("http://localhost:8080/graph"), - DesiredUpdate: &configv1.Update{ - Version: "4.0.3", - }, + // The object passed to status update is the one with desired update cleared + // DesiredUpdate: &configv1.Update{ + // Version: "4.0.4", + // }, }, Status: configv1.ClusterVersionStatus{ + Generation: 2, History: []configv1.UpdateHistory{ - {State: configv1.PartialUpdate, Version: "4.0.3", StartedTime: metav1.Time{Time: time.Unix(1, 0)}}, - {State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", StartedTime: metav1.Time{Time: time.Unix(1, 0)}, CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}}, - }, - Desired: configv1.Update{ - Version: "4.0.3", + {State: configv1.CompletedUpdate, Version: "4.0.1", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, }, + Desired: configv1.Update{Payload: "payload/image:v4.0.1", Version: "4.0.1"}, AvailableUpdates: []configv1.Update{ {Version: "4.0.2", Payload: "test/image:1"}, {Version: "4.0.3", Payload: "test/image:2"}, {Version: "4.0.3", Payload: "test/image:3"}, }, Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, + {Type: ClusterVersionInvalid, Status: configv1.ConditionTrue, Reason: "InvalidClusterVersion", Message: "The cluster version is invalid: spec.desiredUpdate.version: Invalid value: \"4.0.3\": there are multiple possible payloads for this version, specify the exact payload"}, + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 4.0.1"}, {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Reason: "InvalidClusterVersion", Message: "Stopped at payload/image:v4.0.1: the cluster version is invalid"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Reason: "InvalidClusterVersion", Message: "Stopped at 4.0.1: the cluster version is invalid"}, {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, - {Type: ClusterVersionInvalid, Status: configv1.ConditionTrue, Reason: "InvalidClusterVersion", Message: "The cluster version is invalid: spec.desiredUpdate.version: Invalid value: \"4.0.3\": there are multiple possible payloads for this version, specify the exact payload"}, }, }, }) }, }, { - name: "payload hash matches content hash, act as reconcile, no need to apply", - content: content1, + name: "payload hash matches content hash, act as reconcile, no need to apply", + syncStatus: &SyncWorkerStatus{ + Reconciling: true, + Completed: 1, + VersionHash: "y_Kc5IQiIyU=", + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, + }, optr: Operator{ releaseImage: "payload/image:v4.0.1", releaseVersion: "0.0.1-abc", @@ -1542,7 +1640,7 @@ func TestOperator_sync(t *testing.T) { State: configv1.CompletedUpdate, Payload: "payload/image:v4.0.1", Version: "0.0.1-abc", - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, + CompletionTime: &defaultCompletionTime, }, }, Desired: configv1.Update{ @@ -1565,14 +1663,19 @@ func TestOperator_sync(t *testing.T) { f := optr.client.(*fake.Clientset) act := f.Actions() if len(act) != 1 { - t.Fatalf("unknown actions: %d %#v", len(act), act) + t.Fatalf("unknown actions: %d %s", len(act), spew.Sdump(act)) } expectGet(t, act[0], "clusterversions", "", "default") }, }, { - name: "payload hash does not match content hash, act as reconcile, no need to apply", - content: content1, + name: "payload hash does not match content hash, act as reconcile, no need to apply", + syncStatus: &SyncWorkerStatus{ + Reconciling: true, + Completed: 1, + VersionHash: "y_Kc5IQiIyU=", + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, + }, optr: Operator{ releaseImage: "payload/image:v4.0.1", releaseVersion: "0.0.1-abc", @@ -1596,7 +1699,7 @@ func TestOperator_sync(t *testing.T) { State: configv1.CompletedUpdate, Payload: "payload/image:v4.0.1", Version: "0.0.1-abc", - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, + CompletionTime: &defaultCompletionTime, }, }, Desired: configv1.Update{ @@ -1618,8 +1721,8 @@ func TestOperator_sync(t *testing.T) { wantActions: func(t *testing.T, optr *Operator) { f := optr.client.(*fake.Clientset) act := f.Actions() - if len(act) != 3 { - t.Fatalf("unknown actions: %d %#v", len(act), act) + if len(act) != 2 { + t.Fatalf("unknown actions: %d %s", len(act), spew.Sdump(act)) } expectGet(t, act[0], "clusterversions", "", "default") expectUpdateStatus(t, act[1], "clusterversions", "", &configv1.ClusterVersion{ @@ -1637,43 +1740,12 @@ func TestOperator_sync(t *testing.T) { State: configv1.CompletedUpdate, Payload: "payload/image:v4.0.1", Version: "0.0.1-abc", - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, + CompletionTime: &defaultCompletionTime, + StartedTime: metav1.Time{time.Unix(0, 0)}, }, }, Desired: configv1.Update{Version: "0.0.1-abc", Payload: "payload/image:v4.0.1"}, Generation: 2, - VersionHash: "unknown_hash", - Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Message: "Working towards 0.0.1-abc"}, - {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, - }, - }, - }) - expectUpdateStatus(t, act[2], "clusterversions", "", &configv1.ClusterVersion{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default", - Generation: 2, - }, - Spec: configv1.ClusterVersionSpec{ - Upstream: configv1.URL("http://localhost:8080/graph"), - Channel: "fast", - }, - Status: configv1.ClusterVersionStatus{ - History: []configv1.UpdateHistory{ - { - State: configv1.CompletedUpdate, - Payload: "payload/image:v4.0.1", - Version: "0.0.1-abc", - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, - }, - }, - Desired: configv1.Update{ - Version: "0.0.1-abc", - Payload: "payload/image:v4.0.1", - }, - Generation: 2, VersionHash: "y_Kc5IQiIyU=", Conditions: []configv1.ClusterOperatorStatusCondition{ {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 0.0.1-abc"}, @@ -1687,8 +1759,12 @@ func TestOperator_sync(t *testing.T) { }, { - name: "detect invalid cluster version", - content: content1, + name: "detect invalid cluster version", + syncStatus: &SyncWorkerStatus{ + Reconciling: true, + Completed: 1, + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, + }, optr: Operator{ releaseImage: "payload/image:v4.0.1", namespace: "test", @@ -1718,24 +1794,25 @@ func TestOperator_sync(t *testing.T) { ResourceVersion: "1", }, Spec: configv1.ClusterVersionSpec{ - ClusterID: "not-valid-cluster-id", - Upstream: configv1.URL("#%GG"), - Channel: "fast", + // The object passed to status has these spec fields cleared + // ClusterID: "not-valid-cluster-id", + // Upstream: configv1.URL("#%GG"), + Channel: "fast", }, Status: configv1.ClusterVersionStatus{ History: []configv1.UpdateHistory{ - {State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", StartedTime: metav1.Time{Time: time.Unix(1, 0)}}, + {State: configv1.CompletedUpdate, Version: "0.0.1-abc", Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime, CompletionTime: &defaultCompletionTime}, }, Desired: configv1.Update{ - Payload: "payload/image:v4.0.1", + Version: "0.0.1-abc", Payload: "payload/image:v4.0.1", }, VersionHash: "", Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, + {Type: ClusterVersionInvalid, Status: configv1.ConditionTrue, Reason: "InvalidClusterVersion", Message: "The cluster version is invalid:\n* spec.upstream: Invalid value: \"#%GG\": must be a valid URL or empty\n* spec.clusterID: Invalid value: \"not-valid-cluster-id\": must be an RFC4122-variant UUID\n"}, + {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 0.0.1-abc"}, {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Reason: "InvalidClusterVersion", Message: "Stopped at payload/image:v4.0.1: the cluster version is invalid"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Reason: "InvalidClusterVersion", Message: "Stopped at 0.0.1-abc: the cluster version is invalid"}, {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, - {Type: ClusterVersionInvalid, Status: configv1.ConditionTrue, Reason: "InvalidClusterVersion", Message: "The cluster version is invalid:\n* spec.upstream: Invalid value: \"#%GG\": must be a valid URL or empty\n* spec.clusterID: Invalid value: \"not-valid-cluster-id\": must be an RFC4122-variant UUID\n"}, }, }, }) @@ -1743,8 +1820,10 @@ func TestOperator_sync(t *testing.T) { }, { - name: "invalid cluster version should not block initial sync", - content: content1, + name: "invalid cluster version should not block initial sync", + syncStatus: &SyncWorkerStatus{ + Actual: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, + }, optr: Operator{ releaseImage: "payload/image:v4.0.1", namespace: "test", @@ -1761,7 +1840,7 @@ func TestOperator_sync(t *testing.T) { }, Status: configv1.ClusterVersionStatus{ History: []configv1.UpdateHistory{ - {Payload: "payload/image:v4.0.1", StartedTime: metav1.Time{Time: time.Unix(1, 0)}}, + {Payload: "payload/image:v4.0.1", StartedTime: defaultStartedTime}, }, Desired: configv1.Update{Payload: "payload/image:v4.0.1"}, VersionHash: "", @@ -1775,10 +1854,14 @@ func TestOperator_sync(t *testing.T) { }, }), }, + wantSync: []configv1.Update{ + // set by the operator + {Payload: "payload/image:v4.0.1", Version: ""}, + }, wantActions: func(t *testing.T, optr *Operator) { f := optr.client.(*fake.Clientset) act := f.Actions() - if len(act) != 3 { + if len(act) != 2 { t.Fatalf("unknown actions: %d %#v", len(act), act) } expectGet(t, act[0], "clusterversions", "", "default") @@ -1796,50 +1879,14 @@ func TestOperator_sync(t *testing.T) { }, Status: configv1.ClusterVersionStatus{ History: []configv1.UpdateHistory{ - {Payload: "payload/image:v4.0.1", Version: "0.0.1-abc", StartedTime: metav1.Time{Time: time.Unix(1, 0)}}, + {State: configv1.PartialUpdate, Payload: "payload/image:v4.0.1", Version: "0.0.1-abc", StartedTime: defaultStartedTime}, }, Desired: configv1.Update{Payload: "payload/image:v4.0.1", Version: "0.0.1-abc"}, VersionHash: "", Conditions: []configv1.ClusterOperatorStatusCondition{ {Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse}, {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Reason: "", Message: "Reconciling 0.0.1-abc: the cluster version is invalid"}, - {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, - {Type: ClusterVersionInvalid, Status: configv1.ConditionTrue, Reason: "InvalidClusterVersion", Message: "The cluster version is invalid:\n* spec.upstream: Invalid value: \"#%GG\": must be a valid URL or empty\n* spec.clusterID: Invalid value: \"not-valid-cluster-id\": must be an RFC4122-variant UUID\n"}, - }, - }, - }) - expectUpdateStatus(t, act[2], "clusterversions", "", &configv1.ClusterVersion{ - ObjectMeta: metav1.ObjectMeta{ - Name: "default", - ResourceVersion: "2", - }, - Spec: configv1.ClusterVersionSpec{ - // fields are cleared when passed to the client (although server will ignore spec changes) - ClusterID: "", - Upstream: configv1.URL(""), - - Channel: "fast", - }, - Status: configv1.ClusterVersionStatus{ - History: []configv1.UpdateHistory{ - { - State: configv1.CompletedUpdate, - Payload: "payload/image:v4.0.1", - Version: "0.0.1-abc", - StartedTime: metav1.Time{Time: time.Unix(1, 0)}, - CompletionTime: &metav1.Time{Time: time.Unix(2, 0)}, - }, - }, - Desired: configv1.Update{ - Version: "0.0.1-abc", - Payload: "payload/image:v4.0.1", - }, - VersionHash: "y_Kc5IQiIyU=", - Conditions: []configv1.ClusterOperatorStatusCondition{ - {Type: configv1.OperatorAvailable, Status: configv1.ConditionTrue, Message: "Done applying 0.0.1-abc"}, - {Type: configv1.OperatorFailing, Status: configv1.ConditionFalse}, - {Type: configv1.OperatorProgressing, Status: configv1.ConditionFalse, Reason: "InvalidClusterVersion", Message: "Stopped at 0.0.1-abc: the cluster version is invalid"}, + {Type: configv1.OperatorProgressing, Status: configv1.ConditionTrue, Reason: "InvalidClusterVersion", Message: "Reconciling 0.0.1-abc: the cluster version is invalid"}, {Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse}, {Type: ClusterVersionInvalid, Status: configv1.ConditionTrue, Reason: "InvalidClusterVersion", Message: "The cluster version is invalid:\n* spec.upstream: Invalid value: \"#%GG\": must be a valid URL or empty\n* spec.clusterID: Invalid value: \"not-valid-cluster-id\": must be an RFC4122-variant UUID\n"}, }, @@ -1855,21 +1902,23 @@ func TestOperator_sync(t *testing.T) { tt.init(optr) } optr.cvLister = &clientCVLister{client: optr.client} - if optr.updatePayloadHandler == nil { - optr.updatePayloadHandler = func(config *configv1.ClusterVersion, payload *updatePayload) error { - return nil - } - } optr.clusterOperatorLister = &clientCOLister{client: optr.client} dir, err := ioutil.TempDir("", "cvo-test") if err != nil { t.Fatal(err) } defer os.RemoveAll(dir) - optr.payloadDir = dir if err := createContent(dir, tt.content); err != nil { t.Fatal(err) } + if optr.configSync == nil { + expectStatus := tt.syncStatus + if expectStatus == nil { + expectStatus = &SyncWorkerStatus{} + } + optr.configSync = &fakeSyncRecorder{Returns: expectStatus} + } + err = optr.sync(optr.queueKey()) if err != nil && tt.wantErr == nil { t.Fatalf("Operator.sync() unexpected error: %v", err) @@ -1883,6 +1932,12 @@ func TestOperator_sync(t *testing.T) { if tt.wantActions != nil { tt.wantActions(t, optr) } + if tt.wantSync != nil { + actual := optr.configSync.(*fakeSyncRecorder).Updates + if !reflect.DeepEqual(tt.wantSync, actual) { + t.Fatalf("Unexpected updates: %#v", actual) + } + } }) } } @@ -2228,10 +2283,49 @@ func TestOperator_availableUpdatesSync(t *testing.T) { } } -func createContent(baseDir string, content map[string]interface{}) error { +var reVariable = regexp.MustCompile(`\$\([a-zA-Z0-9_\-]+\)`) + +func TestCreateContentReplacement(t *testing.T) { + replacements := []map[string]string{ + {"NS": "other"}, + } + in := `Some stuff $(NS) that should be $(NS)` + out := reVariable.ReplaceAllStringFunc(in, func(key string) string { + key = key[2 : len(key)-1] + for _, r := range replacements { + v, ok := r[key] + if !ok { + continue + } + return v + } + return key + }) + if out != `Some stuff other that should be other` { + t.Fatal(out) + } +} + +func createContent(baseDir string, content map[string]interface{}, replacements ...map[string]string) error { + if err := os.MkdirAll(baseDir, 0750); err != nil { + return err + } for k, v := range content { switch t := v.(type) { case string: + if len(replacements) > 0 { + t = reVariable.ReplaceAllStringFunc(t, func(key string) string { + key = key[2 : len(key)-1] + for _, r := range replacements { + v, ok := r[key] + if !ok { + continue + } + return v + } + return key + }) + } if err := ioutil.WriteFile(filepath.Join(baseDir, k), []byte(t), 0640); err != nil { return err } @@ -2240,7 +2334,7 @@ func createContent(baseDir string, content map[string]interface{}) error { if err := os.Mkdir(dir, 0750); err != nil { return err } - if err := createContent(dir, t); err != nil { + if err := createContent(dir, t, replacements...); err != nil { return err } } @@ -2248,6 +2342,18 @@ func createContent(baseDir string, content map[string]interface{}) error { return nil } +type mapPayloadRetriever struct { + Paths map[string]string +} + +func (r *mapPayloadRetriever) RetrievePayload(ctx context.Context, update configv1.Update) (string, error) { + path, ok := r.Paths[update.Payload] + if !ok { + return "", fmt.Errorf("no payload found for %q", update.Payload) + } + return path, nil +} + func expectGet(t *testing.T, a ktesting.Action, resource, namespace, name string) { t.Helper() if "get" != a.GetVerb() { @@ -2306,7 +2412,9 @@ func expectMutation(t *testing.T, a ktesting.Action, verb string, resource, subr in.Status.Conditions[i].LastTransitionTime.Time = time.Time{} } for i, item := range in.Status.History { - if !item.StartedTime.IsZero() { + if item.StartedTime.IsZero() { + in.Status.History[i].StartedTime.Time = time.Unix(0, 0) + } else { in.Status.History[i].StartedTime.Time = time.Unix(1, 0) } if item.CompletionTime != nil { diff --git a/pkg/cvo/internal/generic.go b/pkg/cvo/internal/generic.go index de2462c8a..939d73e13 100644 --- a/pkg/cvo/internal/generic.go +++ b/pkg/cvo/internal/generic.go @@ -11,12 +11,10 @@ import ( "k8s.io/apimachinery/pkg/util/sets" "k8s.io/apimachinery/pkg/util/strategicpatch" "k8s.io/client-go/dynamic" - "k8s.io/client-go/rest" "github.com/openshift/client-go/config/clientset/versioned/scheme" "github.com/openshift/cluster-version-operator/lib" "github.com/openshift/cluster-version-operator/lib/resourcebuilder" - "github.com/openshift/cluster-version-operator/pkg/cvo/internal/dynamicclient" ) // readUnstructuredV1OrDie reads operatorstatus object from bytes. Panics on error. @@ -67,11 +65,7 @@ type genericBuilder struct { // NewGenericBuilder returns an implentation of resourcebuilder.Interface that // uses dynamic clients for applying. -func NewGenericBuilder(config *rest.Config, m lib.Manifest) (resourcebuilder.Interface, error) { - client, err := dynamicclient.New(config, m.GVK, m.Object().GetNamespace()) - if err != nil { - return nil, err - } +func NewGenericBuilder(client dynamic.ResourceInterface, m lib.Manifest) (resourcebuilder.Interface, error) { return &genericBuilder{ client: client, raw: m.Raw, diff --git a/pkg/cvo/status.go b/pkg/cvo/status.go index 8085f345e..6a6ff103e 100644 --- a/pkg/cvo/status.go +++ b/pkg/cvo/status.go @@ -4,18 +4,18 @@ import ( "bytes" "fmt" - "k8s.io/apimachinery/pkg/util/validation/field" - "k8s.io/apimachinery/pkg/api/equality" apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/validation/field" + "github.com/golang/glog" configv1 "github.com/openshift/api/config/v1" configclientv1 "github.com/openshift/client-go/config/clientset/versioned/typed/config/v1" "github.com/openshift/cluster-version-operator/lib/resourcemerge" ) -func mergeEqualVersions(current *configv1.UpdateHistory, desired *configv1.Update) bool { +func mergeEqualVersions(current *configv1.UpdateHistory, desired configv1.Update) bool { if len(desired.Payload) > 0 && desired.Payload == current.Payload { if len(current.Version) == 0 || desired.Version == current.Version { current.Version = desired.Version @@ -31,11 +31,20 @@ func mergeEqualVersions(current *configv1.UpdateHistory, desired *configv1.Updat return false } -func mergeOperatorHistory(config *configv1.ClusterVersion, current configv1.Update, now metav1.Time) { +func mergeOperatorHistory(config *configv1.ClusterVersion, desired configv1.Update, now metav1.Time, completed bool) { + // if we have no payload, we cannot reproduce the update later and so it cannot be part of the history + if len(desired.Payload) == 0 { + // make the array empty + if config.Status.History == nil { + config.Status.History = []configv1.UpdateHistory{} + } + return + } + if len(config.Status.History) == 0 { config.Status.History = append(config.Status.History, configv1.UpdateHistory{ - Version: current.Version, - Payload: current.Payload, + Version: desired.Version, + Payload: desired.Payload, State: configv1.PartialUpdate, StartedTime: now, @@ -43,10 +52,6 @@ func mergeOperatorHistory(config *configv1.ClusterVersion, current configv1.Upda } last := &config.Status.History[0] - desired := config.Spec.DesiredUpdate - if desired == nil { - desired = ¤t - } if !mergeEqualVersions(last, desired) { last.CompletionTime = &now @@ -66,15 +71,17 @@ func mergeOperatorHistory(config *configv1.ClusterVersion, current configv1.Upda config.Status.History = config.Status.History[:10] } - switch { - case resourcemerge.IsOperatorStatusConditionTrue(config.Status.Conditions, configv1.OperatorAvailable): + if completed { last.State = configv1.CompletedUpdate if last.CompletionTime == nil { last.CompletionTime = &now } } + if len(last.State) == 0 { + last.State = configv1.PartialUpdate + } - config.Status.Desired = *desired + config.Status.Desired = desired } // ClusterVersionInvalid indicates that the cluster version has an error that prevents the server from @@ -82,52 +89,39 @@ func mergeOperatorHistory(config *configv1.ClusterVersion, current configv1.Upda // condition is set. const ClusterVersionInvalid configv1.ClusterStatusConditionType = "Invalid" -// syncInitialObjectStatus ensures that every known condition is either set or primed. errs is a list of -// known validation errors for the cluster version. -func (optr *Operator) syncInitialObjectStatus(original *configv1.ClusterVersion, errs field.ErrorList) (bool, error) { - config := original.DeepCopy() +// syncStatus calculates the new status of the ClusterVersion based on the current sync state and any +// validation errors found. We allow the caller to pass the original object to avoid DeepCopying twice. +func (optr *Operator) syncStatus(original, config *configv1.ClusterVersion, status *SyncWorkerStatus, validationErrs field.ErrorList) error { + glog.V(5).Infof("Synchronizing errs=%#v status=%#v", validationErrs, status) + // update the config with the latest available updates if updated := optr.getAvailableUpdates().NeedsUpdate(config); updated != nil { config = updated + } else if original == nil || original == config { + original = config.DeepCopy() } - now := metav1.Now() - target := optr.desiredVersion(original) - current := optr.currentVersion() - - // ensure the initial state of all conditions is set - if resourcemerge.FindOperatorStatusCondition(config.Status.Conditions, configv1.OperatorAvailable) == nil { - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse, LastTransitionTime: now}) - } - if resourcemerge.FindOperatorStatusCondition(config.Status.Conditions, configv1.OperatorFailing) == nil { - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorFailing, Status: configv1.ConditionFalse, LastTransitionTime: now}) - } - // We default towards progressing because we know we will at least need to sync - if resourcemerge.FindOperatorStatusCondition(config.Status.Conditions, configv1.OperatorProgressing) == nil { - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorProgressing, - Status: configv1.ConditionTrue, - Message: fmt.Sprintf("Initializing, will work towards %s", versionString(target)), - LastTransitionTime: now, - }) - } - // other conditions - if resourcemerge.FindOperatorStatusCondition(config.Status.Conditions, configv1.RetrievedUpdates) == nil { - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse, LastTransitionTime: now}) + config.Status.Generation = config.Generation + if len(status.VersionHash) > 0 { + config.Status.VersionHash = status.VersionHash } - // update the invalid condition - if len(errs) > 0 { + now := metav1.Now() + version := versionString(status.Actual) + + // update validation errors + var reason string + if len(validationErrs) > 0 { buf := &bytes.Buffer{} - if len(errs) == 1 { - fmt.Fprintf(buf, "The cluster version is invalid: %s", errs[0].Error()) + if len(validationErrs) == 1 { + fmt.Fprintf(buf, "The cluster version is invalid: %s", validationErrs[0].Error()) } else { fmt.Fprintf(buf, "The cluster version is invalid:\n") - for _, err := range errs { + for _, err := range validationErrs { fmt.Fprintf(buf, "* %s\n", err.Error()) } } - reason := "InvalidClusterVersion" + reason = "InvalidClusterVersion" resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ Type: ClusterVersionInvalid, @@ -136,227 +130,107 @@ func (optr *Operator) syncInitialObjectStatus(original *configv1.ClusterVersion, Message: buf.String(), LastTransitionTime: now, }) - - if !resourcemerge.IsOperatorStatusConditionTrue(config.Status.Conditions, configv1.OperatorFailing) { - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorProgressing, - Status: configv1.ConditionFalse, - Reason: reason, - Message: fmt.Sprintf("Stopped at %s: the cluster version is invalid", versionString(current)), - LastTransitionTime: now, - }) - } - } else { resourcemerge.RemoveOperatorStatusCondition(&config.Status.Conditions, ClusterVersionInvalid) } - // ensure we record the initial state so the user can roll back - if len(config.Status.History) == 0 { - mergeOperatorHistory(config, current, now) - } - mergeOperatorHistory(config, target, now) - - updated, err := applyClusterVersionStatus(optr.client.ConfigV1(), config, original) - optr.rememberLastUpdate(updated) - return updated != nil && updated.ResourceVersion != original.ResourceVersion, err -} - -func (optr *Operator) syncProgressingStatus(config *configv1.ClusterVersion, update configv1.Update) error { - original := config.DeepCopy() - - config.Status.Generation = config.Generation - - now := metav1.Now() - - // clear the available condition - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse, LastTransitionTime: now}) - - // preserve the most recent failing condition - if resourcemerge.IsOperatorStatusConditionNotIn(config.Status.Conditions, configv1.OperatorFailing, configv1.ConditionTrue) { - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorFailing, Status: configv1.ConditionFalse, LastTransitionTime: now}) - } - - // set progressing with an accurate summary message - if c := resourcemerge.FindOperatorStatusCondition(config.Status.Conditions, configv1.OperatorFailing); c != nil && c.Status == configv1.ConditionTrue { - reason := c.Reason - msg := summaryForReason(reason) - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorProgressing, - Status: configv1.ConditionTrue, - Reason: reason, - Message: fmt.Sprintf("Unable to apply %s: %s", versionString(update), msg), - LastTransitionTime: now, - }) - } else if resourcemerge.IsOperatorStatusConditionTrue(config.Status.Conditions, ClusterVersionInvalid) { - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorProgressing, - Status: configv1.ConditionTrue, - Message: fmt.Sprintf("Reconciling %s: the cluster version is invalid", versionString(update)), - LastTransitionTime: now, - }) - } else { + // set the available condition + if status.Completed > 0 { resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorProgressing, - Status: configv1.ConditionTrue, - Message: fmt.Sprintf("Working towards %s", versionString(update)), + Type: configv1.OperatorAvailable, + Status: configv1.ConditionTrue, + Message: fmt.Sprintf("Done applying %s", version), + LastTransitionTime: now, }) } - - mergeOperatorHistory(config, update, now) - - updated, err := applyClusterVersionStatus(optr.client.ConfigV1(), config, original) - optr.rememberLastUpdate(updated) - return err -} - -func (optr *Operator) syncAvailableStatus(config *configv1.ClusterVersion, current configv1.Update, versionHash string) error { - original := config.DeepCopy() - - config.Status.VersionHash = versionHash - config.Status.Generation = config.Generation - - now := metav1.Now() - version := optr.currentVersionString(config) - - // set the available condition - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorAvailable, - Status: configv1.ConditionTrue, - Message: fmt.Sprintf("Done applying %s", version), - - LastTransitionTime: now, - }) - - // clear the failure condition - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorFailing, Status: configv1.ConditionFalse, LastTransitionTime: now}) - - // clear the progressing condition - if resourcemerge.IsOperatorStatusConditionTrue(config.Status.Conditions, ClusterVersionInvalid) { + // default the available condition if not set + if resourcemerge.FindOperatorStatusCondition(config.Status.Conditions, configv1.OperatorAvailable) == nil { resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorProgressing, + Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse, - Reason: "InvalidClusterVersion", - Message: fmt.Sprintf("Stopped at %s: the cluster version is invalid", versionString(current)), - LastTransitionTime: now, - }) - } else { - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorProgressing, - Status: configv1.ConditionFalse, - Message: fmt.Sprintf("Cluster version is %s", version), - LastTransitionTime: now, }) } - mergeOperatorHistory(config, current, now) - - updated, err := applyClusterVersionStatus(optr.client.ConfigV1(), config, original) - optr.rememberLastUpdate(updated) - return err -} - -func (optr *Operator) syncPayloadFailingStatus(original *configv1.ClusterVersion, err error) error { - config := original.DeepCopy() - - config.Status.Generation = config.Generation - - now := metav1.Now() - var reason string - msg := "an error occurred" - if uErr, ok := err.(*updateError); ok { - reason = uErr.Reason - msg = summaryForReason(reason) - } - - target := optr.desiredVersion(original) - - // leave the available condition alone - - // set the failing condition - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorFailing, - Status: configv1.ConditionTrue, - Reason: reason, - Message: err.Error(), - LastTransitionTime: now, - }) + if err := status.Failure; err != nil { + var reason string + msg := "an error occurred" + if uErr, ok := err.(*updateError); ok { + reason = uErr.Reason + msg = summaryForReason(reason) + } - // update the progressing condition message to indicate there is an error - if resourcemerge.IsOperatorStatusConditionTrue(config.Status.Conditions, configv1.OperatorProgressing) { + // set the failing condition resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorProgressing, + Type: configv1.OperatorFailing, Status: configv1.ConditionTrue, Reason: reason, - Message: fmt.Sprintf("Unable to apply %s: %s", versionString(target), msg), + Message: err.Error(), LastTransitionTime: now, }) - } else { - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorProgressing, - Status: configv1.ConditionFalse, - Reason: reason, - Message: fmt.Sprintf("Error while reconciling %s: %s", versionString(target), msg), - LastTransitionTime: now, - }) - } - - mergeOperatorHistory(config, target, now) - - updated, err := applyClusterVersionStatus(optr.client.ConfigV1(), config, original) - optr.rememberLastUpdate(updated) - return err -} -func (optr *Operator) syncUpdateFailingStatus(original *configv1.ClusterVersion, err error) error { - config := original.DeepCopy() + // update progressing + if status.Reconciling { + resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ + Type: configv1.OperatorProgressing, + Status: configv1.ConditionFalse, + Reason: reason, + Message: fmt.Sprintf("Error while reconciling %s: %s", version, msg), + LastTransitionTime: now, + }) + } else { + resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ + Type: configv1.OperatorProgressing, + Status: configv1.ConditionTrue, + Reason: reason, + Message: fmt.Sprintf("Unable to apply %s: %s", version, msg), + LastTransitionTime: now, + }) + } - config.Status.Generation = config.Generation + } else { + // clear the failure condition + resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorFailing, Status: configv1.ConditionFalse, LastTransitionTime: now}) - now := metav1.Now() - var reason string - msg := "an error occurred" - if uErr, ok := err.(*updateError); ok { - reason = uErr.Reason - msg = summaryForReason(reason) + // update progressing + if status.Reconciling { + message := fmt.Sprintf("Cluster version is %s", version) + if len(validationErrs) > 0 { + message = fmt.Sprintf("Stopped at %s: the cluster version is invalid", version) + } + resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ + Type: configv1.OperatorProgressing, + Status: configv1.ConditionFalse, + Reason: reason, + Message: message, + LastTransitionTime: now, + }) + } else { + message := fmt.Sprintf("Working towards %s", version) + if len(validationErrs) > 0 { + message = fmt.Sprintf("Reconciling %s: the cluster version is invalid", version) + } + resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ + Type: configv1.OperatorProgressing, + Status: configv1.ConditionTrue, + Reason: reason, + Message: message, + LastTransitionTime: now, + }) + } } - target := optr.desiredVersion(original) - - // clear the available condition - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{Type: configv1.OperatorAvailable, Status: configv1.ConditionFalse, LastTransitionTime: now}) - - // set the failing condition - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorFailing, - Status: configv1.ConditionTrue, - Reason: reason, - Message: err.Error(), - LastTransitionTime: now, - }) - - // update the progressing condition message to indicate there is an error - if resourcemerge.IsOperatorStatusConditionTrue(config.Status.Conditions, configv1.OperatorProgressing) { - resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorProgressing, - Status: configv1.ConditionTrue, - Reason: reason, - Message: fmt.Sprintf("Unable to apply %s: %s", versionString(target), msg), - LastTransitionTime: now, - }) - } else { + // default retrieved updates if it is not set + if resourcemerge.FindOperatorStatusCondition(config.Status.Conditions, configv1.RetrievedUpdates) == nil { resourcemerge.SetOperatorStatusCondition(&config.Status.Conditions, configv1.ClusterOperatorStatusCondition{ - Type: configv1.OperatorProgressing, + Type: configv1.RetrievedUpdates, Status: configv1.ConditionFalse, - Reason: reason, - Message: fmt.Sprintf("Error while reconciling %s: %s", versionString(target), msg), LastTransitionTime: now, }) } - mergeOperatorHistory(config, target, now) + mergeOperatorHistory(config, status.Actual, now, status.Completed > 0) updated, err := applyClusterVersionStatus(optr.client.ConfigV1(), config, original) optr.rememberLastUpdate(updated) @@ -413,7 +287,7 @@ func (optr *Operator) syncFailingStatus(config *configv1.ClusterVersion, ierr er LastTransitionTime: now, }) - mergeOperatorHistory(config, optr.currentVersion(), now) + mergeOperatorHistory(config, optr.currentVersion(), now, false) updated, err := applyClusterVersionStatus(optr.client.ConfigV1(), config, original) optr.rememberLastUpdate(updated) diff --git a/pkg/cvo/sync.go b/pkg/cvo/sync.go index 4ca1e0c08..26e816b68 100644 --- a/pkg/cvo/sync.go +++ b/pkg/cvo/sync.go @@ -4,7 +4,8 @@ import ( "fmt" "strings" - "github.com/golang/glog" + "github.com/openshift/cluster-version-operator/pkg/cvo/internal/dynamicclient" + "github.com/pkg/errors" apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -35,64 +36,39 @@ var requeueOnErrorCauseToCheck = map[string]func(error) bool{ RequeueOnErrorCauseNoMatch: meta.IsNoMatchError, } -// loadUpdatePayload reads the payload from disk or remote, as necessary. -func (optr *Operator) loadUpdatePayload(config *configv1.ClusterVersion) (*updatePayload, error) { - payloadDir, err := optr.updatePayloadDir(config) - if err != nil { - return nil, err - } - releaseImage := optr.releaseImage - if config.Spec.DesiredUpdate != nil { - releaseImage = config.Spec.DesiredUpdate.Payload +func (optr *Operator) defaultResourceBuilder() ResourceBuilder { + return &resourceBuilder{ + config: optr.restConfig, } - return loadUpdatePayload(payloadDir, releaseImage) } -// syncUpdatePayload applies the manifests in the payload to the cluster. -func (optr *Operator) syncUpdatePayload(config *configv1.ClusterVersion, payload *updatePayload) error { - version := payload.ReleaseVersion - if len(version) == 0 { - version = payload.ReleaseImage - } +// resourceBuilder provides the default builder implementation for the operator. +// It is abstracted for testing. +type resourceBuilder struct { + config *rest.Config + modifier resourcebuilder.MetaV1ObjectModifierFunc +} - total := len(payload.Manifests) - done := 0 - var tasks []*syncTask - for i := range payload.Manifests { - tasks = append(tasks, &syncTask{ - index: i + 1, - total: total, - manifest: &payload.Manifests[i], - backoff: optr.syncBackoff, - }) +func (b *resourceBuilder) BuilderFor(m *lib.Manifest) (resourcebuilder.Interface, error) { + if resourcebuilder.Mapper.Exists(m.GVK) { + return resourcebuilder.New(resourcebuilder.Mapper, b.config, *m) } + client, err := dynamicclient.New(b.config, m.GVK, m.Object().GetNamespace()) + if err != nil { + return nil, err + } + return internal.NewGenericBuilder(client, *m) +} - for i := 0; i < len(tasks); i++ { - task := tasks[i] - setAppliedAndPending(version, total, done) - glog.V(4).Infof("Running sync for %s", task) - glog.V(6).Infof("Manifest: %s", string(task.manifest.Raw)) - - ov, ok := getOverrideForManifest(config.Spec.Overrides, task.manifest) - if ok && ov.Unmanaged { - glog.V(4).Infof("Skipping %s as unmanaged", task) - continue - } - - if err := task.Run(version, optr.restConfig); err != nil { - cause := errors.Cause(err) - if task.requeued == 0 && shouldRequeueOnErr(cause, task.manifest) { - task.requeued++ - tasks = append(tasks, task) - continue - } - return err - } - done++ - glog.V(4).Infof("Done syncing for %s", task) +func (b *resourceBuilder) Apply(m *lib.Manifest) error { + builder, err := b.BuilderFor(m) + if err != nil { + return err } - setAppliedAndPending(version, total, done) - return nil + if b.modifier != nil { + builder = builder.WithModifier(b.modifier) + } + return builder.Do() } type syncTask struct { @@ -111,28 +87,17 @@ func (st *syncTask) String() string { return fmt.Sprintf("%s \"%s/%s\" (%s, %d of %d)", strings.ToLower(st.manifest.GVK.Kind), ns, st.manifest.Object().GetName(), st.manifest.GVK.GroupVersion().String(), st.index, st.total) } -func (st *syncTask) Run(version string, rc *rest.Config) error { +func (st *syncTask) Run(version string, builder ResourceBuilder) error { var lastErr error if err := wait.ExponentialBackoff(st.backoff, func() (bool, error) { - // build resource builder for manifest - var b resourcebuilder.Interface - var err error - if resourcebuilder.Mapper.Exists(st.manifest.GVK) { - b, err = resourcebuilder.New(resourcebuilder.Mapper, rc, *st.manifest) - } else { - b, err = internal.NewGenericBuilder(rc, *st.manifest) - } - if err != nil { - utilruntime.HandleError(errors.Wrapf(err, "error creating resourcebuilder for %s", st)) - lastErr = err - metricPayloadErrors.WithLabelValues(version).Inc() - return false, nil - } // run builder for the manifest - if err := b.Do(); err != nil { + if err := builder.Apply(st.manifest); err != nil { utilruntime.HandleError(errors.Wrapf(err, "error running apply for %s", st)) lastErr = err metricPayloadErrors.WithLabelValues(version).Inc() + if !shouldRequeueApplyOnErr(err) { + return false, err + } return false, nil } return true, nil @@ -150,6 +115,13 @@ func (st *syncTask) Run(version string, rc *rest.Config) error { return nil } +func shouldRequeueApplyOnErr(err error) bool { + if apierrors.IsInvalid(err) { + return false + } + return true +} + func shouldRequeueOnErr(err error, manifest *lib.Manifest) bool { cause := errors.Cause(err) if _, ok := cause.(*resourcebuilder.RetryLaterError); ok { diff --git a/pkg/cvo/sync_test.go b/pkg/cvo/sync_test.go index 56de65602..58af557ce 100644 --- a/pkg/cvo/sync_test.go +++ b/pkg/cvo/sync_test.go @@ -1,21 +1,29 @@ package cvo import ( + "context" "encoding/json" "fmt" "reflect" + "strings" "testing" "github.com/davecgh/go-spew/spew" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/util/wait" + dynamicfake "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/rest" + clientgotesting "k8s.io/client-go/testing" configv1 "github.com/openshift/api/config/v1" "github.com/openshift/cluster-version-operator/lib" "github.com/openshift/cluster-version-operator/lib/resourcebuilder" + "github.com/openshift/cluster-version-operator/pkg/cvo/internal" ) func TestHasRequeueOnErrorAnnotation(t *testing.T) { @@ -191,12 +199,13 @@ func TestShouldRequeueOnErr(t *testing.T) { } } -func TestSyncUpdatePayload(t *testing.T) { +func Test_SyncWorker_apply(t *testing.T) { tests := []struct { manifests []string reactors map[action]error - check func(*testing.T, []action) + check func(*testing.T, []action) + wantErr bool }{{ manifests: []string{ `{ @@ -220,13 +229,13 @@ func TestSyncUpdatePayload(t *testing.T) { check: func(t *testing.T, actions []action) { if len(actions) != 2 { spew.Dump(actions) - t.Fatal("expected only 2 actions") + t.Fatalf("unexpected %d actions", len(actions)) } - if got, exp := actions[0], (action{schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa"}); !reflect.DeepEqual(got, exp) { + if got, exp := actions[0], (newAction(schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa")); !reflect.DeepEqual(got, exp) { t.Fatalf("expected: %s got: %s", spew.Sdump(exp), spew.Sdump(got)) } - if got, exp := actions[1], (action{schema.GroupVersionKind{"test.cvo.io", "v1", "TestB"}, "default", "testb"}); !reflect.DeepEqual(got, exp) { + if got, exp := actions[1], (newAction(schema.GroupVersionKind{"test.cvo.io", "v1", "TestB"}, "default", "testb")); !reflect.DeepEqual(got, exp) { t.Fatalf("expected: %s got: %s", spew.Sdump(exp), spew.Sdump(got)) } }, @@ -250,15 +259,16 @@ func TestSyncUpdatePayload(t *testing.T) { }`, }, reactors: map[action]error{ - action{schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa"}: &meta.NoResourceMatchError{}, + newAction(schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa"): &meta.NoResourceMatchError{}, }, + wantErr: true, check: func(t *testing.T, actions []action) { if len(actions) != 3 { spew.Dump(actions) - t.Fatal("expected only 3 actions") + t.Fatalf("unexpected %d actions", len(actions)) } - if got, exp := actions[0], (action{schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa"}); !reflect.DeepEqual(got, exp) { + if got, exp := actions[0], (newAction(schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa")); !reflect.DeepEqual(got, exp) { t.Fatalf("expected: %s got: %s", spew.Sdump(exp), spew.Sdump(got)) } }, @@ -285,21 +295,22 @@ func TestSyncUpdatePayload(t *testing.T) { }`, }, reactors: map[action]error{ - action{schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa"}: &meta.NoResourceMatchError{}, + newAction(schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa"): &meta.NoResourceMatchError{}, }, + wantErr: true, check: func(t *testing.T, actions []action) { if len(actions) != 7 { spew.Dump(actions) - t.Fatal("expected only 7 actions") + t.Fatalf("unexpected %d actions", len(actions)) } - if got, exp := actions[0], (action{schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa"}); !reflect.DeepEqual(got, exp) { + if got, exp := actions[0], (newAction(schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa")); !reflect.DeepEqual(got, exp) { t.Fatalf("expected: %s got: %s", spew.Sdump(exp), spew.Sdump(got)) } - if got, exp := actions[3], (action{schema.GroupVersionKind{"test.cvo.io", "v1", "TestB"}, "default", "testb"}); !reflect.DeepEqual(got, exp) { + if got, exp := actions[3], (newAction(schema.GroupVersionKind{"test.cvo.io", "v1", "TestB"}, "default", "testb")); !reflect.DeepEqual(got, exp) { t.Fatalf("expected: %s got: %s", spew.Sdump(exp), spew.Sdump(got)) } - if got, exp := actions[4], (action{schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa"}); !reflect.DeepEqual(got, exp) { + if got, exp := actions[4], (newAction(schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa")); !reflect.DeepEqual(got, exp) { t.Fatalf("expected: %s got: %s", spew.Sdump(exp), spew.Sdump(got)) } }, @@ -329,22 +340,23 @@ func TestSyncUpdatePayload(t *testing.T) { }`, }, reactors: map[action]error{ - action{schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa"}: &meta.NoResourceMatchError{}, - action{schema.GroupVersionKind{"test.cvo.io", "v1", "TestB"}, "default", "testb"}: &meta.NoResourceMatchError{}, + newAction(schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa"): &meta.NoResourceMatchError{}, + newAction(schema.GroupVersionKind{"test.cvo.io", "v1", "TestB"}, "default", "testb"): &meta.NoResourceMatchError{}, }, + wantErr: true, check: func(t *testing.T, actions []action) { if len(actions) != 9 { spew.Dump(actions) - t.Fatal("expected only 12 actions") + t.Fatalf("unexpected %d actions", len(actions)) } - if got, exp := actions[0], (action{schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa"}); !reflect.DeepEqual(got, exp) { + if got, exp := actions[0], (newAction(schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa")); !reflect.DeepEqual(got, exp) { t.Fatalf("expected: %s got: %s", spew.Sdump(exp), spew.Sdump(got)) } - if got, exp := actions[3], (action{schema.GroupVersionKind{"test.cvo.io", "v1", "TestB"}, "default", "testb"}); !reflect.DeepEqual(got, exp) { + if got, exp := actions[3], (newAction(schema.GroupVersionKind{"test.cvo.io", "v1", "TestB"}, "default", "testb")); !reflect.DeepEqual(got, exp) { t.Fatalf("expected: %s got: %s", spew.Sdump(exp), spew.Sdump(got)) } - if got, exp := actions[6], (action{schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa"}); !reflect.DeepEqual(got, exp) { + if got, exp := actions[6], (newAction(schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, "default", "testa")); !reflect.DeepEqual(got, exp) { t.Fatalf("expected: %s got: %s", spew.Sdump(exp), spew.Sdump(got)) } }, @@ -361,29 +373,199 @@ func TestSyncUpdatePayload(t *testing.T) { } up := &updatePayload{ReleaseImage: "test", ReleaseVersion: "v0.0.0", Manifests: manifests} - op := &Operator{} - op.syncBackoff = wait.Backoff{Steps: 3} - config := &configv1.ClusterVersion{} r := &recorder{} testMapper := resourcebuilder.NewResourceMapper() testMapper.RegisterGVK(schema.GroupVersionKind{"test.cvo.io", "v1", "TestA"}, newTestBuilder(r, test.reactors)) testMapper.RegisterGVK(schema.GroupVersionKind{"test.cvo.io", "v1", "TestB"}, newTestBuilder(r, test.reactors)) testMapper.AddToMap(resourcebuilder.Mapper) - op.syncUpdatePayload(config, up) + worker := &SyncWorker{} + worker.backoff.Steps = 3 + worker.builder = (&Operator{}).defaultResourceBuilder() + ctx := context.Background() + worker.apply(ctx, up, &SyncWork{}, &statusWrapper{w: worker, previousStatus: worker.Status()}) test.check(t, r.actions) }) } } +func Test_SyncWorker_apply_generic(t *testing.T) { + tests := []struct { + manifests []string + modifiers []resourcebuilder.MetaV1ObjectModifierFunc + + check func(t *testing.T, client *dynamicfake.FakeDynamicClient) + }{ + { + manifests: []string{ + `{ + "apiVersion": "test.cvo.io/v1", + "kind": "TestA", + "metadata": { + "namespace": "default", + "name": "testa" + } + }`, + `{ + "apiVersion": "test.cvo.io/v1", + "kind": "TestB", + "metadata": { + "namespace": "default", + "name": "testb" + } + }`, + }, + check: func(t *testing.T, client *dynamicfake.FakeDynamicClient) { + actions := client.Actions() + if len(actions) != 4 { + spew.Dump(actions) + t.Fatal("expected only 4 actions") + } + + got := actions[1].(clientgotesting.CreateAction).GetObject() + exp := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "test.cvo.io/v1", + "kind": "TestA", + "metadata": map[string]interface{}{ + "name": "testa", + "namespace": "default", + }, + }, + } + if !reflect.DeepEqual(got, exp) { + t.Fatalf("expected: %s got: %s", spew.Sdump(exp), spew.Sdump(got)) + } + + got = actions[3].(clientgotesting.CreateAction).GetObject() + exp = &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "test.cvo.io/v1", + "kind": "TestB", + "metadata": map[string]interface{}{ + "name": "testb", + "namespace": "default", + }, + }, + } + if !reflect.DeepEqual(got, exp) { + t.Fatalf("expected: %s got: %s", spew.Sdump(exp), spew.Sdump(got)) + } + }, + }, + { + modifiers: []resourcebuilder.MetaV1ObjectModifierFunc{ + func(obj metav1.Object) { + m := obj.GetLabels() + if m == nil { + m = make(map[string]string) + } + m["test/label"] = "a" + obj.SetLabels(m) + }, + }, + manifests: []string{ + `{ + "apiVersion": "test.cvo.io/v1", + "kind": "TestA", + "metadata": { + "namespace": "default", + "name": "testa" + } + }`, + `{ + "apiVersion": "test.cvo.io/v1", + "kind": "TestB", + "metadata": { + "namespace": "default", + "name": "testb" + } + }`, + }, + check: func(t *testing.T, client *dynamicfake.FakeDynamicClient) { + actions := client.Actions() + if len(actions) != 4 { + spew.Dump(actions) + t.Fatalf("got %d actions", len(actions)) + } + + got := actions[1].(clientgotesting.CreateAction).GetObject() + exp := &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "test.cvo.io/v1", + "kind": "TestA", + "metadata": map[string]interface{}{ + "name": "testa", + "namespace": "default", + "labels": map[string]interface{}{"test/label": "a"}, + }, + }, + } + if !reflect.DeepEqual(got, exp) { + t.Fatalf("expected: %s got: %s", spew.Sdump(exp), spew.Sdump(got)) + } + + got = actions[3].(clientgotesting.CreateAction).GetObject() + exp = &unstructured.Unstructured{ + Object: map[string]interface{}{ + "apiVersion": "test.cvo.io/v1", + "kind": "TestB", + "metadata": map[string]interface{}{ + "name": "testb", + "namespace": "default", + "labels": map[string]interface{}{"test/label": "a"}, + }, + }, + } + if !reflect.DeepEqual(got, exp) { + t.Fatalf("expected: %s got: %s", spew.Sdump(exp), spew.Sdump(got)) + } + }, + }, + } + for idx, test := range tests { + t.Run(fmt.Sprintf("test#%d", idx), func(t *testing.T) { + var manifests []lib.Manifest + for _, s := range test.manifests { + m := lib.Manifest{} + if err := json.Unmarshal([]byte(s), &m); err != nil { + t.Fatal(err) + } + manifests = append(manifests, m) + } + + dynamicScheme := runtime.NewScheme() + dynamicScheme.AddKnownTypeWithName(schema.GroupVersionKind{Group: "test.cvo.io", Version: "v1", Kind: "TestA"}, &unstructured.Unstructured{}) + dynamicScheme.AddKnownTypeWithName(schema.GroupVersionKind{Group: "test.cvo.io", Version: "v1", Kind: "TestB"}, &unstructured.Unstructured{}) + dynamicClient := dynamicfake.NewSimpleDynamicClient(dynamicScheme) + + up := &updatePayload{ReleaseImage: "test", ReleaseVersion: "v0.0.0", Manifests: manifests} + worker := &SyncWorker{} + worker.backoff.Steps = 1 + worker.builder = &testResourceBuilder{ + client: dynamicClient, + modifiers: test.modifiers, + } + ctx := context.Background() + err := worker.apply(ctx, up, &SyncWork{}, &statusWrapper{w: worker, previousStatus: worker.Status()}) + if err != nil { + t.Fatal(err) + } + test.check(t, dynamicClient) + }) + } +} + type testBuilder struct { *recorder - reactors map[action]error + reactors map[action]error + modifiers []resourcebuilder.MetaV1ObjectModifierFunc m *lib.Manifest } -func (t *testBuilder) WithModifier(_ resourcebuilder.MetaV1ObjectModifierFunc) resourcebuilder.Interface { +func (t *testBuilder) WithModifier(m resourcebuilder.MetaV1ObjectModifierFunc) resourcebuilder.Interface { + t.modifiers = append(t.modifiers, m) return t } @@ -413,3 +595,73 @@ type action struct { Namespace string Name string } + +func newAction(gvk schema.GroupVersionKind, namespace, name string) action { + return action{GVK: gvk, Namespace: namespace, Name: name} +} + +type fakeSyncRecorder struct { + Returns *SyncWorkerStatus + Updates []configv1.Update +} + +func (r *fakeSyncRecorder) StatusCh() <-chan SyncWorkerStatus { + ch := make(chan SyncWorkerStatus) + close(ch) + return ch +} + +func (r *fakeSyncRecorder) Start(stopCh <-chan struct{}) {} + +func (r *fakeSyncRecorder) Update(desired configv1.Update, overrides []configv1.ComponentOverride, reconciling bool) *SyncWorkerStatus { + r.Updates = append(r.Updates, desired) + return r.Returns +} + +type fakeResourceBuilder struct { + M []*lib.Manifest + Err error +} + +func (b *fakeResourceBuilder) Apply(m *lib.Manifest) error { + b.M = append(b.M, m) + return b.Err +} + +type fakeDirectoryRetriever struct { + Path string + Err error +} + +func (r *fakeDirectoryRetriever) RetrievePayload(ctx context.Context, update configv1.Update) (string, error) { + return r.Path, r.Err +} + +type fakePayloadRetriever struct { + Dir string + Err error +} + +func (r *fakePayloadRetriever) RetrievePayload(ctx context.Context, desired configv1.Update) (string, error) { + return r.Dir, r.Err +} + +// testResourceBuilder uses a fake dynamic client to exercise the generic builder in tests. +type testResourceBuilder struct { + client *dynamicfake.FakeDynamicClient + modifiers []resourcebuilder.MetaV1ObjectModifierFunc +} + +func (b *testResourceBuilder) Apply(m *lib.Manifest) error { + ns := m.Object().GetNamespace() + fakeGVR := schema.GroupVersionResource{Group: m.GVK.Group, Version: m.GVK.Version, Resource: strings.ToLower(m.GVK.Kind)} + client := b.client.Resource(fakeGVR).Namespace(ns) + builder, err := internal.NewGenericBuilder(client, *m) + if err != nil { + return err + } + for _, m := range b.modifiers { + builder = builder.WithModifier(m) + } + return builder.Do() +} diff --git a/pkg/cvo/sync_worker.go b/pkg/cvo/sync_worker.go new file mode 100644 index 000000000..5bc40b065 --- /dev/null +++ b/pkg/cvo/sync_worker.go @@ -0,0 +1,497 @@ +package cvo + +import ( + "context" + "fmt" + "reflect" + "sync" + "time" + + "golang.org/x/time/rate" + + "github.com/golang/glog" + "github.com/pkg/errors" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/cluster-version-operator/lib" +) + +// ConfigSyncWorker abstracts how the payload is synchronized to the server. Introduced for testing. +type ConfigSyncWorker interface { + Start(stopCh <-chan struct{}) + Update(desired configv1.Update, overrides []configv1.ComponentOverride, reconciling bool) *SyncWorkerStatus + StatusCh() <-chan SyncWorkerStatus +} + +// PayloadRetriever abstracts how a desired version is extracted to disk. Introduced for testing. +type PayloadRetriever interface { + RetrievePayload(ctx context.Context, desired configv1.Update) (string, error) +} + +// ResourceBuilder abstracts how a manifest is created on the server. Introduced for testing. +type ResourceBuilder interface { + Apply(*lib.Manifest) error +} + +// StatusReporter abstracts how status is reported by the worker run method. Introduced for testing. +type StatusReporter interface { + Report(status SyncWorkerStatus) +} + +// SyncWork represents the work that should be done in a sync iteration. +type SyncWork struct { + Desired configv1.Update + Overrides []configv1.ComponentOverride + Reconciling bool + Completed int +} + +// Empty returns true if the payload is empty for this work. +func (w SyncWork) Empty() bool { + return len(w.Desired.Payload) == 0 +} + +// SyncWorkerStatus is the status of the sync worker at a given time. +type SyncWorkerStatus struct { + Step string + Failure error + + Fraction float32 + + Completed int + Reconciling bool + VersionHash string + + Actual configv1.Update +} + +// DeepCopy copies the worker status. +func (w SyncWorkerStatus) DeepCopy() *SyncWorkerStatus { + return &w +} + +// SyncWorker retrieves and applies the desired payload, tracking the status for the parent to +// monitor. The worker accepts a desired state via Update() and works to keep that state in +// sync. Once a particular payload version is synced, it will be updated no more often than +// minimumReconcileInterval. +// +// State transitions: +// +// Initial: wait for valid Update(), report empty status +// Update() -> Sync +// Sync: attempt to invoke the syncOnce() method +// syncOnce() returns an error -> Error +// syncOnce() returns nil -> Reconciling +// Reconciling: invoke syncOnce() no more often than reconcileInterval +// Update() with different values -> Sync +// syncOnce() returns an error -> Error +// syncOnce() returns nil -> Reconciling +// Error: backoff until we are attempting every reconcileInterval +// syncOnce() returns an error -> Error +// syncOnce() returns nil -> Reconciling +// +type SyncWorker struct { + backoff wait.Backoff + retriever PayloadRetriever + builder ResourceBuilder + reconciling bool + + // minimumReconcileInterval is the minimum time between reconcile attempts, and is + // used to define the maximum backoff interval when syncOnce() returns an error. + minimumReconcileInterval time.Duration + + // coordination between the sync loop and external callers + notify chan struct{} + report chan SyncWorkerStatus + + // lock guards changes to these fields + lock sync.Mutex + work *SyncWork + cancelFn func() + status SyncWorkerStatus + + // updated by the run method only + payload *updatePayload +} + +// NewSyncWorker initializes a ConfigSyncWorker that will retrieve payloads to disk, apply them via builder +// to a server, and obey limits about how often to reconcile or retry on errors. +func NewSyncWorker(retriever PayloadRetriever, builder ResourceBuilder, reconcileInterval time.Duration, backoff wait.Backoff) ConfigSyncWorker { + return &SyncWorker{ + retriever: retriever, + builder: builder, + backoff: backoff, + + minimumReconcileInterval: reconcileInterval, + + notify: make(chan struct{}, 1), + // report is a large buffered channel to improve local testing - most consumers should invoke + // Status() or use the result of calling Update() instead because the channel can be out of date + // if the reader is not fast enough. + report: make(chan SyncWorkerStatus, 500), + } +} + +// StatusCh returns a channel that reports status from the worker. The channel is buffered and events +// can be lost, so this is best used as a trigger to read the latest status. +func (w *SyncWorker) StatusCh() <-chan SyncWorkerStatus { + return w.report +} + +// Update instructs the sync worker to start synchronizing the desired update. The reconciling boolean is +// ignored unless this is the first time that Update has been called. The returned status represents either +// the initial state or whatever the last recorded status was. +// TODO: in the future it may be desirable for changes that alter desired to wait briefly before returning, +// giving the sync loop the opportunity to observe our change and begin working towards it. +func (w *SyncWorker) Update(desired configv1.Update, overrides []configv1.ComponentOverride, reconciling bool) *SyncWorkerStatus { + w.lock.Lock() + defer w.lock.Unlock() + + work := &SyncWork{ + Desired: desired, + Overrides: overrides, + } + + if work.Empty() || equalSyncWork(w.work, work) { + return w.status.DeepCopy() + } + + // initialize the reconciliation flag and the status the first time + // update is invoked + if w.work == nil { + if reconciling { + work.Reconciling = true + } + w.status = SyncWorkerStatus{Reconciling: work.Reconciling, Actual: work.Desired} + } + + // notify the sync loop that we changed config + w.work = work + if w.cancelFn != nil { + w.cancelFn() + w.cancelFn = nil + } + select { + case w.notify <- struct{}{}: + default: + } + + return w.status.DeepCopy() +} + +// Start periodically invokes run, detecting whether content has changed. +// It is edge-triggered when Update() is invoked and level-driven after the +// syncOnce() has succeeded for a given input (we are said to be "reconciling"). +func (w *SyncWorker) Start(stopCh <-chan struct{}) { + glog.V(5).Infof("Starting sync worker") + + work := &SyncWork{} + + wait.Until(func() { + consecutiveErrors := 0 + errorInterval := w.minimumReconcileInterval / 16 + + var next <-chan time.Time + for { + waitingToReconcile := work.Reconciling + select { + case <-stopCh: + glog.V(5).Infof("Stopped worker") + return + case <-next: + waitingToReconcile = false + glog.V(5).Infof("Wait finished") + case <-w.notify: + glog.V(5).Infof("Work updated") + } + + // determine whether we need to do work + changed := w.calculateNext(work) + if !changed && waitingToReconcile { + glog.V(5).Infof("No change, waiting") + continue + } + + // until Update() has been called at least once, we do nothing + if work.Empty() { + next = time.After(w.minimumReconcileInterval) + glog.V(5).Infof("No work, waiting") + continue + } + + // actually apply the payload, allowing for calls to be cancelled + err := func() error { + ctx, cancelFn := context.WithCancel(context.Background()) + w.lock.Lock() + w.cancelFn = cancelFn + w.lock.Unlock() + defer cancelFn() + + // reporter hides status updates that occur earlier than the previous failure, + // so that we don't fail, then immediately start reporting an earlier status + reporter := &statusWrapper{w: w, previousStatus: w.Status()} + glog.V(5).Infof("Previous sync status: %#v", reporter.previousStatus) + return w.syncOnce(ctx, work, reporter) + }() + if err != nil { + // backoff wait + // TODO: replace with wait.Backoff when 1.13 client-go is available + consecutiveErrors++ + interval := w.minimumReconcileInterval + if consecutiveErrors < 4 { + interval = errorInterval + for i := 0; i < consecutiveErrors; i++ { + interval *= 2 + } + } + next = time.After(wait.Jitter(interval, 0.2)) + + utilruntime.HandleError(fmt.Errorf("unable to synchronize payload (waiting %s): %v", interval, err)) + continue + } + glog.V(5).Infof("Sync succeeded, reconciling") + + work.Reconciling = true + next = time.After(w.minimumReconcileInterval) + } + }, 10*time.Millisecond, stopCh) + + glog.V(5).Infof("Worker shut down") +} + +// statusWrapper prevents a newer status update from overwriting a previous +// failure from later in the sync process. +type statusWrapper struct { + w *SyncWorker + previousStatus *SyncWorkerStatus +} + +func (w *statusWrapper) Report(status SyncWorkerStatus) { + p := w.previousStatus + if p.Failure != nil && status.Failure == nil { + if p.Actual == status.Actual { + if status.Fraction < p.Fraction { + glog.V(5).Infof("Dropping status report from earlier in sync loop") + return + } + } + } + w.w.updateStatus(status) +} + +// calculateNext updates the passed work object with the desired next state and +// returns true if any changes were made. The reconciling flag is set the first +// time work transitions from empty to not empty (as a result of someone invoking +// Update). +func (w *SyncWorker) calculateNext(work *SyncWork) bool { + w.lock.Lock() + defer w.lock.Unlock() + + changed := !equalSyncWork(w.work, work) + + // if this is the first time through the loop, initialize reconciling to + // the state Update() calculated (to allow us to start in reconciling) + if work.Empty() { + work.Reconciling = w.work.Reconciling + } else { + if changed { + work.Reconciling = false + } + } + // always clear the completed variable if we are not reconciling + if !work.Reconciling { + work.Completed = 0 + } + + if w.work != nil { + work.Desired = w.work.Desired + work.Overrides = w.work.Overrides + } + + return changed +} + +// equalUpdate returns true if two updates have the same payload. +func equalUpdate(a, b configv1.Update) bool { + return a.Payload == b.Payload +} + +// equalSyncWork returns true if a and b are equal. +func equalSyncWork(a, b *SyncWork) bool { + if a == b { + return true + } + if (a == nil && b != nil) || (a != nil && b == nil) { + return false + } + return equalUpdate(a.Desired, b.Desired) && reflect.DeepEqual(a.Overrides, b.Overrides) +} + +// updateStatus records the current status of the sync action for observation +// by others. It sends a copy of the update to the report channel for improved +// testability. +func (w *SyncWorker) updateStatus(update SyncWorkerStatus) { + w.lock.Lock() + defer w.lock.Unlock() + + glog.V(5).Infof("Status change %#v", update) + w.status = update + select { + case w.report <- update: + default: + if glog.V(5) { + glog.Infof("Status report channel was full %#v", update) + } + } +} + +// Desired returns the state the SyncWorker is trying to achieve. +func (w *SyncWorker) Desired() configv1.Update { + w.lock.Lock() + defer w.lock.Unlock() + if w.work == nil { + return configv1.Update{} + } + return w.work.Desired +} + +// Status returns a copy of the current worker status. +func (w *SyncWorker) Status() *SyncWorkerStatus { + w.lock.Lock() + defer w.lock.Unlock() + return w.status.DeepCopy() +} + +// sync retrieves the payload and applies it to the server, returning an error if +// the update could not be completely applied. The status is updated as we progress. +// Cancelling the context will abort the execution of the sync. +func (w *SyncWorker) syncOnce(ctx context.Context, work *SyncWork, reporter StatusReporter) error { + glog.V(4).Infof("Running sync %s", versionString(work.Desired)) + update := work.Desired + + // cache the payload until the release image changes + payload := w.payload + if payload == nil || !equalUpdate(configv1.Update{Payload: payload.ReleaseImage}, update) { + glog.V(4).Infof("Loading payload") + reporter.Report(SyncWorkerStatus{Step: "RetrievePayload", Reconciling: work.Reconciling, Actual: update}) + payloadDir, err := w.retriever.RetrievePayload(ctx, update) + if err != nil { + reporter.Report(SyncWorkerStatus{Failure: err, Step: "RetrievePayload", Reconciling: work.Reconciling, Actual: update}) + return err + } + payload, err := loadUpdatePayload(payloadDir, update.Payload) + if err != nil { + reporter.Report(SyncWorkerStatus{Failure: err, Step: "VerifyPayload", Reconciling: work.Reconciling, Actual: update}) + return err + } + w.payload = payload + glog.V(4).Infof("Payload loaded from %s with hash %s", payload.ReleaseImage, payload.ManifestHash) + } + + return w.apply(ctx, w.payload, work, reporter) +} + +// apply updates the server with the contents of the provided payload or returns an error. +// Cancelling the context will abort the execution of the sync. +func (w *SyncWorker) apply(ctx context.Context, payload *updatePayload, work *SyncWork, reporter StatusReporter) error { + update := configv1.Update{ + Version: payload.ReleaseVersion, + Payload: payload.ReleaseImage, + } + + // update each object + version := payload.ReleaseVersion + total := len(payload.Manifests) + done := 0 + var tasks []*syncTask + for i := range payload.Manifests { + tasks = append(tasks, &syncTask{ + index: i + 1, + total: total, + manifest: &payload.Manifests[i], + backoff: w.backoff, + }) + } + + for i := 0; i < len(tasks); i++ { + task := tasks[i] + setAppliedAndPending(version, total, done) + fraction := float32(i) / float32(len(tasks)) + + reporter.Report(SyncWorkerStatus{Fraction: fraction, Step: "ApplyResources", Reconciling: work.Reconciling, VersionHash: payload.ManifestHash, Actual: update}) + + glog.V(4).Infof("Running sync for %s", task) + glog.V(5).Infof("Manifest: %s", string(task.manifest.Raw)) + + if contextIsCancelled(ctx) { + err := fmt.Errorf("update was cancelled at %d/%d", i, len(tasks)) + reporter.Report(SyncWorkerStatus{Failure: err, Fraction: fraction, Step: "ApplyResources", Reconciling: work.Reconciling, VersionHash: payload.ManifestHash, Actual: update}) + return err + } + + ov, ok := getOverrideForManifest(work.Overrides, task.manifest) + if ok && ov.Unmanaged { + glog.V(4).Infof("Skipping %s as unmanaged", task) + continue + } + + if err := task.Run(version, w.builder); err != nil { + reporter.Report(SyncWorkerStatus{Failure: err, Fraction: fraction, Step: "ApplyResources", Reconciling: work.Reconciling, VersionHash: payload.ManifestHash, Actual: update}) + cause := errors.Cause(err) + if task.requeued == 0 && shouldRequeueOnErr(cause, task.manifest) { + task.requeued++ + tasks = append(tasks, task) + continue + } + return err + } + done++ + glog.V(4).Infof("Done syncing for %s", task) + } + + setAppliedAndPending(version, total, done) + work.Completed++ + reporter.Report(SyncWorkerStatus{Fraction: 1, Completed: work.Completed, Reconciling: true, VersionHash: payload.ManifestHash, Actual: update}) + + return nil +} + +// contextIsCancelled returns true if the provided context is cancelled. +func contextIsCancelled(ctx context.Context) bool { + select { + case <-ctx.Done(): + return true + default: + return false + } +} + +// runThrottledStatusNotifier invokes fn every time ch is updated, but no more often than once +// every interval. If bucket is non-zero then the channel is throttled like a rate limiter bucket. +func runThrottledStatusNotifier(stopCh <-chan struct{}, interval time.Duration, bucket int, ch <-chan SyncWorkerStatus, fn func()) { + // notify the status change function fairly infrequently to avoid updating + // the caller status more frequently than is needed + throttle := rate.NewLimiter(rate.Every(interval), bucket) + wait.Until(func() { + ctx := context.Background() + var last SyncWorkerStatus + for { + select { + case <-stopCh: + return + case next := <-ch: + // only throttle if we aren't on an edge + if next.Actual == last.Actual && next.Reconciling == last.Reconciling && (next.Failure != nil) == (last.Failure != nil) { + if err := throttle.Wait(ctx); err != nil { + utilruntime.HandleError(fmt.Errorf("unable to throttle status notification: %v", err)) + } + } + last = next + + fn() + } + } + }, 1*time.Second, stopCh) +} diff --git a/pkg/cvo/sync_worker_test.go b/pkg/cvo/sync_worker_test.go new file mode 100644 index 000000000..1d52fc8eb --- /dev/null +++ b/pkg/cvo/sync_worker_test.go @@ -0,0 +1,131 @@ +package cvo + +import ( + "fmt" + "testing" + "time" + + configv1 "github.com/openshift/api/config/v1" +) + +func Test_statusWrapper_Report(t *testing.T) { + tests := []struct { + name string + previous SyncWorkerStatus + next SyncWorkerStatus + want bool + }{ + { + name: "skip updates that clear an error and are at an earlier fraction", + previous: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Update{Payload: "testing"}, Fraction: 0.1}, + next: SyncWorkerStatus{Actual: configv1.Update{Payload: "testing"}}, + want: false, + }, + { + previous: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Update{Payload: "testing"}, Fraction: 0.1}, + next: SyncWorkerStatus{Actual: configv1.Update{Payload: "testing2"}}, + want: true, + }, + { + previous: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Update{Payload: "testing"}}, + next: SyncWorkerStatus{Actual: configv1.Update{Payload: "testing"}}, + want: true, + }, + { + previous: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Update{Payload: "testing"}, Fraction: 0.1}, + next: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Update{Payload: "testing"}}, + want: true, + }, + { + previous: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Update{Payload: "testing"}, Fraction: 0.1}, + next: SyncWorkerStatus{Failure: fmt.Errorf("b"), Actual: configv1.Update{Payload: "testing"}, Fraction: 0.1}, + want: true, + }, + { + previous: SyncWorkerStatus{Failure: fmt.Errorf("a"), Actual: configv1.Update{Payload: "testing"}, Fraction: 0.1}, + next: SyncWorkerStatus{Failure: fmt.Errorf("b"), Actual: configv1.Update{Payload: "testing"}, Fraction: 0.2}, + want: true, + }, + { + previous: SyncWorkerStatus{Actual: configv1.Update{Payload: "testing"}}, + next: SyncWorkerStatus{Actual: configv1.Update{Payload: "testing"}}, + want: true, + }, + { + next: SyncWorkerStatus{Actual: configv1.Update{Payload: "testing"}}, + want: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + w := &statusWrapper{ + previousStatus: &tt.previous, + } + w.w = &SyncWorker{report: make(chan SyncWorkerStatus, 1)} + w.Report(tt.next) + close(w.w.report) + if tt.want { + select { + case evt, ok := <-w.w.report: + if !ok { + t.Fatalf("no event") + } + if evt != tt.next { + t.Fatalf("unexpected: %#v", evt) + } + } + } else { + select { + case evt, ok := <-w.w.report: + if ok { + t.Fatalf("unexpected event: %#v", evt) + } + } + } + }) + } +} + +func Test_runThrottledStatusNotifier(t *testing.T) { + stopCh := make(chan struct{}) + defer close(stopCh) + in := make(chan SyncWorkerStatus) + out := make(chan struct{}, 100) + + go runThrottledStatusNotifier(stopCh, 30*time.Second, 1, in, func() { out <- struct{}{} }) + + in <- SyncWorkerStatus{Actual: configv1.Update{Payload: "test"}} + select { + case <-out: + case <-time.After(100 * time.Millisecond): + t.Fatalf("should have not throttled") + } + + in <- SyncWorkerStatus{Reconciling: true, Actual: configv1.Update{Payload: "test"}} + select { + case <-out: + case <-time.After(100 * time.Millisecond): + t.Fatalf("should have not throttled") + } + + in <- SyncWorkerStatus{Failure: fmt.Errorf("a"), Reconciling: true, Actual: configv1.Update{Payload: "test"}} + select { + case <-out: + case <-time.After(100 * time.Millisecond): + t.Fatalf("should have not throttled") + } + + in <- SyncWorkerStatus{Failure: fmt.Errorf("a"), Reconciling: true, Actual: configv1.Update{Payload: "test"}} + select { + case <-out: + case <-time.After(100 * time.Millisecond): + t.Fatalf("should have not throttled") + } + + in <- SyncWorkerStatus{Failure: fmt.Errorf("a"), Reconciling: true, Actual: configv1.Update{Payload: "test"}} + select { + case <-out: + t.Fatalf("should have throttled") + case <-time.After(100 * time.Millisecond): + } +} diff --git a/pkg/cvo/updatepayload.go b/pkg/cvo/updatepayload.go index d2605a064..8c8378d94 100644 --- a/pkg/cvo/updatepayload.go +++ b/pkg/cvo/updatepayload.go @@ -2,6 +2,7 @@ package cvo import ( "bytes" + "context" "crypto/md5" "encoding/base64" "fmt" @@ -10,6 +11,8 @@ import ( "os" "path/filepath" + "k8s.io/client-go/kubernetes" + "github.com/golang/glog" imagev1 "github.com/openshift/api/image/v1" "github.com/pkg/errors" @@ -162,43 +165,68 @@ func loadUpdatePayload(dir, releaseImage string) (*updatePayload, error) { return payload, nil } -func (optr *Operator) baseDirectory() string { +func (optr *Operator) defaultPayloadDir() string { if len(optr.payloadDir) == 0 { return defaultUpdatePayloadDir } return optr.payloadDir } -func (optr *Operator) updatePayloadDir(config *configv1.ClusterVersion) (string, error) { - tdir, err := optr.targetUpdatePayloadDir(config) +func (optr *Operator) defaultPayloadRetriever() PayloadRetriever { + return &payloadRetriever{ + kubeClient: optr.kubeClient, + operatorName: optr.name, + releaseImage: optr.releaseImage, + namespace: optr.namespace, + nodeName: optr.nodename, + payloadDir: optr.defaultPayloadDir(), + workingDir: targetUpdatePayloadsDir, + } +} + +type payloadRetriever struct { + // releaseImage and payloadDir are the default payload identifiers - updates that point + // to releaseImage will always use the contents of payloadDir + releaseImage string + payloadDir string + + // these fields are used to retrieve the payload when any other payload is specified + kubeClient kubernetes.Interface + workingDir string + namespace string + nodeName string + operatorName string +} + +func (r *payloadRetriever) RetrievePayload(ctx context.Context, update configv1.Update) (string, error) { + if r.releaseImage == update.Payload { + return r.payloadDir, nil + } + + if len(update.Payload) == 0 { + return "", fmt.Errorf("no payload image has been specified and the contents of the payload cannot be retrieved") + } + + tdir, err := r.targetUpdatePayloadDir(ctx, update) if err != nil { return "", &updateError{ Reason: "UpdatePayloadRetrievalFailed", Message: fmt.Sprintf("Unable to download and prepare the update: %v", err), } } - if len(tdir) > 0 { - return tdir, nil - } - return optr.baseDirectory(), nil + return tdir, nil } -func (optr *Operator) targetUpdatePayloadDir(config *configv1.ClusterVersion) (string, error) { - payload, ok := findUpdatePayload(config) - if !ok { - return "", nil - } +func (r *payloadRetriever) targetUpdatePayloadDir(ctx context.Context, update configv1.Update) (string, error) { hash := md5.New() - hash.Write([]byte(payload)) + hash.Write([]byte(update.Payload)) payloadHash := base64.RawURLEncoding.EncodeToString(hash.Sum(nil)) - tdir := filepath.Join(targetUpdatePayloadsDir, payloadHash) + tdir := filepath.Join(r.workingDir, payloadHash) err := validateUpdatePayload(tdir) if os.IsNotExist(err) { // the dirs don't exist, try fetching the payload to tdir. - if err := optr.fetchUpdatePayloadToDir(tdir, config); err != nil { - return "", err - } + err = r.fetchUpdatePayloadToDir(ctx, tdir, update) } if err != nil { return "", err @@ -234,18 +262,15 @@ func validateUpdatePayload(dir string) error { return nil } -func (optr *Operator) fetchUpdatePayloadToDir(dir string, config *configv1.ClusterVersion) error { - if config.Spec.DesiredUpdate == nil { - return fmt.Errorf("cannot fetch payload for empty desired update") - } +func (r *payloadRetriever) fetchUpdatePayloadToDir(ctx context.Context, dir string, update configv1.Update) error { var ( - version = config.Spec.DesiredUpdate.Version - payload = config.Spec.DesiredUpdate.Payload - name = fmt.Sprintf("%s-%s-%s", optr.name, version, randutil.String(5)) - namespace = optr.namespace + version = update.Version + payload = update.Payload + name = fmt.Sprintf("%s-%s-%s", r.operatorName, version, randutil.String(5)) + namespace = r.namespace deadline = pointer.Int64Ptr(2 * 60) nodeSelectorKey = "node-role.kubernetes.io/master" - nodename = optr.nodename + nodename = r.nodeName cmd = []string{"/bin/sh"} args = []string{"-c", copyPayloadCmd(dir)} ) @@ -293,11 +318,11 @@ func (optr *Operator) fetchUpdatePayloadToDir(dir string, config *configv1.Clust }, } - _, err := optr.kubeClient.BatchV1().Jobs(job.Namespace).Create(job) + _, err := r.kubeClient.BatchV1().Jobs(job.Namespace).Create(job) if err != nil { return err } - return resourcebuilder.WaitForJobCompletion(optr.kubeClient.BatchV1(), job) + return resourcebuilder.WaitForJobCompletion(r.kubeClient.BatchV1(), job) } // copyPayloadCmd returns command that copies cvo and release manifests from deafult location @@ -318,27 +343,30 @@ func copyPayloadCmd(tdir string) string { return fmt.Sprintf("%s && %s", cvoCmd, releaseCmd) } -func findUpdatePayload(config *configv1.ClusterVersion) (string, bool) { +// findUpdateFromConfig identifies a desired update from user input or returns false. It will +// resolve payload if the user specifies a version and a matching available update or previous +// update is in the history. +func findUpdateFromConfig(config *configv1.ClusterVersion) (configv1.Update, bool) { update := config.Spec.DesiredUpdate if update == nil { - return "", false + return configv1.Update{}, false } if len(update.Payload) == 0 { - return findPayloadForVersion(config, update.Version) + return findUpdateFromConfigVersion(config, update.Version) } - return update.Payload, len(update.Payload) > 0 + return *update, true } -func findPayloadForVersion(config *configv1.ClusterVersion, version string) (string, bool) { +func findUpdateFromConfigVersion(config *configv1.ClusterVersion, version string) (configv1.Update, bool) { for _, update := range config.Status.AvailableUpdates { if update.Version == version { - return update.Payload, len(update.Payload) > 0 + return update, len(update.Payload) > 0 } } for _, history := range config.Status.History { if history.Version == version { - return history.Payload, len(history.Payload) > 0 + return configv1.Update{Payload: history.Payload, Version: history.Version}, len(history.Payload) > 0 } } - return "", false + return configv1.Update{}, false } diff --git a/vendor/k8s.io/client-go/dynamic/fake/simple.go b/vendor/k8s.io/client-go/dynamic/fake/simple.go new file mode 100644 index 000000000..a71cec50e --- /dev/null +++ b/vendor/k8s.io/client-go/dynamic/fake/simple.go @@ -0,0 +1,363 @@ +/* +Copyright 2018 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package fake + +import ( + "strings" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/runtime/serializer" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/watch" + "k8s.io/client-go/dynamic" + "k8s.io/client-go/testing" +) + +func NewSimpleDynamicClient(scheme *runtime.Scheme, objects ...runtime.Object) *FakeDynamicClient { + codecs := serializer.NewCodecFactory(scheme) + o := testing.NewObjectTracker(scheme, codecs.UniversalDecoder()) + for _, obj := range objects { + if err := o.Add(obj); err != nil { + panic(err) + } + } + + cs := &FakeDynamicClient{} + cs.AddReactor("*", "*", testing.ObjectReaction(o)) + cs.AddWatchReactor("*", func(action testing.Action) (handled bool, ret watch.Interface, err error) { + gvr := action.GetResource() + ns := action.GetNamespace() + watch, err := o.Watch(gvr, ns) + if err != nil { + return false, nil, err + } + return true, watch, nil + }) + + return cs +} + +// Clientset implements clientset.Interface. Meant to be embedded into a +// struct to get a default implementation. This makes faking out just the method +// you want to test easier. +type FakeDynamicClient struct { + testing.Fake + scheme *runtime.Scheme +} + +type dynamicResourceClient struct { + client *FakeDynamicClient + namespace string + resource schema.GroupVersionResource +} + +var _ dynamic.Interface = &FakeDynamicClient{} + +func (c *FakeDynamicClient) Resource(resource schema.GroupVersionResource) dynamic.NamespaceableResourceInterface { + return &dynamicResourceClient{client: c, resource: resource} +} + +func (c *dynamicResourceClient) Namespace(ns string) dynamic.ResourceInterface { + ret := *c + ret.namespace = ns + return &ret +} + +func (c *dynamicResourceClient) Create(obj *unstructured.Unstructured, subresources ...string) (*unstructured.Unstructured, error) { + var uncastRet runtime.Object + var err error + switch { + case len(c.namespace) == 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootCreateAction(c.resource, obj), obj) + + case len(c.namespace) == 0 && len(subresources) > 0: + accessor, err := meta.Accessor(obj) + if err != nil { + return nil, err + } + name := accessor.GetName() + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootCreateSubresourceAction(c.resource, name, strings.Join(subresources, "/"), obj), obj) + + case len(c.namespace) > 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewCreateAction(c.resource, c.namespace, obj), obj) + + case len(c.namespace) > 0 && len(subresources) > 0: + accessor, err := meta.Accessor(obj) + if err != nil { + return nil, err + } + name := accessor.GetName() + uncastRet, err = c.client.Fake. + Invokes(testing.NewCreateSubresourceAction(c.resource, name, strings.Join(subresources, "/"), c.namespace, obj), obj) + + } + + if err != nil { + return nil, err + } + if uncastRet == nil { + return nil, err + } + + ret := &unstructured.Unstructured{} + if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil { + return nil, err + } + return ret, err +} + +func (c *dynamicResourceClient) Update(obj *unstructured.Unstructured, subresources ...string) (*unstructured.Unstructured, error) { + var uncastRet runtime.Object + var err error + switch { + case len(c.namespace) == 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootUpdateAction(c.resource, obj), obj) + + case len(c.namespace) == 0 && len(subresources) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(c.resource, strings.Join(subresources, "/"), obj), obj) + + case len(c.namespace) > 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewUpdateAction(c.resource, c.namespace, obj), obj) + + case len(c.namespace) > 0 && len(subresources) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewUpdateSubresourceAction(c.resource, strings.Join(subresources, "/"), c.namespace, obj), obj) + + } + + if err != nil { + return nil, err + } + if uncastRet == nil { + return nil, err + } + + ret := &unstructured.Unstructured{} + if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil { + return nil, err + } + return ret, err +} + +func (c *dynamicResourceClient) UpdateStatus(obj *unstructured.Unstructured) (*unstructured.Unstructured, error) { + var uncastRet runtime.Object + var err error + switch { + case len(c.namespace) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootUpdateSubresourceAction(c.resource, "status", obj), obj) + + case len(c.namespace) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewUpdateSubresourceAction(c.resource, "status", c.namespace, obj), obj) + + } + + if err != nil { + return nil, err + } + if uncastRet == nil { + return nil, err + } + + ret := &unstructured.Unstructured{} + if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil { + return nil, err + } + return ret, err +} + +func (c *dynamicResourceClient) Delete(name string, opts *metav1.DeleteOptions, subresources ...string) error { + var err error + switch { + case len(c.namespace) == 0 && len(subresources) == 0: + _, err = c.client.Fake. + Invokes(testing.NewRootDeleteAction(c.resource, name), &metav1.Status{Status: "dynamic delete fail"}) + + case len(c.namespace) == 0 && len(subresources) > 0: + _, err = c.client.Fake. + Invokes(testing.NewRootDeleteSubresourceAction(c.resource, strings.Join(subresources, "/"), name), &metav1.Status{Status: "dynamic delete fail"}) + + case len(c.namespace) > 0 && len(subresources) == 0: + _, err = c.client.Fake. + Invokes(testing.NewDeleteAction(c.resource, c.namespace, name), &metav1.Status{Status: "dynamic delete fail"}) + + case len(c.namespace) > 0 && len(subresources) > 0: + _, err = c.client.Fake. + Invokes(testing.NewDeleteSubresourceAction(c.resource, strings.Join(subresources, "/"), c.namespace, name), &metav1.Status{Status: "dynamic delete fail"}) + } + + return err +} + +func (c *dynamicResourceClient) DeleteCollection(opts *metav1.DeleteOptions, listOptions metav1.ListOptions) error { + var err error + switch { + case len(c.namespace) == 0: + action := testing.NewRootDeleteCollectionAction(c.resource, listOptions) + _, err = c.client.Fake.Invokes(action, &metav1.Status{Status: "dynamic deletecollection fail"}) + + case len(c.namespace) > 0: + action := testing.NewDeleteCollectionAction(c.resource, c.namespace, listOptions) + _, err = c.client.Fake.Invokes(action, &metav1.Status{Status: "dynamic deletecollection fail"}) + + } + + return err +} + +func (c *dynamicResourceClient) Get(name string, opts metav1.GetOptions, subresources ...string) (*unstructured.Unstructured, error) { + var uncastRet runtime.Object + var err error + switch { + case len(c.namespace) == 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootGetAction(c.resource, name), &metav1.Status{Status: "dynamic get fail"}) + + case len(c.namespace) == 0 && len(subresources) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootGetSubresourceAction(c.resource, strings.Join(subresources, "/"), name), &metav1.Status{Status: "dynamic get fail"}) + + case len(c.namespace) > 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewGetAction(c.resource, c.namespace, name), &metav1.Status{Status: "dynamic get fail"}) + + case len(c.namespace) > 0 && len(subresources) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewGetSubresourceAction(c.resource, c.namespace, strings.Join(subresources, "/"), name), &metav1.Status{Status: "dynamic get fail"}) + } + + if err != nil { + return nil, err + } + if uncastRet == nil { + return nil, err + } + + ret := &unstructured.Unstructured{} + if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil { + return nil, err + } + return ret, err +} + +func (c *dynamicResourceClient) List(opts metav1.ListOptions) (*unstructured.UnstructuredList, error) { + var obj runtime.Object + var err error + switch { + case len(c.namespace) == 0: + obj, err = c.client.Fake. + Invokes(testing.NewRootListAction(c.resource, schema.GroupVersionKind{Version: "v1", Kind: "List"}, opts), &metav1.Status{Status: "dynamic list fail"}) + + case len(c.namespace) > 0: + obj, err = c.client.Fake. + Invokes(testing.NewListAction(c.resource, schema.GroupVersionKind{Version: "v1", Kind: "List"}, c.namespace, opts), &metav1.Status{Status: "dynamic list fail"}) + + } + + if obj == nil { + return nil, err + } + + label, _, _ := testing.ExtractFromListOptions(opts) + if label == nil { + label = labels.Everything() + } + + retUnstructured := &unstructured.Unstructured{} + if err := c.client.scheme.Convert(obj, retUnstructured, nil); err != nil { + return nil, err + } + entireList, err := retUnstructured.ToList() + if err != nil { + return nil, err + } + + list := &unstructured.UnstructuredList{} + for _, item := range entireList.Items { + metadata, err := meta.Accessor(item) + if err != nil { + return nil, err + } + if label.Matches(labels.Set(metadata.GetLabels())) { + list.Items = append(list.Items, item) + } + } + return list, nil +} + +func (c *dynamicResourceClient) Watch(opts metav1.ListOptions) (watch.Interface, error) { + switch { + case len(c.namespace) == 0: + return c.client.Fake. + InvokesWatch(testing.NewRootWatchAction(c.resource, opts)) + + case len(c.namespace) > 0: + return c.client.Fake. + InvokesWatch(testing.NewWatchAction(c.resource, c.namespace, opts)) + + } + + panic("math broke") +} + +func (c *dynamicResourceClient) Patch(name string, pt types.PatchType, data []byte, subresources ...string) (*unstructured.Unstructured, error) { + var uncastRet runtime.Object + var err error + switch { + case len(c.namespace) == 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootPatchAction(c.resource, name, data), &metav1.Status{Status: "dynamic patch fail"}) + + case len(c.namespace) == 0 && len(subresources) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewRootPatchSubresourceAction(c.resource, name, data, subresources...), &metav1.Status{Status: "dynamic patch fail"}) + + case len(c.namespace) > 0 && len(subresources) == 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewPatchAction(c.resource, c.namespace, name, data), &metav1.Status{Status: "dynamic patch fail"}) + + case len(c.namespace) > 0 && len(subresources) > 0: + uncastRet, err = c.client.Fake. + Invokes(testing.NewPatchSubresourceAction(c.resource, c.namespace, name, data, subresources...), &metav1.Status{Status: "dynamic patch fail"}) + + } + + if err != nil { + return nil, err + } + if uncastRet == nil { + return nil, err + } + + ret := &unstructured.Unstructured{} + if err := c.client.scheme.Convert(uncastRet, ret, nil); err != nil { + return nil, err + } + return ret, err +}