diff --git a/Makefile b/Makefile index 60ec87f4c8..b10ca3b403 100644 --- a/Makefile +++ b/Makefile @@ -95,4 +95,4 @@ images.rhel7: $(imc7) # This was copied from https://github.com/openshift/cluster-image-registry-operato test-e2e: - go test -timeout 20m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e/ + if ! go test -timeout 20m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e/; then ./test/e2e/debuglog; exit 1; fi diff --git a/pkg/apis/machineconfiguration.openshift.io/v1/types.go b/pkg/apis/machineconfiguration.openshift.io/v1/types.go index aa13fdf0fd..65ca1ba117 100644 --- a/pkg/apis/machineconfiguration.openshift.io/v1/types.go +++ b/pkg/apis/machineconfiguration.openshift.io/v1/types.go @@ -263,8 +263,9 @@ type MachineConfigPoolSpec struct { // Label selector for Machines. MachineSelector *metav1.LabelSelector `json:"machineSelector,omitempty"` - // If true, changes to this machine pool should be stopped. - // This includes generating new desiredMachineConfig and update of machines. + // If true, changes to this machine pool should be stopped, + // including generating new desiredMachineConfig and update of machines. + // This flag is intended for administrators to change. Paused bool `json:"paused"` // MaxUnavailable specifies the percentage or constant number of machines that can be updating at any given time. diff --git a/pkg/controller/node/status.go b/pkg/controller/node/status.go index 780bca5108..71325fc89e 100644 --- a/pkg/controller/node/status.go +++ b/pkg/controller/node/status.go @@ -152,8 +152,13 @@ func getUnavailableMachines(currentConfig string, nodes []*corev1.Node) []*corev if !ok || cconfig == "" { continue } + // We won't have a state annotation on initial bootstrap + dstate, ok := node.Annotations[daemon.MachineConfigDaemonStateAnnotationKey] + if !ok { + dstate = "" + } - if dconfig == currentConfig && (dconfig != cconfig || !isNodeReady(node)) { + if (dstate == daemon.MachineConfigDaemonStateBootstrap) || (dconfig == currentConfig && (dconfig != cconfig || !isNodeReady(node))) { unavail = append(unavail, nodes[idx]) } } diff --git a/pkg/controller/node/status_test.go b/pkg/controller/node/status_test.go index 2caacaf84f..73acfa202c 100644 --- a/pkg/controller/node/status_test.go +++ b/pkg/controller/node/status_test.go @@ -59,6 +59,9 @@ func newNode(name string, currentConfig, desiredConfig string) *corev1.Node { annos = map[string]string{} annos[daemon.CurrentMachineConfigAnnotationKey] = currentConfig annos[daemon.DesiredMachineConfigAnnotationKey] = desiredConfig + if currentConfig == desiredConfig { + annos[daemon.MachineConfigDaemonStateAnnotationKey] = daemon.MachineConfigDaemonStateDone + } } return &corev1.Node{ diff --git a/pkg/controller/render/render_controller.go b/pkg/controller/render/render_controller.go index 719dcf92e9..3638b503a7 100644 --- a/pkg/controller/render/render_controller.go +++ b/pkg/controller/render/render_controller.go @@ -19,7 +19,6 @@ import ( apierrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/types" utilruntime "k8s.io/apimachinery/pkg/util/runtime" "k8s.io/apimachinery/pkg/util/wait" clientset "k8s.io/client-go/kubernetes" @@ -232,12 +231,13 @@ func (ctrl *Controller) deleteMachineConfig(obj interface{}) { } } + glog.Infof("MachineConfig %s deleted", mc.Name) + if controllerRef := metav1.GetControllerOf(mc); controllerRef != nil { pool := ctrl.resolveControllerRef(controllerRef) if pool == nil { return } - glog.V(4).Infof("MachineConfig %s deleted", mc.Name) ctrl.enqueueMachineConfigPool(pool) return } @@ -248,7 +248,6 @@ func (ctrl *Controller) deleteMachineConfig(obj interface{}) { return } - glog.V(4).Infof("MachineConfig %s deleted", mc.Name) for _, p := range pools { ctrl.enqueueMachineConfigPool(p) } @@ -461,33 +460,36 @@ func (ctrl *Controller) syncGeneratedMachineConfig(pool *mcfgv1.MachineConfigPoo return err } - gmcs, err := ctrl.mcLister.List(labels.Everything()) - if err != nil { - return err - } - for _, gmc := range gmcs { - if gmc.Name == generated.Name { - continue - } - - deleteOwnerRefPatch := fmt.Sprintf(`{"metadata":{"ownerReferences":[{"$patch":"delete","uid":"%s"}],"uid":"%s"}}`, pool.UID, gmc.UID) - _, err = ctrl.client.MachineconfigurationV1().MachineConfigs().Patch(gmc.Name, types.JSONPatchType, []byte(deleteOwnerRefPatch)) - if err != nil { - if errors.IsNotFound(err) { - // If the machineconfig no longer exists, ignore it. - continue - } - if errors.IsInvalid(err) { - // Invalid error will be returned in two cases: 1. the machineconfig - // has no owner reference, 2. the uid of the machineconfig doesn't - // match. - // In both cases, the error can be ignored. - continue - } - // Let's not make it fatal for now - glog.Warningf("Failed to delete ownerReference from %s: %v", gmc.Name, err) - } - } + // this code is known broken + // gmcs, err := ctrl.mcLister.List(labels.Everything()) + // if err != nil { + // return err + // } + // for _, gmc := range gmcs { + // if gmc.Name == generated.Name { + // continue + // } + + // deleteOwnerRefPatch := fmt.Sprintf(`{"metadata":{"ownerReferences":[{"$patch":"delete","uid":"%s"}],"uid":"%s"}}`, pool.UID, gmc.UID) + // _, err = ctrl.client.MachineconfigurationV1().MachineConfigs().Patch(gmc.Name, types.JSONPatchType, []byte(deleteOwnerRefPatch)) + // if err != nil { + // if errors.IsNotFound(err) { + // // If the machineconfig no longer exists, ignore it. + // continue + // } + // if errors.IsInvalid(err) { + // // Invalid error will be returned in two cases: 1. the machineconfig + // // has no owner reference, 2. the uid of the machineconfig doesn't + // // match. + // // In both cases, the error can be ignored. + // continue + // } + // // Let's not make it fatal for now + // glog.Warningf("Failed to delete ownerReference from %s: %v", gmc.Name, err) + // } else { + // glog.Infof("Queued for GC: %s", gmc.Name) + // } + // } return nil } diff --git a/pkg/controller/template/render.go b/pkg/controller/template/render.go index 05758bbae6..9882bd1211 100644 --- a/pkg/controller/template/render.go +++ b/pkg/controller/template/render.go @@ -91,21 +91,16 @@ func generateMachineConfigs(config *RenderConfig, templateDir string) ([]*mcfgv1 return cfgs, nil } +// GenerateMachineConfigsForRole is part of generateMachineConfigs; it operates +// on a specific role which has a set of builtin templates. func GenerateMachineConfigsForRole(config *RenderConfig, role string, path string) ([]*mcfgv1.MachineConfig, error) { + cfgs := []*mcfgv1.MachineConfig{} + + // Add our built-in templates infos, err := ioutil.ReadDir(path) if err != nil { return nil, fmt.Errorf("failed to read dir %q: %v", path, err) } - // for each role a machine config is created containing the sshauthorized keys to allow for ssh access - // ex: role = worker -> machine config "00-worker-ssh" created containing user core and ssh key - var tempIgnConfig ignv2_2types.Config - tempUser := ignv2_2types.PasswdUser{Name: "core", SSHAuthorizedKeys: []ignv2_2types.SSHAuthorizedKey{ignv2_2types.SSHAuthorizedKey(config.SSHKey)}} - tempIgnConfig.Passwd.Users = append(tempIgnConfig.Passwd.Users, tempUser) - sshConfigName := "00-" + role + "-ssh" - sshMachineConfigForRole := MachineConfigFromIgnConfig(role, sshConfigName, &tempIgnConfig) - - cfgs := []*mcfgv1.MachineConfig{} - cfgs = append(cfgs, sshMachineConfigForRole) for _, info := range infos { if !info.IsDir() { @@ -121,9 +116,68 @@ func GenerateMachineConfigsForRole(config *RenderConfig, role string, path strin cfgs = append(cfgs, nameConfig) } + // And derived configs + derivedCfgs, err := generateDerivedMachineConfigs(config, role) + if err != nil { + return nil, err + } + cfgs = append(cfgs, derivedCfgs...) + + return cfgs, nil +} + +// machineConfigForOSImageURL generates a MC fragment that just includes the target OSImageURL. +func machineConfigForOSImageURL(role string, url string) *mcfgv1.MachineConfig { + labels := map[string]string{ + machineConfigRoleLabelKey: role, + } + return &mcfgv1.MachineConfig{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Name: "00-" + role + "-osimageurl", + }, + Spec: mcfgv1.MachineConfigSpec{ + OSImageURL: url, + }, + } +} + +// Temporary hack +var doneInitialRender map[string]bool + +// generateDerivedMachineConfigs is part of generateMachineConfigsForRole. It +// takes care of generating MachineConfig objects which are derived from other +// components of the cluster configuration. Currently, that's: +// +// - SSH keys from the install configuration +// - OSImageURL from the machine-config-osimageurl configmap (which comes from the CVO) +func generateDerivedMachineConfigs(config *RenderConfig, role string) ([]*mcfgv1.MachineConfig, error) { + cfgs := []*mcfgv1.MachineConfig{} + + // for each role a machine config is created containing the sshauthorized keys to allow for ssh access + // ex: role = worker -> machine config "00-worker-ssh" created containing user core and ssh key + var tempIgnConfig ignv2_2types.Config + tempUser := ignv2_2types.PasswdUser{Name: "core", SSHAuthorizedKeys: []ignv2_2types.SSHAuthorizedKey{ignv2_2types.SSHAuthorizedKey(config.SSHKey)}} + tempIgnConfig.Passwd.Users = append(tempIgnConfig.Passwd.Users, tempUser) + sshConfigName := "00-" + role + "-ssh" + cfgs = append(cfgs, MachineConfigFromIgnConfig(role, sshConfigName, &tempIgnConfig)) + + if doneInitialRender == nil { + doneInitialRender = make(map[string]bool) + } + + isFirstRun := !doneInitialRender[role] + if config.OSImageURL != "" && !isFirstRun { + cfgs = append(cfgs, machineConfigForOSImageURL(role, config.OSImageURL)) + } + if isFirstRun { + doneInitialRender[role] = true + } + return cfgs, nil } +// generateMachineConfigForName is part of the implementation of generateMachineConfigsForRole func generateMachineConfigForName(config *RenderConfig, role, name, path string) (*mcfgv1.MachineConfig, error) { platformDirs := []string{} for _, dir := range []string{"_base", config.Platform} { diff --git a/pkg/daemon/constants.go b/pkg/daemon/constants.go index 4f5988d660..e4ffa2c8c4 100644 --- a/pkg/daemon/constants.go +++ b/pkg/daemon/constants.go @@ -7,6 +7,8 @@ const ( DesiredMachineConfigAnnotationKey = "machineconfiguration.openshift.io/desiredConfig" // MachineConfigDaemonStateAnnotationKey is used to fetch the state of the daemon on the machine. MachineConfigDaemonStateAnnotationKey = "machineconfiguration.openshift.io/state" + // MachineConfigDaemonStateBootstrap is the initial state of a system on boot + MachineConfigDaemonStateBootstrap = "Bootstrap" // MachineConfigDaemonStateWorking is set by daemon when it is applying an update. MachineConfigDaemonStateWorking = "Working" // MachineConfigDaemonStateDone is set by daemon when it is done applying an update. diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index 5090b48d12..2a2381293c 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -357,6 +357,7 @@ func (dn *Daemon) EnterDegradedState(err error) { // // If any of the object names are the same, they will be pointer-equal. type stateAndConfigs struct { + bootstrapping bool state string currentConfig *mcfgv1.MachineConfig pendingConfig *mcfgv1.MachineConfig @@ -364,14 +365,17 @@ type stateAndConfigs struct { } func (dn *Daemon) getStateAndConfigs(pendingConfigName string) (*stateAndConfigs, error) { - state, err := getNodeAnnotationExt(dn.kubeClient.CoreV1().Nodes(), dn.name, MachineConfigDaemonStateAnnotationKey, true) + _, err := os.Lstat(InitialNodeAnnotationsFilePath) + bootstrapping := false if err != nil { + if os.IsNotExist(err) { + // The node annotation file (laid down by the MCS) + // doesn't exist, we must not be bootstrapping + } return nil, err - } - // Temporary hack: the MCS used to not write the state=done annotation - // key. If it's unset, let's write it now. - if state == "" { - state = MachineConfigDaemonStateDone + } else { + bootstrapping = true + glog.Infof("In bootstrap mode") } currentConfigName, err := getNodeAnnotation(dn.kubeClient.CoreV1().Nodes(), dn.name, CurrentMachineConfigAnnotationKey) @@ -386,6 +390,16 @@ func (dn *Daemon) getStateAndConfigs(pendingConfigName string) (*stateAndConfigs if err != nil { return nil, err } + state, err := getNodeAnnotationExt(dn.kubeClient.CoreV1().Nodes(), dn.name, MachineConfigDaemonStateAnnotationKey, true) + if err != nil { + return nil, err + } + // Temporary hack: the MCS used to not write the state=done annotation + // key. If it's unset, let's write it now. + if state == "" { + state = MachineConfigDaemonStateDone + } + var desiredConfig *mcfgv1.MachineConfig if currentConfigName == desiredConfigName { desiredConfig = currentConfig @@ -415,6 +429,7 @@ func (dn *Daemon) getStateAndConfigs(pendingConfigName string) (*stateAndConfigs } return &stateAndConfigs{ + bootstrapping: bootstrapping, currentConfig: currentConfig, pendingConfig: pendingConfig, desiredConfig: desiredConfig, @@ -540,6 +555,26 @@ func (dn *Daemon) CheckStateOnBoot() error { select {} } + if state.bootstrapping { + if !dn.checkOS(state.currentConfig.Spec.OSImageURL) { + glog.Infof("Bootstrap pivot required") + // This only returns on error + return dn.updateOSAndReboot(state.currentConfig) + } else { + glog.Infof("No bootstrap pivot required; unlinking bootstrap node annotations") + // Delete the bootstrap node annotations; the + // currentConfig's osImageURL should now be *truth*. + // In other words if it drifts somehow, we go degraded. + if err := os.Remove(InitialNodeAnnotationsFilePath); err != nil { + return errors.Wrapf(err, "Removing initial node annotations file") + } + // And add the done state annotation + if err := dn.nodeWriter.SetUpdateDone(dn.kubeClient.CoreV1().Nodes(), dn.name, state.currentConfig.GetName()); err != nil { + return err + } + } + } + // Validate the on-disk state against what we *expect*. // // In the case where we're booting a node for the first time, or the MCD @@ -742,10 +777,6 @@ func (dn *Daemon) completeUpdate(desiredConfigName string) error { // triggerUpdateWithMachineConfig starts the update. It queries the cluster for // the current and desired config if they weren't passed. func (dn *Daemon) triggerUpdateWithMachineConfig(currentConfig *mcfgv1.MachineConfig, desiredConfig *mcfgv1.MachineConfig) error { - if err := dn.nodeWriter.SetUpdateWorking(dn.kubeClient.CoreV1().Nodes(), dn.name); err != nil { - return err - } - if currentConfig == nil { ccAnnotation, err := getNodeAnnotation(dn.kubeClient.CoreV1().Nodes(), dn.name, CurrentMachineConfigAnnotationKey) if err != nil { diff --git a/pkg/daemon/update.go b/pkg/daemon/update.go index a0e52e9a2f..1c230fb5bc 100644 --- a/pkg/daemon/update.go +++ b/pkg/daemon/update.go @@ -65,41 +65,16 @@ func (dn *Daemon) writePendingState(desiredConfig *mcfgv1.MachineConfig) error { return replaceFileContentsAtomically(pathStateJSON, b) } -// update the node to the provided node configuration. -func (dn *Daemon) update(oldConfig, newConfig *mcfgv1.MachineConfig) error { +// updateOSAndReboot is the last step in an update(), and it can also +// be called as a special case for the "bootstrap pivot". +func (dn *Daemon) updateOSAndReboot(newConfig *mcfgv1.MachineConfig) error { var err error - oldConfigName := oldConfig.GetName() - newConfigName := newConfig.GetName() - glog.Infof("Checking reconcilable for config %v to %v", oldConfigName, newConfigName) - // make sure we can actually reconcile this state - reconcilableError := dn.reconcilable(oldConfig, newConfig) - - if reconcilableError != nil { - msg := fmt.Sprintf("Can't reconcile config %v with %v: %v", oldConfigName, newConfigName, *reconcilableError) - if dn.recorder != nil { - dn.recorder.Eventf(newConfig, corev1.EventTypeWarning, "FailedToReconcile", msg) - } - dn.logSystem(msg) - return fmt.Errorf("%s", msg) - } - - // update files on disk that need updating - if err = dn.updateFiles(oldConfig, newConfig); err != nil { - return err - } - if err = dn.updateOS(newConfig); err != nil { return err } - if err = dn.updateSSHKeys(newConfig.Spec.Config.Passwd.Users); err != nil { - return err - } - - // TODO: Change the logic to be clearer - // We need to skip draining of the node when we are running once - // and there is no cluster. + // Skip draining of the node when we're not cluster driven if dn.onceFrom == "" { glog.Info("Update prepared; draining the node") @@ -122,12 +97,49 @@ func (dn *Daemon) update(oldConfig, newConfig *mcfgv1.MachineConfig) error { glog.V(2).Infof("Node successfully drained") } + // reboot. this function shouldn't actually return. + return dn.reboot(fmt.Sprintf("Node will reboot into config %v", newConfig.GetName())) +} + +// update the node to the provided node configuration. +func (dn *Daemon) update(oldConfig, newConfig *mcfgv1.MachineConfig) error { + var err error + + if dn.nodeWriter != nil { + if err = dn.nodeWriter.SetUpdateWorking(dn.kubeClient.CoreV1().Nodes(), dn.name); err != nil { + return err + } + } + + oldConfigName := oldConfig.GetName() + newConfigName := newConfig.GetName() + glog.Infof("Checking reconcilable for config %v to %v", oldConfigName, newConfigName) + // make sure we can actually reconcile this state + reconcilableError := dn.reconcilable(oldConfig, newConfig) + + if reconcilableError != nil { + msg := fmt.Sprintf("Can't reconcile config %v with %v: %v", oldConfigName, newConfigName, *reconcilableError) + if dn.recorder != nil { + dn.recorder.Eventf(newConfig, corev1.EventTypeWarning, "FailedToReconcile", msg) + } + dn.logSystem(msg) + return fmt.Errorf("%s", msg) + } + + // update files on disk that need updating + if err = dn.updateFiles(oldConfig, newConfig); err != nil { + return err + } + + if err = dn.updateSSHKeys(newConfig.Spec.Config.Passwd.Users); err != nil { + return err + } + if err = dn.writePendingState(newConfig); err != nil { return errors.Wrapf(err, "writing pending state") } - // reboot. this function shouldn't actually return. - return dn.reboot(fmt.Sprintf("Node will reboot into config %v", newConfigName)) + return dn.updateOSAndReboot(newConfig) } // reconcilable checks the configs to make sure that the only changes requested diff --git a/pkg/operator/operator.go b/pkg/operator/operator.go index e4d52422eb..94e6f713f9 100644 --- a/pkg/operator/operator.go +++ b/pkg/operator/operator.go @@ -55,6 +55,8 @@ const ( type Operator struct { namespace, name string + inClusterBringup bool + imagesFile string client mcfgclientset.Interface @@ -155,6 +157,17 @@ func (optr *Operator) Run(workers int, stopCh <-chan struct{}) { glog.Info("Starting MachineConfigOperator") defer glog.Info("Shutting down MachineConfigOperator") + apiClient := optr.apiExtClient.ApiextensionsV1beta1() + _, err := apiClient.CustomResourceDefinitions().Get("machineconfigpools.machineconfiguration.openshift.io", metav1.GetOptions{}) + if err != nil { + if apierrors.IsNotFound(err) { + glog.Infof("Couldn't find machineconfigpool CRD, in cluster bringup mode") + optr.inClusterBringup = true + } else { + glog.Errorf("While checking for cluster bringup: %v", err) + } + } + if !cache.WaitForCacheSync(stopCh, optr.crdListerSynced, optr.mcoconfigListerSynced, diff --git a/pkg/operator/sync.go b/pkg/operator/sync.go index 8261c3b596..9a59f373de 100644 --- a/pkg/operator/sync.go +++ b/pkg/operator/sync.go @@ -19,17 +19,20 @@ import ( "github.com/openshift/machine-config-operator/pkg/version" ) -type syncFunc func(config renderConfig) error +type syncFunc struct { + name string + fn func(config renderConfig) error +} func (optr *Operator) syncAll(rconfig renderConfig) error { // syncFuncs is the list of sync functions that are executed in order. // any error marks sync as failure but continues to next syncFunc - syncFuncs := []syncFunc{ - optr.syncMachineConfigPools, - optr.syncMachineConfigController, - optr.syncMachineConfigServer, - optr.syncMachineConfigDaemon, - optr.syncRequiredMachineConfigPools, + var syncFuncs = []syncFunc{ + { "pools", optr.syncMachineConfigPools }, + { "mcs", optr.syncMachineConfigServer }, + { "mcd", optr.syncMachineConfigDaemon }, + { "mcc", optr.syncMachineConfigController }, + { "required-pools", optr.syncRequiredMachineConfigPools }, } if err := optr.syncProgressingStatus(); err != nil { @@ -37,8 +40,12 @@ func (optr *Operator) syncAll(rconfig renderConfig) error { } var errs []error - for _, f := range syncFuncs { - errs = append(errs, f(rconfig)) + for _, sf := range syncFuncs { + startTime := time.Now() + errs = append(errs, sf.fn(rconfig)) + if optr.inClusterBringup { + glog.Infof("[init mode] synced %s in %v", sf.name, time.Since(startTime)) + } optr.syncProgressingStatus() } @@ -47,6 +54,9 @@ func (optr *Operator) syncAll(rconfig renderConfig) error { errs = append(errs, optr.syncFailingStatus(agg)) agg = utilerrors.NewAggregate(errs) return fmt.Errorf("error syncing: %v", agg.Error()) + } else { + glog.Infof("Initialization complete") + optr.inClusterBringup = false } return optr.syncAvailableStatus() diff --git a/pkg/server/server.go b/pkg/server/server.go index 4a8ee75b45..e6b37f2058 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -66,7 +66,7 @@ func getNodeAnnotation(conf string) (string, error) { nodeAnnotations := map[string]string{ daemon.CurrentMachineConfigAnnotationKey: conf, daemon.DesiredMachineConfigAnnotationKey: conf, - daemon.MachineConfigDaemonStateAnnotationKey: daemon.MachineConfigDaemonStateDone, + daemon.MachineConfigDaemonStateAnnotationKey: daemon.MachineConfigDaemonStateBootstrap, } contents, err := json.Marshal(nodeAnnotations) if err != nil { diff --git a/test/e2e/debuglog b/test/e2e/debuglog new file mode 100755 index 0000000000..279c8c6e73 --- /dev/null +++ b/test/e2e/debuglog @@ -0,0 +1,9 @@ +#!/bin/bash +set -xeuo pipefail +oc get nodes +oc -n openshift-machine-config-operator get machineconfigpool +oc -n openshift-machine-config-operator get machineconfigs +oc -n openshift-machine-config-operator get ds +oc -n openshift-machine-config-operator get pods +oc -n openshift-machine-config-operator logs deploy/machine-config-operator +oc -n openshift-machine-config-operator logs deploy/machine-config-controller \ No newline at end of file diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 046fa5a1ab..961f27a3cf 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -5,6 +5,10 @@ import ( "testing" ) +const ( + namespace = "openshift-machine-config-operator" +) + func TestMain(m *testing.M) { os.Exit(m.Run()) } diff --git a/test/e2e/sanity_test.go b/test/e2e/sanity_test.go index 794e377ce8..9dc70f6d4c 100644 --- a/test/e2e/sanity_test.go +++ b/test/e2e/sanity_test.go @@ -2,10 +2,22 @@ package e2e_test import ( "testing" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/informers" + "k8s.io/apimachinery/pkg/labels" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "github.com/openshift/machine-config-operator/cmd/common" + "github.com/openshift/machine-config-operator/pkg/daemon" +) + + +var ( + controllerKind = mcfgv1.SchemeGroupVersion.WithKind("MachineConfigPool") ) // Test case for https://github.com/openshift/machine-config-operator/pull/288/commits/44d5c5215b5450fca32806f796b50a3372daddc2 @@ -16,7 +28,7 @@ func TestOperatorLabel(t *testing.T) { } k := cb.KubeClientOrDie("sanity-test") - d, err := k.AppsV1().DaemonSets("openshift-machine-config-operator").Get("machine-config-daemon", metav1.GetOptions{}) + d, err := k.AppsV1().DaemonSets(namespace).Get("machine-config-daemon", metav1.GetOptions{}) if err != nil { t.Errorf("%#v", err) } @@ -26,3 +38,96 @@ func TestOperatorLabel(t *testing.T) { t.Errorf("Expected node selector 'linux', not '%s'", osSelector) } } + +func TestNoDegraded(t *testing.T) { + cb, err := common.NewClientBuilder("") + if err != nil{ + t.Errorf("%#v", err) + } + k := cb.KubeClientOrDie("sanity-test") + + kubeSharedInformer := informers.NewSharedInformerFactory(k, 20 * time.Minute) + nodeInformer := kubeSharedInformer.Core().V1().Nodes() + nodeLister := nodeInformer.Lister() + nodeListerSynced := nodeInformer.Informer().HasSynced + + stopCh := make(chan struct{}) + kubeSharedInformer.Start(stopCh) + cache.WaitForCacheSync(stopCh, nodeListerSynced) + + nodes, err := nodeLister.List(labels.Everything()) + if err != nil { + t.Errorf("listing nodes: %v", err) + } + + var degraded []*corev1.Node + for _, node := range nodes { + if node.Annotations == nil { + continue + } + dstate, ok := node.Annotations[daemon.MachineConfigDaemonStateAnnotationKey] + if !ok || dstate == "" { + continue + } + + if dstate == daemon.MachineConfigDaemonStateDegraded { + degraded = append(degraded, node) + } + } + + if len(degraded) > 0 { + t.Errorf("%d degraded nodes found", len(degraded)) + } +} + +// func getRenderedMachineConfigs(mcfgs []mcfgv1.MachineConfig) ([]*mcfgv1.MachineConfig, []*mcfgv1.MachineConfig) { +// var masters []*mcfgv1.MachineConfig +// var workers []*mcfgv1.MachineConfig +// for _, mcfg := range mcfgs { +// if controllerRef := metav1.GetControllerOf(&mcfg); controllerRef != nil { +// if controllerRef.Kind != controllerKind.Kind { +// continue +// } +// if strings.HasPrefix(mcfg.Name, "master-") { +// masters = append(masters, &mcfg) +// } else if strings.HasPrefix(mcfg.Name, "worker-") { +// workers = append(workers, &mcfg) +// } +// } +// } +// return masters, workers +// } + +func TestMachineConfigsOSImageURL(t *testing.T) { + cb, err := common.NewClientBuilder("") + if err != nil{ + t.Fatalf("%#v", err) + } + mcClient := cb.MachineConfigClientOrDie("mc-test") + + masterMCP, err := mcClient.MachineconfigurationV1().MachineConfigPools().Get("master", metav1.GetOptions{}) + if err != nil { + t.Fatalf("Getting master MCP: %v", err) + } + workerMCP, err := mcClient.MachineconfigurationV1().MachineConfigPools().Get("worker", metav1.GetOptions{}) + if err != nil { + t.Fatalf("Getting worker MCP: %v", err) + } + + masterMC, err := mcClient.MachineconfigurationV1().MachineConfigs().Get(masterMCP.Status.Configuration.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Getting MC: %v", err) + } + if masterMC.Spec.OSImageURL == "" { + t.Fatalf("master has no OSImageURL") + } + + workerMC, err := mcClient.MachineconfigurationV1().MachineConfigs().Get(workerMCP.Status.Configuration.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Getting MC: %v", err) + } + if workerMC.Spec.OSImageURL == "" { + t.Fatalf("master has no OSImageURL") + } +} +