WIP: Render osImageURL, handle "bootstrap" case in MCD

Makefile

@@ -95,4 +95,4 @@ images.rhel7: $(imc7)
 
 # This was copied from https://github.com/openshift/cluster-image-registry-operato
 test-e2e:
-	go test -timeout 20m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e/
+	if ! go test -timeout 20m -v$${WHAT:+ -run="$$WHAT"} ./test/e2e/; then ./test/e2e/debuglog; exit 1; fi

pkg/apis/machineconfiguration.openshift.io/v1/types.go

@@ -263,8 +263,9 @@ type MachineConfigPoolSpec struct {
 	// Label selector for Machines.
 	MachineSelector *metav1.LabelSelector `json:"machineSelector,omitempty"`
 
-	// If true, changes to this machine pool should be stopped.
-	// This includes generating new desiredMachineConfig and update of machines.
+	// If true, changes to this machine pool should be stopped,
+	// including generating new desiredMachineConfig and update of machines.
+	// This flag is intended for administrators to change.
 	Paused bool `json:"paused"`
 
 	// MaxUnavailable specifies the percentage or constant number of machines that can be updating at any given time.

pkg/controller/node/status.go

@@ -152,8 +152,13 @@ func getUnavailableMachines(currentConfig string, nodes []*corev1.Node) []*corev
 		if !ok || cconfig == "" {
 			continue
 		}
+		// We won't have a state annotation on initial bootstrap
+		dstate, ok := node.Annotations[daemon.MachineConfigDaemonStateAnnotationKey]
+		if !ok {
+			dstate = ""
+		}
 
-		if dconfig == currentConfig && (dconfig != cconfig || !isNodeReady(node)) {
+		if (dstate == daemon.MachineConfigDaemonStateBootstrap) || (dconfig == currentConfig && (dconfig != cconfig || !isNodeReady(node))) {
 			unavail = append(unavail, nodes[idx])
 		}
 	}

pkg/controller/node/status_test.go

@@ -59,6 +59,9 @@ func newNode(name string, currentConfig, desiredConfig string) *corev1.Node {
 		annos = map[string]string{}
 		annos[daemon.CurrentMachineConfigAnnotationKey] = currentConfig
 		annos[daemon.DesiredMachineConfigAnnotationKey] = desiredConfig
+		if currentConfig == desiredConfig {
+			annos[daemon.MachineConfigDaemonStateAnnotationKey] = daemon.MachineConfigDaemonStateDone
+		}
 	}
 
 	return &corev1.Node{

pkg/controller/render/render_controller.go

@@ -19,7 +19,6 @@ import (
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/labels"
-	"k8s.io/apimachinery/pkg/types"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	clientset "k8s.io/client-go/kubernetes"
@@ -232,12 +231,13 @@ func (ctrl *Controller) deleteMachineConfig(obj interface{}) {
 		}
 	}
 
+	glog.Infof("MachineConfig %s deleted", mc.Name)
+
 	if controllerRef := metav1.GetControllerOf(mc); controllerRef != nil {
 		pool := ctrl.resolveControllerRef(controllerRef)
 		if pool == nil {
 			return
 		}
-		glog.V(4).Infof("MachineConfig %s deleted", mc.Name)
 		ctrl.enqueueMachineConfigPool(pool)
 		return
 	}
@@ -248,7 +248,6 @@ func (ctrl *Controller) deleteMachineConfig(obj interface{}) {
 		return
 	}
 
-	glog.V(4).Infof("MachineConfig %s deleted", mc.Name)
 	for _, p := range pools {
 		ctrl.enqueueMachineConfigPool(p)
 	}
@@ -461,33 +460,36 @@ func (ctrl *Controller) syncGeneratedMachineConfig(pool *mcfgv1.MachineConfigPoo
 		return err
 	}
 
-	gmcs, err := ctrl.mcLister.List(labels.Everything())
-	if err != nil {
-		return err
-	}
-	for _, gmc := range gmcs {
-		if gmc.Name == generated.Name {
-			continue
-		}
-
-		deleteOwnerRefPatch := fmt.Sprintf(`{"metadata":{"ownerReferences":[{"$patch":"delete","uid":"%s"}],"uid":"%s"}}`, pool.UID, gmc.UID)
-		_, err = ctrl.client.MachineconfigurationV1().MachineConfigs().Patch(gmc.Name, types.JSONPatchType, []byte(deleteOwnerRefPatch))
-		if err != nil {
-			if errors.IsNotFound(err) {
-				// If the machineconfig no longer exists, ignore it.
-				continue
-			}
-			if errors.IsInvalid(err) {
-				// Invalid error will be returned in two cases: 1. the machineconfig
-				// has no owner reference, 2. the uid of the machineconfig doesn't
-				// match.
-				// In both cases, the error can be ignored.
-				continue
-			}
-			// Let's not make it fatal for now
-			glog.Warningf("Failed to delete ownerReference from %s: %v", gmc.Name, err)
-		}
-	}
+	// this code is known broken
+	// gmcs, err := ctrl.mcLister.List(labels.Everything())
+	// if err != nil {
+	// 	return err
+	// }
+	// for _, gmc := range gmcs {
+	// 	if gmc.Name == generated.Name {
+	// 		continue
+	// 	}
+
+	// 	deleteOwnerRefPatch := fmt.Sprintf(`{"metadata":{"ownerReferences":[{"$patch":"delete","uid":"%s"}],"uid":"%s"}}`, pool.UID, gmc.UID)
+	// 	_, err = ctrl.client.MachineconfigurationV1().MachineConfigs().Patch(gmc.Name, types.JSONPatchType, []byte(deleteOwnerRefPatch))
+	// 	if err != nil {
+	// 		if errors.IsNotFound(err) {
+	// 			// If the machineconfig no longer exists, ignore it.
+	// 			continue
+	// 		}
+	// 		if errors.IsInvalid(err) {
+	// 			// Invalid error will be returned in two cases: 1. the machineconfig
+	// 			// has no owner reference, 2. the uid of the machineconfig doesn't
+	// 			// match.
+	// 			// In both cases, the error can be ignored.
+	// 			continue
+	// 		}
+	// 		// Let's not make it fatal for now
+	// 		glog.Warningf("Failed to delete ownerReference from %s: %v", gmc.Name, err)
+	// 	} else {
+	// 		glog.Infof("Queued for GC: %s", gmc.Name)
+	// 	}
+	// }
 
 	return nil
 }

pkg/controller/template/render.go

@@ -91,21 +91,16 @@ func generateMachineConfigs(config *RenderConfig, templateDir string) ([]*mcfgv1
 	return cfgs, nil
 }
 
+// GenerateMachineConfigsForRole is part of generateMachineConfigs; it operates
+// on a specific role which has a set of builtin templates.
 func GenerateMachineConfigsForRole(config *RenderConfig, role string, path string) ([]*mcfgv1.MachineConfig, error) {
+	cfgs := []*mcfgv1.MachineConfig{}
+
+	// Add our built-in templates
 	infos, err := ioutil.ReadDir(path)
 	if err != nil {
 		return nil, fmt.Errorf("failed to read dir %q: %v", path, err)
 	}
-	// for each role a machine config is created containing the sshauthorized keys to allow for ssh access
-	// ex: role = worker -> machine config "00-worker-ssh" created containing user core and ssh key
-	var tempIgnConfig ignv2_2types.Config
-	tempUser := ignv2_2types.PasswdUser{Name: "core", SSHAuthorizedKeys: []ignv2_2types.SSHAuthorizedKey{ignv2_2types.SSHAuthorizedKey(config.SSHKey)}}
-	tempIgnConfig.Passwd.Users = append(tempIgnConfig.Passwd.Users, tempUser)
-	sshConfigName := "00-" + role + "-ssh"
-	sshMachineConfigForRole := MachineConfigFromIgnConfig(role, sshConfigName, &tempIgnConfig)
-
-	cfgs := []*mcfgv1.MachineConfig{}
-	cfgs = append(cfgs, sshMachineConfigForRole)
 
 	for _, info := range infos {
 		if !info.IsDir() {
@@ -121,9 +116,68 @@ func GenerateMachineConfigsForRole(config *RenderConfig, role string, path strin
 		cfgs = append(cfgs, nameConfig)
 	}
 
+	// And derived configs
+	derivedCfgs, err := generateDerivedMachineConfigs(config, role)
+	if err != nil {
+		return nil, err
+	}
+	cfgs = append(cfgs, derivedCfgs...)
+
+	return cfgs, nil
+}
+
+// machineConfigForOSImageURL generates a MC fragment that just includes the target OSImageURL.
+func machineConfigForOSImageURL(role string, url string) *mcfgv1.MachineConfig {
+	labels := map[string]string{
+		machineConfigRoleLabelKey: role,
+	}
+	return &mcfgv1.MachineConfig{
+		ObjectMeta: metav1.ObjectMeta{
+			Labels: labels,
+			Name:   "00-" + role + "-osimageurl",
+		},
+		Spec: mcfgv1.MachineConfigSpec{
+			OSImageURL: url,
+		},
+	}
+}
+
+// Temporary hack
+var doneInitialRender map[string]bool
+
+// generateDerivedMachineConfigs is part of generateMachineConfigsForRole. It
+// takes care of generating MachineConfig objects which are derived from other
+// components of the cluster configuration. Currently, that's:
+//
+//  - SSH keys from the install configuration
+//  - OSImageURL from the machine-config-osimageurl configmap (which comes from the CVO)
+func generateDerivedMachineConfigs(config *RenderConfig, role string) ([]*mcfgv1.MachineConfig, error) {
+	cfgs := []*mcfgv1.MachineConfig{}
+
+	// for each role a machine config is created containing the sshauthorized keys to allow for ssh access
+	// ex: role = worker -> machine config "00-worker-ssh" created containing user core and ssh key
+	var tempIgnConfig ignv2_2types.Config
+	tempUser := ignv2_2types.PasswdUser{Name: "core", SSHAuthorizedKeys: []ignv2_2types.SSHAuthorizedKey{ignv2_2types.SSHAuthorizedKey(config.SSHKey)}}
+	tempIgnConfig.Passwd.Users = append(tempIgnConfig.Passwd.Users, tempUser)
+	sshConfigName := "00-" + role + "-ssh"
+	cfgs = append(cfgs, MachineConfigFromIgnConfig(role, sshConfigName, &tempIgnConfig))
+
+	if doneInitialRender == nil {
+		doneInitialRender = make(map[string]bool)
+	}
+
+	isFirstRun := !doneInitialRender[role]
+	if config.OSImageURL != "" && !isFirstRun {
+		cfgs = append(cfgs, machineConfigForOSImageURL(role, config.OSImageURL))
+	}
+	if isFirstRun {
+		doneInitialRender[role] = true
+	}
+
 	return cfgs, nil
 }
 
+// generateMachineConfigForName is part of the implementation of generateMachineConfigsForRole
 func generateMachineConfigForName(config *RenderConfig, role, name, path string) (*mcfgv1.MachineConfig, error) {
 	platformDirs := []string{}
 	for _, dir := range []string{"_base", config.Platform} {

pkg/daemon/constants.go

@@ -7,6 +7,8 @@ const (
 	DesiredMachineConfigAnnotationKey = "machineconfiguration.openshift.io/desiredConfig"
 	// MachineConfigDaemonStateAnnotationKey is used to fetch the state of the daemon on the machine.
 	MachineConfigDaemonStateAnnotationKey = "machineconfiguration.openshift.io/state"
+	// MachineConfigDaemonStateBootstrap is the initial state of a system on boot
+	MachineConfigDaemonStateBootstrap = "Bootstrap"
 	// MachineConfigDaemonStateWorking is set by daemon when it is applying an update.
 	MachineConfigDaemonStateWorking = "Working"
 	// MachineConfigDaemonStateDone is set by daemon when it is done applying an update.

pkg/daemon/daemon.go

@@ -357,21 +357,25 @@ func (dn *Daemon) EnterDegradedState(err error) {
 //
 // If any of the object names are the same, they will be pointer-equal.
 type stateAndConfigs struct {
+	bootstrapping bool
 	state         string
 	currentConfig *mcfgv1.MachineConfig
 	pendingConfig *mcfgv1.MachineConfig
 	desiredConfig *mcfgv1.MachineConfig
 }
 
 func (dn *Daemon) getStateAndConfigs(pendingConfigName string) (*stateAndConfigs, error) {
-	state, err := getNodeAnnotationExt(dn.kubeClient.CoreV1().Nodes(), dn.name, MachineConfigDaemonStateAnnotationKey, true)
+	_, err := os.Lstat(InitialNodeAnnotationsFilePath)
+	bootstrapping := false
 	if err != nil {
+		if os.IsNotExist(err) {
+			// The node annotation file (laid down by the MCS)
+			// doesn't exist, we must not be bootstrapping
+		}
 		return nil, err
-	}
-	// Temporary hack: the MCS used to not write the state=done annotation
-	// key.  If it's unset, let's write it now.
-	if state == "" {
-		state = MachineConfigDaemonStateDone
+	} else {
+		bootstrapping = true
+		glog.Infof("In bootstrap mode")
 	}
 
 	currentConfigName, err := getNodeAnnotation(dn.kubeClient.CoreV1().Nodes(), dn.name, CurrentMachineConfigAnnotationKey)
@@ -386,6 +390,16 @@ func (dn *Daemon) getStateAndConfigs(pendingConfigName string) (*stateAndConfigs
 	if err != nil {
 		return nil, err
 	}
+	state, err := getNodeAnnotationExt(dn.kubeClient.CoreV1().Nodes(), dn.name, MachineConfigDaemonStateAnnotationKey, true)
+	if err != nil {
+		return nil, err
+	}
+	// Temporary hack: the MCS used to not write the state=done annotation
+	// key.  If it's unset, let's write it now.
+	if state == "" {
+		state = MachineConfigDaemonStateDone
+	}
+
 	var desiredConfig *mcfgv1.MachineConfig
 	if currentConfigName == desiredConfigName {
 		desiredConfig = currentConfig
@@ -415,6 +429,7 @@ func (dn *Daemon) getStateAndConfigs(pendingConfigName string) (*stateAndConfigs
 	}
 
 	return &stateAndConfigs{
+		bootstrapping: bootstrapping,
 		currentConfig: currentConfig,
 		pendingConfig: pendingConfig,
 		desiredConfig: desiredConfig,
@@ -540,6 +555,26 @@ func (dn *Daemon) CheckStateOnBoot() error {
 		select {}
 	}
 
+	if state.bootstrapping {
+		if !dn.checkOS(state.currentConfig.Spec.OSImageURL) {
+			glog.Infof("Bootstrap pivot required")
+			// This only returns on error
+			return dn.updateOSAndReboot(state.currentConfig)
+		} else {
+			glog.Infof("No bootstrap pivot required; unlinking bootstrap node annotations")
+			// Delete the bootstrap node annotations; the
+			// currentConfig's osImageURL should now be *truth*.
+			// In other words if it drifts somehow, we go degraded.
+			if err := os.Remove(InitialNodeAnnotationsFilePath); err != nil {
+				return errors.Wrapf(err, "Removing initial node annotations file")
+			}
+			// And add the done state annotation
+			if err := dn.nodeWriter.SetUpdateDone(dn.kubeClient.CoreV1().Nodes(), dn.name, state.currentConfig.GetName()); err != nil {
+				return err
+			}
+		}
+	}
+
 	// Validate the on-disk state against what we *expect*.
 	//
 	// In the case where we're booting a node for the first time, or the MCD
@@ -742,10 +777,6 @@ func (dn *Daemon) completeUpdate(desiredConfigName string) error {
 // triggerUpdateWithMachineConfig starts the update. It queries the cluster for
 // the current and desired config if they weren't passed.
 func (dn *Daemon) triggerUpdateWithMachineConfig(currentConfig *mcfgv1.MachineConfig, desiredConfig *mcfgv1.MachineConfig) error {
-	if err := dn.nodeWriter.SetUpdateWorking(dn.kubeClient.CoreV1().Nodes(), dn.name); err != nil {
-		return err
-	}
-
 	if currentConfig == nil {
 		ccAnnotation, err := getNodeAnnotation(dn.kubeClient.CoreV1().Nodes(), dn.name, CurrentMachineConfigAnnotationKey)
 		if err != nil {