server: Implement "early pivot" before the MCD comes up

install/image-references

@@ -33,4 +33,4 @@ spec:
   - name: machine-os-content
     from:
       kind: DockerImage
-      name: registry.svc.ci.openshift.org/rhcos/maipo@sha256:83a6d461628380f1dcc057ffef4909992f593b72c5aa4db2e01c0b130004afea
+      name: registry.svc.ci.openshift.org/rhcos/maipo@sha256:61dc83d62cfb5054c4c5532bd2478742a0711075ef5151572e63f94babeacc1a

pkg/controller/template/render.go

@@ -238,7 +238,11 @@ func generateMachineConfigForName(config *RenderConfig, role, name, path string)
 		return nil, fmt.Errorf("error transpiling ct config to Ignition config: %v", err)
 	}
 
-	return MachineConfigFromIgnConfig(role, name, ignCfg), nil
+	mcfg := MachineConfigFromIgnConfig(role, name, ignCfg)
+	// And inject the osimageurl here
+	mcfg.Spec.OSImageURL = config.OSImageURL
+
+	return mcfg, nil
 }
 
 const (

pkg/daemon/constants/constants.go

@@ -23,4 +23,8 @@ const (
 	// InitialNodeAnnotationsFilePath defines the path at which it will find the node annotations it needs to set on the node once it comes up for the first time.
 	// The Machine Config Server writes the node annotations to this path.
 	InitialNodeAnnotationsFilePath = "/etc/machine-config-daemon/node-annotations.json"
+
+	// EtcPivotFile is used by the `pivot` command
+	// For more information, see https://github.com/openshift/pivot/pull/25/commits/c77788a35d7ee4058d1410e89e6c7937bca89f6c#diff-04c6e90faac2675aa89e2176d2eec7d8R44
+	EtcPivotFile = "/etc/pivot/image-pullspec"
 )

pkg/daemon/rpm-ostree.go

@@ -13,12 +13,13 @@ import (
 	"github.com/coreos/go-systemd/dbus"
 	"github.com/coreos/go-systemd/sdjournal"
 	"github.com/golang/glog"
+
+	"github.com/openshift/machine-config-operator/pkg/daemon/constants"
 )
 
 const (
 	pivotUnit      = "pivot.service"
 	rpmostreedUnit = "rpm-ostreed.service"
-	etcPivotFile   = "/etc/pivot/image-pullspec"
 )
 
 // RpmOstreeState houses zero or more RpmOstreeDeployments
@@ -99,12 +100,12 @@ func (r *RpmOstreeClient) GetBootedOSImageURL(rootMount string) (string, string,
 // RunPivot executes a pivot from one deployment to another as found in the referenced
 // osImageURL. See https://github.com/openshift/pivot.
 func (r *RpmOstreeClient) RunPivot(osImageURL string) error {
-	if err := os.MkdirAll(filepath.Dir(etcPivotFile), os.FileMode(0755)); err != nil {
-		return fmt.Errorf("error creating leading dirs for %s: %v", etcPivotFile, err)
+	if err := os.MkdirAll(filepath.Dir(constants.EtcPivotFile), os.FileMode(0755)); err != nil {
+		return fmt.Errorf("error creating leading dirs for %s: %v", constants.EtcPivotFile, err)
 	}
 
-	if err := ioutil.WriteFile(etcPivotFile, []byte(osImageURL), 0644); err != nil {
-		return fmt.Errorf("error writing to %s: %v", etcPivotFile, err)
+	if err := ioutil.WriteFile(constants.EtcPivotFile, []byte(osImageURL), 0644); err != nil {
+		return fmt.Errorf("error writing to %s: %v", constants.EtcPivotFile, err)
 	}
 
 	journalStopCh := make(chan time.Time)

pkg/server/bootstrap_server.go

@@ -96,7 +96,7 @@ func (bsc *bootstrapServer) GetConfig(cr poolRequest) (*ignv2_2types.Config, err
 		return nil, fmt.Errorf("server: could not unmarshal file %s, err: %v", fileName, err)
 	}
 
-	appenders := getAppenders(cr, currConf, bsc.kubeconfigFunc)
+	appenders := getAppenders(cr, currConf, bsc.kubeconfigFunc, mc.Spec.OSImageURL)
 	for _, a := range appenders {
 		if err := a(&mc.Spec.Config); err != nil {
 			return nil, err

pkg/server/cluster_server.go

@@ -70,7 +70,7 @@ func (cs *clusterServer) GetConfig(cr poolRequest) (*ignv2_2types.Config, error)
 		return nil, fmt.Errorf("could not fetch config %s, err: %v", currConf, err)
 	}
 
-	appenders := getAppenders(cr, currConf, cs.kubeconfigFunc)
+	appenders := getAppenders(cr, currConf, cs.kubeconfigFunc, mc.Spec.OSImageURL)
 	for _, a := range appenders {
 		if err := a(&mc.Spec.Config); err != nil {
 			return nil, err

pkg/server/server.go

@@ -16,6 +16,9 @@ const (
 	// of the KubeConfig file on the machine.
 	defaultMachineKubeConfPath = "/etc/kubernetes/kubeconfig"
 
+	// From https://github.com/openshift/pivot/pull/25/commits/c77788a35d7ee4058d1410e89e6c7937bca89f6c#diff-04c6e90faac2675aa89e2176d2eec7d8R44
+	pivotRebootNeeded = "/run/pivot/reboot-needed"
+
 	// defaultFileSystem defines the default file system to be
 	// used for writing the ignition files created by the
 	// server.
@@ -34,16 +37,51 @@ type Server interface {
 	GetConfig(poolRequest) (*ignv2_2types.Config, error)
 }
 
-func getAppenders(cr poolRequest, currMachineConfig string, f kubeconfigFunc) []appenderFunc {
+func getAppenders(cr poolRequest, currMachineConfig string, f kubeconfigFunc, osimageurl string) []appenderFunc {
 	appenders := []appenderFunc{
 		// append machine annotations file.
 		func(config *ignv2_2types.Config) error { return appendNodeAnnotations(config, currMachineConfig) },
+		// append pivot
+		func(config *ignv2_2types.Config) error { return appendInitialPivot(config, osimageurl) },
 		// append kubeconfig.
 		func(config *ignv2_2types.Config) error { return appendKubeConfig(config, f) },
 	}
 	return appenders
 }
 
+// Golang :cry:
+func boolToPtr(b bool) *bool {
+	return &b
+}
+
+func appendInitialPivot(conf *ignv2_2types.Config, osimageurl string) error {
+	if osimageurl == "" {
+		return nil
+	}
+
+	// Tell pivot.service to pivot early
+	appendFileToIgnition(conf, daemonconsts.EtcPivotFile, osimageurl + "\n")
+	// Awful hack to create a file in /run
+	// https://github.com/openshift/machine-config-operator/pull/363#issuecomment-463397373
+	// "So one gotcha here is that Ignition will actually write `/run/pivot/image-pullspec` to the filesystem rather than the `/run` tmpfs"
+	if len(conf.Systemd.Units) == 0 {
+		conf.Systemd.Units = make([]ignv2_2types.Unit, 0)
+	}
+	unit := ignv2_2types.Unit {
+		Name: "mcd-write-pivot-reboot.service",
+		Enabled: boolToPtr(true),
+		Contents: `[Unit]
+Before=pivot.service
+ConditionFirstBoot=true
+[Service]
+ExecStart=/bin/sh -c 'mkdir /run/pivot && touch /run/pivot/reboot-needed'
+[Install]
+WantedBy=multi-user.target
+`,}
+	conf.Systemd.Units = append(conf.Systemd.Units, unit)
+	return nil
+}
+
 func appendKubeConfig(conf *ignv2_2types.Config, f kubeconfigFunc) error {
 	kcData, _, err := f()
 	if err != nil {

test/e2e/osimageurl_test.go

@@ -0,0 +1,46 @@
+package e2e_test
+
+import (
+	"testing"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/openshift/machine-config-operator/cmd/common"
+)
+
+func TestOSImageURL(t *testing.T) {
+	cb, err := common.NewClientBuilder("")
+	if err != nil {
+		t.Fatalf("%#v", err)
+	}
+	mcClient := cb.MachineConfigClientOrDie("mc-file-add")
+
+	// grab the latest worker- MC
+	mcp, err := mcClient.MachineconfigurationV1().MachineConfigPools().Get("worker", metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("%#v", err)
+	}
+
+	mc, err := mcClient.MachineconfigurationV1().MachineConfigs().Get(mcp.Status.Configuration.Name, metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("%#v", err)
+	}
+
+	if mc.Spec.OSImageURL == "" {
+		t.Fatalf("Empty OSImageURL for %s", mc.Name)
+	}
+
+	// grab the latest master- MC
+	mcp, err = mcClient.MachineconfigurationV1().MachineConfigPools().Get("master", metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("%#v", err)
+	}
+	mc, err = mcClient.MachineconfigurationV1().MachineConfigs().Get(mcp.Status.Configuration.Name, metav1.GetOptions{})
+	if err != nil {
+		t.Fatalf("%#v", err)
+	}
+
+	if mc.Spec.OSImageURL == "" {
+		t.Fatalf("Empty OSImageURL for %s", mc.Name)
+	}
+}