diff --git a/docs/MachineConfigServer.md b/docs/MachineConfigServer.md index 583442b295..4c54043aa8 100644 --- a/docs/MachineConfigServer.md +++ b/docs/MachineConfigServer.md @@ -36,6 +36,10 @@ It performs the following extra actions on the Ignition config defined in the Ma The new machines that come up, will need a KubeConfig file which will be added as an Ignition file. +* *Early pivot* - `/etc/rhcos-initial-pivot-target` + + While the Ignition content configures the node, it may actually be booted into an older OS image than is specified by the release payload managed by the [Cluster Version Operator](https://github.com/openshift/cluster-version-operator/). The MCS writes out the `osImageURL` to this file, and the system will (if necessary) performs an "early pivot" before the node has actually joined the cluster. This then ensures that when the MachineConfigDaemon starts, it will validate the `currentConfig` (including files written by Ignition and the `osImageURL`). + ### Running MachineConfigServer It is recommended that the MachineConfigServer is run as a DaemonSet on all `master` machines with the pods running in host network. So machines can access the Ignition endpoint through load balancer setup for control plane. diff --git a/pkg/controller/template/render.go b/pkg/controller/template/render.go index 05758bbae6..e7f3c5dabf 100644 --- a/pkg/controller/template/render.go +++ b/pkg/controller/template/render.go @@ -91,21 +91,16 @@ func generateMachineConfigs(config *RenderConfig, templateDir string) ([]*mcfgv1 return cfgs, nil } +// GenerateMachineConfigsForRole is part of generateMachineConfigs; it operates +// on a specific role which has a set of builtin templates. func GenerateMachineConfigsForRole(config *RenderConfig, role string, path string) ([]*mcfgv1.MachineConfig, error) { + cfgs := []*mcfgv1.MachineConfig{} + + // Add our built-in templates infos, err := ioutil.ReadDir(path) if err != nil { return nil, fmt.Errorf("failed to read dir %q: %v", path, err) } - // for each role a machine config is created containing the sshauthorized keys to allow for ssh access - // ex: role = worker -> machine config "00-worker-ssh" created containing user core and ssh key - var tempIgnConfig ignv2_2types.Config - tempUser := ignv2_2types.PasswdUser{Name: "core", SSHAuthorizedKeys: []ignv2_2types.SSHAuthorizedKey{ignv2_2types.SSHAuthorizedKey(config.SSHKey)}} - tempIgnConfig.Passwd.Users = append(tempIgnConfig.Passwd.Users, tempUser) - sshConfigName := "00-" + role + "-ssh" - sshMachineConfigForRole := MachineConfigFromIgnConfig(role, sshConfigName, &tempIgnConfig) - - cfgs := []*mcfgv1.MachineConfig{} - cfgs = append(cfgs, sshMachineConfigForRole) for _, info := range infos { if !info.IsDir() { @@ -121,9 +116,57 @@ func GenerateMachineConfigsForRole(config *RenderConfig, role string, path strin cfgs = append(cfgs, nameConfig) } + // And derived configs + derivedCfgs, err := generateDerivedMachineConfigs(config, role) + if err != nil { + return nil, err + } + cfgs = append(cfgs, derivedCfgs...) + + return cfgs, nil +} + +// machineConfigForOSImageURL generates a MC fragment that just includes the target OSImageURL. +func machineConfigForOSImageURL(role string, url string) *mcfgv1.MachineConfig { + labels := map[string]string{ + machineConfigRoleLabelKey: role, + } + return &mcfgv1.MachineConfig{ + ObjectMeta: metav1.ObjectMeta{ + Labels: labels, + Name: "00-" + role + "-osimageurl", + }, + Spec: mcfgv1.MachineConfigSpec{ + OSImageURL: url, + }, + } +} + +// generateDerivedMachineConfigs is part of generateMachineConfigsForRole. It +// takes care of generating MachineConfig objects which are derived from other +// components of the cluster configuration. Currently, that's: +// +// - SSH keys from the install configuration +// - OSImageURL from the machine-config-osimageurl configmap (which comes from the CVO) +func generateDerivedMachineConfigs(config *RenderConfig, role string) ([]*mcfgv1.MachineConfig, error) { + cfgs := []*mcfgv1.MachineConfig{} + + // for each role a machine config is created containing the sshauthorized keys to allow for ssh access + // ex: role = worker -> machine config "00-worker-ssh" created containing user core and ssh key + var tempIgnConfig ignv2_2types.Config + tempUser := ignv2_2types.PasswdUser{Name: "core", SSHAuthorizedKeys: []ignv2_2types.SSHAuthorizedKey{ignv2_2types.SSHAuthorizedKey(config.SSHKey)}} + tempIgnConfig.Passwd.Users = append(tempIgnConfig.Passwd.Users, tempUser) + sshConfigName := "00-" + role + "-ssh" + cfgs = append(cfgs, MachineConfigFromIgnConfig(role, sshConfigName, &tempIgnConfig)) + + if config.OSImageURL != "" { + cfgs = append(cfgs, machineConfigForOSImageURL(role, config.OSImageURL)) + } + return cfgs, nil } +// generateMachineConfigForName is part of the implementation of generateMachineConfigsForRole func generateMachineConfigForName(config *RenderConfig, role, name, path string) (*mcfgv1.MachineConfig, error) { platformDirs := []string{} for _, dir := range []string{"_base", config.Platform} { diff --git a/pkg/controller/template/test_data/templates/master/00-master/aws/units/rhcos-initial-pivot.service b/pkg/controller/template/test_data/templates/master/00-master/aws/units/rhcos-initial-pivot.service new file mode 100644 index 0000000000..bfe687cc45 --- /dev/null +++ b/pkg/controller/template/test_data/templates/master/00-master/aws/units/rhcos-initial-pivot.service @@ -0,0 +1,19 @@ +contents: | + [Unit] + Description=RHCOS initial pivot + # Before we join the cluster + Before=kubelet.service + # But be sure that we've finished Ignition before we reboot + After=ignition-firstboot-complete.service + ConditionPathExists=/etc/rhcos-initial-pivot-target + + [Service] + Type=simple + ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target' + Restart=on-failure + RestartSec=30 + + [Install] + WantedBy=multi-user.target +enabled: true +name: rhcos-initial-pivot.service diff --git a/pkg/controller/template/test_data/templates/master/00-master/libvirt/units/rhcos-initial-pivot.service b/pkg/controller/template/test_data/templates/master/00-master/libvirt/units/rhcos-initial-pivot.service new file mode 100644 index 0000000000..bfe687cc45 --- /dev/null +++ b/pkg/controller/template/test_data/templates/master/00-master/libvirt/units/rhcos-initial-pivot.service @@ -0,0 +1,19 @@ +contents: | + [Unit] + Description=RHCOS initial pivot + # Before we join the cluster + Before=kubelet.service + # But be sure that we've finished Ignition before we reboot + After=ignition-firstboot-complete.service + ConditionPathExists=/etc/rhcos-initial-pivot-target + + [Service] + Type=simple + ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target' + Restart=on-failure + RestartSec=30 + + [Install] + WantedBy=multi-user.target +enabled: true +name: rhcos-initial-pivot.service diff --git a/pkg/controller/template/test_data/templates/master/00-master/none/units/rhcos-initial-pivot.service b/pkg/controller/template/test_data/templates/master/00-master/none/units/rhcos-initial-pivot.service new file mode 100644 index 0000000000..bfe687cc45 --- /dev/null +++ b/pkg/controller/template/test_data/templates/master/00-master/none/units/rhcos-initial-pivot.service @@ -0,0 +1,19 @@ +contents: | + [Unit] + Description=RHCOS initial pivot + # Before we join the cluster + Before=kubelet.service + # But be sure that we've finished Ignition before we reboot + After=ignition-firstboot-complete.service + ConditionPathExists=/etc/rhcos-initial-pivot-target + + [Service] + Type=simple + ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target' + Restart=on-failure + RestartSec=30 + + [Install] + WantedBy=multi-user.target +enabled: true +name: rhcos-initial-pivot.service diff --git a/pkg/controller/template/test_data/templates/master/00-master/openstack/units/rhcos-initial-pivot.service b/pkg/controller/template/test_data/templates/master/00-master/openstack/units/rhcos-initial-pivot.service new file mode 100644 index 0000000000..bfe687cc45 --- /dev/null +++ b/pkg/controller/template/test_data/templates/master/00-master/openstack/units/rhcos-initial-pivot.service @@ -0,0 +1,19 @@ +contents: | + [Unit] + Description=RHCOS initial pivot + # Before we join the cluster + Before=kubelet.service + # But be sure that we've finished Ignition before we reboot + After=ignition-firstboot-complete.service + ConditionPathExists=/etc/rhcos-initial-pivot-target + + [Service] + Type=simple + ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target' + Restart=on-failure + RestartSec=30 + + [Install] + WantedBy=multi-user.target +enabled: true +name: rhcos-initial-pivot.service diff --git a/pkg/controller/template/test_data/templates/worker/00-worker/aws/units/rhcos-initial-pivot.service b/pkg/controller/template/test_data/templates/worker/00-worker/aws/units/rhcos-initial-pivot.service new file mode 100644 index 0000000000..bfe687cc45 --- /dev/null +++ b/pkg/controller/template/test_data/templates/worker/00-worker/aws/units/rhcos-initial-pivot.service @@ -0,0 +1,19 @@ +contents: | + [Unit] + Description=RHCOS initial pivot + # Before we join the cluster + Before=kubelet.service + # But be sure that we've finished Ignition before we reboot + After=ignition-firstboot-complete.service + ConditionPathExists=/etc/rhcos-initial-pivot-target + + [Service] + Type=simple + ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target' + Restart=on-failure + RestartSec=30 + + [Install] + WantedBy=multi-user.target +enabled: true +name: rhcos-initial-pivot.service diff --git a/pkg/controller/template/test_data/templates/worker/00-worker/libvirt/units/rhcos-initial-pivot.service b/pkg/controller/template/test_data/templates/worker/00-worker/libvirt/units/rhcos-initial-pivot.service new file mode 100644 index 0000000000..bfe687cc45 --- /dev/null +++ b/pkg/controller/template/test_data/templates/worker/00-worker/libvirt/units/rhcos-initial-pivot.service @@ -0,0 +1,19 @@ +contents: | + [Unit] + Description=RHCOS initial pivot + # Before we join the cluster + Before=kubelet.service + # But be sure that we've finished Ignition before we reboot + After=ignition-firstboot-complete.service + ConditionPathExists=/etc/rhcos-initial-pivot-target + + [Service] + Type=simple + ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target' + Restart=on-failure + RestartSec=30 + + [Install] + WantedBy=multi-user.target +enabled: true +name: rhcos-initial-pivot.service diff --git a/pkg/controller/template/test_data/templates/worker/00-worker/none/units/rhcos-initial-pivot.service b/pkg/controller/template/test_data/templates/worker/00-worker/none/units/rhcos-initial-pivot.service new file mode 100644 index 0000000000..bfe687cc45 --- /dev/null +++ b/pkg/controller/template/test_data/templates/worker/00-worker/none/units/rhcos-initial-pivot.service @@ -0,0 +1,19 @@ +contents: | + [Unit] + Description=RHCOS initial pivot + # Before we join the cluster + Before=kubelet.service + # But be sure that we've finished Ignition before we reboot + After=ignition-firstboot-complete.service + ConditionPathExists=/etc/rhcos-initial-pivot-target + + [Service] + Type=simple + ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target' + Restart=on-failure + RestartSec=30 + + [Install] + WantedBy=multi-user.target +enabled: true +name: rhcos-initial-pivot.service diff --git a/pkg/controller/template/test_data/templates/worker/00-worker/openstack/units/rhcos-initial-pivot.service b/pkg/controller/template/test_data/templates/worker/00-worker/openstack/units/rhcos-initial-pivot.service new file mode 100644 index 0000000000..bfe687cc45 --- /dev/null +++ b/pkg/controller/template/test_data/templates/worker/00-worker/openstack/units/rhcos-initial-pivot.service @@ -0,0 +1,19 @@ +contents: | + [Unit] + Description=RHCOS initial pivot + # Before we join the cluster + Before=kubelet.service + # But be sure that we've finished Ignition before we reboot + After=ignition-firstboot-complete.service + ConditionPathExists=/etc/rhcos-initial-pivot-target + + [Service] + Type=simple + ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target' + Restart=on-failure + RestartSec=30 + + [Install] + WantedBy=multi-user.target +enabled: true +name: rhcos-initial-pivot.service diff --git a/pkg/server/bootstrap_server.go b/pkg/server/bootstrap_server.go index a163fddd20..fd5116a946 100644 --- a/pkg/server/bootstrap_server.go +++ b/pkg/server/bootstrap_server.go @@ -96,7 +96,7 @@ func (bsc *bootstrapServer) GetConfig(cr poolRequest) (*ignv2_2types.Config, err return nil, fmt.Errorf("server: could not unmarshal file %s, err: %v", fileName, err) } - appenders := getAppenders(cr, currConf, bsc.kubeconfigFunc) + appenders := getAppenders(cr, currConf, bsc.kubeconfigFunc, "") for _, a := range appenders { if err := a(&mc.Spec.Config); err != nil { return nil, err diff --git a/pkg/server/cluster_server.go b/pkg/server/cluster_server.go index 82a19b0197..76c9692afa 100644 --- a/pkg/server/cluster_server.go +++ b/pkg/server/cluster_server.go @@ -70,7 +70,7 @@ func (cs *clusterServer) GetConfig(cr poolRequest) (*ignv2_2types.Config, error) return nil, fmt.Errorf("could not fetch config %s, err: %v", currConf, err) } - appenders := getAppenders(cr, currConf, cs.kubeconfigFunc) + appenders := getAppenders(cr, currConf, cs.kubeconfigFunc, mc.Spec.OSImageURL) for _, a := range appenders { if err := a(&mc.Spec.Config); err != nil { return nil, err diff --git a/pkg/server/server.go b/pkg/server/server.go index 4a8ee75b45..0f266cc299 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -16,6 +16,9 @@ const ( // of the KubeConfig file on the machine. defaultMachineKubeConfPath = "/etc/kubernetes/kubeconfig" + // rhcosInitialPivotPath is processed by rhcos-initial-pivot.service + rhcosInitialPivotPath = "/etc/rhcos-initial-pivot-target" + // defaultFileSystem defines the default file system to be // used for writing the ignition files created by the // server. @@ -34,10 +37,11 @@ type Server interface { GetConfig(poolRequest) (*ignv2_2types.Config, error) } -func getAppenders(cr poolRequest, currMachineConfig string, f kubeconfigFunc) []appenderFunc { +func getAppenders(cr poolRequest, currMachineConfig string, f kubeconfigFunc, osimageurl string) []appenderFunc { appenders := []appenderFunc{ // append machine annotations file. func(config *ignv2_2types.Config) error { return appendNodeAnnotations(config, currMachineConfig) }, + func(config *ignv2_2types.Config) error { return appendInitialPivot(config, osimageurl) }, // append kubeconfig. func(config *ignv2_2types.Config) error { return appendKubeConfig(config, f) }, } @@ -53,6 +57,15 @@ func appendKubeConfig(conf *ignv2_2types.Config, f kubeconfigFunc) error { return nil } +// Ensures that the node is in the OS we expect; for more information see +// rhcos-initial-pivot.service in the templates +func appendInitialPivot(conf *ignv2_2types.Config, osimageurl string) error { + if osimageurl != "" { + appendFileToIgnition(conf, rhcosInitialPivotPath, osimageurl + "\n") + } + return nil +} + func appendNodeAnnotations(conf *ignv2_2types.Config, currConf string) error { anno, err := getNodeAnnotation(currConf) if err != nil { diff --git a/templates/master/00-master/_base/units/rhcos-initial-pivot.yaml b/templates/master/00-master/_base/units/rhcos-initial-pivot.yaml new file mode 100644 index 0000000000..0f74e174b6 --- /dev/null +++ b/templates/master/00-master/_base/units/rhcos-initial-pivot.yaml @@ -0,0 +1,19 @@ +name: "rhcos-initial-pivot.service" +enabled: true +contents: | + [Unit] + Description=RHCOS initial pivot + # Before we join the cluster + Before=kubelet.service + # But be sure that we've finished Ignition before we reboot + After=ignition-firstboot-complete.service + ConditionPathExists=/etc/rhcos-initial-pivot-target + + [Service] + Type=simple + ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target' + Restart=on-failure + RestartSec=30 + + [Install] + WantedBy=multi-user.target diff --git a/templates/worker/00-worker/_base/units/rhcos-initial-pivot.yaml b/templates/worker/00-worker/_base/units/rhcos-initial-pivot.yaml new file mode 100644 index 0000000000..0f74e174b6 --- /dev/null +++ b/templates/worker/00-worker/_base/units/rhcos-initial-pivot.yaml @@ -0,0 +1,19 @@ +name: "rhcos-initial-pivot.service" +enabled: true +contents: | + [Unit] + Description=RHCOS initial pivot + # Before we join the cluster + Before=kubelet.service + # But be sure that we've finished Ignition before we reboot + After=ignition-firstboot-complete.service + ConditionPathExists=/etc/rhcos-initial-pivot-target + + [Service] + Type=simple + ExecStart=/usr/bin/sh -c 'pivot --reboot $(cat /etc/rhcos-initial-pivot-target) && rm /etc/rhcos-initial-pivot-target' + Restart=on-failure + RestartSec=30 + + [Install] + WantedBy=multi-user.target diff --git a/test/e2e/main_test.go b/test/e2e/main_test.go index 046fa5a1ab..961f27a3cf 100644 --- a/test/e2e/main_test.go +++ b/test/e2e/main_test.go @@ -5,6 +5,10 @@ import ( "testing" ) +const ( + namespace = "openshift-machine-config-operator" +) + func TestMain(m *testing.M) { os.Exit(m.Run()) } diff --git a/test/e2e/sanity_test.go b/test/e2e/sanity_test.go index 794e377ce8..9dc70f6d4c 100644 --- a/test/e2e/sanity_test.go +++ b/test/e2e/sanity_test.go @@ -2,10 +2,22 @@ package e2e_test import ( "testing" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/cache" + "k8s.io/client-go/informers" + "k8s.io/apimachinery/pkg/labels" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" "github.com/openshift/machine-config-operator/cmd/common" + "github.com/openshift/machine-config-operator/pkg/daemon" +) + + +var ( + controllerKind = mcfgv1.SchemeGroupVersion.WithKind("MachineConfigPool") ) // Test case for https://github.com/openshift/machine-config-operator/pull/288/commits/44d5c5215b5450fca32806f796b50a3372daddc2 @@ -16,7 +28,7 @@ func TestOperatorLabel(t *testing.T) { } k := cb.KubeClientOrDie("sanity-test") - d, err := k.AppsV1().DaemonSets("openshift-machine-config-operator").Get("machine-config-daemon", metav1.GetOptions{}) + d, err := k.AppsV1().DaemonSets(namespace).Get("machine-config-daemon", metav1.GetOptions{}) if err != nil { t.Errorf("%#v", err) } @@ -26,3 +38,96 @@ func TestOperatorLabel(t *testing.T) { t.Errorf("Expected node selector 'linux', not '%s'", osSelector) } } + +func TestNoDegraded(t *testing.T) { + cb, err := common.NewClientBuilder("") + if err != nil{ + t.Errorf("%#v", err) + } + k := cb.KubeClientOrDie("sanity-test") + + kubeSharedInformer := informers.NewSharedInformerFactory(k, 20 * time.Minute) + nodeInformer := kubeSharedInformer.Core().V1().Nodes() + nodeLister := nodeInformer.Lister() + nodeListerSynced := nodeInformer.Informer().HasSynced + + stopCh := make(chan struct{}) + kubeSharedInformer.Start(stopCh) + cache.WaitForCacheSync(stopCh, nodeListerSynced) + + nodes, err := nodeLister.List(labels.Everything()) + if err != nil { + t.Errorf("listing nodes: %v", err) + } + + var degraded []*corev1.Node + for _, node := range nodes { + if node.Annotations == nil { + continue + } + dstate, ok := node.Annotations[daemon.MachineConfigDaemonStateAnnotationKey] + if !ok || dstate == "" { + continue + } + + if dstate == daemon.MachineConfigDaemonStateDegraded { + degraded = append(degraded, node) + } + } + + if len(degraded) > 0 { + t.Errorf("%d degraded nodes found", len(degraded)) + } +} + +// func getRenderedMachineConfigs(mcfgs []mcfgv1.MachineConfig) ([]*mcfgv1.MachineConfig, []*mcfgv1.MachineConfig) { +// var masters []*mcfgv1.MachineConfig +// var workers []*mcfgv1.MachineConfig +// for _, mcfg := range mcfgs { +// if controllerRef := metav1.GetControllerOf(&mcfg); controllerRef != nil { +// if controllerRef.Kind != controllerKind.Kind { +// continue +// } +// if strings.HasPrefix(mcfg.Name, "master-") { +// masters = append(masters, &mcfg) +// } else if strings.HasPrefix(mcfg.Name, "worker-") { +// workers = append(workers, &mcfg) +// } +// } +// } +// return masters, workers +// } + +func TestMachineConfigsOSImageURL(t *testing.T) { + cb, err := common.NewClientBuilder("") + if err != nil{ + t.Fatalf("%#v", err) + } + mcClient := cb.MachineConfigClientOrDie("mc-test") + + masterMCP, err := mcClient.MachineconfigurationV1().MachineConfigPools().Get("master", metav1.GetOptions{}) + if err != nil { + t.Fatalf("Getting master MCP: %v", err) + } + workerMCP, err := mcClient.MachineconfigurationV1().MachineConfigPools().Get("worker", metav1.GetOptions{}) + if err != nil { + t.Fatalf("Getting worker MCP: %v", err) + } + + masterMC, err := mcClient.MachineconfigurationV1().MachineConfigs().Get(masterMCP.Status.Configuration.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Getting MC: %v", err) + } + if masterMC.Spec.OSImageURL == "" { + t.Fatalf("master has no OSImageURL") + } + + workerMC, err := mcClient.MachineconfigurationV1().MachineConfigs().Get(workerMCP.Status.Configuration.Name, metav1.GetOptions{}) + if err != nil { + t.Fatalf("Getting MC: %v", err) + } + if workerMC.Spec.OSImageURL == "" { + t.Fatalf("master has no OSImageURL") + } +} +