diff --git a/lib/resourcemerge/machineconfig.go b/lib/resourcemerge/machineconfig.go index 9bc9e70846..14f0873202 100644 --- a/lib/resourcemerge/machineconfig.go +++ b/lib/resourcemerge/machineconfig.go @@ -45,6 +45,8 @@ func EnsureMachineConfigPool(modified *bool, existing *mcfgv1.MachineConfigPool, func ensureMachineConfigSpec(modified *bool, existing *mcfgv1.MachineConfigSpec, required mcfgv1.MachineConfigSpec) { setStringIfSet(modified, &existing.OSImageURL, required.OSImageURL) + setStringIfSet(modified, &existing.KernelType, required.KernelType) + if !equality.Semantic.DeepEqual(existing.KernelArguments, required.KernelArguments) { *modified = true (*existing).KernelArguments = required.KernelArguments diff --git a/manifests/machineconfig.crd.yaml b/manifests/machineconfig.crd.yaml index 97272ff324..0b0f2c42b5 100644 --- a/manifests/machineconfig.crd.yaml +++ b/manifests/machineconfig.crd.yaml @@ -233,6 +233,9 @@ spec: fips: description: "fips controls FIPS mode" type: boolean + kernelType: + description: "contains which kernel we want to be running like default (traditional), realtime" + type: string osImageURL: description: "osImageURL specifies the remote location that will be used to fetch the OS" type: string diff --git a/pkg/apis/machineconfiguration.openshift.io/v1/helpers.go b/pkg/apis/machineconfiguration.openshift.io/v1/helpers.go index dc207c495a..dbab8ec7f9 100644 --- a/pkg/apis/machineconfiguration.openshift.io/v1/helpers.go +++ b/pkg/apis/machineconfiguration.openshift.io/v1/helpers.go @@ -21,6 +21,7 @@ func MergeMachineConfigs(configs []*MachineConfig, osImageURL string) *MachineCo sort.Slice(configs, func(i, j int) bool { return configs[i].Name < configs[j].Name }) var fips bool + var kernelType string outIgn := configs[0].Spec.Config for idx := 1; idx < len(configs); idx++ { // if any of the config has FIPS enabled, it'll be set @@ -29,6 +30,23 @@ func MergeMachineConfigs(configs []*MachineConfig, osImageURL string) *MachineCo } outIgn = ign.Append(outIgn, configs[idx].Spec.Config) } + + // sets the KernelType if specified in any of the MachineConfig + // Setting kerneType to realtime in any of MachineConfig takes priority + for _, cfg := range configs { + if cfg.Spec.KernelType == "realtime" { + kernelType = cfg.Spec.KernelType + break + } else if kernelType == "default" { + kernelType = cfg.Spec.KernelType + } + } + + // If no MC sets kerneType, then set it to 'default' since that's what it is using + if kernelType == "" { + kernelType = "default" + } + kargs := []string{} for _, cfg := range configs { kargs = append(kargs, cfg.Spec.KernelArguments...) @@ -40,6 +58,7 @@ func MergeMachineConfigs(configs []*MachineConfig, osImageURL string) *MachineCo KernelArguments: kargs, Config: outIgn, FIPS: fips, + KernelType: kernelType, }, } } diff --git a/pkg/apis/machineconfiguration.openshift.io/v1/types.go b/pkg/apis/machineconfiguration.openshift.io/v1/types.go index 3a35ced5c5..9976cc3f4c 100644 --- a/pkg/apis/machineconfiguration.openshift.io/v1/types.go +++ b/pkg/apis/machineconfiguration.openshift.io/v1/types.go @@ -188,7 +188,8 @@ type MachineConfigSpec struct { KernelArguments []string `json:"kernelArguments"` - FIPS bool `json:"fips"` + FIPS bool `json:"fips"` + KernelType string `json:"kernelType"` } // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object diff --git a/pkg/controller/common/helpers.go b/pkg/controller/common/helpers.go index 355656880e..4ffae16f1b 100644 --- a/pkg/controller/common/helpers.go +++ b/pkg/controller/common/helpers.go @@ -7,6 +7,7 @@ import ( igntypes "github.com/coreos/ignition/config/v2_2/types" validate "github.com/coreos/ignition/config/validate" "github.com/golang/glog" + mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" errors "github.com/pkg/errors" ) @@ -41,3 +42,14 @@ func ValidateIgnition(cfg igntypes.Config) error { } return nil } + +// ValidateMachineConfig validates that given MachineConfig Spec is valid. +func ValidateMachineConfig(cfg mcfgv1.MachineConfigSpec) error { + if !(cfg.KernelType == "" || cfg.KernelType == "default" || cfg.KernelType == "realtime") { + return errors.Errorf("kernelType=%s is invalid", cfg.KernelType) + } + if err := ValidateIgnition(cfg.Config); err != nil { + return err + } + return nil +} diff --git a/pkg/controller/render/render_controller.go b/pkg/controller/render/render_controller.go index 2a617ed59f..c229645438 100644 --- a/pkg/controller/render/render_controller.go +++ b/pkg/controller/render/render_controller.go @@ -524,9 +524,9 @@ func (ctrl *Controller) syncGeneratedMachineConfig(pool *mcfgv1.MachineConfigPoo // generateRenderedMachineConfig takes all MCs for a given pool and returns a single rendered MC. For ex master-XXXX or worker-XXXX func generateRenderedMachineConfig(pool *mcfgv1.MachineConfigPool, configs []*mcfgv1.MachineConfig, cconfig *mcfgv1.ControllerConfig) (*mcfgv1.MachineConfig, error) { - // Before merging all MCs for a specific pool, let's make sure each contains a valid Ignition Config + // Before merging all MCs for a specific pool, let's make sure MachineConfigs are valid for _, config := range configs { - if err := ctrlcommon.ValidateIgnition(config.Spec.Config); err != nil { + if err := ctrlcommon.ValidateMachineConfig(config.Spec); err != nil { return nil, err } } diff --git a/pkg/daemon/update.go b/pkg/daemon/update.go index 7f46103acd..1ae35bacca 100644 --- a/pkg/daemon/update.go +++ b/pkg/daemon/update.go @@ -11,6 +11,7 @@ import ( "os/user" "path/filepath" "reflect" + "regexp" "strconv" "strings" "syscall" @@ -23,10 +24,12 @@ import ( mcfgv1 "github.com/openshift/machine-config-operator/pkg/apis/machineconfiguration.openshift.io/v1" ctrlcommon "github.com/openshift/machine-config-operator/pkg/controller/common" "github.com/openshift/machine-config-operator/pkg/daemon/constants" + pivotutils "github.com/openshift/machine-config-operator/pkg/daemon/pivot/utils" errors "github.com/pkg/errors" "github.com/vincent-petithory/dataurl" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/uuid" "k8s.io/apimachinery/pkg/util/wait" ) @@ -41,6 +44,10 @@ const ( coreUserSSHPath = "/home/core/.ssh/" // fipsFile is the file to check if FIPS is enabled fipsFile = "/proc/sys/crypto/fips_enabled" + // Traditional kernel + defaultKernelType = "default" + // Realtime kernel + realtimeKernelType = "realtime" ) func writeFileAtomicallyWithDefaults(fpath string, b []byte) error { @@ -192,6 +199,32 @@ func canonicalizeEmptyMC(config *mcfgv1.MachineConfig) *mcfgv1.MachineConfig { } } +// Returns true if updated packages are available +func rtKernelUpdateAvailable(rpms []os.FileInfo, rtKernelPkg []string) (bool, error) { + for _, pkg := range rtKernelPkg { + var out []byte + var err error + found := false + + if out, err = exec.Command("rpm", "-q", pkg).Output(); err != nil { + wrappedErr := fmt.Errorf("Failed to run rpm -q : %v", err) + return false, wrappedErr + } + searchRpm := strings.TrimSpace(string(out)) + ".rpm" + for _, rpm := range rpms { + if rpm.Name() == searchRpm { + found = true + break + } + } + if !found { + return true, nil + } + } + + return false, nil +} + func (dn *Daemon) compareMachineConfig(oldConfig, newConfig *mcfgv1.MachineConfig) bool { var diff *MachineConfigDiff oldConfig = canonicalizeEmptyMC(oldConfig) @@ -307,6 +340,20 @@ func (dn *Daemon) update(oldConfig, newConfig *mcfgv1.MachineConfig) (retErr err } }() + // Switch to real time kernel + if err := dn.switchKernel(oldConfig, newConfig); err != nil { + return err + } + + defer func() { + if retErr != nil { + if err := dn.switchKernel(newConfig, oldConfig); err != nil { + retErr = errors.Wrapf(retErr, "error rolling back Real time Kernel %v", err) + return + } + } + }() + return dn.updateOSAndReboot(newConfig) } @@ -315,12 +362,21 @@ func (dn *Daemon) update(oldConfig, newConfig *mcfgv1.MachineConfig) (retErr err // and the MCO would just operate on that. For now we're just doing this to get // improved logging. type MachineConfigDiff struct { - osUpdate bool - kargs bool - fips bool - passwd bool - files bool - units bool + osUpdate bool + kargs bool + fips bool + passwd bool + files bool + units bool + kernelType bool +} + +// canonicalizeKernelType returns a valid kernelType. We consider empty("") and default kernelType as same +func canonicalizeKernelType(kernelType string) string { + if kernelType == realtimeKernelType { + return realtimeKernelType + } + return defaultKernelType } // NewMachineConfigDiff compares two MachineConfig objects. @@ -333,12 +389,13 @@ func NewMachineConfigDiff(oldConfig, newConfig *mcfgv1.MachineConfig) *MachineCo kargsEmpty := len(oldConfig.Spec.KernelArguments) == 0 && len(newConfig.Spec.KernelArguments) == 0 return &MachineConfigDiff{ - osUpdate: oldConfig.Spec.OSImageURL != newConfig.Spec.OSImageURL, - kargs: !(kargsEmpty || reflect.DeepEqual(oldConfig.Spec.KernelArguments, newConfig.Spec.KernelArguments)), - fips: oldConfig.Spec.FIPS != newConfig.Spec.FIPS, - passwd: !reflect.DeepEqual(oldIgn.Passwd, newIgn.Passwd), - files: !reflect.DeepEqual(oldIgn.Storage.Files, newIgn.Storage.Files), - units: !reflect.DeepEqual(oldIgn.Systemd.Units, newIgn.Systemd.Units), + osUpdate: oldConfig.Spec.OSImageURL != newConfig.Spec.OSImageURL, + kargs: !(kargsEmpty || reflect.DeepEqual(oldConfig.Spec.KernelArguments, newConfig.Spec.KernelArguments)), + fips: oldConfig.Spec.FIPS != newConfig.Spec.FIPS, + passwd: !reflect.DeepEqual(oldIgn.Passwd, newIgn.Passwd), + files: !reflect.DeepEqual(oldIgn.Storage.Files, newIgn.Storage.Files), + units: !reflect.DeepEqual(oldIgn.Systemd.Units, newIgn.Systemd.Units), + kernelType: canonicalizeKernelType(oldConfig.Spec.KernelType) != canonicalizeKernelType(newConfig.Spec.KernelType), } } @@ -556,6 +613,136 @@ func (dn *Daemon) updateKernelArguments(oldConfig, newConfig *mcfgv1.MachineConf return exec.Command("rpm-ostree", args...).Run() } +// mountOSContainer mounts the container and returns the mountpoint +func (dn *Daemon) mountOSContainer(container string) (mnt, containerName string, err error) { + var authArgs []string + if _, err = os.Stat(kubeletAuthFile); err == nil { + authArgs = append(authArgs, "--authfile", kubeletAuthFile) + } + // Pull the image + args := []string{"pull", "-q"} + args = append(args, authArgs...) + args = append(args, container) + pivotutils.RunExt(false, numRetriesNetCommands, "podman", args...) + + containerName = "mcd-" + string(uuid.NewUUID()) + // `podman mount` wants a container, so let's create a dummy one, but not run it + var cidBuf []byte + cidBuf, err = runGetOut("podman", "create", "--net=none", "--annotation=org.openshift.machineconfigoperator.pivot=true", "--name", containerName, container) + if err != nil { + return + } + + cid := strings.TrimSpace(string(cidBuf)) + // Use the container ID to find its mount point + mntBuf, err := runGetOut("podman", "mount", cid) + if err != nil { + return + } + mnt = strings.TrimSpace(string(mntBuf)) + return +} + +// switchKernel updates kernel on host with the kernelType specified in MachineConfig. +// Right now it supports default (traditional) and realtime kernel +func (dn *Daemon) switchKernel(oldConfig, newConfig *mcfgv1.MachineConfig) error { + // Do nothing if both old and new KernelType are of type default + if canonicalizeKernelType(oldConfig.Spec.KernelType) == defaultKernelType && canonicalizeKernelType(newConfig.Spec.KernelType) == defaultKernelType { + return nil + } + // We support Kernel update only on RHCOS nodes + if dn.OperatingSystem != machineConfigDaemonOSRHCOS { + return fmt.Errorf("Updating kernel on non-RHCOS nodes is not supported") + } + + defaultKernel := []string{"kernel", "kernel-core", "kernel-modules", "kernel-modules-extra"} + rtKernel := []string{"kernel-rt-core", "kernel-rt-modules", "kernel-rt-modules-extra"} + var args []string + + dn.logSystem("Initiating switch from kernel %s to %s", canonicalizeKernelType(oldConfig.Spec.KernelType), canonicalizeKernelType(newConfig.Spec.KernelType)) + + if canonicalizeKernelType(oldConfig.Spec.KernelType) == realtimeKernelType && canonicalizeKernelType(newConfig.Spec.KernelType) == defaultKernelType { + args = []string{"override", "reset"} + args = append(args, defaultKernel...) + rtKernelUninstall := []string{"--uninstall", "kernel-rt-core", "--uninstall", "kernel-rt-modules", "--uninstall", "kernel-rt-modules-extra"} + args = append(args, rtKernelUninstall...) + dn.logSystem("Switching to kernelType=%s, invoking rpm-ostree %+q", newConfig.Spec.KernelType, args) + if err := exec.Command("rpm-ostree", args...).Run(); err != nil { + return fmt.Errorf("Failed to execute rpm-ostree %+q : %v", args, err) + } + return nil + } + + var mnt, containerName string + var err error + if mnt, containerName, err = dn.mountOSContainer(newConfig.Spec.OSImageURL); err != nil { + return err + } + + defer func() { + // Delete container and remove image once we are done with using rpms available in OSContainer + podmanRemove(containerName) + exec.Command("podman", "rmi", newConfig.Spec.OSImageURL).Run() + dn.logSystem("Deleted container and removed OSContainer image") + }() + + // Get kernel-rt packages from OSContainer + rtRegex := regexp.MustCompile("kernel-rt(.*).rpm") + files, err := ioutil.ReadDir(mnt) + if err != nil { + return err + } + + rtKernelRpms := []os.FileInfo{} + for _, file := range files { + if rtRegex.MatchString(file.Name()) { + rtKernelRpms = append(rtKernelRpms, file) + } + } + + if len(rtKernelRpms) == 0 { + // No kernel-rt rpm package found + return fmt.Errorf("No kernel-rt package available in the OSContainer with URL %s", newConfig.Spec.OSImageURL) + } + + if canonicalizeKernelType(oldConfig.Spec.KernelType) == defaultKernelType && canonicalizeKernelType(newConfig.Spec.KernelType) == realtimeKernelType { + // Switch to RT kernel + args = []string{"override", "remove"} + args = append(args, defaultKernel...) + for _, rpm := range rtKernelRpms { + args = append(args, "--install", fmt.Sprintf("%s/%s", mnt, rpm.Name())) + } + + dn.logSystem("Switching to kernelType=%s, invoking rpm-ostree %+q", newConfig.Spec.KernelType, args) + if err := exec.Command("rpm-ostree", args...).Run(); err != nil { + return fmt.Errorf("Failed to execute rpm-ostree %+q : %v", args, err) + } + } + + if canonicalizeKernelType(oldConfig.Spec.KernelType) == realtimeKernelType && canonicalizeKernelType(newConfig.Spec.KernelType) == realtimeKernelType { + if oldConfig.Spec.OSImageURL != newConfig.Spec.OSImageURL { + args = []string{"uninstall"} + args = append(args, rtKernel...) + // Perform kernel-rt package update only if updated packages are available + var updateAvailable bool + if updateAvailable, err = rtKernelUpdateAvailable(rtKernelRpms, rtKernel); err != nil { + return err + } else if !updateAvailable { + return nil + } + for _, rpm := range rtKernelRpms { + args = append(args, "--install", fmt.Sprintf("%s/%s", mnt, rpm.Name())) + } + dn.logSystem("Updating rt-kernel packages on host: %+q", args) + if err := exec.Command("rpm-ostree", args...).Run(); err != nil { + return fmt.Errorf("Failed to execute rpm-ostree %+q : %v", args, err) + } + } + } + + return nil +} + // updateFiles writes files specified by the nodeconfig to disk. it also writes // systemd units. there is no support for multiple filesystems at this point. // diff --git a/pkg/operator/assets/bindata.go b/pkg/operator/assets/bindata.go index 40e93e158e..696e676064 100644 --- a/pkg/operator/assets/bindata.go +++ b/pkg/operator/assets/bindata.go @@ -821,6 +821,9 @@ spec: fips: description: "fips controls FIPS mode" type: boolean + kernelType: + description: "contains which kernel we want to be running like default (traditional), realtime" + type: string osImageURL: description: "osImageURL specifies the remote location that will be used to fetch the OS" type: string