diff --git a/pkg/controller/common/helpers.go b/pkg/controller/common/helpers.go index 0ae97b7dae..9a0058c1f0 100644 --- a/pkg/controller/common/helpers.go +++ b/pkg/controller/common/helpers.go @@ -54,7 +54,7 @@ func strToPtr(s string) *string { // It sorts all the configs in increasing order of their name. // It uses the Ignition config from first object as base and appends all the rest. // Kernel arguments are concatenated. -// It uses only the OSImageURL provided by the CVO and ignores any MC provided OSImageURL. +// It defaults to the OSImageURL provided by the CVO but allows a MC provided OSImageURL to take precedence. func MergeMachineConfigs(configs []*mcfgv1.MachineConfig, osImageURL string) (*mcfgv1.MachineConfig, error) { if len(configs) == 0 { return nil, nil @@ -125,6 +125,18 @@ func MergeMachineConfigs(configs []*mcfgv1.MachineConfig, osImageURL string) (*m } } + // For layering, we want to let the user override OSImageURL again + overriddenOSImageURL := "" + for _, cfg := range configs { + if cfg.Spec.OSImageURL != "" { + overriddenOSImageURL = cfg.Spec.OSImageURL + } + } + // Make sure it's obvious in the logs that it was overridden + if overriddenOSImageURL != "" && overriddenOSImageURL != osImageURL { + osImageURL = overriddenOSImageURL + } + return &mcfgv1.MachineConfig{ Spec: mcfgv1.MachineConfigSpec{ OSImageURL: osImageURL, diff --git a/pkg/controller/common/helpers_test.go b/pkg/controller/common/helpers_test.go index dd97aa4df2..b876a33d8f 100644 --- a/pkg/controller/common/helpers_test.go +++ b/pkg/controller/common/helpers_test.go @@ -273,11 +273,12 @@ func TestMergeMachineConfigs(t *testing.T) { // Test that all other configs can also be set properly - // osImageURL should be set from the passed variable, make sure that - // setting it via a MachineConfig doesn't do anything + // we previously prevented OSImageURL from being overridden via + // machineconfig, but now that we're doing layering, we want to + // give that functionality back, so make sure we can override it machineConfigOSImageURL := &mcfgv1.MachineConfig{ Spec: mcfgv1.MachineConfigSpec{ - OSImageURL: "badURL", + OSImageURL: "overriddenURL", }, } machineConfigKernelArgs := &mcfgv1.MachineConfig{ @@ -328,7 +329,7 @@ func TestMergeMachineConfigs(t *testing.T) { expectedMachineConfig = &mcfgv1.MachineConfig{ Spec: mcfgv1.MachineConfigSpec{ - OSImageURL: osImageURL, + OSImageURL: "overriddenURL", KernelArguments: kargs, Config: runtime.RawExtension{ Raw: rawOutIgn, diff --git a/pkg/controller/render/render_controller.go b/pkg/controller/render/render_controller.go index 4fa5d6dc50..dd836b4a50 100644 --- a/pkg/controller/render/render_controller.go +++ b/pkg/controller/render/render_controller.go @@ -487,6 +487,11 @@ func (ctrl *Controller) syncGeneratedMachineConfig(pool *mcfgv1.MachineConfigPoo return err } + // Emit an event so it's more visible that OSImageURL was overridden. + if generated.Spec.OSImageURL != cc.Spec.OSImageURL { + ctrl.eventRecorder.Eventf(generated, corev1.EventTypeNormal, "OSImageURLOverridden", "OSImageURL was overridden via machineconfig in %s (was: %s is: %s)", generated.Name, cc.Spec.OSImageURL, generated.Spec.OSImageURL) + } + source := []corev1.ObjectReference{} for _, cfg := range configs { source = append(source, corev1.ObjectReference{Kind: machineconfigKind.Kind, Name: cfg.GetName(), APIVersion: machineconfigKind.GroupVersion().String()}) @@ -570,6 +575,12 @@ func generateRenderedMachineConfig(pool *mcfgv1.MachineConfigPool, configs []*mc merged.Annotations[ctrlcommon.GeneratedByControllerVersionAnnotationKey] = version.Hash merged.Annotations[ctrlcommon.ReleaseImageVersionAnnotationKey] = cconfig.Annotations[ctrlcommon.ReleaseImageVersionAnnotationKey] + // Make it obvious that the OSImageURL has been overridden. If we log this in MergeMachineConfigs, we don't know the name yet, so we're + // logging out here instead so it's actually helpful. + if merged.Spec.OSImageURL != cconfig.Spec.OSImageURL { + glog.Infof("OSImageURL has been overridden via machineconfig in %s (was: %s is: %s)", merged.Name, cconfig.Spec.OSImageURL, merged.Spec.OSImageURL) + } + return merged, nil } diff --git a/pkg/controller/render/render_controller_test.go b/pkg/controller/render/render_controller_test.go index 6fa13b3a87..c8fd5c5f7f 100644 --- a/pkg/controller/render/render_controller_test.go +++ b/pkg/controller/render/render_controller_test.go @@ -370,7 +370,7 @@ func TestUpdatesGeneratedMachineConfig(t *testing.T) { f.run(getKey(mcp, t)) } -func TestGenerateMachineConfigNoOverrideOSImageURL(t *testing.T) { +func TestGenerateMachineConfigOverrideOSImageURL(t *testing.T) { mcp := helpers.NewMachineConfigPool("test-cluster-master", helpers.MasterSelector, nil, "") mcs := []*mcfgv1.MachineConfig{ helpers.NewMachineConfig("00-test-cluster-master", map[string]string{"node-role/master": ""}, "dummy-test-1", []ign3types.File{}), @@ -383,7 +383,7 @@ func TestGenerateMachineConfigNoOverrideOSImageURL(t *testing.T) { if err != nil { t.Fatal(err) } - assert.Equal(t, "dummy", gmc.Spec.OSImageURL) + assert.Equal(t, "dummy-change", gmc.Spec.OSImageURL) } func TestVersionSkew(t *testing.T) { diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index d5708ac77c..7b09fb1992 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -1410,16 +1410,30 @@ func (dn *Daemon) checkStateOnFirstRun() error { osMatch := dn.checkOS(targetOSImageURL) if !osMatch { glog.Infof("Bootstrap pivot required to: %s", targetOSImageURL) - // This only returns on error - osImageContentDir, err := ExtractOSImage(targetOSImageURL) + + // Check to see if we have a layered/new format image + isLayeredImage, err := dn.NodeUpdaterClient.IsBootableImage(targetOSImageURL) if err != nil { - return err - } - if err := updateOS(state.currentConfig, osImageContentDir); err != nil { - return err + return fmt.Errorf("Error checking type of target image: %w", err) } - if err := os.RemoveAll(osImageContentDir); err != nil { - return err + + if isLayeredImage { + // If this is a new format image, we don't have to extract it, + // we can just update it the proper way + if err := updateLayeredOS(state.currentConfig); err != nil { + return err + } + } else { + osImageContentDir, err := ExtractOSImage(targetOSImageURL) + if err != nil { + return err + } + if err := updateOS(state.currentConfig, osImageContentDir); err != nil { + return err + } + if err := os.RemoveAll(osImageContentDir); err != nil { + return err + } } if err := dn.finalizeBeforeReboot(state.currentConfig); err != nil { return err diff --git a/pkg/daemon/rpm-ostree.go b/pkg/daemon/rpm-ostree.go index 1c1d3b1305..708db4bca5 100644 --- a/pkg/daemon/rpm-ostree.go +++ b/pkg/daemon/rpm-ostree.go @@ -2,6 +2,7 @@ package daemon import ( "encoding/json" + "errors" "fmt" "os" "os/exec" @@ -31,15 +32,18 @@ type rpmOstreeState struct { // RpmOstreeDeployment represents a single deployment on a node type RpmOstreeDeployment struct { - ID string `json:"id"` - OSName string `json:"osname"` - Serial int32 `json:"serial"` - Checksum string `json:"checksum"` - Version string `json:"version"` - Timestamp uint64 `json:"timestamp"` - Booted bool `json:"booted"` - Origin string `json:"origin"` - CustomOrigin []string `json:"custom-origin"` + ID string `json:"id"` + OSName string `json:"osname"` + Serial int32 `json:"serial"` + Checksum string `json:"checksum"` + Version string `json:"version"` + Timestamp uint64 `json:"timestamp"` + Booted bool `json:"booted"` + Staged bool `json:"staged"` + LiveReplaced string `json:"live-replaced,omitempty"` + Origin string `json:"origin"` + CustomOrigin []string `json:"custom-origin"` + ContainerImageReference string `json:"container-image-reference"` } // imageInspection is a public implementation of @@ -64,7 +68,9 @@ type NodeUpdaterClient interface { GetStatus() (string, error) GetBootedOSImageURL() (string, string, error) Rebase(string, string) (bool, error) - GetBootedDeployment() (*RpmOstreeDeployment, error) + RebaseLayered(string) error + IsBootableImage(string) (bool, error) + GetBootedAndStagedDeployment() (*RpmOstreeDeployment, *RpmOstreeDeployment, error) } // RpmOstreeClient provides all RpmOstree related methods in one structure. @@ -105,20 +111,16 @@ func (r *RpmOstreeClient) Initialize() error { } // GetBootedDeployment returns the current deployment found -func (r *RpmOstreeClient) GetBootedDeployment() (*RpmOstreeDeployment, error) { +func (r *RpmOstreeClient) GetBootedAndStagedDeployment() (booted, staged *RpmOstreeDeployment, err error) { status, err := r.loadStatus() if err != nil { - return nil, err + return nil, nil, err } - for _, deployment := range status.Deployments { - if deployment.Booted { - deployment := deployment - return &deployment, nil - } - } + booted, err = status.getBootedDeployment() + staged = status.getStagedDeployment() - return nil, fmt.Errorf("not currently booted in a deployment") + return } // GetStatus returns multi-line human-readable text describing system status @@ -135,7 +137,7 @@ func (r *RpmOstreeClient) GetStatus() (string, error) { // Returns the empty string if the host doesn't have a custom origin that matches pivot:// // (This could be the case for e.g. FCOS, or a future RHCOS which comes not-pivoted by default) func (r *RpmOstreeClient) GetBootedOSImageURL() (string, string, error) { - bootedDeployment, err := r.GetBootedDeployment() + bootedDeployment, _, err := r.GetBootedAndStagedDeployment() if err != nil { return "", "", err } @@ -148,6 +150,14 @@ func (r *RpmOstreeClient) GetBootedOSImageURL() (string, string, error) { } } + // we have container images now, make sure we can parse those too + if bootedDeployment.ContainerImageReference != "" { + // right now they start with "ostree-unverified-registry:", so scrape that off + tokens := strings.SplitN(bootedDeployment.ContainerImageReference, ":", 2) + if len(tokens) > 1 { + osImageURL = tokens[1] + } + } return osImageURL, bootedDeployment.Version, nil } @@ -189,7 +199,7 @@ func (r *RpmOstreeClient) Rebase(imgURL, osImageContentDir string) (changed bool ostreeCsum string ostreeVersion string ) - defaultDeployment, err := r.GetBootedDeployment() + defaultDeployment, _, err := r.GetBootedAndStagedDeployment() if err != nil { return } @@ -275,6 +285,66 @@ func (r *RpmOstreeClient) Rebase(imgURL, osImageContentDir string) (changed bool return } +// IsBootableImage determines if the image is a bootable (new container formet) image, or a wrapper (old container format) +func (r *RpmOstreeClient) IsBootableImage(imgURL string) (bool, error) { + + // TODO(jkyros): This is duplicated-ish from Rebase(), do we still need to carry this around? + var isBootableImage string + var imageData *types.ImageInspectInfo + var err error + if imageData, err = imageInspect(imgURL); err != nil { + if err != nil { + var podmanImgData *imageInspection + glog.Infof("Falling back to using podman inspect") + + if podmanImgData, err = podmanInspect(imgURL); err != nil { + return false, err + } + isBootableImage = podmanImgData.Labels["ostree.bootable"] + } + } else { + isBootableImage = imageData.Labels["ostree.bootable"] + } + // We may have pulled in OSContainer image as fallback during podmanCopy() or podmanInspect() + defer exec.Command("podman", "rmi", imgURL).Run() + + return isBootableImage == "true", nil +} + +// RebaseLayered rebases system or errors if already rebased +func (r *RpmOstreeClient) RebaseLayered(imgURL string) (err error) { + glog.Infof("Executing rebase to %s", imgURL) + + // For now, just let ostree use the kublet config.json, + err = useKubeletConfigSecrets() + if err != nil { + return fmt.Errorf("Error while ensuring access to kublet config.json pull secrets: %w", err) + } + + return runRpmOstree("rebase", "--experimental", "ostree-unverified-registry:"+imgURL) +} + +// useKubeletConfigSecrets gives the rpm-ostree client access to secrets in the kubelet config.json by symlinking so that +// rpm-ostree can use those secrets to pull images. It does this by symlinking the kubelet's config.json into /run/ostree. +func useKubeletConfigSecrets() error { + if _, err := os.Stat("/run/ostree/auth.json"); err != nil { + + if errors.Is(err, os.ErrNotExist) { + + err := os.MkdirAll("/run/ostree", 0o544) + if err != nil { + return err + } + + err = os.Symlink(kubeletAuthFile, "/run/ostree/auth.json") + if err != nil { + return err + } + } + } + return nil +} + // truncate a string using runes/codepoints as limits. // This specifically will avoid breaking a UTF-8 value. func truncate(input string, limit int) string { @@ -303,3 +373,23 @@ func runGetOut(command string, args ...string) ([]byte, error) { } return rawOut, nil } + +func (state *rpmOstreeState) getBootedDeployment() (*RpmOstreeDeployment, error) { + for num := range state.Deployments { + deployment := state.Deployments[num] + if deployment.Booted { + return &deployment, nil + } + } + return &RpmOstreeDeployment{}, fmt.Errorf("not currently booted in a deployment") +} + +func (state *rpmOstreeState) getStagedDeployment() *RpmOstreeDeployment { + for num := range state.Deployments { + deployment := state.Deployments[num] + if deployment.Staged { + return &deployment + } + } + return &RpmOstreeDeployment{} +} diff --git a/pkg/daemon/rpm-ostree_test.go b/pkg/daemon/rpm-ostree_test.go index 6159d3d2bf..7d895d55a6 100644 --- a/pkg/daemon/rpm-ostree_test.go +++ b/pkg/daemon/rpm-ostree_test.go @@ -46,3 +46,15 @@ func (r RpmOstreeClientMock) GetStatus() (string, error) { func (r RpmOstreeClientMock) GetBootedDeployment() (*RpmOstreeDeployment, error) { return &RpmOstreeDeployment{}, nil } + +func (r RpmOstreeClientMock) GetBootedAndStagedDeployment() (booted, staged *RpmOstreeDeployment, err error) { + return nil, nil, nil +} + +func (r RpmOstreeClientMock) IsBootableImage(string) (bool, error) { + return false, nil +} + +func (r RpmOstreeClientMock) RebaseLayered(string) error { + return nil +} diff --git a/pkg/daemon/update.go b/pkg/daemon/update.go index 1deef2a1a8..bc4abac4cc 100644 --- a/pkg/daemon/update.go +++ b/pkg/daemon/update.go @@ -337,7 +337,6 @@ func (dn *CoreOSDaemon) applyOSChanges(mcDiff machineConfigDiff, oldConfig, newC dn.nodeWriter.Eventf(corev1.EventTypeNormal, "OSUpdateStarted", mcDiff.osChangesString()) } - var osImageContentDir string if mcDiff.osUpdate || mcDiff.extensions || mcDiff.kernelType { // When we're going to apply an OS update, switch the block // scheduler to BFQ to apply more fairness between etcd @@ -346,6 +345,8 @@ func (dn *CoreOSDaemon) applyOSChanges(mcDiff machineConfigDiff, oldConfig, newC // do this. // Add nil check since firstboot also goes through this path, // which doesn't have a node object yet. + // This is okay because we know if we made it here, we are going + // to reboot and this setting does not persist across reboots. if dn.node != nil { if _, isControlPlane := dn.node.Labels[ctrlcommon.MasterLabel]; isControlPlane { if err := setRootDeviceSchedulerBFQ(); err != nil { @@ -357,72 +358,27 @@ func (dn *CoreOSDaemon) applyOSChanges(mcDiff machineConfigDiff, oldConfig, newC if dn.nodeWriter != nil { dn.nodeWriter.Eventf(corev1.EventTypeNormal, "InClusterUpgrade", fmt.Sprintf("Updating from oscontainer %s", newConfig.Spec.OSImageURL)) } - var err error - if osImageContentDir, err = ExtractOSImage(newConfig.Spec.OSImageURL); err != nil { - return err - } - // Delete extracted OS image once we are done. - defer os.RemoveAll(osImageContentDir) - if err := addExtensionsRepo(osImageContentDir); err != nil { - return err - } - defer os.Remove(extensionsRepo) } - // Update OS - if mcDiff.osUpdate { - if err := updateOS(newConfig, osImageContentDir); err != nil { - nodeName := "" - if dn.node != nil { - nodeName = dn.node.Name - } - MCDPivotErr.WithLabelValues(nodeName, newConfig.Spec.OSImageURL, err.Error()).SetToCurrentTime() - return err - } - if dn.nodeWriter != nil { - dn.nodeWriter.Eventf(corev1.EventTypeNormal, "OSUpgradeApplied", "OS upgrade applied; new MachineConfig (%s) has new OS image (%s)", newConfig.Name, newConfig.Spec.OSImageURL) - } - } else { //nolint:gocritic // The nil check for dn.nodeWriter has nothing to do with an OS update being unavailable. - // An OS upgrade is not available - if dn.nodeWriter != nil { - dn.nodeWriter.Eventf(corev1.EventTypeNormal, "OSUpgradeSkipped", "OS upgrade skipped; new MachineConfig (%s) has same OS image (%s) as old MachineConfig (%s)", newConfig.Name, newConfig.Spec.OSImageURL, oldConfig.Name) - } + // The steps from here on are different depending on the image type, so check the image type + isLayeredImage, err := dn.NodeUpdaterClient.IsBootableImage(newConfig.Spec.OSImageURL) + if err != nil { + return fmt.Errorf("Error checking type of update image: %w", err) } - defer func() { - // Operations performed by rpm-ostree on the booted system are available - // as staged deployment. It gets applied only when we reboot the system. - // In case of an error during any rpm-ostree transaction, removing pending deployment - // should be sufficient to discard any applied changes. - if retErr != nil { - // Print out the error now so that if we fail to cleanup -p, we don't lose it. - glog.Infof("Rolling back applied changes to OS due to error: %v", retErr) - if err := removePendingDeployment(); err != nil { - errs := kubeErrs.NewAggregate([]error{err, retErr}) - retErr = fmt.Errorf("error removing staged deployment: %w", errs) - return - } + if isLayeredImage { + // If it's a layered/bootable image, then apply it the "new" way + if err := dn.applyLayeredOSChanges(mcDiff, oldConfig, newConfig); err != nil { + return err } - }() - - // Apply kargs - if mcDiff.kargs { - if err := dn.updateKernelArguments(oldConfig.Spec.KernelArguments, newConfig.Spec.KernelArguments); err != nil { + } else { + // Otherwise fall back to the old way -- we can take this out someday when it goes away + if err := dn.applyLegacyOSChanges(mcDiff, oldConfig, newConfig); err != nil { return err } } - // Switch to real time kernel - if err := dn.switchKernel(oldConfig, newConfig); err != nil { - return err - } - - // Apply extensions - if err := dn.applyExtensions(oldConfig, newConfig); err != nil { - return err - } - if dn.nodeWriter != nil { var nodeName string var nodeObjRef corev1.ObjectReference @@ -771,6 +727,10 @@ func (mcDiff *machineConfigDiff) osChangesString() string { if mcDiff.kernelType { changes = append(changes, "Changing kernel type") } + if mcDiff.kargs { + changes = append(changes, "Changing kernel arguments") + } + return strings.Join(changes, "; ") } @@ -1889,6 +1849,18 @@ func updateOS(config *mcfgv1.MachineConfig, osImageContentDir string) error { return nil } +// updateLayeredOS updates the system OS to the one specified in newConfig +func updateLayeredOS(config *mcfgv1.MachineConfig) error { + newURL := config.Spec.OSImageURL + glog.Infof("Updating OS to layered image %s", newURL) + client := NewNodeUpdaterClient() + if err := client.RebaseLayered(newURL); err != nil { + return fmt.Errorf("failed to update OS to %s : %w", newURL, err) + } + + return nil +} + func (dn *Daemon) getPendingStateLegacyLogger() (*journalMsg, error) { glog.Info("logger doesn't support --jounald, grepping the journal") @@ -2091,3 +2063,133 @@ func (dn *Daemon) reboot(rationale string) error { MCDRebootErr.WithLabelValues(dn.node.Name, "reboot failed", "this error should be unreachable, something is seriously wrong").SetToCurrentTime() return fmt.Errorf("reboot failed; this error should be unreachable, something is seriously wrong") } + +func (dn *CoreOSDaemon) applyLayeredOSChanges(mcDiff machineConfigDiff, oldConfig, newConfig *mcfgv1.MachineConfig) (retErr error) { + // Update OS + if mcDiff.osUpdate { + if err := updateLayeredOS(newConfig); err != nil { + nodeName := "" + if dn.node != nil { + nodeName = dn.node.Name + } + MCDPivotErr.WithLabelValues(nodeName, newConfig.Spec.OSImageURL, err.Error()).SetToCurrentTime() + return err + } + if dn.nodeWriter != nil { + dn.nodeWriter.Eventf(corev1.EventTypeNormal, "OSUpgradeApplied", "OS upgrade applied; new MachineConfig (%s) has new OS image (%s)", newConfig.Name, newConfig.Spec.OSImageURL) + } + } else { //nolint:gocritic // The nil check for dn.nodeWriter has nothing to do with an OS update being unavailable. + // An OS upgrade is not available + if dn.nodeWriter != nil { + dn.nodeWriter.Eventf(corev1.EventTypeNormal, "OSUpgradeSkipped", "OS upgrade skipped; new MachineConfig (%s) has same OS image (%s) as old MachineConfig (%s)", newConfig.Name, newConfig.Spec.OSImageURL, oldConfig.Name) + } + } + + defer func() { + // Operations performed by rpm-ostree on the booted system are available + // as staged deployment. It gets applied only when we reboot the system. + // In case of an error during any rpm-ostree transaction, removing pending deployment + // should be sufficient to discard any applied changes. + if retErr != nil { + // Print out the error now so that if we fail to cleanup -p, we don't lose it. + glog.Infof("Rolling back applied changes to OS due to error: %v", retErr) + if err := removePendingDeployment(); err != nil { + errs := kubeErrs.NewAggregate([]error{err, retErr}) + retErr = fmt.Errorf("error removing staged deployment: %w", errs) + return + } + } + }() + + if mcDiff.kargs { + if err := dn.updateKernelArguments(oldConfig.Spec.KernelArguments, newConfig.Spec.KernelArguments); err != nil { + return err + } + } + + // TODO(jkyros): We can't currently switch kernels on layered images, so only allow it if they're both default. We'll come back for this when it's supported. + // If you did try to switch kernels when using layered image, you would get a "No enabled repositories" error. + if !(canonicalizeKernelType(oldConfig.Spec.KernelType) == ctrlcommon.KernelTypeDefault && canonicalizeKernelType(newConfig.Spec.KernelType) == ctrlcommon.KernelTypeDefault) { + return fmt.Errorf("Non-default kernels are not currently supported for layered images. (old: %s new %s)", oldConfig.Spec.KernelType, newConfig.Spec.KernelType) + } + + // TODO(jkyros): This is where we will handle Joseph's extensions container + if len(newConfig.Spec.Extensions) > 0 { + return fmt.Errorf("Extensions are not currently supported with layered images, but extensions were supplied: %s", strings.Join(newConfig.Spec.Extensions, " ")) + } + + return nil +} + +func (dn *CoreOSDaemon) applyLegacyOSChanges(mcDiff machineConfigDiff, oldConfig, newConfig *mcfgv1.MachineConfig) (retErr error) { + var osImageContentDir string + var err error + if mcDiff.osUpdate || mcDiff.extensions || mcDiff.kernelType { + + if osImageContentDir, err = ExtractOSImage(newConfig.Spec.OSImageURL); err != nil { + return err + } + // Delete extracted OS image once we are done. + defer os.RemoveAll(osImageContentDir) + + if err := addExtensionsRepo(osImageContentDir); err != nil { + return err + } + defer os.Remove(extensionsRepo) + } + + // Update OS + if mcDiff.osUpdate { + if err := updateOS(newConfig, osImageContentDir); err != nil { + nodeName := "" + if dn.node != nil { + nodeName = dn.node.Name + } + MCDPivotErr.WithLabelValues(nodeName, newConfig.Spec.OSImageURL, err.Error()).SetToCurrentTime() + return err + } + if dn.nodeWriter != nil { + dn.nodeWriter.Eventf(corev1.EventTypeNormal, "OSUpgradeApplied", "OS upgrade applied; new MachineConfig (%s) has new OS image (%s)", newConfig.Name, newConfig.Spec.OSImageURL) + } + } else { //nolint:gocritic // The nil check for dn.nodeWriter has nothing to do with an OS update being unavailable. + // An OS upgrade is not available + if dn.nodeWriter != nil { + dn.nodeWriter.Eventf(corev1.EventTypeNormal, "OSUpgradeSkipped", "OS upgrade skipped; new MachineConfig (%s) has same OS image (%s) as old MachineConfig (%s)", newConfig.Name, newConfig.Spec.OSImageURL, oldConfig.Name) + } + } + + defer func() { + // Operations performed by rpm-ostree on the booted system are available + // as staged deployment. It gets applied only when we reboot the system. + // In case of an error during any rpm-ostree transaction, removing pending deployment + // should be sufficient to discard any applied changes. + if retErr != nil { + // Print out the error now so that if we fail to cleanup -p, we don't lose it. + glog.Infof("Rolling back applied changes to OS due to error: %v", retErr) + if err := removePendingDeployment(); err != nil { + errs := kubeErrs.NewAggregate([]error{err, retErr}) + retErr = fmt.Errorf("error removing staged deployment: %w", errs) + return + } + } + }() + + // Apply kargs + if mcDiff.kargs { + if err := dn.updateKernelArguments(oldConfig.Spec.KernelArguments, newConfig.Spec.KernelArguments); err != nil { + return err + } + } + + // Switch to real time kernel + if err := dn.switchKernel(oldConfig, newConfig); err != nil { + return err + } + + // Apply extensions + if err := dn.applyExtensions(oldConfig, newConfig); err != nil { + return err + } + + return nil +}