diff --git a/cmd/machine-config-daemon/pivot.go b/cmd/machine-config-daemon/pivot.go new file mode 100644 index 0000000000..45fe1cd1e8 --- /dev/null +++ b/cmd/machine-config-daemon/pivot.go @@ -0,0 +1,376 @@ +package main + +import ( + "bufio" + "encoding/json" + "flag" + "fmt" + "io/ioutil" + "os" + "strings" + + // Enable sha256 in container image references + _ "crypto/sha256" + + "github.com/golang/glog" + daemon "github.com/openshift/machine-config-operator/pkg/daemon" + "github.com/openshift/machine-config-operator/pkg/daemon/pivot/types" + "github.com/openshift/machine-config-operator/pkg/daemon/pivot/utils" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// flag storage +var keep bool +var reboot bool +var exit77 bool + +const ( + // the number of times to retry commands that pull data from the network + numRetriesNetCommands = 5 + etcPivotFile = "/etc/pivot/image-pullspec" + runPivotRebootFile = "/run/pivot/reboot-needed" + // Pull secret. Written by the machine-config-operator + kubeletAuthFile = "/var/lib/kubelet/config.json" + // File containing kernel arg changes for tuning + kernelTuningFile = "/etc/pivot/kernel-args" + cmdLineFile = "/proc/cmdline" +) + +// TODO: fill out the whitelist +// tuneableArgsWhitelist contains allowed keys for tunable arguments +var tuneableArgsWhitelist = map[string]bool{ + "nosmt": true, +} + +var pivotCmd = &cobra.Command{ + Use: "pivot", + DisableFlagsInUseLine: true, + Short: "Allows moving from one OSTree deployment to another", + Args: cobra.MaximumNArgs(1), + Run: Execute, +} + +// init executes upon import +func init() { + rootCmd.AddCommand(pivotCmd) + pivotCmd.PersistentFlags().BoolVarP(&keep, "keep", "k", false, "Do not remove container image") + pivotCmd.PersistentFlags().BoolVarP(&reboot, "reboot", "r", false, "Reboot if changed") + pivotCmd.PersistentFlags().BoolVar(&exit77, "unchanged-exit-77", false, "If unchanged, exit 77") + pflag.CommandLine.AddGoFlagSet(flag.CommandLine) +} + +// isArgTuneable returns if the argument provided is allowed to be modified +func isArgTunable(arg string) bool { + return tuneableArgsWhitelist[arg] +} + +// isArgInUse checks to see if the argument is already in use by the system currently +func isArgInUse(arg, cmdLinePath string) (bool, error) { + if cmdLinePath == "" { + cmdLinePath = cmdLineFile + } + content, err := ioutil.ReadFile(cmdLinePath) + if err != nil { + return false, err + } + + checkable := string(content) + if strings.Contains(checkable, arg) { + return true, nil + } + return false, nil +} + +// parseTuningFile parses the kernel argument tuning file +func parseTuningFile(tuningFilePath, cmdLinePath string) ([]types.TuneArgument, []types.TuneArgument, error) { + addArguments := []types.TuneArgument{} + deleteArguments := []types.TuneArgument{} + if tuningFilePath == "" { + tuningFilePath = kernelTuningFile + } + if cmdLinePath == "" { + cmdLinePath = cmdLineFile + } + // Return fast if the file does not exist + if _, err := os.Stat(tuningFilePath); os.IsNotExist(err) { + glog.V(2).Infof("no kernel tuning needed as %s does not exist", tuningFilePath) + // This isn't an error. Return out. + return addArguments, deleteArguments, err + } + // Read and parse the file + file, err := os.Open(tuningFilePath) + if err != nil { + // If we have an issue reading return an error + glog.Infof("Unable to open %s for reading: %v", tuningFilePath, err) + return addArguments, deleteArguments, err + } + // Clean up + defer file.Close() + + // Parse the tuning lines + scanner := bufio.NewScanner(file) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "ADD ") { + // NOTE: Today only specific bare kernel arguments are allowed so + // there is not a need to split on =. + key := strings.TrimSpace(line[len("ADD "):]) + if isArgTunable(key) { + // Find out if the argument is in use + inUse, err := isArgInUse(key, cmdLinePath) + if err != nil { + return addArguments, deleteArguments, err + } + if !inUse { + addArguments = append(addArguments, types.TuneArgument{Key: key, Bare: true}) + } else { + glog.Infof(`skipping "%s" as it is already in use`, key) + } + } else { + glog.Infof("%s not a whitelisted kernel argument", key) + } + } else if strings.HasPrefix(line, "DELETE ") { + // NOTE: Today only specific bare kernel arguments are allowed so + // there is not a need to split on =. + key := strings.TrimSpace(line[len("DELETE "):]) + if isArgTunable(key) { + inUse, err := isArgInUse(key, cmdLinePath) + if err != nil { + return addArguments, deleteArguments, err + } + if inUse { + deleteArguments = append(deleteArguments, types.TuneArgument{Key: key, Bare: true}) + } else { + glog.Infof(`skipping "%s" as it is not present in the current argument list`, key) + } + } else { + glog.Infof("%s not a whitelisted kernel argument", key) + } + } else { + glog.V(2).Infof(`skipping malformed line in %s: "%s"`, tuningFilePath, line) + } + } + return addArguments, deleteArguments, nil +} + +// updateTuningArgs executes additions and removals of kernel tuning arguments +func updateTuningArgs(tuningFilePath, cmdLinePath string) (bool, error) { + if cmdLinePath == "" { + cmdLinePath = cmdLineFile + } + changed := false + additions, deletions, err := parseTuningFile(tuningFilePath, cmdLinePath) + if err != nil { + return changed, err + } + + // Execute additions + for _, toAdd := range additions { + if toAdd.Bare { + changed = true + utils.Run("rpm-ostree", "kargs", fmt.Sprintf("--append=%s", toAdd.Key)) + } else { + panic("Not supported") + } + } + // Execute deletions + for _, toDelete := range deletions { + if toDelete.Bare { + changed = true + utils.Run("rpm-ostree", "kargs", fmt.Sprintf("--delete=%s", toDelete.Key)) + } else { + panic("Not supported") + } + } + return changed, nil +} + +// podmanRemove kills and removes a container +func podmanRemove(cid string) { + utils.RunIgnoreErr("podman", "kill", cid) + utils.RunIgnoreErr("podman", "rm", "-f", cid) +} + +// getDefaultDeployment uses rpm-ostree status --json to get the current deployment +func getDefaultDeployment() types.RpmOstreeDeployment { + // use --status for now, we can switch to D-Bus if we need more info + var rosState types.RpmOstreeState + output := utils.RunGetOut("rpm-ostree", "status", "--json") + if err := json.Unmarshal([]byte(output), &rosState); err != nil { + glog.Fatalf("Failed to parse `rpm-ostree status --json` output: %v", err) + } + + // just make it a hard error if we somehow don't have any deployments + if len(rosState.Deployments) == 0 { + glog.Fatalf("Not currently booted in a deployment") + } + + return rosState.Deployments[0] +} + +// pullAndRebase potentially rebases system if not already rebased. +func pullAndRebase(container string) (imgid string, changed bool) { + defaultDeployment := getDefaultDeployment() + + previousPivot := "" + if len(defaultDeployment.CustomOrigin) > 0 { + if strings.HasPrefix(defaultDeployment.CustomOrigin[0], "pivot://") { + previousPivot = defaultDeployment.CustomOrigin[0][len("pivot://"):] + glog.Infof("Previous pivot: %s", previousPivot) + } + } + + var authArgs []string + if utils.FileExists(kubeletAuthFile) { + authArgs = append(authArgs, "--authfile", kubeletAuthFile) + } + + // If we're passed a non-canonical image, resolve it to its sha256 now + isCanonicalForm := true + if _, err := daemon.GetRefDigest(container); err != nil { + isCanonicalForm = false + // In non-canonical form, we pull unconditionally right now + args := []string{"pull", "-q"} + args = append(args, authArgs...) + args = append(args, container) + utils.RunExt(false, numRetriesNetCommands, "podman", args...) + } else { + targetMatched, err := daemon.CompareOSImageURL(previousPivot, container) + if err != nil { + glog.Fatalf("%v", err) + } + if targetMatched { + changed = false + return + } + + // Pull the image + args := []string{"pull", "-q"} + args = append(args, authArgs...) + args = append(args, container) + utils.RunExt(false, numRetriesNetCommands, "podman", args...) + } + + inspectArgs := []string{"inspect", "--type=image"} + inspectArgs = append(inspectArgs, fmt.Sprintf("%s", container)) + output := utils.RunExt(true, 1, "podman", inspectArgs...) + var imagedataArray []types.ImageInspection + json.Unmarshal([]byte(output), &imagedataArray) + imagedata := imagedataArray[0] + if !isCanonicalForm { + imgid = imagedata.RepoDigests[0] + glog.Infof("Resolved to: %s", imgid) + } else { + imgid = container + } + + // Clean up a previous container + podmanRemove(types.PivotName) + + // `podman mount` wants a container, so let's make create a dummy one, but not run it + cid := utils.RunGetOut("podman", "create", "--net=none", "--name", types.PivotName, imgid) + // Use the container ID to find its mount point + mnt := utils.RunGetOut("podman", "mount", cid) + repo := fmt.Sprintf("%s/srv/repo", mnt) + + // Now we need to figure out the commit to rebase to + + // Commit label takes priority + ostreeCsum, ok := imagedata.Labels["com.coreos.ostree-commit"] + if ok { + if ostreeVersion, ok := imagedata.Labels["version"]; ok { + glog.Infof("Pivoting to: %s (%s)", ostreeVersion, ostreeCsum) + } else { + glog.Infof("Pivoting to: %s", ostreeCsum) + } + } else { + glog.Infof("No com.coreos.ostree-commit label found in metadata! Inspecting...") + refs := strings.Split(utils.RunGetOut("ostree", "refs", "--repo", repo), "\n") + if len(refs) == 1 { + glog.Infof("Using ref %s", refs[0]) + ostreeCsum = utils.RunGetOut("ostree", "rev-parse", "--repo", repo, refs[0]) + } else if len(refs) > 1 { + glog.Fatalf("Multiple refs found in repo!") + } else { + // XXX: in the future, possibly scan the repo to find a unique .commit object + glog.Fatalf("No refs found in repo!") + } + } + + // This will be what will be displayed in `rpm-ostree status` as the "origin spec" + customURL := fmt.Sprintf("pivot://%s", imgid) + + // RPM-OSTree can now directly slurp from the mounted container! + // https://github.com/projectatomic/rpm-ostree/pull/1732 + utils.Run("rpm-ostree", "rebase", "--experimental", + fmt.Sprintf("%s:%s", repo, ostreeCsum), + "--custom-origin-url", customURL, + "--custom-origin-description", "Managed by pivot tool") + + // Kill our dummy container + podmanRemove(types.PivotName) + + changed = true + return +} + +// Execute runs the command +func Execute(cmd *cobra.Command, args []string) { + var fromFile bool + var container string + if len(args) > 0 { + container = args[0] + fromFile = false + } else { + glog.Infof("Using image pullspec from %s", etcPivotFile) + data, err := ioutil.ReadFile(etcPivotFile) + if err != nil { + glog.Fatalf("Failed to read from %s: %v", etcPivotFile, err) + } + container = strings.TrimSpace(string(data)) + fromFile = true + } + + imgid, changed := pullAndRebase(container) + + // Delete the file now that we successfully rebased + if fromFile { + if err := os.Remove(etcPivotFile); err != nil { + if !os.IsNotExist(err) { + glog.Fatalf("Failed to delete %s: %v", etcPivotFile, err) + } + } + } + + // By default, delete the image. + if !keep { + // Related: https://github.com/containers/libpod/issues/2234 + utils.RunIgnoreErr("podman", "rmi", imgid) + } + + // Check to see if we need to tune kernel arguments + tuningChanged, err := updateTuningArgs(kernelTuningFile, cmdLineFile) + if err != nil { + glog.Infof("unable to parse tuning file %s: %s", kernelTuningFile, err) + } + // If tuning changes but the oscontainer didn't we still denote we changed + // for the reboot + if tuningChanged { + changed = true + if err != nil { + glog.Infof(`Unable to remove kernel tuning file %s: "%s"`, kernelTuningFile, err) + } + + } + + if !changed { + glog.Info("Already at target pivot; exiting...") + if exit77 { + os.Exit(77) + } + } else if reboot || utils.FileExists(runPivotRebootFile) { + // Reboot the machine if asked to do so + utils.Run("systemctl", "reboot") + } +} diff --git a/pkg/daemon/daemon.go b/pkg/daemon/daemon.go index af99a8b8e5..6efb871856 100644 --- a/pkg/daemon/daemon.go +++ b/pkg/daemon/daemon.go @@ -1103,10 +1103,10 @@ func (dn *Daemon) validateOnDiskState(currentConfig *mcfgv1.MachineConfig) bool return true } -// getRefDigest parses a Docker/OCI image reference and returns +// GetRefDigest parses a Docker/OCI image reference and returns // its digest, or an error if the string fails to parse as // a "canonical" image reference with a digest. -func getRefDigest(ref string) (string, error) { +func GetRefDigest(ref string) (string, error) { refParsed, err := imgref.ParseNamed(ref) if err != nil { return "", errors.Wrapf(err, "parsing reference: %q", ref) @@ -1119,8 +1119,8 @@ func getRefDigest(ref string) (string, error) { return canon.Digest().String(), nil } -// compareOSImageURL is the backend for checkOS. -func compareOSImageURL(current, desired string) (bool, error) { +// CompareOSImageURL is the backend for checkOS. +func CompareOSImageURL(current, desired string) (bool, error) { // Since https://github.com/openshift/machine-config-operator/pull/426 landed // we don't use the "unspecified" osImageURL anymore, but let's keep supporting // it for now. @@ -1134,11 +1134,11 @@ func compareOSImageURL(current, desired string) (bool, error) { return true, nil } - bootedDigest, err := getRefDigest(current) + bootedDigest, err := GetRefDigest(current) if err != nil { return false, errors.Wrap(err, "parsing booted osImageURL") } - desiredDigest, err := getRefDigest(desired) + desiredDigest, err := GetRefDigest(desired) if err != nil { return false, errors.Wrap(err, "parsing desired osImageURL") } @@ -1164,7 +1164,7 @@ func (dn *Daemon) checkOS(osImageURL string) (bool, error) { return true, nil } - return compareOSImageURL(dn.bootedOSImageURL, osImageURL) + return CompareOSImageURL(dn.bootedOSImageURL, osImageURL) } // checkUnits validates the contents of all the units in the diff --git a/pkg/daemon/daemon_test.go b/pkg/daemon/daemon_test.go index e66a24064a..332d44a584 100644 --- a/pkg/daemon/daemon_test.go +++ b/pkg/daemon/daemon_test.go @@ -112,19 +112,19 @@ func TestCompareOSImageURL(t *testing.T) { refA := "registry.example.com/foo/bar@sha256:0743a3cc3bcf3b4aabb814500c2739f84cb085ff4e7ec7996aef7977c4c19c7f" refB := "registry.example.com/foo/baz@sha256:0743a3cc3bcf3b4aabb814500c2739f84cb085ff4e7ec7996aef7977c4c19c7f" refC := "registry.example.com/foo/bar@sha256:2a76681fd15bfc06fa4aa0ff6913ba17527e075417fc92ea29f6bcc2afca24ff" - m, err := compareOSImageURL(refA, refA) + m, err := CompareOSImageURL(refA, refA) if !m { t.Fatalf("Expected refA ident") } - m, err = compareOSImageURL(refA, refB) + m, err = CompareOSImageURL(refA, refB) if !m { t.Fatalf("Expected refA = refB") } - m, err = compareOSImageURL(refA, refC) + m, err = CompareOSImageURL(refA, refC) if m { t.Fatalf("Expected refA != refC") } - m, err = compareOSImageURL(refA, "registry.example.com/foo/bar") + m, err = CompareOSImageURL(refA, "registry.example.com/foo/bar") if m || err == nil { t.Fatalf("Expected err") } diff --git a/pkg/daemon/pivot/types/imageinspection.go b/pkg/daemon/pivot/types/imageinspection.go new file mode 100644 index 0000000000..62dd71a91d --- /dev/null +++ b/pkg/daemon/pivot/types/imageinspection.go @@ -0,0 +1,47 @@ +package types + +import ( + "time" + + "github.com/opencontainers/go-digest" +) + +const ( + // PivotName is literally the name of the new pivot + PivotName = "ostree-container-pivot" +) + +// ImageInspection is a public implementation of +// https://github.com/containers/skopeo/blob/82186b916faa9c8c70cfa922229bafe5ae024dec/cmd/skopeo/inspect.go#L20-L31 +type ImageInspection struct { + Name string `json:",omitempty"` + Tag string `json:",omitempty"` + Digest digest.Digest + RepoDigests []string + Created *time.Time + DockerVersion string + Labels map[string]string + Architecture string + Os string + Layers []string +} + +// RpmOstreeState houses zero or more deployments +// Subset of `rpm-ostree status --json` +// https://github.com/projectatomic/rpm-ostree/blob/bce966a9812df141d38e3290f845171ec745aa4e/src/daemon/rpmostreed-deployment-utils.c#L227 +type RpmOstreeState struct { + Deployments []RpmOstreeDeployment +} + +// RpmOstreeDeployment abstracts a specific rpm-ostree deployment +type RpmOstreeDeployment struct { + ID string `json:"id"` + OSName string `json:"osname"` + Serial int32 `json:"serial"` + Checksum string `json:"checksum"` + Version string `json:"version"` + Timestamp uint64 `json:"timestamp"` + Booted bool `json:"booted"` + Origin string `json:"origin"` + CustomOrigin []string `json:"custom-origin"` +} diff --git a/pkg/daemon/pivot/types/tuneargument.go b/pkg/daemon/pivot/types/tuneargument.go new file mode 100644 index 0000000000..3f317e63f5 --- /dev/null +++ b/pkg/daemon/pivot/types/tuneargument.go @@ -0,0 +1,8 @@ +package types + +// TuneArgument represents a single tuning argument +type TuneArgument struct { + Key string `json:"key"` // The name of the argument (or argument itself if Bare) + Value string `json:"value"` // The value of the argument + Bare bool `json:"bare"` // If the kernel argument is a bare argument (no value expected) +} diff --git a/pkg/daemon/pivot/utils/fs.go b/pkg/daemon/pivot/utils/fs.go new file mode 100644 index 0000000000..2ced330d34 --- /dev/null +++ b/pkg/daemon/pivot/utils/fs.go @@ -0,0 +1,18 @@ +package utils + +import ( + "os" + + "github.com/golang/glog" +) + +// FileExists checks if the file exists, gracefully handling ENOENT. +func FileExists(path string) bool { + if _, err := os.Stat(path); err != nil { + if os.IsNotExist(err) { + return false + } + glog.Fatalf("Failed to stat %s: %v", path, err) + } + return true +} diff --git a/pkg/daemon/pivot/utils/run.go b/pkg/daemon/pivot/utils/run.go new file mode 100644 index 0000000000..17eb447e20 --- /dev/null +++ b/pkg/daemon/pivot/utils/run.go @@ -0,0 +1,87 @@ +package utils + +import ( + "bytes" + "os" + "os/exec" + "strings" + "time" + + "github.com/golang/glog" + "k8s.io/apimachinery/pkg/util/wait" +) + +// runImpl is the actual shell execution implementation used by other functions. +func runImpl(capture bool, command string, args ...string) ([]byte, error) { + glog.Infof("Running: %s %s\n", command, strings.Join(args, " ")) + cmd := exec.Command(command, args...) + cmd.Stderr = os.Stderr + var stdout bytes.Buffer + if !capture { + cmd.Stdout = os.Stdout + } else { + cmd.Stdout = &stdout + } + err := cmd.Run() + if err != nil { + return nil, err + } + if capture { + return stdout.Bytes(), nil + } + return []byte{}, nil +} + +// runExtBackoff is an extension to runExt that supports configuring retries/duration/backoff. +func runExtBackoff(capture bool, backoff wait.Backoff, command string, args ...string) string { + var output string + err := wait.ExponentialBackoff(backoff, func() (bool, error) { + if out, e := runImpl(capture, command, args...); e != nil { + glog.Warningf("%s failed: %v; retrying...", command, e) + return false, nil + } else if capture { + output = strings.TrimSpace(string(out)) + } + return true, nil + }) + if err != nil { + glog.Fatalf("%s: %s", command, err) + } + return output +} + +// RunExt executes a command, optionally capturing the output and retrying multiple +// times before exiting with a fatal error. +func RunExt(capture bool, retries int, command string, args ...string) string { + return runExtBackoff(capture, wait.Backoff{ + Steps: retries + 1, // times to try + Duration: 5 * time.Second, // sleep between tries + Factor: 2, // factor by which to increase sleep + }, + command, args...) +} + +// Run executes a command, logging it, and exit with a fatal error if +// the command failed. +func Run(command string, args ...string) { + if _, err := runImpl(false, command, args...); err != nil { + glog.Fatalf("%s: %s", command, err) + } +} + +// RunIgnoreErr is like Run(..), but doesn't exit on errors +func RunIgnoreErr(command string, args ...string) { + if _, err := runImpl(false, command, args...); err != nil { + glog.Warningf("(ignored) %s: %s", command, err) + } +} + +// RunGetOut is like Run(..), but get the output as a string +func RunGetOut(command string, args ...string) string { + var err error + var out []byte + if out, err = runImpl(true, command, args...); err != nil { + glog.Fatalf("%s: %s", command, err) + } + return strings.TrimSpace(string(out)) +} diff --git a/pkg/daemon/pivot/utils/run_test.go b/pkg/daemon/pivot/utils/run_test.go new file mode 100644 index 0000000000..dc01392cb7 --- /dev/null +++ b/pkg/daemon/pivot/utils/run_test.go @@ -0,0 +1,60 @@ +package utils + +import ( + "testing" + "os" + "time" + "io/ioutil" + + "k8s.io/apimachinery/pkg/util/wait" +) + +// TestRun should always pass. The function will panic if it is unable to +// execute the shell command(s) or the command returns non-zero. +func TestRun(t *testing.T) { + Run("echo", "echo", "from", "TestRun") +} + +// TestRunGetOut verifies the output of running a command is +// its output, trimmed of whitespace. +func TestRunGetOut(t *testing.T) { + if result := RunGetOut("echo", "hello", "world"); result != "hello world" { + t.Errorf("expected 'hello world', got '%s'", result) + } +} + +// TestRunIgnoreErr verifies the a failed command doesn't cause exit +func TestRunIgnoreErr(t *testing.T) { + // Should succeed and cause no exit + RunIgnoreErr("echo", "test") + // Should fail and cause no exit + RunIgnoreErr("acommandthatdoesNOTEXIST") +} + +// TestRunExt verifies that the wait machinery works, even though we're only +// just testing a single step here since it's tricky to test retries. +func TestRunExt(t *testing.T) { + RunExt(false, 0, "echo", "echo", "from", "TestRunExt") + + if result := RunExt(true, 0, "echo", "hello", "world"); result != "hello world" { + t.Errorf("expected 'hello world', got '%s'", result) + } + + tmpdir, err := ioutil.TempDir("", "run_test") + if err != nil { + t.Fatalf("%v", err) + } + defer os.RemoveAll(tmpdir) + tmpf := tmpdir + "/t" + runExtBackoff(false, wait.Backoff{Steps: 6, + Duration: 1 * time.Second, + Factor: 1.1}, + "sh", "-c", "echo -n x >> " + tmpf + " && test $(stat -c '%s' " + tmpf + ") = 3") + s, err := os.Stat(tmpf) + if err != nil { + t.Fatalf("%v", err) + } + if s.Size() != 3 { + t.Fatalf("Expected size 3") + } +} diff --git a/pkg/daemon/update.go b/pkg/daemon/update.go index 9dc2576e44..05112aebef 100644 --- a/pkg/daemon/update.go +++ b/pkg/daemon/update.go @@ -789,7 +789,7 @@ func (dn *Daemon) updateOS(config *mcfgv1.MachineConfig) error { } newURL := config.Spec.OSImageURL - osMatch, err := compareOSImageURL(dn.bootedOSImageURL, newURL) + osMatch, err := CompareOSImageURL(dn.bootedOSImageURL, newURL) if err != nil { return err } diff --git a/templates/common/_base/units/machine-config-daemon-initial.service b/templates/common/_base/units/machine-config-daemon-initial.service new file mode 100644 index 0000000000..d2fe4ac205 --- /dev/null +++ b/templates/common/_base/units/machine-config-daemon-initial.service @@ -0,0 +1,18 @@ +name: "machine-config-daemon-initial.service" +enabled: true +contents: | + [Unit] + Description=Machine Config Daemon Initial + ConditionPathExists=/etc/pivot/image-pullspec + # If pivot exists, defer to it + ConditionPathExists=!/usr/bin/pivot + After=ignition-firstboot-complete.service + Before=kubelet.service + + [Service] + # Need oneshot to delay kubelet + Type=oneshot + ExecStart=/usr/libexec/machine-config-daemon pivot + + [Install] + WantedBy=multi-user.target