diff --git a/cmd/minikube/cmd/stop.go b/cmd/minikube/cmd/stop.go index a020c203b444..4cce5d09fa86 100644 --- a/cmd/minikube/cmd/stop.go +++ b/cmd/minikube/cmd/stop.go @@ -99,12 +99,13 @@ func runStop(cmd *cobra.Command, args []string) { schedule.KillExisting(profilesToStop) if scheduledStopDuration != 0 { - if runtime.GOOS == "windows" { - exit.Message(reason.Usage, "the --schedule flag is currently not supported on windows") - } if err := schedule.Daemonize(profilesToStop, scheduledStopDuration); err != nil { exit.Message(reason.DaemonizeError, "unable to daemonize: {{.err}}", out.V{"err": err.Error()}) } + // if OS is windows, scheduled stop is now being handled within minikube, so return + if runtime.GOOS == "windows" { + return + } klog.Infof("sleeping %s before completing stop...", scheduledStopDuration.String()) time.Sleep(scheduledStopDuration) } diff --git a/deploy/iso/minikube-iso/package/Config.in b/deploy/iso/minikube-iso/package/Config.in index dc7a8c85c247..27d1403c13f6 100644 --- a/deploy/iso/minikube-iso/package/Config.in +++ b/deploy/iso/minikube-iso/package/Config.in @@ -14,4 +14,5 @@ menu "System tools" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/vbox-guest/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/containerd-bin/Config.in" source "$BR2_EXTERNAL_MINIKUBE_PATH/package/falco-module/Config.in" + source "$BR2_EXTERNAL_MINIKUBE_PATH/package/scheduled-stop/Config.in" endmenu diff --git a/deploy/iso/minikube-iso/package/scheduled-stop/Config.in b/deploy/iso/minikube-iso/package/scheduled-stop/Config.in new file mode 100644 index 000000000000..d4a68f99e8c5 --- /dev/null +++ b/deploy/iso/minikube-iso/package/scheduled-stop/Config.in @@ -0,0 +1,3 @@ +config BR2_PACKAGE_SCHEDULED_STOP + bool "scheduled-stop" + default y diff --git a/deploy/iso/minikube-iso/package/scheduled-stop/minikube-scheduled-stop b/deploy/iso/minikube-iso/package/scheduled-stop/minikube-scheduled-stop new file mode 100755 index 000000000000..a02670900387 --- /dev/null +++ b/deploy/iso/minikube-iso/package/scheduled-stop/minikube-scheduled-stop @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +echo "running scheduled stop ..."; + +echo "sleeping %$SLEEP seconds..." +sleep $SLEEP + +echo "running poweroff..." +sudo systemctl poweroff diff --git a/deploy/iso/minikube-iso/package/scheduled-stop/minikube-scheduled-stop.service b/deploy/iso/minikube-iso/package/scheduled-stop/minikube-scheduled-stop.service new file mode 100644 index 000000000000..12a298415fb3 --- /dev/null +++ b/deploy/iso/minikube-iso/package/scheduled-stop/minikube-scheduled-stop.service @@ -0,0 +1,11 @@ +[Unit] +Description=minikube scheduled stop + +[Install] +WantedBy=multi-user.target + +[Service] +Type=simple +User=root +ExecStart=/usr/sbin/minikube-scheduled-stop +EnvironmentFile=/var/lib/minikube/scheduled-stop/environment diff --git a/deploy/iso/minikube-iso/package/scheduled-stop/scheduled-stop.mk b/deploy/iso/minikube-iso/package/scheduled-stop/scheduled-stop.mk new file mode 100644 index 000000000000..ca1dc31f5302 --- /dev/null +++ b/deploy/iso/minikube-iso/package/scheduled-stop/scheduled-stop.mk @@ -0,0 +1,23 @@ +################################################################################ +# +# minikube scheduled-stop +# +################################################################################ + +define SCHEDULED_STOP_INSTALL_INIT_SYSTEMD + $(INSTALL) -D -m 644 \ + $(SCHEDULED_STOP_PKGDIR)/minikube-scheduled-stop.service \ + $(TARGET_DIR)/usr/lib/systemd/system/minikube-scheduled-stop.service + + mkdir -p $(TARGET_DIR)/etc/systemd/system/multi-user.target.wants + ln -fs /usr/lib/systemd/system/minikube-scheduled-stop.service \ + $(TARGET_DIR)/etc/systemd/system/multi-user.target.wants/minikube-scheduled-stop.service +endef + +define SCHEDULED_STOP_INSTALL_TARGET_CMDS + $(INSTALL) -Dm755 \ + $(SCHEDULED_STOP_PKGDIR)/minikube-scheduled-stop \ + $(TARGET_DIR)/usr/sbin/minikube-scheduled-stop +endef + +$(eval $(generic-package)) diff --git a/deploy/kicbase/Dockerfile b/deploy/kicbase/Dockerfile index 66e6fd7bbe7a..16ad9c4b654a 100644 --- a/deploy/kicbase/Dockerfile +++ b/deploy/kicbase/Dockerfile @@ -130,6 +130,13 @@ COPY automount/minikube-automount.service /usr/lib/systemd/system/minikube-autom RUN ln -fs /usr/lib/systemd/system/minikube-automount.service \ /etc/systemd/system/multi-user.target.wants/minikube-automount.service +# scheduled stop service +COPY scheduled-stop/minikube-scheduled-stop /var/lib/minikube/scheduled-stop/minikube-scheduled-stop +COPY scheduled-stop/minikube-scheduled-stop.service /usr/lib/systemd/system/minikube-scheduled-stop.service +RUN ln -fs /usr/lib/systemd/system/minikube-scheduled-stop.service \ + /etc/systemd/system/multi-user.target.wants/minikube-scheduled-stop.service && \ + chmod +x /var/lib/minikube/scheduled-stop/minikube-scheduled-stop + # disable non-docker runtimes by default RUN systemctl disable containerd && systemctl disable crio && rm /etc/crictl.yaml # enable docker which is default diff --git a/deploy/kicbase/scheduled-stop/minikube-scheduled-stop b/deploy/kicbase/scheduled-stop/minikube-scheduled-stop new file mode 100644 index 000000000000..a02670900387 --- /dev/null +++ b/deploy/kicbase/scheduled-stop/minikube-scheduled-stop @@ -0,0 +1,11 @@ +#!/bin/bash + +set -x + +echo "running scheduled stop ..."; + +echo "sleeping %$SLEEP seconds..." +sleep $SLEEP + +echo "running poweroff..." +sudo systemctl poweroff diff --git a/deploy/kicbase/scheduled-stop/minikube-scheduled-stop.service b/deploy/kicbase/scheduled-stop/minikube-scheduled-stop.service new file mode 100644 index 000000000000..cfc738b9b469 --- /dev/null +++ b/deploy/kicbase/scheduled-stop/minikube-scheduled-stop.service @@ -0,0 +1,11 @@ +[Unit] +Description=minikube scheduled stop + +[Install] +WantedBy=multi-user.target + +[Service] +Type=simple +User=root +ExecStart=/var/lib/minikube/scheduled-stop/minikube-scheduled-stop +EnvironmentFile=/var/lib/minikube/scheduled-stop/environment diff --git a/pkg/minikube/constants/constants.go b/pkg/minikube/constants/constants.go index 99c5128a409b..1b3e794bed76 100644 --- a/pkg/minikube/constants/constants.go +++ b/pkg/minikube/constants/constants.go @@ -80,6 +80,10 @@ const ( // TestDiskUsedEnv is used in integration tests for insufficient storage with 'minikube status' TestDiskUsedEnv = "MINIKUBE_TEST_STORAGE_CAPACITY" + // scheduled stop constants + ScheduledStopEnvFile = "/var/lib/minikube/scheduled-stop/environment" + ScheduledStopSystemdService = "minikube-scheduled-stop" + // MinikubeExistingPrefix is used to save the original environment when executing docker-env MinikubeExistingPrefix = "MINIKUBE_EXISTING_" diff --git a/pkg/minikube/schedule/daemonize_windows.go b/pkg/minikube/schedule/daemonize_windows.go index ea3c7fa23856..d4b077fce049 100644 --- a/pkg/minikube/schedule/daemonize_windows.go +++ b/pkg/minikube/schedule/daemonize_windows.go @@ -20,16 +20,93 @@ package schedule import ( "fmt" + "os/exec" "time" + "github.com/pkg/errors" "k8s.io/klog/v2" + "k8s.io/minikube/pkg/minikube/assets" + "k8s.io/minikube/pkg/minikube/constants" + "k8s.io/minikube/pkg/minikube/machine" + "k8s.io/minikube/pkg/minikube/sysinit" ) // KillExisting will kill existing scheduled stops func KillExisting(profiles []string) { - klog.Errorf("not yet implemented for windows") + for _, profile := range profiles { + if err := killExisting(profile); err != nil { + klog.Errorf("error terminating scheduled stop for profile %s: %v", profile, err) + } + } } +func killExisting(profile string) error { + klog.Infof("trying to kill existing schedule stop for profile %s...", profile) + api, err := machine.NewAPIClient() + if err != nil { + return errors.Wrapf(err, "getting api client for profile %s", profile) + } + h, err := api.Load(profile) + if err != nil { + return errors.Wrap(err, "Error loading existing host. Please try running [minikube delete], then run [minikube start] again.") + } + runner, err := machine.CommandRunner(h) + if err != nil { + return errors.Wrap(err, "getting command runner") + } + // restart scheduled stop service in container + sysManger := sysinit.New(runner) + if err := sysManger.Stop(constants.ScheduledStopSystemdService); err != nil { + return errors.Wrapf(err, "stopping schedule-stop service for profile %s", profile) + } + return nil +} + +// to daemonize on windows, we schedule the stop within minikube itself +// starting the minikube-scheduled-stop systemd service kicks off the scheduled stop func daemonize(profiles []string, duration time.Duration) error { - return fmt.Errorf("not yet implemented for windows") + for _, profile := range profiles { + if err := startSystemdService(profile, duration); err != nil { + return errors.Wrapf(err, "implementing scheduled stop for %s", profile) + } + } + return nil +} + +// to start the systemd service, we first have to tell the systemd service how long to sleep for +// before shutting down minikube from within +// we do this by settig the SLEEP environment variable in the environment file to the users +// requested duration +func startSystemdService(profile string, duration time.Duration) error { + // get ssh runner + klog.Infof("starting systemd service for profile %s...", profile) + api, err := machine.NewAPIClient() + if err != nil { + return errors.Wrapf(err, "getting api client for profile %s", profile) + } + h, err := api.Load(profile) + if err != nil { + return errors.Wrap(err, "Error loading existing host. Please try running [minikube delete], then run [minikube start] again.") + } + runner, err := machine.CommandRunner(h) + if err != nil { + return errors.Wrap(err, "getting command runner") + } + if rr, err := runner.RunCmd(exec.Command("sudo", "mkdir", "-p", "/var/lib/minikube/scheduled-stop")); err != nil { + return errors.Wrapf(err, "creating dirs: %v", rr.Output()) + } + // update environment file to include duration + if err := runner.Copy(environmentFile(duration)); err != nil { + return errors.Wrap(err, "copying scheduled stop env file") + } + // restart scheduled stop service in container + sysManger := sysinit.New(runner) + return sysManger.Restart(constants.ScheduledStopSystemdService) +} + +// return the contents of the environment file for minikube-scheduled-stop systemd service +// should be of the format SLEEP= +func environmentFile(duration time.Duration) assets.CopyableFile { + contents := []byte(fmt.Sprintf("SLEEP=%v", duration.Seconds())) + return assets.NewMemoryAssetTarget(contents, constants.ScheduledStopEnvFile, "0644") } diff --git a/pkg/minikube/schedule/schedule.go b/pkg/minikube/schedule/schedule.go index a016cf00bffa..c8ca7b5f49c4 100644 --- a/pkg/minikube/schedule/schedule.go +++ b/pkg/minikube/schedule/schedule.go @@ -41,11 +41,19 @@ func Daemonize(profiles []string, duration time.Duration) error { continue } daemonizeProfiles = append(daemonizeProfiles, p) + } + + if err := daemonize(daemonizeProfiles, duration); err != nil { + return errors.Wrap(err, "daemonizing") + } + + // save scheduled stop config if daemonize was successful + for _, d := range daemonizeProfiles { + _, cc := mustload.Partial(d) cc.ScheduledStop = scheduledStop - if err := config.SaveProfile(p, cc); err != nil { + if err := config.SaveProfile(d, cc); err != nil { return errors.Wrap(err, "saving profile") } } - - return daemonize(daemonizeProfiles, duration) + return nil } diff --git a/test/integration/scheduled_stop_test.go b/test/integration/scheduled_stop_test.go index 494b6e405961..98e3506eb141 100644 --- a/test/integration/scheduled_stop_test.go +++ b/test/integration/scheduled_stop_test.go @@ -24,19 +24,58 @@ import ( "io/ioutil" "os" "os/exec" + "runtime" "strconv" + "strings" "syscall" "testing" "time" "github.com/docker/machine/libmachine/state" + "k8s.io/minikube/pkg/minikube/constants" "k8s.io/minikube/pkg/minikube/localpath" "k8s.io/minikube/pkg/util/retry" ) -func TestScheduledStop(t *testing.T) { +func TestScheduledStopWindows(t *testing.T) { + if runtime.GOOS != "windows" { + t.Skip("test only runs on windows") + } if NoneDriver() { - t.Skip("--schedule does not apply to none driver ") + t.Skip("--schedule does not work with the none driver") + } + profile := UniqueProfileName("scheduled-stop") + ctx, cancel := context.WithTimeout(context.Background(), Minutes(5)) + defer CleanupWithLogs(t, profile, cancel) + startMinikube(ctx, t, profile) + + // schedule a stop for 5m from now + scheduledStopMinikube(ctx, t, profile, "5m") + + // make sure the systemd service is running + rr, err := Run(t, exec.CommandContext(ctx, Target(), []string{"ssh", "-p", profile, "--", "sudo", "systemctl", "show", constants.ScheduledStopSystemdService, "--no-page"}...)) + if err != nil { + t.Fatalf("getting minikube-scheduled-stop status: %v\n%s", err, rr.Output()) + } + if !strings.Contains(rr.Output(), "ActiveState=active") { + t.Fatalf("minikube-scheduled-stop is not running: %v", rr.Output()) + } + + // reschedule stop for 5 seconds from now + scheduledStopMinikube(ctx, t, profile, "5s") + + // sleep for 5 seconds + time.Sleep(5 * time.Second) + // make sure minikube status is "Stopped" + ensureMinikubeStatusStopped(ctx, t, profile) +} + +func TestScheduledStopUnix(t *testing.T) { + if runtime.GOOS == "windows" { + t.Skip("test only runs on unix") + } + if NoneDriver() { + t.Skip("--schedule does not work with the none driver") } profile := UniqueProfileName("scheduled-stop") ctx, cancel := context.WithTimeout(context.Background(), Minutes(5)) @@ -56,19 +95,8 @@ func TestScheduledStop(t *testing.T) { t.Fatalf("process %v running but should have been killed on reschedule of stop", pid) } checkPID(t, profile) - // wait allotted time to make sure minikube status is "Stopped" - checkStatus := func() error { - ctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Second)) - defer cancel() - got := Status(ctx, t, Target(), profile, "Host", profile) - if got != state.Stopped.String() { - return fmt.Errorf("expected post-stop host status to be -%q- but got *%q*", state.Stopped, got) - } - return nil - } - if err := retry.Expo(checkStatus, time.Second, time.Minute); err != nil { - t.Fatalf("error %v", err) - } + // make sure minikube status is "Stopped" + ensureMinikubeStatusStopped(ctx, t, profile) } func startMinikube(ctx context.Context, t *testing.T, profile string) { @@ -116,3 +144,19 @@ func processRunning(t *testing.T, pid string) bool { t.Log("signal error was: ", err) return err == nil } + +func ensureMinikubeStatusStopped(ctx context.Context, t *testing.T, profile string) { + // wait allotted time to make sure minikube status is "Stopped" + checkStatus := func() error { + ctx, cancel := context.WithDeadline(ctx, time.Now().Add(10*time.Second)) + defer cancel() + got := Status(ctx, t, Target(), profile, "Host", profile) + if got != state.Stopped.String() { + return fmt.Errorf("expected post-stop host status to be -%q- but got *%q*", state.Stopped, got) + } + return nil + } + if err := retry.Expo(checkStatus, time.Second, time.Minute); err != nil { + t.Fatalf("error %v", err) + } +}