diff --git a/checkpoint.go b/checkpoint.go index c1ca44f63ab..9df558c8dba 100644 --- a/checkpoint.go +++ b/checkpoint.go @@ -30,8 +30,14 @@ var checkpointCommand = cli.Command{ if err != nil { fatal(err) } - defer destroy(container) options := criuOptions(context) + status, err := container.Status() + if err != nil { + fatal(err) + } + if status == libcontainer.Checkpointed { + fatal(fmt.Errorf("Container with id %s already checkpointed", context.GlobalString("id"))) + } // these are the mandatory criu options for a container setPageServer(context, options) setManageCgroupsMode(context, options) diff --git a/libcontainer/container.go b/libcontainer/container.go index 051c8cf6821..6292fd1852d 100644 --- a/libcontainer/container.go +++ b/libcontainer/container.go @@ -30,23 +30,6 @@ const ( Destroyed ) -func (s Status) String() string { - switch s { - case Running: - return "running" - case Pausing: - return "pausing" - case Paused: - return "paused" - case Checkpointed: - return "checkpointed" - case Destroyed: - return "destroyed" - default: - return "undefined" - } -} - // BaseState represents the platform agnostic pieces relating to a // running container's state type BaseState struct { diff --git a/libcontainer/container_linux.go b/libcontainer/container_linux.go index 9cf3f9b97c8..5ab4fa2c022 100644 --- a/libcontainer/container_linux.go +++ b/libcontainer/container_linux.go @@ -37,7 +37,6 @@ type linuxContainer struct { criuPath string m sync.Mutex criuVersion int - state containerState } // State represents a running container's state @@ -190,14 +189,7 @@ func (c *linuxContainer) Start(process *Process) error { return newSystemError(err) } if doInit { - if err := c.updateState(parent); err != nil { - return err - } - } else { - c.state.transition(&nullState{ - c: c, - s: Running, - }) + c.updateState(parent) } if c.config.Hooks != nil { s := configs.HookState{ @@ -334,29 +326,48 @@ func newPipe() (parent *os.File, child *os.File, err error) { func (c *linuxContainer) Destroy() error { c.m.Lock() defer c.m.Unlock() - return c.state.destroy() + status, err := c.currentStatus() + if err != nil { + return err + } + if status != Destroyed { + return newGenericError(fmt.Errorf("container is not destroyed"), ContainerNotStopped) + } + if !c.config.Namespaces.Contains(configs.NEWPID) { + if err := killCgroupProcesses(c.cgroupManager); err != nil { + logrus.Warn(err) + } + } + err = c.cgroupManager.Destroy() + if rerr := os.RemoveAll(c.root); err == nil { + err = rerr + } + c.initProcess = nil + if c.config.Hooks != nil { + s := configs.HookState{ + Version: c.config.Version, + ID: c.id, + Root: c.config.Rootfs, + } + for _, hook := range c.config.Hooks.Poststop { + if err := hook.Run(s); err != nil { + return err + } + } + } + return err } func (c *linuxContainer) Pause() error { c.m.Lock() defer c.m.Unlock() - if err := c.cgroupManager.Freeze(configs.Frozen); err != nil { - return err - } - return c.state.transition(&pausedState{ - c: c, - }) + return c.cgroupManager.Freeze(configs.Frozen) } func (c *linuxContainer) Resume() error { c.m.Lock() defer c.m.Unlock() - if err := c.cgroupManager.Freeze(configs.Thawed); err != nil { - return err - } - return c.state.transition(&runningState{ - c: c, - }) + return c.cgroupManager.Freeze(configs.Thawed) } func (c *linuxContainer) NotifyOOM() (<-chan struct{}, error) { @@ -458,7 +469,7 @@ func (c *linuxContainer) Checkpoint(criuOpts *CriuOpts) error { } if criuOpts.ImagesDirectory == "" { - return fmt.Errorf("invalid directory to save checkpoint") + criuOpts.ImagesDirectory = filepath.Join(c.root, "criu.image") } // Since a container can be C/R'ed multiple times, @@ -577,9 +588,11 @@ func (c *linuxContainer) addCriuRestoreMount(req *criurpc.CriuReq, m *configs.Mo func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { c.m.Lock() defer c.m.Unlock() + if err := c.checkCriuVersion("1.5.2"); err != nil { return err } + if criuOpts.WorkDirectory == "" { criuOpts.WorkDirectory = filepath.Join(c.root, "criu.work") } @@ -588,19 +601,22 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { if err := os.Mkdir(criuOpts.WorkDirectory, 0655); err != nil && !os.IsExist(err) { return err } + workDir, err := os.Open(criuOpts.WorkDirectory) if err != nil { return err } defer workDir.Close() + if criuOpts.ImagesDirectory == "" { - return fmt.Errorf("invalid directory to restore checkpoint") + criuOpts.ImagesDirectory = filepath.Join(c.root, "criu.image") } imageDir, err := os.Open(criuOpts.ImagesDirectory) if err != nil { return err } defer imageDir.Close() + // CRIU has a few requirements for a root directory: // * it must be a mount point // * its parent must not be overmounted @@ -611,15 +627,18 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { return err } defer os.Remove(root) + root, err = filepath.EvalSymlinks(root) if err != nil { return err } + err = syscall.Mount(c.config.Rootfs, root, "", syscall.MS_BIND|syscall.MS_REC, "") if err != nil { return err } defer syscall.Unmount(root, syscall.MNT_DETACH) + t := criurpc.CriuReqType_RESTORE req := &criurpc.CriuReq{ Type: &t, @@ -687,13 +706,15 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { fds []string fdJSON []byte ) + if fdJSON, err = ioutil.ReadFile(filepath.Join(criuOpts.ImagesDirectory, descriptorsFilename)); err != nil { return err } - if err := json.Unmarshal(fdJSON, &fds); err != nil { + if err = json.Unmarshal(fdJSON, &fds); err != nil { return err } + for i := range fds { if s := fds[i]; strings.Contains(s, "pipe:") { inheritFd := new(criurpc.InheritFd) @@ -702,7 +723,12 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { req.Opts.InheritFd = append(req.Opts.InheritFd, inheritFd) } } - return c.criuSwrk(process, req, criuOpts, true) + + err = c.criuSwrk(process, req, criuOpts, true) + if err != nil { + return err + } + return nil } func (c *linuxContainer) criuApplyCgroups(pid int, req *criurpc.CriuReq) error { @@ -897,123 +923,82 @@ func (c *linuxContainer) criuNotifications(resp *criurpc.CriuResp, process *Proc if notify == nil { return fmt.Errorf("invalid response: %s", resp.String()) } + switch { case notify.GetScript() == "post-dump": - f, err := os.Create(filepath.Join(c.root, "checkpoint")) - if err != nil { - return err + if !opts.LeaveRunning { + f, err := os.Create(filepath.Join(c.root, "checkpoint")) + if err != nil { + return err + } + f.Close() } - f.Close() + break + case notify.GetScript() == "network-unlock": if err := unlockNetwork(c.config); err != nil { return err } + break + case notify.GetScript() == "network-lock": if err := lockNetwork(c.config); err != nil { return err } + break + case notify.GetScript() == "post-restore": pid := notify.GetPid() r, err := newRestoredProcess(int(pid), fds) if err != nil { return err } - process.ops = r - if err := c.state.transition(&restoredState{ - imageDir: opts.ImagesDirectory, - c: c, - }); err != nil { - return err - } + + // TODO: crosbymichael restore previous process information by saving the init process information in + // the container's state file or separate process state files. if err := c.updateState(r); err != nil { return err } - if err := os.Remove(filepath.Join(c.root, "checkpoint")); err != nil { - if !os.IsNotExist(err) { - logrus.Error(err) - } - } + process.ops = r + break } + return nil } func (c *linuxContainer) updateState(process parentProcess) error { c.initProcess = process - if err := c.refreshState(); err != nil { - return err - } state, err := c.currentState() if err != nil { return err } - return c.saveState(state) -} - -func (c *linuxContainer) saveState(s *State) error { f, err := os.Create(filepath.Join(c.root, stateFilename)) if err != nil { return err } defer f.Close() - return json.NewEncoder(f).Encode(s) -} - -func (c *linuxContainer) deleteState() error { - return os.Remove(filepath.Join(c.root, stateFilename)) + os.Remove(filepath.Join(c.root, "checkpoint")) + return json.NewEncoder(f).Encode(state) } func (c *linuxContainer) currentStatus() (Status, error) { - if err := c.refreshState(); err != nil { - return -1, err - } - return c.state.status(), nil -} - -// refreshState needs to be called to verify that the current state on the -// container is what is true. Because consumers of libcontainer can use it -// out of process we need to verify the container's status based on runtime -// information and not rely on our in process info. -func (c *linuxContainer) refreshState() error { - paused, err := c.isPaused() - if err != nil { - return err + if _, err := os.Stat(filepath.Join(c.root, "checkpoint")); err == nil { + return Checkpointed, nil } - if paused { - return c.state.transition(&pausedState{c: c}) - } - running, err := c.isRunning() - if err != nil { - return err - } - if running { - return c.state.transition(&runningState{c: c}) - } - return c.state.transition(&stoppedState{c: c}) -} - -func (c *linuxContainer) isRunning() (bool, error) { if c.initProcess == nil { - return false, nil + return Destroyed, nil } // return Running if the init process is alive if err := syscall.Kill(c.initProcess.pid(), 0); err != nil { if err == syscall.ESRCH { - return false, nil + return Destroyed, nil } - return false, newSystemError(err) + return 0, newSystemError(err) } - return true, nil -} - -func (c *linuxContainer) isPaused() (bool, error) { - data, err := ioutil.ReadFile(filepath.Join(c.cgroupManager.GetPaths()["freezer"], "freezer.state")) - if err != nil { - if os.IsNotExist(err) { - return false, nil - } - return false, newSystemError(err) + if c.config.Cgroups != nil && c.config.Cgroups.Resources != nil && c.config.Cgroups.Resources.Freezer == configs.Frozen { + return Paused, nil } - return bytes.Equal(bytes.TrimSpace(data), []byte("FROZEN")), nil + return Running, nil } func (c *linuxContainer) currentState() (*State, error) { diff --git a/libcontainer/container_linux_test.go b/libcontainer/container_linux_test.go index a66ea067ca7..18d8d310732 100644 --- a/libcontainer/container_linux_test.go +++ b/libcontainer/container_linux_test.go @@ -166,7 +166,6 @@ func TestGetContainerState(t *testing.T) { }, }, } - container.state = &nullState{c: container} state, err := container.State() if err != nil { t.Fatal(err) diff --git a/libcontainer/factory_linux.go b/libcontainer/factory_linux.go index ae5db9cdbbb..54c13fa9c73 100644 --- a/libcontainer/factory_linux.go +++ b/libcontainer/factory_linux.go @@ -165,7 +165,7 @@ func (l *LinuxFactory) Create(id string, config *configs.Config) (Container, err if err := os.MkdirAll(containerRoot, 0700); err != nil { return nil, newGenericError(err, SystemError) } - c := &linuxContainer{ + return &linuxContainer{ id: id, root: containerRoot, config: config, @@ -173,9 +173,7 @@ func (l *LinuxFactory) Create(id string, config *configs.Config) (Container, err initArgs: l.InitArgs, criuPath: l.CriuPath, cgroupManager: l.NewCgroupsManager(config.Cgroups, nil), - } - c.state = &stoppedState{c: c} - return c, nil + }, nil } func (l *LinuxFactory) Load(id string) (Container, error) { @@ -192,7 +190,7 @@ func (l *LinuxFactory) Load(id string) (Container, error) { processStartTime: state.InitProcessStartTime, fds: state.ExternalDescriptors, } - c := &linuxContainer{ + return &linuxContainer{ initProcess: r, id: id, config: &state.Config, @@ -201,9 +199,7 @@ func (l *LinuxFactory) Load(id string) (Container, error) { criuPath: l.CriuPath, cgroupManager: l.NewCgroupsManager(state.Config.Cgroups, state.CgroupPaths), root: containerRoot, - } - c.state = &nullState{c: c} - return c, nil + }, nil } func (l *LinuxFactory) Type() string { diff --git a/libcontainer/integration/checkpoint_test.go b/libcontainer/integration/checkpoint_test.go index a71c172a3c2..09d506a8a5c 100644 --- a/libcontainer/integration/checkpoint_test.go +++ b/libcontainer/integration/checkpoint_test.go @@ -129,8 +129,8 @@ func TestCheckpoint(t *testing.T) { t.Fatal(err) } - if state != libcontainer.Running { - t.Fatal("Unexpected state checkpoint: ", state) + if state != libcontainer.Checkpointed { + t.Fatal("Unexpected state: ", state) } stdinW.Close() @@ -169,7 +169,7 @@ func TestCheckpoint(t *testing.T) { t.Fatal(err) } if state != libcontainer.Running { - t.Fatal("Unexpected restore state: ", state) + t.Fatal("Unexpected state: ", state) } pid, err = restoreProcessConfig.Pid() diff --git a/libcontainer/state_linux.go b/libcontainer/state_linux.go deleted file mode 100644 index 5e1bb731f8e..00000000000 --- a/libcontainer/state_linux.go +++ /dev/null @@ -1,217 +0,0 @@ -// +build linux - -package libcontainer - -import ( - "fmt" - "os" - "path/filepath" - - "github.com/Sirupsen/logrus" - "github.com/opencontainers/runc/libcontainer/configs" -) - -func newStateTransitionError(from, to containerState) error { - return &stateTransitionError{ - From: from.status().String(), - To: to.status().String(), - } -} - -// stateTransitionError is returned when an invalid state transition happens from one -// state to another. -type stateTransitionError struct { - From string - To string -} - -func (s *stateTransitionError) Error() string { - return fmt.Sprintf("invalid state transition from %s to %s", s.From, s.To) -} - -type containerState interface { - transition(containerState) error - destroy() error - status() Status -} - -func destroy(c *linuxContainer) error { - if !c.config.Namespaces.Contains(configs.NEWPID) { - if err := killCgroupProcesses(c.cgroupManager); err != nil { - logrus.Warn(err) - } - } - err := c.cgroupManager.Destroy() - if rerr := os.RemoveAll(c.root); err == nil { - err = rerr - } - c.initProcess = nil - if herr := runPoststopHooks(c); err == nil { - err = herr - } - return err -} - -func runPoststopHooks(c *linuxContainer) error { - if c.config.Hooks != nil { - s := configs.HookState{ - Version: c.config.Version, - ID: c.id, - Root: c.config.Rootfs, - } - for _, hook := range c.config.Hooks.Poststop { - if err := hook.Run(s); err != nil { - return err - } - } - } - return nil -} - -// stoppedState represents a container is a stopped/destroyed state. -type stoppedState struct { - c *linuxContainer -} - -func (b *stoppedState) status() Status { - return Destroyed -} - -func (b *stoppedState) transition(s containerState) error { - switch s.(type) { - case *runningState: - b.c.state = s - return nil - case *restoredState: - b.c.state = s - return nil - case *stoppedState: - return nil - } - return newStateTransitionError(b, s) -} - -func (b *stoppedState) destroy() error { - return destroy(b.c) -} - -// runningState represents a container that is currently running. -type runningState struct { - c *linuxContainer -} - -func (r *runningState) status() Status { - return Running -} - -func (r *runningState) transition(s containerState) error { - switch s.(type) { - case *stoppedState: - running, err := r.c.isRunning() - if err != nil { - return err - } - if running { - return newGenericError(fmt.Errorf("container still running"), ContainerNotStopped) - } - r.c.state = s - return nil - case *pausedState: - r.c.state = s - return nil - case *runningState, *nullState: - return nil - } - return newStateTransitionError(r, s) -} - -func (r *runningState) destroy() error { - running, err := r.c.isRunning() - if err != nil { - return err - } - if running { - return newGenericError(fmt.Errorf("container is not destroyed"), ContainerNotStopped) - } - return destroy(r.c) -} - -// pausedState represents a container that is currently pause. It cannot be destroyed in a -// paused state and must transition back to running first. -type pausedState struct { - c *linuxContainer -} - -func (p *pausedState) status() Status { - return Paused -} - -func (p *pausedState) transition(s containerState) error { - switch s.(type) { - case *runningState: - p.c.state = s - return nil - case *pausedState: - return nil - } - return newStateTransitionError(p, s) -} - -func (p *pausedState) destroy() error { - return newGenericError(fmt.Errorf("container is paused"), ContainerPaused) -} - -// restoredState is the same as the running state but also has accociated checkpoint -// information that maybe need destroyed when the container is stopped and destory is called. -type restoredState struct { - imageDir string - c *linuxContainer -} - -func (r *restoredState) status() Status { - return Running -} - -func (r *restoredState) transition(s containerState) error { - switch s.(type) { - case *stoppedState: - return nil - case *runningState: - return nil - } - return newStateTransitionError(r, s) -} - -func (r *restoredState) destroy() error { - if _, err := os.Stat(filepath.Join(r.c.root, "checkpoint")); err != nil { - if !os.IsNotExist(err) { - return err - } - } - return destroy(r.c) -} - -// nullState is used whenever a container is restored, loaded, or setting additional -// processes inside and it should not be destroyed when it is exiting. -type nullState struct { - c *linuxContainer - s Status -} - -func (n *nullState) status() Status { - return n.s -} - -func (n *nullState) transition(s containerState) error { - switch s.(type) { - case *restoredState: - n.c.state = s - default: - // do nothing for null states - } - return nil -} - -func (n *nullState) destroy() error { - return nil -} diff --git a/libcontainer/state_linux_test.go b/libcontainer/state_linux_test.go deleted file mode 100644 index e2740ca63a2..00000000000 --- a/libcontainer/state_linux_test.go +++ /dev/null @@ -1,85 +0,0 @@ -// +build linux - -package libcontainer - -import "testing" - -func TestStateStatus(t *testing.T) { - states := map[containerState]Status{ - &stoppedState{}: Destroyed, - &runningState{}: Running, - &restoredState{}: Running, - &pausedState{}: Paused, - } - for s, status := range states { - if s.status() != status { - t.Fatalf("state returned %s but expected %s", s.status(), status) - } - } -} - -func isStateTransitionError(err error) bool { - _, ok := err.(*stateTransitionError) - return ok -} - -func TestStoppedStateTransition(t *testing.T) { - s := &stoppedState{c: &linuxContainer{}} - valid := []containerState{ - &stoppedState{}, - &runningState{}, - &restoredState{}, - } - for _, v := range valid { - if err := s.transition(v); err != nil { - t.Fatal(err) - } - } - err := s.transition(&pausedState{}) - if err == nil { - t.Fatal("transition to paused state should fail") - } - if !isStateTransitionError(err) { - t.Fatal("expected stateTransitionError") - } -} - -func TestPausedStateTransition(t *testing.T) { - s := &pausedState{c: &linuxContainer{}} - valid := []containerState{ - &pausedState{}, - &runningState{}, - } - for _, v := range valid { - if err := s.transition(v); err != nil { - t.Fatal(err) - } - } - err := s.transition(&stoppedState{}) - if err == nil { - t.Fatal("transition to stopped state should fail") - } - if !isStateTransitionError(err) { - t.Fatal("expected stateTransitionError") - } -} - -func TestRestoredStateTransition(t *testing.T) { - s := &restoredState{c: &linuxContainer{}} - valid := []containerState{ - &stoppedState{}, - &runningState{}, - } - for _, v := range valid { - if err := s.transition(v); err != nil { - t.Fatal(err) - } - } - err := s.transition(&nullState{}) - if err == nil { - t.Fatal("transition to null state should fail") - } - if !isStateTransitionError(err) { - t.Fatal("expected stateTransitionError") - } -} diff --git a/restore.go b/restore.go index 55cff56aad7..072b2c4a0aa 100644 --- a/restore.go +++ b/restore.go @@ -5,6 +5,7 @@ package main import ( "fmt" "os" + "path/filepath" "github.com/Sirupsen/logrus" "github.com/codegangsta/cli" @@ -121,6 +122,16 @@ func restoreContainer(context *cli.Context, spec *specs.LinuxSpec, config *confi handler := newSignalHandler(tty) defer handler.Close() if err := container.Restore(process, options); err != nil { + cstatus, cerr := container.Status() + if cerr != nil { + logrus.Error(cerr) + } + if cstatus == libcontainer.Destroyed { + dest := filepath.Join(context.GlobalString("root"), context.GlobalString("id")) + if errVal := os.RemoveAll(dest); errVal != nil { + logrus.Error(errVal) + } + } return -1, err } return handler.forward(process) diff --git a/start.go b/start.go index 8f2d8584530..a713d3c2ef4 100644 --- a/start.go +++ b/start.go @@ -149,7 +149,13 @@ func setupSocketActivation(spec *specs.LinuxSpec, listenFds string) { } func destroy(container libcontainer.Container) { - if err := container.Destroy(); err != nil { + status, err := container.Status() + if err != nil { logrus.Error(err) } + if status != libcontainer.Checkpointed { + if err := container.Destroy(); err != nil { + logrus.Error(err) + } + } }