From 9053cee01298226a2da9f11709a71fcfd6de5e66 Mon Sep 17 00:00:00 2001 From: Saied Kazemi Date: Thu, 5 Feb 2015 20:32:27 -0800 Subject: [PATCH 1/5] Checkpoint/Restore Support: add exec driver methods Methods for checkpointing and restoring containers were added to the native driver. The LXC driver returns an error message that these methods are not implemented yet. Signed-off-by: Saied Kazemi Conflicts: daemon/execdriver/native/create.go daemon/execdriver/native/driver.go daemon/execdriver/native/init.go --- daemon/execdriver/driver.go | 3 + daemon/execdriver/lxc/driver.go | 8 ++ daemon/execdriver/native/create.go | 19 ++++ daemon/execdriver/native/driver.go | 150 +++++++++++++++++++++++++++++ 4 files changed, 180 insertions(+) diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index eca77e921eaac..acb41c5f53376 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -24,6 +24,7 @@ var ( ) type StartCallback func(*ProcessConfig, int) +type RestoreCallback func(*ProcessConfig, int) // Driver specific information based on // processes registered with the driver @@ -59,6 +60,8 @@ type Driver interface { Kill(c *Command, sig int) error Pause(c *Command) error Unpause(c *Command) error + Checkpoint(c *Command) error + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback) (int, error) Name() string // Driver name Info(id string) Info // "temporary" hack (until we move state from core to plugins) GetPidsForContainer(id string) ([]int, error) // Returns a list of pids for the given container. diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 4b5730a3f4f42..91c84078b8456 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -547,6 +547,14 @@ func (d *driver) Unpause(c *execdriver.Command) error { return err } +func (d *driver) Checkpoint(c *execdriver.Command) error { + return fmt.Errorf("Checkpointing lxc containers not supported yet\n") +} + +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + return 0, fmt.Errorf("Restoring lxc containers not supported yet\n") +} + func (d *driver) Terminate(c *execdriver.Command) error { return KillLxc(c.ID, 9) } diff --git a/daemon/execdriver/native/create.go b/daemon/execdriver/native/create.go index 6d2a1e8c4598a..dc98ca8034d54 100644 --- a/daemon/execdriver/native/create.go +++ b/daemon/execdriver/native/create.go @@ -4,6 +4,7 @@ package native import ( "errors" + "encoding/json" "fmt" "net" "strings" @@ -88,6 +89,24 @@ func generateIfaceName() (string, error) { return "", errors.New("Failed to find name for new interface") } +// Re-create the container type from the image that was saved during checkpoint. +func (d *driver) createRestoreContainer(c *execdriver.Command, imageDir string) (*libcontainer.Config, error) { + // Read the container.json. + f1, err := os.Open(filepath.Join(imageDir, "container.json")) + if err != nil { + return nil, err + } + defer f1.Close() + + var container *libcontainer.Config + err = json.NewDecoder(f1).Decode(&container) + if err != nil { + return nil, err + } + + return container, nil +} + func (d *driver) createNetwork(container *configs.Config, c *execdriver.Command) error { if c.Network.ContainerID != "" { d.Lock() diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 1e1df1f21bfbf..2f46786922d47 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -19,6 +19,7 @@ import ( "github.com/docker/docker/pkg/reexec" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" + "github.com/docker/docker/utils" "github.com/docker/libcontainer" "github.com/docker/libcontainer/apparmor" "github.com/docker/libcontainer/cgroups/systemd" @@ -274,6 +275,155 @@ func (d *driver) Unpause(c *execdriver.Command) error { return active.Resume() } +// XXX Where is the right place for the following +// const and getCheckpointImageDir() function? +const ( + containersDir = "/var/lib/docker/containers" + criuImgDir = "criu_img" +) + +func getCheckpointImageDir(containerId string) string { + return filepath.Join(containersDir, containerId, criuImgDir) +} + +func (d *driver) Checkpoint(c *execdriver.Command) error { + active := d.activeContainers[c.ID] + if active == nil { + return fmt.Errorf("active container for %s does not exist", c.ID) + } + container := active.container + + // Create an image directory for this container (which + // may already exist from a previous checkpoint). + imageDir := getCheckpointImageDir(c.ID) + err := os.MkdirAll(imageDir, 0700) + if err != nil && !os.IsExist(err) { + return err + } + + // Copy container.json and state.json files to the CRIU + // image directory for later use during restore. Do this + // before checkpointing because after checkpoint the container + // will exit and these files will be removed. + log.CRDbg("saving container.json and state.json before calling CRIU in %s", imageDir) + srcFiles := []string{"container.json", "state.json"} + for _, f := range srcFiles { + srcFile := filepath.Join(d.root, c.ID, f) + dstFile := filepath.Join(imageDir, f) + if _, err := utils.CopyFile(srcFile, dstFile); err != nil { + return err + } + } + + d.Lock() + defer d.Unlock() + err = namespaces.Checkpoint(container, imageDir, c.ProcessConfig.Process.Pid) + if err != nil { + return err + } + + return nil +} + +type restoreOutput struct { + exitCode int + err error +} + +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + imageDir := getCheckpointImageDir(c.ID) + container, err := d.createRestoreContainer(c, imageDir) + if err != nil { + return 1, err + } + + var term execdriver.Terminal + + if c.ProcessConfig.Tty { + term, err = NewTtyConsole(&c.ProcessConfig, pipes) + } else { + term, err = execdriver.NewStdConsole(&c.ProcessConfig, pipes) + } + if err != nil { + return -1, err + } + c.ProcessConfig.Terminal = term + + d.Lock() + d.activeContainers[c.ID] = &activeContainer{ + container: container, + cmd: &c.ProcessConfig.Cmd, + } + d.Unlock() + defer d.cleanContainer(c.ID) + + // Since the CRIU binary exits after restoring the container, we + // need to reap its child by setting PR_SET_CHILD_SUBREAPER (36) + // so that it'll be owned by this process (Docker daemon) after restore. + // + // XXX This really belongs to where the Docker daemon starts. + if _, _, syserr := syscall.RawSyscall(syscall.SYS_PRCTL, 36, 1, 0); syserr != 0 { + return -1, fmt.Errorf("Could not set PR_SET_CHILD_SUBREAPER (syserr %d)", syserr) + } + + restoreOutputChan := make(chan restoreOutput, 1) + waitForRestore := make(chan struct{}) + + go func() { + exitCode, err := namespaces.Restore(container, c.ProcessConfig.Stdin, c.ProcessConfig.Stdout, c.ProcessConfig.Stderr, c.ProcessConfig.Console, filepath.Join(d.root, c.ID), imageDir, + func(child *os.File, args []string) *exec.Cmd { + cmd := new(exec.Cmd) + cmd.Path = d.initPath + cmd.Args = append([]string{ + DriverName, + "-restore", + "-pipe", "3", + "--", + }, args...) + cmd.ExtraFiles = []*os.File{child} + return cmd + }, + func(restorePid int) error { + log.CRDbg("restorePid=%d", restorePid) + if restorePid == 0 { + restoreCallback(&c.ProcessConfig, 0) + return nil + } + + // The container.json file should be written *after* the container + // has started because its StdFds cannot be initialized before. + // + // XXX How do we handle error here? + d.writeContainerFile(container, c.ID) + close(waitForRestore) + if restoreCallback != nil { + c.ProcessConfig.Process, err = os.FindProcess(restorePid) + if err != nil { + log.Debugf("cannot find restored process %d", restorePid) + return err + } + c.ContainerPid = c.ProcessConfig.Process.Pid + restoreCallback(&c.ProcessConfig, c.ContainerPid) + } + return nil + }) + restoreOutputChan <- restoreOutput{exitCode, err} + }() + + select { + case restoreOutput := <-restoreOutputChan: + // there was an error + return restoreOutput.exitCode, restoreOutput.err + case <-waitForRestore: + // container restored + break + } + + // Wait for the container to exit. + restoreOutput := <-restoreOutputChan + return restoreOutput.exitCode, restoreOutput.err +} + func (d *driver) Terminate(c *execdriver.Command) error { defer d.cleanContainer(c.ID) container, err := d.factory.Load(c.ID) From fc699376cc666ad7b75a9494424712f8100f3467 Mon Sep 17 00:00:00 2001 From: boucher Date: Mon, 25 May 2015 08:32:58 -0700 Subject: [PATCH 2/5] Update checkpoint/restore support to match docker/master Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- daemon/execdriver/driver.go | 4 +- daemon/execdriver/lxc/driver.go | 6 +- daemon/execdriver/native/create.go | 19 --- daemon/execdriver/native/driver.go | 183 ++++++++++------------------- 4 files changed, 69 insertions(+), 143 deletions(-) diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index acb41c5f53376..54d3955bb040c 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -60,8 +60,8 @@ type Driver interface { Kill(c *Command, sig int) error Pause(c *Command) error Unpause(c *Command) error - Checkpoint(c *Command) error - Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback) (int, error) + Checkpoint(c *Command, opts *libcontainer.CriuOpts) error + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (ExitStatus, error) Name() string // Driver name Info(id string) Info // "temporary" hack (until we move state from core to plugins) GetPidsForContainer(id string) ([]int, error) // Returns a list of pids for the given container. diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 91c84078b8456..fe5bdc06b45c4 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -547,12 +547,12 @@ func (d *driver) Unpause(c *execdriver.Command) error { return err } -func (d *driver) Checkpoint(c *execdriver.Command) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { return fmt.Errorf("Checkpointing lxc containers not supported yet\n") } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { - return 0, fmt.Errorf("Restoring lxc containers not supported yet\n") +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { + return execdriver.ExitStatus{ExitCode: 0}, fmt.Errorf("Restoring lxc containers not supported yet\n") } func (d *driver) Terminate(c *execdriver.Command) error { diff --git a/daemon/execdriver/native/create.go b/daemon/execdriver/native/create.go index dc98ca8034d54..6d2a1e8c4598a 100644 --- a/daemon/execdriver/native/create.go +++ b/daemon/execdriver/native/create.go @@ -4,7 +4,6 @@ package native import ( "errors" - "encoding/json" "fmt" "net" "strings" @@ -89,24 +88,6 @@ func generateIfaceName() (string, error) { return "", errors.New("Failed to find name for new interface") } -// Re-create the container type from the image that was saved during checkpoint. -func (d *driver) createRestoreContainer(c *execdriver.Command, imageDir string) (*libcontainer.Config, error) { - // Read the container.json. - f1, err := os.Open(filepath.Join(imageDir, "container.json")) - if err != nil { - return nil, err - } - defer f1.Close() - - var container *libcontainer.Config - err = json.NewDecoder(f1).Decode(&container) - if err != nil { - return nil, err - } - - return container, nil -} - func (d *driver) createNetwork(container *configs.Config, c *execdriver.Command) error { if c.Network.ContainerID != "" { d.Lock() diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 2f46786922d47..8e1bb6c53e1fa 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -19,7 +19,6 @@ import ( "github.com/docker/docker/pkg/reexec" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" - "github.com/docker/docker/utils" "github.com/docker/libcontainer" "github.com/docker/libcontainer/apparmor" "github.com/docker/libcontainer/cgroups/systemd" @@ -275,49 +274,15 @@ func (d *driver) Unpause(c *execdriver.Command) error { return active.Resume() } -// XXX Where is the right place for the following -// const and getCheckpointImageDir() function? -const ( - containersDir = "/var/lib/docker/containers" - criuImgDir = "criu_img" -) - -func getCheckpointImageDir(containerId string) string { - return filepath.Join(containersDir, containerId, criuImgDir) -} - -func (d *driver) Checkpoint(c *execdriver.Command) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { active := d.activeContainers[c.ID] if active == nil { return fmt.Errorf("active container for %s does not exist", c.ID) } - container := active.container - - // Create an image directory for this container (which - // may already exist from a previous checkpoint). - imageDir := getCheckpointImageDir(c.ID) - err := os.MkdirAll(imageDir, 0700) - if err != nil && !os.IsExist(err) { - return err - } - - // Copy container.json and state.json files to the CRIU - // image directory for later use during restore. Do this - // before checkpointing because after checkpoint the container - // will exit and these files will be removed. - log.CRDbg("saving container.json and state.json before calling CRIU in %s", imageDir) - srcFiles := []string{"container.json", "state.json"} - for _, f := range srcFiles { - srcFile := filepath.Join(d.root, c.ID, f) - dstFile := filepath.Join(imageDir, f) - if _, err := utils.CopyFile(srcFile, dstFile); err != nil { - return err - } - } d.Lock() defer d.Unlock() - err = namespaces.Checkpoint(container, imageDir, c.ProcessConfig.Process.Pid) + err := active.Checkpoint(opts) if err != nil { return err } @@ -325,103 +290,83 @@ func (d *driver) Checkpoint(c *execdriver.Command) error { return nil } -type restoreOutput struct { - exitCode int - err error -} +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { + var ( + cont libcontainer.Container + err error + ) -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { - imageDir := getCheckpointImageDir(c.ID) - container, err := d.createRestoreContainer(c, imageDir) + cont, err = d.factory.Load(c.ID) if err != nil { - return 1, err + if forceRestore { + var config *configs.Config + config, err = d.createContainer(c) + if err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err + } + cont, err = d.factory.Create(c.ID, config) + if err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err + } + } else { + return execdriver.ExitStatus{ExitCode: -1}, err + } } - var term execdriver.Terminal - - if c.ProcessConfig.Tty { - term, err = NewTtyConsole(&c.ProcessConfig, pipes) - } else { - term, err = execdriver.NewStdConsole(&c.ProcessConfig, pipes) + p := &libcontainer.Process{ + Args: append([]string{c.ProcessConfig.Entrypoint}, c.ProcessConfig.Arguments...), + Env: c.ProcessConfig.Env, + Cwd: c.WorkingDir, + User: c.ProcessConfig.User, } - if err != nil { - return -1, err + + config := cont.Config() + if err := setupPipes(&config, &c.ProcessConfig, p, pipes); err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err } - c.ProcessConfig.Terminal = term d.Lock() - d.activeContainers[c.ID] = &activeContainer{ - container: container, - cmd: &c.ProcessConfig.Cmd, - } + d.activeContainers[c.ID] = cont d.Unlock() - defer d.cleanContainer(c.ID) + defer func() { + cont.Destroy() + d.cleanContainer(c.ID) + }() - // Since the CRIU binary exits after restoring the container, we - // need to reap its child by setting PR_SET_CHILD_SUBREAPER (36) - // so that it'll be owned by this process (Docker daemon) after restore. - // - // XXX This really belongs to where the Docker daemon starts. - if _, _, syserr := syscall.RawSyscall(syscall.SYS_PRCTL, 36, 1, 0); syserr != 0 { - return -1, fmt.Errorf("Could not set PR_SET_CHILD_SUBREAPER (syserr %d)", syserr) + if err := cont.Restore(p, opts); err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err } - restoreOutputChan := make(chan restoreOutput, 1) - waitForRestore := make(chan struct{}) - - go func() { - exitCode, err := namespaces.Restore(container, c.ProcessConfig.Stdin, c.ProcessConfig.Stdout, c.ProcessConfig.Stderr, c.ProcessConfig.Console, filepath.Join(d.root, c.ID), imageDir, - func(child *os.File, args []string) *exec.Cmd { - cmd := new(exec.Cmd) - cmd.Path = d.initPath - cmd.Args = append([]string{ - DriverName, - "-restore", - "-pipe", "3", - "--", - }, args...) - cmd.ExtraFiles = []*os.File{child} - return cmd - }, - func(restorePid int) error { - log.CRDbg("restorePid=%d", restorePid) - if restorePid == 0 { - restoreCallback(&c.ProcessConfig, 0) - return nil - } - - // The container.json file should be written *after* the container - // has started because its StdFds cannot be initialized before. - // - // XXX How do we handle error here? - d.writeContainerFile(container, c.ID) - close(waitForRestore) - if restoreCallback != nil { - c.ProcessConfig.Process, err = os.FindProcess(restorePid) - if err != nil { - log.Debugf("cannot find restored process %d", restorePid) - return err - } - c.ContainerPid = c.ProcessConfig.Process.Pid - restoreCallback(&c.ProcessConfig, c.ContainerPid) - } - return nil - }) - restoreOutputChan <- restoreOutput{exitCode, err} - }() + // FIXME: no idea if any of this is needed... + if restoreCallback != nil { + pid, err := p.Pid() + if err != nil { + p.Signal(os.Kill) + p.Wait() + return execdriver.ExitStatus{ExitCode: -1}, err + } + restoreCallback(&c.ProcessConfig, pid) + } - select { - case restoreOutput := <-restoreOutputChan: - // there was an error - return restoreOutput.exitCode, restoreOutput.err - case <-waitForRestore: - // container restored - break + oom := notifyOnOOM(cont) + waitF := p.Wait + if nss := cont.Config().Namespaces; !nss.Contains(configs.NEWPID) { + // we need such hack for tracking processes with inherited fds, + // because cmd.Wait() waiting for all streams to be copied + waitF = waitInPIDHost(p, cont) + } + ps, err := waitF() + if err != nil { + execErr, ok := err.(*exec.ExitError) + if !ok { + return execdriver.ExitStatus{ExitCode: -1}, err + } + ps = execErr.ProcessState } - // Wait for the container to exit. - restoreOutput := <-restoreOutputChan - return restoreOutput.exitCode, restoreOutput.err + cont.Destroy() + _, oomKill := <-oom + return execdriver.ExitStatus{ExitCode: utils.ExitStatus(ps.Sys().(syscall.WaitStatus)), OOMKilled: oomKill}, nil } func (d *driver) Terminate(c *execdriver.Command) error { From 5b85c9a78f6e4249bd08623fe42b1208f7cc6425 Mon Sep 17 00:00:00 2001 From: Saied Kazemi Date: Thu, 5 Feb 2015 20:37:07 -0800 Subject: [PATCH 3/5] Checkpoint/Restore Support: add functionality to daemon Support was added to the daemon to use the Checkpoint and Restore methods of the native exec driver for checkpointing and restoring containers. Signed-off-by: Saied Kazemi Conflicts: api/server/server.go daemon/container.go daemon/daemon.go daemon/networkdriver/bridge/driver.go daemon/state.go vendor/src/github.com/docker/libnetwork/ipallocator/allocator.go --- api/server/server.go | 76 +++++++++++++++++++++++++++------------ daemon/checkpoint.go | 55 ++++++++++++++++++++++++++++ daemon/container.go | 66 ++++++++++++++++++++++++++++++++-- daemon/container_linux.go | 48 +++++++++++++++++++++++++ daemon/daemon.go | 31 ++++++++++++++++ daemon/monitor.go | 70 ++++++++++++++++++++++++++++++++++++ daemon/state.go | 23 ++++++++++++ 7 files changed, 345 insertions(+), 24 deletions(-) create mode 100644 daemon/checkpoint.go diff --git a/api/server/server.go b/api/server/server.go index 8d2dafa74004b..6cdb398144135 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -1286,6 +1286,36 @@ func (s *Server) postContainersCopy(version version.Version, w http.ResponseWrit return nil } +func postContainersCheckpoint(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + if err := parseForm(r); err != nil { + return err + } + job := eng.Job("checkpoint", vars["name"]) + if err := job.Run(); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) + return nil +} + +func postContainersRestore(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + if err := parseForm(r); err != nil { + return err + } + job := eng.Job("restore", vars["name"]) + if err := job.Run(); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) + return nil +} + func (s *Server) postContainerExecCreate(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if err := parseForm(r); err != nil { return nil @@ -1488,28 +1518,30 @@ func createRouter(s *Server) *mux.Router { "/exec/{id:.*}/json": s.getExecByID, }, "POST": { - "/auth": s.postAuth, - "/commit": s.postCommit, - "/build": s.postBuild, - "/images/create": s.postImagesCreate, - "/images/load": s.postImagesLoad, - "/images/{name:.*}/push": s.postImagesPush, - "/images/{name:.*}/tag": s.postImagesTag, - "/containers/create": s.postContainersCreate, - "/containers/{name:.*}/kill": s.postContainersKill, - "/containers/{name:.*}/pause": s.postContainersPause, - "/containers/{name:.*}/unpause": s.postContainersUnpause, - "/containers/{name:.*}/restart": s.postContainersRestart, - "/containers/{name:.*}/start": s.postContainersStart, - "/containers/{name:.*}/stop": s.postContainersStop, - "/containers/{name:.*}/wait": s.postContainersWait, - "/containers/{name:.*}/resize": s.postContainersResize, - "/containers/{name:.*}/attach": s.postContainersAttach, - "/containers/{name:.*}/copy": s.postContainersCopy, - "/containers/{name:.*}/exec": s.postContainerExecCreate, - "/exec/{name:.*}/start": s.postContainerExecStart, - "/exec/{name:.*}/resize": s.postContainerExecResize, - "/containers/{name:.*}/rename": s.postContainerRename, + "/auth": s.postAuth, + "/commit": s.postCommit, + "/build": s.postBuild, + "/images/create": s.postImagesCreate, + "/images/load": s.postImagesLoad, + "/images/{name:.*}/push": s.postImagesPush, + "/images/{name:.*}/tag": s.postImagesTag, + "/containers/create": s.postContainersCreate, + "/containers/{name:.*}/kill": s.postContainersKill, + "/containers/{name:.*}/pause": s.postContainersPause, + "/containers/{name:.*}/unpause": s.postContainersUnpause, + "/containers/{name:.*}/restart": s.postContainersRestart, + "/containers/{name:.*}/start": s.postContainersStart, + "/containers/{name:.*}/stop": s.postContainersStop, + "/containers/{name:.*}/wait": s.postContainersWait, + "/containers/{name:.*}/resize": s.postContainersResize, + "/containers/{name:.*}/attach": s.postContainersAttach, + "/containers/{name:.*}/copy": s.postContainersCopy, + "/containers/{name:.*}/exec": s.postContainerExecCreate, + "/exec/{name:.*}/start": s.postContainerExecStart, + "/exec/{name:.*}/resize": s.postContainerExecResize, + "/containers/{name:.*}/rename": s.postContainerRename, + "/containers/{name:.*}/checkpoint": s.postContainersCheckpoint, + "/containers/{name:.*}/restore": s.postContainersRestore, }, "DELETE": { "/containers/{name:.*}": s.deleteContainers, diff --git a/daemon/checkpoint.go b/daemon/checkpoint.go new file mode 100644 index 0000000000000..f6057c6a028f9 --- /dev/null +++ b/daemon/checkpoint.go @@ -0,0 +1,55 @@ +package daemon + +import ( + "github.com/docker/docker/engine" +) + +// Checkpoint a running container. +func (daemon *Daemon) ContainerCheckpoint(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + return job.Errorf("Usage: %s CONTAINER\n", job.Name) + } + + name := job.Args[0] + container, err := daemon.Get(name) + if err != nil { + return job.Error(err) + } + if !container.IsRunning() { + return job.Errorf("Container %s not running", name) + } + + if err := container.Checkpoint(); err != nil { + return job.Errorf("Cannot checkpoint container %s: %s", name, err) + } + + container.LogEvent("checkpoint") + return engine.StatusOK +} + +// Restore a checkpointed container. +func (daemon *Daemon) ContainerRestore(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + return job.Errorf("Usage: %s CONTAINER\n", job.Name) + } + + name := job.Args[0] + container, err := daemon.Get(name) + if err != nil { + return job.Error(err) + } + if container.IsRunning() { + return job.Errorf("Container %s already running", name) + } + if !container.State.IsCheckpointed() { + return job.Errorf("Container %s is not checkpointed", name) + } + + if err := container.Restore(); err != nil { + container.LogEvent("die") + return job.Errorf("Cannot restore container %s: %s", name, err) + } + + container.LogEvent("restore") + return engine.StatusOK +} diff --git a/daemon/container.go b/daemon/container.go index 7a59bec5d78db..11b20b7736576 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -339,10 +339,15 @@ func (container *Container) isNetworkAllocated() bool { return container.NetworkSettings.IPAddress != "" } + // cleanup releases any network resources allocated to the container along with any rules // around how containers are linked together. It also unmounts the container's root filesystem. func (container *Container) cleanup() { - container.ReleaseNetwork() + if container.IsCheckpointed() { + log.CRDbg("not calling ReleaseNetwork() for checkpointed container %s", container.ID) + } else { + container.ReleaseNetwork() + } disableAllActiveLinks(container) @@ -623,6 +628,41 @@ func (container *Container) Copy(resource string) (io.ReadCloser, error) { nil } +func (container *Container) Checkpoint() error { + return container.daemon.Checkpoint(container) +} + +func (container *Container) Restore() error { + var err error + + container.Lock() + defer container.Unlock() + + defer func() { + if err != nil { + container.cleanup() + } + }() + + if err = container.initializeNetworking(); err != nil { + return err + } + + linkedEnv, err := container.setupLinkedContainers() + if err != nil { + return err + } + if err = container.setupWorkingDirectory(); err != nil { + return err + } + env := container.createDaemonEnvironment(linkedEnv) + if err = populateCommandRestore(container, env); err != nil { + return err + } + + return container.waitForRestore() +} + // Returns true if the container exposes a certain port func (container *Container) Exposes(p nat.Port) bool { _, exists := container.Config.ExposedPorts[p] @@ -709,6 +749,29 @@ func (container *Container) waitForStart() error { return nil } +// Like waitForStart() but for restoring a container. +// +// XXX Does RestartPolicy apply here? +func (container *Container) waitForRestore() error { + container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) + + // After calling promise.Go() we'll have two goroutines: + // - The current goroutine that will block in the select + // below until restore is done. + // - A new goroutine that will restore the container and + // wait for it to exit. + select { + case <-container.monitor.restoreSignal: + if container.ExitCode != 0 { + return fmt.Errorf("restore process failed") + } + case err := <-promise.Go(container.monitor.Restore): + return err + } + + return nil +} + func (container *Container) GetProcessLabel() string { // even if we have a process label return "" if we are running // in privileged mode @@ -913,7 +976,6 @@ func attach(streamConfig *StreamConfig, openStdin, stdinOnce, tty bool, stdin io _, err = copyEscapable(cStdin, stdin) } else { _, err = io.Copy(cStdin, stdin) - } if err == io.ErrClosedPipe { err = nil diff --git a/daemon/container_linux.go b/daemon/container_linux.go index 72e840927525a..b12bebb8aec62 100644 --- a/daemon/container_linux.go +++ b/daemon/container_linux.go @@ -313,6 +313,54 @@ func populateCommand(c *Container, env []string) error { return nil } + +// Like populateCommand() but for restoring a container. +// +// XXX populateCommand() does a lot more. Not sure if we have +// to do everything it does. +func populateCommandRestore(c *Container, env []string) error { + resources := &execdriver.Resources{ + Memory: c.Config.Memory, + MemorySwap: c.Config.MemorySwap, + CpuShares: c.Config.CpuShares, + Cpuset: c.Config.Cpuset, + } + + processConfig := execdriver.ProcessConfig{ + Privileged: c.hostConfig.Privileged, + Entrypoint: c.Path, + Arguments: c.Args, + Tty: c.Config.Tty, + User: c.Config.User, + } + + processConfig.SysProcAttr = &syscall.SysProcAttr{Setsid: true} + processConfig.Env = env + + c.command = &execdriver.Command{ + ID: c.ID, + Rootfs: c.RootfsPath(), + ReadonlyRootfs: c.hostConfig.ReadonlyRootfs, + InitPath: "/.dockerinit", + WorkingDir: c.Config.WorkingDir, + // Network: en, + // Ipc: ipc, + // Pid: pid, + Resources: resources, + // AllowedDevices: allowedDevices, + // AutoCreatedDevices: autoCreatedDevices, + CapAdd: c.hostConfig.CapAdd, + CapDrop: c.hostConfig.CapDrop, + ProcessConfig: processConfig, + ProcessLabel: c.GetProcessLabel(), + MountLabel: c.GetMountLabel(), + // LxcConfig: lxcConfig, + AppArmorProfile: c.AppArmorProfile, + } + + return nil +} + // GetSize, return real size, virtual size func (container *Container) GetSize() (int64, int64) { var ( diff --git a/daemon/daemon.go b/daemon/daemon.go index 96c4e9c7365b9..fcbafeb357d5d 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -279,6 +279,18 @@ func (daemon *Daemon) restore() error { logrus.Debugf("Loaded container %v", container.ID) containers[container.ID] = container + + // If the container was checkpointed, we need to reserve + // the IP address that it was using. + // + // XXX We should also reserve host ports (if any). + if container.IsCheckpointed() { + /*err = bridge.ReserveIP(container.ID, container.NetworkSettings.IPAddress) + if err != nil { + log.Errorf("Failed to reserve IP %s for container %s", + container.ID, container.NetworkSettings.IPAddress) + }*/ + } } else { logrus.Debugf("Cannot load container %s because it was created with another graph driver.", container.ID) } @@ -1048,6 +1060,25 @@ func (daemon *Daemon) Run(c *Container, pipes *execdriver.Pipes, startCallback e return daemon.execDriver.Run(c.command, pipes, startCallback) } +func (daemon *Daemon) Checkpoint(c *Container) error { + if err := daemon.execDriver.Checkpoint(c.command); err != nil { + return err + } + c.SetCheckpointed() + return nil +} + +func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + // Mount the container's filesystem (daemon/graphdriver/aufs/aufs.go). + _, err := daemon.driver.Get(c.ID, c.GetMountLabel()) + if err != nil { + return 0, err + } + + exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback) + return exitCode, err +} + func (daemon *Daemon) Kill(c *Container, sig int) error { return daemon.execDriver.Kill(c.command, sig) } diff --git a/daemon/monitor.go b/daemon/monitor.go index dfade8e21847b..241efa32d0eb3 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -44,6 +44,9 @@ type containerMonitor struct { // left waiting for nothing to happen during this time stopChan chan struct{} + // like startSignal but for restoring a container + restoreSignal chan struct{} + // timeIncrement is the amount of time to wait between restarts // this is in milliseconds timeIncrement int @@ -61,6 +64,7 @@ func newContainerMonitor(container *Container, policy runconfig.RestartPolicy) * timeIncrement: defaultTimeIncrement, stopChan: make(chan struct{}), startSignal: make(chan struct{}), + restoreSignal: make(chan struct{}), } } @@ -181,6 +185,49 @@ func (m *containerMonitor) Start() error { } } +// Like Start() but for restoring a container. +func (m *containerMonitor) Restore() error { + var ( + err error + // XXX The following line should be changed to + // exitStatus execdriver.ExitStatus to match Start() + exitCode int + afterRestore bool + ) + + defer func() { + if afterRestore { + m.container.Lock() + m.container.setStopped(&execdriver.ExitStatus{exitCode, false}) + defer m.container.Unlock() + } + m.Close() + }() + + if err := m.container.startLoggingToDisk(); err != nil { + m.resetContainer(false) + return err + } + + pipes := execdriver.NewPipes(m.container.stdin, m.container.stdout, m.container.stderr, m.container.Config.OpenStdin) + + m.container.LogEvent("restore") + m.lastStartTime = time.Now() + if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback); err != nil { + log.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) + m.container.ExitCode = -1 + m.resetContainer(false) + return err + } + afterRestore = true + + m.container.ExitCode = exitCode + m.resetMonitor(err == nil && exitCode == 0) + m.container.LogEvent("die") + m.resetContainer(true) + return err +} + // resetMonitor resets the stateful fields on the containerMonitor based on the // previous runs success or failure. Regardless of success, if the container had // an execution time of more than 10s then reset the timer back to the default @@ -267,6 +314,29 @@ func (m *containerMonitor) callback(processConfig *execdriver.ProcessConfig, pid } } +// Like callback() but for restoring a container. +func (m *containerMonitor) restoreCallback(processConfig *execdriver.ProcessConfig, restorePid int) { + // If restorePid is 0, it means that restore failed. + if restorePid != 0 { + m.container.setRunning(restorePid) + } + + // Unblock the goroutine waiting in waitForRestore(). + select { + case <-m.restoreSignal: + default: + close(m.restoreSignal) + } + + if restorePid != 0 { + // Write config.json and hostconfig.json files + // to /var/lib/docker/containers/. + if err := m.container.ToDisk(); err != nil { + log.Debugf("%s", err) + } + } +} + // resetContainer resets the container's IO and ensures that the command is able to be executed again // by copying the data into a new struct // if lock is true, then container locked during reset diff --git a/daemon/state.go b/daemon/state.go index 4119d0e6cd924..9837cc5b181d0 100644 --- a/daemon/state.go +++ b/daemon/state.go @@ -14,6 +14,7 @@ type State struct { Running bool Paused bool Restarting bool + Checkpointed bool OOMKilled bool removalInProgress bool // Not need for this to be persistent on disk. Dead bool @@ -22,7 +23,9 @@ type State struct { Error string // contains last known error when starting the container StartedAt time.Time FinishedAt time.Time + CheckpointedAt time.Time waitChan chan struct{} + } func NewState() *State { @@ -42,6 +45,8 @@ func (s *State) String() string { } return fmt.Sprintf("Up %s", units.HumanDuration(time.Now().UTC().Sub(s.StartedAt))) + } else if s.Checkpointed { + return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) } if s.removalInProgress { @@ -158,6 +163,7 @@ func (s *State) setRunning(pid int) { s.Error = "" s.Running = true s.Paused = false + s.Checkpointed = false s.Restarting = false s.ExitCode = 0 s.Pid = pid @@ -254,3 +260,20 @@ func (s *State) SetDead() { s.Dead = true s.Unlock() } + +func (s *State) SetCheckpointed() { + s.Lock() + s.CheckpointedAt = time.Now().UTC() + s.Checkpointed = true + s.Running = false + s.Paused = false + s.Restarting = false + // XXX Not sure if we need to close and recreate waitChan. + // close(s.waitChan) + // s.waitChan = make(chan struct{}) + s.Unlock() +} + +func (s *State) IsCheckpointed() bool { + return s.Checkpointed +} From d17726ed268fd5c3ed62bbb61c016c83f913f3f2 Mon Sep 17 00:00:00 2001 From: Hui Kang Date: Tue, 19 May 2015 21:08:04 +0000 Subject: [PATCH 4/5] Release the network resource during checkpoint Restore failed if network resource not released during checkpoint, e.g., a container with port open with -p Signed-off-by: Hui Kang Conflicts: daemon/container.go --- daemon/container.go | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/daemon/container.go b/daemon/container.go index 11b20b7736576..4117557890555 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -569,6 +569,19 @@ func validateID(id string) error { return nil } + +func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { + if err := container.daemon.Checkpoint(container, opts); err != nil { + return err + } + + if opts.LeaveRunning == false { + container.ReleaseNetwork() + } + return nil +} + + func (container *Container) Copy(resource string) (io.ReadCloser, error) { container.Lock() defer container.Unlock() From 5a1f29863b9d86dfd52cf89d396b086c685c1f5d Mon Sep 17 00:00:00 2001 From: boucher Date: Thu, 28 May 2015 10:09:58 -0700 Subject: [PATCH 5/5] Update checkpoint/restore support in the daemon. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- api/server/server.go | 25 ++++++++--- daemon/checkpoint.go | 57 ++++++++++++------------- daemon/container.go | 87 +++++++++++++++++++-------------------- daemon/container_linux.go | 66 ++++++----------------------- daemon/daemon.go | 13 +++--- daemon/monitor.go | 27 ++++++------ daemon/state.go | 21 +++++++--- docker/flags.go | 2 + runconfig/restore.go | 10 +++++ 9 files changed, 153 insertions(+), 155 deletions(-) create mode 100644 runconfig/restore.go diff --git a/api/server/server.go b/api/server/server.go index 6cdb398144135..c1eb40252e27d 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -36,6 +36,7 @@ import ( "github.com/docker/docker/pkg/version" "github.com/docker/docker/runconfig" "github.com/docker/docker/utils" + "github.com/docker/libcontainer" "github.com/docker/libnetwork/portallocator" ) @@ -1286,32 +1287,44 @@ func (s *Server) postContainersCopy(version version.Version, w http.ResponseWrit return nil } -func postContainersCheckpoint(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { +func (s *Server) postContainersCheckpoint(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if vars == nil { return fmt.Errorf("Missing parameter") } if err := parseForm(r); err != nil { return err } - job := eng.Job("checkpoint", vars["name"]) - if err := job.Run(); err != nil { + + criuOpts := &libcontainer.CriuOpts{} + if err := json.NewDecoder(r.Body).Decode(criuOpts); err != nil { return err } + + if err := s.daemon.ContainerCheckpoint(vars["name"], criuOpts); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) return nil } -func postContainersRestore(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { +func (s *Server) postContainersRestore(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if vars == nil { return fmt.Errorf("Missing parameter") } if err := parseForm(r); err != nil { return err } - job := eng.Job("restore", vars["name"]) - if err := job.Run(); err != nil { + + restoreOpts := runconfig.RestoreConfig{} + if err := json.NewDecoder(r.Body).Decode(&restoreOpts); err != nil { return err } + + if err := s.daemon.ContainerRestore(vars["name"], &restoreOpts.CriuOpts, restoreOpts.ForceRestore); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) return nil } diff --git a/daemon/checkpoint.go b/daemon/checkpoint.go index f6057c6a028f9..1d98d3222c787 100644 --- a/daemon/checkpoint.go +++ b/daemon/checkpoint.go @@ -1,55 +1,56 @@ package daemon import ( - "github.com/docker/docker/engine" + "fmt" + + "github.com/docker/libcontainer" ) // Checkpoint a running container. -func (daemon *Daemon) ContainerCheckpoint(job *engine.Job) engine.Status { - if len(job.Args) != 1 { - return job.Errorf("Usage: %s CONTAINER\n", job.Name) - } - - name := job.Args[0] +func (daemon *Daemon) ContainerCheckpoint(name string, opts *libcontainer.CriuOpts) error { container, err := daemon.Get(name) if err != nil { - return job.Error(err) + return err } if !container.IsRunning() { - return job.Errorf("Container %s not running", name) + return fmt.Errorf("Container %s not running", name) } - - if err := container.Checkpoint(); err != nil { - return job.Errorf("Cannot checkpoint container %s: %s", name, err) + if err := container.Checkpoint(opts); err != nil { + return fmt.Errorf("Cannot checkpoint container %s: %s", name, err) } container.LogEvent("checkpoint") - return engine.StatusOK + return nil } // Restore a checkpointed container. -func (daemon *Daemon) ContainerRestore(job *engine.Job) engine.Status { - if len(job.Args) != 1 { - return job.Errorf("Usage: %s CONTAINER\n", job.Name) - } - - name := job.Args[0] +func (daemon *Daemon) ContainerRestore(name string, opts *libcontainer.CriuOpts, forceRestore bool) error { container, err := daemon.Get(name) if err != nil { - return job.Error(err) - } - if container.IsRunning() { - return job.Errorf("Container %s already running", name) + return err } - if !container.State.IsCheckpointed() { - return job.Errorf("Container %s is not checkpointed", name) + + if !forceRestore { + // TODO: It's possible we only want to bypass the checkpointed check, + // I'm not sure how this will work if the container is already running + if container.IsRunning() { + return fmt.Errorf("Container %s already running", name) + } + + if !container.IsCheckpointed() { + return fmt.Errorf("Container %s is not checkpointed", name) + } + } else { + if !container.HasBeenCheckpointed() && opts.ImagesDirectory == "" { + return fmt.Errorf("You must specify an image directory to restore from %s", name) + } } - if err := container.Restore(); err != nil { + if err = container.Restore(opts, forceRestore); err != nil { container.LogEvent("die") - return job.Errorf("Cannot restore container %s: %s", name, err) + return fmt.Errorf("Cannot restore container %s: %s", name, err) } container.LogEvent("restore") - return engine.StatusOK + return nil } diff --git a/daemon/container.go b/daemon/container.go index 4117557890555..53a2886f2547e 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -13,6 +13,7 @@ import ( "syscall" "time" + "github.com/docker/libcontainer" "github.com/docker/libcontainer/label" "github.com/Sirupsen/logrus" @@ -255,7 +256,7 @@ func (container *Container) Start() (err error) { if err := container.Mount(); err != nil { return err } - if err := container.initializeNetworking(); err != nil { + if err := container.initializeNetworking(false); err != nil { return err } container.verifyDaemonSettings() @@ -339,12 +340,11 @@ func (container *Container) isNetworkAllocated() bool { return container.NetworkSettings.IPAddress != "" } - // cleanup releases any network resources allocated to the container along with any rules // around how containers are linked together. It also unmounts the container's root filesystem. func (container *Container) cleanup() { if container.IsCheckpointed() { - log.CRDbg("not calling ReleaseNetwork() for checkpointed container %s", container.ID) + logrus.Debugf("not calling ReleaseNetwork() for checkpointed container %s", container.ID) } else { container.ReleaseNetwork() } @@ -569,7 +569,6 @@ func validateID(id string) error { return nil } - func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { if err := container.daemon.Checkpoint(container, opts); err != nil { return err @@ -581,6 +580,43 @@ func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { return nil } +func (container *Container) Restore(opts *libcontainer.CriuOpts, forceRestore bool) error { + var err error + container.Lock() + defer container.Unlock() + + defer func() { + if err != nil { + container.cleanup() + } + }() + if err := container.Mount(); err != nil { + return err + } + if err = container.initializeNetworking(true); err != nil { + return err + } + container.verifyDaemonSettings() + + linkedEnv, err := container.setupLinkedContainers() + if err != nil { + return err + } + if err = container.setupWorkingDirectory(); err != nil { + return err + } + + env := container.createDaemonEnvironment(linkedEnv) + if err = populateCommand(container, env); err != nil { + return err + } + + if err = container.setupMounts(); err != nil { + return err + } + + return container.waitForRestore(opts, forceRestore) +} func (container *Container) Copy(resource string) (io.ReadCloser, error) { container.Lock() @@ -641,41 +677,6 @@ func (container *Container) Copy(resource string) (io.ReadCloser, error) { nil } -func (container *Container) Checkpoint() error { - return container.daemon.Checkpoint(container) -} - -func (container *Container) Restore() error { - var err error - - container.Lock() - defer container.Unlock() - - defer func() { - if err != nil { - container.cleanup() - } - }() - - if err = container.initializeNetworking(); err != nil { - return err - } - - linkedEnv, err := container.setupLinkedContainers() - if err != nil { - return err - } - if err = container.setupWorkingDirectory(); err != nil { - return err - } - env := container.createDaemonEnvironment(linkedEnv) - if err = populateCommandRestore(container, env); err != nil { - return err - } - - return container.waitForRestore() -} - // Returns true if the container exposes a certain port func (container *Container) Exposes(p nat.Port) bool { _, exists := container.Config.ExposedPorts[p] @@ -762,10 +763,7 @@ func (container *Container) waitForStart() error { return nil } -// Like waitForStart() but for restoring a container. -// -// XXX Does RestartPolicy apply here? -func (container *Container) waitForRestore() error { +func (container *Container) waitForRestore(opts *libcontainer.CriuOpts, forceRestore bool) error { container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) // After calling promise.Go() we'll have two goroutines: @@ -778,7 +776,7 @@ func (container *Container) waitForRestore() error { if container.ExitCode != 0 { return fmt.Errorf("restore process failed") } - case err := <-promise.Go(container.monitor.Restore): + case err := <-promise.Go(func() error { return container.monitor.Restore(opts, forceRestore) }): return err } @@ -989,6 +987,7 @@ func attach(streamConfig *StreamConfig, openStdin, stdinOnce, tty bool, stdin io _, err = copyEscapable(cStdin, stdin) } else { _, err = io.Copy(cStdin, stdin) + } if err == io.ErrClosedPipe { err = nil diff --git a/daemon/container_linux.go b/daemon/container_linux.go index b12bebb8aec62..52b44b879ebe6 100644 --- a/daemon/container_linux.go +++ b/daemon/container_linux.go @@ -313,54 +313,6 @@ func populateCommand(c *Container, env []string) error { return nil } - -// Like populateCommand() but for restoring a container. -// -// XXX populateCommand() does a lot more. Not sure if we have -// to do everything it does. -func populateCommandRestore(c *Container, env []string) error { - resources := &execdriver.Resources{ - Memory: c.Config.Memory, - MemorySwap: c.Config.MemorySwap, - CpuShares: c.Config.CpuShares, - Cpuset: c.Config.Cpuset, - } - - processConfig := execdriver.ProcessConfig{ - Privileged: c.hostConfig.Privileged, - Entrypoint: c.Path, - Arguments: c.Args, - Tty: c.Config.Tty, - User: c.Config.User, - } - - processConfig.SysProcAttr = &syscall.SysProcAttr{Setsid: true} - processConfig.Env = env - - c.command = &execdriver.Command{ - ID: c.ID, - Rootfs: c.RootfsPath(), - ReadonlyRootfs: c.hostConfig.ReadonlyRootfs, - InitPath: "/.dockerinit", - WorkingDir: c.Config.WorkingDir, - // Network: en, - // Ipc: ipc, - // Pid: pid, - Resources: resources, - // AllowedDevices: allowedDevices, - // AutoCreatedDevices: autoCreatedDevices, - CapAdd: c.hostConfig.CapAdd, - CapDrop: c.hostConfig.CapDrop, - ProcessConfig: processConfig, - ProcessLabel: c.GetProcessLabel(), - MountLabel: c.GetMountLabel(), - // LxcConfig: lxcConfig, - AppArmorProfile: c.AppArmorProfile, - } - - return nil -} - // GetSize, return real size, virtual size func (container *Container) GetSize() (int64, int64) { var ( @@ -685,7 +637,7 @@ func (container *Container) UpdateNetwork() error { return nil } -func (container *Container) buildCreateEndpointOptions() ([]libnetwork.EndpointOption, error) { +func (container *Container) buildCreateEndpointOptions(restoring bool) ([]libnetwork.EndpointOption, error) { var ( portSpecs = make(nat.PortSet) bindings = make(nat.PortMap) @@ -766,10 +718,18 @@ func (container *Container) buildCreateEndpointOptions() ([]libnetwork.EndpointO createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(genericOption)) } + /*if restoring && container.NetworkSettings.IPAddress != "" { + genericOption := options.Generic{ + netlabel.IPAddress: net.ParseIP(container.NetworkSettings.IPAddress), + } + + createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(genericOption)) + }*/ + return createOptions, nil } -func (container *Container) AllocateNetwork() error { +func (container *Container) AllocateNetwork(restoring bool) error { mode := container.hostConfig.NetworkMode if container.Config.NetworkDisabled || mode.IsContainer() { return nil @@ -782,7 +742,7 @@ func (container *Container) AllocateNetwork() error { return fmt.Errorf("error locating network with name %s: %v", string(mode), err) } - createOptions, err := container.buildCreateEndpointOptions() + createOptions, err := container.buildCreateEndpointOptions(restoring) if err != nil { return err } @@ -816,7 +776,7 @@ func (container *Container) AllocateNetwork() error { return nil } -func (container *Container) initializeNetworking() error { +func (container *Container) initializeNetworking(restoring bool) error { var err error // Make sure NetworkMode has an acceptable value before @@ -857,7 +817,7 @@ func (container *Container) initializeNetworking() error { } - if err := container.AllocateNetwork(); err != nil { + if err := container.AllocateNetwork(restoring); err != nil { return err } diff --git a/daemon/daemon.go b/daemon/daemon.go index fcbafeb357d5d..43b47585e7bcd 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -14,6 +14,7 @@ import ( "sync" "time" + "github.com/docker/libcontainer" "github.com/docker/libcontainer/label" "github.com/docker/libnetwork" "github.com/docker/libnetwork/netlabel" @@ -1060,22 +1061,22 @@ func (daemon *Daemon) Run(c *Container, pipes *execdriver.Pipes, startCallback e return daemon.execDriver.Run(c.command, pipes, startCallback) } -func (daemon *Daemon) Checkpoint(c *Container) error { - if err := daemon.execDriver.Checkpoint(c.command); err != nil { +func (daemon *Daemon) Checkpoint(c *Container, opts *libcontainer.CriuOpts) error { + if err := daemon.execDriver.Checkpoint(c.command, opts); err != nil { return err } - c.SetCheckpointed() + c.SetCheckpointed(opts.LeaveRunning) return nil } -func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { +func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { // Mount the container's filesystem (daemon/graphdriver/aufs/aufs.go). _, err := daemon.driver.Get(c.ID, c.GetMountLabel()) if err != nil { - return 0, err + return execdriver.ExitStatus{ExitCode: 0}, err } - exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback) + exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback, opts, forceRestore) return exitCode, err } diff --git a/daemon/monitor.go b/daemon/monitor.go index 241efa32d0eb3..42682ee6ec04d 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -10,6 +10,7 @@ import ( "github.com/docker/docker/daemon/execdriver" "github.com/docker/docker/pkg/stringid" "github.com/docker/docker/runconfig" + "github.com/docker/libcontainer" ) const defaultTimeIncrement = 100 @@ -186,43 +187,45 @@ func (m *containerMonitor) Start() error { } // Like Start() but for restoring a container. -func (m *containerMonitor) Restore() error { +func (m *containerMonitor) Restore(opts *libcontainer.CriuOpts, forceRestore bool) error { var ( err error // XXX The following line should be changed to // exitStatus execdriver.ExitStatus to match Start() - exitCode int + exitCode execdriver.ExitStatus afterRestore bool ) - defer func() { if afterRestore { m.container.Lock() - m.container.setStopped(&execdriver.ExitStatus{exitCode, false}) + m.container.setStopped(&execdriver.ExitStatus{exitCode.ExitCode, false}) defer m.container.Unlock() } m.Close() }() - if err := m.container.startLoggingToDisk(); err != nil { - m.resetContainer(false) - return err + // FIXME: right now if we startLogging again we get double logs after a restore + if m.container.logCopier == nil { + if err := m.container.startLogging(); err != nil { + m.resetContainer(false) + return err + } } pipes := execdriver.NewPipes(m.container.stdin, m.container.stdout, m.container.stderr, m.container.Config.OpenStdin) m.container.LogEvent("restore") m.lastStartTime = time.Now() - if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback); err != nil { - log.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) + if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback, opts, forceRestore); err != nil { + logrus.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) m.container.ExitCode = -1 m.resetContainer(false) return err } afterRestore = true - m.container.ExitCode = exitCode - m.resetMonitor(err == nil && exitCode == 0) + m.container.ExitCode = exitCode.ExitCode + m.resetMonitor(err == nil && exitCode.ExitCode == 0) m.container.LogEvent("die") m.resetContainer(true) return err @@ -332,7 +335,7 @@ func (m *containerMonitor) restoreCallback(processConfig *execdriver.ProcessConf // Write config.json and hostconfig.json files // to /var/lib/docker/containers/. if err := m.container.ToDisk(); err != nil { - log.Debugf("%s", err) + logrus.Debugf("%s", err) } } } diff --git a/daemon/state.go b/daemon/state.go index 9837cc5b181d0..aca36a5815b85 100644 --- a/daemon/state.go +++ b/daemon/state.go @@ -25,7 +25,6 @@ type State struct { FinishedAt time.Time CheckpointedAt time.Time waitChan chan struct{} - } func NewState() *State { @@ -45,14 +44,16 @@ func (s *State) String() string { } return fmt.Sprintf("Up %s", units.HumanDuration(time.Now().UTC().Sub(s.StartedAt))) - } else if s.Checkpointed { - return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) } if s.removalInProgress { return "Removal In Progress" } + if s.Checkpointed { + return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) + } + if s.Dead { return "Dead" } @@ -76,6 +77,10 @@ func (s *State) StateString() string { return "running" } + if s.Checkpointed { + return "checkpointed'" + } + if s.Dead { return "dead" } @@ -261,11 +266,11 @@ func (s *State) SetDead() { s.Unlock() } -func (s *State) SetCheckpointed() { +func (s *State) SetCheckpointed(leaveRunning bool) { s.Lock() s.CheckpointedAt = time.Now().UTC() - s.Checkpointed = true - s.Running = false + s.Checkpointed = !leaveRunning + s.Running = leaveRunning s.Paused = false s.Restarting = false // XXX Not sure if we need to close and recreate waitChan. @@ -274,6 +279,10 @@ func (s *State) SetCheckpointed() { s.Unlock() } +func (s *State) HasBeenCheckpointed() bool { + return s.CheckpointedAt != time.Time{} +} + func (s *State) IsCheckpointed() bool { return s.Checkpointed } diff --git a/docker/flags.go b/docker/flags.go index cbdb6a859deb5..fcf3f77db41b9 100644 --- a/docker/flags.go +++ b/docker/flags.go @@ -30,6 +30,7 @@ var ( dockerCommands = []command{ {"attach", "Attach to a running container"}, {"build", "Build an image from a Dockerfile"}, + {"checkpoint", "Checkpoint one or more running containers"}, {"commit", "Create a new image from a container's changes"}, {"cp", "Copy files/folders from a container's filesystem to the host path"}, {"create", "Create a new container"}, @@ -54,6 +55,7 @@ var ( {"push", "Push an image or a repository to a Docker registry server"}, {"rename", "Rename an existing container"}, {"restart", "Restart a running container"}, + {"restore", "Restore one or more checkpointed containers"}, {"rm", "Remove one or more containers"}, {"rmi", "Remove one or more images"}, {"run", "Run a command in a new container"}, diff --git a/runconfig/restore.go b/runconfig/restore.go new file mode 100644 index 0000000000000..18749b67e766f --- /dev/null +++ b/runconfig/restore.go @@ -0,0 +1,10 @@ +package runconfig + +import ( + "github.com/docker/libcontainer" +) + +type RestoreConfig struct { + CriuOpts libcontainer.CriuOpts + ForceRestore bool +}