From 1d2f53b28fefd31aed0311488b789988453196a8 Mon Sep 17 00:00:00 2001 From: Saied Kazemi Date: Thu, 5 Feb 2015 20:32:27 -0800 Subject: [PATCH 01/16] Checkpoint/Restore Support: add exec driver methods Methods for checkpointing and restoring containers were added to the native driver. The LXC driver returns an error message that these methods are not implemented yet. Signed-off-by: Saied Kazemi Conflicts: daemon/execdriver/native/create.go daemon/execdriver/native/driver.go daemon/execdriver/native/init.go --- daemon/execdriver/driver.go | 5 + daemon/execdriver/lxc/driver.go | 9 +- daemon/execdriver/native/create.go | 19 ++++ daemon/execdriver/native/driver.go | 150 +++++++++++++++++++++++++++++ 4 files changed, 182 insertions(+), 1 deletion(-) diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index b865c76cf2e3f..a9f347fa49f1e 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -28,6 +28,7 @@ var ( // It's used by 'Run' and 'Exec', does some work in parent process // after child process is started. type StartCallback func(*ProcessConfig, int) +type RestoreCallback func(*ProcessConfig, int) // Info is driver specific information based on // processes registered with the driver @@ -71,6 +72,10 @@ type Driver interface { // Unpause unpauses a container. Unpause(c *Command) error + Checkpoint(c *Command) error + + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback) (int, error) + // Name returns the name of the driver. Name() string diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 27da7c8c24db8..f8b0511ae0e73 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -560,7 +560,14 @@ func (d *Driver) Unpause(c *execdriver.Command) error { return err } -// Terminate implements the exec driver Driver interface. +func (d *driver) Checkpoint(c *execdriver.Command) error { + return fmt.Errorf("Checkpointing lxc containers not supported yet\n") +} + +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + return 0, fmt.Errorf("Restoring lxc containers not supported yet\n") +} + func (d *Driver) Terminate(c *execdriver.Command) error { return killLxc(c.ID, 9) } diff --git a/daemon/execdriver/native/create.go b/daemon/execdriver/native/create.go index 85f72f8c2cc83..8759135b9c1da 100644 --- a/daemon/execdriver/native/create.go +++ b/daemon/execdriver/native/create.go @@ -4,6 +4,7 @@ package native import ( "errors" + "encoding/json" "fmt" "net" "strings" @@ -113,6 +114,24 @@ func generateIfaceName() (string, error) { return "", errors.New("Failed to find name for new interface") } +// Re-create the container type from the image that was saved during checkpoint. +func (d *driver) createRestoreContainer(c *execdriver.Command, imageDir string) (*libcontainer.Config, error) { + // Read the container.json. + f1, err := os.Open(filepath.Join(imageDir, "container.json")) + if err != nil { + return nil, err + } + defer f1.Close() + + var container *libcontainer.Config + err = json.NewDecoder(f1).Decode(&container) + if err != nil { + return nil, err + } + + return container, nil +} + func (d *Driver) createNetwork(container *configs.Config, c *execdriver.Command) error { if c.Network == nil { return nil diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index b241bdbc504c8..3bd3d5d610bd5 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -20,6 +20,7 @@ import ( "github.com/docker/docker/pkg/reexec" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" + "github.com/docker/docker/utils" "github.com/opencontainers/runc/libcontainer" "github.com/opencontainers/runc/libcontainer/apparmor" "github.com/opencontainers/runc/libcontainer/cgroups/systemd" @@ -298,6 +299,155 @@ func (d *Driver) Unpause(c *execdriver.Command) error { return active.Resume() } +// XXX Where is the right place for the following +// const and getCheckpointImageDir() function? +const ( + containersDir = "/var/lib/docker/containers" + criuImgDir = "criu_img" +) + +func getCheckpointImageDir(containerId string) string { + return filepath.Join(containersDir, containerId, criuImgDir) +} + +func (d *driver) Checkpoint(c *execdriver.Command) error { + active := d.activeContainers[c.ID] + if active == nil { + return fmt.Errorf("active container for %s does not exist", c.ID) + } + container := active.container + + // Create an image directory for this container (which + // may already exist from a previous checkpoint). + imageDir := getCheckpointImageDir(c.ID) + err := os.MkdirAll(imageDir, 0700) + if err != nil && !os.IsExist(err) { + return err + } + + // Copy container.json and state.json files to the CRIU + // image directory for later use during restore. Do this + // before checkpointing because after checkpoint the container + // will exit and these files will be removed. + log.CRDbg("saving container.json and state.json before calling CRIU in %s", imageDir) + srcFiles := []string{"container.json", "state.json"} + for _, f := range srcFiles { + srcFile := filepath.Join(d.root, c.ID, f) + dstFile := filepath.Join(imageDir, f) + if _, err := utils.CopyFile(srcFile, dstFile); err != nil { + return err + } + } + + d.Lock() + defer d.Unlock() + err = namespaces.Checkpoint(container, imageDir, c.ProcessConfig.Process.Pid) + if err != nil { + return err + } + + return nil +} + +type restoreOutput struct { + exitCode int + err error +} + +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + imageDir := getCheckpointImageDir(c.ID) + container, err := d.createRestoreContainer(c, imageDir) + if err != nil { + return 1, err + } + + var term execdriver.Terminal + + if c.ProcessConfig.Tty { + term, err = NewTtyConsole(&c.ProcessConfig, pipes) + } else { + term, err = execdriver.NewStdConsole(&c.ProcessConfig, pipes) + } + if err != nil { + return -1, err + } + c.ProcessConfig.Terminal = term + + d.Lock() + d.activeContainers[c.ID] = &activeContainer{ + container: container, + cmd: &c.ProcessConfig.Cmd, + } + d.Unlock() + defer d.cleanContainer(c.ID) + + // Since the CRIU binary exits after restoring the container, we + // need to reap its child by setting PR_SET_CHILD_SUBREAPER (36) + // so that it'll be owned by this process (Docker daemon) after restore. + // + // XXX This really belongs to where the Docker daemon starts. + if _, _, syserr := syscall.RawSyscall(syscall.SYS_PRCTL, 36, 1, 0); syserr != 0 { + return -1, fmt.Errorf("Could not set PR_SET_CHILD_SUBREAPER (syserr %d)", syserr) + } + + restoreOutputChan := make(chan restoreOutput, 1) + waitForRestore := make(chan struct{}) + + go func() { + exitCode, err := namespaces.Restore(container, c.ProcessConfig.Stdin, c.ProcessConfig.Stdout, c.ProcessConfig.Stderr, c.ProcessConfig.Console, filepath.Join(d.root, c.ID), imageDir, + func(child *os.File, args []string) *exec.Cmd { + cmd := new(exec.Cmd) + cmd.Path = d.initPath + cmd.Args = append([]string{ + DriverName, + "-restore", + "-pipe", "3", + "--", + }, args...) + cmd.ExtraFiles = []*os.File{child} + return cmd + }, + func(restorePid int) error { + log.CRDbg("restorePid=%d", restorePid) + if restorePid == 0 { + restoreCallback(&c.ProcessConfig, 0) + return nil + } + + // The container.json file should be written *after* the container + // has started because its StdFds cannot be initialized before. + // + // XXX How do we handle error here? + d.writeContainerFile(container, c.ID) + close(waitForRestore) + if restoreCallback != nil { + c.ProcessConfig.Process, err = os.FindProcess(restorePid) + if err != nil { + log.Debugf("cannot find restored process %d", restorePid) + return err + } + c.ContainerPid = c.ProcessConfig.Process.Pid + restoreCallback(&c.ProcessConfig, c.ContainerPid) + } + return nil + }) + restoreOutputChan <- restoreOutput{exitCode, err} + }() + + select { + case restoreOutput := <-restoreOutputChan: + // there was an error + return restoreOutput.exitCode, restoreOutput.err + case <-waitForRestore: + // container restored + break + } + + // Wait for the container to exit. + restoreOutput := <-restoreOutputChan + return restoreOutput.exitCode, restoreOutput.err +} + // Terminate implements the exec driver Driver interface. func (d *Driver) Terminate(c *execdriver.Command) error { defer d.cleanContainer(c.ID) From 6e98f20989fa322249ccab6febb58ee7d817cb6a Mon Sep 17 00:00:00 2001 From: boucher Date: Mon, 25 May 2015 08:32:58 -0700 Subject: [PATCH 02/16] Update checkpoint/restore support to match docker/master Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- daemon/execdriver/driver.go | 4 +- daemon/execdriver/lxc/driver.go | 6 +- daemon/execdriver/native/create.go | 1 - daemon/execdriver/native/driver.go | 182 ++++++++++------------------- 4 files changed, 69 insertions(+), 124 deletions(-) diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index a9f347fa49f1e..bb90515395458 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -72,9 +72,9 @@ type Driver interface { // Unpause unpauses a container. Unpause(c *Command) error - Checkpoint(c *Command) error + Checkpoint(c *Command, opts *libcontainer.CriuOpts) error - Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback) (int, error) + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (ExitStatus, error) // Name returns the name of the driver. Name() string diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index f8b0511ae0e73..26866b7efd15f 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -560,12 +560,12 @@ func (d *Driver) Unpause(c *execdriver.Command) error { return err } -func (d *driver) Checkpoint(c *execdriver.Command) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { return fmt.Errorf("Checkpointing lxc containers not supported yet\n") } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { - return 0, fmt.Errorf("Restoring lxc containers not supported yet\n") +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { + return execdriver.ExitStatus{ExitCode: 0}, fmt.Errorf("Restoring lxc containers not supported yet\n") } func (d *Driver) Terminate(c *execdriver.Command) error { diff --git a/daemon/execdriver/native/create.go b/daemon/execdriver/native/create.go index 8759135b9c1da..95c2fd03eb5fc 100644 --- a/daemon/execdriver/native/create.go +++ b/daemon/execdriver/native/create.go @@ -4,7 +4,6 @@ package native import ( "errors" - "encoding/json" "fmt" "net" "strings" diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 3bd3d5d610bd5..90924230538ab 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -299,49 +299,15 @@ func (d *Driver) Unpause(c *execdriver.Command) error { return active.Resume() } -// XXX Where is the right place for the following -// const and getCheckpointImageDir() function? -const ( - containersDir = "/var/lib/docker/containers" - criuImgDir = "criu_img" -) - -func getCheckpointImageDir(containerId string) string { - return filepath.Join(containersDir, containerId, criuImgDir) -} - -func (d *driver) Checkpoint(c *execdriver.Command) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { active := d.activeContainers[c.ID] if active == nil { return fmt.Errorf("active container for %s does not exist", c.ID) } - container := active.container - - // Create an image directory for this container (which - // may already exist from a previous checkpoint). - imageDir := getCheckpointImageDir(c.ID) - err := os.MkdirAll(imageDir, 0700) - if err != nil && !os.IsExist(err) { - return err - } - - // Copy container.json and state.json files to the CRIU - // image directory for later use during restore. Do this - // before checkpointing because after checkpoint the container - // will exit and these files will be removed. - log.CRDbg("saving container.json and state.json before calling CRIU in %s", imageDir) - srcFiles := []string{"container.json", "state.json"} - for _, f := range srcFiles { - srcFile := filepath.Join(d.root, c.ID, f) - dstFile := filepath.Join(imageDir, f) - if _, err := utils.CopyFile(srcFile, dstFile); err != nil { - return err - } - } d.Lock() defer d.Unlock() - err = namespaces.Checkpoint(container, imageDir, c.ProcessConfig.Process.Pid) + err := active.Checkpoint(opts) if err != nil { return err } @@ -349,103 +315,83 @@ func (d *driver) Checkpoint(c *execdriver.Command) error { return nil } -type restoreOutput struct { - exitCode int - err error -} +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { + var ( + cont libcontainer.Container + err error + ) -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { - imageDir := getCheckpointImageDir(c.ID) - container, err := d.createRestoreContainer(c, imageDir) + cont, err = d.factory.Load(c.ID) if err != nil { - return 1, err + if forceRestore { + var config *configs.Config + config, err = d.createContainer(c) + if err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err + } + cont, err = d.factory.Create(c.ID, config) + if err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err + } + } else { + return execdriver.ExitStatus{ExitCode: -1}, err + } } - var term execdriver.Terminal - - if c.ProcessConfig.Tty { - term, err = NewTtyConsole(&c.ProcessConfig, pipes) - } else { - term, err = execdriver.NewStdConsole(&c.ProcessConfig, pipes) + p := &libcontainer.Process{ + Args: append([]string{c.ProcessConfig.Entrypoint}, c.ProcessConfig.Arguments...), + Env: c.ProcessConfig.Env, + Cwd: c.WorkingDir, + User: c.ProcessConfig.User, } - if err != nil { - return -1, err + + config := cont.Config() + if err := setupPipes(&config, &c.ProcessConfig, p, pipes); err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err } - c.ProcessConfig.Terminal = term d.Lock() - d.activeContainers[c.ID] = &activeContainer{ - container: container, - cmd: &c.ProcessConfig.Cmd, - } + d.activeContainers[c.ID] = cont d.Unlock() - defer d.cleanContainer(c.ID) + defer func() { + cont.Destroy() + d.cleanContainer(c.ID) + }() - // Since the CRIU binary exits after restoring the container, we - // need to reap its child by setting PR_SET_CHILD_SUBREAPER (36) - // so that it'll be owned by this process (Docker daemon) after restore. - // - // XXX This really belongs to where the Docker daemon starts. - if _, _, syserr := syscall.RawSyscall(syscall.SYS_PRCTL, 36, 1, 0); syserr != 0 { - return -1, fmt.Errorf("Could not set PR_SET_CHILD_SUBREAPER (syserr %d)", syserr) + if err := cont.Restore(p, opts); err != nil { + return execdriver.ExitStatus{ExitCode: -1}, err } - restoreOutputChan := make(chan restoreOutput, 1) - waitForRestore := make(chan struct{}) - - go func() { - exitCode, err := namespaces.Restore(container, c.ProcessConfig.Stdin, c.ProcessConfig.Stdout, c.ProcessConfig.Stderr, c.ProcessConfig.Console, filepath.Join(d.root, c.ID), imageDir, - func(child *os.File, args []string) *exec.Cmd { - cmd := new(exec.Cmd) - cmd.Path = d.initPath - cmd.Args = append([]string{ - DriverName, - "-restore", - "-pipe", "3", - "--", - }, args...) - cmd.ExtraFiles = []*os.File{child} - return cmd - }, - func(restorePid int) error { - log.CRDbg("restorePid=%d", restorePid) - if restorePid == 0 { - restoreCallback(&c.ProcessConfig, 0) - return nil - } - - // The container.json file should be written *after* the container - // has started because its StdFds cannot be initialized before. - // - // XXX How do we handle error here? - d.writeContainerFile(container, c.ID) - close(waitForRestore) - if restoreCallback != nil { - c.ProcessConfig.Process, err = os.FindProcess(restorePid) - if err != nil { - log.Debugf("cannot find restored process %d", restorePid) - return err - } - c.ContainerPid = c.ProcessConfig.Process.Pid - restoreCallback(&c.ProcessConfig, c.ContainerPid) - } - return nil - }) - restoreOutputChan <- restoreOutput{exitCode, err} - }() + // FIXME: no idea if any of this is needed... + if restoreCallback != nil { + pid, err := p.Pid() + if err != nil { + p.Signal(os.Kill) + p.Wait() + return execdriver.ExitStatus{ExitCode: -1}, err + } + restoreCallback(&c.ProcessConfig, pid) + } - select { - case restoreOutput := <-restoreOutputChan: - // there was an error - return restoreOutput.exitCode, restoreOutput.err - case <-waitForRestore: - // container restored - break + oom := notifyOnOOM(cont) + waitF := p.Wait + if nss := cont.Config().Namespaces; !nss.Contains(configs.NEWPID) { + // we need such hack for tracking processes with inherited fds, + // because cmd.Wait() waiting for all streams to be copied + waitF = waitInPIDHost(p, cont) + } + ps, err := waitF() + if err != nil { + execErr, ok := err.(*exec.ExitError) + if !ok { + return execdriver.ExitStatus{ExitCode: -1}, err + } + ps = execErr.ProcessState } - // Wait for the container to exit. - restoreOutput := <-restoreOutputChan - return restoreOutput.exitCode, restoreOutput.err + cont.Destroy() + _, oomKill := <-oom + return execdriver.ExitStatus{ExitCode: utils.ExitStatus(ps.Sys().(syscall.WaitStatus)), OOMKilled: oomKill}, nil } // Terminate implements the exec driver Driver interface. From 825147f3938bc058f2f581ef2ce754f3b76d736e Mon Sep 17 00:00:00 2001 From: Saied Kazemi Date: Thu, 5 Feb 2015 20:37:07 -0800 Subject: [PATCH 03/16] Checkpoint/Restore Support: add functionality to daemon Support was added to the daemon to use the Checkpoint and Restore methods of the native exec driver for checkpointing and restoring containers. Signed-off-by: Saied Kazemi Conflicts: api/server/server.go daemon/container.go daemon/daemon.go daemon/networkdriver/bridge/driver.go daemon/state.go vendor/src/github.com/docker/libnetwork/ipallocator/allocator.go --- api/server/server.go | 79 ++++++++++++++++++++++++++++------------ daemon/checkpoint.go | 55 ++++++++++++++++++++++++++++ daemon/container.go | 66 ++++++++++++++++++++++++++++++++- daemon/container_unix.go | 49 ++++++++++++++++++++++++- daemon/daemon.go | 31 ++++++++++++++++ daemon/monitor.go | 70 +++++++++++++++++++++++++++++++++++ daemon/state.go | 23 ++++++++++++ 7 files changed, 347 insertions(+), 26 deletions(-) create mode 100644 daemon/checkpoint.go diff --git a/api/server/server.go b/api/server/server.go index ec2f00cbf268f..743653614cd7a 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -215,10 +215,41 @@ func httpError(w http.ResponseWriter, err error) { // json encoding. func writeJSON(w http.ResponseWriter, code int, v interface{}) error { w.Header().Set("Content-Type", "application/json") + w.WriteHeader(code) return json.NewEncoder(w).Encode(v) } +func postContainersCheckpoint(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + if err := parseForm(r); err != nil { + return err + } + job := eng.Job("checkpoint", vars["name"]) + if err := job.Run(); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) + return nil +} + +func postContainersRestore(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + if err := parseForm(r); err != nil { + return err + } + job := eng.Job("restore", vars["name"]) + if err := job.Run(); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) + return nil +} + func (s *Server) optionsHandler(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { w.WriteHeader(http.StatusOK) return nil @@ -332,29 +363,31 @@ func createRouter(s *Server) *mux.Router { "/volumes/{name:.*}": s.getVolumeByName, }, "POST": { - "/auth": s.postAuth, - "/commit": s.postCommit, - "/build": s.postBuild, - "/images/create": s.postImagesCreate, - "/images/load": s.postImagesLoad, - "/images/{name:.*}/push": s.postImagesPush, - "/images/{name:.*}/tag": s.postImagesTag, - "/containers/create": s.postContainersCreate, - "/containers/{name:.*}/kill": s.postContainersKill, - "/containers/{name:.*}/pause": s.postContainersPause, - "/containers/{name:.*}/unpause": s.postContainersUnpause, - "/containers/{name:.*}/restart": s.postContainersRestart, - "/containers/{name:.*}/start": s.postContainersStart, - "/containers/{name:.*}/stop": s.postContainersStop, - "/containers/{name:.*}/wait": s.postContainersWait, - "/containers/{name:.*}/resize": s.postContainersResize, - "/containers/{name:.*}/attach": s.postContainersAttach, - "/containers/{name:.*}/copy": s.postContainersCopy, - "/containers/{name:.*}/exec": s.postContainerExecCreate, - "/exec/{name:.*}/start": s.postContainerExecStart, - "/exec/{name:.*}/resize": s.postContainerExecResize, - "/containers/{name:.*}/rename": s.postContainerRename, - "/volumes": s.postVolumesCreate, + "/auth": s.postAuth, + "/commit": s.postCommit, + "/build": s.postBuild, + "/images/create": s.postImagesCreate, + "/images/load": s.postImagesLoad, + "/images/{name:.*}/push": s.postImagesPush, + "/images/{name:.*}/tag": s.postImagesTag, + "/containers/create": s.postContainersCreate, + "/containers/{name:.*}/kill": s.postContainersKill, + "/containers/{name:.*}/pause": s.postContainersPause, + "/containers/{name:.*}/unpause": s.postContainersUnpause, + "/containers/{name:.*}/restart": s.postContainersRestart, + "/containers/{name:.*}/start": s.postContainersStart, + "/containers/{name:.*}/stop": s.postContainersStop, + "/containers/{name:.*}/wait": s.postContainersWait, + "/containers/{name:.*}/resize": s.postContainersResize, + "/containers/{name:.*}/attach": s.postContainersAttach, + "/containers/{name:.*}/copy": s.postContainersCopy, + "/containers/{name:.*}/exec": s.postContainerExecCreate, + "/exec/{name:.*}/start": s.postContainerExecStart, + "/exec/{name:.*}/resize": s.postContainerExecResize, + "/containers/{name:.*}/rename": s.postContainerRename, + "/containers/{name:.*}/checkpoint": s.postContainersCheckpoint, + "/containers/{name:.*}/restore": s.postContainersRestore, + "/volumes": s.postVolumesCreate, }, "PUT": { "/containers/{name:.*}/archive": s.putContainersArchive, diff --git a/daemon/checkpoint.go b/daemon/checkpoint.go new file mode 100644 index 0000000000000..f6057c6a028f9 --- /dev/null +++ b/daemon/checkpoint.go @@ -0,0 +1,55 @@ +package daemon + +import ( + "github.com/docker/docker/engine" +) + +// Checkpoint a running container. +func (daemon *Daemon) ContainerCheckpoint(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + return job.Errorf("Usage: %s CONTAINER\n", job.Name) + } + + name := job.Args[0] + container, err := daemon.Get(name) + if err != nil { + return job.Error(err) + } + if !container.IsRunning() { + return job.Errorf("Container %s not running", name) + } + + if err := container.Checkpoint(); err != nil { + return job.Errorf("Cannot checkpoint container %s: %s", name, err) + } + + container.LogEvent("checkpoint") + return engine.StatusOK +} + +// Restore a checkpointed container. +func (daemon *Daemon) ContainerRestore(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + return job.Errorf("Usage: %s CONTAINER\n", job.Name) + } + + name := job.Args[0] + container, err := daemon.Get(name) + if err != nil { + return job.Error(err) + } + if container.IsRunning() { + return job.Errorf("Container %s already running", name) + } + if !container.State.IsCheckpointed() { + return job.Errorf("Container %s is not checkpointed", name) + } + + if err := container.Restore(); err != nil { + container.LogEvent("die") + return job.Errorf("Cannot restore container %s: %s", name, err) + } + + container.LogEvent("restore") + return engine.StatusOK +} diff --git a/daemon/container.go b/daemon/container.go index 0f4b81021d092..68a6e251fd2ef 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -354,10 +354,15 @@ func (container *Container) isNetworkAllocated() bool { return container.NetworkSettings.IPAddress != "" } + // cleanup releases any network resources allocated to the container along with any rules // around how containers are linked together. It also unmounts the container's root filesystem. func (container *Container) cleanup() { - container.releaseNetwork() + if container.IsCheckpointed() { + log.CRDbg("not calling ReleaseNetwork() for checkpointed container %s", container.ID) + } else { + container.ReleaseNetwork() + } if err := container.Unmount(); err != nil { logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err) @@ -695,6 +700,41 @@ func (container *Container) copy(resource string) (rc io.ReadCloser, err error) return reader, nil } +func (container *Container) Checkpoint() error { + return container.daemon.Checkpoint(container) +} + +func (container *Container) Restore() error { + var err error + + container.Lock() + defer container.Unlock() + + defer func() { + if err != nil { + container.cleanup() + } + }() + + if err = container.initializeNetworking(); err != nil { + return err + } + + linkedEnv, err := container.setupLinkedContainers() + if err != nil { + return err + } + if err = container.setupWorkingDirectory(); err != nil { + return err + } + env := container.createDaemonEnvironment(linkedEnv) + if err = populateCommandRestore(container, env); err != nil { + return err + } + + return container.waitForRestore() +} + // Returns true if the container exposes a certain port func (container *Container) exposes(p nat.Port) bool { _, exists := container.Config.ExposedPorts[p] @@ -784,6 +824,29 @@ func (container *Container) waitForStart() error { return nil } +// Like waitForStart() but for restoring a container. +// +// XXX Does RestartPolicy apply here? +func (container *Container) waitForRestore() error { + container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) + + // After calling promise.Go() we'll have two goroutines: + // - The current goroutine that will block in the select + // below until restore is done. + // - A new goroutine that will restore the container and + // wait for it to exit. + select { + case <-container.monitor.restoreSignal: + if container.ExitCode != 0 { + return fmt.Errorf("restore process failed") + } + case err := <-promise.Go(container.monitor.Restore): + return err + } + + return nil +} + func (container *Container) getProcessLabel() string { // even if we have a process label return "" if we are running // in privileged mode @@ -983,7 +1046,6 @@ func attach(streamConfig *streamConfig, openStdin, stdinOnce, tty bool, stdin io _, err = copyEscapable(cStdin, stdin) } else { _, err = io.Copy(cStdin, stdin) - } if err == io.ErrClosedPipe { err = nil diff --git a/daemon/container_unix.go b/daemon/container_unix.go index a98acd4f3cf25..38948cf41dad3 100644 --- a/daemon/container_unix.go +++ b/daemon/container_unix.go @@ -328,7 +328,54 @@ func mergeDevices(defaultDevices, userDevices []*configs.Device) []*configs.Devi return append(devs, userDevices...) } -// GetSize returns the real size & virtual size of the container. +// Like populateCommand() but for restoring a container. +// +// XXX populateCommand() does a lot more. Not sure if we have +// to do everything it does. +func populateCommandRestore(c *Container, env []string) error { + resources := &execdriver.Resources{ + Memory: c.Config.Memory, + MemorySwap: c.Config.MemorySwap, + CpuShares: c.Config.CpuShares, + Cpuset: c.Config.Cpuset, + } + + processConfig := execdriver.ProcessConfig{ + Privileged: c.hostConfig.Privileged, + Entrypoint: c.Path, + Arguments: c.Args, + Tty: c.Config.Tty, + User: c.Config.User, + } + + processConfig.SysProcAttr = &syscall.SysProcAttr{Setsid: true} + processConfig.Env = env + + c.command = &execdriver.Command{ + ID: c.ID, + Rootfs: c.RootfsPath(), + ReadonlyRootfs: c.hostConfig.ReadonlyRootfs, + InitPath: "/.dockerinit", + WorkingDir: c.Config.WorkingDir, + // Network: en, + // Ipc: ipc, + // Pid: pid, + Resources: resources, + // AllowedDevices: allowedDevices, + // AutoCreatedDevices: autoCreatedDevices, + CapAdd: c.hostConfig.CapAdd, + CapDrop: c.hostConfig.CapDrop, + ProcessConfig: processConfig, + ProcessLabel: c.GetProcessLabel(), + MountLabel: c.GetMountLabel(), + // LxcConfig: lxcConfig, + AppArmorProfile: c.AppArmorProfile, + } + + return nil +} + +// GetSize, return real size, virtual size func (container *Container) getSize() (int64, int64) { var ( sizeRw, sizeRootfs int64 diff --git a/daemon/daemon.go b/daemon/daemon.go index a05262dbff0d0..532bb44fad27e 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -284,6 +284,18 @@ func (daemon *Daemon) restore() error { logrus.Debugf("Loaded container %v", container.ID) containers[container.ID] = &cr{container: container} + + // If the container was checkpointed, we need to reserve + // the IP address that it was using. + // + // XXX We should also reserve host ports (if any). + if container.IsCheckpointed() { + /*err = bridge.ReserveIP(container.ID, container.NetworkSettings.IPAddress) + if err != nil { + log.Errorf("Failed to reserve IP %s for container %s", + container.ID, container.NetworkSettings.IPAddress) + }*/ + } } else { logrus.Debugf("Cannot load container %s because it was created with another graph driver.", container.ID) } @@ -877,6 +889,25 @@ func (daemon *Daemon) run(c *Container, pipes *execdriver.Pipes, startCallback e return daemon.execDriver.Run(c.command, pipes, startCallback) } +func (daemon *Daemon) Checkpoint(c *Container) error { + if err := daemon.execDriver.Checkpoint(c.command); err != nil { + return err + } + c.SetCheckpointed() + return nil +} + +func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { + // Mount the container's filesystem (daemon/graphdriver/aufs/aufs.go). + _, err := daemon.driver.Get(c.ID, c.GetMountLabel()) + if err != nil { + return 0, err + } + + exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback) + return exitCode, err +} + func (daemon *Daemon) kill(c *Container, sig int) error { return daemon.execDriver.Kill(c.command, sig) } diff --git a/daemon/monitor.go b/daemon/monitor.go index dd5d8c6b255bb..5bf1088f0950e 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -47,6 +47,9 @@ type containerMonitor struct { // left waiting for nothing to happen during this time stopChan chan struct{} + // like startSignal but for restoring a container + restoreSignal chan struct{} + // timeIncrement is the amount of time to wait between restarts // this is in milliseconds timeIncrement int @@ -64,6 +67,7 @@ func newContainerMonitor(container *Container, policy runconfig.RestartPolicy) * timeIncrement: defaultTimeIncrement, stopChan: make(chan struct{}), startSignal: make(chan struct{}), + restoreSignal: make(chan struct{}), } } @@ -188,6 +192,49 @@ func (m *containerMonitor) Start() error { } } +// Like Start() but for restoring a container. +func (m *containerMonitor) Restore() error { + var ( + err error + // XXX The following line should be changed to + // exitStatus execdriver.ExitStatus to match Start() + exitCode int + afterRestore bool + ) + + defer func() { + if afterRestore { + m.container.Lock() + m.container.setStopped(&execdriver.ExitStatus{exitCode, false}) + defer m.container.Unlock() + } + m.Close() + }() + + if err := m.container.startLoggingToDisk(); err != nil { + m.resetContainer(false) + return err + } + + pipes := execdriver.NewPipes(m.container.stdin, m.container.stdout, m.container.stderr, m.container.Config.OpenStdin) + + m.container.LogEvent("restore") + m.lastStartTime = time.Now() + if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback); err != nil { + log.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) + m.container.ExitCode = -1 + m.resetContainer(false) + return err + } + afterRestore = true + + m.container.ExitCode = exitCode + m.resetMonitor(err == nil && exitCode == 0) + m.container.LogEvent("die") + m.resetContainer(true) + return err +} + // resetMonitor resets the stateful fields on the containerMonitor based on the // previous runs success or failure. Regardless of success, if the container had // an execution time of more than 10s then reset the timer back to the default @@ -275,6 +322,29 @@ func (m *containerMonitor) callback(processConfig *execdriver.ProcessConfig, pid } } +// Like callback() but for restoring a container. +func (m *containerMonitor) restoreCallback(processConfig *execdriver.ProcessConfig, restorePid int) { + // If restorePid is 0, it means that restore failed. + if restorePid != 0 { + m.container.setRunning(restorePid) + } + + // Unblock the goroutine waiting in waitForRestore(). + select { + case <-m.restoreSignal: + default: + close(m.restoreSignal) + } + + if restorePid != 0 { + // Write config.json and hostconfig.json files + // to /var/lib/docker/containers/. + if err := m.container.ToDisk(); err != nil { + log.Debugf("%s", err) + } + } +} + // resetContainer resets the container's IO and ensures that the command is able to be executed again // by copying the data into a new struct // if lock is true, then container locked during reset diff --git a/daemon/state.go b/daemon/state.go index 9e2590791f4cf..49e455ecb5c4a 100644 --- a/daemon/state.go +++ b/daemon/state.go @@ -19,6 +19,7 @@ type State struct { Running bool Paused bool Restarting bool + Checkpointed bool OOMKilled bool removalInProgress bool // Not need for this to be persistent on disk. Dead bool @@ -27,7 +28,9 @@ type State struct { Error string // contains last known error when starting the container StartedAt time.Time FinishedAt time.Time + CheckpointedAt time.Time waitChan chan struct{} + } // NewState creates a default state object with a fresh channel for state changes. @@ -48,6 +51,8 @@ func (s *State) String() string { } return fmt.Sprintf("Up %s", units.HumanDuration(time.Now().UTC().Sub(s.StartedAt))) + } else if s.Checkpointed { + return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) } if s.removalInProgress { @@ -187,6 +192,7 @@ func (s *State) setRunning(pid int) { s.Error = "" s.Running = true s.Paused = false + s.Checkpointed = false s.Restarting = false s.ExitCode = 0 s.Pid = pid @@ -268,3 +274,20 @@ func (s *State) setDead() { s.Dead = true s.Unlock() } + +func (s *State) SetCheckpointed() { + s.Lock() + s.CheckpointedAt = time.Now().UTC() + s.Checkpointed = true + s.Running = false + s.Paused = false + s.Restarting = false + // XXX Not sure if we need to close and recreate waitChan. + // close(s.waitChan) + // s.waitChan = make(chan struct{}) + s.Unlock() +} + +func (s *State) IsCheckpointed() bool { + return s.Checkpointed +} From 61a54396b5fee56f562fef2237115d71a0dcfd7f Mon Sep 17 00:00:00 2001 From: Hui Kang Date: Tue, 19 May 2015 21:08:04 +0000 Subject: [PATCH 04/16] Release the network resource during checkpoint Restore failed if network resource not released during checkpoint, e.g., a container with port open with -p Signed-off-by: Hui Kang Conflicts: daemon/container.go --- daemon/container.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/daemon/container.go b/daemon/container.go index 68a6e251fd2ef..b83ec7a171c42 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -635,6 +635,18 @@ func validateID(id string) error { return nil } +func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { + if err := container.daemon.Checkpoint(container, opts); err != nil { + return err + } + + if opts.LeaveRunning == false { + container.ReleaseNetwork() + } + return nil +} + + func (container *Container) copy(resource string) (rc io.ReadCloser, err error) { container.Lock() From edfbd1018d34bca791c7926c79f6e4db0a2b6abf Mon Sep 17 00:00:00 2001 From: boucher Date: Mon, 1 Jun 2015 15:15:02 -0700 Subject: [PATCH 05/16] Update daemon and cli support for checkpoint and restore. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- api/client/checkpoint.go | 52 +++++++++++++++ api/client/restore.go | 54 ++++++++++++++++ api/server/server.go | 24 +++++-- daemon/checkpoint.go | 57 ++++++++--------- daemon/container.go | 94 +++++++++++++++------------- daemon/container_unix.go | 71 +++++---------------- daemon/container_windows.go | 2 +- daemon/daemon.go | 12 ++-- daemon/execdriver/driver.go | 7 ++- daemon/execdriver/lxc/driver.go | 5 +- daemon/execdriver/native/driver.go | 21 +++++-- daemon/execdriver/windows/windows.go | 9 +++ daemon/monitor.go | 26 ++++---- daemon/state.go | 21 +++++-- docker/flags.go | 2 + runconfig/restore.go | 15 +++++ 16 files changed, 305 insertions(+), 167 deletions(-) create mode 100644 api/client/checkpoint.go create mode 100644 api/client/restore.go create mode 100644 runconfig/restore.go diff --git a/api/client/checkpoint.go b/api/client/checkpoint.go new file mode 100644 index 0000000000000..8c681bcf9716f --- /dev/null +++ b/api/client/checkpoint.go @@ -0,0 +1,52 @@ +package client + +import ( + "fmt" + + flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/runconfig" +) + +func (cli *DockerCli) CmdCheckpoint(args ...string) error { + cmd := cli.Subcmd("checkpoint", []string{"CONTAINER [CONTAINER...]"}, "Checkpoint one or more running containers", true) + cmd.Require(flag.Min, 1) + + var ( + flImgDir = cmd.String([]string{"-image-dir"}, "", "directory for storing checkpoint image files") + flWorkDir = cmd.String([]string{"-work-dir"}, "", "directory for storing log file") + flLeaveRunning = cmd.Bool([]string{"-leave-running"}, false, "leave the container running after checkpoint") + flCheckTcp = cmd.Bool([]string{"-allow-tcp"}, false, "allow checkpointing tcp connections") + flExtUnix = cmd.Bool([]string{"-allow-ext-unix"}, false, "allow checkpointing external unix connections") + flShell = cmd.Bool([]string{"-allow-shell"}, false, "allow checkpointing shell jobs") + ) + + if err := cmd.ParseFlags(args, true); err != nil { + return err + } + + if cmd.NArg() < 1 { + cmd.Usage() + return nil + } + + criuOpts := &runconfig.CriuConfig{ + ImagesDirectory: *flImgDir, + WorkDirectory: *flWorkDir, + LeaveRunning: *flLeaveRunning, + TcpEstablished: *flCheckTcp, + ExternalUnixConnections: *flExtUnix, + ShellJob: *flShell, + } + + var encounteredError error + for _, name := range cmd.Args() { + _, _, err := readBody(cli.call("POST", "/containers/"+name+"/checkpoint", criuOpts, nil)) + if err != nil { + fmt.Fprintf(cli.err, "%s\n", err) + encounteredError = fmt.Errorf("Error: failed to checkpoint one or more containers") + } else { + fmt.Fprintf(cli.out, "%s\n", name) + } + } + return encounteredError +} diff --git a/api/client/restore.go b/api/client/restore.go new file mode 100644 index 0000000000000..0c4085fbbbd84 --- /dev/null +++ b/api/client/restore.go @@ -0,0 +1,54 @@ +package client + +import ( + "fmt" + + flag "github.com/docker/docker/pkg/mflag" + "github.com/docker/docker/runconfig" +) + +func (cli *DockerCli) CmdRestore(args ...string) error { + cmd := cli.Subcmd("restore", []string{"CONTAINER [CONTAINER...]"}, "Restore one or more checkpointed containers", true) + cmd.Require(flag.Min, 1) + + var ( + flImgDir = cmd.String([]string{"-image-dir"}, "", "directory to restore image files from") + flWorkDir = cmd.String([]string{"-work-dir"}, "", "directory for restore log") + flCheckTcp = cmd.Bool([]string{"-allow-tcp"}, false, "allow restoring tcp connections") + flExtUnix = cmd.Bool([]string{"-allow-ext-unix"}, false, "allow restoring external unix connections") + flShell = cmd.Bool([]string{"-allow-shell"}, false, "allow restoring shell jobs") + flForce = cmd.Bool([]string{"-force"}, false, "bypass checks for current container state") + ) + + if err := cmd.ParseFlags(args, true); err != nil { + return err + } + + if cmd.NArg() < 1 { + cmd.Usage() + return nil + } + + restoreOpts := &runconfig.RestoreConfig{ + CriuOpts: runconfig.CriuConfig{ + ImagesDirectory: *flImgDir, + WorkDirectory: *flWorkDir, + TcpEstablished: *flCheckTcp, + ExternalUnixConnections: *flExtUnix, + ShellJob: *flShell, + }, + ForceRestore: *flForce, + } + + var encounteredError error + for _, name := range cmd.Args() { + _, _, err := readBody(cli.call("POST", "/containers/"+name+"/restore", restoreOpts, nil)) + if err != nil { + fmt.Fprintf(cli.err, "%s\n", err) + encounteredError = fmt.Errorf("Error: failed to restore one or more containers") + } else { + fmt.Fprintf(cli.out, "%s\n", name) + } + } + return encounteredError +} diff --git a/api/server/server.go b/api/server/server.go index 743653614cd7a..39552eb3c1e93 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -220,32 +220,44 @@ func writeJSON(w http.ResponseWriter, code int, v interface{}) error { return json.NewEncoder(w).Encode(v) } -func postContainersCheckpoint(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { +func (s *Server) postContainersCheckpoint(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if vars == nil { return fmt.Errorf("Missing parameter") } if err := parseForm(r); err != nil { return err } - job := eng.Job("checkpoint", vars["name"]) - if err := job.Run(); err != nil { + + criuOpts := &runconfig.CriuConfig{} + if err := json.NewDecoder(r.Body).Decode(criuOpts); err != nil { return err } + + if err := s.daemon.ContainerCheckpoint(vars["name"], criuOpts); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) return nil } -func postContainersRestore(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { +func (s *Server) postContainersRestore(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if vars == nil { return fmt.Errorf("Missing parameter") } if err := parseForm(r); err != nil { return err } - job := eng.Job("restore", vars["name"]) - if err := job.Run(); err != nil { + + restoreOpts := runconfig.RestoreConfig{} + if err := json.NewDecoder(r.Body).Decode(&restoreOpts); err != nil { return err } + + if err := s.daemon.ContainerRestore(vars["name"], &restoreOpts.CriuOpts, restoreOpts.ForceRestore); err != nil { + return err + } + w.WriteHeader(http.StatusNoContent) return nil } diff --git a/daemon/checkpoint.go b/daemon/checkpoint.go index f6057c6a028f9..a39662cc0f325 100644 --- a/daemon/checkpoint.go +++ b/daemon/checkpoint.go @@ -1,55 +1,56 @@ package daemon import ( - "github.com/docker/docker/engine" + "fmt" + + "github.com/docker/docker/runconfig" ) // Checkpoint a running container. -func (daemon *Daemon) ContainerCheckpoint(job *engine.Job) engine.Status { - if len(job.Args) != 1 { - return job.Errorf("Usage: %s CONTAINER\n", job.Name) - } - - name := job.Args[0] +func (daemon *Daemon) ContainerCheckpoint(name string, opts *runconfig.CriuConfig) error { container, err := daemon.Get(name) if err != nil { - return job.Error(err) + return err } if !container.IsRunning() { - return job.Errorf("Container %s not running", name) + return fmt.Errorf("Container %s not running", name) } - - if err := container.Checkpoint(); err != nil { - return job.Errorf("Cannot checkpoint container %s: %s", name, err) + if err := container.Checkpoint(opts); err != nil { + return fmt.Errorf("Cannot checkpoint container %s: %s", name, err) } container.LogEvent("checkpoint") - return engine.StatusOK + return nil } // Restore a checkpointed container. -func (daemon *Daemon) ContainerRestore(job *engine.Job) engine.Status { - if len(job.Args) != 1 { - return job.Errorf("Usage: %s CONTAINER\n", job.Name) - } - - name := job.Args[0] +func (daemon *Daemon) ContainerRestore(name string, opts *runconfig.CriuConfig, forceRestore bool) error { container, err := daemon.Get(name) if err != nil { - return job.Error(err) - } - if container.IsRunning() { - return job.Errorf("Container %s already running", name) + return err } - if !container.State.IsCheckpointed() { - return job.Errorf("Container %s is not checkpointed", name) + + if !forceRestore { + // TODO: It's possible we only want to bypass the checkpointed check, + // I'm not sure how this will work if the container is already running + if container.IsRunning() { + return fmt.Errorf("Container %s already running", name) + } + + if !container.IsCheckpointed() { + return fmt.Errorf("Container %s is not checkpointed", name) + } + } else { + if !container.HasBeenCheckpointed() && opts.ImagesDirectory == "" { + return fmt.Errorf("You must specify an image directory to restore from %s", name) + } } - if err := container.Restore(); err != nil { + if err = container.Restore(opts, forceRestore); err != nil { container.LogEvent("die") - return job.Errorf("Cannot restore container %s: %s", name, err) + return fmt.Errorf("Cannot restore container %s: %s", name, err) } container.LogEvent("restore") - return engine.StatusOK + return nil } diff --git a/daemon/container.go b/daemon/container.go index b83ec7a171c42..ed2af5fc5d479 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -282,7 +282,7 @@ func (container *Container) Start() (err error) { // backwards API compatibility. container.hostConfig = runconfig.SetDefaultNetModeIfBlank(container.hostConfig) - if err := container.initializeNetworking(); err != nil { + if err := container.initializeNetworking(false); err != nil { return err } linkedEnv, err := container.setupLinkedContainers() @@ -354,12 +354,11 @@ func (container *Container) isNetworkAllocated() bool { return container.NetworkSettings.IPAddress != "" } - // cleanup releases any network resources allocated to the container along with any rules // around how containers are linked together. It also unmounts the container's root filesystem. func (container *Container) cleanup() { if container.IsCheckpointed() { - log.CRDbg("not calling ReleaseNetwork() for checkpointed container %s", container.ID) + logrus.Debugf("not calling ReleaseNetwork() for checkpointed container %s", container.ID) } else { container.ReleaseNetwork() } @@ -635,7 +634,7 @@ func validateID(id string) error { return nil } -func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { +func (container *Container) Checkpoint(opts *runconfig.CriuConfig) error { if err := container.daemon.Checkpoint(container, opts); err != nil { return err } @@ -646,6 +645,50 @@ func (container *Container) Checkpoint(opts *libcontainer.CriuOpts) error { return nil } +func (container *Container) Restore(opts *runconfig.CriuConfig, forceRestore bool) error { + var err error + container.Lock() + defer container.Unlock() + + defer func() { + if err != nil { + container.setError(err) + // if no one else has set it, make sure we don't leave it at zero + if container.ExitCode == 0 { + container.ExitCode = 128 + } + container.toDisk() + container.cleanup() + } + }() + + if err := container.Mount(); err != nil { + return err + } + if err = container.initializeNetworking(true); err != nil { + return err + } + linkedEnv, err := container.setupLinkedContainers() + if err != nil { + return err + } + if err = container.setupWorkingDirectory(); err != nil { + return err + } + + env := container.createDaemonEnvironment(linkedEnv) + if err = populateCommand(container, env); err != nil { + return err + } + + mounts, err := container.setupMounts() + if err != nil { + return err + } + + container.command.Mounts = mounts + return container.waitForRestore(opts, forceRestore) +} func (container *Container) copy(resource string) (rc io.ReadCloser, err error) { container.Lock() @@ -712,41 +755,6 @@ func (container *Container) copy(resource string) (rc io.ReadCloser, err error) return reader, nil } -func (container *Container) Checkpoint() error { - return container.daemon.Checkpoint(container) -} - -func (container *Container) Restore() error { - var err error - - container.Lock() - defer container.Unlock() - - defer func() { - if err != nil { - container.cleanup() - } - }() - - if err = container.initializeNetworking(); err != nil { - return err - } - - linkedEnv, err := container.setupLinkedContainers() - if err != nil { - return err - } - if err = container.setupWorkingDirectory(); err != nil { - return err - } - env := container.createDaemonEnvironment(linkedEnv) - if err = populateCommandRestore(container, env); err != nil { - return err - } - - return container.waitForRestore() -} - // Returns true if the container exposes a certain port func (container *Container) exposes(p nat.Port) bool { _, exists := container.Config.ExposedPorts[p] @@ -836,10 +844,7 @@ func (container *Container) waitForStart() error { return nil } -// Like waitForStart() but for restoring a container. -// -// XXX Does RestartPolicy apply here? -func (container *Container) waitForRestore() error { +func (container *Container) waitForRestore(opts *runconfig.CriuConfig, forceRestore bool) error { container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) // After calling promise.Go() we'll have two goroutines: @@ -852,7 +857,7 @@ func (container *Container) waitForRestore() error { if container.ExitCode != 0 { return fmt.Errorf("restore process failed") } - case err := <-promise.Go(container.monitor.Restore): + case err := <-promise.Go(func() error { return container.monitor.Restore(opts, forceRestore) }): return err } @@ -1058,6 +1063,7 @@ func attach(streamConfig *streamConfig, openStdin, stdinOnce, tty bool, stdin io _, err = copyEscapable(cStdin, stdin) } else { _, err = io.Copy(cStdin, stdin) + } if err == io.ErrClosedPipe { err = nil diff --git a/daemon/container_unix.go b/daemon/container_unix.go index 38948cf41dad3..d1f82d9d58d20 100644 --- a/daemon/container_unix.go +++ b/daemon/container_unix.go @@ -328,53 +328,6 @@ func mergeDevices(defaultDevices, userDevices []*configs.Device) []*configs.Devi return append(devs, userDevices...) } -// Like populateCommand() but for restoring a container. -// -// XXX populateCommand() does a lot more. Not sure if we have -// to do everything it does. -func populateCommandRestore(c *Container, env []string) error { - resources := &execdriver.Resources{ - Memory: c.Config.Memory, - MemorySwap: c.Config.MemorySwap, - CpuShares: c.Config.CpuShares, - Cpuset: c.Config.Cpuset, - } - - processConfig := execdriver.ProcessConfig{ - Privileged: c.hostConfig.Privileged, - Entrypoint: c.Path, - Arguments: c.Args, - Tty: c.Config.Tty, - User: c.Config.User, - } - - processConfig.SysProcAttr = &syscall.SysProcAttr{Setsid: true} - processConfig.Env = env - - c.command = &execdriver.Command{ - ID: c.ID, - Rootfs: c.RootfsPath(), - ReadonlyRootfs: c.hostConfig.ReadonlyRootfs, - InitPath: "/.dockerinit", - WorkingDir: c.Config.WorkingDir, - // Network: en, - // Ipc: ipc, - // Pid: pid, - Resources: resources, - // AllowedDevices: allowedDevices, - // AutoCreatedDevices: autoCreatedDevices, - CapAdd: c.hostConfig.CapAdd, - CapDrop: c.hostConfig.CapDrop, - ProcessConfig: processConfig, - ProcessLabel: c.GetProcessLabel(), - MountLabel: c.GetMountLabel(), - // LxcConfig: lxcConfig, - AppArmorProfile: c.AppArmorProfile, - } - - return nil -} - // GetSize, return real size, virtual size func (container *Container) getSize() (int64, int64) { var ( @@ -737,7 +690,7 @@ func (container *Container) updateNetwork() error { return nil } -func (container *Container) buildCreateEndpointOptions() ([]libnetwork.EndpointOption, error) { +func (container *Container) buildCreateEndpointOptions(restoring bool) ([]libnetwork.EndpointOption, error) { var ( portSpecs = make(nat.PortSet) bindings = make(nat.PortMap) @@ -817,6 +770,14 @@ func (container *Container) buildCreateEndpointOptions() ([]libnetwork.EndpointO createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(genericOption)) } + /*if restoring && container.NetworkSettings.IPAddress != "" { + genericOption := options.Generic{ + netlabel.IPAddress: net.ParseIP(container.NetworkSettings.IPAddress), + } + + createOptions = append(createOptions, libnetwork.EndpointOptionGeneric(genericOption)) + }*/ + return createOptions, nil } @@ -870,7 +831,7 @@ func (container *Container) secondaryNetworkRequired(primaryNetworkType string) return false } -func (container *Container) allocateNetwork() error { +func (container *Container) allocateNetwork(isRestoring bool) error { mode := container.hostConfig.NetworkMode controller := container.daemon.netController if container.Config.NetworkDisabled || mode.IsContainer() { @@ -906,19 +867,19 @@ func (container *Container) allocateNetwork() error { if container.secondaryNetworkRequired(networkDriver) { // Configure Bridge as secondary network for port binding purposes - if err := container.configureNetwork("bridge", service, "bridge", false); err != nil { + if err := container.configureNetwork("bridge", service, "bridge", false, isRestoring); err != nil { return err } } - if err := container.configureNetwork(networkName, service, networkDriver, mode.IsDefault()); err != nil { + if err := container.configureNetwork(networkName, service, networkDriver, mode.IsDefault(), isRestoring); err != nil { return err } return container.writeHostConfig() } -func (container *Container) configureNetwork(networkName, service, networkDriver string, canCreateNetwork bool) error { +func (container *Container) configureNetwork(networkName, service, networkDriver string, canCreateNetwork bool, isRestoring bool) error { controller := container.daemon.netController n, err := controller.NetworkByName(networkName) if err != nil { @@ -937,7 +898,7 @@ func (container *Container) configureNetwork(networkName, service, networkDriver return err } - createOptions, err := container.buildCreateEndpointOptions() + createOptions, err := container.buildCreateEndpointOptions(isRestoring) if err != nil { return err } @@ -968,7 +929,7 @@ func (container *Container) configureNetwork(networkName, service, networkDriver return nil } -func (container *Container) initializeNetworking() error { +func (container *Container) initializeNetworking(restoring bool) error { var err error if container.hostConfig.NetworkMode.IsContainer() { @@ -999,7 +960,7 @@ func (container *Container) initializeNetworking() error { } - if err := container.allocateNetwork(); err != nil { + if err := container.allocateNetwork(restoring); err != nil { return err } diff --git a/daemon/container_windows.go b/daemon/container_windows.go index 03665186c0e32..a7ccd9d1ecbf3 100644 --- a/daemon/container_windows.go +++ b/daemon/container_windows.go @@ -35,7 +35,7 @@ func (container *Container) createDaemonEnvironment(linkedEnv []string) []string return container.Config.Env } -func (container *Container) initializeNetworking() error { +func (container *Container) initializeNetworking(restoring bool) error { return nil } diff --git a/daemon/daemon.go b/daemon/daemon.go index 532bb44fad27e..71f684c3de75e 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -889,22 +889,22 @@ func (daemon *Daemon) run(c *Container, pipes *execdriver.Pipes, startCallback e return daemon.execDriver.Run(c.command, pipes, startCallback) } -func (daemon *Daemon) Checkpoint(c *Container) error { - if err := daemon.execDriver.Checkpoint(c.command); err != nil { +func (daemon *Daemon) Checkpoint(c *Container, opts *runconfig.CriuConfig) error { + if err := daemon.execDriver.Checkpoint(c.command, opts); err != nil { return err } - c.SetCheckpointed() + c.SetCheckpointed(opts.LeaveRunning) return nil } -func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback) (int, error) { +func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { // Mount the container's filesystem (daemon/graphdriver/aufs/aufs.go). _, err := daemon.driver.Get(c.ID, c.GetMountLabel()) if err != nil { - return 0, err + return execdriver.ExitStatus{ExitCode: 0}, err } - exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback) + exitCode, err := daemon.execDriver.Restore(c.command, pipes, restoreCallback, opts, forceRestore) return exitCode, err } diff --git a/daemon/execdriver/driver.go b/daemon/execdriver/driver.go index bb90515395458..4cdea701d1c50 100644 --- a/daemon/execdriver/driver.go +++ b/daemon/execdriver/driver.go @@ -8,6 +8,7 @@ import ( // TODO Windows: Factor out ulimit "github.com/docker/docker/pkg/ulimit" + "github.com/docker/docker/runconfig" "github.com/opencontainers/runc/libcontainer" "github.com/opencontainers/runc/libcontainer/configs" ) @@ -72,9 +73,11 @@ type Driver interface { // Unpause unpauses a container. Unpause(c *Command) error - Checkpoint(c *Command, opts *libcontainer.CriuOpts) error + // Checkpoints a container (with criu). + Checkpoint(c *Command, opts *runconfig.CriuConfig) error - Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (ExitStatus, error) + // Restores a checkpoint image into a container (with criu). + Restore(c *Command, pipes *Pipes, restoreCallback RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (ExitStatus, error) // Name returns the name of the driver. Name() string diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index 26866b7efd15f..e8196316098ad 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -25,6 +25,7 @@ import ( sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" "github.com/docker/docker/pkg/version" + "github.com/docker/docker/runconfig" "github.com/kr/pty" "github.com/opencontainers/runc/libcontainer" "github.com/opencontainers/runc/libcontainer/cgroups" @@ -560,11 +561,11 @@ func (d *Driver) Unpause(c *execdriver.Command) error { return err } -func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { +func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { return fmt.Errorf("Checkpointing lxc containers not supported yet\n") } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { return execdriver.ExitStatus{ExitCode: 0}, fmt.Errorf("Restoring lxc containers not supported yet\n") } diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 90924230538ab..e9bd2d981600f 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -20,7 +20,7 @@ import ( "github.com/docker/docker/pkg/reexec" sysinfo "github.com/docker/docker/pkg/system" "github.com/docker/docker/pkg/term" - "github.com/docker/docker/utils" + "github.com/docker/docker/runconfig" "github.com/opencontainers/runc/libcontainer" "github.com/opencontainers/runc/libcontainer/apparmor" "github.com/opencontainers/runc/libcontainer/cgroups/systemd" @@ -299,7 +299,18 @@ func (d *Driver) Unpause(c *execdriver.Command) error { return active.Resume() } -func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) error { +func libcontainerCriuOpts(runconfigOpts *runconfig.CriuConfig) *libcontainer.CriuOpts { + return &libcontainer.CriuOpts{ + ImagesDirectory: runconfigOpts.ImagesDirectory, + WorkDirectory: runconfigOpts.WorkDirectory, + LeaveRunning: runconfigOpts.LeaveRunning, + TcpEstablished: runconfigOpts.TcpEstablished, + ExternalUnixConnections: runconfigOpts.ExternalUnixConnections, + ShellJob: runconfigOpts.ShellJob, + } +} + +func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { active := d.activeContainers[c.ID] if active == nil { return fmt.Errorf("active container for %s does not exist", c.ID) @@ -307,7 +318,7 @@ func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) d.Lock() defer d.Unlock() - err := active.Checkpoint(opts) + err := active.Checkpoint(libcontainerCriuOpts(opts)) if err != nil { return err } @@ -315,7 +326,7 @@ func (d *driver) Checkpoint(c *execdriver.Command, opts *libcontainer.CriuOpts) return nil } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *libcontainer.CriuOpts, forceRestore bool) (execdriver.ExitStatus, error) { +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { var ( cont libcontainer.Container err error @@ -358,7 +369,7 @@ func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restore d.cleanContainer(c.ID) }() - if err := cont.Restore(p, opts); err != nil { + if err := cont.Restore(p, libcontainerCriuOpts(opts)); err != nil { return execdriver.ExitStatus{ExitCode: -1}, err } diff --git a/daemon/execdriver/windows/windows.go b/daemon/execdriver/windows/windows.go index 198ddc8dd7184..c87c50e230524 100644 --- a/daemon/execdriver/windows/windows.go +++ b/daemon/execdriver/windows/windows.go @@ -11,6 +11,7 @@ import ( "github.com/docker/docker/autogen/dockerversion" "github.com/docker/docker/daemon/execdriver" "github.com/docker/docker/pkg/parsers" + "github.com/docker/docker/runconfig" ) // This is a daemon development variable only and should not be @@ -93,3 +94,11 @@ func setupEnvironmentVariables(a []string) map[string]string { } return r } + +func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { + return fmt.Errorf("Windows: Containers cannot be checkpointed") +} + +func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { + return execdriver.ExitStatus{ExitCode: 0}, fmt.Errorf("Windows: Containers cannot be restored") +} diff --git a/daemon/monitor.go b/daemon/monitor.go index 5bf1088f0950e..fede2fb70035b 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -193,43 +193,45 @@ func (m *containerMonitor) Start() error { } // Like Start() but for restoring a container. -func (m *containerMonitor) Restore() error { +func (m *containerMonitor) Restore(opts *runconfig.CriuConfig, forceRestore bool) error { var ( err error // XXX The following line should be changed to // exitStatus execdriver.ExitStatus to match Start() - exitCode int + exitCode execdriver.ExitStatus afterRestore bool ) - defer func() { if afterRestore { m.container.Lock() - m.container.setStopped(&execdriver.ExitStatus{exitCode, false}) + m.container.setStopped(&execdriver.ExitStatus{exitCode.ExitCode, false}) defer m.container.Unlock() } m.Close() }() - if err := m.container.startLoggingToDisk(); err != nil { - m.resetContainer(false) - return err + // FIXME: right now if we startLogging again we get double logs after a restore + if m.container.logCopier == nil { + if err := m.container.startLogging(); err != nil { + m.resetContainer(false) + return err + } } pipes := execdriver.NewPipes(m.container.stdin, m.container.stdout, m.container.stderr, m.container.Config.OpenStdin) m.container.LogEvent("restore") m.lastStartTime = time.Now() - if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback); err != nil { - log.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) + if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback, opts, forceRestore); err != nil { + logrus.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) m.container.ExitCode = -1 m.resetContainer(false) return err } afterRestore = true - m.container.ExitCode = exitCode - m.resetMonitor(err == nil && exitCode == 0) + m.container.ExitCode = exitCode.ExitCode + m.resetMonitor(err == nil && exitCode.ExitCode == 0) m.container.LogEvent("die") m.resetContainer(true) return err @@ -340,7 +342,7 @@ func (m *containerMonitor) restoreCallback(processConfig *execdriver.ProcessConf // Write config.json and hostconfig.json files // to /var/lib/docker/containers/. if err := m.container.ToDisk(); err != nil { - log.Debugf("%s", err) + logrus.Debugf("%s", err) } } } diff --git a/daemon/state.go b/daemon/state.go index 49e455ecb5c4a..1bddf24dd9e8b 100644 --- a/daemon/state.go +++ b/daemon/state.go @@ -30,7 +30,6 @@ type State struct { FinishedAt time.Time CheckpointedAt time.Time waitChan chan struct{} - } // NewState creates a default state object with a fresh channel for state changes. @@ -51,14 +50,16 @@ func (s *State) String() string { } return fmt.Sprintf("Up %s", units.HumanDuration(time.Now().UTC().Sub(s.StartedAt))) - } else if s.Checkpointed { - return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) } if s.removalInProgress { return "Removal In Progress" } + if s.Checkpointed { + return fmt.Sprintf("Checkpointed %s ago", units.HumanDuration(time.Now().UTC().Sub(s.CheckpointedAt))) + } + if s.Dead { return "Dead" } @@ -86,6 +87,10 @@ func (s *State) StateString() string { return "running" } + if s.Checkpointed { + return "checkpointed'" + } + if s.Dead { return "dead" } @@ -275,11 +280,11 @@ func (s *State) setDead() { s.Unlock() } -func (s *State) SetCheckpointed() { +func (s *State) SetCheckpointed(leaveRunning bool) { s.Lock() s.CheckpointedAt = time.Now().UTC() - s.Checkpointed = true - s.Running = false + s.Checkpointed = !leaveRunning + s.Running = leaveRunning s.Paused = false s.Restarting = false // XXX Not sure if we need to close and recreate waitChan. @@ -288,6 +293,10 @@ func (s *State) SetCheckpointed() { s.Unlock() } +func (s *State) HasBeenCheckpointed() bool { + return s.CheckpointedAt != time.Time{} +} + func (s *State) IsCheckpointed() bool { return s.Checkpointed } diff --git a/docker/flags.go b/docker/flags.go index 1e234a28eada3..ed4ad3b1b7a17 100644 --- a/docker/flags.go +++ b/docker/flags.go @@ -23,6 +23,7 @@ func (a byName) Less(i, j int) bool { return a[i].name < a[j].name } var dockerCommands = []command{ {"attach", "Attach to a running container"}, {"build", "Build an image from a Dockerfile"}, + {"checkpoint", "Checkpoint one or more running containers"}, {"commit", "Create a new image from a container's changes"}, {"cp", "Copy files/folders from a container to a HOSTDIR or to STDOUT"}, {"create", "Create a new container"}, @@ -47,6 +48,7 @@ var dockerCommands = []command{ {"push", "Push an image or a repository to a registry"}, {"rename", "Rename a container"}, {"restart", "Restart a running container"}, + {"restore", "Restore one or more checkpointed containers"}, {"rm", "Remove one or more containers"}, {"rmi", "Remove one or more images"}, {"run", "Run a command in a new container"}, diff --git a/runconfig/restore.go b/runconfig/restore.go new file mode 100644 index 0000000000000..22f8b0ab0a096 --- /dev/null +++ b/runconfig/restore.go @@ -0,0 +1,15 @@ +package runconfig + +type CriuConfig struct { + ImagesDirectory string + WorkDirectory string + LeaveRunning bool + TcpEstablished bool + ExternalUnixConnections bool + ShellJob bool +} + +type RestoreConfig struct { + CriuOpts CriuConfig + ForceRestore bool +} From f8fc24fc1b9eff07347e582a114007b7d4786327 Mon Sep 17 00:00:00 2001 From: boucher Date: Tue, 2 Jun 2015 14:04:14 -0700 Subject: [PATCH 06/16] Add compilation steps for Criu to the Dockerfile Add a basic test for checkpoint/restore to the integration tests Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- Dockerfile | 18 +++++++++ api/types/types.go | 22 ++++++----- daemon/inspect.go | 24 ++++++------ integration-cli/docker_cli_checkpoint_test.go | 37 +++++++++++++++++++ 4 files changed, 80 insertions(+), 21 deletions(-) create mode 100644 integration-cli/docker_cli_checkpoint_test.go diff --git a/Dockerfile b/Dockerfile index 0e58df106531b..22294606cca1f 100644 --- a/Dockerfile +++ b/Dockerfile @@ -32,9 +32,11 @@ RUN echo deb http://ppa.launchpad.net/zfs-native/stable/ubuntu trusty main > /et # Packaged dependencies RUN apt-get update && apt-get install -y \ apparmor \ + asciidoc \ aufs-tools \ automake \ bash-completion \ + bsdmainutils \ btrfs-tools \ build-essential \ createrepo \ @@ -43,19 +45,28 @@ RUN apt-get update && apt-get install -y \ gcc-mingw-w64 \ git \ iptables \ + libaio-dev \ libapparmor-dev \ libcap-dev \ + libprotobuf-c0-dev \ + libprotobuf-dev \ libsqlite3-dev \ mercurial \ parallel \ + pkg-config \ + protobuf-compiler \ + protobuf-c-compiler \ + python-minimal \ python-mock \ python-pip \ + python-protobuf \ python-websocket \ reprepro \ ruby1.9.1 \ ruby1.9.1-dev \ s3cmd=1.1.0* \ ubuntu-zfs \ + xmlto \ libzfs-dev \ --no-install-recommends @@ -80,6 +91,13 @@ RUN cd /usr/src/lxc \ && make install \ && ldconfig +# Install Criu +RUN mkdir -p /usr/src/criu \ + && curl -sSL https://github.com/xemul/criu/archive/v1.6.tar.gz | tar -v -C /usr/src/criu/ -xz --strip-components=1 +RUN cd /usr/src/criu \ + && make \ + && make install + # Install Go ENV GO_VERSION 1.4.2 RUN curl -sSL https://golang.org/dl/go${GO_VERSION}.src.tar.gz | tar -v -C /usr/local -xz \ diff --git a/api/types/types.go b/api/types/types.go index 66c3f820780ca..63f4e133f83b5 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -228,16 +228,18 @@ type ExecStartCheck struct { // it's part of ContainerJSONBase and will return by "inspect" command type ContainerState struct { Status string - Running bool - Paused bool - Restarting bool - OOMKilled bool - Dead bool - Pid int - ExitCode int - Error string - StartedAt string - FinishedAt string + Running bool + Paused bool + Checkpointed bool + Restarting bool + OOMKilled bool + Dead bool + Pid int + ExitCode int + Error string + StartedAt string + FinishedAt string + CheckpointedAt string } // ContainerJSONBase contains response of Remote API: diff --git a/daemon/inspect.go b/daemon/inspect.go index 1c4a65fca7dd1..bb80a24ee3529 100644 --- a/daemon/inspect.go +++ b/daemon/inspect.go @@ -49,17 +49,19 @@ func (daemon *Daemon) getInspectData(container *Container) (*types.ContainerJSON } containerState := &types.ContainerState{ - Status: container.State.StateString(), - Running: container.State.Running, - Paused: container.State.Paused, - Restarting: container.State.Restarting, - OOMKilled: container.State.OOMKilled, - Dead: container.State.Dead, - Pid: container.State.Pid, - ExitCode: container.State.ExitCode, - Error: container.State.Error, - StartedAt: container.State.StartedAt.Format(time.RFC3339Nano), - FinishedAt: container.State.FinishedAt.Format(time.RFC3339Nano), + Status: container.State.StateString(), + Running: container.State.Running, + Paused: container.State.Paused, + Checkpointed: container.State.Checkpointed, + Restarting: container.State.Restarting, + OOMKilled: container.State.OOMKilled, + Dead: container.State.Dead, + Pid: container.State.Pid, + ExitCode: container.State.ExitCode, + Error: container.State.Error, + StartedAt: container.State.StartedAt.Format(time.RFC3339Nano), + FinishedAt: container.State.FinishedAt.Format(time.RFC3339Nano), + CheckpointedAt: container.State.CheckpointedAt.Format(time.RFC3339Nano), } contJSONBase := &types.ContainerJSONBase{ diff --git a/integration-cli/docker_cli_checkpoint_test.go b/integration-cli/docker_cli_checkpoint_test.go new file mode 100644 index 0000000000000..e19ef524efd43 --- /dev/null +++ b/integration-cli/docker_cli_checkpoint_test.go @@ -0,0 +1,37 @@ +package main + +import ( + "os/exec" + "strings" + + "github.com/go-check/check" +) + +func (s *DockerSuite) TestCheckpointAndRestore(c *check.C) { + defer unpauseAllContainers() + + runCmd := exec.Command(dockerBinary, "run", "-d", "busybox", "top") + out, _, err := runCommandWithOutput(runCmd) + if err != nil { + c.Fatalf("failed to run container: %v, output: %q", err, out) + } + + containerID := strings.TrimSpace(out) + checkpointCmd := exec.Command(dockerBinary, "checkpoint", containerID) + out, _, err = runCommandWithOutput(checkpointCmd) + if err != nil { + c.Fatalf("failed to checkpoint container: %v, output: %q", err, out) + } + + out, err = inspectField(containerID, "State.Checkpointed") + c.Assert(out, check.Equals, "true") + + restoreCmd := exec.Command(dockerBinary, "restore", containerID) + out, _, _, err = runCommandWithStdoutStderr(restoreCmd) + if err != nil { + c.Fatalf("failed to restore container: %v, output: %q", err, out) + } + + out, err = inspectField(containerID, "State.Checkpointed") + c.Assert(out, check.Equals, "false") +} From b694a0941ba07fff67dd64c99406e4963d31e6eb Mon Sep 17 00:00:00 2001 From: boucher Date: Tue, 16 Jun 2015 14:41:05 -0700 Subject: [PATCH 07/16] Add optional dependency info to the PACKAGERS file. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- project/PACKAGERS.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/project/PACKAGERS.md b/project/PACKAGERS.md index 22f24b4789b77..5ba406cedf009 100644 --- a/project/PACKAGERS.md +++ b/project/PACKAGERS.md @@ -303,6 +303,9 @@ by having support for them in the kernel or userspace. A few examples include: least the "auplink" utility from aufs-tools) * BTRFS graph driver (requires BTRFS support enabled in the kernel) * ZFS graph driver (requires userspace zfs-utils and a corresponding kernel module) +* Checkpoint/Restore containers: + - requires criu version 1.5.2 or later (criu.org) + - requires kernel version 3.19 or later if using overlay-fs ## Daemon Init Script From 07458debb33a9d9ea538b8f15a9669b759b62ab8 Mon Sep 17 00:00:00 2001 From: boucher Date: Thu, 18 Jun 2015 15:18:09 -0700 Subject: [PATCH 08/16] Don't destroy/delete the container if it has been checkpointed. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- daemon/execdriver/native/driver.go | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index e9bd2d981600f..f4f34095ee50a 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -158,8 +158,11 @@ func (d *Driver) Run(c *execdriver.Command, pipes *execdriver.Pipes, startCallba d.activeContainers[c.ID] = cont d.Unlock() defer func() { - cont.Destroy() - d.cleanContainer(c.ID) + status, err := cont.Status() + if err != nil || status != libcontainer.Checkpointed { + cont.Destroy() + d.cleanContainer(c.ID) + } }() if err := cont.Start(p); err != nil { From 087d6819b037dc767b6f3ed3bc7060a4a5730565 Mon Sep 17 00:00:00 2001 From: boucher Date: Thu, 9 Jul 2015 09:40:43 -0700 Subject: [PATCH 09/16] Move checkpoint methods into a separate container_checkpoint file. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- api/types/types.go | 2 +- daemon/container.go | 60 ++---------------------- daemon/container_checkpoint.go | 84 ++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 57 deletions(-) create mode 100644 daemon/container_checkpoint.go diff --git a/api/types/types.go b/api/types/types.go index 63f4e133f83b5..147fd6c475af2 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -227,7 +227,7 @@ type ExecStartCheck struct { // ContainerState stores container's running state // it's part of ContainerJSONBase and will return by "inspect" command type ContainerState struct { - Status string + Status string Running bool Paused bool Checkpointed bool diff --git a/daemon/container.go b/daemon/container.go index ed2af5fc5d479..dba11aa3b5694 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -634,62 +634,6 @@ func validateID(id string) error { return nil } -func (container *Container) Checkpoint(opts *runconfig.CriuConfig) error { - if err := container.daemon.Checkpoint(container, opts); err != nil { - return err - } - - if opts.LeaveRunning == false { - container.ReleaseNetwork() - } - return nil -} - -func (container *Container) Restore(opts *runconfig.CriuConfig, forceRestore bool) error { - var err error - container.Lock() - defer container.Unlock() - - defer func() { - if err != nil { - container.setError(err) - // if no one else has set it, make sure we don't leave it at zero - if container.ExitCode == 0 { - container.ExitCode = 128 - } - container.toDisk() - container.cleanup() - } - }() - - if err := container.Mount(); err != nil { - return err - } - if err = container.initializeNetworking(true); err != nil { - return err - } - linkedEnv, err := container.setupLinkedContainers() - if err != nil { - return err - } - if err = container.setupWorkingDirectory(); err != nil { - return err - } - - env := container.createDaemonEnvironment(linkedEnv) - if err = populateCommand(container, env); err != nil { - return err - } - - mounts, err := container.setupMounts() - if err != nil { - return err - } - - container.command.Mounts = mounts - return container.waitForRestore(opts, forceRestore) -} - func (container *Container) copy(resource string) (rc io.ReadCloser, err error) { container.Lock() @@ -844,6 +788,7 @@ func (container *Container) waitForStart() error { return nil } +<<<<<<< HEAD func (container *Container) waitForRestore(opts *runconfig.CriuConfig, forceRestore bool) error { container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) @@ -865,6 +810,9 @@ func (container *Container) waitForRestore(opts *runconfig.CriuConfig, forceRest } func (container *Container) getProcessLabel() string { +======= +func (container *Container) GetProcessLabel() string { +>>>>>>> Move checkpoint methods into a separate container_checkpoint file. // even if we have a process label return "" if we are running // in privileged mode if container.hostConfig.Privileged { diff --git a/daemon/container_checkpoint.go b/daemon/container_checkpoint.go new file mode 100644 index 0000000000000..468816e448dc1 --- /dev/null +++ b/daemon/container_checkpoint.go @@ -0,0 +1,84 @@ +package daemon + +import ( + "fmt" + + "github.com/docker/docker/pkg/promise" + "github.com/docker/docker/runconfig" +) + +func (container *Container) Checkpoint(opts *runconfig.CriuConfig) error { + if err := container.daemon.Checkpoint(container, opts); err != nil { + return err + } + + if opts.LeaveRunning == false { + container.ReleaseNetwork() + } + return nil +} + +func (container *Container) Restore(opts *runconfig.CriuConfig, forceRestore bool) error { + var err error + container.Lock() + defer container.Unlock() + + defer func() { + if err != nil { + container.setError(err) + // if no one else has set it, make sure we don't leave it at zero + if container.ExitCode == 0 { + container.ExitCode = 128 + } + container.toDisk() + container.cleanup() + } + }() + + if err := container.Mount(); err != nil { + return err + } + if err = container.initializeNetworking(true); err != nil { + return err + } + linkedEnv, err := container.setupLinkedContainers() + if err != nil { + return err + } + if err = container.setupWorkingDirectory(); err != nil { + return err + } + + env := container.createDaemonEnvironment(linkedEnv) + if err = populateCommand(container, env); err != nil { + return err + } + + mounts, err := container.setupMounts() + if err != nil { + return err + } + + container.command.Mounts = mounts + return container.waitForRestore(opts, forceRestore) +} + +func (container *Container) waitForRestore(opts *runconfig.CriuConfig, forceRestore bool) error { + container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) + + // After calling promise.Go() we'll have two goroutines: + // - The current goroutine that will block in the select + // below until restore is done. + // - A new goroutine that will restore the container and + // wait for it to exit. + select { + case <-container.monitor.restoreSignal: + if container.ExitCode != 0 { + return fmt.Errorf("restore process failed") + } + case err := <-promise.Go(func() error { return container.monitor.Restore(opts, forceRestore) }): + return err + } + + return nil +} From 717dc4b7fc5b7affaff9f16f38c6224250a34c99 Mon Sep 17 00:00:00 2001 From: boucher Date: Fri, 17 Jul 2015 12:31:05 -0700 Subject: [PATCH 10/16] Move checkpoint/restore interface into docker experimental build. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- api/client/checkpoint.go | 2 + api/client/restore.go | 2 + api/server/server.go | 92 +++++-------------- api/server/server_experimental_unix.go | 47 ++++++++++ api/server/server_stub.go | 4 + api/types/types.go | 2 +- daemon/container.go | 5 +- daemon/daemon.go | 8 +- docker/docker.go | 6 +- docker/flags.go | 2 - docker/flags_experimental.go | 10 ++ docker/flags_stub.go | 7 ++ integration-cli/docker_cli_checkpoint_test.go | 2 + .../docker_cli_help_experimental_test.go | 5 + .../docker_cli_help_standard_test.go | 5 + integration-cli/docker_cli_help_test.go | 6 +- 16 files changed, 127 insertions(+), 78 deletions(-) create mode 100644 docker/flags_experimental.go create mode 100644 docker/flags_stub.go create mode 100644 integration-cli/docker_cli_help_experimental_test.go create mode 100644 integration-cli/docker_cli_help_standard_test.go diff --git a/api/client/checkpoint.go b/api/client/checkpoint.go index 8c681bcf9716f..02990d9499adb 100644 --- a/api/client/checkpoint.go +++ b/api/client/checkpoint.go @@ -1,3 +1,5 @@ +// +build experimental + package client import ( diff --git a/api/client/restore.go b/api/client/restore.go index 0c4085fbbbd84..013acb4cf04b0 100644 --- a/api/client/restore.go +++ b/api/client/restore.go @@ -1,3 +1,5 @@ +// +build experimental + package client import ( diff --git a/api/server/server.go b/api/server/server.go index 39552eb3c1e93..e7d11e4aa9ff1 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -220,48 +220,6 @@ func writeJSON(w http.ResponseWriter, code int, v interface{}) error { return json.NewEncoder(w).Encode(v) } -func (s *Server) postContainersCheckpoint(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if vars == nil { - return fmt.Errorf("Missing parameter") - } - if err := parseForm(r); err != nil { - return err - } - - criuOpts := &runconfig.CriuConfig{} - if err := json.NewDecoder(r.Body).Decode(criuOpts); err != nil { - return err - } - - if err := s.daemon.ContainerCheckpoint(vars["name"], criuOpts); err != nil { - return err - } - - w.WriteHeader(http.StatusNoContent) - return nil -} - -func (s *Server) postContainersRestore(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - if vars == nil { - return fmt.Errorf("Missing parameter") - } - if err := parseForm(r); err != nil { - return err - } - - restoreOpts := runconfig.RestoreConfig{} - if err := json.NewDecoder(r.Body).Decode(&restoreOpts); err != nil { - return err - } - - if err := s.daemon.ContainerRestore(vars["name"], &restoreOpts.CriuOpts, restoreOpts.ForceRestore); err != nil { - return err - } - - w.WriteHeader(http.StatusNoContent) - return nil -} - func (s *Server) optionsHandler(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { w.WriteHeader(http.StatusOK) return nil @@ -375,31 +333,29 @@ func createRouter(s *Server) *mux.Router { "/volumes/{name:.*}": s.getVolumeByName, }, "POST": { - "/auth": s.postAuth, - "/commit": s.postCommit, - "/build": s.postBuild, - "/images/create": s.postImagesCreate, - "/images/load": s.postImagesLoad, - "/images/{name:.*}/push": s.postImagesPush, - "/images/{name:.*}/tag": s.postImagesTag, - "/containers/create": s.postContainersCreate, - "/containers/{name:.*}/kill": s.postContainersKill, - "/containers/{name:.*}/pause": s.postContainersPause, - "/containers/{name:.*}/unpause": s.postContainersUnpause, - "/containers/{name:.*}/restart": s.postContainersRestart, - "/containers/{name:.*}/start": s.postContainersStart, - "/containers/{name:.*}/stop": s.postContainersStop, - "/containers/{name:.*}/wait": s.postContainersWait, - "/containers/{name:.*}/resize": s.postContainersResize, - "/containers/{name:.*}/attach": s.postContainersAttach, - "/containers/{name:.*}/copy": s.postContainersCopy, - "/containers/{name:.*}/exec": s.postContainerExecCreate, - "/exec/{name:.*}/start": s.postContainerExecStart, - "/exec/{name:.*}/resize": s.postContainerExecResize, - "/containers/{name:.*}/rename": s.postContainerRename, - "/containers/{name:.*}/checkpoint": s.postContainersCheckpoint, - "/containers/{name:.*}/restore": s.postContainersRestore, - "/volumes": s.postVolumesCreate, + "/auth": s.postAuth, + "/commit": s.postCommit, + "/build": s.postBuild, + "/images/create": s.postImagesCreate, + "/images/load": s.postImagesLoad, + "/images/{name:.*}/push": s.postImagesPush, + "/images/{name:.*}/tag": s.postImagesTag, + "/containers/create": s.postContainersCreate, + "/containers/{name:.*}/kill": s.postContainersKill, + "/containers/{name:.*}/pause": s.postContainersPause, + "/containers/{name:.*}/unpause": s.postContainersUnpause, + "/containers/{name:.*}/restart": s.postContainersRestart, + "/containers/{name:.*}/start": s.postContainersStart, + "/containers/{name:.*}/stop": s.postContainersStop, + "/containers/{name:.*}/wait": s.postContainersWait, + "/containers/{name:.*}/resize": s.postContainersResize, + "/containers/{name:.*}/attach": s.postContainersAttach, + "/containers/{name:.*}/copy": s.postContainersCopy, + "/containers/{name:.*}/exec": s.postContainerExecCreate, + "/exec/{name:.*}/start": s.postContainerExecStart, + "/exec/{name:.*}/resize": s.postContainerExecResize, + "/containers/{name:.*}/rename": s.postContainerRename, + "/volumes": s.postVolumesCreate, }, "PUT": { "/containers/{name:.*}/archive": s.putContainersArchive, @@ -414,6 +370,8 @@ func createRouter(s *Server) *mux.Router { }, } + addExperimentalRoutes(s, &m) + // If "api-cors-header" is not given, but "api-enable-cors" is true, we set cors to "*" // otherwise, all head values will be passed to HTTP handler corsHeaders := s.cfg.CorsHeaders diff --git a/api/server/server_experimental_unix.go b/api/server/server_experimental_unix.go index 20b3292cc6881..d08ff72f03c00 100644 --- a/api/server/server_experimental_unix.go +++ b/api/server/server_experimental_unix.go @@ -2,6 +2,11 @@ package server +func addExperimentalRoutes(s *Server, m *map[string]map[string]HttpApiFunc) { + m["GET"]["/containers/{name:.*}/checkpoint"] = s.postContainersCheckpoint + m["GET"]["/containers/{name:.*}/restore"] = s.postContainersRestore +} + func (s *Server) registerSubRouter() { httpHandler := s.daemon.NetworkAPIRouter() @@ -15,3 +20,45 @@ func (s *Server) registerSubRouter() { subrouter = s.router.PathPrefix("/services").Subrouter() subrouter.Methods("GET", "POST", "PUT", "DELETE").HandlerFunc(httpHandler) } + +func (s *Server) postContainersCheckpoint(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + if err := parseForm(r); err != nil { + return err + } + + criuOpts := &runconfig.CriuConfig{} + if err := json.NewDecoder(r.Body).Decode(criuOpts); err != nil { + return err + } + + if err := s.daemon.ContainerCheckpoint(vars["name"], criuOpts); err != nil { + return err + } + + w.WriteHeader(http.StatusNoContent) + return nil +} + +func (s *Server) postContainersRestore(version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + if err := parseForm(r); err != nil { + return err + } + + restoreOpts := runconfig.RestoreConfig{} + if err := json.NewDecoder(r.Body).Decode(&restoreOpts); err != nil { + return err + } + + if err := s.daemon.ContainerRestore(vars["name"], &restoreOpts.CriuOpts, restoreOpts.ForceRestore); err != nil { + return err + } + + w.WriteHeader(http.StatusNoContent) + return nil +} diff --git a/api/server/server_stub.go b/api/server/server_stub.go index cae28493836f4..aa5c82b952502 100644 --- a/api/server/server_stub.go +++ b/api/server/server_stub.go @@ -2,5 +2,9 @@ package server +func addExperimentalRoutes(s *Server, m *map[string]map[string]HttpApiFunc) { + +} + func (s *Server) registerSubRouter() { } diff --git a/api/types/types.go b/api/types/types.go index 147fd6c475af2..fcdaf776491dc 100644 --- a/api/types/types.go +++ b/api/types/types.go @@ -239,7 +239,7 @@ type ContainerState struct { Error string StartedAt string FinishedAt string - CheckpointedAt string + CheckpointedAt string `json:"-"` } // ContainerJSONBase contains response of Remote API: diff --git a/daemon/container.go b/daemon/container.go index dba11aa3b5694..47f582543ecdc 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -357,11 +357,12 @@ func (container *Container) isNetworkAllocated() bool { // cleanup releases any network resources allocated to the container along with any rules // around how containers are linked together. It also unmounts the container's root filesystem. func (container *Container) cleanup() { - if container.IsCheckpointed() { + /*if container.IsCheckpointed() { logrus.Debugf("not calling ReleaseNetwork() for checkpointed container %s", container.ID) } else { container.ReleaseNetwork() - } + }*/ + container.ReleaseNetwork() if err := container.Unmount(); err != nil { logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err) diff --git a/daemon/daemon.go b/daemon/daemon.go index 71f684c3de75e..f8f53dbadef7b 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -289,13 +289,13 @@ func (daemon *Daemon) restore() error { // the IP address that it was using. // // XXX We should also reserve host ports (if any). - if container.IsCheckpointed() { - /*err = bridge.ReserveIP(container.ID, container.NetworkSettings.IPAddress) + /*if container.IsCheckpointed() { + err = bridge.ReserveIP(container.ID, container.NetworkSettings.IPAddress) if err != nil { log.Errorf("Failed to reserve IP %s for container %s", container.ID, container.NetworkSettings.IPAddress) - }*/ - } + } + }*/ } else { logrus.Debugf("Cannot load container %s because it was created with another graph driver.", container.ID) } diff --git a/docker/docker.go b/docker/docker.go index 8ad0d13c05c1a..4d3477110c03f 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -35,7 +35,11 @@ func main() { help := "\nCommands:\n" - for _, cmd := range dockerCommands { + // TODO(tiborvass): no need to sort if we ensure dockerCommands is sorted + allCommands := append(dockerCommands, experimentalCommands...) + sort.Sort(byName(allCommands)) + + for _, cmd := range allCommands { help += fmt.Sprintf(" %-10.10s%s\n", cmd.name, cmd.description) } diff --git a/docker/flags.go b/docker/flags.go index ed4ad3b1b7a17..1e234a28eada3 100644 --- a/docker/flags.go +++ b/docker/flags.go @@ -23,7 +23,6 @@ func (a byName) Less(i, j int) bool { return a[i].name < a[j].name } var dockerCommands = []command{ {"attach", "Attach to a running container"}, {"build", "Build an image from a Dockerfile"}, - {"checkpoint", "Checkpoint one or more running containers"}, {"commit", "Create a new image from a container's changes"}, {"cp", "Copy files/folders from a container to a HOSTDIR or to STDOUT"}, {"create", "Create a new container"}, @@ -48,7 +47,6 @@ var dockerCommands = []command{ {"push", "Push an image or a repository to a registry"}, {"rename", "Rename a container"}, {"restart", "Restart a running container"}, - {"restore", "Restore one or more checkpointed containers"}, {"rm", "Remove one or more containers"}, {"rmi", "Remove one or more images"}, {"run", "Run a command in a new container"}, diff --git a/docker/flags_experimental.go b/docker/flags_experimental.go new file mode 100644 index 0000000000000..08893a7cf0b49 --- /dev/null +++ b/docker/flags_experimental.go @@ -0,0 +1,10 @@ +// +build experimental + +package main + +var ( + experimentalCommands = []command{ + {"checkpoint", "Checkpoint one or more running containers"}, + {"restore", "Restore one or more checkpointed containers"}, + } +) diff --git a/docker/flags_stub.go b/docker/flags_stub.go new file mode 100644 index 0000000000000..d627b86a3a8f7 --- /dev/null +++ b/docker/flags_stub.go @@ -0,0 +1,7 @@ +// +build !experimental + +package main + +var ( + experimentalCommands = []command{} +) diff --git a/integration-cli/docker_cli_checkpoint_test.go b/integration-cli/docker_cli_checkpoint_test.go index e19ef524efd43..09ec47a9a0d54 100644 --- a/integration-cli/docker_cli_checkpoint_test.go +++ b/integration-cli/docker_cli_checkpoint_test.go @@ -1,3 +1,5 @@ +// +build experimental + package main import ( diff --git a/integration-cli/docker_cli_help_experimental_test.go b/integration-cli/docker_cli_help_experimental_test.go new file mode 100644 index 0000000000000..f6f7ed5bf070d --- /dev/null +++ b/integration-cli/docker_cli_help_experimental_test.go @@ -0,0 +1,5 @@ +// +build experimental + +package main + +var totalDockerCLICommands = 42 diff --git a/integration-cli/docker_cli_help_standard_test.go b/integration-cli/docker_cli_help_standard_test.go new file mode 100644 index 0000000000000..f960f3450a9d6 --- /dev/null +++ b/integration-cli/docker_cli_help_standard_test.go @@ -0,0 +1,5 @@ +// +build !experimental + +package main + +var totalDockerCLICommands = 40 diff --git a/integration-cli/docker_cli_help_test.go b/integration-cli/docker_cli_help_test.go index f6a6f96db60ee..fa2d51cac2b19 100644 --- a/integration-cli/docker_cli_help_test.go +++ b/integration-cli/docker_cli_help_test.go @@ -238,13 +238,17 @@ func (s *DockerSuite) TestHelpTextVerify(c *check.C) { } +<<<<<<< HEAD expected := 40 +======= + expected := totalDockerCLICommands +>>>>>>> Move checkpoint/restore interface into docker experimental build. if isLocalDaemon { expected++ // for the daemon command } if len(cmds) != expected { c.Fatalf("Wrong # of cmds(%d), it should be: %d\nThe list:\n%q", - len(cmds), expected, cmds) + len(cmds), totalDockerCLICommands, cmds) } } From e0749d4a3689fc9670a582ae827f807c223055e5 Mon Sep 17 00:00:00 2001 From: Hui Kang Date: Wed, 19 Aug 2015 04:09:39 +0000 Subject: [PATCH 11/16] Set Checkpointed state to true in config file after checkpoint Docker-DCO-1.1-Signed-off-by: Hui Kang (github: huikang) --- daemon/container.go | 26 +------------------------- daemon/container_checkpoint.go | 5 +++++ 2 files changed, 6 insertions(+), 25 deletions(-) diff --git a/daemon/container.go b/daemon/container.go index 47f582543ecdc..89109443befc0 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -362,7 +362,7 @@ func (container *Container) cleanup() { } else { container.ReleaseNetwork() }*/ - container.ReleaseNetwork() + container.releaseNetwork() if err := container.Unmount(); err != nil { logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err) @@ -789,31 +789,7 @@ func (container *Container) waitForStart() error { return nil } -<<<<<<< HEAD -func (container *Container) waitForRestore(opts *runconfig.CriuConfig, forceRestore bool) error { - container.monitor = newContainerMonitor(container, container.hostConfig.RestartPolicy) - - // After calling promise.Go() we'll have two goroutines: - // - The current goroutine that will block in the select - // below until restore is done. - // - A new goroutine that will restore the container and - // wait for it to exit. - select { - case <-container.monitor.restoreSignal: - if container.ExitCode != 0 { - return fmt.Errorf("restore process failed") - } - case err := <-promise.Go(func() error { return container.monitor.Restore(opts, forceRestore) }): - return err - } - - return nil -} - func (container *Container) getProcessLabel() string { -======= -func (container *Container) GetProcessLabel() string { ->>>>>>> Move checkpoint methods into a separate container_checkpoint file. // even if we have a process label return "" if we are running // in privileged mode if container.hostConfig.Privileged { diff --git a/daemon/container_checkpoint.go b/daemon/container_checkpoint.go index 468816e448dc1..77f3488fbf157 100644 --- a/daemon/container_checkpoint.go +++ b/daemon/container_checkpoint.go @@ -15,6 +15,11 @@ func (container *Container) Checkpoint(opts *runconfig.CriuConfig) error { if opts.LeaveRunning == false { container.ReleaseNetwork() } + + if err := container.ToDisk(); err != nil { + return fmt.Errorf("Cannot update config for container: %s", err) + } + return nil } From 30457157e9e6202862b4da4e78c88553afcb59b2 Mon Sep 17 00:00:00 2001 From: fl0yd Date: Fri, 17 Jul 2015 16:23:01 -0500 Subject: [PATCH 12/16] imports to fix compilation issues and drop * from map Docker-DCO-1.1-Signed-off-by: Mark Oates fl0yd@me.com (github: fl0yd) --- api/server/server.go | 2 +- api/server/server_experimental_unix.go | 14 +++++++++++--- api/server/server_stub.go | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/api/server/server.go b/api/server/server.go index e7d11e4aa9ff1..dfac3d00558a3 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -370,7 +370,7 @@ func createRouter(s *Server) *mux.Router { }, } - addExperimentalRoutes(s, &m) + addExperimentalRoutes(s, m) // If "api-cors-header" is not given, but "api-enable-cors" is true, we set cors to "*" // otherwise, all head values will be passed to HTTP handler diff --git a/api/server/server_experimental_unix.go b/api/server/server_experimental_unix.go index d08ff72f03c00..b49c66ec793f2 100644 --- a/api/server/server_experimental_unix.go +++ b/api/server/server_experimental_unix.go @@ -2,9 +2,17 @@ package server -func addExperimentalRoutes(s *Server, m *map[string]map[string]HttpApiFunc) { - m["GET"]["/containers/{name:.*}/checkpoint"] = s.postContainersCheckpoint - m["GET"]["/containers/{name:.*}/restore"] = s.postContainersRestore +import ( + "encoding/json" + "fmt" + "net/http" + "github.com/docker/docker/pkg/version" + "github.com/docker/docker/runconfig" + ) + +func addExperimentalRoutes(s *Server, m map[string]map[string]HttpApiFunc) { + m["POST"]["/containers/{name:.*}/checkpoint"] = s.postContainersCheckpoint + m["POST"]["/containers/{name:.*}/restore"] = s.postContainersRestore } func (s *Server) registerSubRouter() { diff --git a/api/server/server_stub.go b/api/server/server_stub.go index aa5c82b952502..ce6669da845be 100644 --- a/api/server/server_stub.go +++ b/api/server/server_stub.go @@ -2,7 +2,7 @@ package server -func addExperimentalRoutes(s *Server, m *map[string]map[string]HttpApiFunc) { +func addExperimentalRoutes(s *Server, m map[string]map[string]HttpApiFunc) { } From 090aedcef7b4d973f6d45a32702ad3501ce8da48 Mon Sep 17 00:00:00 2001 From: boucher Date: Wed, 5 Aug 2015 09:32:41 -0700 Subject: [PATCH 13/16] Update checkpoint/restore to match changes in latest docker. Docker-DCO-1.1-Signed-off-by: Ross Boucher (github: boucher) --- api/client/checkpoint.go | 3 ++- api/client/restore.go | 3 ++- api/server/server_experimental_unix.go | 6 +++--- api/server/server_stub.go | 2 +- daemon/checkpoint.go | 6 +++--- daemon/container_checkpoint.go | 4 ++-- daemon/daemon.go | 2 +- daemon/execdriver/lxc/driver.go | 4 ++-- daemon/execdriver/native/create.go | 18 ------------------ daemon/execdriver/native/driver.go | 4 ++-- daemon/inspect.go | 16 ++++++++-------- daemon/monitor.go | 6 +++--- docker/docker.go | 2 +- 13 files changed, 30 insertions(+), 46 deletions(-) diff --git a/api/client/checkpoint.go b/api/client/checkpoint.go index 02990d9499adb..24fed5f68b634 100644 --- a/api/client/checkpoint.go +++ b/api/client/checkpoint.go @@ -5,12 +5,13 @@ package client import ( "fmt" + Cli "github.com/docker/docker/cli" flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/runconfig" ) func (cli *DockerCli) CmdCheckpoint(args ...string) error { - cmd := cli.Subcmd("checkpoint", []string{"CONTAINER [CONTAINER...]"}, "Checkpoint one or more running containers", true) + cmd := Cli.Subcmd("checkpoint", []string{"CONTAINER [CONTAINER...]"}, "Checkpoint one or more running containers", true) cmd.Require(flag.Min, 1) var ( diff --git a/api/client/restore.go b/api/client/restore.go index 013acb4cf04b0..bef78a262b54a 100644 --- a/api/client/restore.go +++ b/api/client/restore.go @@ -5,12 +5,13 @@ package client import ( "fmt" + Cli "github.com/docker/docker/cli" flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/runconfig" ) func (cli *DockerCli) CmdRestore(args ...string) error { - cmd := cli.Subcmd("restore", []string{"CONTAINER [CONTAINER...]"}, "Restore one or more checkpointed containers", true) + cmd := Cli.Subcmd("restore", []string{"CONTAINER [CONTAINER...]"}, "Restore one or more checkpointed containers", true) cmd.Require(flag.Min, 1) var ( diff --git a/api/server/server_experimental_unix.go b/api/server/server_experimental_unix.go index b49c66ec793f2..d41f6b0413235 100644 --- a/api/server/server_experimental_unix.go +++ b/api/server/server_experimental_unix.go @@ -5,12 +5,12 @@ package server import ( "encoding/json" "fmt" - "net/http" "github.com/docker/docker/pkg/version" "github.com/docker/docker/runconfig" - ) + "net/http" +) -func addExperimentalRoutes(s *Server, m map[string]map[string]HttpApiFunc) { +func addExperimentalRoutes(s *Server, m map[string]map[string]HTTPAPIFunc) { m["POST"]["/containers/{name:.*}/checkpoint"] = s.postContainersCheckpoint m["POST"]["/containers/{name:.*}/restore"] = s.postContainersRestore } diff --git a/api/server/server_stub.go b/api/server/server_stub.go index ce6669da845be..6d93d21626dd1 100644 --- a/api/server/server_stub.go +++ b/api/server/server_stub.go @@ -2,7 +2,7 @@ package server -func addExperimentalRoutes(s *Server, m map[string]map[string]HttpApiFunc) { +func addExperimentalRoutes(s *Server, m map[string]map[string]HTTPAPIFunc) { } diff --git a/daemon/checkpoint.go b/daemon/checkpoint.go index a39662cc0f325..6842b2ae58a9e 100644 --- a/daemon/checkpoint.go +++ b/daemon/checkpoint.go @@ -19,7 +19,7 @@ func (daemon *Daemon) ContainerCheckpoint(name string, opts *runconfig.CriuConfi return fmt.Errorf("Cannot checkpoint container %s: %s", name, err) } - container.LogEvent("checkpoint") + container.logEvent("checkpoint") return nil } @@ -47,10 +47,10 @@ func (daemon *Daemon) ContainerRestore(name string, opts *runconfig.CriuConfig, } if err = container.Restore(opts, forceRestore); err != nil { - container.LogEvent("die") + container.logEvent("die") return fmt.Errorf("Cannot restore container %s: %s", name, err) } - container.LogEvent("restore") + container.logEvent("restore") return nil } diff --git a/daemon/container_checkpoint.go b/daemon/container_checkpoint.go index 77f3488fbf157..27b23377de48b 100644 --- a/daemon/container_checkpoint.go +++ b/daemon/container_checkpoint.go @@ -13,10 +13,10 @@ func (container *Container) Checkpoint(opts *runconfig.CriuConfig) error { } if opts.LeaveRunning == false { - container.ReleaseNetwork() + container.releaseNetwork() } - if err := container.ToDisk(); err != nil { + if err := container.toDisk(); err != nil { return fmt.Errorf("Cannot update config for container: %s", err) } diff --git a/daemon/daemon.go b/daemon/daemon.go index f8f53dbadef7b..02a995e78d80f 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -899,7 +899,7 @@ func (daemon *Daemon) Checkpoint(c *Container, opts *runconfig.CriuConfig) error func (daemon *Daemon) Restore(c *Container, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { // Mount the container's filesystem (daemon/graphdriver/aufs/aufs.go). - _, err := daemon.driver.Get(c.ID, c.GetMountLabel()) + _, err := daemon.driver.Get(c.ID, c.getMountLabel()) if err != nil { return execdriver.ExitStatus{ExitCode: 0}, err } diff --git a/daemon/execdriver/lxc/driver.go b/daemon/execdriver/lxc/driver.go index e8196316098ad..dce464b7937db 100644 --- a/daemon/execdriver/lxc/driver.go +++ b/daemon/execdriver/lxc/driver.go @@ -561,11 +561,11 @@ func (d *Driver) Unpause(c *execdriver.Command) error { return err } -func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { +func (d *Driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { return fmt.Errorf("Checkpointing lxc containers not supported yet\n") } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { +func (d *Driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { return execdriver.ExitStatus{ExitCode: 0}, fmt.Errorf("Restoring lxc containers not supported yet\n") } diff --git a/daemon/execdriver/native/create.go b/daemon/execdriver/native/create.go index 95c2fd03eb5fc..85f72f8c2cc83 100644 --- a/daemon/execdriver/native/create.go +++ b/daemon/execdriver/native/create.go @@ -113,24 +113,6 @@ func generateIfaceName() (string, error) { return "", errors.New("Failed to find name for new interface") } -// Re-create the container type from the image that was saved during checkpoint. -func (d *driver) createRestoreContainer(c *execdriver.Command, imageDir string) (*libcontainer.Config, error) { - // Read the container.json. - f1, err := os.Open(filepath.Join(imageDir, "container.json")) - if err != nil { - return nil, err - } - defer f1.Close() - - var container *libcontainer.Config - err = json.NewDecoder(f1).Decode(&container) - if err != nil { - return nil, err - } - - return container, nil -} - func (d *Driver) createNetwork(container *configs.Config, c *execdriver.Command) error { if c.Network == nil { return nil diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index f4f34095ee50a..2b54aa3ff68ce 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -313,7 +313,7 @@ func libcontainerCriuOpts(runconfigOpts *runconfig.CriuConfig) *libcontainer.Cri } } -func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { +func (d *Driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { active := d.activeContainers[c.ID] if active == nil { return fmt.Errorf("active container for %s does not exist", c.ID) @@ -329,7 +329,7 @@ func (d *driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) e return nil } -func (d *driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { +func (d *Driver) Restore(c *execdriver.Command, pipes *execdriver.Pipes, restoreCallback execdriver.RestoreCallback, opts *runconfig.CriuConfig, forceRestore bool) (execdriver.ExitStatus, error) { var ( cont libcontainer.Container err error diff --git a/daemon/inspect.go b/daemon/inspect.go index bb80a24ee3529..00518e9105bc9 100644 --- a/daemon/inspect.go +++ b/daemon/inspect.go @@ -53,14 +53,14 @@ func (daemon *Daemon) getInspectData(container *Container) (*types.ContainerJSON Running: container.State.Running, Paused: container.State.Paused, Checkpointed: container.State.Checkpointed, - Restarting: container.State.Restarting, - OOMKilled: container.State.OOMKilled, - Dead: container.State.Dead, - Pid: container.State.Pid, - ExitCode: container.State.ExitCode, - Error: container.State.Error, - StartedAt: container.State.StartedAt.Format(time.RFC3339Nano), - FinishedAt: container.State.FinishedAt.Format(time.RFC3339Nano), + Restarting: container.State.Restarting, + OOMKilled: container.State.OOMKilled, + Dead: container.State.Dead, + Pid: container.State.Pid, + ExitCode: container.State.ExitCode, + Error: container.State.Error, + StartedAt: container.State.StartedAt.Format(time.RFC3339Nano), + FinishedAt: container.State.FinishedAt.Format(time.RFC3339Nano), CheckpointedAt: container.State.CheckpointedAt.Format(time.RFC3339Nano), } diff --git a/daemon/monitor.go b/daemon/monitor.go index fede2fb70035b..ee250d33d96ec 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -220,7 +220,7 @@ func (m *containerMonitor) Restore(opts *runconfig.CriuConfig, forceRestore bool pipes := execdriver.NewPipes(m.container.stdin, m.container.stdout, m.container.stderr, m.container.Config.OpenStdin) - m.container.LogEvent("restore") + m.container.logEvent("restore") m.lastStartTime = time.Now() if exitCode, err = m.container.daemon.Restore(m.container, pipes, m.restoreCallback, opts, forceRestore); err != nil { logrus.Errorf("Error restoring container: %s, exitCode=%d", err, exitCode) @@ -232,7 +232,7 @@ func (m *containerMonitor) Restore(opts *runconfig.CriuConfig, forceRestore bool m.container.ExitCode = exitCode.ExitCode m.resetMonitor(err == nil && exitCode.ExitCode == 0) - m.container.LogEvent("die") + m.container.logEvent("die") m.resetContainer(true) return err } @@ -341,7 +341,7 @@ func (m *containerMonitor) restoreCallback(processConfig *execdriver.ProcessConf if restorePid != 0 { // Write config.json and hostconfig.json files // to /var/lib/docker/containers/. - if err := m.container.ToDisk(); err != nil { + if err := m.container.toDisk(); err != nil { logrus.Debugf("%s", err) } } diff --git a/docker/docker.go b/docker/docker.go index 4d3477110c03f..fb0a5240e6936 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -3,6 +3,7 @@ package main import ( "fmt" "os" + "sort" "github.com/Sirupsen/logrus" "github.com/docker/docker/api/client" @@ -35,7 +36,6 @@ func main() { help := "\nCommands:\n" - // TODO(tiborvass): no need to sort if we ensure dockerCommands is sorted allCommands := append(dockerCommands, experimentalCommands...) sort.Sort(byName(allCommands)) From 792fc89abbfeaef47138e7f6563332d26de9f745 Mon Sep 17 00:00:00 2001 From: boucher Date: Mon, 31 Aug 2015 10:04:33 -0700 Subject: [PATCH 14/16] Support file locks, and enable all of the CRIU "safety" features by default. Signed-off-by: Ross Boucher --- api/client/checkpoint.go | 7 +++---- api/client/restore.go | 15 +++++++-------- daemon/execdriver/native/driver.go | 1 + runconfig/restore.go | 1 + 4 files changed, 12 insertions(+), 12 deletions(-) diff --git a/api/client/checkpoint.go b/api/client/checkpoint.go index 24fed5f68b634..5edaf7facffb0 100644 --- a/api/client/checkpoint.go +++ b/api/client/checkpoint.go @@ -18,8 +18,6 @@ func (cli *DockerCli) CmdCheckpoint(args ...string) error { flImgDir = cmd.String([]string{"-image-dir"}, "", "directory for storing checkpoint image files") flWorkDir = cmd.String([]string{"-work-dir"}, "", "directory for storing log file") flLeaveRunning = cmd.Bool([]string{"-leave-running"}, false, "leave the container running after checkpoint") - flCheckTcp = cmd.Bool([]string{"-allow-tcp"}, false, "allow checkpointing tcp connections") - flExtUnix = cmd.Bool([]string{"-allow-ext-unix"}, false, "allow checkpointing external unix connections") flShell = cmd.Bool([]string{"-allow-shell"}, false, "allow checkpointing shell jobs") ) @@ -36,9 +34,10 @@ func (cli *DockerCli) CmdCheckpoint(args ...string) error { ImagesDirectory: *flImgDir, WorkDirectory: *flWorkDir, LeaveRunning: *flLeaveRunning, - TcpEstablished: *flCheckTcp, - ExternalUnixConnections: *flExtUnix, ShellJob: *flShell, + TcpEstablished: true, + ExternalUnixConnections: true, + FileLocks: true, } var encounteredError error diff --git a/api/client/restore.go b/api/client/restore.go index bef78a262b54a..32b010060f1f1 100644 --- a/api/client/restore.go +++ b/api/client/restore.go @@ -15,12 +15,10 @@ func (cli *DockerCli) CmdRestore(args ...string) error { cmd.Require(flag.Min, 1) var ( - flImgDir = cmd.String([]string{"-image-dir"}, "", "directory to restore image files from") - flWorkDir = cmd.String([]string{"-work-dir"}, "", "directory for restore log") - flCheckTcp = cmd.Bool([]string{"-allow-tcp"}, false, "allow restoring tcp connections") - flExtUnix = cmd.Bool([]string{"-allow-ext-unix"}, false, "allow restoring external unix connections") - flShell = cmd.Bool([]string{"-allow-shell"}, false, "allow restoring shell jobs") - flForce = cmd.Bool([]string{"-force"}, false, "bypass checks for current container state") + flImgDir = cmd.String([]string{"-image-dir"}, "", "directory to restore image files from") + flWorkDir = cmd.String([]string{"-work-dir"}, "", "directory for restore log") + flForce = cmd.Bool([]string{"-force"}, false, "bypass checks for current container state") + flShell = cmd.Bool([]string{"-allow-shell"}, false, "allow restoring shell jobs") ) if err := cmd.ParseFlags(args, true); err != nil { @@ -36,9 +34,10 @@ func (cli *DockerCli) CmdRestore(args ...string) error { CriuOpts: runconfig.CriuConfig{ ImagesDirectory: *flImgDir, WorkDirectory: *flWorkDir, - TcpEstablished: *flCheckTcp, - ExternalUnixConnections: *flExtUnix, ShellJob: *flShell, + TcpEstablished: true, + ExternalUnixConnections: true, + FileLocks: true, }, ForceRestore: *flForce, } diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index 2b54aa3ff68ce..d6dbc07ae9b74 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -310,6 +310,7 @@ func libcontainerCriuOpts(runconfigOpts *runconfig.CriuConfig) *libcontainer.Cri TcpEstablished: runconfigOpts.TcpEstablished, ExternalUnixConnections: runconfigOpts.ExternalUnixConnections, ShellJob: runconfigOpts.ShellJob, + FileLocks: runconfigOpts.FileLocks, } } diff --git a/runconfig/restore.go b/runconfig/restore.go index 22f8b0ab0a096..a47b4c0d759d0 100644 --- a/runconfig/restore.go +++ b/runconfig/restore.go @@ -7,6 +7,7 @@ type CriuConfig struct { TcpEstablished bool ExternalUnixConnections bool ShellJob bool + FileLocks bool } type RestoreConfig struct { From 291924916a4b1a864e42d6db94f7e2f49cf76125 Mon Sep 17 00:00:00 2001 From: boucher Date: Fri, 4 Sep 2015 15:03:37 -0700 Subject: [PATCH 15/16] Add README information for experimental checkpoint/restore support. Signed-off-by: Ross Boucher --- experimental/README.md | 9 +- experimental/checkpoint_restore.md | 154 +++++++++++++++++++++++++++++ 2 files changed, 159 insertions(+), 4 deletions(-) create mode 100644 experimental/checkpoint_restore.md diff --git a/experimental/README.md b/experimental/README.md index 8da601d6f202d..dbd83bcb570c0 100644 --- a/experimental/README.md +++ b/experimental/README.md @@ -1,8 +1,8 @@ -# Docker Experimental Features +# Docker Experimental Features This page contains a list of features in the Docker engine which are experimental. Experimental features are **not** ready for production. They are -provided for test and evaluation in your sandbox environments. +provided for test and evaluation in your sandbox environments. The information below describes each feature and the GitHub pull requests and issues associated with it. If necessary, links are provided to additional @@ -75,9 +75,10 @@ to build a dynamically-linked Docker binary with the experimental features enabl * [Networking and Services UI](networking.md) * [Native multi-host networking](network_overlay.md) * [Compose, Swarm and networking integration](compose_swarm_networking.md) +* [Checkpoint & Restore](checkpoint_restore.md) ## How to comment on an experimental feature -Each feature's documentation includes a list of proposal pull requests or PRs associated with the feature. If you want to comment on or suggest a change to a feature, please add it to the existing feature PR. +Each feature's documentation includes a list of proposal pull requests or PRs associated with the feature. If you want to comment on or suggest a change to a feature, please add it to the existing feature PR. -Issues or problems with a feature? Inquire for help on the `#docker` IRC channel or in on the [Docker Google group](https://groups.google.com/forum/#!forum/docker-user). +Issues or problems with a feature? Inquire for help on the `#docker` IRC channel or in on the [Docker Google group](https://groups.google.com/forum/#!forum/docker-user). diff --git a/experimental/checkpoint_restore.md b/experimental/checkpoint_restore.md new file mode 100644 index 0000000000000..f3ed0b5898e85 --- /dev/null +++ b/experimental/checkpoint_restore.md @@ -0,0 +1,154 @@ +# Docker Checkpoint & Restore + +Checkpoint & Restore is a new feature that allows you to freeze a running +container by checkpointing it, which turns its state into a collection of files +on disk. Later, the container can be restored from the point it was frozen. + +This is accomplished using a tool called [CRIU](http://criu.org), which is an +external dependency of this feature. A good overview of the history of +checkpoint and restore in Docker is available in this +[Kubernetes blog post](http://blog.kubernetes.io/2015/07/how-did-quake-demo-from-dockercon-work.html). + +## Installing CRIU + +If you use a Debian system, you can add the CRIU PPA and install with apt-get +https://launchpad.net/~criu/+archive/ubuntu/ppa. + +Alternatively, you can [build CRIU from source](http://criu.org/Installation). + +## Use cases for checkpoint & restore + +This feature is currently focused on single-host use cases for checkpoint and +restore. Here are a few: + +- Restarting / upgrading the docker daemon without stopping containers +- Restarting the host machine without stopping/starting containers +- Speeding up the start time of slow start applications +- "Rewinding" processes to an earlier point in time +- "Forensic debugging" of running processes + +Another primary use case of checkpoint & restore outside of Docker is the live +migration of a server from one machine to another. This is possible with the +current implementation, but not currently a priority (and so the workflow is +not optimized for the task). + +## Using Checkpoint & Restore + +Two new top level commands are introduced in the CLI: `checkpoint` & `restore`. +The options for checkpoint: + + Usage: docker checkpoint [OPTIONS] CONTAINER [CONTAINER...] + + Checkpoint one or more running containers + + --allow-shell=false allow checkpointing shell jobs + --image-dir= directory for storing checkpoint image files + --leave-running=false leave the container running after checkpoint + --work-dir= directory for storing log file + +And for restore: + + Usage: docker restore [OPTIONS] CONTAINER [CONTAINER...] + + Restore one or more checkpointed containers + + --allow-shell=false allow restoring shell jobs + --force=false bypass checks for current container state + --image-dir= directory to restore image files from + --work-dir= directory for restore log + +A simple example of using checkpoint & restore on a container: + + $ docker run --name cr -d busybox /bin/sh -c 'i=0; while true; do echo $i; i=$(expr $i + 1); sleep 1; done' + > abc0123 + + $ docker checkpoint cr + > abc0123 + + $ docker restore cr + > abc0123 + +This process just logs an incrementing counter to stdout. If you `docker logs` +in between running/checkpoint/restoring you should see that the counter +increases while the process is running, stops while it's checkpointed, and +resumes from the point it left off once you restore. + +### Same container checkpoint/restore + +The above example falls into the category of "same container" use cases for c/r. +Restarting the daemon is an example of this kind of use case. There is only one +container here at any point in time. That container's status, once it is +checkpointed, will be "Checkpointed" and docker inspect will contain that status +as well as the time of the last checkpoint. The IP address and other container +state do not change (see known issues at the bottom of this document). + +### New container checkpoint/restore + +Here's an example of a "new container" use case for c/r: + + $ docker run some_image + > abc789 + + ## the container runs for a while + + $ docker checkpoint --image-dir=/some/path abc789 + > abc789 + +At this point, we've created a checkpoint image at `/some/path` that encodes a +process at the exact state we want it to be. Now, at some later point in time, +we can put a copy of that exact state into a new container (perhaps many times): + + $ docker create some_image + > def123 + + $ docker restore --force=true --image-dir=/some/path def123 + > def123 + +We created a new container (but didn't start it), and then we restored our +checkpointed process into that container. + +This is obviously more involved than the simple use case shown earlier. It +requires starting subsequent containers with the same configuration (e.g. +the same mounted volumes, the same base image, etc.). + +### Options + +Checkpoint & Restore: + + --image-dir= directory for storing checkpoint image files + +Allows you to specify the path for writing a checkpoint image, or the path for +the image you want to restore. + + --work-dir= directory for storing log file + +Allows you to specify the path for writing the CRIU log. + + --leave-running=false leave the container running after checkpoint + +Normally, when checkpointing a process, the process is stopped aftewrards. +When this flag is enabled, the process keeps running after a checkpoint. This is +useful if you want to capture a process at multiple points in time, for later +use in debugging or rewinding a process for some reason. It's also used for +minimizing downtime when checkpointing processes with a large memory footprint. + +Restore Only: + + --force=false force restoring into a container + +As shown in the "new container" example, this flag allows you to restore a +checkpoint image into a container that was not previously checkpointed. +Normally, docker would return an error when restoring into a container that +has not been previously checkpointed. + +## Known Issues + +- Currently, networking is broken in this PR. Although it's implemented at the +libcontainer level, the method used no longer works since the introduction of +libnetwork. See: + - https://github.com/docker/libnetwork/pull/465 + - https://github.com/boucher/docker/pull/15 +- There are likely several networking related issues to work out, like: + - ensuring IPs are reserved across daemon restarts + - ensuring port maps are reserved + - deciding how to deal with network resources in the "new container" model From 302e026fa4bace653b3b0204ffde3cb56d794359 Mon Sep 17 00:00:00 2001 From: Hui Kang Date: Mon, 7 Sep 2015 00:56:58 +0000 Subject: [PATCH 16/16] Allow restore network to have network connectivity Reuse the endpoint of the checkpointed container when restore. Pass veth pair name to ciur when restore a checkpointed container. Signed-off-by: Hui Kang --- daemon/container.go | 7 +-- daemon/container_checkpoint.go | 28 ++++++++- daemon/container_unix.go | 59 +++++++++++++++---- daemon/execdriver/native/driver.go | 11 +++- runconfig/restore.go | 6 ++ .../github.com/docker/libnetwork/endpoint.go | 14 +++++ .../runc/libcontainer/container_linux.go | 10 ++++ .../runc/libcontainer/criu_opts.go | 6 ++ 8 files changed, 124 insertions(+), 17 deletions(-) diff --git a/daemon/container.go b/daemon/container.go index 89109443befc0..6c31379a5b65f 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -357,12 +357,11 @@ func (container *Container) isNetworkAllocated() bool { // cleanup releases any network resources allocated to the container along with any rules // around how containers are linked together. It also unmounts the container's root filesystem. func (container *Container) cleanup() { - /*if container.IsCheckpointed() { + if container.IsCheckpointed() { logrus.Debugf("not calling ReleaseNetwork() for checkpointed container %s", container.ID) } else { - container.ReleaseNetwork() - }*/ - container.releaseNetwork() + container.releaseNetwork(false) + } if err := container.Unmount(); err != nil { logrus.Errorf("%v: Failed to umount filesystem: %v", container.ID, err) diff --git a/daemon/container_checkpoint.go b/daemon/container_checkpoint.go index 27b23377de48b..9a75022da7f16 100644 --- a/daemon/container_checkpoint.go +++ b/daemon/container_checkpoint.go @@ -5,6 +5,8 @@ import ( "github.com/docker/docker/pkg/promise" "github.com/docker/docker/runconfig" + + "github.com/docker/libnetwork/netutils" ) func (container *Container) Checkpoint(opts *runconfig.CriuConfig) error { @@ -13,7 +15,7 @@ func (container *Container) Checkpoint(opts *runconfig.CriuConfig) error { } if opts.LeaveRunning == false { - container.releaseNetwork() + container.releaseNetwork(true) } if err := container.toDisk(); err != nil { @@ -46,6 +48,30 @@ func (container *Container) Restore(opts *runconfig.CriuConfig, forceRestore boo if err = container.initializeNetworking(true); err != nil { return err } + + nctl := container.daemon.netController + network, err := nctl.NetworkByID(container.NetworkSettings.NetworkID) + if err != nil { + return err + } + + ep_t, err := network.EndpointByID(container.NetworkSettings.EndpointID) + if err != nil { + return err + } + + for _, i := range ep_t.SandboxInterfaces() { + outname, err := netutils.GenerateIfaceName("veth", 7) + if err != nil { + return err + } + vethpair := runconfig.VethPairName{ + InName: i.DstName(), + OutName: outname, + } + opts.VethPairs = append(opts.VethPairs, vethpair) + } + linkedEnv, err := container.setupLinkedContainers() if err != nil { return err diff --git a/daemon/container_unix.go b/daemon/container_unix.go index d1f82d9d58d20..b937d87073ad4 100644 --- a/daemon/container_unix.go +++ b/daemon/container_unix.go @@ -892,20 +892,52 @@ func (container *Container) configureNetwork(networkName, service, networkDriver } } - ep, err := n.EndpointByName(service) - if err != nil { - if _, ok := err.(libnetwork.ErrNoSuchEndpoint); !ok { - return err - } + var ep libnetwork.Endpoint - createOptions, err := container.buildCreateEndpointOptions(isRestoring) - if err != nil { - return err + if isRestoring == true { + // Use existing Endpoint for a checkpointed container + for _, endpoint := range n.Endpoints() { + if endpoint.ID() == container.NetworkSettings.EndpointID { + ep = endpoint + } } - ep, err = n.CreateEndpoint(service, createOptions...) + if ep == nil { + //return fmt.Errorf("Fail to find the Endpoint for the checkpointed container") + fmt.Println("Fail to find the Endpoint for the checkpointed container") + ep, err = n.EndpointByName(service) + if err != nil { + if _, ok := err.(libnetwork.ErrNoSuchEndpoint); !ok { + return err + } + + createOptions, err := container.buildCreateEndpointOptions(isRestoring) + if err != nil { + return err + } + + ep, err = n.CreateEndpoint(service, createOptions...) + if err != nil { + return err + } + } + } + } else { + ep, err = n.EndpointByName(service) if err != nil { - return err + if _, ok := err.(libnetwork.ErrNoSuchEndpoint); !ok { + return err + } + + createOptions, err := container.buildCreateEndpointOptions(isRestoring) + if err != nil { + return err + } + + ep, err = n.CreateEndpoint(service, createOptions...) + if err != nil { + return err + } } } @@ -1043,7 +1075,7 @@ func (container *Container) getNetworkedContainer() (*Container, error) { } } -func (container *Container) releaseNetwork() { +func (container *Container) releaseNetwork(is_checkpoint bool) { if container.hostConfig.NetworkMode.IsContainer() || container.Config.NetworkDisabled { return } @@ -1082,6 +1114,11 @@ func (container *Container) releaseNetwork() { } } + // for checkpointed container, the endpoint will not be deleted + if is_checkpoint == true { + return + } + // In addition to leaving all endpoints, delete implicitly created endpoint if container.Config.PublishService == "" { if err := ep.Delete(); err != nil { diff --git a/daemon/execdriver/native/driver.go b/daemon/execdriver/native/driver.go index d6dbc07ae9b74..91d1dfe594420 100644 --- a/daemon/execdriver/native/driver.go +++ b/daemon/execdriver/native/driver.go @@ -303,7 +303,7 @@ func (d *Driver) Unpause(c *execdriver.Command) error { } func libcontainerCriuOpts(runconfigOpts *runconfig.CriuConfig) *libcontainer.CriuOpts { - return &libcontainer.CriuOpts{ + criuopts := &libcontainer.CriuOpts{ ImagesDirectory: runconfigOpts.ImagesDirectory, WorkDirectory: runconfigOpts.WorkDirectory, LeaveRunning: runconfigOpts.LeaveRunning, @@ -312,6 +312,15 @@ func libcontainerCriuOpts(runconfigOpts *runconfig.CriuConfig) *libcontainer.Cri ShellJob: runconfigOpts.ShellJob, FileLocks: runconfigOpts.FileLocks, } + + for _, i := range runconfigOpts.VethPairs { + criuopts.VethPairs = append(criuopts.VethPairs, + libcontainer.VethPairName{ + InName: i.InName, + OutName: i.OutName, + }) + } + return criuopts } func (d *Driver) Checkpoint(c *execdriver.Command, opts *runconfig.CriuConfig) error { diff --git a/runconfig/restore.go b/runconfig/restore.go index a47b4c0d759d0..dd81d909fcd1a 100644 --- a/runconfig/restore.go +++ b/runconfig/restore.go @@ -1,5 +1,10 @@ package runconfig +type VethPairName struct { + InName string + OutName string +} + type CriuConfig struct { ImagesDirectory string WorkDirectory string @@ -8,6 +13,7 @@ type CriuConfig struct { ExternalUnixConnections bool ShellJob bool FileLocks bool + VethPairs []VethPairName } type RestoreConfig struct { diff --git a/vendor/src/github.com/docker/libnetwork/endpoint.go b/vendor/src/github.com/docker/libnetwork/endpoint.go index 3475e9eafda04..b0354a03bc6e1 100644 --- a/vendor/src/github.com/docker/libnetwork/endpoint.go +++ b/vendor/src/github.com/docker/libnetwork/endpoint.go @@ -54,6 +54,9 @@ type Endpoint interface { // Retrieve the interfaces' statistics from the sandbox Statistics() (map[string]*sandbox.InterfaceStatistics, error) + + // Retrieve the interfaces from sandbox, for restoring checkpointed container + SandboxInterfaces() []sandbox.Interface } // EndpointOption is a option setter function type used to pass varios options to Network @@ -583,6 +586,17 @@ func (ep *endpoint) Statistics() (map[string]*sandbox.InterfaceStatistics, error return m, nil } +func (ep *endpoint) SandboxInterfaces() []sandbox.Interface { + n := ep.network + + n.Lock() + c := n.ctrlr + n.Unlock() + + sbox := c.sandboxGet(ep.container.data.SandboxKey) + return sbox.Info().Interfaces() +} + func (ep *endpoint) deleteEndpoint() error { ep.Lock() n := ep.network diff --git a/vendor/src/github.com/opencontainers/runc/libcontainer/container_linux.go b/vendor/src/github.com/opencontainers/runc/libcontainer/container_linux.go index 9a27eb432faad..935af32cc42b4 100644 --- a/vendor/src/github.com/opencontainers/runc/libcontainer/container_linux.go +++ b/vendor/src/github.com/opencontainers/runc/libcontainer/container_linux.go @@ -520,6 +520,8 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { break } } + + /* XXX, we can not get interface from config */ for _, iface := range c.config.Networks { switch iface.Type { case "veth": @@ -533,6 +535,14 @@ func (c *linuxContainer) Restore(process *Process, criuOpts *CriuOpts) error { } } + for _, i := range criuOpts.VethPairs { + outname := i.OutName + "@docker0" + veth := new(criurpc.CriuVethPair) + veth.IfOut = proto.String(outname) + veth.IfIn = proto.String(i.InName) + req.Opts.Veths = append(req.Opts.Veths, veth) + } + var ( fds []string fdJSON []byte diff --git a/vendor/src/github.com/opencontainers/runc/libcontainer/criu_opts.go b/vendor/src/github.com/opencontainers/runc/libcontainer/criu_opts.go index bca81672eac8b..1e3c79e6ce9db 100644 --- a/vendor/src/github.com/opencontainers/runc/libcontainer/criu_opts.go +++ b/vendor/src/github.com/opencontainers/runc/libcontainer/criu_opts.go @@ -5,6 +5,11 @@ type CriuPageServerInfo struct { Port int32 // port number of CRIU page server } +type VethPairName struct { + InName string + OutName string +} + type CriuOpts struct { ImagesDirectory string // directory for storing image files WorkDirectory string // directory to cd and write logs/pidfiles/stats to @@ -14,4 +19,5 @@ type CriuOpts struct { ShellJob bool // allow to dump and restore shell jobs FileLocks bool // handle file locks, for safety PageServer CriuPageServerInfo // allow to dump to criu page server + VethPairs []VethPairName // pass the veth to criu when restore }