From e0c8c14f5be5a2da38683f35e55d19ffc36659ba Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Tue, 5 Feb 2019 13:35:06 +0000 Subject: Fix restore options help text and comments Signed-off-by: Adrian Reber --- cmd/podman/restore.go | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/cmd/podman/restore.go b/cmd/podman/restore.go index 8cfd5ca0d..36ae16183 100644 --- a/cmd/podman/restore.go +++ b/cmd/podman/restore.go @@ -43,8 +43,7 @@ func init() { flags.BoolVarP(&restoreCommand.All, "all", "a", false, "Restore all checkpointed containers") flags.BoolVarP(&restoreCommand.Keep, "keep", "k", false, "Keep all temporary checkpoint files") flags.BoolVarP(&restoreCommand.Latest, "latest", "l", false, "Act on the latest container podman is aware of") - // TODO: add ContainerStateCheckpointed - flags.BoolVar(&restoreCommand.TcpEstablished, "tcp-established", false, "Checkpoint a container with established TCP connections") + flags.BoolVar(&restoreCommand.TcpEstablished, "tcp-established", false, "Restore a container with established TCP connections") markFlagHiddenForRemoteClient("latest", flags) } -- cgit v1.2.3-54-g00ecf From a05cfd24bb6929ca4431f9169b9b215b0d43d91e Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Wed, 6 Feb 2019 19:17:25 +0000 Subject: Added helper functions for container migration This adds a couple of function in structure members needed in the next commit to make container migration actually work. This just splits of the function which are not modifying existing code. Signed-off-by: Adrian Reber --- libpod/container_api.go | 5 +++ libpod/container_internal.go | 38 ++++++++++++++++++ libpod/container_internal_linux.go | 81 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 120 insertions(+), 4 deletions(-) diff --git a/libpod/container_api.go b/libpod/container_api.go index eff5bfe5f..b8b60a6e2 100644 --- a/libpod/container_api.go +++ b/libpod/container_api.go @@ -815,6 +815,11 @@ type ContainerCheckpointOptions struct { // TCPEstablished tells the API to checkpoint a container // even if it contains established TCP connections TCPEstablished bool + // Export tells the API to write the checkpoint image to + // the filename set in TargetFile + // Import tells the API to read the checkpoint image from + // the filename set in TargetFile + TargetFile string } // Checkpoint checkpoints a container diff --git a/libpod/container_internal.go b/libpod/container_internal.go index 5f8dd1c72..42f56e82f 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -21,6 +21,7 @@ import ( "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/mount" spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-tools/generate" "github.com/opencontainers/selinux/go-selinux/label" opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" @@ -1501,3 +1502,40 @@ func (c *Container) checkReadyForRemoval() error { return nil } + +// writeJSONFile marshalls and writes the given data to a JSON file +// in the bundle path +func (c *Container) writeJSONFile(v interface{}, file string) (err error) { + fileJSON, err := json.MarshalIndent(v, "", " ") + if err != nil { + return errors.Wrapf(err, "error writing JSON to %s for container %s", file, c.ID()) + } + file = filepath.Join(c.bundlePath(), file) + if err := ioutil.WriteFile(file, fileJSON, 0644); err != nil { + return err + } + + return nil +} + +// prepareCheckpointExport writes the config and spec to +// JSON files for later export +func (c *Container) prepareCheckpointExport() (err error) { + // save live config + if err := c.writeJSONFile(c.Config(), "config.dump"); err != nil { + return err + } + + // save spec + jsonPath := filepath.Join(c.bundlePath(), "config.json") + g, err := generate.NewFromFile(jsonPath) + if err != nil { + logrus.Debugf("generating spec for container %q failed with %v", c.ID(), err) + return err + } + if err := c.writeJSONFile(g.Spec(), "spec.dump"); err != nil { + return err + } + + return nil +} diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index f25f76092..3c02e3ae0 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -5,6 +5,7 @@ package libpod import ( "context" "fmt" + "io" "io/ioutil" "net" "os" @@ -25,6 +26,7 @@ import ( "github.com/containers/libpod/pkg/lookup" "github.com/containers/libpod/pkg/resolvconf" "github.com/containers/libpod/pkg/rootless" + "github.com/containers/storage/pkg/archive" securejoin "github.com/cyphar/filepath-securejoin" "github.com/opencontainers/runc/libcontainer/user" spec "github.com/opencontainers/runtime-spec/specs-go" @@ -496,6 +498,42 @@ func (c *Container) addNamespaceContainer(g *generate.Generator, ns LinuxNS, ctr return nil } +func (c *Container) exportCheckpoint(dest string) (err error) { + logrus.Debugf("Exporting checkpoint image of container %q to %q", c.ID(), dest) + input, err := archive.TarWithOptions(c.bundlePath(), &archive.TarOptions{ + Compression: archive.Gzip, + IncludeSourceDir: true, + IncludeFiles: []string{ + "checkpoint", + "artifacts", + "ctr.log", + "config.dump", + "spec.dump", + "network.status"}, + }) + + if err != nil { + return errors.Wrapf(err, "error reading checkpoint directory %q", c.ID()) + } + + outFile, err := os.Create(dest) + if err != nil { + return errors.Wrapf(err, "error creating checkpoint export file %q", dest) + } + defer outFile.Close() + + if err := os.Chmod(dest, 0600); err != nil { + return errors.Wrapf(err, "cannot chmod %q", dest) + } + + _, err = io.Copy(outFile, input) + if err != nil { + return err + } + + return nil +} + func (c *Container) checkpointRestoreSupported() (err error) { if !criu.CheckForCriu() { return errors.Errorf("Checkpoint/Restore requires at least CRIU %d", criu.MinCriuVersion) @@ -561,15 +599,50 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO } if !options.Keep { - // Remove log file - os.Remove(filepath.Join(c.bundlePath(), "dump.log")) - // Remove statistic file - os.Remove(filepath.Join(c.bundlePath(), "stats-dump")) + cleanup := []string{ + "dump.log", + "stats-dump", + "config.dump", + "spec.dump", + } + for _, delete := range cleanup { + file := filepath.Join(c.bundlePath(), delete) + os.Remove(file) + } } return c.save() } +func (c *Container) importCheckpoint(input string) (err error) { + archiveFile, err := os.Open(input) + if err != nil { + return errors.Wrapf(err, "Failed to open checkpoint archive %s for import", input) + } + + defer archiveFile.Close() + options := &archive.TarOptions{ + ExcludePatterns: []string{ + // config.dump and spec.dump are only required + // container creation + "config.dump", + "spec.dump", + }, + } + err = archive.Untar(archiveFile, c.bundlePath(), options) + if err != nil { + return errors.Wrapf(err, "Unpacking of checkpoint archive %s failed", input) + } + + // Make sure the newly created config.json exists on disk + g := generate.NewFromSpec(c.config.Spec) + if err = c.saveSpec(g.Spec()); err != nil { + return errors.Wrap(err, "Saving imported container specification for restore failed") + } + + return nil +} + func (c *Container) restore(ctx context.Context, options ContainerCheckpointOptions) (err error) { if err := c.checkpointRestoreSupported(); err != nil { -- cgit v1.2.3-54-g00ecf From 0028578b432d207e1e5b313c76e587eae275bdac Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Wed, 6 Feb 2019 19:22:46 +0000 Subject: Added support to migrate containers This commit adds an option to the checkpoint command to export a checkpoint into a tar.gz file as well as importing a checkpoint tar.gz file during restore. With all checkpoint artifacts in one file it is possible to easily transfer a checkpoint and thus enabling container migration in Podman. With the following steps it is possible to migrate a running container from one system (source) to another (destination). Source system: * podman container checkpoint -l -e /tmp/checkpoint.tar.gz * scp /tmp/checkpoint.tar.gz destination:/tmp Destination system: * podman pull 'container-image-as-on-source-system' * podman container restore -i /tmp/checkpoint.tar.gz The exported tar.gz file contains the checkpoint image as created by CRIU and a few additional JSON files describing the state of the checkpointed container. Now the container is running on the destination system with the same state just as during checkpointing. If the container is kept running on the source system with the checkpoint flag '-R', the result will be that the same container is running on two different hosts. Signed-off-by: Adrian Reber --- cmd/podman/checkpoint.go | 2 + cmd/podman/cliconfig/config.go | 2 + cmd/podman/restore.go | 14 ++++-- libpod/container_api.go | 7 +++ libpod/container_internal.go | 2 +- libpod/container_internal_linux.go | 44 +++++++++++++++-- libpod/runtime_ctr.go | 60 +++++++++++++++++++---- pkg/adapter/checkpoint_restore.go | 99 ++++++++++++++++++++++++++++++++++++++ pkg/adapter/containers.go | 6 ++- pkg/adapter/containers_remote.go | 10 +++- 10 files changed, 225 insertions(+), 21 deletions(-) create mode 100644 pkg/adapter/checkpoint_restore.go diff --git a/cmd/podman/checkpoint.go b/cmd/podman/checkpoint.go index 234d683bb..86bc8b973 100644 --- a/cmd/podman/checkpoint.go +++ b/cmd/podman/checkpoint.go @@ -46,6 +46,7 @@ func init() { flags.BoolVar(&checkpointCommand.TcpEstablished, "tcp-established", false, "Checkpoint a container with established TCP connections") flags.BoolVarP(&checkpointCommand.All, "all", "a", false, "Checkpoint all running containers") flags.BoolVarP(&checkpointCommand.Latest, "latest", "l", false, "Act on the latest container podman is aware of") + flags.StringVarP(&checkpointCommand.Export, "export", "e", "", "Export the checkpoint image to a tar.gz") markFlagHiddenForRemoteClient("latest", flags) } @@ -64,6 +65,7 @@ func checkpointCmd(c *cliconfig.CheckpointValues) error { Keep: c.Keep, KeepRunning: c.LeaveRunning, TCPEstablished: c.TcpEstablished, + TargetFile: c.Export, } return runtime.Checkpoint(c, options) } diff --git a/cmd/podman/cliconfig/config.go b/cmd/podman/cliconfig/config.go index aaa4513d8..d742830ee 100644 --- a/cmd/podman/cliconfig/config.go +++ b/cmd/podman/cliconfig/config.go @@ -89,6 +89,7 @@ type CheckpointValues struct { TcpEstablished bool All bool Latest bool + Export string } type CommitValues struct { @@ -426,6 +427,7 @@ type RestoreValues struct { Keep bool Latest bool TcpEstablished bool + Import string } type RmValues struct { diff --git a/cmd/podman/restore.go b/cmd/podman/restore.go index 36ae16183..828ae682f 100644 --- a/cmd/podman/restore.go +++ b/cmd/podman/restore.go @@ -24,10 +24,10 @@ var ( restoreCommand.InputArgs = args restoreCommand.GlobalFlags = MainGlobalOpts restoreCommand.Remote = remoteclient - return restoreCmd(&restoreCommand) + return restoreCmd(&restoreCommand, cmd) }, Args: func(cmd *cobra.Command, args []string) error { - return checkAllAndLatest(cmd, args, false) + return checkAllAndLatest(cmd, args, true) }, Example: `podman container restore ctrID podman container restore --latest @@ -44,11 +44,12 @@ func init() { flags.BoolVarP(&restoreCommand.Keep, "keep", "k", false, "Keep all temporary checkpoint files") flags.BoolVarP(&restoreCommand.Latest, "latest", "l", false, "Act on the latest container podman is aware of") flags.BoolVar(&restoreCommand.TcpEstablished, "tcp-established", false, "Restore a container with established TCP connections") + flags.StringVarP(&restoreCommand.Import, "import", "i", "", "Restore from exported checkpoint archive (tar.gz)") markFlagHiddenForRemoteClient("latest", flags) } -func restoreCmd(c *cliconfig.RestoreValues) error { +func restoreCmd(c *cliconfig.RestoreValues, cmd *cobra.Command) error { if rootless.IsRootless() { return errors.New("restoring a container requires root") } @@ -62,6 +63,11 @@ func restoreCmd(c *cliconfig.RestoreValues) error { options := libpod.ContainerCheckpointOptions{ Keep: c.Keep, TCPEstablished: c.TcpEstablished, + TargetFile: c.Import, } - return runtime.Restore(c, options) + + if (c.Import != "") && (c.All || c.Latest) { + return errors.Errorf("Cannot use --import and --all or --latest at the same time") + } + return runtime.Restore(getContext(), c, options) } diff --git a/libpod/container_api.go b/libpod/container_api.go index b8b60a6e2..72c4d7f37 100644 --- a/libpod/container_api.go +++ b/libpod/container_api.go @@ -825,6 +825,13 @@ type ContainerCheckpointOptions struct { // Checkpoint checkpoints a container func (c *Container) Checkpoint(ctx context.Context, options ContainerCheckpointOptions) error { logrus.Debugf("Trying to checkpoint container %s", c.ID()) + + if options.TargetFile != "" { + if err := c.prepareCheckpointExport(); err != nil { + return err + } + } + if !c.batched { c.lock.Lock() defer c.lock.Unlock() diff --git a/libpod/container_internal.go b/libpod/container_internal.go index 42f56e82f..c0b5e4302 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -1346,7 +1346,7 @@ func (c *Container) appendStringToRundir(destFile, output string) (string, error return filepath.Join(c.state.RunDir, destFile), nil } -// Save OCI spec to disk, replacing any existing specs for the container +// saveSpec saves the OCI spec to disk, replacing any existing specs for the container func (c *Container) saveSpec(spec *spec.Spec) error { // If the OCI spec already exists, we need to replace it // Cannot guarantee some things, e.g. network namespaces, have the same diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index 3c02e3ae0..d471d4191 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -499,6 +499,9 @@ func (c *Container) addNamespaceContainer(g *generate.Generator, ns LinuxNS, ctr } func (c *Container) exportCheckpoint(dest string) (err error) { + if (len(c.config.NamedVolumes) > 0) || (len(c.Dependencies()) > 0) { + return errors.Errorf("Cannot export checkpoints of containers with named volumes or dependencies") + } logrus.Debugf("Exporting checkpoint image of container %q to %q", c.ID(), dest) input, err := archive.TarWithOptions(c.bundlePath(), &archive.TarOptions{ Compression: archive.Gzip, @@ -587,6 +590,12 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO return err } + if options.TargetFile != "" { + if err = c.exportCheckpoint(options.TargetFile); err != nil { + return err + } + } + logrus.Debugf("Checkpointed container %s", c.ID()) if !options.KeepRunning { @@ -653,6 +662,12 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti return errors.Wrapf(ErrCtrStateInvalid, "container %s is running or paused, cannot restore", c.ID()) } + if options.TargetFile != "" { + if err = c.importCheckpoint(options.TargetFile); err != nil { + return err + } + } + // Let's try to stat() CRIU's inventory file. If it does not exist, it makes // no sense to try a restore. This is a minimal check if a checkpoint exist. if _, err := os.Stat(filepath.Join(c.CheckpointPath(), "inventory.img")); os.IsNotExist(err) { @@ -710,23 +725,44 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti return err } + // Restoring from an import means that we are doing migration + if options.TargetFile != "" { + g.SetRootPath(c.state.Mountpoint) + } + // We want to have the same network namespace as before. if c.config.CreateNetNS { g.AddOrReplaceLinuxNamespace(spec.NetworkNamespace, c.state.NetNS.Path()) } - // Save the OCI spec to disk - if err := c.saveSpec(g.Spec()); err != nil { + if err := c.makeBindMounts(); err != nil { return err } - if err := c.makeBindMounts(); err != nil { - return err + if options.TargetFile != "" { + for dstPath, srcPath := range c.state.BindMounts { + newMount := spec.Mount{ + Type: "bind", + Source: srcPath, + Destination: dstPath, + Options: []string{"bind", "private"}, + } + if c.IsReadOnly() && dstPath != "/dev/shm" { + newMount.Options = append(newMount.Options, "ro", "nosuid", "noexec", "nodev") + } + if !MountExists(g.Mounts(), dstPath) { + g.AddMount(newMount) + } + } } // Cleanup for a working restore. c.removeConmonFiles() + // Save the OCI spec to disk + if err := c.saveSpec(g.Spec()); err != nil { + return err + } if err := c.runtime.ociRuntime.createContainer(c, c.config.CgroupParent, &options); err != nil { return err } diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 0c8d3edab..9d0bbf7e8 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -14,6 +14,7 @@ import ( "github.com/containers/storage" "github.com/containers/storage/pkg/stringid" spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-tools/generate" opentracing "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -34,7 +35,7 @@ type CtrCreateOption func(*Container) error // A true return will include the container, a false return will exclude it. type ContainerFilter func(*Container) bool -// NewContainer creates a new container from a given OCI config +// NewContainer creates a new container from a given OCI config. func (r *Runtime) NewContainer(ctx context.Context, rSpec *spec.Spec, options ...CtrCreateOption) (c *Container, err error) { r.lock.Lock() defer r.lock.Unlock() @@ -44,20 +45,38 @@ func (r *Runtime) NewContainer(ctx context.Context, rSpec *spec.Spec, options .. return r.newContainer(ctx, rSpec, options...) } -func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options ...CtrCreateOption) (c *Container, err error) { - span, _ := opentracing.StartSpanFromContext(ctx, "newContainer") - span.SetTag("type", "runtime") - defer span.Finish() +// RestoreContainer re-creates a container from an imported checkpoint +func (r *Runtime) RestoreContainer(ctx context.Context, rSpec *spec.Spec, config *ContainerConfig) (c *Container, err error) { + r.lock.Lock() + defer r.lock.Unlock() + if !r.valid { + return nil, ErrRuntimeStopped + } + ctr, err := r.initContainerVariables(rSpec, config) + if err != nil { + return nil, errors.Wrapf(err, "error initializing container variables") + } + return r.setupContainer(ctx, ctr, true) +} + +func (r *Runtime) initContainerVariables(rSpec *spec.Spec, config *ContainerConfig) (c *Container, err error) { if rSpec == nil { return nil, errors.Wrapf(ErrInvalidArg, "must provide a valid runtime spec to create container") } - ctr := new(Container) ctr.config = new(ContainerConfig) ctr.state = new(ContainerState) - ctr.config.ID = stringid.GenerateNonCryptoID() + if config == nil { + ctr.config.ID = stringid.GenerateNonCryptoID() + ctr.config.ShmSize = DefaultShmSize + } else { + // This is a restore from an imported checkpoint + if err := JSONDeepCopy(config, ctr.config); err != nil { + return nil, errors.Wrapf(err, "error copying container config for restore") + } + } ctr.config.Spec = new(spec.Spec) if err := JSONDeepCopy(rSpec, ctr.config.Spec); err != nil { @@ -65,8 +84,6 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. } ctr.config.CreatedTime = time.Now() - ctr.config.ShmSize = DefaultShmSize - ctr.state.BindMounts = make(map[string]string) ctr.config.StopTimeout = CtrRemoveTimeout @@ -80,12 +97,29 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. } ctr.runtime = r + + return ctr, nil +} + +func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options ...CtrCreateOption) (c *Container, err error) { + span, _ := opentracing.StartSpanFromContext(ctx, "newContainer") + span.SetTag("type", "runtime") + defer span.Finish() + + ctr, err := r.initContainerVariables(rSpec, nil) + if err != nil { + return nil, errors.Wrapf(err, "error initializing container variables") + } + for _, option := range options { if err := option(ctr); err != nil { return nil, errors.Wrapf(err, "error running container create option") } } + return r.setupContainer(ctx, ctr, false) +} +func (r *Runtime) setupContainer(ctx context.Context, ctr *Container, restore bool) (c *Container, err error) { // Allocate a lock for the container lock, err := r.lockManager.AllocateLock() if err != nil { @@ -154,6 +188,14 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. return nil, errors.Wrapf(ErrInvalidArg, "unsupported CGroup manager: %s - cannot validate cgroup parent", r.config.CgroupManager) } + if restore { + // Remove information about /dev/shm mount + // for new container from imported checkpoint + g := generate.Generator{Config: ctr.config.Spec} + g.RemoveMount("/dev/shm") + ctr.config.ShmDir = "" + } + // Set up storage for the container if err := ctr.setupStorage(ctx); err != nil { return nil, err diff --git a/pkg/adapter/checkpoint_restore.go b/pkg/adapter/checkpoint_restore.go new file mode 100644 index 000000000..9df1704ea --- /dev/null +++ b/pkg/adapter/checkpoint_restore.go @@ -0,0 +1,99 @@ +// +build !remoteclient + +package adapter + +import ( + "context" + "github.com/containers/libpod/libpod" + "github.com/containers/storage/pkg/archive" + jsoniter "github.com/json-iterator/go" + spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "io/ioutil" + "os" + "path/filepath" +) + +// Prefixing the checkpoint/restore related functions with 'cr' + +// crImportFromJSON imports the JSON files stored in the exported +// checkpoint tarball +func crImportFromJSON(filePath string, v interface{}) error { + jsonFile, err := os.Open(filePath) + if err != nil { + return errors.Wrapf(err, "Failed to open container definition %s for restore", filePath) + } + defer jsonFile.Close() + + content, err := ioutil.ReadAll(jsonFile) + if err != nil { + return errors.Wrapf(err, "Failed to read container definition %s for restore", filePath) + } + json := jsoniter.ConfigCompatibleWithStandardLibrary + if err = json.Unmarshal([]byte(content), v); err != nil { + return errors.Wrapf(err, "Failed to unmarshal container definition %s for restore", filePath) + } + + return nil +} + +// crImportCheckpoint it the function which imports the information +// from checkpoint tarball and re-creates the container from that information +func crImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, input string) ([]*libpod.Container, error) { + // First get the container definition from the + // tarball to a temporary directory + archiveFile, err := os.Open(input) + if err != nil { + return nil, errors.Wrapf(err, "Failed to open checkpoint archive %s for import", input) + } + defer archiveFile.Close() + options := &archive.TarOptions{ + // Here we only need the files config.dump and spec.dump + ExcludePatterns: []string{ + "checkpoint", + "artifacts", + "ctr.log", + "network.status", + }, + } + dir, err := ioutil.TempDir("", "checkpoint") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + err = archive.Untar(archiveFile, dir, options) + if err != nil { + return nil, errors.Wrapf(err, "Unpacking of checkpoint archive %s failed", input) + } + + // Load spec.dump from temporary directory + spec := new(spec.Spec) + if err := crImportFromJSON(filepath.Join(dir, "spec.dump"), spec); err != nil { + return nil, err + } + + // Load config.dump from temporary directory + config := new(libpod.ContainerConfig) + if err = crImportFromJSON(filepath.Join(dir, "config.dump"), config); err != nil { + return nil, err + } + + // This should not happen as checkpoints with these options are not exported. + if (len(config.Dependencies) > 0) || (len(config.NamedVolumes) > 0) { + return nil, errors.Errorf("Cannot import checkpoints of containers with named volumes or dependencies") + } + + // Now create a new container from the just loaded information + container, err := runtime.RestoreContainer(ctx, spec, config) + if err != nil { + return nil, err + } + + var containers []*libpod.Container + if container == nil { + return nil, nil + } + + containers = append(containers, container) + return containers, nil +} diff --git a/pkg/adapter/containers.go b/pkg/adapter/containers.go index 34ee70d3d..b7f4c272b 100644 --- a/pkg/adapter/containers.go +++ b/pkg/adapter/containers.go @@ -526,7 +526,7 @@ func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues, options libpod. } // Restore one or more containers -func (r *LocalRuntime) Restore(c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error { +func (r *LocalRuntime) Restore(ctx context.Context, c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error { var ( containers []*libpod.Container err, lastError error @@ -538,7 +538,9 @@ func (r *LocalRuntime) Restore(c *cliconfig.RestoreValues, options libpod.Contai return state == libpod.ContainerStateExited }) - if c.All { + if c.Import != "" { + containers, err = crImportCheckpoint(ctx, r.Runtime, c.Import) + } else if c.All { containers, err = r.GetContainers(filterFuncs...) } else { containers, err = shortcuts.GetContainersByContext(false, c.Latest, c.InputArgs, r.Runtime) diff --git a/pkg/adapter/containers_remote.go b/pkg/adapter/containers_remote.go index bc6a9cfcd..776fcbb70 100644 --- a/pkg/adapter/containers_remote.go +++ b/pkg/adapter/containers_remote.go @@ -664,6 +664,10 @@ func (r *LocalRuntime) Attach(ctx context.Context, c *cliconfig.AttachValues) er // Checkpoint one or more containers func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues, options libpod.ContainerCheckpointOptions) error { + if c.Export != "" { + return errors.New("the remote client does not support exporting checkpoints") + } + var lastError error ids, err := iopodman.GetContainersByContext().Call(r.Conn, c.All, c.Latest, c.InputArgs) if err != nil { @@ -699,7 +703,11 @@ func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues, options libpod. } // Restore one or more containers -func (r *LocalRuntime) Restore(c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error { +func (r *LocalRuntime) Restore(ctx context.Context, c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error { + if c.Import != "" { + return errors.New("the remote client does not support importing checkpoints") + } + var lastError error ids, err := iopodman.GetContainersByContext().Call(r.Conn, c.All, c.Latest, c.InputArgs) if err != nil { -- cgit v1.2.3-54-g00ecf From 2aa3261744b4247996abf3ef552e03a89f77cfb8 Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Tue, 5 Feb 2019 16:26:30 +0000 Subject: Add test case for container migration The difference between container checkpoint/restore and container migration is that for migration the container which was checkpointed must not exist during restore. To simulate migration the container is remove ('podman rm -fa') before being restored. The migration test does following steps: * podman run * podman container checkpoint -l -e /tmp/checkpoint.tar.gz * podman rm -fa * podman container restore -i /tmp/checkpoint.tar.gz Signed-off-by: Adrian Reber --- test/e2e/checkpoint_test.go | 34 ++++++++++++++++++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/test/e2e/checkpoint_test.go b/test/e2e/checkpoint_test.go index 95ec21433..bd9cdcb8d 100644 --- a/test/e2e/checkpoint_test.go +++ b/test/e2e/checkpoint_test.go @@ -347,4 +347,38 @@ var _ = Describe("Podman checkpoint", func() { Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) }) + // This test does the same steps which are necessary for migrating + // a container from one host to another + It("podman checkpoint container with export (migration)", func() { + // CRIU does not work with seccomp correctly on RHEL7 + session := podmanTest.Podman([]string{"run", "-it", "--security-opt", "seccomp=unconfined", "-d", ALPINE, "top"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1)) + + result := podmanTest.Podman([]string{"container", "checkpoint", "-l", "-e", "/tmp/checkpoint.tar.gz"}) + result.WaitWithDefaultTimeout() + + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) + Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited")) + + // Remove all containers to simulate migration + result = podmanTest.Podman([]string{"rm", "-fa"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) + + result = podmanTest.Podman([]string{"container", "restore", "-i", "/tmp/checkpoint.tar.gz"}) + result.WaitWithDefaultTimeout() + + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1)) + Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up")) + + result = podmanTest.Podman([]string{"rm", "-fa"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) + }) }) -- cgit v1.2.3-54-g00ecf From 7b1ab8a0f4c1dc00e72684e3127ab4a4c1241e73 Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Tue, 5 Feb 2019 16:54:01 +0000 Subject: Added bash completion for container migration Signed-off-by: Adrian Reber --- completions/bash/podman | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/completions/bash/podman b/completions/bash/podman index 49c8c0e52..4972188aa 100644 --- a/completions/bash/podman +++ b/completions/bash/podman @@ -742,6 +742,10 @@ _podman_container_attach() { } _podman_container_checkpoint() { + local options_with_args=" + -e + --export + " local boolean_options=" -a --all @@ -755,9 +759,15 @@ _podman_container_checkpoint() { --leave-running --tcp-established " + case "$prev" in + -e|--export) + _filedir + return + ;; + esac case "$cur" in -*) - COMPREPLY=($(compgen -W "$boolean_options" -- "$cur")) + COMPREPLY=($(compgen -W "$boolean_options $options_with_args" -- "$cur")) ;; *) __podman_complete_containers_running @@ -844,6 +854,10 @@ _podman_container_restart() { } _podman_container_restore() { + local options_with_args=" + -i + --import + " local boolean_options=" -a --all @@ -855,9 +869,15 @@ _podman_container_restore() { --latest --tcp-established " + case "$prev" in + -i|--import) + _filedir + return + ;; + esac case "$cur" in -*) - COMPREPLY=($(compgen -W "$boolean_options" -- "$cur")) + COMPREPLY=($(compgen -W "$boolean_options $options_with_args" -- "$cur")) ;; *) __podman_complete_containers_created -- cgit v1.2.3-54-g00ecf From 42e903decb8765b1c00d8aa9bc004bb25b1d33f9 Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Tue, 5 Feb 2019 17:04:40 +0000 Subject: Add man-pages for container migration Signed-off-by: Adrian Reber --- docs/podman-container-checkpoint.1.md | 6 ++++++ docs/podman-container-restore.1.md | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/docs/podman-container-checkpoint.1.md b/docs/podman-container-checkpoint.1.md index 79dc12261..afccdf59a 100644 --- a/docs/podman-container-checkpoint.1.md +++ b/docs/podman-container-checkpoint.1.md @@ -38,6 +38,12 @@ image contains established TCP connections, this options is required during restore. Defaults to not checkpointing containers with established TCP connections. +**--export, -e** + +Export the checkpoint to a tar.gz file. The exported checkpoint can be used +to import the container on another system and thus enabling container live +migration. + ## EXAMPLE podman container checkpoint mywebserver diff --git a/docs/podman-container-restore.1.md b/docs/podman-container-restore.1.md index e41f7c1d8..578d34ff3 100644 --- a/docs/podman-container-restore.1.md +++ b/docs/podman-container-restore.1.md @@ -42,6 +42,12 @@ If the checkpoint image does not contain established TCP connections this option is ignored. Defaults to not restoring containers with established TCP connections. +**--import, -i** + +Import a checkpoint tar.gz file, which was exported by Podman. This can be used +to import a checkpointed container from another host. It is not necessary to specify +a container when restoring from an exported checkpoint. + ## EXAMPLE podman container restore mywebserver -- cgit v1.2.3-54-g00ecf From a4d33330d6886ab90f7d90ef686f2eee4dda683b Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Tue, 5 Feb 2019 17:20:31 +0000 Subject: Include container migration into tutorial Signed-off-by: Adrian Reber --- docs/tutorials/podman_tutorial.md | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/docs/tutorials/podman_tutorial.md b/docs/tutorials/podman_tutorial.md index 032b7c851..8b29264db 100644 --- a/docs/tutorials/podman_tutorial.md +++ b/docs/tutorials/podman_tutorial.md @@ -96,6 +96,28 @@ After being restored, the container will answer requests again as it did before curl http://:8080 ``` +### Migrate the container +To live migrate a container from one host to another the container is checkpointed on the source +system of the migration, transferred to the destination system and then restored on the destination +system. When transferring the checkpoint, it is possible to specify an output-file. + +On the source system: +```console +sudo podman container checkpoint -e /tmp/checkpoint.tar.gz +scp /tmp/checkpoint.tar.gz :/tmp +``` + +On the destination system: +```console +sudo podman container restore -i /tmp/checkpoint.tar.gz +``` + +After being restored, the container will answer requests again as it did before checkpointing. This +time the container will continue to run on the destination system. +```console +curl http://:8080 +``` + ### Stopping the container To stop the httpd container: ```console -- cgit v1.2.3-54-g00ecf From 0e072f9a9785c67f38859ab989267397b57154c8 Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Mon, 15 Apr 2019 18:45:52 +0000 Subject: Also download container images during restore If restoring a container from a checkpoint it was necessary that the image the container is based was already available (podman pull). This commit adds the image download to podman container restore if it does not exist. Signed-off-by: Adrian Reber --- pkg/adapter/checkpoint_restore.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/pkg/adapter/checkpoint_restore.go b/pkg/adapter/checkpoint_restore.go index 9df1704ea..4ca17dd93 100644 --- a/pkg/adapter/checkpoint_restore.go +++ b/pkg/adapter/checkpoint_restore.go @@ -5,10 +5,12 @@ package adapter import ( "context" "github.com/containers/libpod/libpod" + "github.com/containers/libpod/libpod/image" "github.com/containers/storage/pkg/archive" jsoniter "github.com/json-iterator/go" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" + "io" "io/ioutil" "os" "path/filepath" @@ -83,6 +85,20 @@ func crImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, input stri return nil, errors.Errorf("Cannot import checkpoints of containers with named volumes or dependencies") } + // The code to load the images is copied from create.go + var writer io.Writer + // In create.go this only set if '--quiet' does not exist. + writer = os.Stderr + rtc, err := runtime.GetConfig() + if err != nil { + return nil, err + } + + _, err = runtime.ImageRuntime().New(ctx, config.RootfsImageName, rtc.SignaturePolicyPath, "", writer, nil, image.SigningOptions{}, false, nil) + if err != nil { + return nil, err + } + // Now create a new container from the just loaded information container, err := runtime.RestoreContainer(ctx, spec, config) if err != nil { -- cgit v1.2.3-54-g00ecf From bef83c42eaacd83fb5020395f1bbdeb9a6f0f220 Mon Sep 17 00:00:00 2001 From: Adrian Reber Date: Thu, 14 Mar 2019 07:57:16 +0000 Subject: migration: add possibility to restore a container with a new name The option to restore a container from an external checkpoint archive (podman container restore -i /tmp/checkpoint.tar.gz) restores a container with the same name and same ID as id had before checkpointing. This commit adds the option '--name,-n' to 'podman container restore'. With this option the restored container gets the name specified after '--name,-n' and a new ID. This way it is possible to restore one container multiple times. If a container is restored with a new name Podman will not try to request the same IP address for the container as it had during checkpointing. This implicitly assumes that if a container is restored from a checkpoint archive with a different name, that it will be restored multiple times and restoring a container multiple times with the same IP address will fail as each IP address can only be used once. Signed-off-by: Adrian Reber --- cmd/podman/cliconfig/config.go | 1 + cmd/podman/restore.go | 10 ++++++++++ completions/bash/podman | 2 ++ docs/podman-container-restore.1.md | 12 ++++++++++++ libpod/container_api.go | 4 ++++ libpod/container_internal_linux.go | 8 +++++++- libpod/runtime_ctr.go | 15 ++++++++++++++- pkg/adapter/checkpoint_restore.go | 32 +++++++++++++++++++++++++++++++- pkg/adapter/containers.go | 2 +- test/e2e/checkpoint_test.go | 11 +++++++++++ 10 files changed, 93 insertions(+), 4 deletions(-) diff --git a/cmd/podman/cliconfig/config.go b/cmd/podman/cliconfig/config.go index d742830ee..466164cc1 100644 --- a/cmd/podman/cliconfig/config.go +++ b/cmd/podman/cliconfig/config.go @@ -428,6 +428,7 @@ type RestoreValues struct { Latest bool TcpEstablished bool Import string + Name string } type RmValues struct { diff --git a/cmd/podman/restore.go b/cmd/podman/restore.go index 828ae682f..9c77d4a5e 100644 --- a/cmd/podman/restore.go +++ b/cmd/podman/restore.go @@ -45,6 +45,7 @@ func init() { flags.BoolVarP(&restoreCommand.Latest, "latest", "l", false, "Act on the latest container podman is aware of") flags.BoolVar(&restoreCommand.TcpEstablished, "tcp-established", false, "Restore a container with established TCP connections") flags.StringVarP(&restoreCommand.Import, "import", "i", "", "Restore from exported checkpoint archive (tar.gz)") + flags.StringVarP(&restoreCommand.Name, "name", "n", "", "Specify new name for container restored from exported checkpoint (only works with --import)") markFlagHiddenForRemoteClient("latest", flags) } @@ -64,6 +65,15 @@ func restoreCmd(c *cliconfig.RestoreValues, cmd *cobra.Command) error { Keep: c.Keep, TCPEstablished: c.TcpEstablished, TargetFile: c.Import, + Name: c.Name, + } + + if c.Import == "" && c.Name != "" { + return errors.Errorf("--name can only used with --import") + } + + if c.Name != "" && c.TcpEstablished { + return errors.Errorf("--tcp-established cannot be used with --name") } if (c.Import != "") && (c.All || c.Latest) { diff --git a/completions/bash/podman b/completions/bash/podman index 4972188aa..efb8a6a9b 100644 --- a/completions/bash/podman +++ b/completions/bash/podman @@ -857,6 +857,8 @@ _podman_container_restore() { local options_with_args=" -i --import + -n + --name " local boolean_options=" -a diff --git a/docs/podman-container-restore.1.md b/docs/podman-container-restore.1.md index 578d34ff3..5efc280fe 100644 --- a/docs/podman-container-restore.1.md +++ b/docs/podman-container-restore.1.md @@ -48,6 +48,18 @@ Import a checkpoint tar.gz file, which was exported by Podman. This can be used to import a checkpointed container from another host. It is not necessary to specify a container when restoring from an exported checkpoint. +**--name, -n** + +This is only available in combination with **--import, -i**. If a container is restored +from a checkpoint tar.gz file it is possible to rename it with **--name, -n**. This +way it is possible to restore a container from a checkpoint multiple times with different +names. + +If the **--name, -n** option is used, Podman will not attempt to assign the same IP +address to the container it was using before checkpointing as each IP address can only +be used once and the restored container will have another IP address. This also means +that **--name, -n** cannot be used in combination with **--tcp-established**. + ## EXAMPLE podman container restore mywebserver diff --git a/libpod/container_api.go b/libpod/container_api.go index 72c4d7f37..c27cb85ea 100644 --- a/libpod/container_api.go +++ b/libpod/container_api.go @@ -820,6 +820,10 @@ type ContainerCheckpointOptions struct { // Import tells the API to read the checkpoint image from // the filename set in TargetFile TargetFile string + // Name tells the API that during restore from an exported + // checkpoint archive a new name should be used for the + // restored container + Name string } // Checkpoint checkpoints a container diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index d471d4191..4acc77afa 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -681,7 +681,13 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti // Read network configuration from checkpoint // Currently only one interface with one IP is supported. networkStatusFile, err := os.Open(filepath.Join(c.bundlePath(), "network.status")) - if err == nil { + // If the restored container should get a new name, the IP address of + // the container will not be restored. This assumes that if a new name is + // specified, the container is restored multiple times. + // TODO: This implicit restoring with or without IP depending on an + // unrelated restore parameter (--name) does not seem like the + // best solution. + if err == nil && options.Name == "" { // The file with the network.status does exist. Let's restore the // container with the same IP address as during checkpointing. defer networkStatusFile.Close() diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 9d0bbf7e8..cf1f5701d 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -76,6 +76,14 @@ func (r *Runtime) initContainerVariables(rSpec *spec.Spec, config *ContainerConf if err := JSONDeepCopy(config, ctr.config); err != nil { return nil, errors.Wrapf(err, "error copying container config for restore") } + // If the ID is empty a new name for the restored container was requested + if ctr.config.ID == "" { + ctr.config.ID = stringid.GenerateNonCryptoID() + // Fixup ExitCommand with new ID + ctr.config.ExitCommand[len(ctr.config.ExitCommand)-1] = ctr.config.ID + } + // Reset the log path to point to the default + ctr.config.LogPath = "" } ctr.config.Spec = new(spec.Spec) @@ -189,11 +197,16 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container, restore bo } if restore { - // Remove information about /dev/shm mount + // Remove information about bind mount // for new container from imported checkpoint g := generate.Generator{Config: ctr.config.Spec} g.RemoveMount("/dev/shm") ctr.config.ShmDir = "" + g.RemoveMount("/etc/resolv.conf") + g.RemoveMount("/etc/hostname") + g.RemoveMount("/etc/hosts") + g.RemoveMount("/run/.containerenv") + g.RemoveMount("/run/secrets") } // Set up storage for the container diff --git a/pkg/adapter/checkpoint_restore.go b/pkg/adapter/checkpoint_restore.go index 4ca17dd93..97ba5ecf7 100644 --- a/pkg/adapter/checkpoint_restore.go +++ b/pkg/adapter/checkpoint_restore.go @@ -41,7 +41,7 @@ func crImportFromJSON(filePath string, v interface{}) error { // crImportCheckpoint it the function which imports the information // from checkpoint tarball and re-creates the container from that information -func crImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, input string) ([]*libpod.Container, error) { +func crImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, input string, name string) ([]*libpod.Container, error) { // First get the container definition from the // tarball to a temporary directory archiveFile, err := os.Open(input) @@ -85,6 +85,18 @@ func crImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, input stri return nil, errors.Errorf("Cannot import checkpoints of containers with named volumes or dependencies") } + ctrID := config.ID + newName := false + + // Check if the restored container gets a new name + if name != "" { + config.ID = "" + config.Name = name + newName = true + } + + ctrName := config.Name + // The code to load the images is copied from create.go var writer io.Writer // In create.go this only set if '--quiet' does not exist. @@ -110,6 +122,24 @@ func crImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, input stri return nil, nil } + containerConfig := container.Config() + if containerConfig.Name != ctrName { + return nil, errors.Errorf("Name of restored container (%s) does not match requested name (%s)", containerConfig.Name, ctrName) + } + + if newName == false { + // Only check ID for a restore with the same name. + // Using -n to request a new name for the restored container, will also create a new ID + if containerConfig.ID != ctrID { + return nil, errors.Errorf("ID of restored container (%s) does not match requested ID (%s)", containerConfig.ID, ctrID) + } + } + + // Check if the ExitCommand points to the correct container ID + if containerConfig.ExitCommand[len(containerConfig.ExitCommand)-1] != containerConfig.ID { + return nil, errors.Errorf("'ExitCommandID' uses ID %s instead of container ID %s", containerConfig.ExitCommand[len(containerConfig.ExitCommand)-1], containerConfig.ID) + } + containers = append(containers, container) return containers, nil } diff --git a/pkg/adapter/containers.go b/pkg/adapter/containers.go index b7f4c272b..29297fbd5 100644 --- a/pkg/adapter/containers.go +++ b/pkg/adapter/containers.go @@ -539,7 +539,7 @@ func (r *LocalRuntime) Restore(ctx context.Context, c *cliconfig.RestoreValues, }) if c.Import != "" { - containers, err = crImportCheckpoint(ctx, r.Runtime, c.Import) + containers, err = crImportCheckpoint(ctx, r.Runtime, c.Import, c.Name) } else if c.All { containers, err = r.GetContainers(filterFuncs...) } else { diff --git a/test/e2e/checkpoint_test.go b/test/e2e/checkpoint_test.go index bd9cdcb8d..d452a062b 100644 --- a/test/e2e/checkpoint_test.go +++ b/test/e2e/checkpoint_test.go @@ -376,9 +376,20 @@ var _ = Describe("Podman checkpoint", func() { Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1)) Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up")) + // Restore container a second time with different name + result = podmanTest.Podman([]string{"container", "restore", "-i", "/tmp/checkpoint.tar.gz", "-n", "restore_again"}) + result.WaitWithDefaultTimeout() + + Expect(result.ExitCode()).To(Equal(0)) + Expect(podmanTest.NumberOfContainersRunning()).To(Equal(2)) + Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up")) + result = podmanTest.Podman([]string{"rm", "-fa"}) result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0)) + + // Remove exported checkpoint + os.Remove("/tmp/checkpoint.tar.gz") }) }) -- cgit v1.2.3-54-g00ecf