Merge pull request #2272 from adrianreber/migration

Add support to migrate containers

4 files changed, 243 insertions, 19 deletions

diff --git a/libpod/container_api.go b/libpod/container_api.go
index eff5bfe5f..c27cb85ea 100644
--- a/libpod/container_api.go
+++ b/libpod/container_api.go
@@ -815,11 +815,27 @@ type ContainerCheckpointOptions struct {
 	// TCPEstablished tells the API to checkpoint a container
 	// even if it contains established TCP connections
 	TCPEstablished bool
+	// Export tells the API to write the checkpoint image to
+	// the filename set in TargetFile
+	// Import tells the API to read the checkpoint image from
+	// the filename set in TargetFile
+	TargetFile string
+	// Name tells the API that during restore from an exported
+	// checkpoint archive a new name should be used for the
+	// restored container
+	Name string
 }
 
 // Checkpoint checkpoints a container
 func (c *Container) Checkpoint(ctx context.Context, options ContainerCheckpointOptions) error {
 	logrus.Debugf("Trying to checkpoint container %s", c.ID())
+
+	if options.TargetFile != "" {
+		if err := c.prepareCheckpointExport(); err != nil {
+			return err
+		}
+	}
+
 	if !c.batched {
 		c.lock.Lock()
 		defer c.lock.Unlock()
diff --git a/libpod/container_internal.go b/libpod/container_internal.go
index 5f8dd1c72..c0b5e4302 100644
--- a/libpod/container_internal.go
+++ b/libpod/container_internal.go
@@ -21,6 +21,7 @@ import (
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/mount"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/generate"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
@@ -1345,7 +1346,7 @@ func (c *Container) appendStringToRundir(destFile, output string) (string, error
 	return filepath.Join(c.state.RunDir, destFile), nil
 }
 
-// Save OCI spec to disk, replacing any existing specs for the container
+// saveSpec saves the OCI spec to disk, replacing any existing specs for the container
 func (c *Container) saveSpec(spec *spec.Spec) error {
 	// If the OCI spec already exists, we need to replace it
 	// Cannot guarantee some things, e.g. network namespaces, have the same
@@ -1501,3 +1502,40 @@ func (c *Container) checkReadyForRemoval() error {
 
 	return nil
 }
+
+// writeJSONFile marshalls and writes the given data to a JSON file
+// in the bundle path
+func (c *Container) writeJSONFile(v interface{}, file string) (err error) {
+	fileJSON, err := json.MarshalIndent(v, "", "  ")
+	if err != nil {
+		return errors.Wrapf(err, "error writing JSON to %s for container %s", file, c.ID())
+	}
+	file = filepath.Join(c.bundlePath(), file)
+	if err := ioutil.WriteFile(file, fileJSON, 0644); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// prepareCheckpointExport writes the config and spec to
+// JSON files for later export
+func (c *Container) prepareCheckpointExport() (err error) {
+	// save live config
+	if err := c.writeJSONFile(c.Config(), "config.dump"); err != nil {
+		return err
+	}
+
+	// save spec
+	jsonPath := filepath.Join(c.bundlePath(), "config.json")
+	g, err := generate.NewFromFile(jsonPath)
+	if err != nil {
+		logrus.Debugf("generating spec for container %q failed with %v", c.ID(), err)
+		return err
+	}
+	if err := c.writeJSONFile(g.Spec(), "spec.dump"); err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go
index f25f76092..4acc77afa 100644
--- a/libpod/container_internal_linux.go
+++ b/libpod/container_internal_linux.go
@@ -5,6 +5,7 @@ package libpod
 import (
 	"context"
 	"fmt"
+	"io"
 	"io/ioutil"
 	"net"
 	"os"
@@ -25,6 +26,7 @@ import (
 	"github.com/containers/libpod/pkg/lookup"
 	"github.com/containers/libpod/pkg/resolvconf"
 	"github.com/containers/libpod/pkg/rootless"
+	"github.com/containers/storage/pkg/archive"
 	securejoin "github.com/cyphar/filepath-securejoin"
 	"github.com/opencontainers/runc/libcontainer/user"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
@@ -496,6 +498,45 @@ func (c *Container) addNamespaceContainer(g *generate.Generator, ns LinuxNS, ctr
 	return nil
 }
 
+func (c *Container) exportCheckpoint(dest string) (err error) {
+	if (len(c.config.NamedVolumes) > 0) || (len(c.Dependencies()) > 0) {
+		return errors.Errorf("Cannot export checkpoints of containers with named volumes or dependencies")
+	}
+	logrus.Debugf("Exporting checkpoint image of container %q to %q", c.ID(), dest)
+	input, err := archive.TarWithOptions(c.bundlePath(), &archive.TarOptions{
+		Compression:      archive.Gzip,
+		IncludeSourceDir: true,
+		IncludeFiles: []string{
+			"checkpoint",
+			"artifacts",
+			"ctr.log",
+			"config.dump",
+			"spec.dump",
+			"network.status"},
+	})
+
+	if err != nil {
+		return errors.Wrapf(err, "error reading checkpoint directory %q", c.ID())
+	}
+
+	outFile, err := os.Create(dest)
+	if err != nil {
+		return errors.Wrapf(err, "error creating checkpoint export file %q", dest)
+	}
+	defer outFile.Close()
+
+	if err := os.Chmod(dest, 0600); err != nil {
+		return errors.Wrapf(err, "cannot chmod %q", dest)
+	}
+
+	_, err = io.Copy(outFile, input)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func (c *Container) checkpointRestoreSupported() (err error) {
 	if !criu.CheckForCriu() {
 		return errors.Errorf("Checkpoint/Restore requires at least CRIU %d", criu.MinCriuVersion)
@@ -549,6 +590,12 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO
 		return err
 	}
 
+	if options.TargetFile != "" {
+		if err = c.exportCheckpoint(options.TargetFile); err != nil {
+			return err
+		}
+	}
+
 	logrus.Debugf("Checkpointed container %s", c.ID())
 
 	if !options.KeepRunning {
@@ -561,15 +608,50 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO
 	}
 
 	if !options.Keep {
-		// Remove log file
-		os.Remove(filepath.Join(c.bundlePath(), "dump.log"))
-		// Remove statistic file
-		os.Remove(filepath.Join(c.bundlePath(), "stats-dump"))
+		cleanup := []string{
+			"dump.log",
+			"stats-dump",
+			"config.dump",
+			"spec.dump",
+		}
+		for _, delete := range cleanup {
+			file := filepath.Join(c.bundlePath(), delete)
+			os.Remove(file)
+		}
 	}
 
 	return c.save()
 }
 
+func (c *Container) importCheckpoint(input string) (err error) {
+	archiveFile, err := os.Open(input)
+	if err != nil {
+		return errors.Wrapf(err, "Failed to open checkpoint archive %s for import", input)
+	}
+
+	defer archiveFile.Close()
+	options := &archive.TarOptions{
+		ExcludePatterns: []string{
+			// config.dump and spec.dump are only required
+			// container creation
+			"config.dump",
+			"spec.dump",
+		},
+	}
+	err = archive.Untar(archiveFile, c.bundlePath(), options)
+	if err != nil {
+		return errors.Wrapf(err, "Unpacking of checkpoint archive %s failed", input)
+	}
+
+	// Make sure the newly created config.json exists on disk
+	g := generate.NewFromSpec(c.config.Spec)
+	if err = c.saveSpec(g.Spec()); err != nil {
+		return errors.Wrap(err, "Saving imported container specification for restore failed")
+	}
+
+	return nil
+}
+
 func (c *Container) restore(ctx context.Context, options ContainerCheckpointOptions) (err error) {
 
 	if err := c.checkpointRestoreSupported(); err != nil {
@@ -580,6 +662,12 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti
 		return errors.Wrapf(ErrCtrStateInvalid, "container %s is running or paused, cannot restore", c.ID())
 	}
 
+	if options.TargetFile != "" {
+		if err = c.importCheckpoint(options.TargetFile); err != nil {
+			return err
+		}
+	}
+
 	// Let's try to stat() CRIU's inventory file. If it does not exist, it makes
 	// no sense to try a restore. This is a minimal check if a checkpoint exist.
 	if _, err := os.Stat(filepath.Join(c.CheckpointPath(), "inventory.img")); os.IsNotExist(err) {
@@ -593,7 +681,13 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti
 	// Read network configuration from checkpoint
 	// Currently only one interface with one IP is supported.
 	networkStatusFile, err := os.Open(filepath.Join(c.bundlePath(), "network.status"))
-	if err == nil {
+	// If the restored container should get a new name, the IP address of
+	// the container will not be restored. This assumes that if a new name is
+	// specified, the container is restored multiple times.
+	// TODO: This implicit restoring with or without IP depending on an
+	//       unrelated restore parameter (--name) does not seem like the
+	//       best solution.
+	if err == nil && options.Name == "" {
 		// The file with the network.status does exist. Let's restore the
 		// container with the same IP address as during checkpointing.
 		defer networkStatusFile.Close()
@@ -637,23 +731,44 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti
 		return err
 	}
 
+	// Restoring from an import means that we are doing migration
+	if options.TargetFile != "" {
+		g.SetRootPath(c.state.Mountpoint)
+	}
+
 	// We want to have the same network namespace as before.
 	if c.config.CreateNetNS {
 		g.AddOrReplaceLinuxNamespace(spec.NetworkNamespace, c.state.NetNS.Path())
 	}
 
-	// Save the OCI spec to disk
-	if err := c.saveSpec(g.Spec()); err != nil {
+	if err := c.makeBindMounts(); err != nil {
 		return err
 	}
 
-	if err := c.makeBindMounts(); err != nil {
-		return err
+	if options.TargetFile != "" {
+		for dstPath, srcPath := range c.state.BindMounts {
+			newMount := spec.Mount{
+				Type:        "bind",
+				Source:      srcPath,
+				Destination: dstPath,
+				Options:     []string{"bind", "private"},
+			}
+			if c.IsReadOnly() && dstPath != "/dev/shm" {
+				newMount.Options = append(newMount.Options, "ro", "nosuid", "noexec", "nodev")
+			}
+			if !MountExists(g.Mounts(), dstPath) {
+				g.AddMount(newMount)
+			}
+		}
 	}
 
 	// Cleanup for a working restore.
 	c.removeConmonFiles()
 
+	// Save the OCI spec to disk
+	if err := c.saveSpec(g.Spec()); err != nil {
+		return err
+	}
 	if err := c.runtime.ociRuntime.createContainer(c, c.config.CgroupParent, &options); err != nil {
 		return err
 	}
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
index 0c8d3edab..cf1f5701d 100644
--- a/libpod/runtime_ctr.go
+++ b/libpod/runtime_ctr.go
@@ -14,6 +14,7 @@ import (
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/stringid"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/generate"
 	opentracing "github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -34,7 +35,7 @@ type CtrCreateOption func(*Container) error
 // A true return will include the container, a false return will exclude it.
 type ContainerFilter func(*Container) bool
 
-// NewContainer creates a new container from a given OCI config
+// NewContainer creates a new container from a given OCI config.
 func (r *Runtime) NewContainer(ctx context.Context, rSpec *spec.Spec, options ...CtrCreateOption) (c *Container, err error) {
 	r.lock.Lock()
 	defer r.lock.Unlock()
@@ -44,20 +45,46 @@ func (r *Runtime) NewContainer(ctx context.Context, rSpec *spec.Spec, options ..
 	return r.newContainer(ctx, rSpec, options...)
 }
 
-func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options ...CtrCreateOption) (c *Container, err error) {
-	span, _ := opentracing.StartSpanFromContext(ctx, "newContainer")
-	span.SetTag("type", "runtime")
-	defer span.Finish()
+// RestoreContainer re-creates a container from an imported checkpoint
+func (r *Runtime) RestoreContainer(ctx context.Context, rSpec *spec.Spec, config *ContainerConfig) (c *Container, err error) {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+	if !r.valid {
+		return nil, ErrRuntimeStopped
+	}
 
+	ctr, err := r.initContainerVariables(rSpec, config)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error initializing container variables")
+	}
+	return r.setupContainer(ctx, ctr, true)
+}
+
+func (r *Runtime) initContainerVariables(rSpec *spec.Spec, config *ContainerConfig) (c *Container, err error) {
 	if rSpec == nil {
 		return nil, errors.Wrapf(ErrInvalidArg, "must provide a valid runtime spec to create container")
 	}
-
 	ctr := new(Container)
 	ctr.config = new(ContainerConfig)
 	ctr.state = new(ContainerState)
 
-	ctr.config.ID = stringid.GenerateNonCryptoID()
+	if config == nil {
+		ctr.config.ID = stringid.GenerateNonCryptoID()
+		ctr.config.ShmSize = DefaultShmSize
+	} else {
+		// This is a restore from an imported checkpoint
+		if err := JSONDeepCopy(config, ctr.config); err != nil {
+			return nil, errors.Wrapf(err, "error copying container config for restore")
+		}
+		// If the ID is empty a new name for the restored container was requested
+		if ctr.config.ID == "" {
+			ctr.config.ID = stringid.GenerateNonCryptoID()
+			// Fixup ExitCommand with new ID
+			ctr.config.ExitCommand[len(ctr.config.ExitCommand)-1] = ctr.config.ID
+		}
+		// Reset the log path to point to the default
+		ctr.config.LogPath = ""
+	}
 
 	ctr.config.Spec = new(spec.Spec)
 	if err := JSONDeepCopy(rSpec, ctr.config.Spec); err != nil {
@@ -65,8 +92,6 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options ..
 	}
 	ctr.config.CreatedTime = time.Now()
 
-	ctr.config.ShmSize = DefaultShmSize
-
 	ctr.state.BindMounts = make(map[string]string)
 
 	ctr.config.StopTimeout = CtrRemoveTimeout
@@ -80,12 +105,29 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options ..
 	}
 
 	ctr.runtime = r
+
+	return ctr, nil
+}
+
+func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options ...CtrCreateOption) (c *Container, err error) {
+	span, _ := opentracing.StartSpanFromContext(ctx, "newContainer")
+	span.SetTag("type", "runtime")
+	defer span.Finish()
+
+	ctr, err := r.initContainerVariables(rSpec, nil)
+	if err != nil {
+		return nil, errors.Wrapf(err, "error initializing container variables")
+	}
+
 	for _, option := range options {
 		if err := option(ctr); err != nil {
 			return nil, errors.Wrapf(err, "error running container create option")
 		}
 	}
+	return r.setupContainer(ctx, ctr, false)
+}
 
+func (r *Runtime) setupContainer(ctx context.Context, ctr *Container, restore bool) (c *Container, err error) {
 	// Allocate a lock for the container
 	lock, err := r.lockManager.AllocateLock()
 	if err != nil {
@@ -154,6 +196,19 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options ..
 		return nil, errors.Wrapf(ErrInvalidArg, "unsupported CGroup manager: %s - cannot validate cgroup parent", r.config.CgroupManager)
 	}
 
+	if restore {
+		// Remove information about bind mount
+		// for new container from imported checkpoint
+		g := generate.Generator{Config: ctr.config.Spec}
+		g.RemoveMount("/dev/shm")
+		ctr.config.ShmDir = ""
+		g.RemoveMount("/etc/resolv.conf")
+		g.RemoveMount("/etc/hostname")
+		g.RemoveMount("/etc/hosts")
+		g.RemoveMount("/run/.containerenv")
+		g.RemoveMount("/run/secrets")
+	}
+
 	// Set up storage for the container
 	if err := ctr.setupStorage(ctx); err != nil {
 		return nil, err