Merge pull request #8819 from chen-zhuohan/add-pre-checkpoint

Add pre-checkpoint and restore with previous

11 files changed, 192 insertions, 2 deletions

diff --git a/cmd/podman/containers/checkpoint.go b/cmd/podman/containers/checkpoint.go
index 4a477eb10..14abfd5a7 100644
--- a/cmd/podman/containers/checkpoint.go
+++ b/cmd/podman/containers/checkpoint.go
@@ -58,6 +58,9 @@ func init() {
 
 	flags.BoolVar(&checkpointOptions.IgnoreRootFS, "ignore-rootfs", false, "Do not include root file-system changes when exporting")
 	flags.BoolVar(&checkpointOptions.IgnoreVolumes, "ignore-volumes", false, "Do not export volumes associated with container")
+	flags.BoolVarP(&checkpointOptions.PreCheckPoint, "pre-checkpoint", "P", false, "Dump container's memory information only, leave the container running")
+	flags.BoolVar(&checkpointOptions.WithPrevious, "with-previous", false, "Checkpoint container with pre-checkpoint images")
+
 	validate.AddLatestFlag(checkpointCommand, &checkpointOptions.Latest)
 }
 
@@ -72,6 +75,9 @@ func checkpoint(cmd *cobra.Command, args []string) error {
 	if checkpointOptions.Export == "" && checkpointOptions.IgnoreVolumes {
 		return errors.Errorf("--ignore-volumes can only be used with --export")
 	}
+	if checkpointOptions.WithPrevious && checkpointOptions.PreCheckPoint {
+		return errors.Errorf("--with-previous can not be used with --pre-checkpoint")
+	}
 	responses, err := registry.ContainerEngine().ContainerCheckpoint(context.Background(), args, checkpointOptions)
 	if err != nil {
 		return err
diff --git a/cmd/podman/containers/restore.go b/cmd/podman/containers/restore.go
index 5245a68fa..49c0be88e 100644
--- a/cmd/podman/containers/restore.go
+++ b/cmd/podman/containers/restore.go
@@ -59,6 +59,10 @@ func init() {
 	flags.StringVarP(&restoreOptions.Name, nameFlagName, "n", "", "Specify new name for container restored from exported checkpoint (only works with --import)")
 	_ = restoreCommand.RegisterFlagCompletionFunc(nameFlagName, completion.AutocompleteNone)
 
+	importPreviousFlagName := "import-previous"
+	flags.StringVar(&restoreOptions.ImportPrevious, importPreviousFlagName, "", "Restore from exported pre-checkpoint archive (tar.gz)")
+	_ = restoreCommand.RegisterFlagCompletionFunc(importPreviousFlagName, completion.AutocompleteDefault)
+
 	flags.BoolVar(&restoreOptions.IgnoreRootFS, "ignore-rootfs", false, "Do not apply root file-system changes when importing from exported checkpoint")
 	flags.BoolVar(&restoreOptions.IgnoreStaticIP, "ignore-static-ip", false, "Ignore IP address set via --static-ip")
 	flags.BoolVar(&restoreOptions.IgnoreStaticMAC, "ignore-static-mac", false, "Ignore MAC address set via --mac-address")
@@ -71,6 +75,9 @@ func restore(_ *cobra.Command, args []string) error {
 	if rootless.IsRootless() {
 		return errors.New("restoring a container requires root")
 	}
+	if restoreOptions.Import == "" && restoreOptions.ImportPrevious != "" {
+		return errors.Errorf("--import-previous can only be used with --import")
+	}
 	if restoreOptions.Import == "" && restoreOptions.IgnoreRootFS {
 		return errors.Errorf("--ignore-rootfs can only be used with --import")
 	}
diff --git a/docs/source/markdown/podman-container-checkpoint.1.md b/docs/source/markdown/podman-container-checkpoint.1.md
index 6a9469156..ea05979cd 100644
--- a/docs/source/markdown/podman-container-checkpoint.1.md
+++ b/docs/source/markdown/podman-container-checkpoint.1.md
@@ -58,12 +58,26 @@ This option must be used in combination with the **--export, -e** option.
 When this option is specified, the content of volumes associated with
 the container will not be included into the checkpoint tar.gz file.
 
+#### **--pre-checkpoint**, **-P**
+
+Dump the container's memory information only, leaving the container running. Later
+operations will supersede prior dumps. It only works on runc 1.0-rc3 or higher.
+
+#### **--with-previous**
+
+Check out the container with previous criu image files in pre-dump. It only works
+without **--pre-checkpoint** or **-P**. It only works on runc 1.0-rc3 or higher.
+
 ## EXAMPLE
 
 podman container checkpoint mywebserver
 
 podman container checkpoint 860a4b23
 
+podman container checkpoint -P -e pre-checkpoint.tar.gz -l
+
+podman container checkpoint --with-previous -e checkpoint.tar.gz -l
+
 ## SEE ALSO
 podman(1), podman-container-restore(1)
 
diff --git a/docs/source/markdown/podman-container-restore.1.md b/docs/source/markdown/podman-container-restore.1.md
index 0593e6fe9..192b8765b 100644
--- a/docs/source/markdown/podman-container-restore.1.md
+++ b/docs/source/markdown/podman-container-restore.1.md
@@ -48,6 +48,11 @@ Import a checkpoint tar.gz file, which was exported by Podman. This can be used
 to import a checkpointed container from another host. Do not specify a *container*
 argument when using this option.
 
+#### **--import-previous**
+
+Import a pre-checkpoint tar.gz file which was exported by Podman. This option
+must be used with **-i** or **--import**. It only works on runc 1.0-rc3 or higher.
+
 #### **--name**, **-n**
 
 This is only available in combination with **--import, -i**. If a container is restored
@@ -98,6 +103,8 @@ podman container restore mywebserver
 
 podman container restore 860a4b23
 
+podman container restore --import-previous pre-checkpoint.tar.gz --import checkpoint.tar.gz
+
 ## SEE ALSO
 podman(1), podman-container-checkpoint(1)
 
diff --git a/libpod/container_api.go b/libpod/container_api.go
index 2c7dc79c9..87ff764e3 100644
--- a/libpod/container_api.go
+++ b/libpod/container_api.go
@@ -706,6 +706,13 @@ type ContainerCheckpointOptions struct {
 	// IgnoreVolumes tells the API to not export or not to import
 	// the content of volumes associated with the container
 	IgnoreVolumes bool
+	// Pre Checkpoint container and leave container running
+	PreCheckPoint bool
+	// Dump container with Pre Checkpoint images
+	WithPrevious bool
+	// ImportPrevious tells the API to restore container with two
+	// images. One is TargetFile, the other is ImportPrevious.
+	ImportPrevious string
 }
 
 // Checkpoint checkpoints a container
@@ -718,6 +725,12 @@ func (c *Container) Checkpoint(ctx context.Context, options ContainerCheckpointO
 		}
 	}
 
+	if options.WithPrevious {
+		if err := c.canWithPrevious(); err != nil {
+			return err
+		}
+	}
+
 	if !c.batched {
 		c.lock.Lock()
 		defer c.lock.Unlock()
diff --git a/libpod/container_internal.go b/libpod/container_internal.go
index 540230c26..c7548e0e5 100644
--- a/libpod/container_internal.go
+++ b/libpod/container_internal.go
@@ -134,6 +134,11 @@ func (c *Container) CheckpointPath() string {
 	return filepath.Join(c.bundlePath(), "checkpoint")
 }
 
+// PreCheckpointPath returns the path to the directory containing the pre-checkpoint-images
+func (c *Container) PreCheckPointPath() string {
+	return filepath.Join(c.bundlePath(), "pre-checkpoint")
+}
+
 // AttachSocketPath retrieves the path of the container's attach socket
 func (c *Container) AttachSocketPath() (string, error) {
 	return c.ociRuntime.AttachSocketPath(c)
@@ -2023,6 +2028,12 @@ func (c *Container) checkReadyForRemoval() error {
 	return nil
 }
 
+// canWithPrevious return the stat of the preCheckPoint dir
+func (c *Container) canWithPrevious() error {
+	_, err := os.Stat(c.PreCheckPointPath())
+	return err
+}
+
 // writeJSONFile marshalls and writes the given data to a JSON file
 // in the bundle path
 func (c *Container) writeJSONFile(v interface{}, file string) error {
diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go
index ac20e1f25..705086bda 100644
--- a/libpod/container_internal_linux.go
+++ b/libpod/container_internal_linux.go
@@ -812,6 +812,9 @@ func (c *Container) exportCheckpoint(options ContainerCheckpointOptions) error {
 		"spec.dump",
 		"network.status"}
 
+	if options.PreCheckPoint {
+		includeFiles[0] = "pre-checkpoint"
+	}
 	// Get root file-system changes included in the checkpoint archive
 	rootfsDiffPath := filepath.Join(c.bundlePath(), "rootfs-diff.tar")
 	deleteFilesList := filepath.Join(c.bundlePath(), "deleted.files")
@@ -1015,6 +1018,15 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO
 
 	defer c.newContainerEvent(events.Checkpoint)
 
+	// There is a bug from criu: https://github.com/checkpoint-restore/criu/issues/116
+	// We have to change the symbolic link from absolute path to relative path
+	if options.WithPrevious {
+		os.Remove(path.Join(c.CheckpointPath(), "parent"))
+		if err := os.Symlink("../pre-checkpoint", path.Join(c.CheckpointPath(), "parent")); err != nil {
+			return err
+		}
+	}
+
 	if options.TargetFile != "" {
 		if err = c.exportCheckpoint(options); err != nil {
 			return err
@@ -1023,7 +1035,7 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO
 
 	logrus.Debugf("Checkpointed container %s", c.ID())
 
-	if !options.KeepRunning {
+	if !options.KeepRunning && !options.PreCheckPoint {
 		c.state.State = define.ContainerStateStopped
 
 		// Cleanup Storage and Network
@@ -1032,7 +1044,7 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO
 		}
 	}
 
-	if !options.Keep {
+	if !options.Keep && !options.PreCheckPoint {
 		cleanup := []string{
 			"dump.log",
 			"stats-dump",
@@ -1080,6 +1092,21 @@ func (c *Container) importCheckpoint(input string) error {
 	return nil
 }
 
+func (c *Container) importPreCheckpoint(input string) error {
+	archiveFile, err := os.Open(input)
+	if err != nil {
+		return errors.Wrap(err, "failed to open pre-checkpoint archive for import")
+	}
+
+	defer archiveFile.Close()
+
+	err = archive.Untar(archiveFile, c.bundlePath(), nil)
+	if err != nil {
+		return errors.Wrapf(err, "Unpacking of pre-checkpoint archive %s failed", input)
+	}
+	return nil
+}
+
 func (c *Container) restore(ctx context.Context, options ContainerCheckpointOptions) (retErr error) {
 	if err := c.checkpointRestoreSupported(); err != nil {
 		return err
@@ -1089,6 +1116,12 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti
 		return errors.Wrapf(define.ErrCtrStateInvalid, "container %s is running or paused, cannot restore", c.ID())
 	}
 
+	if options.ImportPrevious != "" {
+		if err := c.importPreCheckpoint(options.ImportPrevious); err != nil {
+			return err
+		}
+	}
+
 	if options.TargetFile != "" {
 		if err := c.importCheckpoint(options.TargetFile); err != nil {
 			return err
@@ -1322,6 +1355,10 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti
 		if err != nil {
 			logrus.Debugf("Non-fatal: removal of checkpoint directory (%s) failed: %v", c.CheckpointPath(), err)
 		}
+		err = os.RemoveAll(c.PreCheckPointPath())
+		if err != nil {
+			logrus.Debugf("Non-fatal: removal of pre-checkpoint directory (%s) failed: %v", c.PreCheckPointPath(), err)
+		}
 		cleanup := [...]string{"restore.log", "dump.log", "stats-dump", "stats-restore", "network.status", "rootfs-diff.tar", "deleted.files"}
 		for _, del := range cleanup {
 			file := filepath.Join(c.bundlePath(), del)
diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go
index b7598edad..70896cda4 100644
--- a/libpod/oci_conmon_linux.go
+++ b/libpod/oci_conmon_linux.go
@@ -768,10 +768,14 @@ func (r *ConmonOCIRuntime) CheckpointContainer(ctr *Container, options Container
 	}
 	// imagePath is used by CRIU to store the actual checkpoint files
 	imagePath := ctr.CheckpointPath()
+	if options.PreCheckPoint {
+		imagePath = ctr.PreCheckPointPath()
+	}
 	// workPath will be used to store dump.log and stats-dump
 	workPath := ctr.bundlePath()
 	logrus.Debugf("Writing checkpoint to %s", imagePath)
 	logrus.Debugf("Writing checkpoint logs to %s", workPath)
+	logrus.Debugf("Pre-dump the container %t", options.PreCheckPoint)
 	args := []string{}
 	args = append(args, r.runtimeFlags...)
 	args = append(args, "checkpoint")
@@ -785,6 +789,15 @@ func (r *ConmonOCIRuntime) CheckpointContainer(ctr *Container, options Container
 	if options.TCPEstablished {
 		args = append(args, "--tcp-established")
 	}
+	if !options.PreCheckPoint && options.KeepRunning {
+		args = append(args, "--leave-running")
+	}
+	if options.PreCheckPoint {
+		args = append(args, "--pre-dump")
+	}
+	if !options.PreCheckPoint && options.WithPrevious {
+		args = append(args, "--parent-path", ctr.PreCheckPointPath())
+	}
 	runtimeDir, err := util.GetRuntimeDir()
 	if err != nil {
 		return err
@@ -793,6 +806,7 @@ func (r *ConmonOCIRuntime) CheckpointContainer(ctr *Container, options Container
 		return errors.Wrapf(err, "cannot set XDG_RUNTIME_DIR")
 	}
 	args = append(args, ctr.ID())
+	logrus.Debugf("the args to checkpoint: %s %s", r.path, strings.Join(args, " "))
 	return utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, nil, r.path, args...)
 }
 
diff --git a/pkg/domain/entities/containers.go b/pkg/domain/entities/containers.go
index a67ecebd5..96687b1de 100644
--- a/pkg/domain/entities/containers.go
+++ b/pkg/domain/entities/containers.go
@@ -178,6 +178,8 @@ type CheckpointOptions struct {
 	Latest         bool
 	LeaveRunning   bool
 	TCPEstablished bool
+	PreCheckPoint  bool
+	WithPrevious   bool
 }
 
 type CheckpointReport struct {
@@ -196,6 +198,7 @@ type RestoreOptions struct {
 	Latest          bool
 	Name            string
 	TCPEstablished  bool
+	ImportPrevious  string
 }
 
 type RestoreReport struct {
diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go
index f7a538934..2c79b6187 100644
--- a/pkg/domain/infra/abi/containers.go
+++ b/pkg/domain/infra/abi/containers.go
@@ -489,6 +489,8 @@ func (ic *ContainerEngine) ContainerCheckpoint(ctx context.Context, namesOrIds [
 		IgnoreRootfs:   options.IgnoreRootFS,
 		IgnoreVolumes:  options.IgnoreVolumes,
 		KeepRunning:    options.LeaveRunning,
+		PreCheckPoint:  options.PreCheckPoint,
+		WithPrevious:   options.WithPrevious,
 	}
 
 	if options.All {
@@ -529,6 +531,7 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st
 		IgnoreVolumes:   options.IgnoreVolumes,
 		IgnoreStaticIP:  options.IgnoreStaticIP,
 		IgnoreStaticMAC: options.IgnoreStaticMAC,
+		ImportPrevious:  options.ImportPrevious,
 	}
 
 	filterFuncs := []libpod.ContainerFilter{
diff --git a/test/e2e/checkpoint_test.go b/test/e2e/checkpoint_test.go
index 4c6b2cf5c..abc37792a 100644
--- a/test/e2e/checkpoint_test.go
+++ b/test/e2e/checkpoint_test.go
@@ -4,6 +4,7 @@ import (
 	"net"
 	"os"
 	"os/exec"
+	"strings"
 
 	"github.com/containers/podman/v2/pkg/criu"
 	. "github.com/containers/podman/v2/test/utils"
@@ -747,4 +748,78 @@ var _ = Describe("Podman checkpoint", func() {
 		// Remove exported checkpoint
 		os.Remove(checkpointFileName)
 	})
+
+	It("podman checkpoint container with --pre-checkpoint", func() {
+		if !strings.Contains(podmanTest.OCIRuntime, "runc") {
+			Skip("Test only works on runc 1.0-rc3 or higher.")
+		}
+		localRunString := getRunString([]string{ALPINE, "top"})
+		session := podmanTest.Podman(localRunString)
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		cid := session.OutputToString()
+
+		result := podmanTest.Podman([]string{"container", "checkpoint", "-P", cid})
+		result.WaitWithDefaultTimeout()
+
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
+		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
+
+		result = podmanTest.Podman([]string{"container", "checkpoint", "--with-previous", cid})
+		result.WaitWithDefaultTimeout()
+
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
+		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
+
+		result = podmanTest.Podman([]string{"container", "restore", cid})
+		result.WaitWithDefaultTimeout()
+
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
+		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
+	})
+
+	It("podman checkpoint container with --pre-checkpoint and export (migration)", func() {
+		if !strings.Contains(podmanTest.OCIRuntime, "runc") {
+			Skip("Test only works on runc 1.0-rc3 or higher.")
+		}
+		localRunString := getRunString([]string{ALPINE, "top"})
+		session := podmanTest.Podman(localRunString)
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(0))
+		cid := session.OutputToString()
+		preCheckpointFileName := "/tmp/pre-checkpoint-" + cid + ".tar.gz"
+		checkpointFileName := "/tmp/checkpoint-" + cid + ".tar.gz"
+
+		result := podmanTest.Podman([]string{"container", "checkpoint", "-P", "-e", preCheckpointFileName, cid})
+		result.WaitWithDefaultTimeout()
+
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
+		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
+
+		result = podmanTest.Podman([]string{"container", "checkpoint", "--with-previous", "-e", checkpointFileName, cid})
+		result.WaitWithDefaultTimeout()
+
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
+		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Exited"))
+
+		result = podmanTest.Podman([]string{"rm", "-f", cid})
+		result.WaitWithDefaultTimeout()
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(0))
+
+		result = podmanTest.Podman([]string{"container", "restore", "-i", checkpointFileName, "--import-previous", preCheckpointFileName})
+		result.WaitWithDefaultTimeout()
+
+		Expect(result.ExitCode()).To(Equal(0))
+		Expect(podmanTest.NumberOfContainersRunning()).To(Equal(1))
+		Expect(podmanTest.GetContainerStatus()).To(ContainSubstring("Up"))
+
+		os.Remove(checkpointFileName)
+		os.Remove(preCheckpointFileName)
+	})
 })