4 files changed, 77 insertions, 2 deletions
diff --git a/libpod/container_api.go b/libpod/container_api.go
index 2c7dc79c9..87ff764e3 100644
--- a/libpod/container_api.go
+++ b/libpod/container_api.go
@@ -706,6 +706,13 @@ type ContainerCheckpointOptions struct {
 	// IgnoreVolumes tells the API to not export or not to import
 	// the content of volumes associated with the container
 	IgnoreVolumes bool
+	// Pre Checkpoint container and leave container running
+	PreCheckPoint bool
+	// Dump container with Pre Checkpoint images
+	WithPrevious bool
+	// ImportPrevious tells the API to restore container with two
+	// images. One is TargetFile, the other is ImportPrevious.
+	ImportPrevious string
 }
 
 // Checkpoint checkpoints a container
@@ -718,6 +725,12 @@ func (c *Container) Checkpoint(ctx context.Context, options ContainerCheckpointO
 		}
 	}
 
+	if options.WithPrevious {
+		if err := c.canWithPrevious(); err != nil {
+			return err
+		}
+	}
+
 	if !c.batched {
 		c.lock.Lock()
 		defer c.lock.Unlock()
diff --git a/libpod/container_internal.go b/libpod/container_internal.go
index 540230c26..c7548e0e5 100644
--- a/libpod/container_internal.go
+++ b/libpod/container_internal.go
@@ -134,6 +134,11 @@ func (c *Container) CheckpointPath() string {
 	return filepath.Join(c.bundlePath(), "checkpoint")
 }
 
+// PreCheckpointPath returns the path to the directory containing the pre-checkpoint-images
+func (c *Container) PreCheckPointPath() string {
+	return filepath.Join(c.bundlePath(), "pre-checkpoint")
+}
+
 // AttachSocketPath retrieves the path of the container's attach socket
 func (c *Container) AttachSocketPath() (string, error) {
 	return c.ociRuntime.AttachSocketPath(c)
@@ -2023,6 +2028,12 @@ func (c *Container) checkReadyForRemoval() error {
 	return nil
 }
 
+// canWithPrevious return the stat of the preCheckPoint dir
+func (c *Container) canWithPrevious() error {
+	_, err := os.Stat(c.PreCheckPointPath())
+	return err
+}
+
 // writeJSONFile marshalls and writes the given data to a JSON file
 // in the bundle path
 func (c *Container) writeJSONFile(v interface{}, file string) error {
diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go
index ac20e1f25..705086bda 100644
--- a/libpod/container_internal_linux.go
+++ b/libpod/container_internal_linux.go
@@ -812,6 +812,9 @@ func (c *Container) exportCheckpoint(options ContainerCheckpointOptions) error {
 		"spec.dump",
 		"network.status"}
 
+	if options.PreCheckPoint {
+		includeFiles[0] = "pre-checkpoint"
+	}
 	// Get root file-system changes included in the checkpoint archive
 	rootfsDiffPath := filepath.Join(c.bundlePath(), "rootfs-diff.tar")
 	deleteFilesList := filepath.Join(c.bundlePath(), "deleted.files")
@@ -1015,6 +1018,15 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO
 
 	defer c.newContainerEvent(events.Checkpoint)
 
+	// There is a bug from criu: https://github.com/checkpoint-restore/criu/issues/116
+	// We have to change the symbolic link from absolute path to relative path
+	if options.WithPrevious {
+		os.Remove(path.Join(c.CheckpointPath(), "parent"))
+		if err := os.Symlink("../pre-checkpoint", path.Join(c.CheckpointPath(), "parent")); err != nil {
+			return err
+		}
+	}
+
 	if options.TargetFile != "" {
 		if err = c.exportCheckpoint(options); err != nil {
 			return err
@@ -1023,7 +1035,7 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO
 
 	logrus.Debugf("Checkpointed container %s", c.ID())
 
-	if !options.KeepRunning {
+	if !options.KeepRunning && !options.PreCheckPoint {
 		c.state.State = define.ContainerStateStopped
 
 		// Cleanup Storage and Network
@@ -1032,7 +1044,7 @@ func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointO
 		}
 	}
 
-	if !options.Keep {
+	if !options.Keep && !options.PreCheckPoint {
 		cleanup := []string{
 			"dump.log",
 			"stats-dump",
@@ -1080,6 +1092,21 @@ func (c *Container) importCheckpoint(input string) error {
 	return nil
 }
 
+func (c *Container) importPreCheckpoint(input string) error {
+	archiveFile, err := os.Open(input)
+	if err != nil {
+		return errors.Wrap(err, "failed to open pre-checkpoint archive for import")
+	}
+
+	defer archiveFile.Close()
+
+	err = archive.Untar(archiveFile, c.bundlePath(), nil)
+	if err != nil {
+		return errors.Wrapf(err, "Unpacking of pre-checkpoint archive %s failed", input)
+	}
+	return nil
+}
+
 func (c *Container) restore(ctx context.Context, options ContainerCheckpointOptions) (retErr error) {
 	if err := c.checkpointRestoreSupported(); err != nil {
 		return err
@@ -1089,6 +1116,12 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti
 		return errors.Wrapf(define.ErrCtrStateInvalid, "container %s is running or paused, cannot restore", c.ID())
 	}
 
+	if options.ImportPrevious != "" {
+		if err := c.importPreCheckpoint(options.ImportPrevious); err != nil {
+			return err
+		}
+	}
+
 	if options.TargetFile != "" {
 		if err := c.importCheckpoint(options.TargetFile); err != nil {
 			return err
@@ -1322,6 +1355,10 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti
 		if err != nil {
 			logrus.Debugf("Non-fatal: removal of checkpoint directory (%s) failed: %v", c.CheckpointPath(), err)
 		}
+		err = os.RemoveAll(c.PreCheckPointPath())
+		if err != nil {
+			logrus.Debugf("Non-fatal: removal of pre-checkpoint directory (%s) failed: %v", c.PreCheckPointPath(), err)
+		}
 		cleanup := [...]string{"restore.log", "dump.log", "stats-dump", "stats-restore", "network.status", "rootfs-diff.tar", "deleted.files"}
 		for _, del := range cleanup {
 			file := filepath.Join(c.bundlePath(), del)
diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go
index 6b5da439a..66c8141c2 100644
--- a/libpod/oci_conmon_linux.go
+++ b/libpod/oci_conmon_linux.go
@@ -769,10 +769,14 @@ func (r *ConmonOCIRuntime) CheckpointContainer(ctr *Container, options Container
 	}
 	// imagePath is used by CRIU to store the actual checkpoint files
 	imagePath := ctr.CheckpointPath()
+	if options.PreCheckPoint {
+		imagePath = ctr.PreCheckPointPath()
+	}
 	// workPath will be used to store dump.log and stats-dump
 	workPath := ctr.bundlePath()
 	logrus.Debugf("Writing checkpoint to %s", imagePath)
 	logrus.Debugf("Writing checkpoint logs to %s", workPath)
+	logrus.Debugf("Pre-dump the container %t", options.PreCheckPoint)
 	args := []string{}
 	args = append(args, r.runtimeFlags...)
 	args = append(args, "checkpoint")
@@ -786,6 +790,15 @@ func (r *ConmonOCIRuntime) CheckpointContainer(ctr *Container, options Container
 	if options.TCPEstablished {
 		args = append(args, "--tcp-established")
 	}
+	if !options.PreCheckPoint && options.KeepRunning {
+		args = append(args, "--leave-running")
+	}
+	if options.PreCheckPoint {
+		args = append(args, "--pre-dump")
+	}
+	if !options.PreCheckPoint && options.WithPrevious {
+		args = append(args, "--parent-path", ctr.PreCheckPointPath())
+	}
 	runtimeDir, err := util.GetRuntimeDir()
 	if err != nil {
 		return err
@@ -794,6 +807,7 @@ func (r *ConmonOCIRuntime) CheckpointContainer(ctr *Container, options Container
 		return errors.Wrapf(err, "cannot set XDG_RUNTIME_DIR")
 	}
 	args = append(args, ctr.ID())
+	logrus.Debugf("the args to checkpoint: %s %s", r.path, strings.Join(args, " "))
 	return utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, nil, r.path, args...)
 }