Add support for checkpoint image

This is an enhancement proposal for the checkpoint / restore feature of
Podman that enables container migration across multiple systems with
standard image distribution infrastructure.

A new option `--create-image <image>` has been added to the
`podman container checkpoint` command. This option tells Podman to
create a container image.  This is a standard image with a single layer,
tar archive, that that contains all checkpoint files. This is similar to
the current approach with checkpoint `--export`/`--import`.

This image can be pushed to a container registry and pulled on a
different system.  It can also be exported locally with `podman image
save` and inspected with `podman inspect`. Inspecting the image would
display additional information about the host and the versions of
Podman, criu, crun/runc, kernel, etc.

`podman container restore` has also been extended to support image
name or ID as input.

Suggested-by: Adrian Reber <areber@redhat.com>
Signed-off-by: Radostin Stoyanov <radostin@redhat.com>

9 files changed, 132 insertions, 31 deletions

diff --git a/pkg/api/handlers/libpod/containers.go b/pkg/api/handlers/libpod/containers.go
index dfa09b8b8..03dd436f6 100644
--- a/pkg/api/handlers/libpod/containers.go
+++ b/pkg/api/handlers/libpod/containers.go
@@ -209,15 +209,16 @@ func Checkpoint(w http.ResponseWriter, r *http.Request) {
 
 	decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder)
 	query := struct {
-		Keep           bool `schema:"keep"`
-		LeaveRunning   bool `schema:"leaveRunning"`
-		TCPEstablished bool `schema:"tcpEstablished"`
-		Export         bool `schema:"export"`
-		IgnoreRootFS   bool `schema:"ignoreRootFS"`
-		PrintStats     bool `schema:"printStats"`
-		PreCheckpoint  bool `schema:"preCheckpoint"`
-		WithPrevious   bool `schema:"withPrevious"`
-		FileLocks      bool `schema:"fileLocks"`
+		Keep           bool   `schema:"keep"`
+		LeaveRunning   bool   `schema:"leaveRunning"`
+		TCPEstablished bool   `schema:"tcpEstablished"`
+		Export         bool   `schema:"export"`
+		IgnoreRootFS   bool   `schema:"ignoreRootFS"`
+		PrintStats     bool   `schema:"printStats"`
+		PreCheckpoint  bool   `schema:"preCheckpoint"`
+		WithPrevious   bool   `schema:"withPrevious"`
+		FileLocks      bool   `schema:"fileLocks"`
+		CreateImage    string `schema:"createImage"`
 	}{
 		// override any golang type defaults
 	}
@@ -243,6 +244,7 @@ func Checkpoint(w http.ResponseWriter, r *http.Request) {
 		PreCheckPoint:  query.PreCheckpoint,
 		WithPrevious:   query.WithPrevious,
 		FileLocks:      query.FileLocks,
+		CreateImage:    query.CreateImage,
 	}
 
 	if query.Export {
@@ -341,8 +343,17 @@ func Restore(w http.ResponseWriter, r *http.Request) {
 	} else {
 		name := utils.GetName(r)
 		if _, err := runtime.LookupContainer(name); err != nil {
-			utils.ContainerNotFound(w, name, err)
-			return
+			// If container was not found, check if this is a checkpoint image
+			ir := abi.ImageEngine{Libpod: runtime}
+			report, err := ir.Exists(r.Context(), name)
+			if err != nil {
+				utils.Error(w, http.StatusNotFound, errors.Wrapf(err, "failed to find container or checkpoint image %s", name))
+				return
+			}
+			if !report.Value {
+				utils.Error(w, http.StatusNotFound, errors.Errorf("failed to find container or checkpoint image %s", name))
+				return
+			}
 		}
 		names = []string{name}
 	}
diff --git a/pkg/bindings/containers/types.go b/pkg/bindings/containers/types.go
index c87f82bf4..81d491bb7 100644
--- a/pkg/bindings/containers/types.go
+++ b/pkg/bindings/containers/types.go
@@ -47,6 +47,7 @@ type AttachOptions struct {
 // CheckpointOptions are optional options for checkpointing containers
 type CheckpointOptions struct {
 	Export         *string
+	CreateImage    *string
 	IgnoreRootfs   *bool
 	Keep           *bool
 	LeaveRunning   *bool
diff --git a/pkg/bindings/containers/types_checkpoint_options.go b/pkg/bindings/containers/types_checkpoint_options.go
index e717daf9f..d5f6e541d 100644
--- a/pkg/bindings/containers/types_checkpoint_options.go
+++ b/pkg/bindings/containers/types_checkpoint_options.go
@@ -32,6 +32,21 @@ func (o *CheckpointOptions) GetExport() string {
 	return *o.Export
 }
 
+// WithCreateImage set field CreateImage to given value
+func (o *CheckpointOptions) WithCreateImage(value string) *CheckpointOptions {
+	o.CreateImage = &value
+	return o
+}
+
+// GetCreateImage returns value of field CreateImage
+func (o *CheckpointOptions) GetCreateImage() string {
+	if o.CreateImage == nil {
+		var z string
+		return z
+	}
+	return *o.CreateImage
+}
+
 // WithIgnoreRootfs set field IgnoreRootfs to given value
 func (o *CheckpointOptions) WithIgnoreRootfs(value bool) *CheckpointOptions {
 	o.IgnoreRootfs = &value
diff --git a/pkg/checkpoint/checkpoint_restore.go b/pkg/checkpoint/checkpoint_restore.go
index 270b5b6c4..396b521a1 100644
--- a/pkg/checkpoint/checkpoint_restore.go
+++ b/pkg/checkpoint/checkpoint_restore.go
@@ -22,9 +22,7 @@ import (
 
 // Prefixing the checkpoint/restore related functions with 'cr'
 
-// CRImportCheckpoint it the function which imports the information
-// from checkpoint tarball and re-creates the container from that information
-func CRImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, restoreOptions entities.RestoreOptions) ([]*libpod.Container, error) {
+func CRImportCheckpointTar(ctx context.Context, runtime *libpod.Runtime, restoreOptions entities.RestoreOptions) ([]*libpod.Container, error) {
 	// First get the container definition from the
 	// tarball to a temporary directory
 	dir, err := ioutil.TempDir("", "checkpoint")
@@ -39,7 +37,12 @@ func CRImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, restoreOpt
 	if err := crutils.CRImportCheckpointConfigOnly(dir, restoreOptions.Import); err != nil {
 		return nil, err
 	}
+	return CRImportCheckpoint(ctx, runtime, restoreOptions, dir)
+}
 
+// CRImportCheckpoint it the function which imports the information
+// from checkpoint tarball and re-creates the container from that information
+func CRImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, restoreOptions entities.RestoreOptions, dir string) ([]*libpod.Container, error) {
 	// Load spec.dump from temporary directory
 	dumpSpec := new(spec.Spec)
 	if _, err := metadata.ReadJSONFile(dumpSpec, dir, metadata.SpecDumpFile); err != nil {
@@ -48,7 +51,7 @@ func CRImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, restoreOpt
 
 	// Load config.dump from temporary directory
 	ctrConfig := new(libpod.ContainerConfig)
-	if _, err = metadata.ReadJSONFile(ctrConfig, dir, metadata.ConfigDumpFile); err != nil {
+	if _, err := metadata.ReadJSONFile(ctrConfig, dir, metadata.ConfigDumpFile); err != nil {
 		return nil, err
 	}
 
diff --git a/pkg/checkpoint/crutils/checkpoint_restore_utils.go b/pkg/checkpoint/crutils/checkpoint_restore_utils.go
index 6a8a7894a..76c868cee 100644
--- a/pkg/checkpoint/crutils/checkpoint_restore_utils.go
+++ b/pkg/checkpoint/crutils/checkpoint_restore_utils.go
@@ -54,7 +54,6 @@ func CRImportCheckpointConfigOnly(destination, input string) error {
 	options := &archive.TarOptions{
 		// Here we only need the files config.dump and spec.dump
 		ExcludePatterns: []string{
-			"volumes",
 			"ctr.log",
 			"artifacts",
 			stats.StatsDump,
@@ -62,6 +61,7 @@ func CRImportCheckpointConfigOnly(destination, input string) error {
 			metadata.DeletedFilesFile,
 			metadata.NetworkStatusFile,
 			metadata.CheckpointDirectory,
+			metadata.CheckpointVolumesDirectory,
 		},
 	}
 	if err = archive.Untar(archiveFile, destination, options); err != nil {
diff --git a/pkg/criu/criu.go b/pkg/criu/criu.go
index b54870abc..6570159d7 100644
--- a/pkg/criu/criu.go
+++ b/pkg/criu/criu.go
@@ -28,6 +28,11 @@ func CheckForCriu(version int) bool {
 	return result
 }
 
+func GetCriuVestion() (int, error) {
+	c := criu.MakeCriu()
+	return c.GetCriuVersion()
+}
+
 func MemTrack() bool {
 	features, err := criu.MakeCriu().FeatureCheck(
 		&rpc.CriuFeatures{
diff --git a/pkg/domain/entities/containers.go b/pkg/domain/entities/containers.go
index 072514d0f..3d1d7a6d2 100644
--- a/pkg/domain/entities/containers.go
+++ b/pkg/domain/entities/containers.go
@@ -178,6 +178,7 @@ type ContainerExportOptions struct {
 type CheckpointOptions struct {
 	All            bool
 	Export         string
+	CreateImage    string
 	IgnoreRootFS   bool
 	IgnoreVolumes  bool
 	Keep           bool
@@ -205,6 +206,7 @@ type RestoreOptions struct {
 	IgnoreStaticIP  bool
 	IgnoreStaticMAC bool
 	Import          string
+	CheckpointImage bool
 	Keep            bool
 	Latest          bool
 	Name            string
diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go
index 100842c69..46ef01b80 100644
--- a/pkg/domain/infra/abi/containers.go
+++ b/pkg/domain/infra/abi/containers.go
@@ -563,6 +563,7 @@ func (ic *ContainerEngine) ContainerCheckpoint(ctx context.Context, namesOrIds [
 		Compression:    options.Compression,
 		PrintStats:     options.PrintStats,
 		FileLocks:      options.FileLocks,
+		CreateImage:    options.CreateImage,
 	}
 
 	if options.All {
@@ -592,8 +593,9 @@ func (ic *ContainerEngine) ContainerCheckpoint(ctx context.Context, namesOrIds [
 
 func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []string, options entities.RestoreOptions) ([]*entities.RestoreReport, error) {
 	var (
-		cons []*libpod.Container
-		err  error
+		containers                  []*libpod.Container
+		checkpointImageImportErrors []error
+		err                         error
 	)
 
 	restoreOptions := libpod.ContainerCheckpointOptions{
@@ -619,17 +621,49 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st
 
 	switch {
 	case options.Import != "":
-		cons, err = checkpoint.CRImportCheckpoint(ctx, ic.Libpod, options)
+		containers, err = checkpoint.CRImportCheckpointTar(ctx, ic.Libpod, options)
 	case options.All:
-		cons, err = ic.Libpod.GetContainers(filterFuncs...)
+		containers, err = ic.Libpod.GetContainers(filterFuncs...)
+	case options.Latest:
+		containers, err = getContainersByContext(false, options.Latest, namesOrIds, ic.Libpod)
 	default:
-		cons, err = getContainersByContext(false, options.Latest, namesOrIds, ic.Libpod)
+		for _, nameOrID := range namesOrIds {
+			logrus.Debugf("lookup container: %q", nameOrID)
+			ctr, err := ic.Libpod.LookupContainer(nameOrID)
+			if err == nil {
+				containers = append(containers, ctr)
+			} else {
+				// If container was not found, check if this is a checkpoint image
+				logrus.Debugf("lookup image: %q", nameOrID)
+				img, _, err := ic.Libpod.LibimageRuntime().LookupImage(nameOrID, nil)
+				if err != nil {
+					return nil, fmt.Errorf("no such container or image: %s", nameOrID)
+				}
+				restoreOptions.CheckpointImageID = img.ID()
+				mountPoint, err := img.Mount(ctx, nil, "")
+				defer img.Unmount(true)
+				if err != nil {
+					return nil, err
+				}
+				importedContainers, err := checkpoint.CRImportCheckpoint(ctx, ic.Libpod, options, mountPoint)
+				if err != nil {
+					// CRImportCheckpoint is expected to import exactly one container from checkpoint image
+					checkpointImageImportErrors = append(
+						checkpointImageImportErrors,
+						errors.Errorf("unable to import checkpoint from image: %q: %v", nameOrID, err),
+					)
+				} else {
+					containers = append(containers, importedContainers[0])
+				}
+			}
+		}
 	}
 	if err != nil {
 		return nil, err
 	}
-	reports := make([]*entities.RestoreReport, 0, len(cons))
-	for _, con := range cons {
+
+	reports := make([]*entities.RestoreReport, 0, len(containers))
+	for _, con := range containers {
 		criuStatistics, runtimeRestoreDuration, err := con.Restore(ctx, restoreOptions)
 		reports = append(reports, &entities.RestoreReport{
 			Err:             err,
@@ -638,6 +672,13 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st
 			CRIUStatistics:  criuStatistics,
 		})
 	}
+
+	for _, importErr := range checkpointImageImportErrors {
+		reports = append(reports, &entities.RestoreReport{
+			Err: importErr,
+		})
+	}
+
 	return reports, nil
 }
 
diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go
index 10bfb3984..82e8fbb5b 100644
--- a/pkg/domain/infra/tunnel/containers.go
+++ b/pkg/domain/infra/tunnel/containers.go
@@ -16,6 +16,7 @@ import (
 	"github.com/containers/podman/v4/libpod/events"
 	"github.com/containers/podman/v4/pkg/api/handlers"
 	"github.com/containers/podman/v4/pkg/bindings/containers"
+	"github.com/containers/podman/v4/pkg/bindings/images"
 	"github.com/containers/podman/v4/pkg/domain/entities"
 	"github.com/containers/podman/v4/pkg/domain/entities/reports"
 	"github.com/containers/podman/v4/pkg/errorhandling"
@@ -331,6 +332,7 @@ func (ic *ContainerEngine) ContainerCheckpoint(ctx context.Context, namesOrIds [
 	options.WithIgnoreRootfs(opts.IgnoreRootFS)
 	options.WithKeep(opts.Keep)
 	options.WithExport(opts.Export)
+	options.WithCreateImage(opts.CreateImage)
 	options.WithTCPEstablished(opts.TCPEstablished)
 	options.WithPrintStats(opts.PrintStats)
 	options.WithPreCheckpoint(opts.PreCheckPoint)
@@ -396,8 +398,7 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st
 	}
 
 	var (
-		err  error
-		ctrs = []entities.ListContainer{}
+		ids = []string{}
 	)
 	if opts.All {
 		allCtrs, err := getContainersByContext(ic.ClientCtx, true, false, []string{})
@@ -407,20 +408,42 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st
 		// narrow the list to exited only
 		for _, c := range allCtrs {
 			if c.State == define.ContainerStateExited.String() {
-				ctrs = append(ctrs, c)
+				ids = append(ids, c.ID)
 			}
 		}
 	} else {
-		ctrs, err = getContainersByContext(ic.ClientCtx, false, false, namesOrIds)
+		getImageOptions := new(images.GetOptions).WithSize(false)
+		hostInfo, err := ic.Info(context.Background())
 		if err != nil {
 			return nil, err
 		}
+
+		for _, nameOrID := range namesOrIds {
+			ctrData, _, err := ic.ContainerInspect(ic.ClientCtx, []string{nameOrID}, entities.InspectOptions{})
+			if err == nil && len(ctrData) > 0 {
+				ids = append(ids, ctrData[0].ID)
+			} else {
+				// If container was not found, check if this is a checkpoint image
+				inspectReport, err := images.GetImage(ic.ClientCtx, nameOrID, getImageOptions)
+				if err != nil {
+					return nil, fmt.Errorf("no such container or image: %s", nameOrID)
+				}
+				checkpointRuntimeName, found := inspectReport.Annotations[define.CheckpointAnnotationRuntimeName]
+				if !found {
+					return nil, fmt.Errorf("image is not a checkpoint: %s", nameOrID)
+				}
+				if hostInfo.Host.OCIRuntime.Name != checkpointRuntimeName {
+					return nil, fmt.Errorf("container image \"%s\" requires runtime: \"%s\"", nameOrID, checkpointRuntimeName)
+				}
+				ids = append(ids, inspectReport.ID)
+			}
+		}
 	}
-	reports := make([]*entities.RestoreReport, 0, len(ctrs))
-	for _, c := range ctrs {
-		report, err := containers.Restore(ic.ClientCtx, c.ID, options)
+	reports := make([]*entities.RestoreReport, 0, len(ids))
+	for _, id := range ids {
+		report, err := containers.Restore(ic.ClientCtx, id, options)
 		if err != nil {
-			reports = append(reports, &entities.RestoreReport{Id: c.ID, Err: err})
+			reports = append(reports, &entities.RestoreReport{Id: id, Err: err})
 		}
 		reports = append(reports, report)
 	}