support multi-image (docker) archives

Support loading and saving tarballs with more than one image.
Add a new `/libpod/images/export` endpoint to the rest API to
allow for exporting/saving multiple images into an archive.

Note that a non-release version of containers/image is vendored.
A release version must be vendored before cutting a new Podman
release.  We force the containers/image version via a replace in
the go.mod file; this way go won't try to match the versions.

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

3 files changed, 273 insertions, 50 deletions

diff --git a/libpod/image/image.go b/libpod/image/image.go
index 2d055cc44..9dd04e7c7 100644
--- a/libpod/image/image.go
+++ b/libpod/image/image.go
@@ -17,6 +17,7 @@ import (
 	"github.com/containers/common/pkg/retry"
 	cp "github.com/containers/image/v5/copy"
 	"github.com/containers/image/v5/directory"
+	"github.com/containers/image/v5/docker/archive"
 	dockerarchive "github.com/containers/image/v5/docker/archive"
 	"github.com/containers/image/v5/docker/reference"
 	"github.com/containers/image/v5/image"
@@ -173,13 +174,182 @@ func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile
 	return newImage, nil
 }
 
+// SaveImages stores one more images in a multi-image archive.
+// Note that only `docker-archive` supports storing multiple
+// image.
+func (ir *Runtime) SaveImages(ctx context.Context, namesOrIDs []string, format string, outputFile string, quiet bool) (finalErr error) {
+	if format != DockerArchive {
+		return errors.Errorf("multi-image archives are only supported in in the %q format", DockerArchive)
+	}
+
+	sys := GetSystemContext("", "", false)
+
+	archWriter, err := archive.NewWriter(sys, outputFile)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		err := archWriter.Close()
+		if err == nil {
+			return
+		}
+		if finalErr == nil {
+			finalErr = err
+			return
+		}
+		finalErr = errors.Wrap(finalErr, err.Error())
+	}()
+
+	// Decide whether c/image's progress bars should use stderr or stdout.
+	// Use stderr in case we need to be quiet or if the output is set to
+	// stdout.  If the output is set of stdout, any log message there would
+	// corrupt the tarfile.
+	writer := os.Stdout
+	if quiet {
+		writer = os.Stderr
+	}
+
+	// extend an image with additional tags
+	type imageData struct {
+		*Image
+		tags []reference.NamedTagged
+	}
+
+	// Look up the images (and their tags) in the local storage.
+	imageMap := make(map[string]*imageData) // to group tags for an image
+	imageQueue := []string{}                // to preserve relative image order
+	for _, nameOrID := range namesOrIDs {
+		// Look up the name or ID in the local image storage.
+		localImage, err := ir.NewFromLocal(nameOrID)
+		if err != nil {
+			return err
+		}
+		id := localImage.ID()
+
+		iData, exists := imageMap[id]
+		if !exists {
+			imageQueue = append(imageQueue, id)
+			iData = &imageData{Image: localImage}
+			imageMap[id] = iData
+		}
+
+		// Unless we referred to an ID, add the input as a tag.
+		if !strings.HasPrefix(id, nameOrID) {
+			tag, err := NormalizedTag(nameOrID)
+			if err != nil {
+				return err
+			}
+			refTagged, isTagged := tag.(reference.NamedTagged)
+			if isTagged {
+				iData.tags = append(iData.tags, refTagged)
+			}
+		}
+	}
+
+	policyContext, err := getPolicyContext(sys)
+	if err != nil {
+		return err
+	}
+	defer func() {
+		if err := policyContext.Destroy(); err != nil {
+			logrus.Errorf("failed to destroy policy context: %q", err)
+		}
+	}()
+
+	// Now copy the images one-by-one.
+	for _, id := range imageQueue {
+		dest, err := archWriter.NewReference(nil)
+		if err != nil {
+			return err
+		}
+
+		img := imageMap[id]
+		copyOptions := getCopyOptions(sys, writer, nil, nil, SigningOptions{}, "", img.tags)
+		copyOptions.DestinationCtx.SystemRegistriesConfPath = registries.SystemRegistriesConfPath()
+
+		// For copying, we need a source reference that we can create
+		// from the image.
+		src, err := is.Transport.NewStoreReference(img.imageruntime.store, nil, id)
+		if err != nil {
+			return errors.Wrapf(err, "error getting source imageReference for %q", img.InputName)
+		}
+		_, err = cp.Image(ctx, policyContext, dest, src, copyOptions)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+// LoadAllImagesFromDockerArchive loads all images from the docker archive that
+// fileName points to.
+func (ir *Runtime) LoadAllImagesFromDockerArchive(ctx context.Context, fileName string, signaturePolicyPath string, writer io.Writer) ([]*Image, error) {
+	if signaturePolicyPath == "" {
+		signaturePolicyPath = ir.SignaturePolicyPath
+	}
+
+	sc := GetSystemContext(signaturePolicyPath, "", false)
+	reader, err := archive.NewReader(sc, fileName)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err := reader.Close(); err != nil {
+			logrus.Errorf(err.Error())
+		}
+	}()
+
+	refLists, err := reader.List()
+	if err != nil {
+		return nil, err
+	}
+
+	refPairs := []pullRefPair{}
+	for _, refList := range refLists {
+		for _, ref := range refList {
+			pairs, err := ir.getPullRefPairsFromDockerArchiveReference(ctx, reader, ref, sc)
+			if err != nil {
+				return nil, err
+			}
+			refPairs = append(refPairs, pairs...)
+		}
+	}
+
+	goal := pullGoal{
+		pullAllPairs:         true,
+		usedSearchRegistries: false,
+		refPairs:             refPairs,
+		searchedRegistries:   nil,
+	}
+
+	defer goal.cleanUp()
+	imageNames, err := ir.doPullImage(ctx, sc, goal, writer, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{}, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	newImages := make([]*Image, 0, len(imageNames))
+	for _, name := range imageNames {
+		newImage, err := ir.NewFromLocal(name)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error retrieving local image after pulling %s", name)
+		}
+		newImages = append(newImages, newImage)
+	}
+	ir.newImageEvent(events.LoadFromArchive, "")
+	return newImages, nil
+}
+
 // LoadFromArchiveReference creates a new image object for images pulled from a tar archive and the like (podman load)
 // This function is needed because it is possible for a tar archive to have multiple tags for one image
 func (ir *Runtime) LoadFromArchiveReference(ctx context.Context, srcRef types.ImageReference, signaturePolicyPath string, writer io.Writer) ([]*Image, error) {
 	if signaturePolicyPath == "" {
 		signaturePolicyPath = ir.SignaturePolicyPath
 	}
-	imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{MaxRetry: maxRetry})
+
+	imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}, &retry.RetryOptions{})
 	if err != nil {
 		return nil, errors.Wrapf(err, "unable to pull %s", transports.ImageName(srcRef))
 	}
diff --git a/libpod/image/pull.go b/libpod/image/pull.go
index bdcda4016..94d6af4c2 100644
--- a/libpod/image/pull.go
+++ b/libpod/image/pull.go
@@ -11,8 +11,8 @@ import (
 	cp "github.com/containers/image/v5/copy"
 	"github.com/containers/image/v5/directory"
 	"github.com/containers/image/v5/docker"
+	"github.com/containers/image/v5/docker/archive"
 	dockerarchive "github.com/containers/image/v5/docker/archive"
-	"github.com/containers/image/v5/docker/tarfile"
 	ociarchive "github.com/containers/image/v5/oci/archive"
 	oci "github.com/containers/image/v5/oci/layout"
 	is "github.com/containers/image/v5/storage"
@@ -61,12 +61,26 @@ type pullRefPair struct {
 	dstRef types.ImageReference
 }
 
+// cleanUpFunc is a function prototype for clean-up functions.
+type cleanUpFunc func() error
+
 // pullGoal represents the prepared image references and decided behavior to be executed by imagePull
 type pullGoal struct {
 	refPairs             []pullRefPair
-	pullAllPairs         bool     // Pull all refPairs instead of stopping on first success.
-	usedSearchRegistries bool     // refPairs construction has depended on registries.GetRegistries()
-	searchedRegistries   []string // The list of search registries used; set only if usedSearchRegistries
+	pullAllPairs         bool          // Pull all refPairs instead of stopping on first success.
+	usedSearchRegistries bool          // refPairs construction has depended on registries.GetRegistries()
+	searchedRegistries   []string      // The list of search registries used; set only if usedSearchRegistries
+	cleanUpFuncs         []cleanUpFunc // Mainly used to close long-lived objects (e.g., an archive.Reader)
+}
+
+// cleanUp invokes all cleanUpFuncs.  Certain resources may not be available
+// anymore.  Errors are logged.
+func (p *pullGoal) cleanUp() {
+	for _, f := range p.cleanUpFuncs {
+		if err := f(); err != nil {
+			logrus.Error(err.Error())
+		}
+	}
 }
 
 // singlePullRefPairGoal returns a no-frills pull goal for the specified reference pair.
@@ -114,7 +128,49 @@ func (ir *Runtime) getSinglePullRefPairGoal(srcRef types.ImageReference, destNam
 	return singlePullRefPairGoal(rp), nil
 }
 
+// getPullRefPairsFromDockerArchiveReference returns a slice of pullRefPairs
+// for the specified docker reference and the corresponding archive.Reader.
+func (ir *Runtime) getPullRefPairsFromDockerArchiveReference(ctx context.Context, reader *archive.Reader, ref types.ImageReference, sc *types.SystemContext) ([]pullRefPair, error) {
+	destNames, err := reader.ManifestTagsForReference(ref)
+	if err != nil {
+		return nil, err
+	}
+
+	if len(destNames) == 0 {
+		destName, err := getImageDigest(ctx, ref, sc)
+		if err != nil {
+			return nil, err
+		}
+		destNames = append(destNames, destName)
+	} else {
+		for i := range destNames {
+			ref, err := NormalizedTag(destNames[i])
+			if err != nil {
+				return nil, err
+			}
+			destNames[i] = ref.String()
+		}
+	}
+
+	refPairs := []pullRefPair{}
+	for _, destName := range destNames {
+		destRef, err := is.Transport.ParseStoreReference(ir.store, destName)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error parsing dest reference name %#v", destName)
+		}
+		pair := pullRefPair{
+			image:  destName,
+			srcRef: ref,
+			dstRef: destRef,
+		}
+		refPairs = append(refPairs, pair)
+	}
+
+	return refPairs, nil
+}
+
 // pullGoalFromImageReference returns a pull goal for a single ImageReference, depending on the used transport.
+// Note that callers are responsible for invoking (*pullGoal).cleanUp() to clean up possibly open resources.
 func (ir *Runtime) pullGoalFromImageReference(ctx context.Context, srcRef types.ImageReference, imgName string, sc *types.SystemContext) (*pullGoal, error) {
 	span, _ := opentracing.StartSpanFromContext(ctx, "pullGoalFromImageReference")
 	defer span.Finish()
@@ -122,57 +178,26 @@ func (ir *Runtime) pullGoalFromImageReference(ctx context.Context, srcRef types.
 	// supports pulling from docker-archive, oci, and registries
 	switch srcRef.Transport().Name() {
 	case DockerArchive:
-		archivePath := srcRef.StringWithinTransport()
-		tarSource, err := tarfile.NewSourceFromFile(archivePath)
+		reader, readerRef, err := archive.NewReaderForReference(sc, srcRef)
 		if err != nil {
 			return nil, err
 		}
-		defer tarSource.Close()
-		manifest, err := tarSource.LoadTarManifest()
 
+		pairs, err := ir.getPullRefPairsFromDockerArchiveReference(ctx, reader, readerRef, sc)
 		if err != nil {
-			return nil, errors.Wrapf(err, "error retrieving manifest.json")
-		}
-		// to pull the first image stored in the tar file
-		if len(manifest) == 0 {
-			// use the hex of the digest if no manifest is found
-			reference, err := getImageDigest(ctx, srcRef, sc)
-			if err != nil {
-				return nil, err
-			}
-			return ir.getSinglePullRefPairGoal(srcRef, reference)
-		}
-
-		if len(manifest[0].RepoTags) == 0 {
-			// If the input image has no repotags, we need to feed it a dest anyways
-			digest, err := getImageDigest(ctx, srcRef, sc)
-			if err != nil {
-				return nil, err
+			// No need to defer for a single error path.
+			if err := reader.Close(); err != nil {
+				logrus.Error(err.Error())
 			}
-			return ir.getSinglePullRefPairGoal(srcRef, digest)
+			return nil, err
 		}
 
-		// Need to load in all the repo tags from the manifest
-		res := []pullRefPair{}
-		for _, dst := range manifest[0].RepoTags {
-			//check if image exists and gives a warning of untagging
-			localImage, err := ir.NewFromLocal(dst)
-			imageID := strings.TrimSuffix(manifest[0].Config, ".json")
-			if err == nil && imageID != localImage.ID() {
-				logrus.Errorf("the image %s already exists, renaming the old one with ID %s to empty string", dst, localImage.ID())
-			}
-
-			pullInfo, err := ir.getPullRefPair(srcRef, dst)
-			if err != nil {
-				return nil, err
-			}
-			res = append(res, pullInfo)
-		}
 		return &pullGoal{
-			refPairs:             res,
 			pullAllPairs:         true,
 			usedSearchRegistries: false,
+			refPairs:             pairs,
 			searchedRegistries:   nil,
+			cleanUpFuncs:         []cleanUpFunc{reader.Close},
 		}, nil
 
 	case OCIArchive:
@@ -249,6 +274,7 @@ func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName s
 			return nil, errors.Wrapf(err, "error determining pull goal for image %q", inputName)
 		}
 	}
+	defer goal.cleanUp()
 	return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, retryOptions, label)
 }
 
@@ -267,6 +293,7 @@ func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.Imag
 	if err != nil {
 		return nil, errors.Wrapf(err, "error determining pull goal for image %q", transports.ImageName(srcRef))
 	}
+	defer goal.cleanUp()
 	return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, retryOptions, nil)
 }
 
diff --git a/libpod/image/pull_test.go b/libpod/image/pull_test.go
index 0046cdfef..6cb80e8b5 100644
--- a/libpod/image/pull_test.go
+++ b/libpod/image/pull_test.go
@@ -150,7 +150,7 @@ func TestPullGoalFromImageReference(t *testing.T) {
 		{ // RepoTags is empty
 			"docker-archive:testdata/docker-unnamed.tar.xz",
 			[]expected{{"@ec9293436c2e66da44edb9efb8d41f6b13baf62283ebe846468bc992d76d7951", "@ec9293436c2e66da44edb9efb8d41f6b13baf62283ebe846468bc992d76d7951"}},
-			false,
+			true,
 		},
 		{ // RepoTags is a [docker.io/library/]name:latest, normalized to the short format.
 			"docker-archive:testdata/docker-name-only.tar.xz",
@@ -170,11 +170,37 @@ func TestPullGoalFromImageReference(t *testing.T) {
 			},
 			true,
 		},
-		{ // FIXME: Two images in a single archive - only the "first" one (whichever it is) is returned
-			// (and docker-archive: then refuses to read anything when the manifest has more than 1 item)
+		{ // Reference image by name in multi-image archive
+			"docker-archive:testdata/docker-two-images.tar.xz:example.com/empty:latest",
+			[]expected{
+				{"example.com/empty:latest", "example.com/empty:latest"},
+			},
+			true,
+		},
+		{ // Reference image by name in multi-image archive
+			"docker-archive:testdata/docker-two-images.tar.xz:example.com/empty/but:different",
+			[]expected{
+				{"example.com/empty/but:different", "example.com/empty/but:different"},
+			},
+			true,
+		},
+		{ // Reference image by index in multi-image archive
+			"docker-archive:testdata/docker-two-images.tar.xz:@0",
+			[]expected{
+				{"example.com/empty:latest", "example.com/empty:latest"},
+			},
+			true,
+		},
+		{ // Reference image by index in multi-image archive
+			"docker-archive:testdata/docker-two-images.tar.xz:@1",
+			[]expected{
+				{"example.com/empty/but:different", "example.com/empty/but:different"},
+			},
+			true,
+		},
+		{ // Reference entire multi-image archive must fail (more than one manifest)
 			"docker-archive:testdata/docker-two-images.tar.xz",
-			[]expected{{"example.com/empty:latest", "example.com/empty:latest"}},
-			// "example.com/empty/but:different" exists but is ignored
+			[]expected{},
 			true,
 		},
 
@@ -248,7 +274,7 @@ func TestPullGoalFromImageReference(t *testing.T) {
 			for i, e := range c.expected {
 				testDescription := fmt.Sprintf("%s #%d", c.srcName, i)
 				assert.Equal(t, e.image, res.refPairs[i].image, testDescription)
-				assert.Equal(t, srcRef, res.refPairs[i].srcRef, testDescription)
+				assert.Equal(t, transports.ImageName(srcRef), transports.ImageName(res.refPairs[i].srcRef), testDescription)
 				assert.Equal(t, e.dstName, storageReferenceWithoutLocation(res.refPairs[i].dstRef), testDescription)
 			}
 			assert.Equal(t, c.expectedPullAllPairs, res.pullAllPairs, c.srcName)