rewrite podman-cp

* Add a new `pkg/copy` to centralize all container-copy related code.

* The new code is based on Buildah's `copier` package.

* The compat `/archive` endpoints use the new `copy` package.

* Update docs and an several new tests.

* Includes many fixes, most notably, the look-up of volumes and mounts.

Breaking changes:

 * Podman is now expecting that container-destination paths exist.
   Before, Podman created the paths if needed.  Docker does not do
   that and I believe Podman should not either as it's a recipe for
   masking errors.  These errors may be user induced (e.g., a path
   typo), or internal typos (e.g., when the destination may be a
   mistakenly unmounted volume).  Let's keep the magic low for such
   a security sensitive feature.

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

4 files changed, 42 insertions, 413 deletions

diff --git a/pkg/domain/entities/containers.go b/pkg/domain/entities/containers.go
index 3fd7c79f4..39d679eaf 100644
--- a/pkg/domain/entities/containers.go
+++ b/pkg/domain/entities/containers.go
@@ -403,16 +403,14 @@ type ContainerPortReport struct {
 	Ports []ocicni.PortMapping
 }
 
-// ContainerCpOptions describes input options for cp
+// ContainerCpOptions describes input options for cp.
 type ContainerCpOptions struct {
-	Pause   bool
+	// Pause the container while copying.
+	Pause bool
+	// Extract the tarfile into the destination directory.
 	Extract bool
 }
 
-// ContainerCpReport describes the output from a cp operation
-type ContainerCpReport struct {
-}
-
 // ContainerStatsOptions describes input options for getting
 // stats on containers
 type ContainerStatsOptions struct {
diff --git a/pkg/domain/entities/engine_container.go b/pkg/domain/entities/engine_container.go
index df7da616a..e1f40e307 100644
--- a/pkg/domain/entities/engine_container.go
+++ b/pkg/domain/entities/engine_container.go
@@ -16,7 +16,7 @@ type ContainerEngine interface {
 	ContainerCheckpoint(ctx context.Context, namesOrIds []string, options CheckpointOptions) ([]*CheckpointReport, error)
 	ContainerCleanup(ctx context.Context, namesOrIds []string, options ContainerCleanupOptions) ([]*ContainerCleanupReport, error)
 	ContainerCommit(ctx context.Context, nameOrID string, options CommitOptions) (*CommitReport, error)
-	ContainerCp(ctx context.Context, source, dest string, options ContainerCpOptions) (*ContainerCpReport, error)
+	ContainerCp(ctx context.Context, source, dest string, options ContainerCpOptions) error
 	ContainerCreate(ctx context.Context, s *specgen.SpecGenerator) (*ContainerCreateReport, error)
 	ContainerDiff(ctx context.Context, nameOrID string, options DiffOptions) (*DiffReport, error)
 	ContainerExec(ctx context.Context, nameOrID string, options ExecOptions, streams define.AttachStreams) (int, error)
diff --git a/pkg/domain/infra/abi/cp.go b/pkg/domain/infra/abi/cp.go
index 8f4f5d3d7..9409df743 100644
--- a/pkg/domain/infra/abi/cp.go
+++ b/pkg/domain/infra/abi/cp.go
@@ -1,195 +1,70 @@
 package abi
 
 import (
-	"archive/tar"
 	"context"
-	"fmt"
-	"io"
-	"os"
-	"path/filepath"
 	"strings"
 
-	"github.com/containers/buildah/pkg/chrootuser"
-	"github.com/containers/buildah/util"
 	"github.com/containers/podman/v2/libpod"
-	"github.com/containers/podman/v2/libpod/define"
+	"github.com/containers/podman/v2/pkg/copy"
 	"github.com/containers/podman/v2/pkg/domain/entities"
-	"github.com/containers/storage"
-	"github.com/containers/storage/pkg/chrootarchive"
-	"github.com/containers/storage/pkg/idtools"
-	securejoin "github.com/cyphar/filepath-securejoin"
-	"github.com/docker/docker/pkg/archive"
-	"github.com/opencontainers/go-digest"
-	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
 )
 
-func (ic *ContainerEngine) ContainerCp(ctx context.Context, source, dest string, options entities.ContainerCpOptions) (*entities.ContainerCpReport, error) {
-	extract := options.Extract
-
+func (ic *ContainerEngine) ContainerCp(ctx context.Context, source, dest string, options entities.ContainerCpOptions) error {
 	srcCtr, srcPath := parsePath(ic.Libpod, source)
 	destCtr, destPath := parsePath(ic.Libpod, dest)
 
-	if (srcCtr == nil && destCtr == nil) || (srcCtr != nil && destCtr != nil) {
-		return nil, errors.Errorf("invalid arguments %s, %s you must use just one container", source, dest)
+	if srcCtr != nil && destCtr != nil {
+		return errors.Errorf("invalid arguments %q, %q: you must use just one container", source, dest)
 	}
-
-	if len(srcPath) == 0 || len(destPath) == 0 {
-		return nil, errors.Errorf("invalid arguments %s, %s you must specify paths", source, dest)
+	if srcCtr == nil && destCtr == nil {
+		return errors.Errorf("invalid arguments %q, %q: you must specify one container", source, dest)
 	}
-	ctr := srcCtr
-	isFromHostToCtr := ctr == nil
-	if isFromHostToCtr {
-		ctr = destCtr
+	if len(srcPath) == 0 || len(destPath) == 0 {
+		return errors.Errorf("invalid arguments %q, %q: you must specify paths", source, dest)
 	}
 
-	mountPoint, err := ctr.Mount()
-	if err != nil {
-		return nil, err
-	}
-	defer func() {
-		if err := ctr.Unmount(false); err != nil {
-			logrus.Errorf("unable to umount container '%s': %q", ctr.ID(), err)
+	var sourceItem, destinationItem copy.CopyItem
+	var err error
+	// Copy from the container to the host.
+	if srcCtr != nil {
+		sourceItem, err = copy.CopyItemForContainer(srcCtr, srcPath, options.Pause, true)
+		defer sourceItem.CleanUp()
+		if err != nil {
+			return err
 		}
-	}()
-
-	if options.Pause {
-		if err := ctr.Pause(); err != nil {
-			// An invalid state error is fine.
-			// The container isn't running or is already paused.
-			// TODO: We can potentially start the container while
-			// the copy is running, which still allows a race where
-			// malicious code could mess with the symlink.
-			if errors.Cause(err) != define.ErrCtrStateInvalid {
-				return nil, err
-			}
-		} else {
-			// Only add the defer if we actually paused
-			defer func() {
-				if err := ctr.Unpause(); err != nil {
-					logrus.Errorf("Error unpausing container after copying: %v", err)
-				}
-			}()
+	} else {
+		sourceItem, err = copy.CopyItemForHost(srcPath, true)
+		if err != nil {
+			return err
 		}
 	}
 
-	user, err := getUser(mountPoint, ctr.User())
-	if err != nil {
-		return nil, err
-	}
-	idMappingOpts, err := ctr.IDMappings()
-	if err != nil {
-		return nil, errors.Wrapf(err, "error getting IDMappingOptions")
-	}
-	destOwner := idtools.IDPair{UID: int(user.UID), GID: int(user.GID)}
-	hostUID, hostGID, err := util.GetHostIDs(convertIDMap(idMappingOpts.UIDMap), convertIDMap(idMappingOpts.GIDMap), user.UID, user.GID)
-	if err != nil {
-		return nil, err
-	}
-
-	hostOwner := idtools.IDPair{UID: int(hostUID), GID: int(hostGID)}
-
-	if isFromHostToCtr {
-		if isVol, volDestName, volName := isVolumeDestName(destPath, ctr); isVol { //nolint(gocritic)
-			path, err := pathWithVolumeMount(ic.Libpod, volDestName, volName, destPath)
-			if err != nil {
-				return nil, errors.Wrapf(err, "error getting destination path from volume %s", volDestName)
-			}
-			destPath = path
-		} else if isBindMount, mount := isBindMountDestName(destPath, ctr); isBindMount { //nolint(gocritic)
-			path, err := pathWithBindMountSource(mount, destPath)
-			if err != nil {
-				return nil, errors.Wrapf(err, "error getting destination path from bind mount %s", mount.Destination)
-			}
-			destPath = path
-		} else if filepath.IsAbs(destPath) { //nolint(gocritic)
-			cleanedPath, err := securejoin.SecureJoin(mountPoint, destPath)
-			if err != nil {
-				return nil, err
-			}
-			destPath = cleanedPath
-		} else { //nolint(gocritic)
-			ctrWorkDir, err := securejoin.SecureJoin(mountPoint, ctr.WorkingDir())
-			if err != nil {
-				return nil, err
-			}
-			if err = idtools.MkdirAllAndChownNew(ctrWorkDir, 0755, hostOwner); err != nil {
-				return nil, err
-			}
-			cleanedPath, err := securejoin.SecureJoin(mountPoint, filepath.Join(ctr.WorkingDir(), destPath))
-			if err != nil {
-				return nil, err
-			}
-			destPath = cleanedPath
+	if destCtr != nil {
+		destinationItem, err = copy.CopyItemForContainer(destCtr, destPath, options.Pause, false)
+		defer destinationItem.CleanUp()
+		if err != nil {
+			return err
 		}
 	} else {
-		destOwner = idtools.IDPair{UID: os.Getuid(), GID: os.Getgid()}
-		if isVol, volDestName, volName := isVolumeDestName(srcPath, ctr); isVol { //nolint(gocritic)
-			path, err := pathWithVolumeMount(ic.Libpod, volDestName, volName, srcPath)
-			if err != nil {
-				return nil, errors.Wrapf(err, "error getting source path from volume %s", volDestName)
-			}
-			srcPath = path
-		} else if isBindMount, mount := isBindMountDestName(srcPath, ctr); isBindMount { //nolint(gocritic)
-			path, err := pathWithBindMountSource(mount, srcPath)
-			if err != nil {
-				return nil, errors.Wrapf(err, "error getting source path from bind mount %s", mount.Destination)
-			}
-			srcPath = path
-		} else if filepath.IsAbs(srcPath) { //nolint(gocritic)
-			cleanedPath, err := securejoin.SecureJoin(mountPoint, srcPath)
-			if err != nil {
-				return nil, err
-			}
-			srcPath = cleanedPath
-		} else { //nolint(gocritic)
-			cleanedPath, err := securejoin.SecureJoin(mountPoint, filepath.Join(ctr.WorkingDir(), srcPath))
-			if err != nil {
-				return nil, err
-			}
-			srcPath = cleanedPath
-		}
-	}
-
-	if !filepath.IsAbs(destPath) {
-		dir, err := os.Getwd()
+		destinationItem, err = copy.CopyItemForHost(destPath, false)
+		defer destinationItem.CleanUp()
 		if err != nil {
-			return nil, errors.Wrapf(err, "err getting current working directory")
+			return err
 		}
-		destPath = filepath.Join(dir, destPath)
 	}
 
-	if source == "-" {
-		srcPath = os.Stdin.Name()
-		extract = true
-	}
-	err = containerCopy(srcPath, destPath, source, dest, idMappingOpts, &destOwner, extract, isFromHostToCtr)
-	return &entities.ContainerCpReport{}, err
+	// Copy from the host to the container.
+	return copy.Copy(&sourceItem, &destinationItem, options.Extract)
 }
 
-func getUser(mountPoint string, userspec string) (specs.User, error) {
-	uid, gid, _, err := chrootuser.GetUser(mountPoint, userspec)
-	u := specs.User{
-		UID:      uid,
-		GID:      gid,
-		Username: userspec,
+func parsePath(runtime *libpod.Runtime, path string) (*libpod.Container, string) {
+	if len(path) == 0 {
+		return nil, ""
 	}
-	if !strings.Contains(userspec, ":") {
-		groups, err2 := chrootuser.GetAdditionalGroupsForUser(mountPoint, uint64(u.UID))
-		if err2 != nil {
-			if errors.Cause(err2) != chrootuser.ErrNoSuchUser && err == nil {
-				err = err2
-			}
-		} else {
-			u.AdditionalGids = groups
-		}
-
+	if path[0] == '.' || path[0] == '/' { // A path cannot point to a container.
+		return nil, path
 	}
-	return u, err
-}
-
-func parsePath(runtime *libpod.Runtime, path string) (*libpod.Container, string) {
 	pathArr := strings.SplitN(path, ":", 2)
 	if len(pathArr) == 2 {
 		ctr, err := runtime.LookupContainer(pathArr[0])
@@ -199,247 +74,3 @@ func parsePath(runtime *libpod.Runtime, path string) (*libpod.Container, string)
 	}
 	return nil, path
 }
-
-func evalSymlinks(path string) (string, error) {
-	if path == os.Stdin.Name() {
-		return path, nil
-	}
-	return filepath.EvalSymlinks(path)
-}
-
-func getPathInfo(path string) (string, os.FileInfo, error) {
-	path, err := evalSymlinks(path)
-	if err != nil {
-		return "", nil, errors.Wrapf(err, "error evaluating symlinks %q", path)
-	}
-	srcfi, err := os.Stat(path)
-	if err != nil {
-		return "", nil, err
-	}
-	return path, srcfi, nil
-}
-
-func containerCopy(srcPath, destPath, src, dest string, idMappingOpts storage.IDMappingOptions, chownOpts *idtools.IDPair, extract, isFromHostToCtr bool) error {
-	srcPath, err := evalSymlinks(srcPath)
-	if err != nil {
-		return errors.Wrapf(err, "error evaluating symlinks %q", srcPath)
-	}
-
-	srcPath, srcfi, err := getPathInfo(srcPath)
-	if err != nil {
-		return err
-	}
-
-	filename := filepath.Base(destPath)
-	if filename == "-" && !isFromHostToCtr {
-		err := streamFileToStdout(srcPath, srcfi)
-		if err != nil {
-			return errors.Wrapf(err, "error streaming source file %s to Stdout", srcPath)
-		}
-		return nil
-	}
-
-	destdir := destPath
-	if !srcfi.IsDir() {
-		destdir = filepath.Dir(destPath)
-	}
-	_, err = os.Stat(destdir)
-	if err != nil && !os.IsNotExist(err) {
-		return err
-	}
-	destDirIsExist := err == nil
-	if err = os.MkdirAll(destdir, 0755); err != nil {
-		return err
-	}
-
-	// return functions for copying items
-	copyFileWithTar := chrootarchive.CopyFileWithTarAndChown(chownOpts, digest.Canonical.Digester().Hash(), idMappingOpts.UIDMap, idMappingOpts.GIDMap)
-	copyWithTar := chrootarchive.CopyWithTarAndChown(chownOpts, digest.Canonical.Digester().Hash(), idMappingOpts.UIDMap, idMappingOpts.GIDMap)
-	untarPath := chrootarchive.UntarPathAndChown(chownOpts, digest.Canonical.Digester().Hash(), idMappingOpts.UIDMap, idMappingOpts.GIDMap)
-
-	if srcfi.IsDir() {
-		logrus.Debugf("copying %q to %q", srcPath+string(os.PathSeparator)+"*", dest+string(os.PathSeparator)+"*")
-		if destDirIsExist && !strings.HasSuffix(src, fmt.Sprintf("%s.", string(os.PathSeparator))) {
-			srcPathBase := filepath.Base(srcPath)
-			if !isFromHostToCtr {
-				pathArr := strings.SplitN(src, ":", 2)
-				if len(pathArr) != 2 {
-					return errors.Errorf("invalid arguments %s, you must specify source path", src)
-				}
-				if pathArr[1] == "/" {
-					// If `srcPath` is the root directory of the container,
-					// `srcPath` will be `.../${sha256_ID}/merged/`, so do not join it
-					srcPathBase = ""
-				}
-			}
-			destPath = filepath.Join(destPath, srcPathBase)
-		}
-		if err = copyWithTar(srcPath, destPath); err != nil {
-			return errors.Wrapf(err, "error copying %q to %q", srcPath, dest)
-		}
-		return nil
-	}
-
-	if extract {
-		// We're extracting an archive into the destination directory.
-		logrus.Debugf("extracting contents of %q into %q", srcPath, destPath)
-		if err = untarPath(srcPath, destPath); err != nil {
-			return errors.Wrapf(err, "error extracting %q into %q", srcPath, destPath)
-		}
-		return nil
-	}
-
-	destfi, err := os.Stat(destPath)
-	if err != nil {
-		if !os.IsNotExist(err) || strings.HasSuffix(dest, string(os.PathSeparator)) {
-			return err
-		}
-	}
-	if destfi != nil && destfi.IsDir() {
-		destPath = filepath.Join(destPath, filepath.Base(srcPath))
-	}
-
-	// Copy the file, preserving attributes.
-	logrus.Debugf("copying %q to %q", srcPath, destPath)
-	if err = copyFileWithTar(srcPath, destPath); err != nil {
-		return errors.Wrapf(err, "error copying %q to %q", srcPath, destPath)
-	}
-	return nil
-}
-
-func convertIDMap(idMaps []idtools.IDMap) (convertedIDMap []specs.LinuxIDMapping) {
-	for _, idmap := range idMaps {
-		tempIDMap := specs.LinuxIDMapping{
-			ContainerID: uint32(idmap.ContainerID),
-			HostID:      uint32(idmap.HostID),
-			Size:        uint32(idmap.Size),
-		}
-		convertedIDMap = append(convertedIDMap, tempIDMap)
-	}
-	return convertedIDMap
-}
-
-func streamFileToStdout(srcPath string, srcfi os.FileInfo) error {
-	if srcfi.IsDir() {
-		tw := tar.NewWriter(os.Stdout)
-		err := filepath.Walk(srcPath, func(path string, info os.FileInfo, err error) error {
-			if err != nil || !info.Mode().IsRegular() || path == srcPath {
-				return err
-			}
-			hdr, err := tar.FileInfoHeader(info, "")
-			if err != nil {
-				return err
-			}
-
-			if err = tw.WriteHeader(hdr); err != nil {
-				return err
-			}
-			fh, err := os.Open(path)
-			if err != nil {
-				return err
-			}
-			defer fh.Close()
-
-			_, err = io.Copy(tw, fh)
-			return err
-		})
-		if err != nil {
-			return errors.Wrapf(err, "error streaming directory %s to Stdout", srcPath)
-		}
-		return nil
-	}
-
-	file, err := os.Open(srcPath)
-	if err != nil {
-		return err
-	}
-	defer file.Close()
-	if !archive.IsArchivePath(srcPath) {
-		tw := tar.NewWriter(os.Stdout)
-		hdr, err := tar.FileInfoHeader(srcfi, "")
-		if err != nil {
-			return err
-		}
-		err = tw.WriteHeader(hdr)
-		if err != nil {
-			return err
-		}
-		_, err = io.Copy(tw, file)
-		if err != nil {
-			return errors.Wrapf(err, "error streaming archive %s to Stdout", srcPath)
-		}
-		return nil
-	}
-
-	_, err = io.Copy(os.Stdout, file)
-	if err != nil {
-		return errors.Wrapf(err, "error streaming file to Stdout")
-	}
-	return nil
-}
-
-func isVolumeDestName(path string, ctr *libpod.Container) (bool, string, string) {
-	separator := string(os.PathSeparator)
-	if filepath.IsAbs(path) {
-		path = strings.TrimPrefix(path, separator)
-	}
-	if path == "" {
-		return false, "", ""
-	}
-	for _, vol := range ctr.Config().NamedVolumes {
-		volNamePath := strings.TrimPrefix(vol.Dest, separator)
-		if matchVolumePath(path, volNamePath) {
-			return true, vol.Dest, vol.Name
-		}
-	}
-	return false, "", ""
-}
-
-// if SRCPATH or DESTPATH is from volume mount's destination -v or --mount type=volume, generates the path with volume mount point
-func pathWithVolumeMount(runtime *libpod.Runtime, volDestName, volName, path string) (string, error) {
-	destVolume, err := runtime.GetVolume(volName)
-	if err != nil {
-		return "", errors.Wrapf(err, "error getting volume destination %s", volName)
-	}
-	if !filepath.IsAbs(path) {
-		path = filepath.Join(string(os.PathSeparator), path)
-	}
-	path, err = securejoin.SecureJoin(destVolume.MountPoint(), strings.TrimPrefix(path, volDestName))
-	return path, err
-}
-
-func isBindMountDestName(path string, ctr *libpod.Container) (bool, specs.Mount) {
-	separator := string(os.PathSeparator)
-	if filepath.IsAbs(path) {
-		path = strings.TrimPrefix(path, string(os.PathSeparator))
-	}
-	if path == "" {
-		return false, specs.Mount{}
-	}
-	for _, m := range ctr.Config().Spec.Mounts {
-		if m.Type != "bind" {
-			continue
-		}
-		mDest := strings.TrimPrefix(m.Destination, separator)
-		if matchVolumePath(path, mDest) {
-			return true, m
-		}
-	}
-	return false, specs.Mount{}
-}
-
-func matchVolumePath(path, target string) bool {
-	pathStr := filepath.Clean(path)
-	target = filepath.Clean(target)
-	for len(pathStr) > len(target) && strings.Contains(pathStr, string(os.PathSeparator)) {
-		pathStr = pathStr[:strings.LastIndex(pathStr, string(os.PathSeparator))]
-	}
-	return pathStr == target
-}
-
-func pathWithBindMountSource(m specs.Mount, path string) (string, error) {
-	if !filepath.IsAbs(path) {
-		path = filepath.Join(string(os.PathSeparator), path)
-	}
-	return securejoin.SecureJoin(m.Source, strings.TrimPrefix(path, m.Destination))
-}
diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go
index 63677719b..3584668c7 100644
--- a/pkg/domain/infra/tunnel/containers.go
+++ b/pkg/domain/infra/tunnel/containers.go
@@ -731,8 +731,8 @@ func (ic *ContainerEngine) ContainerPort(ctx context.Context, nameOrID string, o
 	return reports, nil
 }
 
-func (ic *ContainerEngine) ContainerCp(ctx context.Context, source, dest string, options entities.ContainerCpOptions) (*entities.ContainerCpReport, error) {
-	return nil, errors.New("not implemented")
+func (ic *ContainerEngine) ContainerCp(ctx context.Context, source, dest string, options entities.ContainerCpOptions) error {
+	return errors.New("not implemented")
 }
 
 // Shutdown Libpod engine