Vendor in latest containers/storage

This should improve performance on vfs images on top of xfs/reflink drives.

Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>

7 files changed, 397 insertions, 6 deletions

diff --git a/vendor/github.com/containers/storage/drivers/copy/copy.go b/vendor/github.com/containers/storage/drivers/copy/copy.go
new file mode 100644
index 000000000..2617824c5
--- /dev/null
+++ b/vendor/github.com/containers/storage/drivers/copy/copy.go
@@ -0,0 +1,277 @@
+// +build linux
+
+package copy
+
+/*
+#include <linux/fs.h>
+
+#ifndef FICLONE
+#define FICLONE		_IOW(0x94, 9, int)
+#endif
+*/
+import "C"
+import (
+	"container/list"
+	"fmt"
+	"io"
+	"os"
+	"path/filepath"
+	"syscall"
+	"time"
+
+	"github.com/containers/storage/pkg/pools"
+	"github.com/containers/storage/pkg/system"
+	rsystem "github.com/opencontainers/runc/libcontainer/system"
+	"golang.org/x/sys/unix"
+)
+
+// Mode indicates whether to use hardlink or copy content
+type Mode int
+
+const (
+	// Content creates a new file, and copies the content of the file
+	Content Mode = iota
+	// Hardlink creates a new hardlink to the existing file
+	Hardlink
+)
+
+func copyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error {
+	srcFile, err := os.Open(srcPath)
+	if err != nil {
+		return err
+	}
+	defer srcFile.Close()
+
+	// If the destination file already exists, we shouldn't blow it away
+	dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, fileinfo.Mode())
+	if err != nil {
+		return err
+	}
+	defer dstFile.Close()
+
+	if *copyWithFileClone {
+		_, _, err = unix.Syscall(unix.SYS_IOCTL, dstFile.Fd(), C.FICLONE, srcFile.Fd())
+		if err == nil {
+			return nil
+		}
+
+		*copyWithFileClone = false
+		if err == unix.EXDEV {
+			*copyWithFileRange = false
+		}
+	}
+	if *copyWithFileRange {
+		err = doCopyWithFileRange(srcFile, dstFile, fileinfo)
+		// Trying the file_clone may not have caught the exdev case
+		// as the ioctl may not have been available (therefore EINVAL)
+		if err == unix.EXDEV || err == unix.ENOSYS {
+			*copyWithFileRange = false
+		} else {
+			return err
+		}
+	}
+	return legacyCopy(srcFile, dstFile)
+}
+
+func doCopyWithFileRange(srcFile, dstFile *os.File, fileinfo os.FileInfo) error {
+	amountLeftToCopy := fileinfo.Size()
+
+	for amountLeftToCopy > 0 {
+		n, err := unix.CopyFileRange(int(srcFile.Fd()), nil, int(dstFile.Fd()), nil, int(amountLeftToCopy), 0)
+		if err != nil {
+			return err
+		}
+
+		amountLeftToCopy = amountLeftToCopy - int64(n)
+	}
+
+	return nil
+}
+
+func legacyCopy(srcFile io.Reader, dstFile io.Writer) error {
+	_, err := pools.Copy(dstFile, srcFile)
+
+	return err
+}
+
+func copyXattr(srcPath, dstPath, attr string) error {
+	data, err := system.Lgetxattr(srcPath, attr)
+	if err != nil {
+		return err
+	}
+	if data != nil {
+		if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+type fileID struct {
+	dev uint64
+	ino uint64
+}
+
+type dirMtimeInfo struct {
+	dstPath *string
+	stat    *syscall.Stat_t
+}
+
+// DirCopy copies or hardlinks the contents of one directory to another,
+// properly handling xattrs, and soft links
+//
+// Copying xattrs can be opted out of by passing false for copyXattrs.
+func DirCopy(srcDir, dstDir string, copyMode Mode, copyXattrs bool) error {
+	copyWithFileRange := true
+	copyWithFileClone := true
+
+	// This is a map of source file inodes to dst file paths
+	copiedFiles := make(map[fileID]string)
+
+	dirsToSetMtimes := list.New()
+	err := filepath.Walk(srcDir, func(srcPath string, f os.FileInfo, err error) error {
+		if err != nil {
+			return err
+		}
+
+		// Rebase path
+		relPath, err := filepath.Rel(srcDir, srcPath)
+		if err != nil {
+			return err
+		}
+
+		dstPath := filepath.Join(dstDir, relPath)
+		if err != nil {
+			return err
+		}
+
+		stat, ok := f.Sys().(*syscall.Stat_t)
+		if !ok {
+			return fmt.Errorf("Unable to get raw syscall.Stat_t data for %s", srcPath)
+		}
+
+		isHardlink := false
+
+		switch f.Mode() & os.ModeType {
+		case 0: // Regular file
+			id := fileID{dev: stat.Dev, ino: stat.Ino}
+			if copyMode == Hardlink {
+				isHardlink = true
+				if err2 := os.Link(srcPath, dstPath); err2 != nil {
+					return err2
+				}
+			} else if hardLinkDstPath, ok := copiedFiles[id]; ok {
+				if err2 := os.Link(hardLinkDstPath, dstPath); err2 != nil {
+					return err2
+				}
+			} else {
+				if err2 := copyRegular(srcPath, dstPath, f, &copyWithFileRange, &copyWithFileClone); err2 != nil {
+					return err2
+				}
+				copiedFiles[id] = dstPath
+			}
+
+		case os.ModeDir:
+			if err := os.Mkdir(dstPath, f.Mode()); err != nil && !os.IsExist(err) {
+				return err
+			}
+
+		case os.ModeSymlink:
+			link, err := os.Readlink(srcPath)
+			if err != nil {
+				return err
+			}
+
+			if err := os.Symlink(link, dstPath); err != nil {
+				return err
+			}
+
+		case os.ModeNamedPipe:
+			fallthrough
+		case os.ModeSocket:
+			if err := unix.Mkfifo(dstPath, stat.Mode); err != nil {
+				return err
+			}
+
+		case os.ModeDevice:
+			if rsystem.RunningInUserNS() {
+				// cannot create a device if running in user namespace
+				return nil
+			}
+			if err := unix.Mknod(dstPath, stat.Mode, int(stat.Rdev)); err != nil {
+				return err
+			}
+
+		default:
+			return fmt.Errorf("unknown file type for %s", srcPath)
+		}
+
+		// Everything below is copying metadata from src to dst. All this metadata
+		// already shares an inode for hardlinks.
+		if isHardlink {
+			return nil
+		}
+
+		if err := os.Lchown(dstPath, int(stat.Uid), int(stat.Gid)); err != nil {
+			return err
+		}
+
+		if copyXattrs {
+			if err := doCopyXattrs(srcPath, dstPath); err != nil {
+				return err
+			}
+		}
+
+		isSymlink := f.Mode()&os.ModeSymlink != 0
+
+		// There is no LChmod, so ignore mode for symlink. Also, this
+		// must happen after chown, as that can modify the file mode
+		if !isSymlink {
+			if err := os.Chmod(dstPath, f.Mode()); err != nil {
+				return err
+			}
+		}
+
+		// system.Chtimes doesn't support a NOFOLLOW flag atm
+		// nolint: unconvert
+		if f.IsDir() {
+			dirsToSetMtimes.PushFront(&dirMtimeInfo{dstPath: &dstPath, stat: stat})
+		} else if !isSymlink {
+			aTime := time.Unix(int64(stat.Atim.Sec), int64(stat.Atim.Nsec))
+			mTime := time.Unix(int64(stat.Mtim.Sec), int64(stat.Mtim.Nsec))
+			if err := system.Chtimes(dstPath, aTime, mTime); err != nil {
+				return err
+			}
+		} else {
+			ts := []syscall.Timespec{stat.Atim, stat.Mtim}
+			if err := system.LUtimesNano(dstPath, ts); err != nil {
+				return err
+			}
+		}
+		return nil
+	})
+	if err != nil {
+		return err
+	}
+	for e := dirsToSetMtimes.Front(); e != nil; e = e.Next() {
+		mtimeInfo := e.Value.(*dirMtimeInfo)
+		ts := []syscall.Timespec{mtimeInfo.stat.Atim, mtimeInfo.stat.Mtim}
+		if err := system.LUtimesNano(*mtimeInfo.dstPath, ts); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func doCopyXattrs(srcPath, dstPath string) error {
+	if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil {
+		return err
+	}
+
+	// We need to copy this attribute if it appears in an overlay upper layer, as
+	// this function is used to copy those. It is set by overlay if a directory
+	// is removed and then re-created and should not inherit anything from the
+	// same dir in the lower dir.
+	return copyXattr(srcPath, dstPath, "trusted.overlay.opaque")
+}
diff --git a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go
index 2801dfdc5..b6f22e90a 100644
--- a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go
+++ b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go
@@ -2401,7 +2401,7 @@ func (devices *DeviceSet) MountDevice(hash, path string, moptions graphdriver.Mo
 		addNouuid := strings.Contains("nouuid", mountOptions)
 		mountOptions = strings.Join(moptions.Options, ",")
 		if addNouuid {
-			mountOptions = fmt.Sprintf("nouuid,", mountOptions)
+			mountOptions = fmt.Sprintf("nouuid,%s", mountOptions)
 		}
 	}
 
diff --git a/vendor/github.com/containers/storage/drivers/vfs/copy_linux.go b/vendor/github.com/containers/storage/drivers/vfs/copy_linux.go
new file mode 100644
index 000000000..8137fcf67
--- /dev/null
+++ b/vendor/github.com/containers/storage/drivers/vfs/copy_linux.go
@@ -0,0 +1,7 @@
+package vfs
+
+import "github.com/containers/storage/drivers/copy"
+
+func dirCopy(srcDir, dstDir string) error {
+	return copy.DirCopy(srcDir, dstDir, copy.Content, false)
+}
diff --git a/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go b/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go
new file mode 100644
index 000000000..8ac80ee1d
--- /dev/null
+++ b/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go
@@ -0,0 +1,9 @@
+// +build !linux
+
+package vfs // import "github.com/containers/storage/drivers/vfs"
+
+import "github.com/containers/storage/pkg/chrootarchive"
+
+func dirCopy(srcDir, dstDir string) error {
+	return chrootarchive.NewArchiver(nil).CopyWithTar(srcDir, dstDir)
+}
diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go
index e3a67a69b..f7f3c75ba 100644
--- a/vendor/github.com/containers/storage/drivers/vfs/driver.go
+++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go
@@ -7,7 +7,6 @@ import (
 	"strings"
 
 	"github.com/containers/storage/drivers"
-	"github.com/containers/storage/pkg/chrootarchive"
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/containers/storage/pkg/ostree"
 	"github.com/containers/storage/pkg/system"
@@ -15,8 +14,8 @@ import (
 )
 
 var (
-	// CopyWithTar defines the copy method to use.
-	CopyWithTar = chrootarchive.NewArchiver(nil).CopyWithTar
+	// CopyDir defines the copy method to use.
+	CopyDir = dirCopy
 )
 
 func init() {
@@ -141,7 +140,7 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, ro bool
 		if err != nil {
 			return fmt.Errorf("%s: %s", parent, err)
 		}
-		if err := CopyWithTar(parentDir, dir); err != nil {
+		if err := dirCopy(parentDir, dir); err != nil {
 			return err
 		}
 	}
diff --git a/vendor/github.com/containers/storage/pkg/archive/example_changes.go b/vendor/github.com/containers/storage/pkg/archive/example_changes.go
new file mode 100644
index 000000000..70f9c5564
--- /dev/null
+++ b/vendor/github.com/containers/storage/pkg/archive/example_changes.go
@@ -0,0 +1,97 @@
+// +build ignore
+
+// Simple tool to create an archive stream from an old and new directory
+//
+// By default it will stream the comparison of two temporary directories with junk files
+package main
+
+import (
+	"flag"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"github.com/containers/storage/pkg/archive"
+	"github.com/sirupsen/logrus"
+)
+
+var (
+	flDebug  = flag.Bool("D", false, "debugging output")
+	flNewDir = flag.String("newdir", "", "")
+	flOldDir = flag.String("olddir", "", "")
+	log      = logrus.New()
+)
+
+func main() {
+	flag.Usage = func() {
+		fmt.Println("Produce a tar from comparing two directory paths. By default a demo tar is created of around 200 files (including hardlinks)")
+		fmt.Printf("%s [OPTIONS]\n", os.Args[0])
+		flag.PrintDefaults()
+	}
+	flag.Parse()
+	log.Out = os.Stderr
+	if (len(os.Getenv("DEBUG")) > 0) || *flDebug {
+		logrus.SetLevel(logrus.DebugLevel)
+	}
+	var newDir, oldDir string
+
+	if len(*flNewDir) == 0 {
+		var err error
+		newDir, err = ioutil.TempDir("", "storage-test-newDir")
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer os.RemoveAll(newDir)
+		if _, err := prepareUntarSourceDirectory(100, newDir, true); err != nil {
+			log.Fatal(err)
+		}
+	} else {
+		newDir = *flNewDir
+	}
+
+	if len(*flOldDir) == 0 {
+		oldDir, err := ioutil.TempDir("", "storage-test-oldDir")
+		if err != nil {
+			log.Fatal(err)
+		}
+		defer os.RemoveAll(oldDir)
+	} else {
+		oldDir = *flOldDir
+	}
+
+	changes, err := archive.ChangesDirs(newDir, oldDir)
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	a, err := archive.ExportChanges(newDir, changes)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer a.Close()
+
+	i, err := io.Copy(os.Stdout, a)
+	if err != nil && err != io.EOF {
+		log.Fatal(err)
+	}
+	fmt.Fprintf(os.Stderr, "wrote archive of %d bytes", i)
+}
+
+func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) {
+	fileData := []byte("fooo")
+	for n := 0; n < numberOfFiles; n++ {
+		fileName := fmt.Sprintf("file-%d", n)
+		if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil {
+			return 0, err
+		}
+		if makeLinks {
+			if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil {
+				return 0, err
+			}
+		}
+	}
+	totalSize := numberOfFiles * len(fileData)
+	return totalSize, nil
+}
diff --git a/vendor/github.com/containers/storage/vendor.conf b/vendor/github.com/containers/storage/vendor.conf
index 059ae94f0..fa52584d7 100644
--- a/vendor/github.com/containers/storage/vendor.conf
+++ b/vendor/github.com/containers/storage/vendor.conf
@@ -9,7 +9,7 @@ github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062
 github.com/opencontainers/go-digest master
 github.com/opencontainers/runc 6c22e77604689db8725fa866f0f2ec0b3e8c3a07
 github.com/opencontainers/selinux 36a9bc45a08c85f2c52bd9eb32e20267876773bd
-github.com/ostreedev/ostree-go aeb02c6b6aa2889db3ef62f7855650755befd460
+github.com/ostreedev/ostree-go master
 github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9
 github.com/pkg/errors master
 github.com/pmezard/go-difflib v1.0.0
@@ -21,3 +21,5 @@ github.com/tchap/go-patricia v2.2.6
 github.com/vbatts/tar-split v0.10.2
 golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6
 golang.org/x/sys 07c182904dbd53199946ba614a412c61d3c548f5
+gotest.tools master
+github.com/google/go-cmp master