From eb4c2035aef32fd7f0037bbea0447205fcd29d33 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Fri, 26 Oct 2018 11:17:25 -0400 Subject: Vendor in latest containers/storage We need this to start testing metacopy up for podman. Signed-off-by: Daniel J Walsh --- vendor.conf | 2 +- .../checkpoint-restore/go-criu/test/main.go | 133 -------------- .../checkpoint-restore/go-criu/test/phaul-main.go | 192 --------------------- .../checkpoint-restore/go-criu/test/piggie.c | 57 ------ .../containers/storage/drivers/overlay/check.go | 5 +- .../containers/storage/drivers/overlay/overlay.go | 14 +- 6 files changed, 12 insertions(+), 391 deletions(-) delete mode 100644 vendor/github.com/checkpoint-restore/go-criu/test/main.go delete mode 100644 vendor/github.com/checkpoint-restore/go-criu/test/phaul-main.go delete mode 100644 vendor/github.com/checkpoint-restore/go-criu/test/piggie.c diff --git a/vendor.conf b/vendor.conf index dfcdbbe80..85b784d9b 100644 --- a/vendor.conf +++ b/vendor.conf @@ -12,7 +12,7 @@ github.com/containerd/continuity master github.com/containernetworking/cni v0.7.0-alpha1 github.com/containernetworking/plugins 1562a1e60ed101aacc5e08ed9dbeba8e9f3d4ec1 github.com/containers/image bd10b1b53b2976f215b3f2f848fb8e7cad779aeb -github.com/containers/storage bd5818eda84012cf1db4dafbddd4b7509bb77142 +github.com/containers/storage 09abf3a26b8a3aa69e29fd7faeb260b98d675759 github.com/containers/psgo 5dde6da0bc8831b35243a847625bcf18183bd1ee github.com/coreos/go-systemd v14 github.com/cri-o/ocicni 2d2983e40c242322a56c22a903785e7f83eb378c diff --git a/vendor/github.com/checkpoint-restore/go-criu/test/main.go b/vendor/github.com/checkpoint-restore/go-criu/test/main.go deleted file mode 100644 index 418ebb843..000000000 --- a/vendor/github.com/checkpoint-restore/go-criu/test/main.go +++ /dev/null @@ -1,133 +0,0 @@ -package main - -import ( - "fmt" - "github.com/checkpoint-restore/go-criu" - "github.com/checkpoint-restore/go-criu/rpc" - "github.com/golang/protobuf/proto" - "os" - "strconv" -) - -// TestNfy struct -type TestNfy struct { - criu.NoNotify -} - -// PreDump test function -func (c TestNfy) PreDump() error { - fmt.Printf("TEST PRE DUMP\n") - return nil -} - -func doDump(c *criu.Criu, pidS string, imgDir string, pre bool, prevImg string) error { - fmt.Printf("Dumping\n") - pid, _ := strconv.Atoi(pidS) - img, err := os.Open(imgDir) - if err != nil { - return fmt.Errorf("can't open image dir (%s)", err) - } - defer img.Close() - - opts := rpc.CriuOpts{ - Pid: proto.Int32(int32(pid)), - ImagesDirFd: proto.Int32(int32(img.Fd())), - LogLevel: proto.Int32(4), - LogFile: proto.String("dump.log"), - } - - if prevImg != "" { - opts.ParentImg = proto.String(prevImg) - opts.TrackMem = proto.Bool(true) - } - - if pre { - err = c.PreDump(opts, TestNfy{}) - } else { - err = c.Dump(opts, TestNfy{}) - } - if err != nil { - return fmt.Errorf("dump fail (%s)", err) - } - - return nil -} - -// Usage: test $act $pid $images_dir -func main() { - c := criu.MakeCriu() - // Read out CRIU version - version, err := c.GetCriuVersion() - if err != nil { - fmt.Println(err) - os.Exit(1) - } - fmt.Println("CRIU version", version) - // Check if version at least 3.2 - result, err := c.IsCriuAtLeast(30200) - if err != nil { - fmt.Println(err) - os.Exit(1) - } - if !result { - fmt.Println("CRIU too old") - os.Exit(1) - } - act := os.Args[1] - switch act { - case "dump": - err := doDump(c, os.Args[2], os.Args[3], false, "") - if err != nil { - fmt.Print(err) - os.Exit(1) - } - case "dump2": - err := c.Prepare() - if err != nil { - fmt.Print(err) - os.Exit(1) - } - - err = doDump(c, os.Args[2], os.Args[3]+"/pre", true, "") - if err != nil { - fmt.Printf("pre-dump failed") - fmt.Print(err) - os.Exit(1) - } - err = doDump(c, os.Args[2], os.Args[3], false, "./pre") - if err != nil { - fmt.Printf("dump failed") - fmt.Print(err) - os.Exit(1) - } - - c.Cleanup() - case "restore": - fmt.Printf("Restoring\n") - img, err := os.Open(os.Args[2]) - if err != nil { - fmt.Printf("can't open image dir") - os.Exit(1) - } - defer img.Close() - - opts := rpc.CriuOpts{ - ImagesDirFd: proto.Int32(int32(img.Fd())), - LogLevel: proto.Int32(4), - LogFile: proto.String("restore.log"), - } - - err = c.Restore(opts, nil) - if err != nil { - fmt.Printf("Error:") - fmt.Print(err) - fmt.Printf("\n") - os.Exit(1) - } - default: - fmt.Printf("unknown action\n") - os.Exit(1) - } - - fmt.Printf("Success\n") -} diff --git a/vendor/github.com/checkpoint-restore/go-criu/test/phaul-main.go b/vendor/github.com/checkpoint-restore/go-criu/test/phaul-main.go deleted file mode 100644 index f1bec2c55..000000000 --- a/vendor/github.com/checkpoint-restore/go-criu/test/phaul-main.go +++ /dev/null @@ -1,192 +0,0 @@ -package main - -import ( - "fmt" - "os" - "strconv" - "strings" - "syscall" - - "github.com/checkpoint-restore/go-criu" - "github.com/checkpoint-restore/go-criu/phaul" - "github.com/checkpoint-restore/go-criu/rpc" - "github.com/golang/protobuf/proto" -) - -type testLocal struct { - criu.NoNotify - r *testRemote -} - -type testRemote struct { - srv *phaul.Server -} - -/* Dir where test will put dump images */ -const imagesDir = "image" - -func prepareImages() error { - err := os.Mkdir(imagesDir, 0700) - if err != nil { - return err - } - - /* Work dir for PhaulClient */ - err = os.Mkdir(imagesDir+"/local", 0700) - if err != nil { - return err - } - - /* Work dir for PhaulServer */ - err = os.Mkdir(imagesDir+"/remote", 0700) - if err != nil { - return err - } - - /* Work dir for DumpCopyRestore */ - err = os.Mkdir(imagesDir+"/test", 0700) - if err != nil { - return err - } - - return nil -} - -func mergeImages(dumpDir, lastPreDumpDir string) error { - idir, err := os.Open(dumpDir) - if err != nil { - return err - } - - defer idir.Close() - - imgs, err := idir.Readdirnames(0) - if err != nil { - return err - } - - for _, fname := range imgs { - if !strings.HasSuffix(fname, ".img") { - continue - } - - fmt.Printf("\t%s -> %s/\n", fname, lastPreDumpDir) - err = syscall.Link(dumpDir+"/"+fname, lastPreDumpDir+"/"+fname) - if err != nil { - return err - } - } - - return nil -} - -func (r *testRemote) doRestore() error { - lastSrvImagesDir := r.srv.LastImagesDir() - /* - * In imagesDir we have images from dump, in the - * lastSrvImagesDir -- where server-side images - * (from page server, with pages and pagemaps) are. - * Need to put former into latter and restore from - * them. - */ - err := mergeImages(imagesDir+"/test", lastSrvImagesDir) - if err != nil { - return err - } - - imgDir, err := os.Open(lastSrvImagesDir) - if err != nil { - return err - } - defer imgDir.Close() - - opts := rpc.CriuOpts{ - LogLevel: proto.Int32(4), - LogFile: proto.String("restore.log"), - ImagesDirFd: proto.Int32(int32(imgDir.Fd())), - } - - cr := r.srv.GetCriu() - fmt.Printf("Do restore\n") - return cr.Restore(opts, nil) -} - -func (l *testLocal) PostDump() error { - return l.r.doRestore() -} - -func (l *testLocal) DumpCopyRestore(cr *criu.Criu, cfg phaul.Config, lastClnImagesDir string) error { - fmt.Printf("Final stage\n") - - imgDir, err := os.Open(imagesDir + "/test") - if err != nil { - return err - } - defer imgDir.Close() - - psi := rpc.CriuPageServerInfo{ - Fd: proto.Int32(int32(cfg.Memfd)), - } - - opts := rpc.CriuOpts{ - Pid: proto.Int32(int32(cfg.Pid)), - LogLevel: proto.Int32(4), - LogFile: proto.String("dump.log"), - ImagesDirFd: proto.Int32(int32(imgDir.Fd())), - TrackMem: proto.Bool(true), - ParentImg: proto.String(lastClnImagesDir), - Ps: &psi, - } - - fmt.Printf("Do dump\n") - return cr.Dump(opts, l) -} - -func main() { - pid, _ := strconv.Atoi(os.Args[1]) - fds, err := syscall.Socketpair(syscall.AF_LOCAL, syscall.SOCK_STREAM, 0) - if err != nil { - fmt.Printf("Can't make socketpair: %v\n", err) - os.Exit(1) - } - - err = prepareImages() - if err != nil { - fmt.Printf("Can't prepare dirs for images: %v\n", err) - os.Exit(1) - return - } - - fmt.Printf("Make server part (socket %d)\n", fds[1]) - srv, err := phaul.MakePhaulServer(phaul.Config{ - Pid: pid, - Memfd: fds[1], - Wdir: imagesDir + "/remote"}) - if err != nil { - fmt.Printf("Unable to run a server: %v", err) - os.Exit(1) - return - } - - r := &testRemote{srv} - - fmt.Printf("Make client part (socket %d)\n", fds[0]) - cln, err := phaul.MakePhaulClient(&testLocal{r: r}, srv, - phaul.Config{ - Pid: pid, - Memfd: fds[0], - Wdir: imagesDir + "/local"}) - if err != nil { - fmt.Printf("Unable to run a client: %v\n", err) - os.Exit(1) - } - - fmt.Printf("Migrate\n") - err = cln.Migrate() - if err != nil { - fmt.Printf("Failed: %v\n", err) - os.Exit(1) - } - - fmt.Printf("SUCCESS!\n") -} diff --git a/vendor/github.com/checkpoint-restore/go-criu/test/piggie.c b/vendor/github.com/checkpoint-restore/go-criu/test/piggie.c deleted file mode 100644 index 1dc0801c0..000000000 --- a/vendor/github.com/checkpoint-restore/go-criu/test/piggie.c +++ /dev/null @@ -1,57 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include - -#define STKS (4*4096) - -#ifndef CLONE_NEWPID -#define CLONE_NEWPID 0x20000000 -#endif - -static int do_test(void *logf) -{ - int fd, i = 0; - - setsid(); - - close(0); - close(1); - close(2); - - fd = open("/dev/null", O_RDONLY); - if (fd != 0) { - dup2(fd, 0); - close(fd); - } - - fd = open(logf, O_WRONLY | O_TRUNC | O_CREAT, 0600); - dup2(fd, 1); - dup2(fd, 2); - if (fd != 1 && fd != 2) - close(fd); - - while (1) { - sleep(1); - printf("%d\n", i++); - fflush(stdout); - } - - return 0; -} - -int main(int argc, char **argv) -{ - int pid; - void *stk; - - stk = mmap(NULL, STKS, PROT_READ | PROT_WRITE, - MAP_PRIVATE | MAP_ANON | MAP_GROWSDOWN, 0, 0); - pid = clone(do_test, stk + STKS, SIGCHLD | CLONE_NEWPID, argv[1]); - printf("Child forked, pid %d\n", pid); - - return 0; -} diff --git a/vendor/github.com/containers/storage/drivers/overlay/check.go b/vendor/github.com/containers/storage/drivers/overlay/check.go index 2a096edf6..590d517fa 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/check.go +++ b/vendor/github.com/containers/storage/drivers/overlay/check.go @@ -20,7 +20,7 @@ import ( // which copies up the opaque flag when copying up an opaque // directory or the kernel enable CONFIG_OVERLAY_FS_REDIRECT_DIR. // When these exist naive diff should be used. -func doesSupportNativeDiff(d string) error { +func doesSupportNativeDiff(d, mountOpts string) error { td, err := ioutil.TempDir(d, "opaque-bug-check") if err != nil { return err @@ -57,6 +57,9 @@ func doesSupportNativeDiff(d string) error { } opts := fmt.Sprintf("lowerdir=%s:%s,upperdir=%s,workdir=%s", path.Join(td, "l2"), path.Join(td, "l1"), path.Join(td, "l3"), path.Join(td, "work")) + if mountOpts != "" { + opts = fmt.Sprintf("%s,%s", opts, mountOpts) + } if err := unix.Mount("overlay", filepath.Join(td, "merged"), "overlay", 0, opts); err != nil { return errors.Wrap(err, "failed to mount overlay") } diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 2e0498f51..d2cc65bca 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -206,7 +206,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, fmt.Errorf("Storage option overlay.size only supported for backingFS XFS. Found %v", backingFs) } - logrus.Debugf("backingFs=%s, projectQuotaSupported=%v, useNativeDiff=%v", backingFs, projectQuotaSupported, !useNaiveDiff(home)) + logrus.Debugf("backingFs=%s, projectQuotaSupported=%v, useNativeDiff=%v", backingFs, projectQuotaSupported, !d.useNaiveDiff()) return d, nil } @@ -338,9 +338,9 @@ func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGI return supportsDType, errors.Wrap(graphdriver.ErrNotSupported, "'overlay' not found as a supported filesystem on this host. Please ensure kernel is new enough and has overlay support loaded.") } -func useNaiveDiff(home string) bool { +func (d *Driver) useNaiveDiff() bool { useNaiveDiffLock.Do(func() { - if err := doesSupportNativeDiff(home); err != nil { + if err := doesSupportNativeDiff(d.home, d.options.mountOptions); err != nil { logrus.Warnf("Not using native diff for overlay, this may cause degraded performance for building images: %v", err) useNaiveDiffOnly = true } @@ -358,7 +358,7 @@ func (d *Driver) Status() [][2]string { return [][2]string{ {"Backing Filesystem", backingFs}, {"Supports d_type", strconv.FormatBool(d.supportsDType)}, - {"Native Overlay Diff", strconv.FormatBool(!useNaiveDiff(d.home))}, + {"Native Overlay Diff", strconv.FormatBool(!d.useNaiveDiff())}, } } @@ -883,7 +883,7 @@ func (d *Driver) getDiffPath(id string) string { // and its parent and returns the size in bytes of the changes // relative to its base filesystem directory. func (d *Driver) DiffSize(id string, idMappings *idtools.IDMappings, parent string, parentMappings *idtools.IDMappings, mountLabel string) (size int64, err error) { - if useNaiveDiff(d.home) || !d.isParent(id, parent) { + if d.useNaiveDiff() || !d.isParent(id, parent) { return d.naiveDiff.DiffSize(id, idMappings, parent, parentMappings, mountLabel) } return directory.Size(d.getDiffPath(id)) @@ -892,7 +892,7 @@ func (d *Driver) DiffSize(id string, idMappings *idtools.IDMappings, parent stri // Diff produces an archive of the changes between the specified // layer and its parent layer which may be "". func (d *Driver) Diff(id string, idMappings *idtools.IDMappings, parent string, parentMappings *idtools.IDMappings, mountLabel string) (io.ReadCloser, error) { - if useNaiveDiff(d.home) || !d.isParent(id, parent) { + if d.useNaiveDiff() || !d.isParent(id, parent) { return d.naiveDiff.Diff(id, idMappings, parent, parentMappings, mountLabel) } @@ -919,7 +919,7 @@ func (d *Driver) Diff(id string, idMappings *idtools.IDMappings, parent string, // Changes produces a list of changes between the specified layer // and its parent layer. If parent is "", then all changes will be ADD changes. func (d *Driver) Changes(id string, idMappings *idtools.IDMappings, parent string, parentMappings *idtools.IDMappings, mountLabel string) ([]archive.Change, error) { - if useNaiveDiff(d.home) || !d.isParent(id, parent) { + if d.useNaiveDiff() || !d.isParent(id, parent) { return d.naiveDiff.Changes(id, idMappings, parent, parentMappings, mountLabel) } // Overlay doesn't have snapshots, so we need to get changes from all parent -- cgit v1.2.3-54-g00ecf From 5f480f5f75b94fb3398884977589d75d5ccf19c6 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Wed, 5 Dec 2018 13:56:38 -0500 Subject: Vendor in latest containers/storage This should improve performance on vfs images on top of xfs/reflink drives. Signed-off-by: Daniel J Walsh --- vendor.conf | 2 +- .../containers/storage/drivers/copy/copy.go | 277 +++++++++++++++++++++ .../storage/drivers/devmapper/deviceset.go | 2 +- .../containers/storage/drivers/vfs/copy_linux.go | 7 + .../storage/drivers/vfs/copy_unsupported.go | 9 + .../containers/storage/drivers/vfs/driver.go | 7 +- .../storage/pkg/archive/example_changes.go | 97 ++++++++ vendor/github.com/containers/storage/vendor.conf | 4 +- 8 files changed, 398 insertions(+), 7 deletions(-) create mode 100644 vendor/github.com/containers/storage/drivers/copy/copy.go create mode 100644 vendor/github.com/containers/storage/drivers/vfs/copy_linux.go create mode 100644 vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go create mode 100644 vendor/github.com/containers/storage/pkg/archive/example_changes.go diff --git a/vendor.conf b/vendor.conf index 51907f763..94eb6fccc 100644 --- a/vendor.conf +++ b/vendor.conf @@ -12,7 +12,7 @@ github.com/containerd/continuity master github.com/containernetworking/cni v0.7.0-alpha1 github.com/containernetworking/plugins 1562a1e60ed101aacc5e08ed9dbeba8e9f3d4ec1 github.com/containers/image bd10b1b53b2976f215b3f2f848fb8e7cad779aeb -github.com/containers/storage ad0f9c4dfa38fcb160f430ff1d653dc3dae03810 +github.com/containers/storage db40f96d853dfced60c563e61fb66ba231ce7c8d github.com/containers/psgo 5dde6da0bc8831b35243a847625bcf18183bd1ee github.com/coreos/go-systemd v14 github.com/cri-o/ocicni 2d2983e40c242322a56c22a903785e7f83eb378c diff --git a/vendor/github.com/containers/storage/drivers/copy/copy.go b/vendor/github.com/containers/storage/drivers/copy/copy.go new file mode 100644 index 000000000..2617824c5 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/copy/copy.go @@ -0,0 +1,277 @@ +// +build linux + +package copy + +/* +#include + +#ifndef FICLONE +#define FICLONE _IOW(0x94, 9, int) +#endif +*/ +import "C" +import ( + "container/list" + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/containers/storage/pkg/pools" + "github.com/containers/storage/pkg/system" + rsystem "github.com/opencontainers/runc/libcontainer/system" + "golang.org/x/sys/unix" +) + +// Mode indicates whether to use hardlink or copy content +type Mode int + +const ( + // Content creates a new file, and copies the content of the file + Content Mode = iota + // Hardlink creates a new hardlink to the existing file + Hardlink +) + +func copyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { + srcFile, err := os.Open(srcPath) + if err != nil { + return err + } + defer srcFile.Close() + + // If the destination file already exists, we shouldn't blow it away + dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, fileinfo.Mode()) + if err != nil { + return err + } + defer dstFile.Close() + + if *copyWithFileClone { + _, _, err = unix.Syscall(unix.SYS_IOCTL, dstFile.Fd(), C.FICLONE, srcFile.Fd()) + if err == nil { + return nil + } + + *copyWithFileClone = false + if err == unix.EXDEV { + *copyWithFileRange = false + } + } + if *copyWithFileRange { + err = doCopyWithFileRange(srcFile, dstFile, fileinfo) + // Trying the file_clone may not have caught the exdev case + // as the ioctl may not have been available (therefore EINVAL) + if err == unix.EXDEV || err == unix.ENOSYS { + *copyWithFileRange = false + } else { + return err + } + } + return legacyCopy(srcFile, dstFile) +} + +func doCopyWithFileRange(srcFile, dstFile *os.File, fileinfo os.FileInfo) error { + amountLeftToCopy := fileinfo.Size() + + for amountLeftToCopy > 0 { + n, err := unix.CopyFileRange(int(srcFile.Fd()), nil, int(dstFile.Fd()), nil, int(amountLeftToCopy), 0) + if err != nil { + return err + } + + amountLeftToCopy = amountLeftToCopy - int64(n) + } + + return nil +} + +func legacyCopy(srcFile io.Reader, dstFile io.Writer) error { + _, err := pools.Copy(dstFile, srcFile) + + return err +} + +func copyXattr(srcPath, dstPath, attr string) error { + data, err := system.Lgetxattr(srcPath, attr) + if err != nil { + return err + } + if data != nil { + if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil { + return err + } + } + return nil +} + +type fileID struct { + dev uint64 + ino uint64 +} + +type dirMtimeInfo struct { + dstPath *string + stat *syscall.Stat_t +} + +// DirCopy copies or hardlinks the contents of one directory to another, +// properly handling xattrs, and soft links +// +// Copying xattrs can be opted out of by passing false for copyXattrs. +func DirCopy(srcDir, dstDir string, copyMode Mode, copyXattrs bool) error { + copyWithFileRange := true + copyWithFileClone := true + + // This is a map of source file inodes to dst file paths + copiedFiles := make(map[fileID]string) + + dirsToSetMtimes := list.New() + err := filepath.Walk(srcDir, func(srcPath string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + relPath, err := filepath.Rel(srcDir, srcPath) + if err != nil { + return err + } + + dstPath := filepath.Join(dstDir, relPath) + if err != nil { + return err + } + + stat, ok := f.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("Unable to get raw syscall.Stat_t data for %s", srcPath) + } + + isHardlink := false + + switch f.Mode() & os.ModeType { + case 0: // Regular file + id := fileID{dev: stat.Dev, ino: stat.Ino} + if copyMode == Hardlink { + isHardlink = true + if err2 := os.Link(srcPath, dstPath); err2 != nil { + return err2 + } + } else if hardLinkDstPath, ok := copiedFiles[id]; ok { + if err2 := os.Link(hardLinkDstPath, dstPath); err2 != nil { + return err2 + } + } else { + if err2 := copyRegular(srcPath, dstPath, f, ©WithFileRange, ©WithFileClone); err2 != nil { + return err2 + } + copiedFiles[id] = dstPath + } + + case os.ModeDir: + if err := os.Mkdir(dstPath, f.Mode()); err != nil && !os.IsExist(err) { + return err + } + + case os.ModeSymlink: + link, err := os.Readlink(srcPath) + if err != nil { + return err + } + + if err := os.Symlink(link, dstPath); err != nil { + return err + } + + case os.ModeNamedPipe: + fallthrough + case os.ModeSocket: + if err := unix.Mkfifo(dstPath, stat.Mode); err != nil { + return err + } + + case os.ModeDevice: + if rsystem.RunningInUserNS() { + // cannot create a device if running in user namespace + return nil + } + if err := unix.Mknod(dstPath, stat.Mode, int(stat.Rdev)); err != nil { + return err + } + + default: + return fmt.Errorf("unknown file type for %s", srcPath) + } + + // Everything below is copying metadata from src to dst. All this metadata + // already shares an inode for hardlinks. + if isHardlink { + return nil + } + + if err := os.Lchown(dstPath, int(stat.Uid), int(stat.Gid)); err != nil { + return err + } + + if copyXattrs { + if err := doCopyXattrs(srcPath, dstPath); err != nil { + return err + } + } + + isSymlink := f.Mode()&os.ModeSymlink != 0 + + // There is no LChmod, so ignore mode for symlink. Also, this + // must happen after chown, as that can modify the file mode + if !isSymlink { + if err := os.Chmod(dstPath, f.Mode()); err != nil { + return err + } + } + + // system.Chtimes doesn't support a NOFOLLOW flag atm + // nolint: unconvert + if f.IsDir() { + dirsToSetMtimes.PushFront(&dirMtimeInfo{dstPath: &dstPath, stat: stat}) + } else if !isSymlink { + aTime := time.Unix(int64(stat.Atim.Sec), int64(stat.Atim.Nsec)) + mTime := time.Unix(int64(stat.Mtim.Sec), int64(stat.Mtim.Nsec)) + if err := system.Chtimes(dstPath, aTime, mTime); err != nil { + return err + } + } else { + ts := []syscall.Timespec{stat.Atim, stat.Mtim} + if err := system.LUtimesNano(dstPath, ts); err != nil { + return err + } + } + return nil + }) + if err != nil { + return err + } + for e := dirsToSetMtimes.Front(); e != nil; e = e.Next() { + mtimeInfo := e.Value.(*dirMtimeInfo) + ts := []syscall.Timespec{mtimeInfo.stat.Atim, mtimeInfo.stat.Mtim} + if err := system.LUtimesNano(*mtimeInfo.dstPath, ts); err != nil { + return err + } + } + + return nil +} + +func doCopyXattrs(srcPath, dstPath string) error { + if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil { + return err + } + + // We need to copy this attribute if it appears in an overlay upper layer, as + // this function is used to copy those. It is set by overlay if a directory + // is removed and then re-created and should not inherit anything from the + // same dir in the lower dir. + return copyXattr(srcPath, dstPath, "trusted.overlay.opaque") +} diff --git a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go index 2801dfdc5..b6f22e90a 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go @@ -2401,7 +2401,7 @@ func (devices *DeviceSet) MountDevice(hash, path string, moptions graphdriver.Mo addNouuid := strings.Contains("nouuid", mountOptions) mountOptions = strings.Join(moptions.Options, ",") if addNouuid { - mountOptions = fmt.Sprintf("nouuid,", mountOptions) + mountOptions = fmt.Sprintf("nouuid,%s", mountOptions) } } diff --git a/vendor/github.com/containers/storage/drivers/vfs/copy_linux.go b/vendor/github.com/containers/storage/drivers/vfs/copy_linux.go new file mode 100644 index 000000000..8137fcf67 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/vfs/copy_linux.go @@ -0,0 +1,7 @@ +package vfs + +import "github.com/containers/storage/drivers/copy" + +func dirCopy(srcDir, dstDir string) error { + return copy.DirCopy(srcDir, dstDir, copy.Content, false) +} diff --git a/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go b/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go new file mode 100644 index 000000000..8ac80ee1d --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go @@ -0,0 +1,9 @@ +// +build !linux + +package vfs // import "github.com/containers/storage/drivers/vfs" + +import "github.com/containers/storage/pkg/chrootarchive" + +func dirCopy(srcDir, dstDir string) error { + return chrootarchive.NewArchiver(nil).CopyWithTar(srcDir, dstDir) +} diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index e3a67a69b..f7f3c75ba 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/containers/storage/drivers" - "github.com/containers/storage/pkg/chrootarchive" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/ostree" "github.com/containers/storage/pkg/system" @@ -15,8 +14,8 @@ import ( ) var ( - // CopyWithTar defines the copy method to use. - CopyWithTar = chrootarchive.NewArchiver(nil).CopyWithTar + // CopyDir defines the copy method to use. + CopyDir = dirCopy ) func init() { @@ -141,7 +140,7 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, ro bool if err != nil { return fmt.Errorf("%s: %s", parent, err) } - if err := CopyWithTar(parentDir, dir); err != nil { + if err := dirCopy(parentDir, dir); err != nil { return err } } diff --git a/vendor/github.com/containers/storage/pkg/archive/example_changes.go b/vendor/github.com/containers/storage/pkg/archive/example_changes.go new file mode 100644 index 000000000..70f9c5564 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/archive/example_changes.go @@ -0,0 +1,97 @@ +// +build ignore + +// Simple tool to create an archive stream from an old and new directory +// +// By default it will stream the comparison of two temporary directories with junk files +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "path" + + "github.com/containers/storage/pkg/archive" + "github.com/sirupsen/logrus" +) + +var ( + flDebug = flag.Bool("D", false, "debugging output") + flNewDir = flag.String("newdir", "", "") + flOldDir = flag.String("olddir", "", "") + log = logrus.New() +) + +func main() { + flag.Usage = func() { + fmt.Println("Produce a tar from comparing two directory paths. By default a demo tar is created of around 200 files (including hardlinks)") + fmt.Printf("%s [OPTIONS]\n", os.Args[0]) + flag.PrintDefaults() + } + flag.Parse() + log.Out = os.Stderr + if (len(os.Getenv("DEBUG")) > 0) || *flDebug { + logrus.SetLevel(logrus.DebugLevel) + } + var newDir, oldDir string + + if len(*flNewDir) == 0 { + var err error + newDir, err = ioutil.TempDir("", "storage-test-newDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(newDir) + if _, err := prepareUntarSourceDirectory(100, newDir, true); err != nil { + log.Fatal(err) + } + } else { + newDir = *flNewDir + } + + if len(*flOldDir) == 0 { + oldDir, err := ioutil.TempDir("", "storage-test-oldDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(oldDir) + } else { + oldDir = *flOldDir + } + + changes, err := archive.ChangesDirs(newDir, oldDir) + if err != nil { + log.Fatal(err) + } + + a, err := archive.ExportChanges(newDir, changes) + if err != nil { + log.Fatal(err) + } + defer a.Close() + + i, err := io.Copy(os.Stdout, a) + if err != nil && err != io.EOF { + log.Fatal(err) + } + fmt.Fprintf(os.Stderr, "wrote archive of %d bytes", i) +} + +func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) { + fileData := []byte("fooo") + for n := 0; n < numberOfFiles; n++ { + fileName := fmt.Sprintf("file-%d", n) + if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil { + return 0, err + } + if makeLinks { + if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil { + return 0, err + } + } + } + totalSize := numberOfFiles * len(fileData) + return totalSize, nil +} diff --git a/vendor/github.com/containers/storage/vendor.conf b/vendor/github.com/containers/storage/vendor.conf index 059ae94f0..fa52584d7 100644 --- a/vendor/github.com/containers/storage/vendor.conf +++ b/vendor/github.com/containers/storage/vendor.conf @@ -9,7 +9,7 @@ github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062 github.com/opencontainers/go-digest master github.com/opencontainers/runc 6c22e77604689db8725fa866f0f2ec0b3e8c3a07 github.com/opencontainers/selinux 36a9bc45a08c85f2c52bd9eb32e20267876773bd -github.com/ostreedev/ostree-go aeb02c6b6aa2889db3ef62f7855650755befd460 +github.com/ostreedev/ostree-go master github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9 github.com/pkg/errors master github.com/pmezard/go-difflib v1.0.0 @@ -21,3 +21,5 @@ github.com/tchap/go-patricia v2.2.6 github.com/vbatts/tar-split v0.10.2 golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 golang.org/x/sys 07c182904dbd53199946ba614a412c61d3c548f5 +gotest.tools master +github.com/google/go-cmp master -- cgit v1.2.3-54-g00ecf