From f661e1d21d5f22d889842642dbd612ead54a7612 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Sun, 8 Jul 2018 07:55:30 -0400 Subject: Vendor in latest containers/storage Signed-off-by: Daniel J Walsh Closes: #1061 Approved by: baude --- vendor/github.com/containers/storage/README.md | 3 + .../containers/storage/drivers/overlay/overlay.go | 85 +++++++-- .../containers/storage/drivers/vfs/driver.go | 54 +++++- .../storage/pkg/archive/example_changes.go | 97 ++++++++++ .../containers/storage/pkg/ostree/no_ostree.go | 19 ++ .../containers/storage/pkg/ostree/ostree.go | 198 +++++++++++++++++++++ vendor/github.com/containers/storage/store.go | 11 ++ vendor/github.com/containers/storage/vendor.conf | 1 + 8 files changed, 451 insertions(+), 17 deletions(-) create mode 100644 vendor/github.com/containers/storage/pkg/archive/example_changes.go create mode 100644 vendor/github.com/containers/storage/pkg/ostree/no_ostree.go create mode 100644 vendor/github.com/containers/storage/pkg/ostree/ostree.go (limited to 'vendor/github.com') diff --git a/vendor/github.com/containers/storage/README.md b/vendor/github.com/containers/storage/README.md index 00a47fd98..f68cc55c3 100644 --- a/vendor/github.com/containers/storage/README.md +++ b/vendor/github.com/containers/storage/README.md @@ -41,3 +41,6 @@ memory and stored along with the library's own bookkeeping information. Additionally, the library can store one or more of what it calls *big data* for images and containers. This is a named chunk of larger data, which is only in memory when it is being read from or being written to its own disk file. + +**[Contributing](CONTRIBUTING.md)** +Information about contributing to this project. diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 6b5f6912f..5d5d61d9c 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -24,6 +24,7 @@ import ( "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/locker" "github.com/containers/storage/pkg/mount" + "github.com/containers/storage/pkg/ostree" "github.com/containers/storage/pkg/parsers" "github.com/containers/storage/pkg/system" units "github.com/docker/go-units" @@ -84,6 +85,9 @@ type overlayOptions struct { overrideKernelCheck bool imageStores []string quota quota.Quota + fuseProgram string + ostreeRepo string + skipMountHome bool } // Driver contains information about the home directory and the list of active mounts that are created using this driver. @@ -98,6 +102,7 @@ type Driver struct { naiveDiff graphdriver.DiffDriver supportsDType bool locker *locker.Locker + convert map[string]bool } var ( @@ -147,15 +152,28 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap return nil, err } - supportsDType, err := supportsOverlay(home, fsMagic, rootUID, rootGID) - if err != nil { - os.Remove(filepath.Join(home, linkDir)) - os.Remove(home) - return nil, errors.Wrap(err, "kernel does not support overlay fs") + var supportsDType bool + if opts.fuseProgram != "" { + supportsDType = true + } else { + supportsDType, err = supportsOverlay(home, fsMagic, rootUID, rootGID) + if err != nil { + os.Remove(filepath.Join(home, linkDir)) + os.Remove(home) + return nil, errors.Wrap(err, "kernel does not support overlay fs") + } } - if err := mount.MakePrivate(home); err != nil { - return nil, err + if !opts.skipMountHome { + if err := mount.MakePrivate(home); err != nil { + return nil, err + } + } + + if opts.ostreeRepo != "" { + if err := ostree.CreateOSTreeRepository(opts.ostreeRepo, rootUID, rootGID); err != nil { + return nil, err + } } d := &Driver{ @@ -167,6 +185,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap supportsDType: supportsDType, locker: locker.New(), options: *opts, + convert: make(map[string]bool), } d.naiveDiff = graphdriver.NewNaiveDiffDriver(d, d) @@ -227,6 +246,25 @@ func parseOptions(options []string) (*overlayOptions, error) { } o.imageStores = append(o.imageStores, store) } + case ".fuse_program", "overlay.fuse_program", "overlay2.fuse_program": + logrus.Debugf("overlay: fuse_program=%s", val) + _, err := os.Stat(val) + if err != nil { + return nil, fmt.Errorf("overlay: can't stat FUSE program %s: %v", val, err) + } + o.fuseProgram = val + case "overlay2.ostree_repo", "overlay.ostree_repo", ".ostree_repo": + logrus.Debugf("overlay: ostree_repo=%s", val) + if !ostree.OstreeSupport() { + return nil, fmt.Errorf("overlay: ostree_repo specified but support for ostree is missing") + } + o.ostreeRepo = val + case "overlay2.skip_mount_home", "overlay.skip_mount_home", ".skip_mount_home": + logrus.Debugf("overlay: skip_mount_home=%s", val) + o.skipMountHome, err = strconv.ParseBool(val) + if err != nil { + return nil, err + } default: return nil, fmt.Errorf("overlay: Unknown option %s", key) } @@ -236,6 +274,7 @@ func parseOptions(options []string) (*overlayOptions, error) { func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGID int) (supportsDType bool, err error) { // We can try to modprobe overlay first + exec.Command("modprobe", "overlay").Run() layerDir, err := ioutil.TempDir(home, "compat") @@ -380,6 +419,11 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) (retErr return fmt.Errorf("--storage-opt size is only supported for ReadWrite Layers") } } + + if d.options.ostreeRepo != "" { + d.convert[id] = true + } + return d.create(id, parent, opts) } @@ -547,6 +591,12 @@ func (d *Driver) getLowerDirs(id string) ([]string, error) { func (d *Driver) Remove(id string) error { d.locker.Lock(id) defer d.locker.Unlock(id) + + // Ignore errors, we don't want to fail if the ostree branch doesn't exist, + if d.options.ostreeRepo != "" { + ostree.DeleteOSTree(d.options.ostreeRepo, id) + } + dir := d.dir(id) lid, err := ioutil.ReadFile(path.Join(dir, "link")) if err == nil { @@ -663,7 +713,7 @@ func (d *Driver) Get(id, mountLabel string) (_ string, retErr error) { // the page size. The mount syscall fails if the mount data cannot // fit within a page and relative links make the mount data much // smaller at the expense of requiring a fork exec to chroot. - if len(mountData) > pageSize { + if len(mountData) > pageSize || d.options.fuseProgram != "" { //FIXME: We need to figure out to get this to work with additional stores opts = fmt.Sprintf("lowerdir=%s,upperdir=%s,workdir=%s", strings.Join(relLowers, ":"), path.Join(id, "diff"), path.Join(id, "work")) mountData = label.FormatMountLabel(opts, mountLabel) @@ -671,8 +721,16 @@ func (d *Driver) Get(id, mountLabel string) (_ string, retErr error) { return "", fmt.Errorf("cannot mount layer, mount label too large %d", len(mountData)) } - mount = func(source string, target string, mType string, flags uintptr, label string) error { - return mountFrom(d.home, source, target, mType, flags, label) + if d.options.fuseProgram != "" { + mount = func(source string, target string, mType string, flags uintptr, label string) error { + cmdRootless := exec.Command(d.options.fuseProgram, "-o", label, target) + cmdRootless.Dir = d.home + return cmdRootless.Run() + } + } else { + mount = func(source string, target string, mType string, flags uintptr, label string) error { + return mountFrom(d.home, source, target, mType, flags, label) + } } mountTarget = path.Join(id, "merged") } @@ -764,6 +822,13 @@ func (d *Driver) ApplyDiff(id string, idMappings *idtools.IDMappings, parent str return 0, err } + _, convert := d.convert[id] + if convert { + if err := ostree.ConvertToOSTree(d.options.ostreeRepo, applyDir, id); err != nil { + return 0, err + } + } + return directory.Size(applyDir) } diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index cee55f8d1..9314cb876 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -9,6 +9,7 @@ import ( "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/chrootarchive" "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/pkg/ostree" "github.com/containers/storage/pkg/system" "github.com/opencontainers/selinux/go-selinux/label" ) @@ -42,6 +43,27 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap d.homes = append(d.homes, strings.Split(option[12:], ",")...) continue } + if strings.HasPrefix(option, "vfs.ostree_repo=") { + if !ostree.OstreeSupport() { + return nil, fmt.Errorf("vfs: ostree_repo specified but support for ostree is missing") + } + d.ostreeRepo = option[16:] + } + if strings.HasPrefix(option, ".ostree_repo=") { + if !ostree.OstreeSupport() { + return nil, fmt.Errorf("vfs: ostree_repo specified but support for ostree is missing") + } + d.ostreeRepo = option[13:] + } + } + if d.ostreeRepo != "" { + rootUID, rootGID, err := idtools.GetRootUIDGID(uidMaps, gidMaps) + if err != nil { + return nil, err + } + if err := ostree.CreateOSTreeRepository(d.ostreeRepo, rootUID, rootGID); err != nil { + return nil, err + } } return graphdriver.NewNaiveDiffDriver(d, graphdriver.NewNaiveLayerIDMapUpdater(d)), nil } @@ -53,6 +75,7 @@ func Init(home string, options []string, uidMaps, gidMaps []idtools.IDMap) (grap type Driver struct { homes []string idMappings *idtools.IDMappings + ostreeRepo string } func (d *Driver) String() string { @@ -77,11 +100,15 @@ func (d *Driver) Cleanup() error { // CreateReadWrite creates a layer that is writable for use as a container // file system. func (d *Driver) CreateReadWrite(id, parent string, opts *graphdriver.CreateOpts) error { - return d.Create(id, parent, opts) + return d.create(id, parent, opts, false) } // Create prepares the filesystem for the VFS driver and copies the directory for the given id under the parent. func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { + return d.create(id, parent, opts, true) +} + +func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, ro bool) error { if opts != nil && len(opts.StorageOpt) != 0 { return fmt.Errorf("--storage-opt is not supported for vfs") } @@ -106,14 +133,23 @@ func (d *Driver) Create(id, parent string, opts *graphdriver.CreateOpts) error { if _, mountLabel, err := label.InitLabels(labelOpts); err == nil { label.SetFileLabel(dir, mountLabel) } - if parent == "" { - return nil + if parent != "" { + parentDir, err := d.Get(parent, "") + if err != nil { + return fmt.Errorf("%s: %s", parent, err) + } + if err := CopyWithTar(parentDir, dir); err != nil { + return err + } } - parentDir, err := d.Get(parent, "") - if err != nil { - return fmt.Errorf("%s: %s", parent, err) + + if ro && d.ostreeRepo != "" { + if err := ostree.ConvertToOSTree(d.ostreeRepo, dir, id); err != nil { + return err + } } - return CopyWithTar(parentDir, dir) + return nil + } func (d *Driver) dir(id string) string { @@ -132,6 +168,10 @@ func (d *Driver) dir(id string) string { // Remove deletes the content from the directory for a given id. func (d *Driver) Remove(id string) error { + if d.ostreeRepo != "" { + // Ignore errors, we don't want to fail if the ostree branch doesn't exist, + ostree.DeleteOSTree(d.ostreeRepo, id) + } return system.EnsureRemoveAll(d.dir(id)) } diff --git a/vendor/github.com/containers/storage/pkg/archive/example_changes.go b/vendor/github.com/containers/storage/pkg/archive/example_changes.go new file mode 100644 index 000000000..70f9c5564 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/archive/example_changes.go @@ -0,0 +1,97 @@ +// +build ignore + +// Simple tool to create an archive stream from an old and new directory +// +// By default it will stream the comparison of two temporary directories with junk files +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "path" + + "github.com/containers/storage/pkg/archive" + "github.com/sirupsen/logrus" +) + +var ( + flDebug = flag.Bool("D", false, "debugging output") + flNewDir = flag.String("newdir", "", "") + flOldDir = flag.String("olddir", "", "") + log = logrus.New() +) + +func main() { + flag.Usage = func() { + fmt.Println("Produce a tar from comparing two directory paths. By default a demo tar is created of around 200 files (including hardlinks)") + fmt.Printf("%s [OPTIONS]\n", os.Args[0]) + flag.PrintDefaults() + } + flag.Parse() + log.Out = os.Stderr + if (len(os.Getenv("DEBUG")) > 0) || *flDebug { + logrus.SetLevel(logrus.DebugLevel) + } + var newDir, oldDir string + + if len(*flNewDir) == 0 { + var err error + newDir, err = ioutil.TempDir("", "storage-test-newDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(newDir) + if _, err := prepareUntarSourceDirectory(100, newDir, true); err != nil { + log.Fatal(err) + } + } else { + newDir = *flNewDir + } + + if len(*flOldDir) == 0 { + oldDir, err := ioutil.TempDir("", "storage-test-oldDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(oldDir) + } else { + oldDir = *flOldDir + } + + changes, err := archive.ChangesDirs(newDir, oldDir) + if err != nil { + log.Fatal(err) + } + + a, err := archive.ExportChanges(newDir, changes) + if err != nil { + log.Fatal(err) + } + defer a.Close() + + i, err := io.Copy(os.Stdout, a) + if err != nil && err != io.EOF { + log.Fatal(err) + } + fmt.Fprintf(os.Stderr, "wrote archive of %d bytes", i) +} + +func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) { + fileData := []byte("fooo") + for n := 0; n < numberOfFiles; n++ { + fileName := fmt.Sprintf("file-%d", n) + if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil { + return 0, err + } + if makeLinks { + if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil { + return 0, err + } + } + } + totalSize := numberOfFiles * len(fileData) + return totalSize, nil +} diff --git a/vendor/github.com/containers/storage/pkg/ostree/no_ostree.go b/vendor/github.com/containers/storage/pkg/ostree/no_ostree.go new file mode 100644 index 000000000..0a8e7d679 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/ostree/no_ostree.go @@ -0,0 +1,19 @@ +// +build !ostree + +package ostree + +func OstreeSupport() bool { + return false +} + +func DeleteOSTree(repoLocation, id string) error { + return nil +} + +func CreateOSTreeRepository(repoLocation string, rootUID int, rootGID int) error { + return nil +} + +func ConvertToOSTree(repoLocation, root, id string) error { + return nil +} diff --git a/vendor/github.com/containers/storage/pkg/ostree/ostree.go b/vendor/github.com/containers/storage/pkg/ostree/ostree.go new file mode 100644 index 000000000..e9b57a0fb --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/ostree/ostree.go @@ -0,0 +1,198 @@ +// +build ostree + +package ostree + +import ( + "fmt" + "golang.org/x/sys/unix" + "os" + "path/filepath" + "runtime" + "syscall" + "time" + "unsafe" + + "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/pkg/system" + glib "github.com/ostreedev/ostree-go/pkg/glibobject" + "github.com/ostreedev/ostree-go/pkg/otbuiltin" + "github.com/pkg/errors" +) + +// #cgo pkg-config: glib-2.0 gobject-2.0 ostree-1 +// #include +// #include +// #include +// #include +// #include +// #include +import "C" + +func OstreeSupport() bool { + return true +} + +func fixFiles(dir string, usermode bool) (bool, []string, error) { + var SkipOstree = errors.New("skip ostree deduplication") + + var whiteouts []string + + err := filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + if info.Mode()&(os.ModeNamedPipe|os.ModeSocket|os.ModeDevice) != 0 { + if !usermode { + stat, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return errors.New("not syscall.Stat_t") + } + + if stat.Rdev == 0 && (stat.Mode&unix.S_IFCHR) != 0 { + whiteouts = append(whiteouts, path) + return nil + } + } + // Skip the ostree deduplication if we encounter a file type that + // ostree does not manage. + return SkipOstree + } + if info.IsDir() { + if usermode { + if err := os.Chmod(path, info.Mode()|0700); err != nil { + return err + } + } + } else if usermode && (info.Mode().IsRegular()) { + if err := os.Chmod(path, info.Mode()|0600); err != nil { + return err + } + } + return nil + }) + if err == SkipOstree { + return true, nil, nil + } + if err != nil { + return false, nil, err + } + return false, whiteouts, nil +} + +// Create prepares the filesystem for the OSTREE driver and copies the directory for the given id under the parent. +func ConvertToOSTree(repoLocation, root, id string) error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + repo, err := otbuiltin.OpenRepo(repoLocation) + if err != nil { + return errors.Wrap(err, "could not open the OSTree repository") + } + + skip, whiteouts, err := fixFiles(root, os.Getuid() != 0) + if err != nil { + return errors.Wrap(err, "could not prepare the OSTree directory") + } + if skip { + return nil + } + + if _, err := repo.PrepareTransaction(); err != nil { + return errors.Wrap(err, "could not prepare the OSTree transaction") + } + + if skip { + return nil + } + + commitOpts := otbuiltin.NewCommitOptions() + commitOpts.Timestamp = time.Now() + commitOpts.LinkCheckoutSpeedup = true + commitOpts.Parent = "0000000000000000000000000000000000000000000000000000000000000000" + branch := fmt.Sprintf("containers-storage/%s", id) + + for _, w := range whiteouts { + if err := os.Remove(w); err != nil { + return errors.Wrap(err, "could not delete whiteout file") + } + } + + if _, err := repo.Commit(root, branch, commitOpts); err != nil { + return errors.Wrap(err, "could not commit the layer") + } + + if _, err := repo.CommitTransaction(); err != nil { + return errors.Wrap(err, "could not complete the OSTree transaction") + } + + if err := system.EnsureRemoveAll(root); err != nil { + return errors.Wrap(err, "could not delete layer") + } + + checkoutOpts := otbuiltin.NewCheckoutOptions() + checkoutOpts.RequireHardlinks = true + checkoutOpts.Whiteouts = false + if err := otbuiltin.Checkout(repoLocation, root, branch, checkoutOpts); err != nil { + return errors.Wrap(err, "could not checkout from OSTree") + } + + for _, w := range whiteouts { + if err := unix.Mknod(w, unix.S_IFCHR, 0); err != nil { + return errors.Wrap(err, "could not recreate whiteout file") + } + } + return nil +} + +func CreateOSTreeRepository(repoLocation string, rootUID int, rootGID int) error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + _, err := os.Stat(repoLocation) + if err != nil && !os.IsNotExist(err) { + return err + } else if err != nil { + if err := idtools.MkdirAllAs(repoLocation, 0700, rootUID, rootGID); err != nil { + return errors.Wrap(err, "could not create OSTree repository directory: %v") + } + + if _, err := otbuiltin.Init(repoLocation, otbuiltin.NewInitOptions()); err != nil { + return errors.Wrap(err, "could not create OSTree repository") + } + } + return nil +} + +func openRepo(path string) (*C.struct_OstreeRepo, error) { + var cerr *C.GError + cpath := C.CString(path) + defer C.free(unsafe.Pointer(cpath)) + pathc := C.g_file_new_for_path(cpath) + defer C.g_object_unref(C.gpointer(pathc)) + repo := C.ostree_repo_new(pathc) + r := glib.GoBool(glib.GBoolean(C.ostree_repo_open(repo, nil, &cerr))) + if !r { + C.g_object_unref(C.gpointer(repo)) + return nil, glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr))) + } + return repo, nil +} + +func DeleteOSTree(repoLocation, id string) error { + runtime.LockOSThread() + defer runtime.UnlockOSThread() + + repo, err := openRepo(repoLocation) + if err != nil { + return err + } + defer C.g_object_unref(C.gpointer(repo)) + + branch := fmt.Sprintf("containers-storage/%s", id) + + cbranch := C.CString(branch) + defer C.free(unsafe.Pointer(cbranch)) + + var cerr *C.GError + r := glib.GoBool(glib.GBoolean(C.ostree_repo_set_ref_immediate(repo, nil, cbranch, nil, nil, &cerr))) + if !r { + return glib.ConvertGError(glib.ToGError(unsafe.Pointer(cerr))) + } + return nil +} diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 29972b690..870ccf7b0 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -2987,6 +2987,11 @@ type OptionsConfig struct { RemapGroup string `toml:"remap-group"` // Thinpool container options to be handed to thinpool drivers Thinpool struct{ ThinpoolOptionsConfig } `toml:"thinpool"` + // OSTree repository + OstreeRepo string `toml:"ostree_repo"` + + // Do not create a bind mount on the storage home + SkipMountHome string `toml:"skip_mount_home"` } // TOML-friendly explicit tables used for conversions. @@ -3077,6 +3082,12 @@ func init() { if config.Storage.Options.Size != "" { DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.size=%s", config.Storage.Driver, config.Storage.Options.Size)) } + if config.Storage.Options.OstreeRepo != "" { + DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.ostree_repo=%s", config.Storage.Driver, config.Storage.Options.OstreeRepo)) + } + if config.Storage.Options.SkipMountHome != "" { + DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.skip_mount_home=%s", config.Storage.Driver, config.Storage.Options.SkipMountHome)) + } if config.Storage.Options.OverrideKernelCheck != "" { DefaultStoreOptions.GraphDriverOptions = append(DefaultStoreOptions.GraphDriverOptions, fmt.Sprintf("%s.override_kernel_check=%s", config.Storage.Driver, config.Storage.Options.OverrideKernelCheck)) } diff --git a/vendor/github.com/containers/storage/vendor.conf b/vendor/github.com/containers/storage/vendor.conf index 54c78ab1c..c0498a02d 100644 --- a/vendor/github.com/containers/storage/vendor.conf +++ b/vendor/github.com/containers/storage/vendor.conf @@ -20,3 +20,4 @@ github.com/tchap/go-patricia v2.2.6 github.com/vbatts/tar-split v0.10.2 golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 golang.org/x/sys 07c182904dbd53199946ba614a412c61d3c548f5 +github.com/ostreedev/ostree-go aeb02c6b6aa2889db3ef62f7855650755befd460 -- cgit v1.2.3-54-g00ecf