diff options
40 files changed, 864 insertions, 103 deletions
@@ -35,7 +35,11 @@ PKG_MANAGER ?= $(shell command -v dnf yum|head -n1) # ~/.local/bin is not in PATH on all systems PRE_COMMIT = $(shell command -v bin/venv/bin/pre-commit ~/.local/bin/pre-commit pre-commit | head -n1) -SOURCES = $(shell find . -path './.*' -prune -o -name "*.go") +# This isn't what we actually build; it's a superset, used for target +# dependencies. Basically: all *.go files, except *_test.go, and except +# anything in a dot subdirectory. If any of these files is newer than +# our target (bin/podman{,-remote}), a rebuild is triggered. +SOURCES = $(shell find . -path './.*' -prune -o \( -name '*.go' -a ! -name '*_test.go' \) -print) BUILDFLAGS := -mod=vendor $(BUILDFLAGS) @@ -195,7 +199,6 @@ test/goecho/goecho: .gopathok $(wildcard test/goecho/*.go) $(GO) build $(BUILDFLAGS) -ldflags '$(LDFLAGS_PODMAN)' -o $@ ./test/goecho -.PHONY: bin/podman bin/podman: .gopathok $(SOURCES) go.mod go.sum ## Build with podman # Make sure to warn in case we're building without the systemd buildtag. ifeq (,$(findstring systemd,$(BUILDTAGS))) @@ -207,7 +210,6 @@ endif .PHONY: podman podman: bin/podman -.PHONY: bin/podman-remote bin/podman-remote: .gopathok $(SOURCES) go.mod go.sum ## Build with podman on remote environment $(GO) build $(BUILDFLAGS) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "${REMOTETAGS}" -o $@ ./cmd/podman @@ -390,7 +392,7 @@ MANPAGES_DEST ?= $(subst markdown,man, $(subst source,build,$(MANPAGES))) $(MANPAGES): %: %.md .install.md2man docdir @sed -e 's/\((podman.*\.md)\)//' -e 's/\[\(podman.*\)\]/\1/' -e 's;<\(/\)\?\(a[^>]*\|sup\)>;;g' $< | $(GOMD2MAN) -in /dev/stdin -out $(subst source/markdown,build/man,$@) -.PHONY: docs +.PHONY: docdir docdir: mkdir -p docs/build/man @@ -404,7 +406,7 @@ install-podman-remote-%-docs: podman-remote docs $(MANPAGES) docs/remote-docs.sh $* docs/build/remote/$* $(if $(findstring windows,$*),docs/source/markdown,docs/build/man) .PHONY: man-page-check -man-page-check: +man-page-check: bin/podman hack/man-page-checker hack/xref-helpmsgs-manpages diff --git a/docs/source/markdown/podman-network-create.1.md b/docs/source/markdown/podman-network-create.1.md index 86b15162a..3d5d98055 100644 --- a/docs/source/markdown/podman-network-create.1.md +++ b/docs/source/markdown/podman-network-create.1.md @@ -9,7 +9,7 @@ podman\-network-create - Create a Podman CNI network ## DESCRIPTION Create a CNI-network configuration for use with Podman. By default, Podman creates a bridge connection. A *Macvlan* connection can be created with the *-d macvlan* option. A parent device for macvlan can -be designated with the *-o parent=<device>* option. In the case of *Macvlan* connections, the +be designated with the *-o parent=\<device>* option. In the case of *Macvlan* connections, the CNI *dhcp* plugin needs to be activated or the container image must have a DHCP client to interact with the host network's DHCP server. @@ -8,14 +8,14 @@ require ( github.com/buger/goterm v0.0.0-20181115115552-c206103e1f37 github.com/checkpoint-restore/go-criu v0.0.0-20190109184317-bdb7599cd87b github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd // indirect - github.com/containernetworking/cni v0.8.0 + github.com/containernetworking/cni v0.8.1 github.com/containernetworking/plugins v0.9.0 github.com/containers/buildah v1.19.3 github.com/containers/common v0.33.1 github.com/containers/conmon v2.0.20+incompatible github.com/containers/image/v5 v5.10.1 github.com/containers/psgo v1.5.2 - github.com/containers/storage v1.24.5 + github.com/containers/storage v1.25.0 github.com/coreos/go-systemd/v22 v22.1.0 github.com/cri-o/ocicni v0.2.1-0.20201204103948-b6cbe99b9756 github.com/cyphar/filepath-securejoin v0.2.2 @@ -92,6 +92,8 @@ github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kw github.com/containernetworking/cni v0.7.2-0.20190904153231-83439463f784/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.0 h1:BT9lpgGoH4jw3lFC7Odz2prU5ruiYKcgAjMCbgybcKI= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= +github.com/containernetworking/cni v0.8.1 h1:7zpDnQ3T3s4ucOuJ/ZCLrYBxzkg0AELFfII3Epo9TmI= +github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/plugins v0.8.7/go.mod h1:R7lXeZaBzpfqapcAbHRW8/CYwm0dHzbz0XEjofx0uB0= github.com/containernetworking/plugins v0.9.0 h1:c+1gegKhR7+d0Caum9pEHugZlyhXPOG6v3V6xJgIGCI= github.com/containernetworking/plugins v0.9.0/go.mod h1:dbWv4dI0QrBGuVgj+TuVQ6wJRZVOhrCQj91YyC92sxg= @@ -114,6 +116,8 @@ github.com/containers/psgo v1.5.2/go.mod h1:2ubh0SsreMZjSXW1Hif58JrEcFudQyIy9EzP github.com/containers/storage v1.23.7/go.mod h1:cUT2zHjtx+WlVri30obWmM2gpqpi8jfPsmIzP1TVpEI= github.com/containers/storage v1.24.5 h1:BusfdU0rCS2/Daa/DPw+0iLfGRlYA7UVF7D0el3N7Vk= github.com/containers/storage v1.24.5/go.mod h1:YC+2pY8SkfEAcZkwycxYbpK8EiRbx5soPPwz9dxe4IQ= +github.com/containers/storage v1.25.0 h1:p0PLlQcWmtE+7XLfOCR0WuYyMTby1yozpI4DaKOtWTA= +github.com/containers/storage v1.25.0/go.mod h1:UxTYd5F4mPVqmDRcRL0PBS8+HP74aBn96eahnhEvPtk= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5 h1:DpHb9vJrZQEFMcVLFKAAGMUVX0XoRC0ptCthinRYm38= @@ -373,6 +377,8 @@ github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mattn/go-shellwords v1.0.11 h1:vCoR9VPpsk/TZFW2JwK5I9S0xdrtUq2bph6/YjEPnaw= +github.com/mattn/go-shellwords v1.0.11/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mdlayher/ethernet v0.0.0-20190606142754-0394541c37b7/go.mod h1:U6ZQobyTjI/tJyq2HG+i/dfSoFUt8/aZCM+GKtmFk/Y= diff --git a/pkg/api/handlers/compat/containers.go b/pkg/api/handlers/compat/containers.go index b41987800..86508f938 100644 --- a/pkg/api/handlers/compat/containers.go +++ b/pkg/api/handlers/compat/containers.go @@ -233,8 +233,8 @@ func KillContainer(w http.ResponseWriter, r *http.Request) { return } if sig == 0 || syscall.Signal(sig) == syscall.SIGKILL { - var opts entities.WaitOptions - if _, err := containerEngine.ContainerWait(r.Context(), []string{name}, opts); err != nil { + if _, err := utils.WaitContainer(w, r); err != nil { + utils.Error(w, "Something went wrong.", http.StatusInternalServerError, err) return } diff --git a/test/python/docker/test_containers.py b/test/python/docker/test_containers.py index 01e049ed4..5c2a5fef2 100644 --- a/test/python/docker/test_containers.py +++ b/test/python/docker/test_containers.py @@ -95,6 +95,15 @@ class TestContainers(unittest.TestCase): top.reload() self.assertIn(top.status, ("stopped", "exited")) + def test_kill_container(self): + top = self.client.containers.get(TestContainers.topContainerId) + self.assertEqual(top.status, "running") + + # Kill a running container and validate the state + top.kill() + top.reload() + self.assertIn(top.status, ("stopped", "exited")) + def test_restart_container(self): # Validate the container state top = self.client.containers.get(TestContainers.topContainerId) diff --git a/test/system/030-run.bats b/test/system/030-run.bats index dcf1da370..6c3812dce 100644 --- a/test/system/030-run.bats +++ b/test/system/030-run.bats @@ -610,4 +610,43 @@ json-file | f is "$output" "$randomcontent" "cat random content" } +# https://github.com/containers/podman/issues/9096 +# podman exec may truncate stdout/stderr; actually a bug in conmon: +# https://github.com/containers/conmon/issues/236 +@test "podman run - does not truncate or hang with big output" { + # Size, in bytes, to dd and to expect in return + char_count=700000 + + # Container name; primarily needed when running podman-remote + cname=mybigdatacontainer + + # This is one of those cases where BATS is not the best test framework. + # We can't do any output redirection, because 'run' overrides it so + # as to preserve $output. We can't _not_ do redirection, because BATS + # doesn't like NULs in $output (and neither would humans who might + # have to read them in an error log). + # Workaround: write to a log file, and don't attach stdout. + run_podman run --name $cname --attach stderr --log-driver k8s-file \ + $IMAGE dd if=/dev/zero count=$char_count bs=1 + is "${lines[0]}" "$char_count+0 records in" "dd: number of records in" + is "${lines[1]}" "$char_count+0 records out" "dd: number of records out" + + # We don't have many tests for '-l'. This is as good a place as any + if ! is_remote; then + cname=-l + fi + + # Now find that log file, and count the NULs in it. + # The log file is of the form '<timestamp> <P|F> <data>', where P|F + # is Partial/Full; I think that's called "kubernetes log format"? + run_podman inspect $cname --format '{{.HostConfig.LogConfig.Path}}' + logfile="$output" + + count_zero=$(tr -cd '\0' <$logfile | wc -c) + is "$count_zero" "$char_count" "count of NULL characters in log" + + # Clean up + run_podman rm $cname +} + # vim: filetype=sh diff --git a/vendor/github.com/containernetworking/cni/pkg/invoke/find.go b/vendor/github.com/containernetworking/cni/pkg/invoke/find.go index e815404c8..e62029eb7 100644 --- a/vendor/github.com/containernetworking/cni/pkg/invoke/find.go +++ b/vendor/github.com/containernetworking/cni/pkg/invoke/find.go @@ -18,6 +18,7 @@ import ( "fmt" "os" "path/filepath" + "strings" ) // FindInPath returns the full path of the plugin by searching in the provided path @@ -26,6 +27,10 @@ func FindInPath(plugin string, paths []string) (string, error) { return "", fmt.Errorf("no plugin name provided") } + if strings.ContainsRune(plugin, os.PathSeparator) { + return "", fmt.Errorf("invalid plugin name: %s", plugin) + } + if len(paths) == 0 { return "", fmt.Errorf("no paths provided") } diff --git a/vendor/github.com/containers/storage/.cirrus.yml b/vendor/github.com/containers/storage/.cirrus.yml index cc2dd7952..836bf659a 100644 --- a/vendor/github.com/containers/storage/.cirrus.yml +++ b/vendor/github.com/containers/storage/.cirrus.yml @@ -25,7 +25,7 @@ env: # GCE project where images live IMAGE_PROJECT: "libpod-218412" # VM Image built in containers/automation_images - _BUILT_IMAGE_SUFFIX: "c6233039174893568" + _BUILT_IMAGE_SUFFIX: "c5744859501821952" FEDORA_CACHE_IMAGE_NAME: "fedora-${_BUILT_IMAGE_SUFFIX}" PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${_BUILT_IMAGE_SUFFIX}" UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${_BUILT_IMAGE_SUFFIX}" diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index 6521720b4..ad2191947 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.24.5 +1.25.0 diff --git a/vendor/github.com/containers/storage/drivers/aufs/aufs.go b/vendor/github.com/containers/storage/drivers/aufs/aufs.go index c4ced0488..5310b5ccc 100644 --- a/vendor/github.com/containers/storage/drivers/aufs/aufs.go +++ b/vendor/github.com/containers/storage/drivers/aufs/aufs.go @@ -482,6 +482,21 @@ func (a *Driver) Put(id string) error { return err } +// ReadWriteDiskUsage returns the disk usage of the writable directory for the ID. +// For AUFS, it queries the mountpoint for this ID. +func (a *Driver) ReadWriteDiskUsage(id string) (*directory.DiskUsage, error) { + a.locker.Lock(id) + defer a.locker.Unlock(id) + a.pathCacheLock.Lock() + m, exists := a.pathCache[id] + if !exists { + m = a.getMountpoint(id) + a.pathCache[id] = m + } + a.pathCacheLock.Unlock() + return directory.Usage(m) +} + // isParent returns if the passed in parent is the direct parent of the passed in layer func (a *Driver) isParent(id, parent string) bool { parents, _ := getParentIDs(a.rootPath(), id) diff --git a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go index e71ad69e2..7aba695ff 100644 --- a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go +++ b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go @@ -27,6 +27,7 @@ import ( "unsafe" graphdriver "github.com/containers/storage/drivers" + "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/mount" "github.com/containers/storage/pkg/parsers" @@ -687,6 +688,12 @@ func (d *Driver) Put(id string) error { return nil } +// ReadWriteDiskUsage returns the disk usage of the writable directory for the ID. +// For BTRFS, it queries the subvolumes path for this ID. +func (d *Driver) ReadWriteDiskUsage(id string) (*directory.DiskUsage, error) { + return directory.Usage(d.subvolumesDirID(id)) +} + // Exists checks if the id exists in the filesystem. func (d *Driver) Exists(id string) bool { dir := d.subvolumesDirID(id) diff --git a/vendor/github.com/containers/storage/drivers/devmapper/driver.go b/vendor/github.com/containers/storage/drivers/devmapper/driver.go index 4149979a5..e0bef1301 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/driver.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/driver.go @@ -11,6 +11,7 @@ import ( graphdriver "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/devicemapper" + "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/locker" "github.com/containers/storage/pkg/mount" @@ -251,6 +252,14 @@ func (d *Driver) Put(id string) error { return err } +// ReadWriteDiskUsage returns the disk usage of the writable directory for the ID. +// For devmapper, it queries the mnt path for this ID. +func (d *Driver) ReadWriteDiskUsage(id string) (*directory.DiskUsage, error) { + d.locker.Lock(id) + defer d.locker.Unlock(id) + return directory.Usage(path.Join(d.home, "mnt", id)) +} + // Exists checks to see if the device exists. func (d *Driver) Exists(id string) bool { return d.DeviceSet.HasDevice(id) diff --git a/vendor/github.com/containers/storage/drivers/driver.go b/vendor/github.com/containers/storage/drivers/driver.go index 2d6485e80..0a676f6ac 100644 --- a/vendor/github.com/containers/storage/drivers/driver.go +++ b/vendor/github.com/containers/storage/drivers/driver.go @@ -12,6 +12,7 @@ import ( "github.com/vbatts/tar-split/tar/storage" "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/idtools" ) @@ -105,6 +106,8 @@ type ProtoDriver interface { // Returns a set of key-value pairs which give low level information // about the image/container driver is managing. Metadata(id string) (map[string]string, error) + // ReadWriteDiskUsage returns the disk usage of the writable directory for the specified ID. + ReadWriteDiskUsage(id string) (*directory.DiskUsage, error) // Cleanup performs necessary tasks to release resources // held by the driver, e.g., unmounting all layered filesystems // known to this driver. diff --git a/vendor/github.com/containers/storage/drivers/driver_freebsd.go b/vendor/github.com/containers/storage/drivers/driver_freebsd.go index 53394b738..e1320ee07 100644 --- a/vendor/github.com/containers/storage/drivers/driver_freebsd.go +++ b/vendor/github.com/containers/storage/drivers/driver_freebsd.go @@ -1,8 +1,6 @@ package graphdriver import ( - "syscall" - "golang.org/x/sys/unix" ) @@ -16,7 +14,7 @@ var ( // Mounted checks if the given path is mounted as the fs type func Mounted(fsType FsMagic, mountPath string) (bool, error) { var buf unix.Statfs_t - if err := syscall.Statfs(mountPath, &buf); err != nil { + if err := unix.Statfs(mountPath, &buf); err != nil { return false, err } return FsMagic(buf.Type) == fsType, nil diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 6e5a76cf3..864e0af12 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -521,6 +521,18 @@ func (d *Driver) Metadata(id string) (map[string]string, error) { return metadata, nil } +// ReadWriteDiskUsage returns the disk usage of the writable directory for the ID. +// For Overlay, it attempts to check the XFS quota for size, and falls back to +// finding the size of the "diff" directory. +func (d *Driver) ReadWriteDiskUsage(id string) (*directory.DiskUsage, error) { + usage := &directory.DiskUsage{} + if d.quotaCtl != nil { + err := d.quotaCtl.GetDiskUsage(d.dir(id), usage) + return usage, err + } + return directory.Usage(path.Join(d.dir(id), "diff")) +} + // Cleanup any state created by overlay which should be cleaned when daemon // is being shutdown. For now, we just have to unmount the bind mounted // we had created. @@ -612,17 +624,22 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts) (retErr } }() - if opts != nil && len(opts.StorageOpt) > 0 { - driver := &Driver{} - if err := d.parseStorageOpt(opts.StorageOpt, driver); err != nil { - return err - } - - if driver.options.quota.Size > 0 { - // Set container disk quota limit - if err := d.quotaCtl.SetQuota(dir, driver.options.quota); err != nil { + if d.quotaCtl != nil { + quota := quota.Quota{} + if opts != nil && len(opts.StorageOpt) > 0 { + driver := &Driver{} + if err := d.parseStorageOpt(opts.StorageOpt, driver); err != nil { return err } + if driver.options.quota.Size > 0 { + quota.Size = driver.options.quota.Size + } + + } + // Set container disk quota limit + // If it is set to 0, we will track the disk usage, but not enforce a limit + if err := d.quotaCtl.SetQuota(dir, quota); err != nil { + return err } } @@ -1221,6 +1238,7 @@ func (d *Driver) DiffSize(id string, idMappings *idtools.IDMappings, parent stri if d.useNaiveDiff() || !d.isParent(id, parent) { return d.naiveDiff.DiffSize(id, idMappings, parent, parentMappings, mountLabel) } + return directory.Size(d.getDiffPath(id)) } diff --git a/vendor/github.com/containers/storage/drivers/quota/projectquota.go b/vendor/github.com/containers/storage/drivers/quota/projectquota.go index d5aa0f891..d805623b5 100644 --- a/vendor/github.com/containers/storage/drivers/quota/projectquota.go +++ b/vendor/github.com/containers/storage/drivers/quota/projectquota.go @@ -56,6 +56,7 @@ import ( "path/filepath" "unsafe" + "github.com/containers/storage/pkg/directory" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) @@ -196,17 +197,37 @@ func setProjectQuota(backingFsBlockDev string, projectID uint32, quota Quota) er // GetQuota - get the quota limits of a directory that was configured with SetQuota func (q *Control) GetQuota(targetPath string, quota *Quota) error { + d, err := q.fsDiskQuotaFromPath(targetPath) + if err != nil { + return err + } + quota.Size = uint64(d.d_blk_hardlimit) * 512 + return nil +} + +// GetDiskUsage - get the current disk usage of a directory that was configured with SetQuota +func (q *Control) GetDiskUsage(targetPath string, usage *directory.DiskUsage) error { + d, err := q.fsDiskQuotaFromPath(targetPath) + if err != nil { + return err + } + usage.Size = int64(d.d_bcount) * 512 + usage.InodeCount = int64(d.d_icount) + + return nil +} + +func (q *Control) fsDiskQuotaFromPath(targetPath string) (C.fs_disk_quota_t, error) { + var d C.fs_disk_quota_t projectID, ok := q.quotas[targetPath] if !ok { - return fmt.Errorf("quota not found for path : %s", targetPath) + return d, fmt.Errorf("quota not found for path : %s", targetPath) } // // get the quota limit for the container's project id // - var d C.fs_disk_quota_t - var cs = C.CString(q.backingFsBlockDev) defer C.free(unsafe.Pointer(cs)) @@ -214,12 +235,11 @@ func (q *Control) GetQuota(targetPath string, quota *Quota) error { uintptr(unsafe.Pointer(cs)), uintptr(C.__u32(projectID)), uintptr(unsafe.Pointer(&d)), 0, 0) if errno != 0 { - return fmt.Errorf("Failed to get quota limit for projid %d on %s: %v", + return d, fmt.Errorf("Failed to get quota limit for projid %d on %s: %v", projectID, q.backingFsBlockDev, errno.Error()) } - quota.Size = uint64(d.d_blk_hardlimit) * 512 - return nil + return d, nil } // getProjectID - get the project id of path on xfs diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index 679d89112..ec437e14e 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -10,6 +10,7 @@ import ( graphdriver "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/parsers" "github.com/containers/storage/pkg/system" @@ -243,6 +244,12 @@ func (d *Driver) Put(id string) error { return nil } +// ReadWriteDiskUsage returns the disk usage of the writable directory for the ID. +// For VFS, it queries the directory for this ID. +func (d *Driver) ReadWriteDiskUsage(id string) (*directory.DiskUsage, error) { + return directory.Usage(d.dir(id)) +} + // Exists checks to see if the directory exists for the given id. func (d *Driver) Exists(id string) bool { _, err := os.Stat(d.dir(id)) diff --git a/vendor/github.com/containers/storage/drivers/windows/windows.go b/vendor/github.com/containers/storage/drivers/windows/windows.go index 1fd84e3b4..c8a340801 100644 --- a/vendor/github.com/containers/storage/drivers/windows/windows.go +++ b/vendor/github.com/containers/storage/drivers/windows/windows.go @@ -26,6 +26,7 @@ import ( "github.com/Microsoft/hcsshim" "github.com/containers/storage/drivers" "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/ioutils" "github.com/containers/storage/pkg/longpath" @@ -436,6 +437,12 @@ func (d *Driver) Get(id string, options graphdriver.MountOpts) (string, error) { return dir, nil } +// ReadWriteDiskUsage returns the disk usage of the writable directory for the ID. +// For VFS, it queries the directory for this ID. +func (d *Driver) ReadWriteDiskUsage(id string) (*directory.DiskUsage, error) { + return directory.Usage(d.dir(id)) +} + // Put adds a new layer to the driver. func (d *Driver) Put(id string) error { panicIfUsedByLcow() diff --git a/vendor/github.com/containers/storage/drivers/zfs/zfs.go b/vendor/github.com/containers/storage/drivers/zfs/zfs.go index 4e7290efc..185a8b074 100644 --- a/vendor/github.com/containers/storage/drivers/zfs/zfs.go +++ b/vendor/github.com/containers/storage/drivers/zfs/zfs.go @@ -13,6 +13,7 @@ import ( "time" graphdriver "github.com/containers/storage/drivers" + "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/mount" "github.com/containers/storage/pkg/parsers" @@ -455,6 +456,12 @@ func (d *Driver) Put(id string) error { return nil } +// ReadWriteDiskUsage returns the disk usage of the writable directory for the ID. +// For ZFS, it queries the full mount path for this ID. +func (d *Driver) ReadWriteDiskUsage(id string) (*directory.DiskUsage, error) { + return directory.Usage(d.mountPath(id)) +} + // Exists checks to see if the cache entry exists for the given id. func (d *Driver) Exists(id string) bool { d.Lock() diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index 8af8ceddb..78f6e71d3 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -8,9 +8,9 @@ require ( github.com/Microsoft/hcsshim v0.8.14 github.com/docker/go-units v0.4.0 github.com/hashicorp/go-multierror v1.1.0 - github.com/klauspost/compress v1.11.5 + github.com/klauspost/compress v1.11.7 github.com/klauspost/pgzip v1.2.5 - github.com/mattn/go-shellwords v1.0.10 + github.com/mattn/go-shellwords v1.0.11 github.com/mistifyio/go-zfs v2.1.1+incompatible github.com/moby/sys/mountinfo v0.4.0 github.com/opencontainers/go-digest v1.0.0 @@ -20,7 +20,7 @@ require ( github.com/pkg/errors v0.9.1 github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 github.com/sirupsen/logrus v1.7.0 - github.com/stretchr/testify v1.6.1 + github.com/stretchr/testify v1.7.0 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 github.com/tchap/go-patricia v2.3.0+incompatible github.com/vbatts/tar-split v0.11.1 diff --git a/vendor/github.com/containers/storage/go.sum b/vendor/github.com/containers/storage/go.sum index c786686bc..233191548 100644 --- a/vendor/github.com/containers/storage/go.sum +++ b/vendor/github.com/containers/storage/go.sum @@ -58,8 +58,8 @@ github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.11.5 h1:xNCE0uE6yvTPRS+0wGNMHPo3NIpwnk6aluQZ6R6kRcc= -github.com/klauspost/compress v1.11.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.7 h1:0hzRabrMN4tSTvMfnL3SCv1ZGeAP23ynzodBgaHeMeg= +github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -71,8 +71,8 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= -github.com/mattn/go-shellwords v1.0.10 h1:Y7Xqm8piKOO3v10Thp7Z36h4FYFjt5xB//6XvOrs2Gw= -github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mattn/go-shellwords v1.0.11 h1:vCoR9VPpsk/TZFW2JwK5I9S0xdrtUq2bph6/YjEPnaw= +github.com/mattn/go-shellwords v1.0.11/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/mistifyio/go-zfs v2.1.1+incompatible h1:gAMO1HM9xBRONLHHYnu5iFsOJUiJdNZo6oqSENd4eW8= github.com/mistifyio/go-zfs v2.1.1+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/moby/sys/mountinfo v0.1.3 h1:KIrhRO14+AkwKvG/g2yIpNMOUVZ02xNhOw8KY1WsLOI= @@ -118,8 +118,8 @@ github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= -github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tchap/go-patricia v2.3.0+incompatible h1:GkY4dP3cEfEASBPPkWd+AmjYxhmDkqO9/zg7R0lSQRs= diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index 72f00b8d6..ce059318d 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -117,6 +117,11 @@ type Layer struct { // ReadOnly is true if this layer resides in a read-only layer store. ReadOnly bool `json:"-"` + + // BigDataNames is a list of names of data items that we keep for the + // convenience of the caller. They can be large, and are only in + // memory when being read from or written to disk. + BigDataNames []string `json:"big-data-names,omitempty"` } type layerMountPoint struct { @@ -137,6 +142,7 @@ type DiffOptions struct { type ROLayerStore interface { ROFileBasedStore ROMetadataStore + ROLayerBigDataStore // Exists checks if a layer with the specified name or ID is known. Exists(id string) bool @@ -194,6 +200,7 @@ type LayerStore interface { RWFileBasedStore RWMetadataStore FlaggableStore + RWLayerBigDataStore // Create creates a new layer, optionally giving it a specified ID rather than // a randomly-generated one, either inheriting data from another specified @@ -278,6 +285,7 @@ func copyLayer(l *Layer) *Layer { UncompressedSize: l.UncompressedSize, CompressionType: l.CompressionType, ReadOnly: l.ReadOnly, + BigDataNames: copyStringSlice(l.BigDataNames), Flags: copyStringInterfaceMap(l.Flags), UIDMap: copyIDMap(l.UIDMap), GIDMap: copyIDMap(l.GIDMap), @@ -694,14 +702,15 @@ func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLab } if err == nil { layer = &Layer{ - ID: id, - Parent: parent, - Names: names, - MountLabel: mountLabel, - Created: time.Now().UTC(), - Flags: make(map[string]interface{}), - UIDMap: copyIDMap(moreOptions.UIDMap), - GIDMap: copyIDMap(moreOptions.GIDMap), + ID: id, + Parent: parent, + Names: names, + MountLabel: mountLabel, + Created: time.Now().UTC(), + Flags: make(map[string]interface{}), + UIDMap: copyIDMap(moreOptions.UIDMap), + GIDMap: copyIDMap(moreOptions.GIDMap), + BigDataNames: []string{}, } r.layers = append(r.layers, layer) r.idindex.Add(id) @@ -970,6 +979,80 @@ func (r *layerStore) SetNames(id string, names []string) error { return ErrLayerUnknown } +func (r *layerStore) datadir(id string) string { + return filepath.Join(r.layerdir, id) +} + +func (r *layerStore) datapath(id, key string) string { + return filepath.Join(r.datadir(id), makeBigDataBaseName(key)) +} + +func (r *layerStore) BigData(id, key string) (io.ReadCloser, error) { + if key == "" { + return nil, errors.Wrapf(ErrInvalidBigDataName, "can't retrieve layer big data value for empty name") + } + layer, ok := r.lookup(id) + if !ok { + return nil, errors.Wrapf(ErrLayerUnknown, "error locating layer with ID %q", id) + } + return os.Open(r.datapath(layer.ID, key)) +} + +func (r *layerStore) SetBigData(id, key string, data io.Reader) error { + if key == "" { + return errors.Wrapf(ErrInvalidBigDataName, "can't set empty name for layer big data item") + } + if !r.IsReadWrite() { + return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to save data items associated with layers at %q", r.layerspath()) + } + layer, ok := r.lookup(id) + if !ok { + return errors.Wrapf(ErrLayerUnknown, "error locating layer with ID %q to write bigdata", id) + } + err := os.MkdirAll(r.datadir(layer.ID), 0700) + if err != nil { + return err + } + + // NewAtomicFileWriter doesn't overwrite/truncate the existing inode. + // BigData() relies on this behaviour when opening the file for read + // so that it is either accessing the old data or the new one. + writer, err := ioutils.NewAtomicFileWriter(r.datapath(layer.ID, key), 0600) + if err != nil { + return errors.Wrapf(err, "error opening bigdata file") + } + + if _, err := io.Copy(writer, data); err != nil { + writer.Close() + return errors.Wrapf(err, "error copying bigdata for the layer") + + } + if err := writer.Close(); err != nil { + return errors.Wrapf(err, "error closing bigdata file for the layer") + } + + addName := true + for _, name := range layer.BigDataNames { + if name == key { + addName = false + break + } + } + if addName { + layer.BigDataNames = append(layer.BigDataNames, key) + return r.Save() + } + return nil +} + +func (r *layerStore) BigDataNames(id string) ([]string, error) { + layer, ok := r.lookup(id) + if !ok { + return nil, errors.Wrapf(ErrImageUnknown, "error locating layer with ID %q to retrieve bigdata names", id) + } + return copyStringSlice(layer.BigDataNames), nil +} + func (r *layerStore) Metadata(id string) (string, error) { if layer, ok := r.lookup(id); ok { return layer.Metadata, nil @@ -1004,6 +1087,7 @@ func (r *layerStore) deleteInternal(id string) error { err := r.driver.Remove(id) if err == nil { os.Remove(r.tspath(id)) + os.RemoveAll(r.datadir(id)) delete(r.byid, id) for _, name := range layer.Names { delete(r.byname, name) diff --git a/vendor/github.com/containers/storage/layers_ffjson.go b/vendor/github.com/containers/storage/layers_ffjson.go index 3a1095226..f61e68a21 100644 --- a/vendor/github.com/containers/storage/layers_ffjson.go +++ b/vendor/github.com/containers/storage/layers_ffjson.go @@ -396,6 +396,22 @@ func (j *Layer) MarshalJSONBuf(buf fflib.EncodingBuffer) error { } buf.WriteByte(',') } + if len(j.BigDataNames) != 0 { + buf.WriteString(`"big-data-names":`) + if j.BigDataNames != nil { + buf.WriteString(`[`) + for i, v := range j.BigDataNames { + if i != 0 { + buf.WriteString(`,`) + } + fflib.WriteJsonString(buf, string(v)) + } + buf.WriteString(`]`) + } else { + buf.WriteString(`null`) + } + buf.WriteByte(',') + } buf.Rewind(1) buf.WriteByte('}') return nil @@ -436,6 +452,8 @@ const ( ffjtLayerUIDMap ffjtLayerGIDMap + + ffjtLayerBigDataNames ) var ffjKeyLayerID = []byte("id") @@ -470,6 +488,8 @@ var ffjKeyLayerUIDMap = []byte("uidmap") var ffjKeyLayerGIDMap = []byte("gidmap") +var ffjKeyLayerBigDataNames = []byte("big-data-names") + // UnmarshalJSON umarshall json - template of ffjson func (j *Layer) UnmarshalJSON(input []byte) error { fs := fflib.NewFFLexer(input) @@ -531,6 +551,14 @@ mainparse: } else { switch kn[0] { + case 'b': + + if bytes.Equal(ffjKeyLayerBigDataNames, kn) { + currentKey = ffjtLayerBigDataNames + state = fflib.FFParse_want_colon + goto mainparse + } + case 'c': if bytes.Equal(ffjKeyLayerCreated, kn) { @@ -640,6 +668,12 @@ mainparse: } + if fflib.EqualFoldRight(ffjKeyLayerBigDataNames, kn) { + currentKey = ffjtLayerBigDataNames + state = fflib.FFParse_want_colon + goto mainparse + } + if fflib.SimpleLetterEqualFold(ffjKeyLayerGIDMap, kn) { currentKey = ffjtLayerGIDMap state = fflib.FFParse_want_colon @@ -801,6 +835,9 @@ mainparse: case ffjtLayerGIDMap: goto handle_GIDMap + case ffjtLayerBigDataNames: + goto handle_BigDataNames + case ffjtLayernosuchkey: err = fs.SkipField(tok) if err != nil { @@ -1551,6 +1588,80 @@ handle_GIDMap: state = fflib.FFParse_after_value goto mainparse +handle_BigDataNames: + + /* handler: j.BigDataNames type=[]string kind=slice quoted=false*/ + + { + + { + if tok != fflib.FFTok_left_brace && tok != fflib.FFTok_null { + return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for ", tok)) + } + } + + if tok == fflib.FFTok_null { + j.BigDataNames = nil + } else { + + j.BigDataNames = []string{} + + wantVal := true + + for { + + var tmpJBigDataNames string + + tok = fs.Scan() + if tok == fflib.FFTok_error { + goto tokerror + } + if tok == fflib.FFTok_right_brace { + break + } + + if tok == fflib.FFTok_comma { + if wantVal == true { + // TODO(pquerna): this isn't an ideal error message, this handles + // things like [,,,] as an array value. + return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) + } + continue + } else { + wantVal = true + } + + /* handler: tmpJBigDataNames type=string kind=string quoted=false*/ + + { + + { + if tok != fflib.FFTok_string && tok != fflib.FFTok_null { + return fs.WrapErr(fmt.Errorf("cannot unmarshal %s into Go value for string", tok)) + } + } + + if tok == fflib.FFTok_null { + + } else { + + outBuf := fs.Output.Bytes() + + tmpJBigDataNames = string(string(outBuf)) + + } + } + + j.BigDataNames = append(j.BigDataNames, tmpJBigDataNames) + + wantVal = false + } + } + } + + state = fflib.FFParse_after_value + goto mainparse + wantedvalue: return fs.WrapErr(fmt.Errorf("wanted value token, but got token: %v", tok)) wrongtokenerror: diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_freebsd.go b/vendor/github.com/containers/storage/pkg/archive/archive_freebsd.go new file mode 100644 index 000000000..1953b4051 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/archive/archive_freebsd.go @@ -0,0 +1,125 @@ +// +build freebsd + +package archive + +import ( + "archive/tar" + "errors" + "os" + "path/filepath" + "syscall" + + "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/pkg/system" + rsystem "github.com/opencontainers/runc/libcontainer/system" + "golang.org/x/sys/unix" +) + +// fixVolumePathPrefix does platform specific processing to ensure that if +// the path being passed in is not in a volume path format, convert it to one. +func fixVolumePathPrefix(srcPath string) string { + return srcPath +} + +// getWalkRoot calculates the root path when performing a TarWithOptions. +// We use a separate function as this is platform specific. On Linux, we +// can't use filepath.Join(srcPath,include) because this will clean away +// a trailing "." or "/" which may be important. +func getWalkRoot(srcPath string, include string) string { + return srcPath + string(filepath.Separator) + include +} + +// CanonicalTarNameForPath returns platform-specific filepath +// to canonical posix-style path for tar archival. p is relative +// path. +func CanonicalTarNameForPath(p string) (string, error) { + return p, nil // already unix-style +} + +// chmodTarEntry is used to adjust the file permissions used in tar header based +// on the platform the archival is done. +func chmodTarEntry(perm os.FileMode) os.FileMode { + return perm // noop for unix as golang APIs provide perm bits correctly +} + +func setHeaderForSpecialDevice(hdr *tar.Header, name string, stat interface{}) (err error) { + s, ok := stat.(*syscall.Stat_t) + + if ok { + // Currently go does not fill in the major/minors + if s.Mode&unix.S_IFBLK != 0 || + s.Mode&unix.S_IFCHR != 0 { + hdr.Devmajor = int64(major(uint64(s.Rdev))) // nolint: unconvert + hdr.Devminor = int64(minor(uint64(s.Rdev))) // nolint: unconvert + } + } + + return +} + +func getInodeFromStat(stat interface{}) (inode uint64, err error) { + s, ok := stat.(*syscall.Stat_t) + + if ok { + inode = s.Ino + } + + return +} + +func getFileUIDGID(stat interface{}) (idtools.IDPair, error) { + s, ok := stat.(*syscall.Stat_t) + + if !ok { + return idtools.IDPair{}, errors.New("cannot convert stat value to syscall.Stat_t") + } + return idtools.IDPair{UID: int(s.Uid), GID: int(s.Gid)}, nil +} + +func major(device uint64) uint64 { + return (device >> 8) & 0xfff +} + +func minor(device uint64) uint64 { + return (device & 0xff) | ((device >> 12) & 0xfff00) +} + +// handleTarTypeBlockCharFifo is an OS-specific helper function used by +// createTarFile to handle the following types of header: Block; Char; Fifo +func handleTarTypeBlockCharFifo(hdr *tar.Header, path string) error { + if rsystem.RunningInUserNS() { + // cannot create a device if running in user namespace + return nil + } + + mode := uint32(hdr.Mode & 07777) + switch hdr.Typeflag { + case tar.TypeBlock: + mode |= unix.S_IFBLK + case tar.TypeChar: + mode |= unix.S_IFCHR + case tar.TypeFifo: + mode |= unix.S_IFIFO + } + + return system.Mknod(path, mode, uint64(system.Mkdev(hdr.Devmajor, hdr.Devminor))) +} + +func handleLChmod(hdr *tar.Header, path string, hdrInfo os.FileInfo, forceMask *os.FileMode) error { + permissionsMask := hdrInfo.Mode() + if forceMask != nil { + permissionsMask = *forceMask + } + if hdr.Typeflag == tar.TypeLink { + if fi, err := os.Lstat(hdr.Linkname); err == nil && (fi.Mode()&os.ModeSymlink == 0) { + if err := os.Chmod(path, permissionsMask); err != nil { + return err + } + } + } else if hdr.Typeflag != tar.TypeSymlink { + if err := os.Chmod(path, permissionsMask); err != nil { + return err + } + } + return nil +} diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_unix.go b/vendor/github.com/containers/storage/pkg/archive/archive_unix.go index ecb704b64..5438700ab 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive_unix.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive_unix.go @@ -1,4 +1,4 @@ -// +build !windows +// +build !windows,!freebsd package archive diff --git a/vendor/github.com/containers/storage/pkg/directory/directory.go b/vendor/github.com/containers/storage/pkg/directory/directory.go index 1715ef45d..b0ce706e5 100644 --- a/vendor/github.com/containers/storage/pkg/directory/directory.go +++ b/vendor/github.com/containers/storage/pkg/directory/directory.go @@ -6,9 +6,15 @@ import ( "path/filepath" ) +// DiskUsage is a structure that describes the disk usage (size and inode count) +// of a particular directory. +type DiskUsage struct { + Size int64 + InodeCount int64 +} + // MoveToSubdir moves all contents of a directory to a subdirectory underneath the original path func MoveToSubdir(oldpath, subdir string) error { - infos, err := ioutil.ReadDir(oldpath) if err != nil { return err diff --git a/vendor/github.com/containers/storage/pkg/directory/directory_unix.go b/vendor/github.com/containers/storage/pkg/directory/directory_unix.go index 05522d68b..245a70c00 100644 --- a/vendor/github.com/containers/storage/pkg/directory/directory_unix.go +++ b/vendor/github.com/containers/storage/pkg/directory/directory_unix.go @@ -10,37 +10,50 @@ import ( // Size walks a directory tree and returns its total size in bytes. func Size(dir string) (size int64, err error) { + usage, err := Usage(dir) + if err != nil { + return 0, err + } + return usage.Size, nil +} + +// Usage walks a directory tree and returns its total size in bytes and the number of inodes. +func Usage(dir string) (usage *DiskUsage, err error) { + usage = &DiskUsage{} data := make(map[uint64]struct{}) err = filepath.Walk(dir, func(d string, fileInfo os.FileInfo, err error) error { if err != nil { - // if dir does not exist, Size() returns the error. - // if dir/x disappeared while walking, Size() ignores dir/x. + // if dir does not exist, Usage() returns the error. + // if dir/x disappeared while walking, Usage() ignores dir/x. if os.IsNotExist(err) && d != dir { return nil } return err } - // Ignore directory sizes if fileInfo == nil { return nil } - s := fileInfo.Size() - if fileInfo.IsDir() || s == 0 { - return nil - } - // Check inode to handle hard links correctly inode := fileInfo.Sys().(*syscall.Stat_t).Ino // inode is not a uint64 on all platforms. Cast it to avoid issues. if _, exists := data[uint64(inode)]; exists { return nil } + // inode is not a uint64 on all platforms. Cast it to avoid issues. data[uint64(inode)] = struct{}{} - size += s + // Count the unique inode + usage.InodeCount++ + + // Ignore directory sizes + if fileInfo.IsDir() { + return nil + } + + usage.Size += fileInfo.Size() return nil }) diff --git a/vendor/github.com/containers/storage/pkg/directory/directory_windows.go b/vendor/github.com/containers/storage/pkg/directory/directory_windows.go index 6fb0917c4..4ff98ddf4 100644 --- a/vendor/github.com/containers/storage/pkg/directory/directory_windows.go +++ b/vendor/github.com/containers/storage/pkg/directory/directory_windows.go @@ -7,8 +7,18 @@ import ( "path/filepath" ) -// Size walks a directory tree and returns its total size in bytes. +// Size walks a directory tree and returns its total size in bytes func Size(dir string) (size int64, err error) { + usage, err := Usage(dir) + if err != nil { + return 0, nil + } + return usage.Size, nil +} + +// Usage walks a directory tree and returns its total size in bytes and the number of inodes. +func Usage(dir string) (usage *DiskUsage, err error) { + usage = &DiskUsage{} err = filepath.Walk(dir, func(d string, fileInfo os.FileInfo, err error) error { if err != nil { // if dir does not exist, Size() returns the error. @@ -29,7 +39,8 @@ func Size(dir string) (size int64, err error) { return nil } - size += s + usage.Size += s + usage.InodeCount++ return nil }) diff --git a/vendor/github.com/containers/storage/pkg/homedir/homedir_others.go b/vendor/github.com/containers/storage/pkg/homedir/homedir_others.go index 4f778c858..607c5feb5 100644 --- a/vendor/github.com/containers/storage/pkg/homedir/homedir_others.go +++ b/vendor/github.com/containers/storage/pkg/homedir/homedir_others.go @@ -1,4 +1,4 @@ -// +build !linux,!darwin +// +build !linux,!darwin,!freebsd package homedir diff --git a/vendor/github.com/containers/storage/pkg/mount/mounter_unsupported.go b/vendor/github.com/containers/storage/pkg/mount/mounter_unsupported.go index 42d1d422c..9d20cfbf8 100644 --- a/vendor/github.com/containers/storage/pkg/mount/mounter_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/mount/mounter_unsupported.go @@ -1,4 +1,4 @@ -// +build !linux +// +build !linux,!freebsd package mount diff --git a/vendor/github.com/containers/storage/pkg/system/mknod.go b/vendor/github.com/containers/storage/pkg/system/mknod.go index af79a6538..c276ce8e8 100644 --- a/vendor/github.com/containers/storage/pkg/system/mknod.go +++ b/vendor/github.com/containers/storage/pkg/system/mknod.go @@ -1,4 +1,4 @@ -// +build !windows +// +build !windows,!freebsd package system diff --git a/vendor/github.com/containers/storage/pkg/system/mknod_freebsd.go b/vendor/github.com/containers/storage/pkg/system/mknod_freebsd.go new file mode 100644 index 000000000..d09005589 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/system/mknod_freebsd.go @@ -0,0 +1,22 @@ +// +build freebsd + +package system + +import ( + "golang.org/x/sys/unix" +) + +// Mknod creates a filesystem node (file, device special file or named pipe) named path +// with attributes specified by mode and dev. +func Mknod(path string, mode uint32, dev uint64) error { + return unix.Mknod(path, mode, dev) +} + +// Mkdev is used to build the value of linux devices (in /dev/) which specifies major +// and minor number of the newly created device special file. +// Linux device nodes are a bit weird due to backwards compat with 16 bit device nodes. +// They are, from low to high: the lower 8 bits of the minor, then 12 bits of the major, +// then the top 12 bits of the minor. +func Mkdev(major int64, minor int64) uint32 { + return uint32(((minor & 0xfff00) << 12) | ((major & 0xfff) << 8) | (minor & 0xff)) +} diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index fa595355d..e9c3de5fa 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -122,6 +122,30 @@ type ContainerBigDataStore interface { SetBigData(id, key string, data []byte) error } +// A ROLayerBigDataStore wraps up how we store RO big-data associated with layers. +type ROLayerBigDataStore interface { + // SetBigData stores a (potentially large) piece of data associated + // with this ID. + BigData(id, key string) (io.ReadCloser, error) + + // BigDataNames() returns a list of the names of previously-stored pieces of + // data. + BigDataNames(id string) ([]string, error) +} + +// A RWLayerBigDataStore wraps up how we store big-data associated with layers. +type RWLayerBigDataStore interface { + // SetBigData stores a (potentially large) piece of data associated + // with this ID. + SetBigData(id, key string, data io.Reader) error +} + +// A LayerBigDataStore wraps up how we store big-data associated with layers. +type LayerBigDataStore interface { + ROLayerBigDataStore + RWLayerBigDataStore +} + // A FlaggableStore can have flags set and cleared on items which it manages. type FlaggableStore interface { // ClearFlag removes a named flag from an item in the store. @@ -385,6 +409,18 @@ type Store interface { // allow ImagesByDigest to find images by their correct digests. SetImageBigData(id, key string, data []byte, digestManifest func([]byte) (digest.Digest, error)) error + // ListLayerBigData retrieves a list of the (possibly large) chunks of + // named data associated with an layer. + ListLayerBigData(id string) ([]string, error) + + // LayerBigData retrieves a (possibly large) chunk of named data + // associated with a layer. + LayerBigData(id, key string) (io.ReadCloser, error) + + // SetLayerBigData stores a (possibly large) chunk of named data + // associated with a layer. + SetLayerBigData(id, key string, data io.Reader) error + // ImageSize computes the size of the image's layers and ancillary data. ImageSize(id string) (int64, error) @@ -1627,6 +1663,95 @@ func (s *store) ImageBigData(id, key string) ([]byte, error) { return nil, errors.Wrapf(ErrImageUnknown, "error locating image with ID %q", id) } +// ListLayerBigData retrieves a list of the (possibly large) chunks of +// named data associated with an layer. +func (s *store) ListLayerBigData(id string) ([]string, error) { + lstore, err := s.LayerStore() + if err != nil { + return nil, err + } + lstores, err := s.ROLayerStores() + if err != nil { + return nil, err + } + foundLayer := false + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s + store.RLock() + defer store.Unlock() + if modified, err := store.Modified(); modified || err != nil { + if err = store.Load(); err != nil { + return nil, err + } + } + data, err := store.BigDataNames(id) + if err == nil { + return data, nil + } + if store.Exists(id) { + foundLayer = true + } + } + if foundLayer { + return nil, errors.Wrapf(os.ErrNotExist, "error locating big data for layer with ID %q", id) + } + return nil, errors.Wrapf(ErrLayerUnknown, "error locating layer with ID %q", id) +} + +// LayerBigData retrieves a (possibly large) chunk of named data +// associated with a layer. +func (s *store) LayerBigData(id, key string) (io.ReadCloser, error) { + lstore, err := s.LayerStore() + if err != nil { + return nil, err + } + lstores, err := s.ROLayerStores() + if err != nil { + return nil, err + } + foundLayer := false + for _, s := range append([]ROLayerStore{lstore}, lstores...) { + store := s + store.RLock() + defer store.Unlock() + if modified, err := store.Modified(); modified || err != nil { + if err = store.Load(); err != nil { + return nil, err + } + } + data, err := store.BigData(id, key) + if err == nil { + return data, nil + } + if store.Exists(id) { + foundLayer = true + } + } + if foundLayer { + return nil, errors.Wrapf(os.ErrNotExist, "error locating item named %q for layer with ID %q", key, id) + } + return nil, errors.Wrapf(ErrLayerUnknown, "error locating layer with ID %q", id) +} + +// SetLayerBigData stores a (possibly large) chunk of named data +// associated with a layer. +func (s *store) SetLayerBigData(id, key string, data io.Reader) error { + store, err := s.LayerStore() + if err != nil { + return err + } + + store.Lock() + defer store.Unlock() + if modified, err := store.Modified(); modified || err != nil { + if err = store.Load(); err != nil { + return nil + } + } + + return store.SetBigData(id, key, data) +} + func (s *store) SetImageBigData(id, key string, data []byte, digestManifest func([]byte) (digest.Digest, error)) error { ristore, err := s.ImageStore() if err != nil { diff --git a/vendor/github.com/mattn/go-shellwords/.travis.yml b/vendor/github.com/mattn/go-shellwords/.travis.yml index b2904bffc..ebd5edd89 100644 --- a/vendor/github.com/mattn/go-shellwords/.travis.yml +++ b/vendor/github.com/mattn/go-shellwords/.travis.yml @@ -1,3 +1,6 @@ +arch: + - amd64 + - ppc64le language: go sudo: false go: diff --git a/vendor/github.com/mattn/go-shellwords/README.md b/vendor/github.com/mattn/go-shellwords/README.md index e91902f40..bdd531918 100644 --- a/vendor/github.com/mattn/go-shellwords/README.md +++ b/vendor/github.com/mattn/go-shellwords/README.md @@ -2,7 +2,8 @@ [](https://codecov.io/gh/mattn/go-shellwords) [](https://travis-ci.org/mattn/go-shellwords) -[](http://godoc.org/github.com/mattn/go-shellwords) +[](https://pkg.go.dev/github.com/mattn/go-shellwords) +[](https://github.com/mattn/go-shellwords/actions) Parse line as shell words. @@ -14,6 +15,12 @@ args, err := shellwords.Parse("./foo --bar=baz") ``` ```go +envs, args, err := shellwords.ParseWithEnvs("FOO=foo BAR=baz ./foo --bar=baz") +// envs should be ["FOO=foo", "BAR=baz"] +// args should be ["./foo", "--bar=baz"] +``` + +```go os.Setenv("FOO", "bar") p := shellwords.NewParser() p.ParseEnv = true diff --git a/vendor/github.com/mattn/go-shellwords/shellwords.go b/vendor/github.com/mattn/go-shellwords/shellwords.go index ef080861a..01afd94d1 100644 --- a/vendor/github.com/mattn/go-shellwords/shellwords.go +++ b/vendor/github.com/mattn/go-shellwords/shellwords.go @@ -1,10 +1,11 @@ package shellwords import ( + "bytes" "errors" "os" - "regexp" "strings" + "unicode" ) var ( @@ -12,8 +13,6 @@ var ( ParseBacktick bool = false ) -var envRe = regexp.MustCompile(`\$({[a-zA-Z0-9_]+}|[a-zA-Z0-9_]+)`) - func isSpace(r rune) bool { switch r { case ' ', '\t', '\r', '\n': @@ -27,13 +26,72 @@ func replaceEnv(getenv func(string) string, s string) string { getenv = os.Getenv } - return envRe.ReplaceAllStringFunc(s, func(s string) string { - s = s[1:] - if s[0] == '{' { - s = s[1 : len(s)-1] + var buf bytes.Buffer + rs := []rune(s) + for i := 0; i < len(rs); i++ { + r := rs[i] + if r == '\\' { + i++ + if i == len(rs) { + break + } + buf.WriteRune(rs[i]) + continue + } else if r == '$' { + i++ + if i == len(rs) { + buf.WriteRune(r) + break + } + if rs[i] == 0x7b { + i++ + p := i + for ; i < len(rs); i++ { + r = rs[i] + if r == '\\' { + i++ + if i == len(rs) { + return s + } + continue + } + if r == 0x7d || (!unicode.IsLetter(r) && r != '_' && !unicode.IsDigit(r)) { + break + } + } + if r != 0x7d { + return s + } + if i > p { + buf.WriteString(getenv(s[p:i])) + } + } else { + p := i + for ; i < len(rs); i++ { + r := rs[i] + if r == '\\' { + i++ + if i == len(rs) { + return s + } + continue + } + if !unicode.IsLetter(r) && r != '_' && !unicode.IsDigit(r) { + break + } + } + if i > p { + buf.WriteString(getenv(s[p:i])) + i-- + } else { + buf.WriteString(s[p:]) + } + } + } else { + buf.WriteRune(r) } - return getenv(s) - }) + } + return buf.String() } type Parser struct { @@ -56,6 +114,14 @@ func NewParser() *Parser { } } +type argType int + +const ( + argNo argType = iota + argSingle + argQuoted +) + func (p *Parser) Parse(line string) ([]string, error) { args := []string{} buf := "" @@ -63,13 +129,16 @@ func (p *Parser) Parse(line string) ([]string, error) { backtick := "" pos := -1 - got := false + got := argNo + i := -1 loop: - for i, r := range line { + for _, r := range line { + i++ if escaped { buf += string(r) escaped = false + got = argSingle continue } @@ -86,21 +155,23 @@ loop: if singleQuoted || doubleQuoted || backQuote || dollarQuote { buf += string(r) backtick += string(r) - } else if got { + } else if got != argNo { if p.ParseEnv { - parser := &Parser{ParseEnv: false, ParseBacktick: false, Position: 0, Dir: p.Dir} - strs, err := parser.Parse(replaceEnv(p.Getenv, buf)) - if err != nil { - return nil, err - } - for _, str := range strs { - args = append(args, str) + if got == argSingle { + parser := &Parser{ParseEnv: false, ParseBacktick: false, Position: 0, Dir: p.Dir} + strs, err := parser.Parse(replaceEnv(p.Getenv, buf)) + if err != nil { + return nil, err + } + args = append(args, strs...) + } else { + args = append(args, replaceEnv(p.Getenv, buf)) } } else { args = append(args, buf) } buf = "" - got = false + got = argNo } continue } @@ -153,7 +224,7 @@ loop: case '"': if !singleQuoted && !dollarQuote { if doubleQuoted { - got = true + got = argQuoted } doubleQuoted = !doubleQuoted continue @@ -161,7 +232,7 @@ loop: case '\'': if !doubleQuoted && !dollarQuote { if singleQuoted { - got = true + got = argSingle } singleQuoted = !singleQuoted continue @@ -171,7 +242,7 @@ loop: if r == '>' && len(buf) > 0 { if c := buf[0]; '0' <= c && c <= '9' { i -= 1 - got = false + got = argNo } } pos = i @@ -179,22 +250,24 @@ loop: } } - got = true + got = argSingle buf += string(r) if backQuote || dollarQuote { backtick += string(r) } } - if got { + if got != argNo { if p.ParseEnv { - parser := &Parser{ParseEnv: false, ParseBacktick: false, Position: 0, Dir: p.Dir} - strs, err := parser.Parse(replaceEnv(p.Getenv, buf)) - if err != nil { - return nil, err - } - for _, str := range strs { - args = append(args, str) + if got == argSingle { + parser := &Parser{ParseEnv: false, ParseBacktick: false, Position: 0, Dir: p.Dir} + strs, err := parser.Parse(replaceEnv(p.Getenv, buf)) + if err != nil { + return nil, err + } + args = append(args, strs...) + } else { + args = append(args, replaceEnv(p.Getenv, buf)) } } else { args = append(args, buf) @@ -210,6 +283,35 @@ loop: return args, nil } +func (p *Parser) ParseWithEnvs(line string) (envs []string, args []string, err error) { + _args, err := p.Parse(line) + if err != nil { + return nil, nil, err + } + envs = []string{} + args = []string{} + parsingEnv := true + for _, arg := range _args { + if parsingEnv && isEnv(arg) { + envs = append(envs, arg) + } else { + if parsingEnv { + parsingEnv = false + } + args = append(args, arg) + } + } + return envs, args, nil +} + +func isEnv(arg string) bool { + return len(strings.Split(arg, "=")) == 2 +} + func Parse(line string) ([]string, error) { return NewParser().Parse(line) } + +func ParseWithEnvs(line string) (envs []string, args []string, err error) { + return NewParser().ParseWithEnvs(line) +} diff --git a/vendor/github.com/mattn/go-shellwords/util_posix.go b/vendor/github.com/mattn/go-shellwords/util_posix.go index 988fc9ed2..b56a90120 100644 --- a/vendor/github.com/mattn/go-shellwords/util_posix.go +++ b/vendor/github.com/mattn/go-shellwords/util_posix.go @@ -3,7 +3,7 @@ package shellwords import ( - "errors" + "fmt" "os" "os/exec" "strings" @@ -23,7 +23,7 @@ func shellRun(line, dir string) (string, error) { if eerr, ok := err.(*exec.ExitError); ok { b = eerr.Stderr } - return "", errors.New(err.Error() + ":" + string(b)) + return "", fmt.Errorf("%s: %w", string(b), err) } return strings.TrimSpace(string(b)), nil } diff --git a/vendor/github.com/mattn/go-shellwords/util_windows.go b/vendor/github.com/mattn/go-shellwords/util_windows.go index 20546737c..fd738a721 100644 --- a/vendor/github.com/mattn/go-shellwords/util_windows.go +++ b/vendor/github.com/mattn/go-shellwords/util_windows.go @@ -3,7 +3,7 @@ package shellwords import ( - "errors" + "fmt" "os" "os/exec" "strings" @@ -23,7 +23,7 @@ func shellRun(line, dir string) (string, error) { if eerr, ok := err.(*exec.ExitError); ok { b = eerr.Stderr } - return "", errors.New(err.Error() + ":" + string(b)) + return "", fmt.Errorf("%s: %w", string(b), err) } return strings.TrimSpace(string(b)), nil } diff --git a/vendor/modules.txt b/vendor/modules.txt index 4b0beca94..ef33a0dcc 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -55,7 +55,7 @@ github.com/containerd/containerd/sys # github.com/containerd/continuity v0.0.0-20200413184840-d3ef23f19fbb github.com/containerd/continuity/fs github.com/containerd/continuity/sysx -# github.com/containernetworking/cni v0.8.0 +# github.com/containernetworking/cni v0.8.1 github.com/containernetworking/cni/libcni github.com/containernetworking/cni/pkg/invoke github.com/containernetworking/cni/pkg/types @@ -173,7 +173,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.24.5 +# github.com/containers/storage v1.25.0 github.com/containers/storage github.com/containers/storage/drivers github.com/containers/storage/drivers/aufs @@ -370,7 +370,7 @@ github.com/mattn/go-colorable github.com/mattn/go-isatty # github.com/mattn/go-runewidth v0.0.9 github.com/mattn/go-runewidth -# github.com/mattn/go-shellwords v1.0.10 +# github.com/mattn/go-shellwords v1.0.11 github.com/mattn/go-shellwords # github.com/matttproud/golang_protobuf_extensions v1.0.1 github.com/matttproud/golang_protobuf_extensions/pbutil |