From 82a6b373a59b82532263b19c0a8abff9af4a96db Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Thu, 30 Aug 2018 17:27:54 -0400 Subject: Vendor in latest containers/storage and containers/image Update container/image to address a commit error when copying layers and metadata. This change may require users to recreate containers. container/storage added some new lock protection to prevent possible deadlock and data corruption. Signed-off-by: Daniel J Walsh Closes: #1381 Approved by: mheon --- vendor.conf | 4 +- vendor/github.com/containers/image/README.md | 8 +- vendor/github.com/containers/image/copy/copy.go | 2 - .../containers/image/docker/docker_client.go | 3 +- .../containers/image/docker/docker_image_src.go | 9 +- .../containers/image/manifest/docker_schema2.go | 5 +- .../image/pkg/tlsclientconfig/tlsclientconfig.go | 10 ++- vendor/github.com/containers/storage/containers.go | 7 ++ vendor/github.com/containers/storage/images.go | 7 ++ vendor/github.com/containers/storage/layers.go | 7 ++ vendor/github.com/containers/storage/lockfile.go | 3 + .../github.com/containers/storage/lockfile_unix.go | 12 ++- .../containers/storage/lockfile_windows.go | 17 +++- .../storage/pkg/archive/example_changes.go | 97 ++++++++++++++++++++++ .../github.com/projectatomic/buildah/vendor.conf | 6 +- 15 files changed, 174 insertions(+), 23 deletions(-) create mode 100644 vendor/github.com/containers/storage/pkg/archive/example_changes.go diff --git a/vendor.conf b/vendor.conf index c97aa4145..1302a786f 100644 --- a/vendor.conf +++ b/vendor.conf @@ -10,8 +10,8 @@ github.com/containerd/cgroups 77e628511d924b13a77cebdc73b757a47f6d751b github.com/containerd/continuity master github.com/containernetworking/cni v0.7.0-alpha1 github.com/containernetworking/plugins 1562a1e60ed101aacc5e08ed9dbeba8e9f3d4ec1 -github.com/containers/image 134f99bed228d6297dc01d152804f6f09f185418 -github.com/containers/storage 17c7d1fee5603ccf6dd97edc14162fc1510e7e23 +github.com/containers/image 5df44e095ed826fbe2beeaabb329c749d7d6c3b6 +github.com/containers/storage d0cb0107646058ad96ce90630fee2bd5709ee637 github.com/containers/psgo 5dde6da0bc8831b35243a847625bcf18183bd1ee github.com/coreos/go-systemd v14 github.com/cri-o/ocicni master diff --git a/vendor/github.com/containers/image/README.md b/vendor/github.com/containers/image/README.md index 017211b2c..4ed8c93b7 100644 --- a/vendor/github.com/containers/image/README.md +++ b/vendor/github.com/containers/image/README.md @@ -25,7 +25,7 @@ them as necessary, and to sign and verify images. The containers/image project is only a library with no user interface; you can either incorporate it into your Go programs, or use the `skopeo` tool: -The [skopeo](https://github.com/projectatomic/skopeo) tool uses the +The [skopeo](https://github.com/containers/skopeo) tool uses the containers/image library and takes advantage of many of its features, e.g. `skopeo copy` exposes the `containers/image/copy.Image` functionality. @@ -42,7 +42,7 @@ What this project tests against dependencies-wise is located ## Building If you want to see what the library can do, or an example of how it is called, -consider starting with the [skopeo](https://github.com/projectatomic/skopeo) tool +consider starting with the [skopeo](https://github.com/containers/skopeo) tool instead. To integrate this library into your project, put it into `$GOPATH` or use @@ -73,7 +73,9 @@ When developing this library, please use `make` (or `make … BUILDTAGS=…`) to ## License -ASL 2.0 +Apache License 2.0 + +SPDX-License-Identifier: Apache-2.0 ## Contact diff --git a/vendor/github.com/containers/image/copy/copy.go b/vendor/github.com/containers/image/copy/copy.go index 6b1df75a9..183993c5d 100644 --- a/vendor/github.com/containers/image/copy/copy.go +++ b/vendor/github.com/containers/image/copy/copy.go @@ -71,7 +71,6 @@ func (d *digestingReader) Read(p []byte) (int, error) { // copier allows us to keep track of diffID values for blobs, and other // data shared across one or more images in a possible manifest list. type copier struct { - copiedBlobs map[digest.Digest]digest.Digest cachedDiffIDs map[digest.Digest]digest.Digest dest types.ImageDestination rawSource types.ImageSource @@ -141,7 +140,6 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, }() c := &copier{ - copiedBlobs: make(map[digest.Digest]digest.Digest), cachedDiffIDs: make(map[digest.Digest]digest.Digest), dest: dest, rawSource: rawSource, diff --git a/vendor/github.com/containers/image/docker/docker_client.go b/vendor/github.com/containers/image/docker/docker_client.go index 9825b40cf..cae73a6cd 100644 --- a/vendor/github.com/containers/image/docker/docker_client.go +++ b/vendor/github.com/containers/image/docker/docker_client.go @@ -462,6 +462,7 @@ func (c *dockerClient) getBearerToken(ctx context.Context, realm, service, scope if c.username != "" && c.password != "" { authReq.SetBasicAuth(c.username, c.password) } + logrus.Debugf("%s %s", authReq.Method, authReq.URL.String()) tr := tlsclientconfig.NewTransport() // TODO(runcom): insecure for now to contact the external token service tr.TLSClientConfig = &tls.Config{InsecureSkipVerify: true} @@ -497,8 +498,8 @@ func (c *dockerClient) detectProperties(ctx context.Context) error { ping := func(scheme string) error { url := fmt.Sprintf(resolvedPingV2URL, scheme, c.registry) resp, err := c.makeRequestToResolvedURL(ctx, "GET", url, nil, nil, -1, true) - logrus.Debugf("Ping %s err %#v", url, err) if err != nil { + logrus.Debugf("Ping %s err %#v", url, err) return err } defer resp.Body.Close() diff --git a/vendor/github.com/containers/image/docker/docker_image_src.go b/vendor/github.com/containers/image/docker/docker_image_src.go index 98f6067e1..46145dff6 100644 --- a/vendor/github.com/containers/image/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/docker/docker_image_src.go @@ -310,7 +310,14 @@ func (s *dockerImageSource) getSignaturesFromAPIExtension(ctx context.Context, i // deleteImage deletes the named image from the registry, if supported. func deleteImage(ctx context.Context, sys *types.SystemContext, ref dockerReference) error { - c, err := newDockerClientFromRef(sys, ref, true, "push") + // docker/distribution does not document what action should be used for deleting images. + // + // Current docker/distribution requires "pull" for reading the manifest and "delete" for deleting it. + // quay.io requires "push" (an explicit "pull" is unnecessary), does not grant any token (fails parsing the request) if "delete" is included. + // OpenShift ignores the action string (both the password and the token is an OpenShift API token identifying a user). + // + // We have to hard-code a single string, luckily both docker/distribution and quay.io support "*" to mean "everything". + c, err := newDockerClientFromRef(sys, ref, true, "*") if err != nil { return err } diff --git a/vendor/github.com/containers/image/manifest/docker_schema2.go b/vendor/github.com/containers/image/manifest/docker_schema2.go index a7f9451bd..0671aed9f 100644 --- a/vendor/github.com/containers/image/manifest/docker_schema2.go +++ b/vendor/github.com/containers/image/manifest/docker_schema2.go @@ -57,8 +57,9 @@ type Schema2HealthConfig struct { Test []string `json:",omitempty"` // Zero means to inherit. Durations are expressed as integer nanoseconds. - Interval time.Duration `json:",omitempty"` // Interval is the time to wait between checks. - Timeout time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung. + StartPeriod time.Duration `json:",omitempty"` // StartPeriod is the time to wait after starting before running the first check. + Interval time.Duration `json:",omitempty"` // Interval is the time to wait between checks. + Timeout time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung. // Retries is the number of consecutive failures needed to consider a container as unhealthy. // Zero means inherit. diff --git a/vendor/github.com/containers/image/pkg/tlsclientconfig/tlsclientconfig.go b/vendor/github.com/containers/image/pkg/tlsclientconfig/tlsclientconfig.go index 0a32861ce..156651173 100644 --- a/vendor/github.com/containers/image/pkg/tlsclientconfig/tlsclientconfig.go +++ b/vendor/github.com/containers/image/pkg/tlsclientconfig/tlsclientconfig.go @@ -34,11 +34,13 @@ func SetupCertificates(dir string, tlsc *tls.Config) error { for _, f := range fs { fullPath := filepath.Join(dir, f.Name()) if strings.HasSuffix(f.Name(), ".crt") { - systemPool, err := tlsconfig.SystemCertPool() - if err != nil { - return errors.Wrap(err, "unable to get system cert pool") + if tlsc.RootCAs == nil { + systemPool, err := tlsconfig.SystemCertPool() + if err != nil { + return errors.Wrap(err, "unable to get system cert pool") + } + tlsc.RootCAs = systemPool } - tlsc.RootCAs = systemPool logrus.Debugf(" crt: %s", fullPath) data, err := ioutil.ReadFile(fullPath) if err != nil { diff --git a/vendor/github.com/containers/storage/containers.go b/vendor/github.com/containers/storage/containers.go index ebc1e99a0..f87ea15be 100644 --- a/vendor/github.com/containers/storage/containers.go +++ b/vendor/github.com/containers/storage/containers.go @@ -192,6 +192,9 @@ func (r *containerStore) Load() error { } func (r *containerStore) Save() error { + if !r.Locked() { + return errors.New("container store is not locked") + } rpath := r.containerspath() if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil { return err @@ -560,3 +563,7 @@ func (r *containerStore) IsReadWrite() bool { func (r *containerStore) TouchedSince(when time.Time) bool { return r.lockfile.TouchedSince(when) } + +func (r *containerStore) Locked() bool { + return r.lockfile.Locked() +} diff --git a/vendor/github.com/containers/storage/images.go b/vendor/github.com/containers/storage/images.go index 80fae6dce..b10501b08 100644 --- a/vendor/github.com/containers/storage/images.go +++ b/vendor/github.com/containers/storage/images.go @@ -219,6 +219,9 @@ func (r *imageStore) Save() error { if !r.IsReadWrite() { return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify the image store at %q", r.imagespath()) } + if !r.Locked() { + return errors.New("image store is not locked") + } rpath := r.imagespath() if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil { return err @@ -701,3 +704,7 @@ func (r *imageStore) IsReadWrite() bool { func (r *imageStore) TouchedSince(when time.Time) bool { return r.lockfile.TouchedSince(when) } + +func (r *imageStore) Locked() bool { + return r.lockfile.Locked() +} diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index c5f926273..9989bd6be 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -371,6 +371,9 @@ func (r *layerStore) Save() error { if !r.IsReadWrite() { return errors.Wrapf(ErrStoreIsReadOnly, "not allowed to modify the layer store at %q", r.layerspath()) } + if !r.Locked() { + return errors.New("layer store is not locked") + } rpath := r.layerspath() if err := os.MkdirAll(filepath.Dir(rpath), 0700); err != nil { return err @@ -1181,3 +1184,7 @@ func (r *layerStore) IsReadWrite() bool { func (r *layerStore) TouchedSince(when time.Time) bool { return r.lockfile.TouchedSince(when) } + +func (r *layerStore) Locked() bool { + return r.lockfile.Locked() +} diff --git a/vendor/github.com/containers/storage/lockfile.go b/vendor/github.com/containers/storage/lockfile.go index c1aa482f8..9f6a18144 100644 --- a/vendor/github.com/containers/storage/lockfile.go +++ b/vendor/github.com/containers/storage/lockfile.go @@ -28,6 +28,9 @@ type Locker interface { // IsReadWrite() checks if the lock file is read-write IsReadWrite() bool + + // Locked() checks if lock is locked + Locked() bool } var ( diff --git a/vendor/github.com/containers/storage/lockfile_unix.go b/vendor/github.com/containers/storage/lockfile_unix.go index 679259234..31e39c02e 100644 --- a/vendor/github.com/containers/storage/lockfile_unix.go +++ b/vendor/github.com/containers/storage/lockfile_unix.go @@ -25,9 +25,9 @@ func getLockFile(path string, ro bool) (Locker, error) { } unix.CloseOnExec(fd) if ro { - return &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_RDLCK}, nil + return &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_RDLCK, locked: false}, nil } - return &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_WRLCK}, nil + return &lockfile{file: path, fd: uintptr(fd), lw: stringid.GenerateRandomID(), locktype: unix.F_WRLCK, locked: false}, nil } type lockfile struct { @@ -36,6 +36,7 @@ type lockfile struct { fd uintptr lw string locktype int16 + locked bool } // Lock locks the lock file @@ -48,6 +49,7 @@ func (l *lockfile) Lock() { Pid: int32(os.Getpid()), } l.mu.Lock() + l.locked = true for unix.FcntlFlock(l.fd, unix.F_SETLKW, &lk) != nil { time.Sleep(10 * time.Millisecond) } @@ -65,9 +67,15 @@ func (l *lockfile) Unlock() { for unix.FcntlFlock(l.fd, unix.F_SETLKW, &lk) != nil { time.Sleep(10 * time.Millisecond) } + l.locked = false l.mu.Unlock() } +// Check if lock is locked +func (l *lockfile) Locked() bool { + return l.locked +} + // Touch updates the lock file with the UID of the user func (l *lockfile) Touch() error { l.lw = stringid.GenerateRandomID() diff --git a/vendor/github.com/containers/storage/lockfile_windows.go b/vendor/github.com/containers/storage/lockfile_windows.go index ed6c5c4b2..77490b851 100644 --- a/vendor/github.com/containers/storage/lockfile_windows.go +++ b/vendor/github.com/containers/storage/lockfile_windows.go @@ -9,18 +9,29 @@ import ( ) func getLockFile(path string, ro bool) (Locker, error) { - return &lockfile{}, nil + return &lockfile{locked: false}, nil } type lockfile struct { - mu sync.Mutex - file string + mu sync.Mutex + file string + locked bool } func (l *lockfile) Lock() { + l.mu.Lock() + l.locked = true } + func (l *lockfile) Unlock() { + l.locked = false + l.mu.Unlock() +} + +func (l *lockfile) Locked() bool { + return l.locked } + func (l *lockfile) Modified() (bool, error) { return false, nil } diff --git a/vendor/github.com/containers/storage/pkg/archive/example_changes.go b/vendor/github.com/containers/storage/pkg/archive/example_changes.go new file mode 100644 index 000000000..70f9c5564 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/archive/example_changes.go @@ -0,0 +1,97 @@ +// +build ignore + +// Simple tool to create an archive stream from an old and new directory +// +// By default it will stream the comparison of two temporary directories with junk files +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "path" + + "github.com/containers/storage/pkg/archive" + "github.com/sirupsen/logrus" +) + +var ( + flDebug = flag.Bool("D", false, "debugging output") + flNewDir = flag.String("newdir", "", "") + flOldDir = flag.String("olddir", "", "") + log = logrus.New() +) + +func main() { + flag.Usage = func() { + fmt.Println("Produce a tar from comparing two directory paths. By default a demo tar is created of around 200 files (including hardlinks)") + fmt.Printf("%s [OPTIONS]\n", os.Args[0]) + flag.PrintDefaults() + } + flag.Parse() + log.Out = os.Stderr + if (len(os.Getenv("DEBUG")) > 0) || *flDebug { + logrus.SetLevel(logrus.DebugLevel) + } + var newDir, oldDir string + + if len(*flNewDir) == 0 { + var err error + newDir, err = ioutil.TempDir("", "storage-test-newDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(newDir) + if _, err := prepareUntarSourceDirectory(100, newDir, true); err != nil { + log.Fatal(err) + } + } else { + newDir = *flNewDir + } + + if len(*flOldDir) == 0 { + oldDir, err := ioutil.TempDir("", "storage-test-oldDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(oldDir) + } else { + oldDir = *flOldDir + } + + changes, err := archive.ChangesDirs(newDir, oldDir) + if err != nil { + log.Fatal(err) + } + + a, err := archive.ExportChanges(newDir, changes) + if err != nil { + log.Fatal(err) + } + defer a.Close() + + i, err := io.Copy(os.Stdout, a) + if err != nil && err != io.EOF { + log.Fatal(err) + } + fmt.Fprintf(os.Stderr, "wrote archive of %d bytes", i) +} + +func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) { + fileData := []byte("fooo") + for n := 0; n < numberOfFiles; n++ { + fileName := fmt.Sprintf("file-%d", n) + if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil { + return 0, err + } + if makeLinks { + if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil { + return 0, err + } + } + } + totalSize := numberOfFiles * len(fileData) + return totalSize, nil +} diff --git a/vendor/github.com/projectatomic/buildah/vendor.conf b/vendor/github.com/projectatomic/buildah/vendor.conf index ec1d5ec9c..870fb4bdd 100644 --- a/vendor/github.com/projectatomic/buildah/vendor.conf +++ b/vendor/github.com/projectatomic/buildah/vendor.conf @@ -4,8 +4,8 @@ github.com/BurntSushi/toml master github.com/containerd/continuity master github.com/containernetworking/cni v0.7.0-alpha1 github.com/seccomp/containers-golang master -github.com/containers/image 216acb1bcd2c1abef736ee322e17147ee2b7d76c -github.com/containers/storage 17c7d1fee5603ccf6dd97edc14162fc1510e7e23 +github.com/containers/image 5df44e095ed826fbe2beeaabb329c749d7d6c3b6 +github.com/containers/storage 9fcbb57eb6c732e7b67003bb8ed861f169d33d63 github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716 github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00 github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1 @@ -46,12 +46,12 @@ github.com/containers/libpod d20f3a51463ce75d139dd830e19a173906b0b0cb github.com/sirupsen/logrus master github.com/syndtr/gocapability master github.com/tchap/go-patricia master +github.com/ulikunitz/xz v0.5.4 github.com/urfave/cli 934abfb2f102315b5794e15ebc7949e4ca253920 github.com/vbatts/tar-split v0.10.2 github.com/xeipuuv/gojsonpointer master github.com/xeipuuv/gojsonreference master github.com/xeipuuv/gojsonschema master -github.com/ulikunitz/xz v0.5.4 golang.org/x/crypto master golang.org/x/net master golang.org/x/sys master -- cgit v1.2.3-54-g00ecf