From c1b792c1e0326b95e1d37733731f2d8a130bebbd Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Thu, 18 Jul 2019 16:50:34 +0200 Subject: vendor github.com/containers/image@v2.0.1 * progress bar: use spinners for unknown blob sizes * use 'containers_image_ostree' as build tag * ostree: default is no OStree support * Add "Env" to ImageInspectInfo * config.go: improve debug message * config.go: log where credentials come from * Fix typo in docs/containers-registries.conf.5.md * docker: delete: support all MIME types * Try harder in storageImageDestination.TryReusingBlob * docker: allow deleting OCI images * ostree: improve error message Signed-off-by: Valentin Rothberg --- go.mod | 2 +- go.sum | 2 + hack/ostree_tag.sh | 2 +- vendor/github.com/containers/image/copy/copy.go | 35 ++++++--- .../containers/image/docker/docker_image_src.go | 13 ++-- .../containers/image/docker/tarfile/src.go | 83 +++++++++++----------- .../containers/image/manifest/docker_schema1.go | 1 + .../containers/image/manifest/docker_schema2.go | 1 + vendor/github.com/containers/image/manifest/oci.go | 1 + .../containers/image/ostree/ostree_dest.go | 4 +- .../containers/image/ostree/ostree_src.go | 2 +- .../containers/image/ostree/ostree_transport.go | 2 +- .../containers/image/pkg/docker/config/config.go | 4 ++ .../pkg/sysregistriesv2/system_registries_v2.go | 22 +++--- .../containers/image/storage/storage_image.go | 13 +++- .../image/transports/alltransports/ostree.go | 2 +- .../image/transports/alltransports/ostree_stub.go | 2 +- vendor/github.com/containers/image/types/types.go | 1 + .../github.com/containers/image/version/version.go | 2 +- vendor/modules.txt | 2 +- 20 files changed, 116 insertions(+), 80 deletions(-) diff --git a/go.mod b/go.mod index b6c400f18..23249a807 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/containernetworking/cni v0.7.1 github.com/containernetworking/plugins v0.8.1 github.com/containers/buildah v1.9.0 - github.com/containers/image v2.0.0+incompatible + github.com/containers/image v2.0.1+incompatible github.com/containers/psgo v1.3.1 github.com/containers/storage v1.12.13 github.com/coreos/bbolt v1.3.3 // indirect diff --git a/go.sum b/go.sum index 3964158c3..8f86d3757 100644 --- a/go.sum +++ b/go.sum @@ -70,6 +70,8 @@ github.com/containers/buildah v1.9.0 h1:ktVRCGNoVfW8PlTuCKUeh+zGdqn1Nik80DSWvGX+ github.com/containers/buildah v1.9.0/go.mod h1:1CsiLJvyU+h+wOjnqJJOWuJCVcMxZOr5HN/gHGdzJxY= github.com/containers/image v2.0.0+incompatible h1:FTr6Br7jlIKNCKMjSOMbAxKp2keQ0//jzJaYNTVhauk= github.com/containers/image v2.0.0+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= +github.com/containers/image v2.0.1+incompatible h1:w39mlElA/aSFZ6moFa5N+A4MWu9c8hgdMiMMYnH94Hs= +github.com/containers/image v2.0.1+incompatible/go.mod h1:8Vtij257IWSanUQKe1tAeNOm2sRVkSqQTVQ1IlwI3+M= github.com/containers/psgo v1.3.0 h1:kDhiA4gNNyJ2qCzmOuBf6AmrF/Pp+6Jo98P68R7fB8I= github.com/containers/psgo v1.3.0/go.mod h1:7MELvPTW1fj6yMrwD9I1Iasx1vU+hKlRkHXAJ51sFtU= github.com/containers/psgo v1.3.1-0.20190626112706-fbef66e4ce92 h1:aVJs/Av0Yc9uNoWnIwmG+6Z+XozuRXFwvLwAOVmwlvI= diff --git a/hack/ostree_tag.sh b/hack/ostree_tag.sh index b08441ef9..06a5bc68c 100755 --- a/hack/ostree_tag.sh +++ b/hack/ostree_tag.sh @@ -2,5 +2,5 @@ if ! pkg-config glib-2.0 gobject-2.0 ostree-1 libselinux 2> /dev/null ; then echo containers_image_ostree_stub else - echo ostree + echo containers_image_ostree fi diff --git a/vendor/github.com/containers/image/copy/copy.go b/vendor/github.com/containers/image/copy/copy.go index 3ed8a2b82..f1b029f97 100644 --- a/vendor/github.com/containers/image/copy/copy.go +++ b/vendor/github.com/containers/image/copy/copy.go @@ -597,15 +597,32 @@ func (c *copier) createProgressBar(pool *mpb.Progress, info types.BlobInfo, kind prefix = prefix[:maxPrefixLen] } - bar := pool.AddBar(info.Size, - mpb.BarClearOnComplete(), - mpb.PrependDecorators( - decor.Name(prefix), - ), - mpb.AppendDecorators( - decor.OnComplete(decor.CountersKibiByte("%.1f / %.1f"), " "+onComplete), - ), - ) + // Use a normal progress bar when we know the size (i.e., size > 0). + // Otherwise, use a spinner to indicate that something's happening. + var bar *mpb.Bar + if info.Size > 0 { + bar = pool.AddBar(info.Size, + mpb.BarClearOnComplete(), + mpb.PrependDecorators( + decor.Name(prefix), + ), + mpb.AppendDecorators( + decor.OnComplete(decor.CountersKibiByte("%.1f / %.1f"), " "+onComplete), + ), + ) + } else { + bar = pool.AddSpinner(info.Size, + mpb.SpinnerOnLeft, + mpb.BarClearOnComplete(), + mpb.SpinnerStyle([]string{".", "..", "...", "....", ""}), + mpb.PrependDecorators( + decor.Name(prefix), + ), + mpb.AppendDecorators( + decor.OnComplete(decor.Name(""), " "+onComplete), + ), + ) + } if c.progressOutput == ioutil.Discard { c.Printf("Copying %s %s\n", kind, info.Digest) } diff --git a/vendor/github.com/containers/image/docker/docker_image_src.go b/vendor/github.com/containers/image/docker/docker_image_src.go index c43e6e7ca..6951f31e9 100644 --- a/vendor/github.com/containers/image/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/docker/docker_image_src.go @@ -138,8 +138,9 @@ func (s *dockerImageSource) GetManifest(ctx context.Context, instanceDigest *dig func (s *dockerImageSource) fetchManifest(ctx context.Context, tagOrDigest string) ([]byte, string, error) { path := fmt.Sprintf(manifestPath, reference.Path(s.ref.ref), tagOrDigest) - headers := make(map[string][]string) - headers["Accept"] = manifest.DefaultRequestedManifestMIMETypes + headers := map[string][]string{ + "Accept": manifest.DefaultRequestedManifestMIMETypes, + } res, err := s.c.makeRequest(ctx, "GET", path, headers, nil, v2Auth, nil) if err != nil { return nil, "", err @@ -381,11 +382,9 @@ func deleteImage(ctx context.Context, sys *types.SystemContext, ref dockerRefere return err } - // When retrieving the digest from a registry >= 2.3 use the following header: - // "Accept": "application/vnd.docker.distribution.manifest.v2+json" - headers := make(map[string][]string) - headers["Accept"] = []string{manifest.DockerV2Schema2MediaType} - + headers := map[string][]string{ + "Accept": manifest.DefaultRequestedManifestMIMETypes, + } refTail, err := ref.tagOrDigest() if err != nil { return err diff --git a/vendor/github.com/containers/image/docker/tarfile/src.go b/vendor/github.com/containers/image/docker/tarfile/src.go index 03735f8a4..dd5d78fe8 100644 --- a/vendor/github.com/containers/image/docker/tarfile/src.go +++ b/vendor/github.com/containers/image/docker/tarfile/src.go @@ -15,24 +15,24 @@ import ( "github.com/containers/image/manifest" "github.com/containers/image/pkg/compression" "github.com/containers/image/types" - "github.com/opencontainers/go-digest" + digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" ) // Source is a partial implementation of types.ImageSource for reading from tarPath. type Source struct { tarPath string - removeTarPathOnClose bool // Remove temp file on close if true - cacheDataLock sync.Once // Atomic way to ensure that ensureCachedDataIsPresent is only invoked once + removeTarPathOnClose bool // Remove temp file on close if true // The following data is only available after ensureCachedDataIsPresent() succeeds - cacheDataResult error // The return value of ensureCachedDataIsPresent, since it should be as safe to cache as the side effects tarManifest *ManifestItem // nil if not available yet. configBytes []byte configDigest digest.Digest orderedDiffIDList []digest.Digest knownLayers map[digest.Digest]*layerInfo // Other state - generatedManifest []byte // Private cache for GetManifest(), nil if not set yet. + generatedManifest []byte // Private cache for GetManifest(), nil if not set yet. + cacheDataLock sync.Once // Private state for ensureCachedDataIsPresent to make it concurrency-safe + cacheDataResult error // Private state for ensureCachedDataIsPresent } type layerInfo struct { @@ -201,47 +201,50 @@ func (s *Source) readTarComponent(path string) ([]byte, error) { } // ensureCachedDataIsPresent loads data necessary for any of the public accessors. +// It is safe to call this from multi-threaded code. func (s *Source) ensureCachedDataIsPresent() error { s.cacheDataLock.Do(func() { - // Read and parse manifest.json - tarManifest, err := s.loadTarManifest() - if err != nil { - s.cacheDataResult = err - return - } + s.cacheDataResult = s.ensureCachedDataIsPresentPrivate() + }) + return s.cacheDataResult +} - // Check to make sure length is 1 - if len(tarManifest) != 1 { - s.cacheDataResult = errors.Errorf("Unexpected tar manifest.json: expected 1 item, got %d", len(tarManifest)) - return - } +// ensureCachedDataIsPresentPrivate is a private implementation detail of ensureCachedDataIsPresent. +// Call ensureCachedDataIsPresent instead. +func (s *Source) ensureCachedDataIsPresentPrivate() error { + // Read and parse manifest.json + tarManifest, err := s.loadTarManifest() + if err != nil { + return err + } - // Read and parse config. - configBytes, err := s.readTarComponent(tarManifest[0].Config) - if err != nil { - s.cacheDataResult = err - return - } - var parsedConfig manifest.Schema2Image // There's a lot of info there, but we only really care about layer DiffIDs. - if err := json.Unmarshal(configBytes, &parsedConfig); err != nil { - s.cacheDataResult = errors.Wrapf(err, "Error decoding tar config %s", tarManifest[0].Config) - return - } + // Check to make sure length is 1 + if len(tarManifest) != 1 { + return errors.Errorf("Unexpected tar manifest.json: expected 1 item, got %d", len(tarManifest)) + } - knownLayers, err := s.prepareLayerData(&tarManifest[0], &parsedConfig) - if err != nil { - s.cacheDataResult = err - return - } + // Read and parse config. + configBytes, err := s.readTarComponent(tarManifest[0].Config) + if err != nil { + return err + } + var parsedConfig manifest.Schema2Image // There's a lot of info there, but we only really care about layer DiffIDs. + if err := json.Unmarshal(configBytes, &parsedConfig); err != nil { + return errors.Wrapf(err, "Error decoding tar config %s", tarManifest[0].Config) + } - // Success; commit. - s.tarManifest = &tarManifest[0] - s.configBytes = configBytes - s.configDigest = digest.FromBytes(configBytes) - s.orderedDiffIDList = parsedConfig.RootFS.DiffIDs - s.knownLayers = knownLayers - }) - return s.cacheDataResult + knownLayers, err := s.prepareLayerData(&tarManifest[0], &parsedConfig) + if err != nil { + return err + } + + // Success; commit. + s.tarManifest = &tarManifest[0] + s.configBytes = configBytes + s.configDigest = digest.FromBytes(configBytes) + s.orderedDiffIDList = parsedConfig.RootFS.DiffIDs + s.knownLayers = knownLayers + return nil } // loadTarManifest loads and decodes the manifest.json. diff --git a/vendor/github.com/containers/image/manifest/docker_schema1.go b/vendor/github.com/containers/image/manifest/docker_schema1.go index 1ba65d5be..b76286b8f 100644 --- a/vendor/github.com/containers/image/manifest/docker_schema1.go +++ b/vendor/github.com/containers/image/manifest/docker_schema1.go @@ -226,6 +226,7 @@ func (m *Schema1) Inspect(_ func(types.BlobInfo) ([]byte, error)) (*types.ImageI } if s1.Config != nil { i.Labels = s1.Config.Labels + i.Env = s1.Config.Env } return i, nil } diff --git a/vendor/github.com/containers/image/manifest/docker_schema2.go b/vendor/github.com/containers/image/manifest/docker_schema2.go index 0671aed9f..76a80e5a6 100644 --- a/vendor/github.com/containers/image/manifest/docker_schema2.go +++ b/vendor/github.com/containers/image/manifest/docker_schema2.go @@ -241,6 +241,7 @@ func (m *Schema2) Inspect(configGetter func(types.BlobInfo) ([]byte, error)) (*t } if s2.Config != nil { i.Labels = s2.Config.Labels + i.Env = s2.Config.Env } return i, nil } diff --git a/vendor/github.com/containers/image/manifest/oci.go b/vendor/github.com/containers/image/manifest/oci.go index 91705045b..dd65e0ba2 100644 --- a/vendor/github.com/containers/image/manifest/oci.go +++ b/vendor/github.com/containers/image/manifest/oci.go @@ -116,6 +116,7 @@ func (m *OCI1) Inspect(configGetter func(types.BlobInfo) ([]byte, error)) (*type Architecture: v1.Architecture, Os: v1.OS, Layers: layerInfosToStrings(m.LayerInfos()), + Env: d1.Config.Env, } return i, nil } diff --git a/vendor/github.com/containers/image/ostree/ostree_dest.go b/vendor/github.com/containers/image/ostree/ostree_dest.go index d69f4fa33..06a905aed 100644 --- a/vendor/github.com/containers/image/ostree/ostree_dest.go +++ b/vendor/github.com/containers/image/ostree/ostree_dest.go @@ -1,4 +1,4 @@ -// +build !containers_image_ostree_stub +// +build containers_image_ostree package ostree @@ -218,7 +218,7 @@ func fixFiles(selinuxHnd *C.struct_selabel_handle, root string, dir string, user defer C.free(unsafe.Pointer(fullpathC)) res, err = C.lsetfilecon_raw(fullpathC, context) if int(res) < 0 { - return errors.Wrapf(err, "cannot setfilecon_raw %s", fullpath) + return errors.Wrapf(err, "cannot setfilecon_raw %s to %s", fullpath, C.GoString(context)) } } } diff --git a/vendor/github.com/containers/image/ostree/ostree_src.go b/vendor/github.com/containers/image/ostree/ostree_src.go index 35d852139..dc52ccb6e 100644 --- a/vendor/github.com/containers/image/ostree/ostree_src.go +++ b/vendor/github.com/containers/image/ostree/ostree_src.go @@ -1,4 +1,4 @@ -// +build !containers_image_ostree_stub +// +build containers_image_ostree package ostree diff --git a/vendor/github.com/containers/image/ostree/ostree_transport.go b/vendor/github.com/containers/image/ostree/ostree_transport.go index c9856530b..2e86623ac 100644 --- a/vendor/github.com/containers/image/ostree/ostree_transport.go +++ b/vendor/github.com/containers/image/ostree/ostree_transport.go @@ -1,4 +1,4 @@ -// +build !containers_image_ostree_stub +// +build containers_image_ostree package ostree diff --git a/vendor/github.com/containers/image/pkg/docker/config/config.go b/vendor/github.com/containers/image/pkg/docker/config/config.go index 57b548e26..2e6bb378f 100644 --- a/vendor/github.com/containers/image/pkg/docker/config/config.go +++ b/vendor/github.com/containers/image/pkg/docker/config/config.go @@ -56,6 +56,7 @@ func SetAuthentication(sys *types.SystemContext, registry, username, password st // If an entry is not found empty strings are returned for the username and password func GetAuthentication(sys *types.SystemContext, registry string) (string, string, error) { if sys != nil && sys.DockerAuthConfig != nil { + logrus.Debug("Returning credentials from DockerAuthConfig") return sys.DockerAuthConfig.Username, sys.DockerAuthConfig.Password, nil } @@ -76,12 +77,15 @@ func GetAuthentication(sys *types.SystemContext, registry string) (string, strin legacyFormat := path == dockerLegacyPath username, password, err := findAuthentication(registry, path, legacyFormat) if err != nil { + logrus.Debugf("Credentials not found") return "", "", err } if username != "" && password != "" { + logrus.Debugf("Returning credentials from %s", path) return username, password, nil } } + logrus.Debugf("Credentials not found") return "", "", nil } diff --git a/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go b/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go index 361e6fc60..0c13913ed 100644 --- a/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go +++ b/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go @@ -30,10 +30,10 @@ const builtinRegistriesConfPath = "/etc/containers/registries.conf" // Endpoint describes a remote location of a registry. type Endpoint struct { // The endpoint's remote location. - Location string `toml:"location"` + Location string `toml:"location,omitempty"` // If true, certs verification will be skipped and HTTP (non-TLS) // connections will be allowed. - Insecure bool `toml:"insecure"` + Insecure bool `toml:"insecure,omitempty"` } // rewriteReference will substitute the provided reference `prefix` to the @@ -56,22 +56,22 @@ func (e *Endpoint) rewriteReference(ref reference.Named, prefix string) (referen // Registry represents a registry. type Registry struct { + // Prefix is used for matching images, and to translate one namespace to + // another. If `Prefix="example.com/bar"`, `location="example.com/foo/bar"` + // and we pull from "example.com/bar/myimage:latest", the image will + // effectively be pulled from "example.com/foo/bar/myimage:latest". + // If no Prefix is specified, it defaults to the specified location. + Prefix string `toml:"prefix"` // A registry is an Endpoint too Endpoint // The registry's mirrors. - Mirrors []Endpoint `toml:"mirror"` + Mirrors []Endpoint `toml:"mirror,omitempty"` // If true, pulling from the registry will be blocked. - Blocked bool `toml:"blocked"` + Blocked bool `toml:"blocked,omitempty"` // If true, mirrors will only be used for digest pulls. Pulling images by // tag can potentially yield different images, depending on which endpoint // we pull from. Forcing digest-pulls for mirrors avoids that issue. - MirrorByDigestOnly bool `toml:"mirror-by-digest-only"` - // Prefix is used for matching images, and to translate one namespace to - // another. If `Prefix="example.com/bar"`, `location="example.com/foo/bar"` - // and we pull from "example.com/bar/myimage:latest", the image will - // effectively be pulled from "example.com/foo/bar/myimage:latest". - // If no Prefix is specified, it defaults to the specified location. - Prefix string `toml:"prefix"` + MirrorByDigestOnly bool `toml:"mirror-by-digest-only,omitempty"` } // PullSource consists of an Endpoint and a Reference. Note that the reference is diff --git a/vendor/github.com/containers/image/storage/storage_image.go b/vendor/github.com/containers/image/storage/storage_image.go index b39d2bcc0..946a85f7b 100644 --- a/vendor/github.com/containers/image/storage/storage_image.go +++ b/vendor/github.com/containers/image/storage/storage_image.go @@ -491,14 +491,21 @@ func (s *storageImageDestination) TryReusingBlob(ctx context.Context, blobinfo t // Does the blob correspond to a known DiffID which we already have available? // Because we must return the size, which is unknown for unavailable compressed blobs, the returned BlobInfo refers to the - // uncompressed layer, and that can happen only if canSubstitute. - if canSubstitute { + // uncompressed layer, and that can happen only if canSubstitute, or if the incoming manifest already specifies the size. + if canSubstitute || blobinfo.Size != -1 { if uncompressedDigest := cache.UncompressedDigest(blobinfo.Digest); uncompressedDigest != "" && uncompressedDigest != blobinfo.Digest { layers, err := s.imageRef.transport.store.LayersByUncompressedDigest(uncompressedDigest) if err != nil && errors.Cause(err) != storage.ErrLayerUnknown { return false, types.BlobInfo{}, errors.Wrapf(err, `Error looking for layers with digest %q`, uncompressedDigest) } if len(layers) > 0 { + if blobinfo.Size != -1 { + s.blobDiffIDs[blobinfo.Digest] = layers[0].UncompressedDigest + return true, blobinfo, nil + } + if !canSubstitute { + return false, types.BlobInfo{}, fmt.Errorf("Internal error: canSubstitute was expected to be true for blobInfo %v", blobinfo) + } s.blobDiffIDs[uncompressedDigest] = layers[0].UncompressedDigest return true, types.BlobInfo{ Digest: uncompressedDigest, @@ -627,7 +634,7 @@ func (s *storageImageDestination) Commit(ctx context.Context) error { if !ok { // Try to find the layer with contents matching that blobsum. layer := "" - layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(blob.Digest) + layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(diffID) if err2 == nil && len(layers) > 0 { layer = layers[0].ID } else { diff --git a/vendor/github.com/containers/image/transports/alltransports/ostree.go b/vendor/github.com/containers/image/transports/alltransports/ostree.go index 4a3b29a00..cc4d69fe8 100644 --- a/vendor/github.com/containers/image/transports/alltransports/ostree.go +++ b/vendor/github.com/containers/image/transports/alltransports/ostree.go @@ -1,4 +1,4 @@ -// +build !containers_image_ostree_stub,linux +// +build containers_image_ostree,linux package alltransports diff --git a/vendor/github.com/containers/image/transports/alltransports/ostree_stub.go b/vendor/github.com/containers/image/transports/alltransports/ostree_stub.go index 48fcaa58d..fb5b96e54 100644 --- a/vendor/github.com/containers/image/transports/alltransports/ostree_stub.go +++ b/vendor/github.com/containers/image/transports/alltransports/ostree_stub.go @@ -1,4 +1,4 @@ -// +build containers_image_ostree_stub !linux +// +build !containers_image_ostree !linux package alltransports diff --git a/vendor/github.com/containers/image/types/types.go b/vendor/github.com/containers/image/types/types.go index 789504348..08b4241e0 100644 --- a/vendor/github.com/containers/image/types/types.go +++ b/vendor/github.com/containers/image/types/types.go @@ -398,6 +398,7 @@ type ImageInspectInfo struct { Architecture string Os string Layers []string + Env []string } // DockerAuthConfig contains authorization information for connecting to a registry. diff --git a/vendor/github.com/containers/image/version/version.go b/vendor/github.com/containers/image/version/version.go index 62b2c8bc5..807daf7a2 100644 --- a/vendor/github.com/containers/image/version/version.go +++ b/vendor/github.com/containers/image/version/version.go @@ -8,7 +8,7 @@ const ( // VersionMinor is for functionality in a backwards-compatible manner VersionMinor = 0 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 0 + VersionPatch = 1 // VersionDev indicates development branch. Releases will be empty string. VersionDev = "" diff --git a/vendor/modules.txt b/vendor/modules.txt index 462ba1408..17b11fd58 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -62,7 +62,7 @@ github.com/containers/buildah/docker github.com/containers/buildah/pkg/blobcache github.com/containers/buildah/pkg/overlay github.com/containers/buildah/pkg/unshare -# github.com/containers/image v2.0.0+incompatible +# github.com/containers/image v2.0.1+incompatible github.com/containers/image/directory github.com/containers/image/docker github.com/containers/image/docker/archive -- cgit v1.2.3-54-g00ecf