diff options
Diffstat (limited to 'vendor/github.com')
58 files changed, 690 insertions, 877 deletions
diff --git a/vendor/github.com/containers/buildah/config.go b/vendor/github.com/containers/buildah/config.go index 3609694f6..114b8ca37 100644 --- a/vendor/github.com/containers/buildah/config.go +++ b/vendor/github.com/containers/buildah/config.go @@ -474,9 +474,6 @@ func (b *Builder) Hostname() string { // Note: this setting is not present in the OCIv1 image format, so it is // discarded when writing images using OCIv1 formats. func (b *Builder) SetHostname(name string) { - if name != "" && b.Format != Dockerv2ImageManifest { - logrus.Errorf("HOSTNAME is not supported for OCI image format, hostname %s will be ignored. Must use `docker` format", name) - } b.Docker.Config.Hostname = name } diff --git a/vendor/github.com/containers/buildah/image.go b/vendor/github.com/containers/buildah/image.go index 163d34269..47842db7d 100644 --- a/vendor/github.com/containers/buildah/image.go +++ b/vendor/github.com/containers/buildah/image.go @@ -557,6 +557,10 @@ func (i *containerImageSource) LayerInfosForCopy(ctx context.Context) ([]types.B return nil, nil } +func (i *containerImageSource) HasThreadSafeGetBlob() bool { + return false +} + func (i *containerImageSource) GetBlob(ctx context.Context, blob types.BlobInfo, cache types.BlobInfoCache) (reader io.ReadCloser, size int64, err error) { if blob.Digest == i.configDigest { logrus.Debugf("start reading config") diff --git a/vendor/github.com/containers/buildah/imagebuildah/build.go b/vendor/github.com/containers/buildah/imagebuildah/build.go index 2681bc198..d838260e7 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/build.go +++ b/vendor/github.com/containers/buildah/imagebuildah/build.go @@ -1307,7 +1307,12 @@ func (b *Executor) Build(ctx context.Context, stages imagebuilder.Stages) (strin var imageRef reference.Canonical imageID := "" - if !b.layers && !b.noCache { + + // Check if we have a one line Dockerfile making layers irrelevant + // or the user told us to ignore layers. + ignoreLayers := (len(stages) < 2 && len(stages[0].Node.Children) < 2) || (!b.layers && !b.noCache) + + if ignoreLayers { imgID, ref, err := stageExecutor.Commit(ctx, stages[len(stages)-1].Builder, "") if err != nil { return "", nil, err diff --git a/vendor/github.com/containers/buildah/imagebuildah/chroot_symlink.go b/vendor/github.com/containers/buildah/imagebuildah/chroot_symlink.go index edb5837db..6feedf6a5 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/chroot_symlink.go +++ b/vendor/github.com/containers/buildah/imagebuildah/chroot_symlink.go @@ -131,6 +131,11 @@ func resolveModifiedTime(rootdir, filename, historyTime string) (bool, error) { func modTimeIsGreater(rootdir, path string, historyTime string) (bool, error) { var timeIsGreater bool + // the Walk below doesn't work if rootdir and path are equal + if rootdir == path { + return false, nil + } + // Convert historyTime from string to time.Time for comparison histTime, err := time.Parse(time.RFC3339Nano, historyTime) if err != nil { diff --git a/vendor/github.com/containers/buildah/pkg/blobcache/blobcache.go b/vendor/github.com/containers/buildah/pkg/blobcache/blobcache.go index 63022f15d..ae55316b0 100644 --- a/vendor/github.com/containers/buildah/pkg/blobcache/blobcache.go +++ b/vendor/github.com/containers/buildah/pkg/blobcache/blobcache.go @@ -218,6 +218,10 @@ func (s *blobCacheSource) GetManifest(ctx context.Context, instanceDigest *diges return s.source.GetManifest(ctx, instanceDigest) } +func (s *blobCacheSource) HasThreadSafeGetBlob() bool { + return false +} + func (s *blobCacheSource) GetBlob(ctx context.Context, blobinfo types.BlobInfo, cache types.BlobInfoCache) (io.ReadCloser, int64, error) { present, size, err := s.reference.HasBlob(blobinfo) if err != nil { @@ -398,6 +402,10 @@ func saveStream(wg *sync.WaitGroup, decompressReader io.ReadCloser, tempFile *os } } +func (s *blobCacheDestination) HasThreadSafePutBlob() bool { + return false +} + func (d *blobCacheDestination) PutBlob(ctx context.Context, stream io.Reader, inputInfo types.BlobInfo, cache types.BlobInfoCache, isConfig bool) (types.BlobInfo, error) { var tempfile *os.File var err error diff --git a/vendor/github.com/containers/buildah/pkg/cli/common.go b/vendor/github.com/containers/buildah/pkg/cli/common.go index e3a4fe62a..dc1b2bc69 100644 --- a/vendor/github.com/containers/buildah/pkg/cli/common.go +++ b/vendor/github.com/containers/buildah/pkg/cli/common.go @@ -148,6 +148,10 @@ var ( Name: "loglevel", Usage: "adjust logging level (range from -2 to 3)", }, + cli.StringFlag{ + Name: "platform", + Usage: "CLI compatibility: no action or effect", + }, cli.BoolTFlag{ Name: "pull", Usage: "pull the image if not present", diff --git a/vendor/github.com/containers/buildah/run.go b/vendor/github.com/containers/buildah/run.go index 5d2cd6a32..a0627f489 100644 --- a/vendor/github.com/containers/buildah/run.go +++ b/vendor/github.com/containers/buildah/run.go @@ -26,6 +26,7 @@ import ( "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/ioutils" "github.com/containers/storage/pkg/reexec" + "github.com/containers/storage/pkg/stringid" units "github.com/docker/go-units" digest "github.com/opencontainers/go-digest" "github.com/opencontainers/runtime-spec/specs-go" @@ -944,10 +945,25 @@ func (b *Builder) configureNamespaces(g *generate.Generator, options RunOptions) g.SetHostname(options.Hostname) } else if b.Hostname() != "" { g.SetHostname(b.Hostname()) + } else { + g.SetHostname(stringid.TruncateID(b.ContainerID)) } } else { g.SetHostname("") } + + found := false + spec := g.Spec() + for i := range spec.Process.Env { + if strings.HasPrefix(spec.Process.Env[i], "HOSTNAME=") { + found = true + break + } + } + if !found { + spec.Process.Env = append(spec.Process.Env, fmt.Sprintf("HOSTNAME=%s", spec.Hostname)) + } + return configureNetwork, configureNetworks, nil } diff --git a/vendor/github.com/containers/buildah/unshare/unshare.c b/vendor/github.com/containers/buildah/unshare/unshare.c index 6873e49c2..3865e414f 100644 --- a/vendor/github.com/containers/buildah/unshare/unshare.c +++ b/vendor/github.com/containers/buildah/unshare/unshare.c @@ -79,7 +79,8 @@ void _buildah_unshare(void) pidfd = _buildah_unshare_parse_envint("_Buildah-pid-pipe"); if (pidfd != -1) { snprintf(buf, sizeof(buf), "%llu", (unsigned long long) getpid()); - if (write(pidfd, buf, strlen(buf)) != strlen(buf)) { + size_t size = write(pidfd, buf, strlen(buf)); + if (size != strlen(buf)) { fprintf(stderr, "Error writing PID to pipe on fd %d: %m\n", pidfd); _exit(1); } diff --git a/vendor/github.com/containers/buildah/util/util.go b/vendor/github.com/containers/buildah/util/util.go index 427c8db28..f4cac522e 100644 --- a/vendor/github.com/containers/buildah/util/util.go +++ b/vendor/github.com/containers/buildah/util/util.go @@ -395,57 +395,11 @@ func GetSubIDMappings(user, group string) ([]specs.LinuxIDMapping, []specs.Linux // ParseIDMappings parses mapping triples. func ParseIDMappings(uidmap, gidmap []string) ([]idtools.IDMap, []idtools.IDMap, error) { - nonDigitsToWhitespace := func(r rune) rune { - if strings.IndexRune("0123456789", r) == -1 { - return ' ' - } else { - return r - } - } - parseTriple := func(spec []string) (container, host, size uint32, err error) { - cid, err := strconv.ParseUint(spec[0], 10, 32) - if err != nil { - return 0, 0, 0, fmt.Errorf("error parsing id map value %q: %v", spec[0], err) - } - hid, err := strconv.ParseUint(spec[1], 10, 32) - if err != nil { - return 0, 0, 0, fmt.Errorf("error parsing id map value %q: %v", spec[1], err) - } - sz, err := strconv.ParseUint(spec[2], 10, 32) - if err != nil { - return 0, 0, 0, fmt.Errorf("error parsing id map value %q: %v", spec[2], err) - } - return uint32(cid), uint32(hid), uint32(sz), nil - } - parseIDMap := func(mapSpec []string, mapSetting string) (idmap []idtools.IDMap, err error) { - for _, idMapSpec := range mapSpec { - idSpec := strings.Fields(strings.Map(nonDigitsToWhitespace, idMapSpec)) - if len(idSpec)%3 != 0 { - return nil, errors.Errorf("error initializing ID mappings: %s setting is malformed", mapSetting) - } - for i := range idSpec { - if i%3 != 0 { - continue - } - cid, hid, size, err := parseTriple(idSpec[i : i+3]) - if err != nil { - return nil, errors.Errorf("error initializing ID mappings: %s setting is malformed", mapSetting) - } - mapping := idtools.IDMap{ - ContainerID: int(cid), - HostID: int(hid), - Size: int(size), - } - idmap = append(idmap, mapping) - } - } - return idmap, nil - } - uid, err := parseIDMap(uidmap, "userns-uid-map") + uid, err := idtools.ParseIDMap(uidmap, "userns-uid-map") if err != nil { return nil, nil, err } - gid, err := parseIDMap(gidmap, "userns-gid-map") + gid, err := idtools.ParseIDMap(gidmap, "userns-gid-map") if err != nil { return nil, nil, err } diff --git a/vendor/github.com/containers/buildah/vendor.conf b/vendor/github.com/containers/buildah/vendor.conf index 61325114c..ee8192935 100644 --- a/vendor/github.com/containers/buildah/vendor.conf +++ b/vendor/github.com/containers/buildah/vendor.conf @@ -3,10 +3,10 @@ github.com/blang/semver master github.com/BurntSushi/toml master github.com/containerd/continuity master github.com/containernetworking/cni v0.7.0-alpha1 -github.com/containers/image d53afe179b381fafb427e6b9cf9b1996a98c1067 +github.com/containers/image 0c6cc8e1420001ae39fa89d308b3b3bc5ee81c57 github.com/boltdb/bolt master -github.com/containers/libpod fe4f09493f41f675d24c969d1b60d1a6a45ddb9e -github.com/containers/storage db40f96d853dfced60c563e61fb66ba231ce7c8d +github.com/containers/libpod c8eaf59d5f4bec249db8134c6a9fcfbcac792519 +github.com/containers/storage 60a692f7ce891feb91ce0eda87bd06bfd5651dff github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716 github.com/docker/docker 86f080cff0914e9694068ed78d503701667c4c00 github.com/docker/docker-credential-helpers d68f9aeca33f5fd3f08eeae5e9d175edf4e731d1 @@ -32,7 +32,7 @@ github.com/mistifyio/go-zfs master github.com/moby/moby f8806b18b4b92c5e1980f6e11c917fad201cd73c github.com/mtrmac/gpgme master github.com/Nvveen/Gotty master -github.com/opencontainers/go-digest aa2ec055abd10d26d539eb630a92241b781ce4bc +github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7 github.com/opencontainers/image-spec v1.0.0 github.com/opencontainers/runc master github.com/opencontainers/runtime-spec v1.0.0 @@ -55,10 +55,14 @@ github.com/xeipuuv/gojsonreference master github.com/xeipuuv/gojsonschema master golang.org/x/crypto master golang.org/x/net master +golang.org/x/sync 42b317875d0fa942474b76e1b46a6060d720ae6e golang.org/x/sys master golang.org/x/text master -gopkg.in/cheggaaa/pb.v1 v1.0.13 +gopkg.in/cheggaaa/pb.v1 v1.0.27 gopkg.in/yaml.v2 cd8b52f8269e0feb286dfeef29f8fe4d5b397e0b k8s.io/apimachinery master k8s.io/client-go master k8s.io/kubernetes master +github.com/klauspost/pgzip v1.2.1 +github.com/klauspost/compress v1.4.1 +github.com/klauspost/cpuid v1.2.0 diff --git a/vendor/github.com/containers/image/copy/copy.go b/vendor/github.com/containers/image/copy/copy.go index 013080e8d..89c7e580f 100644 --- a/vendor/github.com/containers/image/copy/copy.go +++ b/vendor/github.com/containers/image/copy/copy.go @@ -2,7 +2,6 @@ package copy import ( "bytes" - "compress/gzip" "context" "fmt" "io" @@ -10,6 +9,7 @@ import ( "reflect" "runtime" "strings" + "sync" "time" "github.com/containers/image/image" @@ -18,9 +18,11 @@ import ( "github.com/containers/image/signature" "github.com/containers/image/transports" "github.com/containers/image/types" + "github.com/klauspost/pgzip" "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/sirupsen/logrus" + "golang.org/x/sync/semaphore" pb "gopkg.in/cheggaaa/pb.v1" ) @@ -32,6 +34,10 @@ type digestingReader struct { validationSucceeded bool } +// maxParallelDownloads is used to limit the maxmimum number of parallel +// downloads. Let's follow Firefox by limiting it to 6. +var maxParallelDownloads = 6 + // newDigestingReader returns an io.Reader implementation with contents of source, which will eventually return a non-EOF error // or set validationSucceeded/validationFailed to true if the source stream does/does not match expectedDigest. // (neither is set if EOF is never reached). @@ -81,6 +87,7 @@ type copier struct { progressInterval time.Duration progress chan types.ProgressProperties blobInfoCache types.BlobInfoCache + copyInParallel bool } // imageCopier tracks state specific to a single image (possibly an item of a manifest list) @@ -145,12 +152,14 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, } }() + copyInParallel := dest.HasThreadSafePutBlob() && rawSource.HasThreadSafeGetBlob() c := &copier{ dest: dest, rawSource: rawSource, reportWriter: reportWriter, progressInterval: options.ProgressInterval, progress: options.Progress, + copyInParallel: copyInParallel, // FIXME? The cache is used for sources and destinations equally, but we only have a SourceCtx and DestinationCtx. // For now, use DestinationCtx (because blob reuse changes the behavior of the destination side more); eventually // we might want to add a separate CommonCtx — or would that be too confusing? @@ -380,11 +389,26 @@ func (ic *imageCopier) updateEmbeddedDockerReference() error { return nil } +// shortDigest returns the first 12 characters of the digest. +func shortDigest(d digest.Digest) string { + return d.Encoded()[:12] +} + +// createProgressBar creates a pb.ProgressBar. +func createProgressBar(srcInfo types.BlobInfo, kind string, writer io.Writer) *pb.ProgressBar { + bar := pb.New(int(srcInfo.Size)).SetUnits(pb.U_BYTES) + bar.SetMaxWidth(80) + bar.ShowTimeLeft = false + bar.ShowPercent = false + bar.Prefix(fmt.Sprintf("Copying %s %s:", kind, shortDigest(srcInfo.Digest))) + bar.Output = writer + return bar +} + // copyLayers copies layers from ic.src/ic.c.rawSource to dest, using and updating ic.manifestUpdates if necessary and ic.canModifyManifest. func (ic *imageCopier) copyLayers(ctx context.Context) error { srcInfos := ic.src.LayerInfos() - destInfos := []types.BlobInfo{} - diffIDs := []digest.Digest{} + numLayers := len(srcInfos) updatedSrcInfos, err := ic.src.LayerInfosForCopy(ctx) if err != nil { return err @@ -397,30 +421,83 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { srcInfos = updatedSrcInfos srcInfosUpdated = true } - for _, srcLayer := range srcInfos { - var ( - destInfo types.BlobInfo - diffID digest.Digest - err error - ) + + type copyLayerData struct { + destInfo types.BlobInfo + diffID digest.Digest + err error + } + + // copyGroup is used to determine if all layers are copied + copyGroup := sync.WaitGroup{} + copyGroup.Add(numLayers) + // copySemaphore is used to limit the number of parallel downloads to + // avoid malicious images causing troubles and to be nice to servers. + var copySemaphore *semaphore.Weighted + if ic.c.copyInParallel { + copySemaphore = semaphore.NewWeighted(int64(maxParallelDownloads)) + } else { + copySemaphore = semaphore.NewWeighted(int64(1)) + } + + data := make([]copyLayerData, numLayers) + copyLayerHelper := func(index int, srcLayer types.BlobInfo, bar *pb.ProgressBar) { + defer bar.Finish() + defer copySemaphore.Release(1) + defer copyGroup.Done() + cld := copyLayerData{} if ic.c.dest.AcceptsForeignLayerURLs() && len(srcLayer.URLs) != 0 { // DiffIDs are, currently, needed only when converting from schema1. // In which case src.LayerInfos will not have URLs because schema1 // does not support them. if ic.diffIDsAreNeeded { - return errors.New("getting DiffID for foreign layers is unimplemented") + cld.err = errors.New("getting DiffID for foreign layers is unimplemented") + bar.Prefix(fmt.Sprintf("Skipping blob %s (DiffID foreign layer unimplemented):", shortDigest(srcLayer.Digest))) + bar.Finish() + } else { + cld.destInfo = srcLayer + logrus.Debugf("Skipping foreign layer %q copy to %s\n", cld.destInfo.Digest, ic.c.dest.Reference().Transport().Name()) + bar.Prefix(fmt.Sprintf("Skipping blob %s (foreign layer):", shortDigest(srcLayer.Digest))) + bar.Add64(bar.Total) + bar.Finish() } - destInfo = srcLayer - ic.c.Printf("Skipping foreign layer %q copy to %s\n", destInfo.Digest, ic.c.dest.Reference().Transport().Name()) } else { - destInfo, diffID, err = ic.copyLayer(ctx, srcLayer) - if err != nil { - return err - } + cld.destInfo, cld.diffID, cld.err = ic.copyLayer(ctx, srcLayer, bar) } - destInfos = append(destInfos, destInfo) - diffIDs = append(diffIDs, diffID) + data[index] = cld } + + progressBars := make([]*pb.ProgressBar, numLayers) + for i, srcInfo := range srcInfos { + bar := createProgressBar(srcInfo, "blob", nil) + progressBars[i] = bar + } + + progressPool := pb.NewPool(progressBars...) + progressPool.Output = ic.c.reportWriter + if err := progressPool.Start(); err != nil { + return errors.Wrapf(err, "error creating progress-bar pool") + } + + for i, srcLayer := range srcInfos { + copySemaphore.Acquire(ctx, 1) + go copyLayerHelper(i, srcLayer, progressBars[i]) + } + + destInfos := make([]types.BlobInfo, numLayers) + diffIDs := make([]digest.Digest, numLayers) + + copyGroup.Wait() + progressPool.Stop() + + for i, cld := range data { + if cld.err != nil { + return cld.err + } + destInfos[i] = cld.destInfo + diffIDs[i] = cld.diffID + } + ic.manifestUpdates.InformationOnly.LayerInfos = destInfos if ic.diffIDsAreNeeded { ic.manifestUpdates.InformationOnly.LayerDiffIDs = diffIDs @@ -487,12 +564,14 @@ func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context) ([]byte func (c *copier) copyConfig(ctx context.Context, src types.Image) error { srcInfo := src.ConfigInfo() if srcInfo.Digest != "" { - c.Printf("Copying config %s\n", srcInfo.Digest) configBlob, err := src.ConfigBlob(ctx) if err != nil { return errors.Wrapf(err, "Error reading config blob %s", srcInfo.Digest) } - destInfo, err := c.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, false, true) + bar := createProgressBar(srcInfo, "config", c.reportWriter) + defer bar.Finish() + bar.Start() + destInfo, err := c.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, false, true, bar) if err != nil { return err } @@ -512,7 +591,7 @@ type diffIDResult struct { // copyLayer copies a layer with srcInfo (with known Digest and possibly known Size) in src to dest, perhaps compressing it if canCompress, // and returns a complete blobInfo of the copied layer, and a value for LayerDiffIDs if diffIDIsNeeded -func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo) (types.BlobInfo, digest.Digest, error) { +func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, bar *pb.ProgressBar) (types.BlobInfo, digest.Digest, error) { cachedDiffID := ic.c.blobInfoCache.UncompressedDigest(srcInfo.Digest) // May be "" diffIDIsNeeded := ic.diffIDsAreNeeded && cachedDiffID == "" @@ -523,13 +602,14 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo) (t return types.BlobInfo{}, "", errors.Wrapf(err, "Error trying to reuse blob %s at destination", srcInfo.Digest) } if reused { - ic.c.Printf("Skipping fetch of repeat blob %s\n", srcInfo.Digest) + bar.Prefix(fmt.Sprintf("Skipping blob %s (already present):", shortDigest(srcInfo.Digest))) + bar.Add64(bar.Total) + bar.Finish() return blobInfo, cachedDiffID, nil } } // Fallback: copy the layer, computing the diffID if we need to do so - ic.c.Printf("Copying blob %s\n", srcInfo.Digest) srcStream, srcBlobSize, err := ic.c.rawSource.GetBlob(ctx, srcInfo, ic.c.blobInfoCache) if err != nil { return types.BlobInfo{}, "", errors.Wrapf(err, "Error reading blob %s", srcInfo.Digest) @@ -537,7 +617,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo) (t defer srcStream.Close() blobInfo, diffIDChan, err := ic.copyLayerFromStream(ctx, srcStream, types.BlobInfo{Digest: srcInfo.Digest, Size: srcBlobSize}, - diffIDIsNeeded) + diffIDIsNeeded, bar) if err != nil { return types.BlobInfo{}, "", err } @@ -565,7 +645,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo) (t // perhaps compressing the stream if canCompress, // and returns a complete blobInfo of the copied blob and perhaps a <-chan diffIDResult if diffIDIsNeeded, to be read by the caller. func (ic *imageCopier) copyLayerFromStream(ctx context.Context, srcStream io.Reader, srcInfo types.BlobInfo, - diffIDIsNeeded bool) (types.BlobInfo, <-chan diffIDResult, error) { + diffIDIsNeeded bool, bar *pb.ProgressBar) (types.BlobInfo, <-chan diffIDResult, error) { var getDiffIDRecorder func(compression.DecompressorFunc) io.Writer // = nil var diffIDChan chan diffIDResult @@ -589,7 +669,7 @@ func (ic *imageCopier) copyLayerFromStream(ctx context.Context, srcStream io.Rea return pipeWriter } } - blobInfo, err := ic.c.copyBlobFromStream(ctx, srcStream, srcInfo, getDiffIDRecorder, ic.canModifyManifest, false) // Sets err to nil on success + blobInfo, err := ic.c.copyBlobFromStream(ctx, srcStream, srcInfo, getDiffIDRecorder, ic.canModifyManifest, false, bar) // Sets err to nil on success return blobInfo, diffIDChan, err // We need the defer … pipeWriter.CloseWithError() to happen HERE so that the caller can block on reading from diffIDChan } @@ -626,7 +706,7 @@ func computeDiffID(stream io.Reader, decompressor compression.DecompressorFunc) // and returns a complete blobInfo of the copied blob. func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, srcInfo types.BlobInfo, getOriginalLayerCopyWriter func(decompressor compression.DecompressorFunc) io.Writer, - canModifyBlob bool, isConfig bool) (types.BlobInfo, error) { + canModifyBlob bool, isConfig bool, bar *pb.ProgressBar) (types.BlobInfo, error) { // The copying happens through a pipeline of connected io.Readers. // === Input: srcStream @@ -649,16 +729,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr return types.BlobInfo{}, errors.Wrapf(err, "Error reading blob %s", srcInfo.Digest) } isCompressed := decompressor != nil - - // === Report progress using a pb.Reader. - bar := pb.New(int(srcInfo.Size)).SetUnits(pb.U_BYTES) - bar.Output = c.reportWriter - bar.SetMaxWidth(80) - bar.ShowTimeLeft = false - bar.ShowPercent = false - bar.Start() destStream = bar.NewProxyReader(destStream) - defer bar.Finish() // === Send a copy of the original, uncompressed, stream, to a separate path if necessary. var originalLayerReader io.Reader // DO NOT USE this other than to drain the input if no other consumer in the pipeline has done so. @@ -761,7 +832,7 @@ func compressGoroutine(dest *io.PipeWriter, src io.Reader) { dest.CloseWithError(err) // CloseWithError(nil) is equivalent to Close() }() - zipper := gzip.NewWriter(dest) + zipper := pgzip.NewWriter(dest) defer zipper.Close() _, err = io.Copy(zipper, src) // Sets err to nil, i.e. causes dest.Close() diff --git a/vendor/github.com/containers/image/directory/directory_dest.go b/vendor/github.com/containers/image/directory/directory_dest.go index d75c195b2..4b2ab022e 100644 --- a/vendor/github.com/containers/image/directory/directory_dest.go +++ b/vendor/github.com/containers/image/directory/directory_dest.go @@ -124,6 +124,11 @@ func (d *dirImageDestination) IgnoresEmbeddedDockerReference() bool { return false // N/A, DockerReference() returns nil. } +// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. +func (d *dirImageDestination) HasThreadSafePutBlob() bool { + return false +} + // PutBlob writes contents of stream and returns data representing the result (with all data filled in). // inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it. // inputInfo.Size is the expected length of stream, if known. diff --git a/vendor/github.com/containers/image/directory/directory_src.go b/vendor/github.com/containers/image/directory/directory_src.go index 3625def80..59b625397 100644 --- a/vendor/github.com/containers/image/directory/directory_src.go +++ b/vendor/github.com/containers/image/directory/directory_src.go @@ -48,6 +48,11 @@ func (s *dirImageSource) GetManifest(ctx context.Context, instanceDigest *digest return m, manifest.GuessMIMEType(m), err } +// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. +func (s *dirImageSource) HasThreadSafeGetBlob() bool { + return false +} + // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. diff --git a/vendor/github.com/containers/image/docker/docker_client.go b/vendor/github.com/containers/image/docker/docker_client.go index 7f55dbe7f..23d2ac70f 100644 --- a/vendor/github.com/containers/image/docker/docker_client.go +++ b/vendor/github.com/containers/image/docker/docker_client.go @@ -13,6 +13,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "time" "github.com/containers/image/docker/reference" @@ -84,6 +85,7 @@ type dockerClient struct { registry string client *http.Client insecureSkipTLSVerify bool + // The following members are not set by newDockerClient and must be set by callers if needed. username string password string @@ -95,8 +97,13 @@ type dockerClient struct { scheme string // Empty value also used to indicate detectProperties() has not yet succeeded. challenges []challenge supportsSignatures bool - // Private state for setupRequestAuth - tokenCache map[string]bearerToken + + // Private state for setupRequestAuth (key: string, value: bearerToken) + tokenCache sync.Map + // detectPropertiesError caches the initial error. + detectPropertiesError error + // detectPropertiesOnce is used to execuute detectProperties() at most once in in makeRequest(). + detectPropertiesOnce sync.Once } type authScope struct { @@ -262,7 +269,6 @@ func newDockerClient(sys *types.SystemContext, registry, reference string) (*doc registry: registry, client: &http.Client{Transport: tr}, insecureSkipTLSVerify: skipVerify, - tokenCache: map[string]bearerToken{}, }, nil } @@ -473,14 +479,18 @@ func (c *dockerClient) setupRequestAuth(req *http.Request) error { cacheKey = fmt.Sprintf("%s:%s", c.extraScope.remoteName, c.extraScope.actions) scopes = append(scopes, *c.extraScope) } - token, ok := c.tokenCache[cacheKey] - if !ok || time.Now().After(token.expirationTime) { + var token bearerToken + t, inCache := c.tokenCache.Load(cacheKey) + if inCache { + token = t.(bearerToken) + } + if !inCache || time.Now().After(token.expirationTime) { t, err := c.getBearerToken(req.Context(), challenge, scopes) if err != nil { return err } token = *t - c.tokenCache[cacheKey] = token + c.tokenCache.Store(cacheKey, token) } req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token.Token)) return nil @@ -545,9 +555,9 @@ func (c *dockerClient) getBearerToken(ctx context.Context, challenge challenge, return newBearerTokenFromJSONBlob(tokenBlob) } -// detectProperties detects various properties of the registry. -// See the dockerClient documentation for members which are affected by this. -func (c *dockerClient) detectProperties(ctx context.Context) error { +// detectPropertiesHelper performs the work of detectProperties which executes +// it at most once. +func (c *dockerClient) detectPropertiesHelper(ctx context.Context) error { if c.scheme != "" { return nil } @@ -604,6 +614,13 @@ func (c *dockerClient) detectProperties(ctx context.Context) error { return err } +// detectProperties detects various properties of the registry. +// See the dockerClient documentation for members which are affected by this. +func (c *dockerClient) detectProperties(ctx context.Context) error { + c.detectPropertiesOnce.Do(func() { c.detectPropertiesError = c.detectPropertiesHelper(ctx) }) + return c.detectPropertiesError +} + // getExtensionsSignatures returns signatures from the X-Registry-Supports-Signatures API extension, // using the original data structures. func (c *dockerClient) getExtensionsSignatures(ctx context.Context, ref dockerReference, manifestDigest digest.Digest) (*extensionSignatureList, error) { diff --git a/vendor/github.com/containers/image/docker/docker_image_dest.go b/vendor/github.com/containers/image/docker/docker_image_dest.go index 2f471f648..973d160d0 100644 --- a/vendor/github.com/containers/image/docker/docker_image_dest.go +++ b/vendor/github.com/containers/image/docker/docker_image_dest.go @@ -111,6 +111,11 @@ func (c *sizeCounter) Write(p []byte) (n int, err error) { return len(p), nil } +// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. +func (d *dockerImageDestination) HasThreadSafePutBlob() bool { + return false +} + // PutBlob writes contents of stream and returns data representing the result (with all data filled in). // inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it. // inputInfo.Size is the expected length of stream, if known. diff --git a/vendor/github.com/containers/image/docker/docker_image_src.go b/vendor/github.com/containers/image/docker/docker_image_src.go index fbed6297f..c88ff2f34 100644 --- a/vendor/github.com/containers/image/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/docker/docker_image_src.go @@ -161,6 +161,11 @@ func getBlobSize(resp *http.Response) int64 { return size } +// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. +func (s *dockerImageSource) HasThreadSafeGetBlob() bool { + return true +} + // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. diff --git a/vendor/github.com/containers/image/docker/tarfile/dest.go b/vendor/github.com/containers/image/docker/tarfile/dest.go index ad8a48a03..5f30eddbc 100644 --- a/vendor/github.com/containers/image/docker/tarfile/dest.go +++ b/vendor/github.com/containers/image/docker/tarfile/dest.go @@ -82,6 +82,11 @@ func (d *Destination) IgnoresEmbeddedDockerReference() bool { return false // N/A, we only accept schema2 images where EmbeddedDockerReferenceConflicts() is always false. } +// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. +func (d *Destination) HasThreadSafePutBlob() bool { + return false +} + // PutBlob writes contents of stream and returns data representing the result (with all data filled in). // inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it. // inputInfo.Size is the expected length of stream, if known. diff --git a/vendor/github.com/containers/image/docker/tarfile/src.go b/vendor/github.com/containers/image/docker/tarfile/src.go index d94ed9783..889e5f8e8 100644 --- a/vendor/github.com/containers/image/docker/tarfile/src.go +++ b/vendor/github.com/containers/image/docker/tarfile/src.go @@ -397,6 +397,11 @@ func (r uncompressedReadCloser) Close() error { return res } +// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. +func (s *Source) HasThreadSafeGetBlob() bool { + return false +} + // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. diff --git a/vendor/github.com/containers/image/oci/archive/oci_dest.go b/vendor/github.com/containers/image/oci/archive/oci_dest.go index 3997ac2ee..9571c37e2 100644 --- a/vendor/github.com/containers/image/oci/archive/oci_dest.go +++ b/vendor/github.com/containers/image/oci/archive/oci_dest.go @@ -77,6 +77,11 @@ func (d *ociArchiveImageDestination) IgnoresEmbeddedDockerReference() bool { return d.unpackedDest.IgnoresEmbeddedDockerReference() } +// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. +func (d *ociArchiveImageDestination) HasThreadSafePutBlob() bool { + return false +} + // PutBlob writes contents of stream and returns data representing the result. // inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it. // inputInfo.Size is the expected length of stream, if known. diff --git a/vendor/github.com/containers/image/oci/archive/oci_src.go b/vendor/github.com/containers/image/oci/archive/oci_src.go index 084d818f7..ca74f950b 100644 --- a/vendor/github.com/containers/image/oci/archive/oci_src.go +++ b/vendor/github.com/containers/image/oci/archive/oci_src.go @@ -76,6 +76,11 @@ func (s *ociArchiveImageSource) GetManifest(ctx context.Context, instanceDigest return s.unpackedSrc.GetManifest(ctx, instanceDigest) } +// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. +func (s *ociArchiveImageSource) HasThreadSafeGetBlob() bool { + return false +} + // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. diff --git a/vendor/github.com/containers/image/oci/layout/oci_dest.go b/vendor/github.com/containers/image/oci/layout/oci_dest.go index b5a6e08f8..db102184d 100644 --- a/vendor/github.com/containers/image/oci/layout/oci_dest.go +++ b/vendor/github.com/containers/image/oci/layout/oci_dest.go @@ -107,6 +107,11 @@ func (d *ociImageDestination) IgnoresEmbeddedDockerReference() bool { return false // N/A, DockerReference() returns nil. } +// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. +func (d *ociImageDestination) HasThreadSafePutBlob() bool { + return false +} + // PutBlob writes contents of stream and returns data representing the result. // inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it. // inputInfo.Size is the expected length of stream, if known. diff --git a/vendor/github.com/containers/image/oci/layout/oci_src.go b/vendor/github.com/containers/image/oci/layout/oci_src.go index 086a7040d..cc536f69e 100644 --- a/vendor/github.com/containers/image/oci/layout/oci_src.go +++ b/vendor/github.com/containers/image/oci/layout/oci_src.go @@ -92,6 +92,11 @@ func (s *ociImageSource) GetManifest(ctx context.Context, instanceDigest *digest return m, mimeType, nil } +// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. +func (s *ociImageSource) HasThreadSafeGetBlob() bool { + return false +} + // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. diff --git a/vendor/github.com/containers/image/openshift/openshift.go b/vendor/github.com/containers/image/openshift/openshift.go index 0cce1e6c7..814c3eea1 100644 --- a/vendor/github.com/containers/image/openshift/openshift.go +++ b/vendor/github.com/containers/image/openshift/openshift.go @@ -211,6 +211,11 @@ func (s *openshiftImageSource) GetManifest(ctx context.Context, instanceDigest * return s.docker.GetManifest(ctx, instanceDigest) } +// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. +func (s *openshiftImageSource) HasThreadSafeGetBlob() bool { + return false +} + // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. @@ -378,6 +383,11 @@ func (d *openshiftImageDestination) IgnoresEmbeddedDockerReference() bool { return d.docker.IgnoresEmbeddedDockerReference() } +// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. +func (d *openshiftImageDestination) HasThreadSafePutBlob() bool { + return false +} + // PutBlob writes contents of stream and returns data representing the result (with all data filled in). // inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it. // inputInfo.Size is the expected length of stream, if known. diff --git a/vendor/github.com/containers/image/ostree/ostree_dest.go b/vendor/github.com/containers/image/ostree/ostree_dest.go index 064898948..d69f4fa33 100644 --- a/vendor/github.com/containers/image/ostree/ostree_dest.go +++ b/vendor/github.com/containers/image/ostree/ostree_dest.go @@ -4,7 +4,6 @@ package ostree import ( "bytes" - "compress/gzip" "context" "encoding/base64" "encoding/json" @@ -24,6 +23,7 @@ import ( "github.com/containers/image/manifest" "github.com/containers/image/types" "github.com/containers/storage/pkg/archive" + "github.com/klauspost/pgzip" "github.com/opencontainers/go-digest" selinux "github.com/opencontainers/selinux/go-selinux" "github.com/ostreedev/ostree-go/pkg/otbuiltin" @@ -132,6 +132,11 @@ func (d *ostreeImageDestination) IgnoresEmbeddedDockerReference() bool { return false // N/A, DockerReference() returns nil. } +// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. +func (d *ostreeImageDestination) HasThreadSafePutBlob() bool { + return false +} + // PutBlob writes contents of stream and returns data representing the result. // inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it. // inputInfo.Size is the expected length of stream, if known. @@ -249,7 +254,7 @@ func (d *ostreeImageDestination) ostreeCommit(repo *otbuiltin.Repo, branch strin } func generateTarSplitMetadata(output *bytes.Buffer, file string) (digest.Digest, int64, error) { - mfz := gzip.NewWriter(output) + mfz := pgzip.NewWriter(output) defer mfz.Close() metaPacker := storage.NewJSONPacker(mfz) diff --git a/vendor/github.com/containers/image/ostree/ostree_src.go b/vendor/github.com/containers/image/ostree/ostree_src.go index e73cae198..df432c9f3 100644 --- a/vendor/github.com/containers/image/ostree/ostree_src.go +++ b/vendor/github.com/containers/image/ostree/ostree_src.go @@ -4,7 +4,6 @@ package ostree import ( "bytes" - "compress/gzip" "context" "encoding/base64" "fmt" @@ -17,6 +16,7 @@ import ( "github.com/containers/image/manifest" "github.com/containers/image/types" "github.com/containers/storage/pkg/ioutils" + "github.com/klauspost/pgzip" "github.com/opencontainers/go-digest" glib "github.com/ostreedev/ostree-go/pkg/glibobject" "github.com/pkg/errors" @@ -255,6 +255,11 @@ func (s *ostreeImageSource) readSingleFile(commit, path string) (io.ReadCloser, return getter.Get(path) } +// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. +func (s *ostreeImageSource) HasThreadSafeGetBlob() bool { + return false +} + // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. @@ -304,7 +309,7 @@ func (s *ostreeImageSource) GetBlob(ctx context.Context, info types.BlobInfo, ca } mf := bytes.NewReader(tarsplit) - mfz, err := gzip.NewReader(mf) + mfz, err := pgzip.NewReader(mf) if err != nil { return nil, 0, err } diff --git a/vendor/github.com/containers/image/pkg/blobinfocache/default.go b/vendor/github.com/containers/image/pkg/blobinfocache/default.go index 6da9f2805..459ae5c06 100644 --- a/vendor/github.com/containers/image/pkg/blobinfocache/default.go +++ b/vendor/github.com/containers/image/pkg/blobinfocache/default.go @@ -54,7 +54,7 @@ func DefaultCache(sys *types.SystemContext) types.BlobInfoCache { } path := filepath.Join(dir, blobInfoCacheFilename) if err := os.MkdirAll(dir, 0700); err != nil { - logrus.Debugf("Error creating parent directories for %s, using a memory-only cache: %v", err) + logrus.Debugf("Error creating parent directories for %s, using a memory-only cache: %v", blobInfoCacheFilename, err) return NewMemoryCache() } diff --git a/vendor/github.com/containers/image/pkg/compression/compression.go b/vendor/github.com/containers/image/pkg/compression/compression.go index a39523e4f..aad2bfcf2 100644 --- a/vendor/github.com/containers/image/pkg/compression/compression.go +++ b/vendor/github.com/containers/image/pkg/compression/compression.go @@ -3,10 +3,10 @@ package compression import ( "bytes" "compress/bzip2" - "compress/gzip" "io" "io/ioutil" + "github.com/klauspost/pgzip" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/ulikunitz/xz" @@ -18,7 +18,7 @@ type DecompressorFunc func(io.Reader) (io.ReadCloser, error) // GzipDecompressor is a DecompressorFunc for the gzip compression algorithm. func GzipDecompressor(r io.Reader) (io.ReadCloser, error) { - return gzip.NewReader(r) + return pgzip.NewReader(r) } // Bzip2Decompressor is a DecompressorFunc for the bzip2 compression algorithm. diff --git a/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go b/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go index afc7312d1..9e3e9cfe1 100644 --- a/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go +++ b/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go @@ -318,8 +318,37 @@ func FindUnqualifiedSearchRegistries(ctx *types.SystemContext) ([]Registry, erro return unqualified, nil } -// FindRegistry returns the Registry with the longest prefix for ref. If no -// Registry prefixes the image, nil is returned. +// refMatchesPrefix returns true iff ref, +// which is a registry, repository namespace, repository or image reference (as formatted by +// reference.Domain(), reference.Named.Name() or reference.Reference.String() +// — note that this requires the name to start with an explicit hostname!), +// matches a Registry.Prefix value. +// (This is split from the caller primarily to make testing easier.) +func refMatchesPrefix(ref, prefix string) bool { + switch { + case len(ref) < len(prefix): + return false + case len(ref) == len(prefix): + return ref == prefix + case len(ref) > len(prefix): + if !strings.HasPrefix(ref, prefix) { + return false + } + c := ref[len(prefix)] + // This allows "example.com:5000" to match "example.com", + // which is unintended; that will get fixed eventually, DON'T RELY + // ON THE CURRENT BEHAVIOR. + return c == ':' || c == '/' || c == '@' + default: + panic("Internal error: impossible comparison outcome") + } +} + +// FindRegistry returns the Registry with the longest prefix for ref, +// which is a registry, repository namespace repository or image reference (as formatted by +// reference.Domain(), reference.Named.Name() or reference.Reference.String() +// — note that this requires the name to start with an explicit hostname!). +// If no Registry prefixes the image, nil is returned. func FindRegistry(ctx *types.SystemContext, ref string) (*Registry, error) { registries, err := GetRegistries(ctx) if err != nil { @@ -329,7 +358,7 @@ func FindRegistry(ctx *types.SystemContext, ref string) (*Registry, error) { reg := Registry{} prefixLen := 0 for _, r := range registries { - if strings.HasPrefix(ref, r.Prefix+"/") || ref == r.Prefix { + if refMatchesPrefix(ref, r.Prefix) { length := len(r.Prefix) if length > prefixLen { reg = r diff --git a/vendor/github.com/containers/image/storage/storage_image.go b/vendor/github.com/containers/image/storage/storage_image.go index bd6813119..b53fbdf6e 100644 --- a/vendor/github.com/containers/image/storage/storage_image.go +++ b/vendor/github.com/containers/image/storage/storage_image.go @@ -11,6 +11,7 @@ import ( "io/ioutil" "os" "path/filepath" + "sync" "sync/atomic" "github.com/containers/image/image" @@ -47,6 +48,7 @@ type storageImageSource struct { image *storage.Image layerPosition map[digest.Digest]int // Where we are in reading a blob's layers cachedManifest []byte // A cached copy of the manifest, if already known, or nil + getBlobMutex sync.Mutex // Mutex to sync state for parallel GetBlob executions SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice } @@ -56,6 +58,7 @@ type storageImageDestination struct { nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs manifest []byte // Manifest contents, temporary signatures []byte // Signature contents, temporary + putBlobMutex sync.Mutex // Mutex to sync state for parallel PutBlob executions blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them @@ -91,15 +94,20 @@ func newImageSource(imageRef storageReference) (*storageImageSource, error) { } // Reference returns the image reference that we used to find this image. -func (s storageImageSource) Reference() types.ImageReference { +func (s *storageImageSource) Reference() types.ImageReference { return s.imageRef } // Close cleans up any resources we tied up while reading the image. -func (s storageImageSource) Close() error { +func (s *storageImageSource) Close() error { return nil } +// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. +func (s *storageImageSource) HasThreadSafeGetBlob() bool { + return true +} + // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. @@ -137,8 +145,10 @@ func (s *storageImageSource) getBlobAndLayerID(info types.BlobInfo) (rc io.ReadC // Step through the list of matching layers. Tests may want to verify that if we have multiple layers // which claim to have the same contents, that we actually do have multiple layers, otherwise we could // just go ahead and use the first one every time. + s.getBlobMutex.Lock() i := s.layerPosition[info.Digest] s.layerPosition[info.Digest] = i + 1 + s.getBlobMutex.Unlock() if len(layers) > 0 { layer = layers[i%len(layers)] } @@ -300,7 +310,7 @@ func newImageDestination(imageRef storageReference) (*storageImageDestination, e // Reference returns the reference used to set up this destination. Note that this should directly correspond to user's intent, // e.g. it should use the public hostname instead of the result of resolving CNAMEs or following redirects. -func (s storageImageDestination) Reference() types.ImageReference { +func (s *storageImageDestination) Reference() types.ImageReference { return s.imageRef } @@ -309,7 +319,7 @@ func (s *storageImageDestination) Close() error { return os.RemoveAll(s.directory) } -func (s storageImageDestination) DesiredLayerCompression() types.LayerCompression { +func (s *storageImageDestination) DesiredLayerCompression() types.LayerCompression { // We ultimately have to decompress layers to populate trees on disk, // so callers shouldn't bother compressing them before handing them to // us, if they're not already compressed. @@ -320,6 +330,11 @@ func (s *storageImageDestination) computeNextBlobCacheFile() string { return filepath.Join(s.directory, fmt.Sprintf("%d", atomic.AddInt32(&s.nextTempFileID, 1))) } +// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. +func (s *storageImageDestination) HasThreadSafePutBlob() bool { + return true +} + // PutBlob writes contents of stream and returns data representing the result. // inputInfo.Digest can be optionally provided if known; it is not mandatory for the implementation to verify it. // inputInfo.Size is the expected length of stream, if known. @@ -370,9 +385,11 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader, return errorBlobInfo, ErrBlobSizeMismatch } // Record information about the blob. + s.putBlobMutex.Lock() s.blobDiffIDs[hasher.Digest()] = diffID.Digest() s.fileSizes[hasher.Digest()] = counter.Count s.filenames[hasher.Digest()] = filename + s.putBlobMutex.Unlock() blobDigest := blobinfo.Digest if blobDigest.Validate() != nil { blobDigest = hasher.Digest() @@ -398,6 +415,9 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader, // If the transport can not reuse the requested blob, TryReusingBlob returns (false, {}, nil); it returns a non-nil error only on an unexpected failure. // May use and/or update cache. func (s *storageImageDestination) TryReusingBlob(ctx context.Context, blobinfo types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) { + // lock the entire method as it executes fairly quickly + s.putBlobMutex.Lock() + defer s.putBlobMutex.Unlock() if blobinfo.Digest == "" { return false, types.BlobInfo{}, errors.Errorf(`Can not check for a blob with unknown digest`) } diff --git a/vendor/github.com/containers/image/tarball/tarball_src.go b/vendor/github.com/containers/image/tarball/tarball_src.go index ee963b8d8..76e3e755f 100644 --- a/vendor/github.com/containers/image/tarball/tarball_src.go +++ b/vendor/github.com/containers/image/tarball/tarball_src.go @@ -2,7 +2,6 @@ package tarball import ( "bytes" - "compress/gzip" "context" "encoding/json" "fmt" @@ -14,7 +13,7 @@ import ( "time" "github.com/containers/image/types" - + "github.com/klauspost/pgzip" digest "github.com/opencontainers/go-digest" imgspecs "github.com/opencontainers/image-spec/specs-go" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -77,7 +76,7 @@ func (r *tarballReference) NewImageSource(ctx context.Context, sys *types.System // Set up to digest the file after we maybe decompress it. diffIDdigester := digest.Canonical.Digester() - uncompressed, err := gzip.NewReader(reader) + uncompressed, err := pgzip.NewReader(reader) if err == nil { // It is compressed, so the diffID is the digest of the uncompressed version reader = io.TeeReader(uncompressed, diffIDdigester.Hash()) @@ -207,6 +206,11 @@ func (is *tarballImageSource) Close() error { return nil } +// HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. +func (is *tarballImageSource) HasThreadSafeGetBlob() bool { + return false +} + // GetBlob returns a stream for the specified blob, and the blob’s size (or -1 if unknown). // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. diff --git a/vendor/github.com/containers/image/types/types.go b/vendor/github.com/containers/image/types/types.go index dda332776..9fdab2314 100644 --- a/vendor/github.com/containers/image/types/types.go +++ b/vendor/github.com/containers/image/types/types.go @@ -198,6 +198,8 @@ type ImageSource interface { // The Digest field in BlobInfo is guaranteed to be provided, Size may be -1 and MediaType may be optionally provided. // May update BlobInfoCache, preferably after it knows for certain that a blob truly exists at a specific location. GetBlob(context.Context, BlobInfo, BlobInfoCache) (io.ReadCloser, int64, error) + // HasThreadSafeGetBlob indicates whether GetBlob can be executed concurrently. + HasThreadSafeGetBlob() bool // GetSignatures returns the image's signatures. It may use a remote (= slow) service. // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve signatures for // (when the primary manifest is a manifest list); this never happens if the primary manifest is not a manifest list @@ -264,6 +266,8 @@ type ImageDestination interface { // to any other readers for download using the supplied digest. // If stream.Read() at any time, ESPECIALLY at end of input, returns an error, PutBlob MUST 1) fail, and 2) delete any data stored so far. PutBlob(ctx context.Context, stream io.Reader, inputInfo BlobInfo, cache BlobInfoCache, isConfig bool) (BlobInfo, error) + // HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. + HasThreadSafePutBlob() bool // TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). // info.Digest must not be empty. diff --git a/vendor/github.com/containers/image/vendor.conf b/vendor/github.com/containers/image/vendor.conf index 88537981a..bbbb1a458 100644 --- a/vendor/github.com/containers/image/vendor.conf +++ b/vendor/github.com/containers/image/vendor.conf @@ -16,7 +16,7 @@ github.com/imdario/mergo 50d4dbd4eb0e84778abe37cefef140271d96fade github.com/mattn/go-runewidth 14207d285c6c197daabb5c9793d63e7af9ab2d50 github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062 github.com/mtrmac/gpgme b2432428689ca58c2b8e8dea9449d3295cf96fc9 -github.com/opencontainers/go-digest aa2ec055abd10d26d539eb630a92241b781ce4bc +github.com/opencontainers/go-digest c9281466c8b2f606084ac71339773efd177436e7 github.com/opencontainers/image-spec v1.0.0 github.com/opencontainers/runc 6b1d0e76f239ffb435445e5ae316d2676c07c6e3 github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9 @@ -26,8 +26,9 @@ github.com/stretchr/testify 4d4bfba8f1d1027c4fdbe371823030df51419987 github.com/vbatts/tar-split v0.10.2 golang.org/x/crypto 453249f01cfeb54c3d549ddb75ff152ca243f9d8 golang.org/x/net 6b27048ae5e6ad1ef927e72e437531493de612fe +golang.org/x/sync 42b317875d0fa942474b76e1b46a6060d720ae6e golang.org/x/sys 43e60d72a8e2bd92ee98319ba9a384a0e9837c08 -gopkg.in/cheggaaa/pb.v1 d7e6ca3010b6f084d8056847f55d7f572f180678 +gopkg.in/cheggaaa/pb.v1 v1.0.27 gopkg.in/yaml.v2 a3f3340b5840cee44f372bddb5880fcbc419b46a k8s.io/client-go bcde30fb7eaed76fd98a36b4120321b94995ffb6 github.com/xeipuuv/gojsonschema master @@ -36,7 +37,7 @@ github.com/xeipuuv/gojsonpointer master github.com/tchap/go-patricia v2.2.6 github.com/opencontainers/selinux 077c8b6d1c18456fb7c792bc0de52295a0d1900e github.com/BurntSushi/toml b26d9c308763d68093482582cea63d69be07a0f0 -github.com/ostreedev/ostree-go aeb02c6b6aa2889db3ef62f7855650755befd460 +github.com/ostreedev/ostree-go 56f3a639dbc0f2f5051c6d52dade28a882ba78ce github.com/gogo/protobuf fcdc5011193ff531a548e9b0301828d5a5b97fd8 github.com/pquerna/ffjson master github.com/syndtr/gocapability master @@ -44,3 +45,6 @@ github.com/Microsoft/go-winio ab35fc04b6365e8fcb18e6e9e41ea4a02b10b175 github.com/Microsoft/hcsshim eca7177590cdcbd25bbc5df27e3b693a54b53a6a github.com/ulikunitz/xz v0.5.4 github.com/boltdb/bolt master +github.com/klauspost/pgzip v1.2.1 +github.com/klauspost/compress v1.4.1 +github.com/klauspost/cpuid v1.2.0 diff --git a/vendor/github.com/docker/docker/pkg/archive/example_changes.go b/vendor/github.com/docker/docker/pkg/archive/example_changes.go new file mode 100644 index 000000000..495db809e --- /dev/null +++ b/vendor/github.com/docker/docker/pkg/archive/example_changes.go @@ -0,0 +1,97 @@ +// +build ignore + +// Simple tool to create an archive stream from an old and new directory +// +// By default it will stream the comparison of two temporary directories with junk files +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "path" + + "github.com/docker/docker/pkg/archive" + "github.com/sirupsen/logrus" +) + +var ( + flDebug = flag.Bool("D", false, "debugging output") + flNewDir = flag.String("newdir", "", "") + flOldDir = flag.String("olddir", "", "") + log = logrus.New() +) + +func main() { + flag.Usage = func() { + fmt.Println("Produce a tar from comparing two directory paths. By default a demo tar is created of around 200 files (including hardlinks)") + fmt.Printf("%s [OPTIONS]\n", os.Args[0]) + flag.PrintDefaults() + } + flag.Parse() + log.Out = os.Stderr + if (len(os.Getenv("DEBUG")) > 0) || *flDebug { + logrus.SetLevel(logrus.DebugLevel) + } + var newDir, oldDir string + + if len(*flNewDir) == 0 { + var err error + newDir, err = ioutil.TempDir("", "docker-test-newDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(newDir) + if _, err := prepareUntarSourceDirectory(100, newDir, true); err != nil { + log.Fatal(err) + } + } else { + newDir = *flNewDir + } + + if len(*flOldDir) == 0 { + oldDir, err := ioutil.TempDir("", "docker-test-oldDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(oldDir) + } else { + oldDir = *flOldDir + } + + changes, err := archive.ChangesDirs(newDir, oldDir) + if err != nil { + log.Fatal(err) + } + + a, err := archive.ExportChanges(newDir, changes) + if err != nil { + log.Fatal(err) + } + defer a.Close() + + i, err := io.Copy(os.Stdout, a) + if err != nil && err != io.EOF { + log.Fatal(err) + } + fmt.Fprintf(os.Stderr, "wrote archive of %d bytes", i) +} + +func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) { + fileData := []byte("fooo") + for n := 0; n < numberOfFiles; n++ { + fileName := fmt.Sprintf("file-%d", n) + if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil { + return 0, err + } + if makeLinks { + if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil { + return 0, err + } + } + } + totalSize := numberOfFiles * len(fileData) + return totalSize, nil +} diff --git a/vendor/github.com/docker/docker/pkg/sysinfo/README.md b/vendor/github.com/docker/docker/pkg/sysinfo/README.md deleted file mode 100644 index c1530cef0..000000000 --- a/vendor/github.com/docker/docker/pkg/sysinfo/README.md +++ /dev/null @@ -1 +0,0 @@ -SysInfo stores information about which features a kernel supports. diff --git a/vendor/github.com/docker/docker/pkg/sysinfo/numcpu.go b/vendor/github.com/docker/docker/pkg/sysinfo/numcpu.go deleted file mode 100644 index aeb1a3a80..000000000 --- a/vendor/github.com/docker/docker/pkg/sysinfo/numcpu.go +++ /dev/null @@ -1,12 +0,0 @@ -// +build !linux,!windows - -package sysinfo - -import ( - "runtime" -) - -// NumCPU returns the number of CPUs -func NumCPU() int { - return runtime.NumCPU() -} diff --git a/vendor/github.com/docker/docker/pkg/sysinfo/numcpu_linux.go b/vendor/github.com/docker/docker/pkg/sysinfo/numcpu_linux.go deleted file mode 100644 index f1d2d9db3..000000000 --- a/vendor/github.com/docker/docker/pkg/sysinfo/numcpu_linux.go +++ /dev/null @@ -1,44 +0,0 @@ -// +build linux - -package sysinfo - -import ( - "runtime" - "unsafe" - - "golang.org/x/sys/unix" -) - -// numCPU queries the system for the count of threads available -// for use to this process. -// -// Issues two syscalls. -// Returns 0 on errors. Use |runtime.NumCPU| in that case. -func numCPU() int { - // Gets the affinity mask for a process: The very one invoking this function. - pid, _, _ := unix.RawSyscall(unix.SYS_GETPID, 0, 0, 0) - - var mask [1024 / 64]uintptr - _, _, err := unix.RawSyscall(unix.SYS_SCHED_GETAFFINITY, pid, uintptr(len(mask)*8), uintptr(unsafe.Pointer(&mask[0]))) - if err != 0 { - return 0 - } - - // For every available thread a bit is set in the mask. - ncpu := 0 - for _, e := range mask { - if e == 0 { - continue - } - ncpu += int(popcnt(uint64(e))) - } - return ncpu -} - -// NumCPU returns the number of CPUs which are currently online -func NumCPU() int { - if ncpu := numCPU(); ncpu > 0 { - return ncpu - } - return runtime.NumCPU() -} diff --git a/vendor/github.com/docker/docker/pkg/sysinfo/numcpu_windows.go b/vendor/github.com/docker/docker/pkg/sysinfo/numcpu_windows.go deleted file mode 100644 index 1d89dd550..000000000 --- a/vendor/github.com/docker/docker/pkg/sysinfo/numcpu_windows.go +++ /dev/null @@ -1,37 +0,0 @@ -// +build windows - -package sysinfo - -import ( - "runtime" - "unsafe" - - "golang.org/x/sys/windows" -) - -var ( - kernel32 = windows.NewLazySystemDLL("kernel32.dll") - getCurrentProcess = kernel32.NewProc("GetCurrentProcess") - getProcessAffinityMask = kernel32.NewProc("GetProcessAffinityMask") -) - -func numCPU() int { - // Gets the affinity mask for a process - var mask, sysmask uintptr - currentProcess, _, _ := getCurrentProcess.Call() - ret, _, _ := getProcessAffinityMask.Call(currentProcess, uintptr(unsafe.Pointer(&mask)), uintptr(unsafe.Pointer(&sysmask))) - if ret == 0 { - return 0 - } - // For every available thread a bit is set in the mask. - ncpu := int(popcnt(uint64(mask))) - return ncpu -} - -// NumCPU returns the number of CPUs which are currently online -func NumCPU() int { - if ncpu := numCPU(); ncpu > 0 { - return ncpu - } - return runtime.NumCPU() -} diff --git a/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo.go b/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo.go deleted file mode 100644 index f046de4b1..000000000 --- a/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo.go +++ /dev/null @@ -1,144 +0,0 @@ -package sysinfo - -import "github.com/docker/docker/pkg/parsers" - -// SysInfo stores information about which features a kernel supports. -// TODO Windows: Factor out platform specific capabilities. -type SysInfo struct { - // Whether the kernel supports AppArmor or not - AppArmor bool - // Whether the kernel supports Seccomp or not - Seccomp bool - - cgroupMemInfo - cgroupCPUInfo - cgroupBlkioInfo - cgroupCpusetInfo - cgroupPids - - // Whether IPv4 forwarding is supported or not, if this was disabled, networking will not work - IPv4ForwardingDisabled bool - - // Whether bridge-nf-call-iptables is supported or not - BridgeNFCallIPTablesDisabled bool - - // Whether bridge-nf-call-ip6tables is supported or not - BridgeNFCallIP6TablesDisabled bool - - // Whether the cgroup has the mountpoint of "devices" or not - CgroupDevicesEnabled bool -} - -type cgroupMemInfo struct { - // Whether memory limit is supported or not - MemoryLimit bool - - // Whether swap limit is supported or not - SwapLimit bool - - // Whether soft limit is supported or not - MemoryReservation bool - - // Whether OOM killer disable is supported or not - OomKillDisable bool - - // Whether memory swappiness is supported or not - MemorySwappiness bool - - // Whether kernel memory limit is supported or not - KernelMemory bool -} - -type cgroupCPUInfo struct { - // Whether CPU shares is supported or not - CPUShares bool - - // Whether CPU CFS(Completely Fair Scheduler) period is supported or not - CPUCfsPeriod bool - - // Whether CPU CFS(Completely Fair Scheduler) quota is supported or not - CPUCfsQuota bool - - // Whether CPU real-time period is supported or not - CPURealtimePeriod bool - - // Whether CPU real-time runtime is supported or not - CPURealtimeRuntime bool -} - -type cgroupBlkioInfo struct { - // Whether Block IO weight is supported or not - BlkioWeight bool - - // Whether Block IO weight_device is supported or not - BlkioWeightDevice bool - - // Whether Block IO read limit in bytes per second is supported or not - BlkioReadBpsDevice bool - - // Whether Block IO write limit in bytes per second is supported or not - BlkioWriteBpsDevice bool - - // Whether Block IO read limit in IO per second is supported or not - BlkioReadIOpsDevice bool - - // Whether Block IO write limit in IO per second is supported or not - BlkioWriteIOpsDevice bool -} - -type cgroupCpusetInfo struct { - // Whether Cpuset is supported or not - Cpuset bool - - // Available Cpuset's cpus - Cpus string - - // Available Cpuset's memory nodes - Mems string -} - -type cgroupPids struct { - // Whether Pids Limit is supported or not - PidsLimit bool -} - -// IsCpusetCpusAvailable returns `true` if the provided string set is contained -// in cgroup's cpuset.cpus set, `false` otherwise. -// If error is not nil a parsing error occurred. -func (c cgroupCpusetInfo) IsCpusetCpusAvailable(provided string) (bool, error) { - return isCpusetListAvailable(provided, c.Cpus) -} - -// IsCpusetMemsAvailable returns `true` if the provided string set is contained -// in cgroup's cpuset.mems set, `false` otherwise. -// If error is not nil a parsing error occurred. -func (c cgroupCpusetInfo) IsCpusetMemsAvailable(provided string) (bool, error) { - return isCpusetListAvailable(provided, c.Mems) -} - -func isCpusetListAvailable(provided, available string) (bool, error) { - parsedProvided, err := parsers.ParseUintList(provided) - if err != nil { - return false, err - } - parsedAvailable, err := parsers.ParseUintList(available) - if err != nil { - return false, err - } - for k := range parsedProvided { - if !parsedAvailable[k] { - return false, nil - } - } - return true, nil -} - -// Returns bit count of 1, used by NumCPU -func popcnt(x uint64) (n byte) { - x -= (x >> 1) & 0x5555555555555555 - x = (x>>2)&0x3333333333333333 + x&0x3333333333333333 - x += x >> 4 - x &= 0x0f0f0f0f0f0f0f0f - x *= 0x0101010101010101 - return byte(x >> 56) -} diff --git a/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo_linux.go b/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo_linux.go deleted file mode 100644 index 471f786a7..000000000 --- a/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo_linux.go +++ /dev/null @@ -1,254 +0,0 @@ -package sysinfo - -import ( - "fmt" - "io/ioutil" - "os" - "path" - "strings" - - "github.com/opencontainers/runc/libcontainer/cgroups" - "github.com/sirupsen/logrus" - "golang.org/x/sys/unix" -) - -func findCgroupMountpoints() (map[string]string, error) { - cgMounts, err := cgroups.GetCgroupMounts(false) - if err != nil { - return nil, fmt.Errorf("Failed to parse cgroup information: %v", err) - } - mps := make(map[string]string) - for _, m := range cgMounts { - for _, ss := range m.Subsystems { - mps[ss] = m.Mountpoint - } - } - return mps, nil -} - -// New returns a new SysInfo, using the filesystem to detect which features -// the kernel supports. If `quiet` is `false` warnings are printed in logs -// whenever an error occurs or misconfigurations are present. -func New(quiet bool) *SysInfo { - sysInfo := &SysInfo{} - cgMounts, err := findCgroupMountpoints() - if err != nil { - logrus.Warnf("Failed to parse cgroup information: %v", err) - } else { - sysInfo.cgroupMemInfo = checkCgroupMem(cgMounts, quiet) - sysInfo.cgroupCPUInfo = checkCgroupCPU(cgMounts, quiet) - sysInfo.cgroupBlkioInfo = checkCgroupBlkioInfo(cgMounts, quiet) - sysInfo.cgroupCpusetInfo = checkCgroupCpusetInfo(cgMounts, quiet) - sysInfo.cgroupPids = checkCgroupPids(quiet) - } - - _, ok := cgMounts["devices"] - sysInfo.CgroupDevicesEnabled = ok - - sysInfo.IPv4ForwardingDisabled = !readProcBool("/proc/sys/net/ipv4/ip_forward") - sysInfo.BridgeNFCallIPTablesDisabled = !readProcBool("/proc/sys/net/bridge/bridge-nf-call-iptables") - sysInfo.BridgeNFCallIP6TablesDisabled = !readProcBool("/proc/sys/net/bridge/bridge-nf-call-ip6tables") - - // Check if AppArmor is supported. - if _, err := os.Stat("/sys/kernel/security/apparmor"); !os.IsNotExist(err) { - sysInfo.AppArmor = true - } - - // Check if Seccomp is supported, via CONFIG_SECCOMP. - if err := unix.Prctl(unix.PR_GET_SECCOMP, 0, 0, 0, 0); err != unix.EINVAL { - // Make sure the kernel has CONFIG_SECCOMP_FILTER. - if err := unix.Prctl(unix.PR_SET_SECCOMP, unix.SECCOMP_MODE_FILTER, 0, 0, 0); err != unix.EINVAL { - sysInfo.Seccomp = true - } - } - - return sysInfo -} - -// checkCgroupMem reads the memory information from the memory cgroup mount point. -func checkCgroupMem(cgMounts map[string]string, quiet bool) cgroupMemInfo { - mountPoint, ok := cgMounts["memory"] - if !ok { - if !quiet { - logrus.Warn("Your kernel does not support cgroup memory limit") - } - return cgroupMemInfo{} - } - - swapLimit := cgroupEnabled(mountPoint, "memory.memsw.limit_in_bytes") - if !quiet && !swapLimit { - logrus.Warn("Your kernel does not support swap memory limit") - } - memoryReservation := cgroupEnabled(mountPoint, "memory.soft_limit_in_bytes") - if !quiet && !memoryReservation { - logrus.Warn("Your kernel does not support memory reservation") - } - oomKillDisable := cgroupEnabled(mountPoint, "memory.oom_control") - if !quiet && !oomKillDisable { - logrus.Warn("Your kernel does not support oom control") - } - memorySwappiness := cgroupEnabled(mountPoint, "memory.swappiness") - if !quiet && !memorySwappiness { - logrus.Warn("Your kernel does not support memory swappiness") - } - kernelMemory := cgroupEnabled(mountPoint, "memory.kmem.limit_in_bytes") - if !quiet && !kernelMemory { - logrus.Warn("Your kernel does not support kernel memory limit") - } - - return cgroupMemInfo{ - MemoryLimit: true, - SwapLimit: swapLimit, - MemoryReservation: memoryReservation, - OomKillDisable: oomKillDisable, - MemorySwappiness: memorySwappiness, - KernelMemory: kernelMemory, - } -} - -// checkCgroupCPU reads the cpu information from the cpu cgroup mount point. -func checkCgroupCPU(cgMounts map[string]string, quiet bool) cgroupCPUInfo { - mountPoint, ok := cgMounts["cpu"] - if !ok { - if !quiet { - logrus.Warn("Unable to find cpu cgroup in mounts") - } - return cgroupCPUInfo{} - } - - cpuShares := cgroupEnabled(mountPoint, "cpu.shares") - if !quiet && !cpuShares { - logrus.Warn("Your kernel does not support cgroup cpu shares") - } - - cpuCfsPeriod := cgroupEnabled(mountPoint, "cpu.cfs_period_us") - if !quiet && !cpuCfsPeriod { - logrus.Warn("Your kernel does not support cgroup cfs period") - } - - cpuCfsQuota := cgroupEnabled(mountPoint, "cpu.cfs_quota_us") - if !quiet && !cpuCfsQuota { - logrus.Warn("Your kernel does not support cgroup cfs quotas") - } - - cpuRealtimePeriod := cgroupEnabled(mountPoint, "cpu.rt_period_us") - if !quiet && !cpuRealtimePeriod { - logrus.Warn("Your kernel does not support cgroup rt period") - } - - cpuRealtimeRuntime := cgroupEnabled(mountPoint, "cpu.rt_runtime_us") - if !quiet && !cpuRealtimeRuntime { - logrus.Warn("Your kernel does not support cgroup rt runtime") - } - - return cgroupCPUInfo{ - CPUShares: cpuShares, - CPUCfsPeriod: cpuCfsPeriod, - CPUCfsQuota: cpuCfsQuota, - CPURealtimePeriod: cpuRealtimePeriod, - CPURealtimeRuntime: cpuRealtimeRuntime, - } -} - -// checkCgroupBlkioInfo reads the blkio information from the blkio cgroup mount point. -func checkCgroupBlkioInfo(cgMounts map[string]string, quiet bool) cgroupBlkioInfo { - mountPoint, ok := cgMounts["blkio"] - if !ok { - if !quiet { - logrus.Warn("Unable to find blkio cgroup in mounts") - } - return cgroupBlkioInfo{} - } - - weight := cgroupEnabled(mountPoint, "blkio.weight") - if !quiet && !weight { - logrus.Warn("Your kernel does not support cgroup blkio weight") - } - - weightDevice := cgroupEnabled(mountPoint, "blkio.weight_device") - if !quiet && !weightDevice { - logrus.Warn("Your kernel does not support cgroup blkio weight_device") - } - - readBpsDevice := cgroupEnabled(mountPoint, "blkio.throttle.read_bps_device") - if !quiet && !readBpsDevice { - logrus.Warn("Your kernel does not support cgroup blkio throttle.read_bps_device") - } - - writeBpsDevice := cgroupEnabled(mountPoint, "blkio.throttle.write_bps_device") - if !quiet && !writeBpsDevice { - logrus.Warn("Your kernel does not support cgroup blkio throttle.write_bps_device") - } - readIOpsDevice := cgroupEnabled(mountPoint, "blkio.throttle.read_iops_device") - if !quiet && !readIOpsDevice { - logrus.Warn("Your kernel does not support cgroup blkio throttle.read_iops_device") - } - - writeIOpsDevice := cgroupEnabled(mountPoint, "blkio.throttle.write_iops_device") - if !quiet && !writeIOpsDevice { - logrus.Warn("Your kernel does not support cgroup blkio throttle.write_iops_device") - } - return cgroupBlkioInfo{ - BlkioWeight: weight, - BlkioWeightDevice: weightDevice, - BlkioReadBpsDevice: readBpsDevice, - BlkioWriteBpsDevice: writeBpsDevice, - BlkioReadIOpsDevice: readIOpsDevice, - BlkioWriteIOpsDevice: writeIOpsDevice, - } -} - -// checkCgroupCpusetInfo reads the cpuset information from the cpuset cgroup mount point. -func checkCgroupCpusetInfo(cgMounts map[string]string, quiet bool) cgroupCpusetInfo { - mountPoint, ok := cgMounts["cpuset"] - if !ok { - if !quiet { - logrus.Warn("Unable to find cpuset cgroup in mounts") - } - return cgroupCpusetInfo{} - } - - cpus, err := ioutil.ReadFile(path.Join(mountPoint, "cpuset.cpus")) - if err != nil { - return cgroupCpusetInfo{} - } - - mems, err := ioutil.ReadFile(path.Join(mountPoint, "cpuset.mems")) - if err != nil { - return cgroupCpusetInfo{} - } - - return cgroupCpusetInfo{ - Cpuset: true, - Cpus: strings.TrimSpace(string(cpus)), - Mems: strings.TrimSpace(string(mems)), - } -} - -// checkCgroupPids reads the pids information from the pids cgroup mount point. -func checkCgroupPids(quiet bool) cgroupPids { - _, err := cgroups.FindCgroupMountpoint("pids") - if err != nil { - if !quiet { - logrus.Warn(err) - } - return cgroupPids{} - } - - return cgroupPids{ - PidsLimit: true, - } -} - -func cgroupEnabled(mountPoint, name string) bool { - _, err := os.Stat(path.Join(mountPoint, name)) - return err == nil -} - -func readProcBool(path string) bool { - val, err := ioutil.ReadFile(path) - if err != nil { - return false - } - return strings.TrimSpace(string(val)) == "1" -} diff --git a/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo_solaris.go b/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo_solaris.go deleted file mode 100644 index c858d57e0..000000000 --- a/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo_solaris.go +++ /dev/null @@ -1,121 +0,0 @@ -// +build solaris,cgo - -package sysinfo - -import ( - "bytes" - "os/exec" - "strconv" - "strings" -) - -/* -#cgo LDFLAGS: -llgrp -#include <unistd.h> -#include <stdlib.h> -#include <sys/lgrp_user.h> -int getLgrpCount() { - lgrp_cookie_t lgrpcookie = LGRP_COOKIE_NONE; - uint_t nlgrps; - - if ((lgrpcookie = lgrp_init(LGRP_VIEW_OS)) == LGRP_COOKIE_NONE) { - return -1; - } - nlgrps = lgrp_nlgrps(lgrpcookie); - return nlgrps; -} -*/ -import "C" - -// IsCPUSharesAvailable returns whether CPUShares setting is supported. -// We need FSS to be set as default scheduling class to support CPU Shares -func IsCPUSharesAvailable() bool { - cmd := exec.Command("/usr/sbin/dispadmin", "-d") - outBuf := new(bytes.Buffer) - errBuf := new(bytes.Buffer) - cmd.Stderr = errBuf - cmd.Stdout = outBuf - - if err := cmd.Run(); err != nil { - return false - } - return (strings.Contains(outBuf.String(), "FSS")) -} - -// New returns a new SysInfo, using the filesystem to detect which features -// the kernel supports. -//NOTE Solaris: If we change the below capabilities be sure -// to update verifyPlatformContainerSettings() in daemon_solaris.go -func New(quiet bool) *SysInfo { - sysInfo := &SysInfo{} - sysInfo.cgroupMemInfo = setCgroupMem(quiet) - sysInfo.cgroupCPUInfo = setCgroupCPU(quiet) - sysInfo.cgroupBlkioInfo = setCgroupBlkioInfo(quiet) - sysInfo.cgroupCpusetInfo = setCgroupCPUsetInfo(quiet) - - sysInfo.IPv4ForwardingDisabled = false - - sysInfo.AppArmor = false - - return sysInfo -} - -// setCgroupMem reads the memory information for Solaris. -func setCgroupMem(quiet bool) cgroupMemInfo { - - return cgroupMemInfo{ - MemoryLimit: true, - SwapLimit: true, - MemoryReservation: false, - OomKillDisable: false, - MemorySwappiness: false, - KernelMemory: false, - } -} - -// setCgroupCPU reads the cpu information for Solaris. -func setCgroupCPU(quiet bool) cgroupCPUInfo { - - return cgroupCPUInfo{ - CPUShares: true, - CPUCfsPeriod: false, - CPUCfsQuota: true, - CPURealtimePeriod: false, - CPURealtimeRuntime: false, - } -} - -// blkio switches are not supported in Solaris. -func setCgroupBlkioInfo(quiet bool) cgroupBlkioInfo { - - return cgroupBlkioInfo{ - BlkioWeight: false, - BlkioWeightDevice: false, - } -} - -// setCgroupCPUsetInfo reads the cpuset information for Solaris. -func setCgroupCPUsetInfo(quiet bool) cgroupCpusetInfo { - - return cgroupCpusetInfo{ - Cpuset: true, - Cpus: getCPUCount(), - Mems: getLgrpCount(), - } -} - -func getCPUCount() string { - ncpus := C.sysconf(C._SC_NPROCESSORS_ONLN) - if ncpus <= 0 { - return "" - } - return strconv.FormatInt(int64(ncpus), 16) -} - -func getLgrpCount() string { - nlgrps := C.getLgrpCount() - if nlgrps <= 0 { - return "" - } - return strconv.FormatInt(int64(nlgrps), 16) -} diff --git a/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo_unix.go b/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo_unix.go deleted file mode 100644 index 45f3ef1c6..000000000 --- a/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo_unix.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build !linux,!solaris,!windows - -package sysinfo - -// New returns an empty SysInfo for non linux nor solaris for now. -func New(quiet bool) *SysInfo { - sysInfo := &SysInfo{} - return sysInfo -} diff --git a/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo_windows.go b/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo_windows.go deleted file mode 100644 index 4e6255bc5..000000000 --- a/vendor/github.com/docker/docker/pkg/sysinfo/sysinfo_windows.go +++ /dev/null @@ -1,9 +0,0 @@ -// +build windows - -package sysinfo - -// New returns an empty SysInfo for windows for now. -func New(quiet bool) *SysInfo { - sysInfo := &SysInfo{} - return sysInfo -} diff --git a/vendor/github.com/docker/docker/profiles/seccomp/generate.go b/vendor/github.com/docker/docker/profiles/seccomp/generate.go new file mode 100644 index 000000000..32f22bb37 --- /dev/null +++ b/vendor/github.com/docker/docker/profiles/seccomp/generate.go @@ -0,0 +1,32 @@ +// +build ignore + +package main + +import ( + "encoding/json" + "io/ioutil" + "os" + "path/filepath" + + "github.com/docker/docker/profiles/seccomp" +) + +// saves the default seccomp profile as a json file so people can use it as a +// base for their own custom profiles +func main() { + wd, err := os.Getwd() + if err != nil { + panic(err) + } + f := filepath.Join(wd, "default.json") + + // write the default profile to the file + b, err := json.MarshalIndent(seccomp.DefaultProfile(), "", "\t") + if err != nil { + panic(err) + } + + if err := ioutil.WriteFile(f, b, 0644); err != nil { + panic(err) + } +} diff --git a/vendor/github.com/opencontainers/go-digest/LICENSE.code b/vendor/github.com/opencontainers/go-digest/LICENSE index 0ea3ff81e..0ea3ff81e 100644 --- a/vendor/github.com/opencontainers/go-digest/LICENSE.code +++ b/vendor/github.com/opencontainers/go-digest/LICENSE diff --git a/vendor/github.com/opencontainers/go-digest/README.md b/vendor/github.com/opencontainers/go-digest/README.md index 0f5a04092..25aac3470 100644 --- a/vendor/github.com/opencontainers/go-digest/README.md +++ b/vendor/github.com/opencontainers/go-digest/README.md @@ -101,4 +101,4 @@ the various OCI projects). # Copyright and license -Copyright © 2016 Docker, Inc. All rights reserved, except as follows. Code is released under the [Apache 2.0 license](LICENSE.code). This `README.md` file and the [`CONTRIBUTING.md`](CONTRIBUTING.md) file are licensed under the Creative Commons Attribution 4.0 International License under the terms and conditions set forth in the file [`LICENSE.docs`](LICENSE.docs). You may obtain a duplicate copy of the same license, titled CC BY-SA 4.0, at http://creativecommons.org/licenses/by-sa/4.0/. +Copyright © 2016 Docker, Inc. All rights reserved, except as follows. Code is released under the [Apache 2.0 license](LICENSE). This `README.md` file and the [`CONTRIBUTING.md`](CONTRIBUTING.md) file are licensed under the Creative Commons Attribution 4.0 International License under the terms and conditions set forth in the file [`LICENSE.docs`](LICENSE.docs). You may obtain a duplicate copy of the same license, titled CC BY-SA 4.0, at http://creativecommons.org/licenses/by-sa/4.0/. diff --git a/vendor/github.com/opencontainers/go-digest/algorithm.go b/vendor/github.com/opencontainers/go-digest/algorithm.go index bdff42d92..8813bd26f 100644 --- a/vendor/github.com/opencontainers/go-digest/algorithm.go +++ b/vendor/github.com/opencontainers/go-digest/algorithm.go @@ -19,6 +19,7 @@ import ( "fmt" "hash" "io" + "regexp" ) // Algorithm identifies and implementation of a digester by an identifier. @@ -28,9 +29,9 @@ type Algorithm string // supported digest types const ( - SHA256 Algorithm = "sha256" // sha256 with hex encoding - SHA384 Algorithm = "sha384" // sha384 with hex encoding - SHA512 Algorithm = "sha512" // sha512 with hex encoding + SHA256 Algorithm = "sha256" // sha256 with hex encoding (lower case only) + SHA384 Algorithm = "sha384" // sha384 with hex encoding (lower case only) + SHA512 Algorithm = "sha512" // sha512 with hex encoding (lower case only) // Canonical is the primary digest algorithm used with the distribution // project. Other digests may be used but this one is the primary storage @@ -50,6 +51,14 @@ var ( SHA384: crypto.SHA384, SHA512: crypto.SHA512, } + + // anchoredEncodedRegexps contains anchored regular expressions for hex-encoded digests. + // Note that /A-F/ disallowed. + anchoredEncodedRegexps = map[Algorithm]*regexp.Regexp{ + SHA256: regexp.MustCompile(`^[a-f0-9]{64}$`), + SHA384: regexp.MustCompile(`^[a-f0-9]{96}$`), + SHA512: regexp.MustCompile(`^[a-f0-9]{128}$`), + } ) // Available returns true if the digest type is available for use. If this @@ -125,6 +134,14 @@ func (a Algorithm) Hash() hash.Hash { return algorithms[a].New() } +// Encode encodes the raw bytes of a digest, typically from a hash.Hash, into +// the encoded portion of the digest. +func (a Algorithm) Encode(d []byte) string { + // TODO(stevvooe): Currently, all algorithms use a hex encoding. When we + // add support for back registration, we can modify this accordingly. + return fmt.Sprintf("%x", d) +} + // FromReader returns the digest of the reader using the algorithm. func (a Algorithm) FromReader(rd io.Reader) (Digest, error) { digester := a.Digester() @@ -156,3 +173,20 @@ func (a Algorithm) FromBytes(p []byte) Digest { func (a Algorithm) FromString(s string) Digest { return a.FromBytes([]byte(s)) } + +// Validate validates the encoded portion string +func (a Algorithm) Validate(encoded string) error { + r, ok := anchoredEncodedRegexps[a] + if !ok { + return ErrDigestUnsupported + } + // Digests much always be hex-encoded, ensuring that their hex portion will + // always be size*2 + if a.Size()*2 != len(encoded) { + return ErrDigestInvalidLength + } + if r.MatchString(encoded) { + return nil + } + return ErrDigestInvalidFormat +} diff --git a/vendor/github.com/opencontainers/go-digest/digest.go b/vendor/github.com/opencontainers/go-digest/digest.go index 69e1d2b54..ad398cba2 100644 --- a/vendor/github.com/opencontainers/go-digest/digest.go +++ b/vendor/github.com/opencontainers/go-digest/digest.go @@ -45,16 +45,21 @@ func NewDigest(alg Algorithm, h hash.Hash) Digest { // functions. This is also useful for rebuilding digests from binary // serializations. func NewDigestFromBytes(alg Algorithm, p []byte) Digest { - return Digest(fmt.Sprintf("%s:%x", alg, p)) + return NewDigestFromEncoded(alg, alg.Encode(p)) } -// NewDigestFromHex returns a Digest from alg and a the hex encoded digest. +// NewDigestFromHex is deprecated. Please use NewDigestFromEncoded. func NewDigestFromHex(alg, hex string) Digest { - return Digest(fmt.Sprintf("%s:%s", alg, hex)) + return NewDigestFromEncoded(Algorithm(alg), hex) +} + +// NewDigestFromEncoded returns a Digest from alg and the encoded digest. +func NewDigestFromEncoded(alg Algorithm, encoded string) Digest { + return Digest(fmt.Sprintf("%s:%s", alg, encoded)) } // DigestRegexp matches valid digest types. -var DigestRegexp = regexp.MustCompile(`[a-zA-Z0-9-_+.]+:[a-fA-F0-9]+`) +var DigestRegexp = regexp.MustCompile(`[a-z0-9]+(?:[.+_-][a-z0-9]+)*:[a-zA-Z0-9=_-]+`) // DigestRegexpAnchored matches valid digest types, anchored to the start and end of the match. var DigestRegexpAnchored = regexp.MustCompile(`^` + DigestRegexp.String() + `$`) @@ -96,26 +101,18 @@ func FromString(s string) Digest { // error if not. func (d Digest) Validate() error { s := string(d) - i := strings.Index(s, ":") - - // validate i then run through regexp - if i < 0 || i+1 == len(s) || !DigestRegexpAnchored.MatchString(s) { + if i <= 0 || i+1 == len(s) { return ErrDigestInvalidFormat } - - algorithm := Algorithm(s[:i]) + algorithm, encoded := Algorithm(s[:i]), s[i+1:] if !algorithm.Available() { + if !DigestRegexpAnchored.MatchString(s) { + return ErrDigestInvalidFormat + } return ErrDigestUnsupported } - - // Digests much always be hex-encoded, ensuring that their hex portion will - // always be size*2 - if algorithm.Size()*2 != len(s[i+1:]) { - return ErrDigestInvalidLength - } - - return nil + return algorithm.Validate(encoded) } // Algorithm returns the algorithm portion of the digest. This will panic if @@ -133,12 +130,17 @@ func (d Digest) Verifier() Verifier { } } -// Hex returns the hex digest portion of the digest. This will panic if the +// Encoded returns the encoded portion of the digest. This will panic if the // underlying digest is not in a valid format. -func (d Digest) Hex() string { +func (d Digest) Encoded() string { return string(d[d.sepIndex()+1:]) } +// Hex is deprecated. Please use Digest.Encoded. +func (d Digest) Hex() string { + return d.Encoded() +} + func (d Digest) String() string { return string(d) } diff --git a/vendor/github.com/opencontainers/runc/README.md b/vendor/github.com/opencontainers/runc/README.md index 5215e32c1..e755fb7bc 100644 --- a/vendor/github.com/opencontainers/runc/README.md +++ b/vendor/github.com/opencontainers/runc/README.md @@ -68,6 +68,7 @@ make BUILDTAGS='seccomp apparmor' | selinux | selinux process and mount labeling | <none> | | apparmor | apparmor profile support | <none> | | ambient | ambient capability support | kernel 4.3 | +| nokmem | disable kernel memory account | <none> | ### Running the test suite @@ -87,6 +88,18 @@ You can run a specific test case by setting the `TESTFLAGS` variable. # make test TESTFLAGS="-run=SomeTestFunction" ``` +You can run a specific integration test by setting the `TESTPATH` variable. + +```bash +# make test TESTPATH="/checkpoint.bats" +``` + +You can run a test in your proxy environment by setting `DOCKER_BUILD_PROXY` and `DOCKER_RUN_PROXY` variables. + +```bash +# make test DOCKER_BUILD_PROXY="--build-arg HTTP_PROXY=http://yourproxy/" DOCKER_RUN_PROXY="-e HTTP_PROXY=http://yourproxy/" +``` + ### Dependencies Management `runc` uses [vndr](https://github.com/LK4D4/vndr) for dependencies management. @@ -251,3 +264,7 @@ PIDFile=/run/mycontainerid.pid [Install] WantedBy=multi-user.target ``` + +## License + +The code and docs are released under the [Apache 2.0 license](LICENSE). diff --git a/vendor/github.com/opencontainers/runc/libcontainer/README.md b/vendor/github.com/opencontainers/runc/libcontainer/README.md index 42f3efe56..1d7fa04c0 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/README.md +++ b/vendor/github.com/opencontainers/runc/libcontainer/README.md @@ -148,6 +148,7 @@ config := &configs.Config{ {Type: configs.NEWPID}, {Type: configs.NEWUSER}, {Type: configs.NEWNET}, + {Type: configs.NEWCGROUP}, }), Cgroups: &configs.Cgroup{ Name: "test-container", @@ -323,6 +324,7 @@ generated when building libcontainer with docker. ## Copyright and license -Code and documentation copyright 2014 Docker, inc. Code released under the Apache 2.0 license. -Docs released under Creative commons. - +Code and documentation copyright 2014 Docker, inc. +The code and documentation are released under the [Apache 2.0 license](../LICENSE). +The documentation is also released under Creative Commons Attribution 4.0 International License. +You may obtain a copy of the license, titled CC-BY-4.0, at http://creativecommons.org/licenses/by/4.0/. diff --git a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils.go b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils.go index 7c995efee..ea571ad93 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils.go @@ -13,40 +13,50 @@ import ( "strings" "time" - "github.com/docker/go-units" + units "github.com/docker/go-units" ) const ( - cgroupNamePrefix = "name=" + CgroupNamePrefix = "name=" CgroupProcesses = "cgroup.procs" ) // https://www.kernel.org/doc/Documentation/cgroup-v1/cgroups.txt -func FindCgroupMountpoint(subsystem string) (string, error) { - mnt, _, err := FindCgroupMountpointAndRoot(subsystem) +func FindCgroupMountpoint(cgroupPath, subsystem string) (string, error) { + mnt, _, err := FindCgroupMountpointAndRoot(cgroupPath, subsystem) return mnt, err } -func FindCgroupMountpointAndRoot(subsystem string) (string, string, error) { +func FindCgroupMountpointAndRoot(cgroupPath, subsystem string) (string, string, error) { // We are not using mount.GetMounts() because it's super-inefficient, // parsing it directly sped up x10 times because of not using Sscanf. // It was one of two major performance drawbacks in container start. if !isSubsystemAvailable(subsystem) { return "", "", NewNotFoundError(subsystem) } + f, err := os.Open("/proc/self/mountinfo") if err != nil { return "", "", err } defer f.Close() - scanner := bufio.NewScanner(f) + return findCgroupMountpointAndRootFromReader(f, cgroupPath, subsystem) +} + +func findCgroupMountpointAndRootFromReader(reader io.Reader, cgroupPath, subsystem string) (string, string, error) { + scanner := bufio.NewScanner(reader) for scanner.Scan() { txt := scanner.Text() - fields := strings.Split(txt, " ") - for _, opt := range strings.Split(fields[len(fields)-1], ",") { - if opt == subsystem { - return fields[4], fields[3], nil + fields := strings.Fields(txt) + if len(fields) < 5 { + continue + } + if strings.HasPrefix(fields[4], cgroupPath) { + for _, opt := range strings.Split(fields[len(fields)-1], ",") { + if opt == subsystem { + return fields[4], fields[3], nil + } } } } @@ -103,7 +113,7 @@ func FindCgroupMountpointDir() (string, error) { } if postSeparatorFields[0] == "cgroup" { - // Check that the mount is properly formated. + // Check that the mount is properly formatted. if numPostFields < 3 { return "", fmt.Errorf("Error found less than 3 fields post '-' in %q", text) } @@ -151,19 +161,20 @@ func getCgroupMountsHelper(ss map[string]bool, mi io.Reader, all bool) ([]Mount, Root: fields[3], } for _, opt := range strings.Split(fields[len(fields)-1], ",") { - if !ss[opt] { + seen, known := ss[opt] + if !known || (!all && seen) { continue } - if strings.HasPrefix(opt, cgroupNamePrefix) { - m.Subsystems = append(m.Subsystems, opt[len(cgroupNamePrefix):]) - } else { - m.Subsystems = append(m.Subsystems, opt) - } - if !all { - numFound++ + ss[opt] = true + if strings.HasPrefix(opt, CgroupNamePrefix) { + opt = opt[len(CgroupNamePrefix):] } + m.Subsystems = append(m.Subsystems, opt) + numFound++ + } + if len(m.Subsystems) > 0 || all { + res = append(res, m) } - res = append(res, m) } if err := scanner.Err(); err != nil { return nil, err @@ -187,7 +198,7 @@ func GetCgroupMounts(all bool) ([]Mount, error) { allMap := make(map[string]bool) for s := range allSubsystems { - allMap[s] = true + allMap[s] = false } return getCgroupMountsHelper(allMap, f, all) } @@ -256,13 +267,13 @@ func GetInitCgroupPath(subsystem string) (string, error) { } func getCgroupPathHelper(subsystem, cgroup string) (string, error) { - mnt, root, err := FindCgroupMountpointAndRoot(subsystem) + mnt, root, err := FindCgroupMountpointAndRoot("", subsystem) if err != nil { return "", err } // This is needed for nested containers, because in /proc/self/cgroup we - // see pathes from host, which don't exist in container. + // see paths from host, which don't exist in container. relCgroup, err := filepath.Rel(root, cgroup) if err != nil { return "", err @@ -342,7 +353,7 @@ func getControllerPath(subsystem string, cgroups map[string]string) (string, err return p, nil } - if p, ok := cgroups[cgroupNamePrefix+subsystem]; ok { + if p, ok := cgroups[CgroupNamePrefix+subsystem]; ok { return p, nil } @@ -452,7 +463,7 @@ func WriteCgroupProc(dir string, pid int) error { return fmt.Errorf("no such directory for %s", CgroupProcesses) } - // Dont attach any pid to the cgroup if -1 is specified as a pid + // Don't attach any pid to the cgroup if -1 is specified as a pid if pid != -1 { if err := ioutil.WriteFile(filepath.Join(dir, CgroupProcesses), []byte(strconv.Itoa(pid)), 0700); err != nil { return fmt.Errorf("failed to write %v to %v: %v", pid, CgroupProcesses, err) diff --git a/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go b/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go index b1c4762fe..7728522fe 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/configs/config.go @@ -186,12 +186,19 @@ type Config struct { // callers keyring in this case. NoNewKeyring bool `json:"no_new_keyring"` - // Rootless specifies whether the container is a rootless container. - Rootless bool `json:"rootless"` - - // IntelRdt specifies settings for Intel RDT/CAT group that the container is placed into - // to limit the resources (e.g., L3 cache) the container has available + // IntelRdt specifies settings for Intel RDT group that the container is placed into + // to limit the resources (e.g., L3 cache, memory bandwidth) the container has available IntelRdt *IntelRdt `json:"intel_rdt,omitempty"` + + // RootlessEUID is set when the runc was launched with non-zero EUID. + // Note that RootlessEUID is set to false when launched with EUID=0 in userns. + // When RootlessEUID is set, runc creates a new userns for the container. + // (config.json needs to contain userns settings) + RootlessEUID bool `json:"rootless_euid,omitempty"` + + // RootlessCgroups is set when unlikely to have the full access to cgroups. + // When RootlessCgroups is set, cgroups errors are ignored. + RootlessCgroups bool `json:"rootless_cgroups,omitempty"` } type Hooks struct { @@ -265,26 +272,23 @@ func (hooks Hooks) MarshalJSON() ([]byte, error) { }) } -// HookState is the payload provided to a hook on execution. -type HookState specs.State - type Hook interface { // Run executes the hook with the provided state. - Run(HookState) error + Run(*specs.State) error } // NewFunctionHook will call the provided function when the hook is run. -func NewFunctionHook(f func(HookState) error) FuncHook { +func NewFunctionHook(f func(*specs.State) error) FuncHook { return FuncHook{ run: f, } } type FuncHook struct { - run func(HookState) error + run func(*specs.State) error } -func (f FuncHook) Run(s HookState) error { +func (f FuncHook) Run(s *specs.State) error { return f.run(s) } @@ -307,7 +311,7 @@ type CommandHook struct { Command } -func (c Command) Run(s HookState) error { +func (c Command) Run(s *specs.State) error { b, err := json.Marshal(s) if err != nil { return err diff --git a/vendor/github.com/opencontainers/runc/libcontainer/configs/intelrdt.go b/vendor/github.com/opencontainers/runc/libcontainer/configs/intelrdt.go index 36bd5f96a..57e9f037d 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/configs/intelrdt.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/configs/intelrdt.go @@ -4,4 +4,10 @@ type IntelRdt struct { // The schema for L3 cache id and capacity bitmask (CBM) // Format: "L3:<cache_id0>=<cbm0>;<cache_id1>=<cbm1>;..." L3CacheSchema string `json:"l3_cache_schema,omitempty"` + + // The schema of memory bandwidth per L3 cache id + // Format: "MB:<cache_id0>=bandwidth0;<cache_id1>=bandwidth1;..." + // The unit of memory bandwidth is specified in "percentages" by + // default, and in "MBps" if MBA Software Controller is enabled. + MemBwSchema string `json:"memBwSchema,omitempty"` } diff --git a/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_linux.go b/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_linux.go index 5fc171a57..1bbaef9bd 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_linux.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_linux.go @@ -7,12 +7,13 @@ import ( ) const ( - NEWNET NamespaceType = "NEWNET" - NEWPID NamespaceType = "NEWPID" - NEWNS NamespaceType = "NEWNS" - NEWUTS NamespaceType = "NEWUTS" - NEWIPC NamespaceType = "NEWIPC" - NEWUSER NamespaceType = "NEWUSER" + NEWNET NamespaceType = "NEWNET" + NEWPID NamespaceType = "NEWPID" + NEWNS NamespaceType = "NEWNS" + NEWUTS NamespaceType = "NEWUTS" + NEWIPC NamespaceType = "NEWIPC" + NEWUSER NamespaceType = "NEWUSER" + NEWCGROUP NamespaceType = "NEWCGROUP" ) var ( @@ -35,6 +36,8 @@ func NsName(ns NamespaceType) string { return "user" case NEWUTS: return "uts" + case NEWCGROUP: + return "cgroup" } return "" } @@ -68,6 +71,7 @@ func NamespaceTypes() []NamespaceType { NEWNET, NEWPID, NEWNS, + NEWCGROUP, } } diff --git a/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_syscall.go b/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_syscall.go index 4ce6813d2..2dc7adfc9 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_syscall.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/configs/namespaces_syscall.go @@ -9,12 +9,13 @@ func (n *Namespace) Syscall() int { } var namespaceInfo = map[NamespaceType]int{ - NEWNET: unix.CLONE_NEWNET, - NEWNS: unix.CLONE_NEWNS, - NEWUSER: unix.CLONE_NEWUSER, - NEWIPC: unix.CLONE_NEWIPC, - NEWUTS: unix.CLONE_NEWUTS, - NEWPID: unix.CLONE_NEWPID, + NEWNET: unix.CLONE_NEWNET, + NEWNS: unix.CLONE_NEWNS, + NEWUSER: unix.CLONE_NEWUSER, + NEWIPC: unix.CLONE_NEWIPC, + NEWUTS: unix.CLONE_NEWUTS, + NEWPID: unix.CLONE_NEWPID, + NEWCGROUP: unix.CLONE_NEWCGROUP, } // CloneFlags parses the container's Namespaces options to set the correct diff --git a/vendor/github.com/opencontainers/runc/libcontainer/nsenter/README.md b/vendor/github.com/opencontainers/runc/libcontainer/nsenter/README.md index 575701371..9ec6c3931 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/nsenter/README.md +++ b/vendor/github.com/opencontainers/runc/libcontainer/nsenter/README.md @@ -10,8 +10,8 @@ The `nsenter` package will `import "C"` and it uses [cgo](https://golang.org/cmd package. In cgo, if the import of "C" is immediately preceded by a comment, that comment, called the preamble, is used as a header when compiling the C parts of the package. So every time we import package `nsenter`, the C code function `nsexec()` would be -called. And package `nsenter` is now only imported in `main_unix.go`, so every time -before we call `cmd.Start` on linux, that C code would run. +called. And package `nsenter` is only imported in `init.go`, so every time the runc +`init` command is invoked, that C code is run. Because `nsexec()` must be run before the Go runtime in order to use the Linux kernel namespace, you must `import` this library into a package if @@ -37,7 +37,7 @@ the parent `nsexec()` will exit and the child `nsexec()` process will return to allow the Go runtime take over. NOTE: We do both `setns(2)` and `clone(2)` even if we don't have any -CLONE_NEW* clone flags because we must fork a new process in order to +`CLONE_NEW*` clone flags because we must fork a new process in order to enter the PID namespace. diff --git a/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c b/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c index a4cd1399d..28269dfc0 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c +++ b/vendor/github.com/opencontainers/runc/libcontainer/nsenter/nsexec.c @@ -42,6 +42,12 @@ enum sync_t { SYNC_ERR = 0xFF, /* Fatal error, no turning back. The error code follows. */ }; +/* + * Synchronisation value for cgroup namespace setup. + * The same constant is defined in process_linux.go as "createCgroupns". + */ +#define CREATECGROUPNS 0x80 + /* longjmp() arguments. */ #define JUMP_PARENT 0x00 #define JUMP_CHILD 0xA0 @@ -82,7 +88,7 @@ struct nlconfig_t { uint8_t is_setgroup; /* Rootless container settings. */ - uint8_t is_rootless; + uint8_t is_rootless_euid; /* boolean */ char *uidmappath; size_t uidmappath_len; char *gidmappath; @@ -100,7 +106,7 @@ struct nlconfig_t { #define GIDMAP_ATTR 27284 #define SETGROUP_ATTR 27285 #define OOM_SCORE_ADJ_ATTR 27286 -#define ROOTLESS_ATTR 27287 +#define ROOTLESS_EUID_ATTR 27287 #define UIDMAPPATH_ATTR 27288 #define GIDMAPPATH_ATTR 27289 @@ -211,7 +217,7 @@ static int try_mapping_tool(const char *app, int pid, char *map, size_t map_len) /* * If @app is NULL, execve will segfault. Just check it here and bail (if - * we're in this path, the caller is already getting desparate and there + * we're in this path, the caller is already getting desperate and there * isn't a backup to this failing). This usually would be a configuration * or programming issue. */ @@ -419,8 +425,8 @@ static void nl_parse(int fd, struct nlconfig_t *config) case CLONE_FLAGS_ATTR: config->cloneflags = readint32(current); break; - case ROOTLESS_ATTR: - config->is_rootless = readint8(current); + case ROOTLESS_EUID_ATTR: + config->is_rootless_euid = readint8(current); /* boolean */ break; case OOM_SCORE_ADJ_ATTR: config->oom_score_adj = current; @@ -640,7 +646,6 @@ void nsexec(void) case JUMP_PARENT:{ int len; pid_t child, first_child = -1; - char buf[JSON_MAX]; bool ready = false; /* For debugging. */ @@ -687,7 +692,7 @@ void nsexec(void) * newuidmap/newgidmap shall be used. */ - if (config.is_rootless && !config.is_setgroup) + if (config.is_rootless_euid && !config.is_setgroup) update_setgroups(child, SETGROUPS_DENY); /* Set up mappings. */ @@ -716,6 +721,18 @@ void nsexec(void) kill(child, SIGKILL); bail("failed to sync with child: write(SYNC_RECVPID_ACK)"); } + + /* Send the init_func pid back to our parent. + * + * Send the init_func pid and the pid of the first child back to our parent. + * We need to send both back because we can't reap the first child we created (CLONE_PARENT). + * It becomes the responsibility of our parent to reap the first child. + */ + len = dprintf(pipenum, "{\"pid\": %d, \"pid_first\": %d}\n", child, first_child); + if (len < 0) { + kill(child, SIGKILL); + bail("unable to generate JSON for child pid"); + } } break; case SYNC_CHILD_READY: @@ -759,23 +776,6 @@ void nsexec(void) bail("unexpected sync value: %u", s); } } - - /* - * Send the init_func pid and the pid of the first child back to our parent. - * - * We need to send both back because we can't reap the first child we created (CLONE_PARENT). - * It becomes the responsibility of our parent to reap the first child. - */ - len = snprintf(buf, JSON_MAX, "{\"pid\": %d, \"pid_first\": %d}\n", child, first_child); - if (len < 0) { - kill(child, SIGKILL); - bail("unable to generate JSON for child pid"); - } - if (write(pipenum, buf, len) != len) { - kill(child, SIGKILL); - bail("unable to send child pid to bootstrapper"); - } - exit(0); } @@ -862,14 +862,17 @@ void nsexec(void) if (setresuid(0, 0, 0) < 0) bail("failed to become root in user namespace"); } - /* - * Unshare all of the namespaces. Note that we don't merge this - * with clone() because there were some old kernel versions where - * clone(CLONE_PARENT | CLONE_NEWPID) was broken, so we'll just do - * it the long way. + * Unshare all of the namespaces. Now, it should be noted that this + * ordering might break in the future (especially with rootless + * containers). But for now, it's not possible to split this into + * CLONE_NEWUSER + [the rest] because of some RHEL SELinux issues. + * + * Note that we don't merge this with clone() because there were + * some old kernel versions where clone(CLONE_PARENT | CLONE_NEWPID) + * was broken, so we'll just do it the long way anyway. */ - if (unshare(config.cloneflags) < 0) + if (unshare(config.cloneflags & ~CLONE_NEWCGROUP) < 0) bail("failed to unshare namespaces"); /* @@ -953,11 +956,23 @@ void nsexec(void) if (setgid(0) < 0) bail("setgid failed"); - if (!config.is_rootless && config.is_setgroup) { + if (!config.is_rootless_euid && config.is_setgroup) { if (setgroups(0, NULL) < 0) bail("setgroups failed"); } + /* ... wait until our topmost parent has finished cgroup setup in p.manager.Apply() ... */ + if (config.cloneflags & CLONE_NEWCGROUP) { + uint8_t value; + if (read(pipenum, &value, sizeof(value)) != sizeof(value)) + bail("read synchronisation value failed"); + if (value == CREATECGROUPNS) { + if (unshare(CLONE_NEWCGROUP) < 0) + bail("failed to unshare cgroup namespace"); + } else + bail("received unknown synchronisation value"); + } + s = SYNC_CHILD_READY; if (write(syncfd, &s, sizeof(s)) != sizeof(s)) bail("failed to sync with patent: write(SYNC_CHILD_READY)"); diff --git a/vendor/github.com/opencontainers/runc/libcontainer/user/lookup_unix.go b/vendor/github.com/opencontainers/runc/libcontainer/user/lookup_unix.go index c1e634c94..92b5ae8de 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/user/lookup_unix.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/user/lookup_unix.go @@ -5,6 +5,7 @@ package user import ( "io" "os" + "strconv" "golang.org/x/sys/unix" ) @@ -115,22 +116,23 @@ func CurrentGroup() (Group, error) { return LookupGid(unix.Getgid()) } -func CurrentUserSubUIDs() ([]SubID, error) { +func currentUserSubIDs(fileName string) ([]SubID, error) { u, err := CurrentUser() if err != nil { return nil, err } - return ParseSubIDFileFilter("/etc/subuid", - func(entry SubID) bool { return entry.Name == u.Name }) + filter := func(entry SubID) bool { + return entry.Name == u.Name || entry.Name == strconv.Itoa(u.Uid) + } + return ParseSubIDFileFilter(fileName, filter) } -func CurrentGroupSubGIDs() ([]SubID, error) { - g, err := CurrentGroup() - if err != nil { - return nil, err - } - return ParseSubIDFileFilter("/etc/subgid", - func(entry SubID) bool { return entry.Name == g.Name }) +func CurrentUserSubUIDs() ([]SubID, error) { + return currentUserSubIDs("/etc/subuid") +} + +func CurrentUserSubGIDs() ([]SubID, error) { + return currentUserSubIDs("/etc/subgid") } func CurrentProcessUIDMap() ([]IDMap, error) { diff --git a/vendor/github.com/opencontainers/runc/vendor.conf b/vendor/github.com/opencontainers/runc/vendor.conf index e2b519e67..fadbe0707 100644 --- a/vendor/github.com/opencontainers/runc/vendor.conf +++ b/vendor/github.com/opencontainers/runc/vendor.conf @@ -1,7 +1,7 @@ # OCI runtime-spec. When updating this, make sure you use a version tag rather # than a commit ID so it's much more obvious what version of the spec we are # using. -github.com/opencontainers/runtime-spec v1.0.0 +github.com/opencontainers/runtime-spec 5684b8af48c1ac3b1451fa499724e30e3c20a294 # Core libcontainer functionality. github.com/mrunalp/fileutils ed869b029674c0e9ce4c0dfa781405c2d9946d08 github.com/opencontainers/selinux v1.0.0-rc1 |