summaryrefslogtreecommitdiff
path: root/vendor/github.com
diff options
context:
space:
mode:
authorOpenShift Merge Robot <openshift-merge-robot@users.noreply.github.com>2021-04-12 18:25:00 +0200
committerGitHub <noreply@github.com>2021-04-12 18:25:00 +0200
commit7203e0a76a82e278b6a41e7a5851463a0b2f571e (patch)
tree04252a3261ecba007ace7055154fc8d2ce8a9aea /vendor/github.com
parent3803a2630f2bc53f18cbf2b7a825ff1b068a0fb7 (diff)
parent3627c4b691a357fc531c2761f07ccfecf937dd12 (diff)
downloadpodman-7203e0a76a82e278b6a41e7a5851463a0b2f571e.tar.gz
podman-7203e0a76a82e278b6a41e7a5851463a0b2f571e.tar.bz2
podman-7203e0a76a82e278b6a41e7a5851463a0b2f571e.zip
Merge pull request #9994 from containers/dependabot/go_modules/github.com/containers/image/v5-5.11.0
Bump github.com/containers/image/v5 from 5.10.5 to 5.11.0
Diffstat (limited to 'vendor/github.com')
-rw-r--r--vendor/github.com/containers/image/v5/copy/copy.go192
-rw-r--r--vendor/github.com/containers/image/v5/docker/docker_client.go2
-rw-r--r--vendor/github.com/containers/image/v5/docker/docker_image.go1
-rw-r--r--vendor/github.com/containers/image/v5/docker/docker_image_dest.go2
-rw-r--r--vendor/github.com/containers/image/v5/docker/docker_image_src.go2
-rw-r--r--vendor/github.com/containers/image/v5/internal/pkg/platform/platform_matcher.go20
-rw-r--r--vendor/github.com/containers/image/v5/internal/types/types.go53
-rw-r--r--vendor/github.com/containers/image/v5/manifest/manifest.go17
-rw-r--r--vendor/github.com/containers/image/v5/oci/layout/oci_src.go2
-rw-r--r--vendor/github.com/containers/image/v5/openshift/openshift-copies.go24
-rw-r--r--vendor/github.com/containers/image/v5/pkg/blobinfocache/boltdb/boltdb.go4
-rw-r--r--vendor/github.com/containers/image/v5/pkg/blobinfocache/default.go2
-rw-r--r--vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go4
-rw-r--r--vendor/github.com/containers/image/v5/pkg/blobinfocache/none/none.go2
-rw-r--r--vendor/github.com/containers/image/v5/pkg/docker/config/config.go387
-rw-r--r--vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go16
-rw-r--r--vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go5
-rw-r--r--vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go178
-rw-r--r--vendor/github.com/containers/image/v5/storage/storage_image.go427
-rw-r--r--vendor/github.com/containers/image/v5/types/types.go10
-rw-r--r--vendor/github.com/containers/image/v5/version/version.go4
-rw-r--r--vendor/github.com/klauspost/compress/flate/deflate.go6
-rw-r--r--vendor/github.com/klauspost/compress/flate/fast_encoder.go26
-rw-r--r--vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go39
-rw-r--r--vendor/github.com/klauspost/compress/flate/huffman_code.go53
-rw-r--r--vendor/github.com/klauspost/compress/flate/level2.go2
-rw-r--r--vendor/github.com/klauspost/compress/fse/compress.go10
-rw-r--r--vendor/github.com/klauspost/compress/fse/decompress.go4
-rw-r--r--vendor/github.com/klauspost/compress/huff0/compress.go4
-rw-r--r--vendor/github.com/klauspost/compress/snappy/snappy.go2
-rw-r--r--vendor/github.com/klauspost/compress/zstd/blockenc.go42
-rw-r--r--vendor/github.com/klauspost/compress/zstd/enc_base.go40
-rw-r--r--vendor/github.com/klauspost/compress/zstd/enc_best.go1
-rw-r--r--vendor/github.com/klauspost/compress/zstd/enc_better.go589
-rw-r--r--vendor/github.com/klauspost/compress/zstd/enc_dfast.go414
-rw-r--r--vendor/github.com/klauspost/compress/zstd/enc_fast.go371
-rw-r--r--vendor/github.com/klauspost/compress/zstd/encoder.go10
-rw-r--r--vendor/github.com/klauspost/compress/zstd/encoder_options.go34
-rw-r--r--vendor/github.com/klauspost/compress/zstd/fse_encoder.go12
-rw-r--r--vendor/github.com/klauspost/compress/zstd/snappy.go2
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/.gitignore5
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/.travis.yml11
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/README.md118
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/UNLICENSE24
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/bar.go490
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/bar_filler.go30
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/bar_filler_bar.go173
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/bar_filler_spinner.go63
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/bar_option.go213
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/container_option.go102
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/cwriter/util_bsd.go7
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/cwriter/util_linux.go7
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/cwriter/util_solaris.go7
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/cwriter/writer.go84
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/cwriter/writer_posix.go26
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/cwriter/writer_windows.go73
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/any.go21
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/counters.go243
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/decorator.go191
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/doc.go21
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/elapsed.go35
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/eta.go203
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/merge.go107
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/moving_average.go68
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/name.go12
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/on_complete.go37
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/percentage.go58
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/size_type.go109
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/sizeb1000_string.go41
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/sizeb1024_string.go41
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/speed.go171
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/decor/spinner.go21
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/doc.go2
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/go.mod10
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/go.sum8
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/internal/percentage.go18
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/internal/width.go8
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/priority_queue.go32
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/progress.go390
-rw-r--r--vendor/github.com/vbauerster/mpb/v5/proxyreader.go90
80 files changed, 2536 insertions, 3849 deletions
diff --git a/vendor/github.com/containers/image/v5/copy/copy.go b/vendor/github.com/containers/image/v5/copy/copy.go
index 3a2ee9a8f..165a8be4b 100644
--- a/vendor/github.com/containers/image/v5/copy/copy.go
+++ b/vendor/github.com/containers/image/v5/copy/copy.go
@@ -16,6 +16,7 @@ import (
"github.com/containers/image/v5/image"
internalblobinfocache "github.com/containers/image/v5/internal/blobinfocache"
"github.com/containers/image/v5/internal/pkg/platform"
+ internalTypes "github.com/containers/image/v5/internal/types"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/pkg/blobinfocache"
"github.com/containers/image/v5/pkg/compression"
@@ -28,8 +29,8 @@ import (
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
- "github.com/vbauerster/mpb/v5"
- "github.com/vbauerster/mpb/v5/decor"
+ "github.com/vbauerster/mpb/v6"
+ "github.com/vbauerster/mpb/v6/decor"
"golang.org/x/crypto/ssh/terminal"
"golang.org/x/sync/semaphore"
)
@@ -46,7 +47,7 @@ var (
// ErrDecryptParamsMissing is returned if there is missing decryption parameters
ErrDecryptParamsMissing = errors.New("Necessary DecryptParameters not present")
- // maxParallelDownloads is used to limit the maxmimum number of parallel
+ // maxParallelDownloads is used to limit the maximum number of parallel
// downloads. Let's follow Firefox by limiting it to 6.
maxParallelDownloads = uint(6)
)
@@ -108,19 +109,20 @@ func (d *digestingReader) Read(p []byte) (int, error) {
// copier allows us to keep track of diffID values for blobs, and other
// data shared across one or more images in a possible manifest list.
type copier struct {
- dest types.ImageDestination
- rawSource types.ImageSource
- reportWriter io.Writer
- progressOutput io.Writer
- progressInterval time.Duration
- progress chan types.ProgressProperties
- blobInfoCache internalblobinfocache.BlobInfoCache2
- copyInParallel bool
- compressionFormat compression.Algorithm
- compressionLevel *int
- ociDecryptConfig *encconfig.DecryptConfig
- ociEncryptConfig *encconfig.EncryptConfig
- maxParallelDownloads uint
+ dest types.ImageDestination
+ rawSource types.ImageSource
+ reportWriter io.Writer
+ progressOutput io.Writer
+ progressInterval time.Duration
+ progress chan types.ProgressProperties
+ blobInfoCache internalblobinfocache.BlobInfoCache2
+ copyInParallel bool
+ compressionFormat compression.Algorithm
+ compressionLevel *int
+ ociDecryptConfig *encconfig.DecryptConfig
+ ociEncryptConfig *encconfig.EncryptConfig
+ maxParallelDownloads uint
+ downloadForeignLayers bool
}
// imageCopier tracks state specific to a single image (possibly an item of a manifest list)
@@ -194,6 +196,13 @@ type Options struct {
OciDecryptConfig *encconfig.DecryptConfig
// MaxParallelDownloads indicates the maximum layers to pull at the same time. A reasonable default is used if this is left as 0.
MaxParallelDownloads uint
+ // When OptimizeDestinationImageAlreadyExists is set, optimize the copy assuming that the destination image already
+ // exists (and is equivalent). Making the eventual (no-op) copy more performant for this case. Enabling the option
+ // is slightly pessimistic if the destination image doesn't exist, or is not equivalent.
+ OptimizeDestinationImageAlreadyExists bool
+ // Download layer contents with "nondistributable" media types ("foreign" layers) and translate the layer media type
+ // to not indicate "nondistributable".
+ DownloadForeignLayers bool
}
// validateImageListSelection returns an error if the passed-in value is not one that we recognize as a valid ImageListSelection value
@@ -269,10 +278,11 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef,
// FIXME? The cache is used for sources and destinations equally, but we only have a SourceCtx and DestinationCtx.
// For now, use DestinationCtx (because blob reuse changes the behavior of the destination side more); eventually
// we might want to add a separate CommonCtx — or would that be too confusing?
- blobInfoCache: internalblobinfocache.FromBlobInfoCache(blobinfocache.DefaultCache(options.DestinationCtx)),
- ociDecryptConfig: options.OciDecryptConfig,
- ociEncryptConfig: options.OciEncryptConfig,
- maxParallelDownloads: options.MaxParallelDownloads,
+ blobInfoCache: internalblobinfocache.FromBlobInfoCache(blobinfocache.DefaultCache(options.DestinationCtx)),
+ ociDecryptConfig: options.OciDecryptConfig,
+ ociEncryptConfig: options.OciEncryptConfig,
+ maxParallelDownloads: options.MaxParallelDownloads,
+ downloadForeignLayers: options.DownloadForeignLayers,
}
// Default to using gzip compression unless specified otherwise.
if options.DestinationCtx == nil || options.DestinationCtx.CompressionFormat == nil {
@@ -361,6 +371,45 @@ func supportsMultipleImages(dest types.ImageDestination) bool {
return false
}
+// compareImageDestinationManifestEqual compares the `src` and `dest` image manifests (reading the manifest from the
+// (possibly remote) destination). Returning true and the destination's manifest, type and digest if they compare equal.
+func compareImageDestinationManifestEqual(ctx context.Context, options *Options, src types.Image, targetInstance *digest.Digest, dest types.ImageDestination) (bool, []byte, string, digest.Digest, error) {
+ srcManifest, _, err := src.Manifest(ctx)
+ if err != nil {
+ return false, nil, "", "", errors.Wrapf(err, "Error reading manifest from image")
+ }
+
+ srcManifestDigest, err := manifest.Digest(srcManifest)
+ if err != nil {
+ return false, nil, "", "", errors.Wrapf(err, "Error calculating manifest digest")
+ }
+
+ destImageSource, err := dest.Reference().NewImageSource(ctx, options.DestinationCtx)
+ if err != nil {
+ logrus.Debugf("Unable to create destination image %s source: %v", dest.Reference(), err)
+ return false, nil, "", "", nil
+ }
+
+ destManifest, destManifestType, err := destImageSource.GetManifest(ctx, targetInstance)
+ if err != nil {
+ logrus.Debugf("Unable to get destination image %s/%s manifest: %v", destImageSource, targetInstance, err)
+ return false, nil, "", "", nil
+ }
+
+ destManifestDigest, err := manifest.Digest(destManifest)
+ if err != nil {
+ return false, nil, "", "", errors.Wrapf(err, "Error calculating manifest digest")
+ }
+
+ logrus.Debugf("Comparing source and destination manifest digests: %v vs. %v", srcManifestDigest, destManifestDigest)
+ if srcManifestDigest != destManifestDigest {
+ return false, nil, "", "", nil
+ }
+
+ // Destination and source manifests, types and digests should all be equivalent
+ return true, destManifest, destManifestType, destManifestDigest, nil
+}
+
// copyMultipleImages copies some or all of an image list's instances, using
// policyContext to validate source image admissibility.
func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedToplevel *image.UnparsedImage) (copiedManifest []byte, copiedManifestType string, retErr error) {
@@ -646,6 +695,26 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
// If encrypted and decryption keys provided, we should try to decrypt
ic.diffIDsAreNeeded = ic.diffIDsAreNeeded || (isEncrypted(src) && ic.c.ociDecryptConfig != nil) || ic.c.ociEncryptConfig != nil
+ // If enabled, fetch and compare the destination's manifest. And as an optimization skip updating the destination iff equal
+ if options.OptimizeDestinationImageAlreadyExists {
+ shouldUpdateSigs := len(sigs) > 0 || options.SignBy != "" // TODO: Consider allowing signatures updates only and skipping the image's layers/manifest copy if possible
+ noPendingManifestUpdates := ic.noPendingManifestUpdates()
+
+ logrus.Debugf("Checking if we can skip copying: has signatures=%t, OCI encryption=%t, no manifest updates=%t", shouldUpdateSigs, destRequiresOciEncryption, noPendingManifestUpdates)
+ if !shouldUpdateSigs && !destRequiresOciEncryption && noPendingManifestUpdates {
+ isSrcDestManifestEqual, retManifest, retManifestType, retManifestDigest, err := compareImageDestinationManifestEqual(ctx, options, src, targetInstance, c.dest)
+ if err != nil {
+ logrus.Warnf("Failed to compare destination image manifest: %v", err)
+ return nil, "", "", err
+ }
+
+ if isSrcDestManifestEqual {
+ c.Printf("Skipping: image already present at destination\n")
+ return retManifest, retManifestType, retManifestDigest, nil
+ }
+ }
+ }
+
if err := ic.copyLayers(ctx); err != nil {
return nil, "", "", err
}
@@ -702,6 +771,9 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli
return nil, "", "", fmt.Errorf("Uploading manifest failed, attempted the following formats: %s", strings.Join(errs, ", "))
}
}
+ if targetInstance != nil {
+ targetInstance = &retManifestDigest
+ }
if options.SignBy != "" {
newSig, err := c.createSignature(manifestBytes, options.SignBy)
@@ -781,6 +853,10 @@ func (ic *imageCopier) updateEmbeddedDockerReference() error {
return nil
}
+func (ic *imageCopier) noPendingManifestUpdates() bool {
+ return reflect.DeepEqual(*ic.manifestUpdates, types.ManifestUpdateOptions{InformationOnly: ic.manifestUpdates.InformationOnly})
+}
+
// isTTY returns true if the io.Writer is a file and a tty.
func isTTY(w io.Writer) bool {
if f, ok := w.(*os.File); ok {
@@ -834,7 +910,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error {
defer copySemaphore.Release(1)
defer copyGroup.Done()
cld := copyLayerData{}
- if ic.c.dest.AcceptsForeignLayerURLs() && len(srcLayer.URLs) != 0 {
+ if !ic.c.downloadForeignLayers && ic.c.dest.AcceptsForeignLayerURLs() && len(srcLayer.URLs) != 0 {
// DiffIDs are, currently, needed only when converting from schema1.
// In which case src.LayerInfos will not have URLs because schema1
// does not support them.
@@ -845,7 +921,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error {
logrus.Debugf("Skipping foreign layer %q copy to %s", cld.destInfo.Digest, ic.c.dest.Reference().Transport().Name())
}
} else {
- cld.destInfo, cld.diffID, cld.err = ic.copyLayer(ctx, srcLayer, toEncrypt, pool)
+ cld.destInfo, cld.diffID, cld.err = ic.copyLayer(ctx, srcLayer, toEncrypt, pool, index)
}
data[index] = cld
}
@@ -901,6 +977,8 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error {
diffIDs[i] = cld.diffID
}
+ // WARNING: If you are adding new reasons to change ic.manifestUpdates, also update the
+ // OptimizeDestinationImageAlreadyExists short-circuit conditions
ic.manifestUpdates.InformationOnly.LayerInfos = destInfos
if ic.diffIDsAreNeeded {
ic.manifestUpdates.InformationOnly.LayerDiffIDs = diffIDs
@@ -929,7 +1007,7 @@ func layerDigestsDiffer(a, b []types.BlobInfo) bool {
// and its digest.
func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, digest.Digest, error) {
pendingImage := ic.src
- if !reflect.DeepEqual(*ic.manifestUpdates, types.ManifestUpdateOptions{InformationOnly: ic.manifestUpdates.InformationOnly}) {
+ if !ic.noPendingManifestUpdates() {
if !ic.canModifyManifest {
return nil, "", errors.Errorf("Internal error: copy needs an updated manifest but that was known to be forbidden")
}
@@ -1012,10 +1090,9 @@ func (c *copier) createProgressBar(pool *mpb.Progress, info types.BlobInfo, kind
),
)
} else {
- bar = pool.AddSpinner(info.Size,
- mpb.SpinnerOnLeft,
+ bar = pool.Add(0,
+ mpb.NewSpinnerFiller([]string{".", "..", "...", "....", ""}, mpb.SpinnerOnLeft),
mpb.BarFillerClearOnComplete(),
- mpb.SpinnerStyle([]string{".", "..", "...", "....", ""}),
mpb.PrependDecorators(
decor.OnComplete(decor.Name(prefix), onComplete),
),
@@ -1040,7 +1117,7 @@ func (c *copier) copyConfig(ctx context.Context, src types.Image) error {
progressPool, progressCleanup := c.newProgressPool(ctx)
defer progressCleanup()
bar := c.createProgressBar(progressPool, srcInfo, "config", "done")
- destInfo, err := c.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, false, true, false, bar)
+ destInfo, err := c.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, false, true, false, bar, -1)
if err != nil {
return types.BlobInfo{}, err
}
@@ -1066,7 +1143,7 @@ type diffIDResult struct {
// copyLayer copies a layer with srcInfo (with known Digest and Annotations and possibly known Size) in src to dest, perhaps (de/re/)compressing it,
// and returns a complete blobInfo of the copied layer, and a value for LayerDiffIDs if diffIDIsNeeded
-func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, toEncrypt bool, pool *mpb.Progress) (types.BlobInfo, digest.Digest, error) {
+func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, toEncrypt bool, pool *mpb.Progress, layerIndex int) (types.BlobInfo, digest.Digest, error) {
// If the srcInfo doesn't contain compression information, try to compute it from the
// MediaType, which was either read from a manifest by way of LayerInfos() or constructed
// by LayerInfosForCopy(), if it was supplied at all. If we succeed in copying the blob,
@@ -1099,7 +1176,26 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
// a failure when we eventually try to update the manifest with the digest and MIME type of the reused blob.
// Fixing that will probably require passing more information to TryReusingBlob() than the current version of
// the ImageDestination interface lets us pass in.
- reused, blobInfo, err := ic.c.dest.TryReusingBlob(ctx, srcInfo, ic.c.blobInfoCache, ic.canSubstituteBlobs)
+ var (
+ blobInfo types.BlobInfo
+ reused bool
+ err error
+ )
+ // Note: the storage destination optimizes the committing of
+ // layers which requires passing the index of the layer.
+ // Hence, we need to special case and cast.
+ dest, ok := ic.c.dest.(internalTypes.ImageDestinationWithOptions)
+ if ok {
+ options := internalTypes.TryReusingBlobOptions{
+ Cache: ic.c.blobInfoCache,
+ CanSubstitute: ic.canSubstituteBlobs,
+ LayerIndex: &layerIndex,
+ }
+ reused, blobInfo, err = dest.TryReusingBlobWithOptions(ctx, srcInfo, options)
+ } else {
+ reused, blobInfo, err = ic.c.dest.TryReusingBlob(ctx, srcInfo, ic.c.blobInfoCache, ic.canSubstituteBlobs)
+ }
+
if err != nil {
return types.BlobInfo{}, "", errors.Wrapf(err, "Error trying to reuse blob %s at destination", srcInfo.Digest)
}
@@ -1141,7 +1237,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
bar := ic.c.createProgressBar(pool, srcInfo, "blob", "done")
- blobInfo, diffIDChan, err := ic.copyLayerFromStream(ctx, srcStream, types.BlobInfo{Digest: srcInfo.Digest, Size: srcBlobSize, MediaType: srcInfo.MediaType, Annotations: srcInfo.Annotations}, diffIDIsNeeded, toEncrypt, bar)
+ blobInfo, diffIDChan, err := ic.copyLayerFromStream(ctx, srcStream, types.BlobInfo{Digest: srcInfo.Digest, Size: srcBlobSize, MediaType: srcInfo.MediaType, Annotations: srcInfo.Annotations}, diffIDIsNeeded, toEncrypt, bar, layerIndex)
if err != nil {
return types.BlobInfo{}, "", err
}
@@ -1172,7 +1268,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to
// perhaps (de/re/)compressing the stream,
// and returns a complete blobInfo of the copied blob and perhaps a <-chan diffIDResult if diffIDIsNeeded, to be read by the caller.
func (ic *imageCopier) copyLayerFromStream(ctx context.Context, srcStream io.Reader, srcInfo types.BlobInfo,
- diffIDIsNeeded bool, toEncrypt bool, bar *mpb.Bar) (types.BlobInfo, <-chan diffIDResult, error) {
+ diffIDIsNeeded bool, toEncrypt bool, bar *mpb.Bar, layerIndex int) (types.BlobInfo, <-chan diffIDResult, error) {
var getDiffIDRecorder func(compression.DecompressorFunc) io.Writer // = nil
var diffIDChan chan diffIDResult
@@ -1197,7 +1293,7 @@ func (ic *imageCopier) copyLayerFromStream(ctx context.Context, srcStream io.Rea
}
}
- blobInfo, err := ic.c.copyBlobFromStream(ctx, srcStream, srcInfo, getDiffIDRecorder, ic.canModifyManifest, false, toEncrypt, bar) // Sets err to nil on success
+ blobInfo, err := ic.c.copyBlobFromStream(ctx, srcStream, srcInfo, getDiffIDRecorder, ic.canModifyManifest, false, toEncrypt, bar, layerIndex) // Sets err to nil on success
return blobInfo, diffIDChan, err
// We need the defer … pipeWriter.CloseWithError() to happen HERE so that the caller can block on reading from diffIDChan
}
@@ -1249,7 +1345,7 @@ func (r errorAnnotationReader) Read(b []byte) (n int, err error) {
// and returns a complete blobInfo of the copied blob.
func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, srcInfo types.BlobInfo,
getOriginalLayerCopyWriter func(decompressor compression.DecompressorFunc) io.Writer,
- canModifyBlob bool, isConfig bool, toEncrypt bool, bar *mpb.Bar) (types.BlobInfo, error) {
+ canModifyBlob bool, isConfig bool, toEncrypt bool, bar *mpb.Bar, layerIndex int) (types.BlobInfo, error) {
if isConfig { // This is guaranteed by the caller, but set it here to be explicit.
canModifyBlob = false
}
@@ -1267,8 +1363,9 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
if err != nil {
return types.BlobInfo{}, errors.Wrapf(err, "Error preparing to verify blob %s", srcInfo.Digest)
}
-
var destStream io.Reader = digestingReader
+
+ // === Decrypt the stream, if required.
var decrypted bool
if isOciEncrypted(srcInfo.MediaType) && c.ociDecryptConfig != nil {
newDesc := imgspecv1.Descriptor{
@@ -1298,12 +1395,13 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
return types.BlobInfo{}, errors.Wrapf(err, "Error reading blob %s", srcInfo.Digest)
}
isCompressed := decompressor != nil
- destStream = bar.ProxyReader(destStream)
-
if expectedCompressionFormat, known := expectedCompressionFormats[srcInfo.MediaType]; known && isCompressed && compressionFormat.Name() != expectedCompressionFormat.Name() {
logrus.Debugf("blob %s with type %s should be compressed with %s, but compressor appears to be %s", srcInfo.Digest.String(), srcInfo.MediaType, expectedCompressionFormat.Name(), compressionFormat.Name())
}
+ // === Update progress bars
+ destStream = bar.ProxyReader(destStream)
+
// === Send a copy of the original, uncompressed, stream, to a separate path if necessary.
var originalLayerReader io.Reader // DO NOT USE this other than to drain the input if no other consumer in the pipeline has done so.
if getOriginalLayerCopyWriter != nil {
@@ -1312,6 +1410,8 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
}
// === Deal with layer compression/decompression if necessary
+ // WARNING: If you are adding new reasons to change the blob, update also the OptimizeDestinationImageAlreadyExists
+ // short-circuit conditions
var inputInfo types.BlobInfo
var compressionOperation types.LayerCompression
uploadCompressionFormat := &c.compressionFormat
@@ -1393,7 +1493,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
}
}
- // Perform image encryption for valid mediatypes if ociEncryptConfig provided
+ // === Encrypt the stream for valid mediatypes if ociEncryptConfig provided
var (
encrypted bool
finalizer ocicrypt.EncryptLayerFinalizer
@@ -1441,7 +1541,23 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr
}
// === Finally, send the layer stream to dest.
- uploadedInfo, err := c.dest.PutBlob(ctx, &errorAnnotationReader{destStream}, inputInfo, c.blobInfoCache, isConfig)
+ var uploadedInfo types.BlobInfo
+ // Note: the storage destination optimizes the committing of layers
+ // which requires passing the index of the layer. Hence, we need to
+ // special case and cast.
+ dest, ok := c.dest.(internalTypes.ImageDestinationWithOptions)
+ if ok {
+ options := internalTypes.PutBlobOptions{
+ Cache: c.blobInfoCache,
+ IsConfig: isConfig,
+ }
+ if !isConfig {
+ options.LayerIndex = &layerIndex
+ }
+ uploadedInfo, err = dest.PutBlobWithOptions(ctx, &errorAnnotationReader{destStream}, inputInfo, options)
+ } else {
+ uploadedInfo, err = c.dest.PutBlob(ctx, &errorAnnotationReader{destStream}, inputInfo, c.blobInfoCache, isConfig)
+ }
if err != nil {
return types.BlobInfo{}, errors.Wrap(err, "Error writing blob")
}
diff --git a/vendor/github.com/containers/image/v5/docker/docker_client.go b/vendor/github.com/containers/image/v5/docker/docker_client.go
index be46508de..a9533ea39 100644
--- a/vendor/github.com/containers/image/v5/docker/docker_client.go
+++ b/vendor/github.com/containers/image/v5/docker/docker_client.go
@@ -502,6 +502,8 @@ func (c *dockerClient) makeRequestToResolvedURL(ctx context.Context, method, url
attempts == backoffNumIterations {
return res, err
}
+ // close response body before retry or context done
+ res.Body.Close()
delay = parseRetryAfter(res, delay)
if delay > backoffMaxDelay {
diff --git a/vendor/github.com/containers/image/v5/docker/docker_image.go b/vendor/github.com/containers/image/v5/docker/docker_image.go
index 0c1cee0d3..f9fe4e8a3 100644
--- a/vendor/github.com/containers/image/v5/docker/docker_image.go
+++ b/vendor/github.com/containers/image/v5/docker/docker_image.go
@@ -139,6 +139,7 @@ func GetDigest(ctx context.Context, sys *types.SystemContext, ref types.ImageRef
return "", err
}
+ defer res.Body.Close()
if res.StatusCode != http.StatusOK {
return "", errors.Wrapf(registryHTTPResponseToError(res), "Error reading digest %s in %s", tagOrDigest, dr.ref.Name())
}
diff --git a/vendor/github.com/containers/image/v5/docker/docker_image_dest.go b/vendor/github.com/containers/image/v5/docker/docker_image_dest.go
index 842dcfba6..e11084dc8 100644
--- a/vendor/github.com/containers/image/v5/docker/docker_image_dest.go
+++ b/vendor/github.com/containers/image/v5/docker/docker_image_dest.go
@@ -445,7 +445,7 @@ func successStatus(status int) bool {
return status >= 200 && status <= 399
}
-// isManifestInvalidError returns true iff err from client.HandleErrorReponse is a “manifest invalid” error.
+// isManifestInvalidError returns true iff err from client.HandleErrorResponse is a “manifest invalid” error.
func isManifestInvalidError(err error) bool {
errors, ok := err.(errcode.Errors)
if !ok || len(errors) == 0 {
diff --git a/vendor/github.com/containers/image/v5/docker/docker_image_src.go b/vendor/github.com/containers/image/v5/docker/docker_image_src.go
index bff950bb0..6916b7dad 100644
--- a/vendor/github.com/containers/image/v5/docker/docker_image_src.go
+++ b/vendor/github.com/containers/image/v5/docker/docker_image_src.go
@@ -251,6 +251,7 @@ func (s *dockerImageSource) getExternalBlob(ctx context.Context, urls []string)
if resp.StatusCode != http.StatusOK {
err = errors.Errorf("error fetching external blob from %q: %d (%s)", url, resp.StatusCode, http.StatusText(resp.StatusCode))
logrus.Debug(err)
+ resp.Body.Close()
continue
}
break
@@ -290,6 +291,7 @@ func (s *dockerImageSource) GetBlob(ctx context.Context, info types.BlobInfo, ca
return nil, 0, err
}
if err := httpResponseToError(res, "Error fetching blob"); err != nil {
+ res.Body.Close()
return nil, 0, err
}
cache.RecordKnownLocation(s.physicalRef.Transport(), bicTransportScope(s.physicalRef), info.Digest, newBICLocationReference(s.physicalRef))
diff --git a/vendor/github.com/containers/image/v5/internal/pkg/platform/platform_matcher.go b/vendor/github.com/containers/image/v5/internal/pkg/platform/platform_matcher.go
index 5ea542bcf..3e81e06c0 100644
--- a/vendor/github.com/containers/image/v5/internal/pkg/platform/platform_matcher.go
+++ b/vendor/github.com/containers/image/v5/internal/pkg/platform/platform_matcher.go
@@ -123,14 +123,6 @@ var compatibility = map[string][]string{
"arm64": {"v8"},
}
-// baseVariants contains, for a specified architecture, a variant that is known to be
-// supported by _all_ machines using that architecture.
-// Architectures that don’t have variants, or where there are possible versions without
-// an established variant name, should not have an entry here.
-var baseVariants = map[string]string{
- "arm64": "v8",
-}
-
// WantedPlatforms returns all compatible platforms with the platform specifics possibly overridden by user,
// the most compatible platform is first.
// If some option (arch, os, variant) is not present, a value from current platform is detected.
@@ -158,6 +150,8 @@ func WantedPlatforms(ctx *types.SystemContext) ([]imgspecv1.Platform, error) {
var variants []string = nil
if wantedVariant != "" {
+ // If the user requested a specific variant, we'll walk down
+ // the list from most to least compatible.
if compatibility[wantedArch] != nil {
variantOrder := compatibility[wantedArch]
for i, v := range variantOrder {
@@ -171,12 +165,14 @@ func WantedPlatforms(ctx *types.SystemContext) ([]imgspecv1.Platform, error) {
// user wants a variant which we know nothing about - not even compatibility
variants = []string{wantedVariant}
}
+ // Make sure to have a candidate with an empty variant as well.
variants = append(variants, "")
} else {
- variants = append(variants, "") // No variant specified, use a “no variant specified” image if present
- if baseVariant, ok := baseVariants[wantedArch]; ok {
- // But also accept an image with the “base” variant for the architecture, if it exists.
- variants = append(variants, baseVariant)
+ // Make sure to have a candidate with an empty variant as well.
+ variants = append(variants, "")
+ // If available add the entire compatibility matrix for the specific architecture.
+ if possibleVariants, ok := compatibility[wantedArch]; ok {
+ variants = append(variants, possibleVariants...)
}
}
diff --git a/vendor/github.com/containers/image/v5/internal/types/types.go b/vendor/github.com/containers/image/v5/internal/types/types.go
new file mode 100644
index 000000000..9adf0d536
--- /dev/null
+++ b/vendor/github.com/containers/image/v5/internal/types/types.go
@@ -0,0 +1,53 @@
+package types
+
+import (
+ "context"
+ "io"
+
+ publicTypes "github.com/containers/image/v5/types"
+)
+
+// ImageDestinationWithOptions is an internal extension to the ImageDestination
+// interface.
+type ImageDestinationWithOptions interface {
+ publicTypes.ImageDestination
+
+ // PutBlobWithOptions is a wrapper around PutBlob. If
+ // options.LayerIndex is set, the blob will be committed directly.
+ // Either by the calling goroutine or by another goroutine already
+ // committing layers.
+ //
+ // Please note that TryReusingBlobWithOptions and PutBlobWithOptions
+ // *must* be used the together. Mixing the two with non "WithOptions"
+ // functions is not supported.
+ PutBlobWithOptions(ctx context.Context, stream io.Reader, blobinfo publicTypes.BlobInfo, options PutBlobOptions) (publicTypes.BlobInfo, error)
+
+ // TryReusingBlobWithOptions is a wrapper around TryReusingBlob. If
+ // options.LayerIndex is set, the reused blob will be recoreded as
+ // already pulled.
+ //
+ // Please note that TryReusingBlobWithOptions and PutBlobWithOptions
+ // *must* be used the together. Mixing the two with non "WithOptions"
+ // functions is not supported.
+ TryReusingBlobWithOptions(ctx context.Context, blobinfo publicTypes.BlobInfo, options TryReusingBlobOptions) (bool, publicTypes.BlobInfo, error)
+}
+
+// PutBlobOptions are used in PutBlobWithOptions.
+type PutBlobOptions struct {
+ // Cache to look up blob infos.
+ Cache publicTypes.BlobInfoCache
+ // Denotes whether the blob is a config or not.
+ IsConfig bool
+ // The corresponding index in the layer slice.
+ LayerIndex *int
+}
+
+// TryReusingBlobOptions are used in TryReusingBlobWithOptions.
+type TryReusingBlobOptions struct {
+ // Cache to look up blob infos.
+ Cache publicTypes.BlobInfoCache
+ // Use an equivalent of the desired blob.
+ CanSubstitute bool
+ // The corresponding index in the layer slice.
+ LayerIndex *int
+}
diff --git a/vendor/github.com/containers/image/v5/manifest/manifest.go b/vendor/github.com/containers/image/v5/manifest/manifest.go
index 7b0758873..32680e09d 100644
--- a/vendor/github.com/containers/image/v5/manifest/manifest.go
+++ b/vendor/github.com/containers/image/v5/manifest/manifest.go
@@ -30,7 +30,7 @@ const (
DockerV2ListMediaType = "application/vnd.docker.distribution.manifest.list.v2+json"
// DockerV2Schema2ForeignLayerMediaType is the MIME type used for schema 2 foreign layers.
DockerV2Schema2ForeignLayerMediaType = "application/vnd.docker.image.rootfs.foreign.diff.tar"
- // DockerV2Schema2ForeignLayerMediaType is the MIME type used for gzippped schema 2 foreign layers.
+ // DockerV2Schema2ForeignLayerMediaType is the MIME type used for gzipped schema 2 foreign layers.
DockerV2Schema2ForeignLayerMediaTypeGzip = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip"
)
@@ -132,9 +132,16 @@ func GuessMIMEType(manifest []byte) string {
if err := json.Unmarshal(manifest, &ociMan); err != nil {
return ""
}
- if ociMan.Config.MediaType == imgspecv1.MediaTypeImageConfig {
+ switch ociMan.Config.MediaType {
+ case imgspecv1.MediaTypeImageConfig:
return imgspecv1.MediaTypeImageManifest
+ case DockerV2Schema2ConfigMediaType:
+ // This case should not happen since a Docker image
+ // must declare a top-level media type and
+ // `meta.MediaType` has already been checked.
+ return DockerV2Schema2MediaType
}
+ // Maybe an image index or an OCI artifact.
ociIndex := struct {
Manifests []imgspecv1.Descriptor `json:"manifests"`
}{}
@@ -145,9 +152,13 @@ func GuessMIMEType(manifest []byte) string {
if ociMan.Config.MediaType == "" {
return imgspecv1.MediaTypeImageIndex
}
+ // FIXME: this is mixing media types of manifests and configs.
return ociMan.Config.MediaType
}
- return DockerV2Schema2MediaType
+ // It's most likely an OCI artifact with a custom config media
+ // type which is not (and cannot) be covered by the media-type
+ // checks cabove.
+ return imgspecv1.MediaTypeImageManifest
}
return ""
}
diff --git a/vendor/github.com/containers/image/v5/oci/layout/oci_src.go b/vendor/github.com/containers/image/v5/oci/layout/oci_src.go
index 9925aeda7..6801c8432 100644
--- a/vendor/github.com/containers/image/v5/oci/layout/oci_src.go
+++ b/vendor/github.com/containers/image/v5/oci/layout/oci_src.go
@@ -15,6 +15,7 @@ import (
"github.com/opencontainers/go-digest"
imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1"
"github.com/pkg/errors"
+ "github.com/sirupsen/logrus"
)
type ociImageSource struct {
@@ -94,6 +95,7 @@ func (s *ociImageSource) GetManifest(ctx context.Context, instanceDigest *digest
m, err := ioutil.ReadFile(manifestPath)
if err != nil {
+ logrus.Errorf("Error HERE")
return nil, "", err
}
if mimeType == "" {
diff --git a/vendor/github.com/containers/image/v5/openshift/openshift-copies.go b/vendor/github.com/containers/image/v5/openshift/openshift-copies.go
index d87f748e6..ec88b4ebf 100644
--- a/vendor/github.com/containers/image/v5/openshift/openshift-copies.go
+++ b/vendor/github.com/containers/image/v5/openshift/openshift-copies.go
@@ -24,7 +24,7 @@ import (
"golang.org/x/net/http2"
)
-// restTLSClientConfig is a modified copy of k8s.io/kubernets/pkg/client/restclient.TLSClientConfig.
+// restTLSClientConfig is a modified copy of k8s.io/kubernetes/pkg/client/restclient.TLSClientConfig.
// restTLSClientConfig contains settings to enable transport layer security
type restTLSClientConfig struct {
// Server requires TLS client certificate authentication
@@ -45,7 +45,7 @@ type restTLSClientConfig struct {
CAData []byte
}
-// restConfig is a modified copy of k8s.io/kubernets/pkg/client/restclient.Config.
+// restConfig is a modified copy of k8s.io/kubernetes/pkg/client/restclient.Config.
// Config holds the common attributes that can be passed to a Kubernetes client on
// initialization.
type restConfig struct {
@@ -254,7 +254,7 @@ func getServerIdentificationPartialConfig(configAuthInfo clientcmdAuthInfo, conf
// we want this order of precedence for user identification
// 1. configAuthInfo minus auth-path (the final result of command line flags and merged .kubeconfig files)
// 2. configAuthInfo.auth-path (this file can contain information that conflicts with #1, and we want #1 to win the priority)
-// 3. if there is not enough information to idenfity the user, load try the ~/.kubernetes_auth file
+// 3. if there is not enough information to identify the user, load try the ~/.kubernetes_auth file
// 4. if there is not enough information to identify the user, prompt if possible
func getUserIdentificationPartialConfig(configAuthInfo clientcmdAuthInfo) (*restConfig, error) {
mergedConfig := &restConfig{}
@@ -538,7 +538,7 @@ func (e errConfigurationInvalid) Error() string {
// ClientConfigLoadingRules is an ExplicitPath and string slice of specific locations that are used for merging together a Config
// Callers can put the chain together however they want, but we'd recommend:
// EnvVarPathFiles if set (a list of files if set) OR the HomeDirectoryPath
-// ExplicitPath is special, because if a user specifically requests a certain file be used and error is reported if thie file is not present
+// ExplicitPath is special, because if a user specifically requests a certain file be used and error is reported if this file is not present
type clientConfigLoadingRules struct {
Precedence []string
}
@@ -741,7 +741,7 @@ func resolvePaths(refs []*string, base string) error {
return nil
}
-// restClientFor is a modified copy of k8s.io/kubernets/pkg/client/restclient.RESTClientFor.
+// restClientFor is a modified copy of k8s.io/kubernetes/pkg/client/restclient.RESTClientFor.
// RESTClientFor returns a RESTClient that satisfies the requested attributes on a client Config
// object. Note that a RESTClient may require fields that are optional when initializing a Client.
// A RESTClient created by this method is generic - it expects to operate on an API that follows
@@ -769,7 +769,7 @@ func restClientFor(config *restConfig) (*url.URL, *http.Client, error) {
return baseURL, httpClient, nil
}
-// defaultServerURL is a modified copy of k8s.io/kubernets/pkg/client/restclient.DefaultServerURL.
+// defaultServerURL is a modified copy of k8s.io/kubernetes/pkg/client/restclient.DefaultServerURL.
// DefaultServerURL converts a host, host:port, or URL string to the default base server API path
// to use with a Client at a given API version following the standard conventions for a
// Kubernetes API.
@@ -800,7 +800,7 @@ func defaultServerURL(host string, defaultTLS bool) (*url.URL, error) {
return hostURL, nil
}
-// defaultServerURLFor is a modified copy of k8s.io/kubernets/pkg/client/restclient.defaultServerURLFor.
+// defaultServerURLFor is a modified copy of k8s.io/kubernetes/pkg/client/restclient.defaultServerURLFor.
// defaultServerUrlFor is shared between IsConfigTransportTLS and RESTClientFor. It
// requires Host and Version to be set prior to being called.
func defaultServerURLFor(config *restConfig) (*url.URL, error) {
@@ -818,7 +818,7 @@ func defaultServerURLFor(config *restConfig) (*url.URL, error) {
return defaultServerURL(host, defaultTLS)
}
-// transportFor is a modified copy of k8s.io/kubernets/pkg/client/restclient.transportFor.
+// transportFor is a modified copy of k8s.io/kubernetes/pkg/client/restclient.transportFor.
// TransportFor returns an http.RoundTripper that will provide the authentication
// or transport level security defined by the provided Config. Will return the
// default http.DefaultTransport if no special case behavior is needed.
@@ -827,7 +827,7 @@ func transportFor(config *restConfig) (http.RoundTripper, error) {
return transportNew(config)
}
-// isConfigTransportTLS is a modified copy of k8s.io/kubernets/pkg/client/restclient.IsConfigTransportTLS.
+// isConfigTransportTLS is a modified copy of k8s.io/kubernetes/pkg/client/restclient.IsConfigTransportTLS.
// IsConfigTransportTLS returns true if and only if the provided
// config will result in a protected connection to the server when it
// is passed to restclient.RESTClientFor(). Use to determine when to
@@ -1055,11 +1055,11 @@ func (c *restConfig) HasCertAuth() bool {
// Config holds the information needed to build connect to remote kubernetes clusters as a given user
// IMPORTANT if you add fields to this struct, please update IsConfigEmpty()
type clientcmdConfig struct {
- // Clusters is a map of referencable names to cluster configs
+ // Clusters is a map of referenceable names to cluster configs
Clusters clustersMap `json:"clusters"`
- // AuthInfos is a map of referencable names to user configs
+ // AuthInfos is a map of referenceable names to user configs
AuthInfos authInfosMap `json:"users"`
- // Contexts is a map of referencable names to context configs
+ // Contexts is a map of referenceable names to context configs
Contexts contextsMap `json:"contexts"`
// CurrentContext is the name of the context that you would like to use by default
CurrentContext string `json:"current-context"`
diff --git a/vendor/github.com/containers/image/v5/pkg/blobinfocache/boltdb/boltdb.go b/vendor/github.com/containers/image/v5/pkg/blobinfocache/boltdb/boltdb.go
index 2c211b8b8..a472efd95 100644
--- a/vendor/github.com/containers/image/v5/pkg/blobinfocache/boltdb/boltdb.go
+++ b/vendor/github.com/containers/image/v5/pkg/blobinfocache/boltdb/boltdb.go
@@ -282,7 +282,7 @@ func (bdc *cache) RecordKnownLocation(transport types.ImageTransport, scope type
}) // FIXME? Log error (but throttle the log volume on repeated accesses)?
}
-// appendReplacementCandiates creates prioritize.CandidateWithTime values for digest in scopeBucket with corresponding compression info from compressionBucket (if compressionBucket is not nil), and returns the result of appending them to candidates.
+// appendReplacementCandidates creates prioritize.CandidateWithTime values for digest in scopeBucket with corresponding compression info from compressionBucket (if compressionBucket is not nil), and returns the result of appending them to candidates.
func (bdc *cache) appendReplacementCandidates(candidates []prioritize.CandidateWithTime, scopeBucket, compressionBucket *bolt.Bucket, digest digest.Digest, requireCompressionInfo bool) []prioritize.CandidateWithTime {
digestKey := []byte(digest.String())
b := scopeBucket.Bucket(digestKey)
@@ -321,7 +321,7 @@ func (bdc *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
// CandidateLocations2 returns a prioritized, limited, number of blobs and their locations that could possibly be reused
// within the specified (transport scope) (if they still exist, which is not guaranteed).
//
-// If !canSubstitute, the returned cadidates will match the submitted digest exactly; if canSubstitute,
+// If !canSubstitute, the returned candidates will match the submitted digest exactly; if canSubstitute,
// data from previous RecordDigestUncompressedPair calls is used to also look up variants of the blob which have the same
// uncompressed digest.
func (bdc *cache) CandidateLocations2(transport types.ImageTransport, scope types.BICTransportScope, primaryDigest digest.Digest, canSubstitute bool) []blobinfocache.BICReplacementCandidate2 {
diff --git a/vendor/github.com/containers/image/v5/pkg/blobinfocache/default.go b/vendor/github.com/containers/image/v5/pkg/blobinfocache/default.go
index 01abb8d1e..83034b618 100644
--- a/vendor/github.com/containers/image/v5/pkg/blobinfocache/default.go
+++ b/vendor/github.com/containers/image/v5/pkg/blobinfocache/default.go
@@ -20,7 +20,7 @@ const (
systemBlobInfoCacheDir = "/var/lib/containers/cache"
)
-// blobInfoCacheDir returns a path to a blob info cache appropripate for sys and euid.
+// blobInfoCacheDir returns a path to a blob info cache appropriate for sys and euid.
// euid is used so that (sudo …) does not write root-owned files into the unprivileged users’ home directory.
func blobInfoCacheDir(sys *types.SystemContext, euid int) (string, error) {
if sys != nil && sys.BlobInfoCacheDir != "" {
diff --git a/vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go b/vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go
index 3d598057e..426640366 100644
--- a/vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go
+++ b/vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go
@@ -120,7 +120,7 @@ func (mem *cache) RecordDigestCompressorName(blobDigest digest.Digest, compresso
mem.compressors[blobDigest] = compressorName
}
-// appendReplacementCandiates creates prioritize.CandidateWithTime values for (transport, scope, digest), and returns the result of appending them to candidates.
+// appendReplacementCandidates creates prioritize.CandidateWithTime values for (transport, scope, digest), and returns the result of appending them to candidates.
func (mem *cache) appendReplacementCandidates(candidates []prioritize.CandidateWithTime, transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, requireCompressionInfo bool) []prioritize.CandidateWithTime {
locations := mem.knownLocations[locationKey{transport: transport.Name(), scope: scope, blobDigest: digest}] // nil if not present
for l, t := range locations {
@@ -146,7 +146,7 @@ func (mem *cache) appendReplacementCandidates(candidates []prioritize.CandidateW
// CandidateLocations returns a prioritized, limited, number of blobs and their locations that could possibly be reused
// within the specified (transport scope) (if they still exist, which is not guaranteed).
//
-// If !canSubstitute, the returned cadidates will match the submitted digest exactly; if canSubstitute,
+// If !canSubstitute, the returned candidates will match the submitted digest exactly; if canSubstitute,
// data from previous RecordDigestUncompressedPair calls is used to also look up variants of the blob which have the same
// uncompressed digest.
func (mem *cache) CandidateLocations(transport types.ImageTransport, scope types.BICTransportScope, primaryDigest digest.Digest, canSubstitute bool) []types.BICReplacementCandidate {
diff --git a/vendor/github.com/containers/image/v5/pkg/blobinfocache/none/none.go b/vendor/github.com/containers/image/v5/pkg/blobinfocache/none/none.go
index 2a54ff312..4b7122f92 100644
--- a/vendor/github.com/containers/image/v5/pkg/blobinfocache/none/none.go
+++ b/vendor/github.com/containers/image/v5/pkg/blobinfocache/none/none.go
@@ -42,7 +42,7 @@ func (noCache) RecordKnownLocation(transport types.ImageTransport, scope types.B
// CandidateLocations returns a prioritized, limited, number of blobs and their locations that could possibly be reused
// within the specified (transport scope) (if they still exist, which is not guaranteed).
//
-// If !canSubstitute, the returned cadidates will match the submitted digest exactly; if canSubstitute,
+// If !canSubstitute, the returned candidates will match the submitted digest exactly; if canSubstitute,
// data from previous RecordDigestUncompressedPair calls is used to also look up variants of the blob which have the same
// uncompressed digest.
func (noCache) CandidateLocations(transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, canSubstitute bool) []types.BICReplacementCandidate {
diff --git a/vendor/github.com/containers/image/v5/pkg/docker/config/config.go b/vendor/github.com/containers/image/v5/pkg/docker/config/config.go
index 983df41d8..b84aac6e4 100644
--- a/vendor/github.com/containers/image/v5/pkg/docker/config/config.go
+++ b/vendor/github.com/containers/image/v5/pkg/docker/config/config.go
@@ -6,14 +6,17 @@ import (
"fmt"
"io/ioutil"
"os"
+ "os/exec"
"path/filepath"
"runtime"
"strings"
+ "github.com/containers/image/v5/pkg/sysregistriesv2"
"github.com/containers/image/v5/types"
"github.com/containers/storage/pkg/homedir"
helperclient "github.com/docker/docker-credential-helpers/client"
"github.com/docker/docker-credential-helpers/credentials"
+ "github.com/hashicorp/go-multierror"
"github.com/pkg/errors"
"github.com/sirupsen/logrus"
)
@@ -41,12 +44,6 @@ var (
dockerLegacyHomePath = ".dockercfg"
nonLinuxAuthFilePath = filepath.FromSlash(".config/containers/auth.json")
- // Note that the keyring support has been disabled as it was causing
- // regressions. Before enabling, please revisit TODO(keyring) comments
- // which need to be addressed if the need remerged to support the
- // kernel keyring.
- enableKeyring = false
-
// ErrNotLoggedIn is returned for users not logged into a registry
// that they are trying to logout of
ErrNotLoggedIn = errors.New("not logged in")
@@ -54,72 +51,114 @@ var (
ErrNotSupported = errors.New("not supported")
)
-// SetAuthentication stores the username and password in the auth.json file
+// SetAuthentication stores the username and password in the credential helper or file
func SetAuthentication(sys *types.SystemContext, registry, username, password string) error {
- return modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
- if ch, exists := auths.CredHelpers[registry]; exists {
- return false, setAuthToCredHelper(ch, registry, username, password)
- }
+ helpers, err := sysregistriesv2.CredentialHelpers(sys)
+ if err != nil {
+ return err
+ }
- // Set the credentials to kernel keyring if enableKeyring is true.
- // The keyring might not work in all environments (e.g., missing capability) and isn't supported on all platforms.
- // Hence, we want to fall-back to using the authfile in case the keyring failed.
- // However, if the enableKeyring is false, we want adhere to the user specification and not use the keyring.
- if enableKeyring {
- err := setAuthToKernelKeyring(registry, username, password)
- if err == nil {
- logrus.Debugf("credentials for (%s, %s) were stored in the kernel keyring\n", registry, username)
- return false, nil
- }
- logrus.Debugf("failed to authenticate with the kernel keyring, falling back to authfiles. %v", err)
+ // Make sure to collect all errors.
+ var multiErr error
+ for _, helper := range helpers {
+ var err error
+ switch helper {
+ // Special-case the built-in helpers for auth files.
+ case sysregistriesv2.AuthenticationFileHelper:
+ err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
+ if ch, exists := auths.CredHelpers[registry]; exists {
+ return false, setAuthToCredHelper(ch, registry, username, password)
+ }
+ creds := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
+ newCreds := dockerAuthConfig{Auth: creds}
+ auths.AuthConfigs[registry] = newCreds
+ return true, nil
+ })
+ // External helpers.
+ default:
+ err = setAuthToCredHelper(helper, registry, username, password)
+ }
+ if err != nil {
+ multiErr = multierror.Append(multiErr, err)
+ logrus.Debugf("Error storing credentials for %s in credential helper %s: %v", registry, helper, err)
+ continue
}
- creds := base64.StdEncoding.EncodeToString([]byte(username + ":" + password))
- newCreds := dockerAuthConfig{Auth: creds}
- auths.AuthConfigs[registry] = newCreds
- return true, nil
- })
+ logrus.Debugf("Stored credentials for %s in credential helper %s", registry, helper)
+ return nil
+ }
+ return multiErr
}
// GetAllCredentials returns the registry credentials for all registries stored
-// in either the auth.json file or the docker/config.json.
+// in any of the configured credential helpers.
func GetAllCredentials(sys *types.SystemContext) (map[string]types.DockerAuthConfig, error) {
- // Note: we need to read the auth files in the inverse order to prevent
- // a priority inversion when writing to the map.
- authConfigs := make(map[string]types.DockerAuthConfig)
- paths := getAuthFilePaths(sys, homedir.Get())
- for i := len(paths) - 1; i >= 0; i-- {
- path := paths[i]
- // readJSONFile returns an empty map in case the path doesn't exist.
- auths, err := readJSONFile(path.path, path.legacyFormat)
- if err != nil {
- return nil, errors.Wrapf(err, "error reading JSON file %q", path.path)
- }
+ // To keep things simple, let's first extract all registries from all
+ // possible sources, and then call `GetCredentials` on them. That
+ // prevents us from having to reverse engineer the logic in
+ // `GetCredentials`.
+ allRegistries := make(map[string]bool)
+ addRegistry := func(s string) {
+ allRegistries[s] = true
+ }
- for registry, data := range auths.AuthConfigs {
- conf, err := decodeDockerAuth(data)
- if err != nil {
- return nil, err
+ helpers, err := sysregistriesv2.CredentialHelpers(sys)
+ if err != nil {
+ return nil, err
+ }
+ for _, helper := range helpers {
+ switch helper {
+ // Special-case the built-in helper for auth files.
+ case sysregistriesv2.AuthenticationFileHelper:
+ for _, path := range getAuthFilePaths(sys, homedir.Get()) {
+ // readJSONFile returns an empty map in case the path doesn't exist.
+ auths, err := readJSONFile(path.path, path.legacyFormat)
+ if err != nil {
+ return nil, errors.Wrapf(err, "error reading JSON file %q", path.path)
+ }
+ // Credential helpers in the auth file have a
+ // direct mapping to a registry, so we can just
+ // walk the map.
+ for registry := range auths.CredHelpers {
+ addRegistry(registry)
+ }
+ for registry := range auths.AuthConfigs {
+ addRegistry(registry)
+ }
}
- authConfigs[normalizeRegistry(registry)] = conf
- }
-
- // Credential helpers may override credentials from the auth file.
- for registry, credHelper := range auths.CredHelpers {
- username, password, err := getAuthFromCredHelper(credHelper, registry)
+ // External helpers.
+ default:
+ creds, err := listAuthsFromCredHelper(helper)
if err != nil {
- if credentials.IsErrCredentialsNotFoundMessage(err.Error()) {
- continue
+ logrus.Debugf("Error listing credentials stored in credential helper %s: %v", helper, err)
+ }
+ switch errors.Cause(err) {
+ case nil:
+ for registry := range creds {
+ addRegistry(registry)
}
+ case exec.ErrNotFound:
+ // It's okay if the helper doesn't exist.
+ default:
return nil, err
}
-
- conf := types.DockerAuthConfig{Username: username, Password: password}
- authConfigs[normalizeRegistry(registry)] = conf
}
}
- // TODO(keyring): if we ever re-enable the keyring support, we had to
- // query all credentials from the keyring here.
+ // Now use `GetCredentials` to the specific auth configs for each
+ // previously listed registry.
+ authConfigs := make(map[string]types.DockerAuthConfig)
+ for registry := range allRegistries {
+ authConf, err := GetCredentials(sys, registry)
+ if err != nil {
+ if credentials.IsErrCredentialsNotFoundMessage(err.Error()) {
+ // Ignore if the credentials could not be found (anymore).
+ continue
+ }
+ // Note: we rely on the logging in `GetCredentials`.
+ return nil, err
+ }
+ authConfigs[registry] = authConf
+ }
return authConfigs, nil
}
@@ -159,7 +198,9 @@ func getAuthFilePaths(sys *types.SystemContext, homeDir string) []authPath {
return paths
}
-// GetCredentials returns the registry credentials stored in either auth.json
+// GetCredentials returns the registry credentials stored in the
+// registry-specific credential helpers or in the default global credentials
+// helpers with falling back to using either auth.json
// file or .docker/config.json, including support for OAuth2 and IdentityToken.
// If an entry is not found, an empty struct is returned.
func GetCredentials(sys *types.SystemContext, registry string) (types.DockerAuthConfig, error) {
@@ -170,41 +211,65 @@ func GetCredentials(sys *types.SystemContext, registry string) (types.DockerAuth
// it exists only to allow testing it with an artificial home directory.
func getCredentialsWithHomeDir(sys *types.SystemContext, registry, homeDir string) (types.DockerAuthConfig, error) {
if sys != nil && sys.DockerAuthConfig != nil {
- logrus.Debug("Returning credentials from DockerAuthConfig")
+ logrus.Debugf("Returning credentials for %s from DockerAuthConfig", registry)
return *sys.DockerAuthConfig, nil
}
- if enableKeyring {
- username, password, err := getAuthFromKernelKeyring(registry)
- if err == nil {
- logrus.Debug("returning credentials from kernel keyring")
- return types.DockerAuthConfig{
- Username: username,
- Password: password,
- }, nil
+ // Anonymous function to query credentials from auth files.
+ getCredentialsFromAuthFiles := func() (types.DockerAuthConfig, error) {
+ for _, path := range getAuthFilePaths(sys, homeDir) {
+ authConfig, err := findAuthentication(registry, path.path, path.legacyFormat)
+ if err != nil {
+ return types.DockerAuthConfig{}, err
+ }
+
+ if (authConfig.Username != "" && authConfig.Password != "") || authConfig.IdentityToken != "" {
+ return authConfig, nil
+ }
}
+ return types.DockerAuthConfig{}, nil
+ }
+
+ helpers, err := sysregistriesv2.CredentialHelpers(sys)
+ if err != nil {
+ return types.DockerAuthConfig{}, err
}
- for _, path := range getAuthFilePaths(sys, homeDir) {
- authConfig, err := findAuthentication(registry, path.path, path.legacyFormat)
+ var multiErr error
+ for _, helper := range helpers {
+ var creds types.DockerAuthConfig
+ var err error
+ switch helper {
+ // Special-case the built-in helper for auth files.
+ case sysregistriesv2.AuthenticationFileHelper:
+ creds, err = getCredentialsFromAuthFiles()
+ // External helpers.
+ default:
+ creds, err = getAuthFromCredHelper(helper, registry)
+ }
if err != nil {
- logrus.Debugf("Credentials not found")
- return types.DockerAuthConfig{}, err
+ logrus.Debugf("Error looking up credentials for %s in credential helper %s: %v", registry, helper, err)
+ multiErr = multierror.Append(multiErr, err)
+ continue
}
-
- if (authConfig.Username != "" && authConfig.Password != "") || authConfig.IdentityToken != "" {
- logrus.Debugf("Returning credentials from %s", path.path)
- return authConfig, nil
+ if len(creds.Username)+len(creds.Password)+len(creds.IdentityToken) == 0 {
+ continue
}
+ logrus.Debugf("Found credentials for %s in credential helper %s", registry, helper)
+ return creds, nil
+ }
+ if multiErr != nil {
+ return types.DockerAuthConfig{}, multiErr
}
- logrus.Debugf("Credentials not found")
+ logrus.Debugf("No credentials for %s found", registry)
return types.DockerAuthConfig{}, nil
}
-// GetAuthentication returns the registry credentials stored in
-// either auth.json file or .docker/config.json
-// If an entry is not found empty strings are returned for the username and password
+// GetAuthentication returns the registry credentials stored in the
+// registry-specific credential helpers or in the default global credentials
+// helpers with falling back to using either auth.json file or
+// .docker/config.json
//
// Deprecated: This API only has support for username and password. To get the
// support for oauth2 in docker registry authentication, we added the new
@@ -227,53 +292,132 @@ func getAuthenticationWithHomeDir(sys *types.SystemContext, registry, homeDir st
return auth.Username, auth.Password, nil
}
-// RemoveAuthentication deletes the credentials stored in auth.json
+// RemoveAuthentication removes credentials for `registry` from all possible
+// sources such as credential helpers and auth files.
func RemoveAuthentication(sys *types.SystemContext, registry string) error {
- return modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
- // First try cred helpers.
- if ch, exists := auths.CredHelpers[registry]; exists {
- return false, deleteAuthFromCredHelper(ch, registry)
+ helpers, err := sysregistriesv2.CredentialHelpers(sys)
+ if err != nil {
+ return err
+ }
+
+ var multiErr error
+ isLoggedIn := false
+
+ removeFromCredHelper := func(helper string) {
+ err := deleteAuthFromCredHelper(helper, registry)
+ if err == nil {
+ logrus.Debugf("Credentials for %q were deleted from credential helper %s", registry, helper)
+ isLoggedIn = true
+ return
+ }
+ if credentials.IsErrCredentialsNotFoundMessage(err.Error()) {
+ logrus.Debugf("Not logged in to %s with credential helper %s", registry, helper)
+ return
}
+ multiErr = multierror.Append(multiErr, errors.Wrapf(err, "error removing credentials for %s from credential helper %s", registry, helper))
+ }
- // Next if keyring is enabled try kernel keyring
- if enableKeyring {
- err := deleteAuthFromKernelKeyring(registry)
- if err == nil {
- logrus.Debugf("credentials for %s were deleted from the kernel keyring", registry)
- return false, nil
+ for _, helper := range helpers {
+ var err error
+ switch helper {
+ // Special-case the built-in helper for auth files.
+ case sysregistriesv2.AuthenticationFileHelper:
+ err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
+ if innerHelper, exists := auths.CredHelpers[registry]; exists {
+ removeFromCredHelper(innerHelper)
+ }
+ if _, ok := auths.AuthConfigs[registry]; ok {
+ isLoggedIn = true
+ delete(auths.AuthConfigs, registry)
+ } else if _, ok := auths.AuthConfigs[normalizeRegistry(registry)]; ok {
+ isLoggedIn = true
+ delete(auths.AuthConfigs, normalizeRegistry(registry))
+ }
+ return true, multiErr
+ })
+ if err != nil {
+ multiErr = multierror.Append(multiErr, err)
}
- logrus.Debugf("failed to delete credentials from the kernel keyring, falling back to authfiles")
+ // External helpers.
+ default:
+ removeFromCredHelper(helper)
}
+ }
- if _, ok := auths.AuthConfigs[registry]; ok {
- delete(auths.AuthConfigs, registry)
- } else if _, ok := auths.AuthConfigs[normalizeRegistry(registry)]; ok {
- delete(auths.AuthConfigs, normalizeRegistry(registry))
- } else {
- return false, ErrNotLoggedIn
- }
- return true, nil
- })
+ if multiErr != nil {
+ return multiErr
+ }
+ if !isLoggedIn {
+ return ErrNotLoggedIn
+ }
+
+ return nil
}
-// RemoveAllAuthentication deletes all the credentials stored in auth.json and kernel keyring
+// RemoveAllAuthentication deletes all the credentials stored in credential
+// helpers and auth files.
func RemoveAllAuthentication(sys *types.SystemContext) error {
- return modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
- if enableKeyring {
- err := removeAllAuthFromKernelKeyring()
- if err == nil {
- logrus.Debugf("removing all credentials from kernel keyring")
- return false, nil
+ helpers, err := sysregistriesv2.CredentialHelpers(sys)
+ if err != nil {
+ return err
+ }
+
+ var multiErr error
+ for _, helper := range helpers {
+ var err error
+ switch helper {
+ // Special-case the built-in helper for auth files.
+ case sysregistriesv2.AuthenticationFileHelper:
+ err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) {
+ for registry, helper := range auths.CredHelpers {
+ // Helpers in auth files are expected
+ // to exist, so no special treatment
+ // for them.
+ if err := deleteAuthFromCredHelper(helper, registry); err != nil {
+ return false, err
+ }
+ }
+ auths.CredHelpers = make(map[string]string)
+ auths.AuthConfigs = make(map[string]dockerAuthConfig)
+ return true, nil
+ })
+ // External helpers.
+ default:
+ var creds map[string]string
+ creds, err = listAuthsFromCredHelper(helper)
+ switch errors.Cause(err) {
+ case nil:
+ for registry := range creds {
+ err = deleteAuthFromCredHelper(helper, registry)
+ if err != nil {
+ break
+ }
+ }
+ case exec.ErrNotFound:
+ // It's okay if the helper doesn't exist.
+ continue
+ default:
+ // fall through
}
- logrus.Debugf("error removing credentials from kernel keyring")
}
- auths.CredHelpers = make(map[string]string)
- auths.AuthConfigs = make(map[string]dockerAuthConfig)
- return true, nil
- })
+ if err != nil {
+ logrus.Debugf("Error removing credentials from credential helper %s: %v", helper, err)
+ multiErr = multierror.Append(multiErr, err)
+ continue
+ }
+ logrus.Debugf("All credentials removed from credential helper %s", helper)
+ }
+
+ return multiErr
+}
+
+func listAuthsFromCredHelper(credHelper string) (map[string]string, error) {
+ helperName := fmt.Sprintf("docker-credential-%s", credHelper)
+ p := helperclient.NewShellProgramFunc(helperName)
+ return helperclient.List(p)
}
-// getPathToAuth gets the path of the auth.json file used for reading and writing credentials
+// getPathToAuth gets the path of the auth.json file used for reading and writting credentials
// returns the path, and a bool specifies whether the file is in legacy format
func getPathToAuth(sys *types.SystemContext) (string, bool, error) {
return getPathToAuthWithOS(sys, runtime.GOOS)
@@ -387,14 +531,17 @@ func modifyJSON(sys *types.SystemContext, editor func(auths *dockerConfigFile) (
return nil
}
-func getAuthFromCredHelper(credHelper, registry string) (string, string, error) {
+func getAuthFromCredHelper(credHelper, registry string) (types.DockerAuthConfig, error) {
helperName := fmt.Sprintf("docker-credential-%s", credHelper)
p := helperclient.NewShellProgramFunc(helperName)
creds, err := helperclient.Get(p, registry)
if err != nil {
- return "", "", err
+ return types.DockerAuthConfig{}, err
}
- return creds.Username, creds.Secret, nil
+ return types.DockerAuthConfig{
+ Username: creds.Username,
+ Password: creds.Secret,
+ }, nil
}
func setAuthToCredHelper(credHelper, registry, username, password string) error {
@@ -423,15 +570,7 @@ func findAuthentication(registry, path string, legacyFormat bool) (types.DockerA
// First try cred helpers. They should always be normalized.
if ch, exists := auths.CredHelpers[registry]; exists {
- username, password, err := getAuthFromCredHelper(ch, registry)
- if err != nil {
- return types.DockerAuthConfig{}, err
- }
-
- return types.DockerAuthConfig{
- Username: username,
- Password: password,
- }, nil
+ return getAuthFromCredHelper(ch, registry)
}
// I'm feeling lucky
diff --git a/vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go b/vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go
index 1531d6943..5bbfb450f 100644
--- a/vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go
+++ b/vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go
@@ -9,9 +9,13 @@ import (
"github.com/sirupsen/logrus"
)
-const keyDescribePrefix = "container-registry-login:"
+// NOTE: none of the functions here are currently used. If we ever want to
+// reenable keyring support, we should introduce a similar built-in credential
+// helpers as for `sysregistriesv2.AuthenticationFileHelper`.
-func getAuthFromKernelKeyring(registry string) (string, string, error) {
+const keyDescribePrefix = "container-registry-login:" // nolint
+
+func getAuthFromKernelKeyring(registry string) (string, string, error) { // nolint
userkeyring, err := keyctl.UserKeyring()
if err != nil {
return "", "", err
@@ -31,7 +35,7 @@ func getAuthFromKernelKeyring(registry string) (string, string, error) {
return parts[0], parts[1], nil
}
-func deleteAuthFromKernelKeyring(registry string) error {
+func deleteAuthFromKernelKeyring(registry string) error { // nolint
userkeyring, err := keyctl.UserKeyring()
if err != nil {
@@ -44,7 +48,7 @@ func deleteAuthFromKernelKeyring(registry string) error {
return key.Unlink()
}
-func removeAllAuthFromKernelKeyring() error {
+func removeAllAuthFromKernelKeyring() error { // nolint
keys, err := keyctl.ReadUserKeyring()
if err != nil {
return err
@@ -77,7 +81,7 @@ func removeAllAuthFromKernelKeyring() error {
return nil
}
-func setAuthToKernelKeyring(registry, username, password string) error {
+func setAuthToKernelKeyring(registry, username, password string) error { // nolint
keyring, err := keyctl.SessionKeyring()
if err != nil {
return err
@@ -110,6 +114,6 @@ func setAuthToKernelKeyring(registry, username, password string) error {
return nil
}
-func genDescription(registry string) string {
+func genDescription(registry string) string { // nolint
return fmt.Sprintf("%s%s", keyDescribePrefix, registry)
}
diff --git a/vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go b/vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go
index f1e5c453e..b64f44674 100644
--- a/vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go
+++ b/vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go
@@ -313,7 +313,10 @@ func Resolve(ctx *types.SystemContext, name string) (*Resolved, error) {
}
// Error out if there's no matching alias and no search registries.
if len(unqualifiedSearchRegistries) == 0 {
- return nil, errors.Errorf("short-name %q did not resolve to an alias and no unqualified-search registries are defined in %q", name, usrConfig)
+ if usrConfig != "" {
+ return nil, errors.Errorf("short-name %q did not resolve to an alias and no unqualified-search registries are defined in %q", name, usrConfig)
+ }
+ return nil, errors.Errorf("short-name %q did not resolve to an alias and no containers-registries.conf(5) was found", name)
}
resolved.originDescription = usrConfig
diff --git a/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go b/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go
index 3312237ef..880f8c871 100644
--- a/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go
+++ b/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go
@@ -30,16 +30,24 @@ const builtinRegistriesConfPath = "/etc/containers/registries.conf"
// systemRegistriesConfDirPath is the path to the system-wide registry
// configuration directory and is used to add/subtract potential registries for
// obtaining images. You can override this at build time with
-// -ldflags '-X github.com/containers/image/v5/sysregistries.systemRegistriesConfDirecotyPath=$your_path'
+// -ldflags '-X github.com/containers/image/v5/sysregistries.systemRegistriesConfDirectoryPath=$your_path'
var systemRegistriesConfDirPath = builtinRegistriesConfDirPath
// builtinRegistriesConfDirPath is the path to the registry configuration directory.
// DO NOT change this, instead see systemRegistriesConfDirectoryPath above.
const builtinRegistriesConfDirPath = "/etc/containers/registries.conf.d"
+// AuthenticationFileHelper is a special key for credential helpers indicating
+// the usage of consulting containers-auth.json files instead of a credential
+// helper.
+const AuthenticationFileHelper = "containers-auth.json"
+
// Endpoint describes a remote location of a registry.
type Endpoint struct {
- // The endpoint's remote location.
+ // The endpoint's remote location. Can be empty iff Prefix contains
+ // wildcard in the format: "*.example.com" for subdomain matching.
+ // Please refer to FindRegistry / PullSourcesFromReference instead
+ // of accessing/interpreting `Location` directly.
Location string `toml:"location,omitempty"`
// If true, certs verification will be skipped and HTTP (non-TLS)
// connections will be allowed.
@@ -57,11 +65,26 @@ var userRegistriesDir = filepath.FromSlash(".config/containers/registries.conf.d
// The function errors if the newly created reference is not parsable.
func (e *Endpoint) rewriteReference(ref reference.Named, prefix string) (reference.Named, error) {
refString := ref.String()
- if !refMatchesPrefix(refString, prefix) {
+ var newNamedRef string
+ // refMatchingPrefix returns the length of the match. Everything that
+ // follows the match gets appended to registries location.
+ prefixLen := refMatchingPrefix(refString, prefix)
+ if prefixLen == -1 {
return nil, fmt.Errorf("invalid prefix '%v' for reference '%v'", prefix, refString)
}
-
- newNamedRef := strings.Replace(refString, prefix, e.Location, 1)
+ // In the case of an empty `location` field, simply return the original
+ // input ref as-is.
+ //
+ // FIXME: already validated in postProcessRegistries, so check can probably
+ // be dropped.
+ // https://github.com/containers/image/pull/1191#discussion_r610621608
+ if e.Location == "" {
+ if prefix[:2] != "*." {
+ return nil, fmt.Errorf("invalid prefix '%v' for empty location, should be in the format: *.example.com", prefix)
+ }
+ return ref, nil
+ }
+ newNamedRef = e.Location + refString[prefixLen:]
newParsedRef, err := reference.ParseNamed(newNamedRef)
if err != nil {
return nil, errors.Wrapf(err, "error rewriting reference")
@@ -77,6 +100,11 @@ type Registry struct {
// and we pull from "example.com/bar/myimage:latest", the image will
// effectively be pulled from "example.com/foo/bar/myimage:latest".
// If no Prefix is specified, it defaults to the specified location.
+ // Prefix can also be in the format: "*.example.com" for matching
+ // subdomains. The wildcard should only be in the beginning and should also
+ // not contain any namespaces or special characters: "/", "@" or ":".
+ // Please refer to FindRegistry / PullSourcesFromReference instead
+ // of accessing/interpreting `Prefix` directly.
Prefix string `toml:"prefix"`
// A registry is an Endpoint too
Endpoint
@@ -154,6 +182,14 @@ type V2RegistriesConf struct {
Registries []Registry `toml:"registry"`
// An array of host[:port] (not prefix!) entries to use for resolving unqualified image references
UnqualifiedSearchRegistries []string `toml:"unqualified-search-registries"`
+ // An array of global credential helpers to use for authentication
+ // (e.g., ["pass", "secretservice"]). The helpers are consulted in the
+ // specified order. Note that "containers-auth.json" is a reserved
+ // value for consulting auth files as specified in
+ // containers-auth.json(5).
+ //
+ // If empty, CredentialHelpers defaults to ["containers-auth.json"].
+ CredentialHelpers []string `toml:"credential-helpers"`
// ShortNameMode defines how short-name resolution should be handled by
// _consumers_ of this package. Depending on the mode, the user should
@@ -177,7 +213,7 @@ func (config *V2RegistriesConf) Nonempty() bool {
// parsedConfig is the result of parsing, and possibly merging, configuration files;
// it is the boundary between the process of reading+ingesting the files, and
-// later interpreting the configuraiton based on caller’s requests.
+// later interpreting the configuration based on caller’s requests.
type parsedConfig struct {
// NOTE: Update also parsedConfig.updateWithConfigurationFrom!
@@ -212,9 +248,15 @@ func (e *InvalidRegistries) Error() string {
func parseLocation(input string) (string, error) {
trimmed := strings.TrimRight(input, "/")
- if trimmed == "" {
- return "", &InvalidRegistries{s: "invalid location: cannot be empty"}
- }
+ // FIXME: This check needs to exist but fails for empty Location field with
+ // wildcarded prefix. Removal of this check "only" allows invalid input in,
+ // and does not prevent correct operation.
+ // https://github.com/containers/image/pull/1191#discussion_r610122617
+ //
+ // if trimmed == "" {
+ // return "", &InvalidRegistries{s: "invalid location: cannot be empty"}
+ // }
+ //
if strings.HasPrefix(trimmed, "http://") || strings.HasPrefix(trimmed, "https://") {
msg := fmt.Sprintf("invalid location '%s': URI schemes are not supported", input)
@@ -293,12 +335,20 @@ func (config *V2RegistriesConf) postProcessRegistries() error {
}
if reg.Prefix == "" {
+ if reg.Location == "" {
+ return &InvalidRegistries{s: "invalid condition: both location and prefix are unset"}
+ }
reg.Prefix = reg.Location
} else {
reg.Prefix, err = parseLocation(reg.Prefix)
if err != nil {
return err
}
+ // FIXME: allow config authors to always use Prefix.
+ // https://github.com/containers/image/pull/1191#discussion_r610622495
+ if reg.Prefix[:2] != "*." && reg.Location == "" {
+ return &InvalidRegistries{s: "invalid condition: location is unset and prefix is not in the format: *.example.com"}
+ }
}
// make sure mirrors are valid
@@ -307,8 +357,19 @@ func (config *V2RegistriesConf) postProcessRegistries() error {
if err != nil {
return err
}
+
+ //FIXME: unqualifiedSearchRegistries now also accepts empty values
+ //and shouldn't
+ // https://github.com/containers/image/pull/1191#discussion_r610623216
+ if mir.Location == "" {
+ return &InvalidRegistries{s: "invalid condition: mirror location is unset"}
+ }
+ }
+ if reg.Location == "" {
+ regMap[reg.Prefix] = append(regMap[reg.Prefix], reg)
+ } else {
+ regMap[reg.Location] = append(regMap[reg.Location], reg)
}
- regMap[reg.Location] = append(regMap[reg.Location], reg)
}
// Given a registry can be mentioned multiple times (e.g., to have
@@ -318,7 +379,13 @@ func (config *V2RegistriesConf) postProcessRegistries() error {
// Note: we need to iterate over the registries array to ensure a
// deterministic behavior which is not guaranteed by maps.
for _, reg := range config.Registries {
- others, ok := regMap[reg.Location]
+ var others []*Registry
+ var ok bool
+ if reg.Location == "" {
+ others, ok = regMap[reg.Prefix]
+ } else {
+ others, ok = regMap[reg.Location]
+ }
if !ok {
return fmt.Errorf("Internal error in V2RegistriesConf.PostProcess: entry in regMap is missing")
}
@@ -450,7 +517,7 @@ func newConfigWrapperWithHomeDir(ctx *types.SystemContext, homeDir string) confi
return wrapper
}
-// ConfigurationSourceDescription returns a string containres paths of registries.conf and registries.conf.d
+// ConfigurationSourceDescription returns a string containers paths of registries.conf and registries.conf.d
func ConfigurationSourceDescription(ctx *types.SystemContext) string {
wrapper := newConfigWrapper(ctx)
configSources := []string{wrapper.configPath}
@@ -601,11 +668,17 @@ func tryUpdatingCache(ctx *types.SystemContext, wrapper configWrapper) (*parsedC
config.shortNameMode = defaultShortNameMode
}
+ if len(config.partialV2.CredentialHelpers) == 0 {
+ config.partialV2.CredentialHelpers = []string{AuthenticationFileHelper}
+ }
+
// populate the cache
configCache[wrapper] = config
return config, nil
}
+// GetRegistries has been deprecated. Use FindRegistry instead.
+//
// GetRegistries loads and returns the registries specified in the config.
// Note the parsed content of registry config files is cached. For reloading,
// use `InvalidateCache` and re-call `GetRegistries`.
@@ -663,27 +736,72 @@ func GetShortNameMode(ctx *types.SystemContext) (types.ShortNameMode, error) {
return config.shortNameMode, err
}
-// refMatchesPrefix returns true iff ref,
+// CredentialHelpers returns the global top-level credential helpers.
+func CredentialHelpers(sys *types.SystemContext) ([]string, error) {
+ config, err := getConfig(sys)
+ if err != nil {
+ return nil, err
+ }
+ return config.partialV2.CredentialHelpers, nil
+}
+
+// refMatchingSubdomainPrefix returns the length of ref
+// iff ref, which is a registry, repository namespace, repository or image reference (as formatted by
+// reference.Domain(), reference.Named.Name() or reference.Reference.String()
+// — note that this requires the name to start with an explicit hostname!),
+// matches a Registry.Prefix value containing wildcarded subdomains in the
+// format: *.example.com. Wildcards are only accepted at the beginning, so
+// other formats like example.*.com will not work. Wildcarded prefixes also
+// cannot contain port numbers or namespaces in them.
+func refMatchingSubdomainPrefix(ref, prefix string) int {
+ index := strings.Index(ref, prefix[1:])
+ if index == -1 {
+ return -1
+ }
+ if strings.Contains(ref[:index], "/") {
+ return -1
+ }
+ index += len(prefix[1:])
+ if index == len(ref) {
+ return index
+ }
+ switch ref[index] {
+ case ':', '/', '@':
+ return index
+ default:
+ return -1
+ }
+}
+
+// refMatchingPrefix returns the length of the prefix iff ref,
// which is a registry, repository namespace, repository or image reference (as formatted by
// reference.Domain(), reference.Named.Name() or reference.Reference.String()
// — note that this requires the name to start with an explicit hostname!),
// matches a Registry.Prefix value.
// (This is split from the caller primarily to make testing easier.)
-func refMatchesPrefix(ref, prefix string) bool {
+func refMatchingPrefix(ref, prefix string) int {
switch {
+ case prefix[0:2] == "*.":
+ return refMatchingSubdomainPrefix(ref, prefix)
case len(ref) < len(prefix):
- return false
+ return -1
case len(ref) == len(prefix):
- return ref == prefix
+ if ref == prefix {
+ return len(prefix)
+ }
+ return -1
case len(ref) > len(prefix):
if !strings.HasPrefix(ref, prefix) {
- return false
+ return -1
}
c := ref[len(prefix)]
// This allows "example.com:5000" to match "example.com",
// which is unintended; that will get fixed eventually, DON'T RELY
// ON THE CURRENT BEHAVIOR.
- return c == ':' || c == '/' || c == '@'
+ if c == ':' || c == '/' || c == '@' {
+ return len(prefix)
+ }
+ return -1
default:
panic("Internal error: impossible comparison outcome")
}
@@ -700,10 +818,16 @@ func FindRegistry(ctx *types.SystemContext, ref string) (*Registry, error) {
return nil, err
}
+ return findRegistryWithParsedConfig(config, ref)
+}
+
+// findRegistryWithParsedConfig implements `FindRegistry` with a pre-loaded
+// parseConfig.
+func findRegistryWithParsedConfig(config *parsedConfig, ref string) (*Registry, error) {
reg := Registry{}
prefixLen := 0
for _, r := range config.partialV2.Registries {
- if refMatchesPrefix(ref, r.Prefix) {
+ if refMatchingPrefix(ref, r.Prefix) != -1 {
length := len(r.Prefix)
if length > prefixLen {
reg = r
@@ -772,6 +896,17 @@ func loadConfigFile(path string, forceV2 bool) (*parsedConfig, error) {
res.shortNameMode = types.ShortNameModeInvalid
}
+ // Valid wildcarded prefixes must be in the format: *.example.com
+ // FIXME: Move to postProcessRegistries
+ // https://github.com/containers/image/pull/1191#discussion_r610623829
+ for i := range res.partialV2.Registries {
+ prefix := res.partialV2.Registries[i].Prefix
+ if prefix[:2] == "*." && strings.ContainsAny(prefix, "/@:") {
+ msg := fmt.Sprintf("Wildcarded prefix should be in the format: *.example.com. Current prefix %q is incorrectly formatted", prefix)
+ return nil, &InvalidRegistries{s: msg}
+ }
+ }
+
// Parse and validate short-name aliases.
cache, err := newShortNameAliasCache(path, &res.partialV2.shortNameAliasConf)
if err != nil {
@@ -825,6 +960,11 @@ func (c *parsedConfig) updateWithConfigurationFrom(updates *parsedConfig) {
c.unqualifiedSearchRegistriesOrigin = updates.unqualifiedSearchRegistriesOrigin
}
+ // == Merge credential helpers:
+ if updates.partialV2.CredentialHelpers != nil {
+ c.partialV2.CredentialHelpers = updates.partialV2.CredentialHelpers
+ }
+
// == Merge shortNameMode:
// We don’t maintain c.partialV2.ShortNameMode.
if updates.shortNameMode != types.ShortNameModeInvalid {
diff --git a/vendor/github.com/containers/image/v5/storage/storage_image.go b/vendor/github.com/containers/image/v5/storage/storage_image.go
index 5340690bc..ae020dd66 100644
--- a/vendor/github.com/containers/image/v5/storage/storage_image.go
+++ b/vendor/github.com/containers/image/v5/storage/storage_image.go
@@ -18,6 +18,7 @@ import (
"github.com/containers/image/v5/docker/reference"
"github.com/containers/image/v5/image"
"github.com/containers/image/v5/internal/tmpdir"
+ internalTypes "github.com/containers/image/v5/internal/types"
"github.com/containers/image/v5/manifest"
"github.com/containers/image/v5/pkg/blobinfocache/none"
"github.com/containers/image/v5/types"
@@ -45,6 +46,7 @@ var (
type storageImageSource struct {
imageRef storageReference
image *storage.Image
+ systemContext *types.SystemContext // SystemContext used in GetBlob() to create temporary files
layerPosition map[digest.Digest]int // Where we are in reading a blob's layers
cachedManifest []byte // A cached copy of the manifest, if already known, or nil
getBlobMutex sync.Mutex // Mutex to sync state for parallel GetBlob executions
@@ -54,17 +56,31 @@ type storageImageSource struct {
type storageImageDestination struct {
imageRef storageReference
- directory string // Temporary directory where we store blobs until Commit() time
- nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs
- manifest []byte // Manifest contents, temporary
- signatures []byte // Signature contents, temporary
- signatureses map[digest.Digest][]byte // Instance signature contents, temporary
- putBlobMutex sync.Mutex // Mutex to sync state for parallel PutBlob executions
- blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs
- fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes
- filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them
- SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice
- SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // Sizes of each manifest's signature slice
+ directory string // Temporary directory where we store blobs until Commit() time
+ nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs
+ manifest []byte // Manifest contents, temporary
+ signatures []byte // Signature contents, temporary
+ signatureses map[digest.Digest][]byte // Instance signature contents, temporary
+ SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice
+ SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // Sizes of each manifest's signature slice
+
+ // A storage destination may be used concurrently. Accesses are
+ // serialized via a mutex. Please refer to the individual comments
+ // below for details.
+ lock sync.Mutex
+ // Mapping from layer (by index) to the associated ID in the storage.
+ // It's protected *implicitly* since `commitLayer()`, at any given
+ // time, can only be executed by *one* goroutine. Please refer to
+ // `queueOrCommit()` for further details on how the single-caller
+ // guarantee is implemented.
+ indexToStorageID map[int]*string
+ // All accesses to below data are protected by `lock` which is made
+ // *explicit* in the code.
+ blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs
+ fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes
+ filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them
+ currentIndex int // The index of the layer to be committed (i.e., lower indices have already been committed)
+ indexToPulledBlob map[int]*types.BlobInfo // Mapping from layer (by index) to pulled down blob
}
type storageImageCloser struct {
@@ -96,6 +112,7 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, imageRef stor
// Build the reader object.
image := &storageImageSource{
imageRef: imageRef,
+ systemContext: sys,
image: img,
layerPosition: make(map[digest.Digest]int),
SignatureSizes: []int{},
@@ -131,8 +148,35 @@ func (s *storageImageSource) GetBlob(ctx context.Context, info types.BlobInfo, c
if info.Digest == image.GzippedEmptyLayerDigest {
return ioutil.NopCloser(bytes.NewReader(image.GzippedEmptyLayer)), int64(len(image.GzippedEmptyLayer)), nil
}
+
+ // NOTE: the blob is first written to a temporary file and subsequently
+ // closed. The intention is to keep the time we own the storage lock
+ // as short as possible to allow other processes to access the storage.
rc, n, _, err = s.getBlobAndLayerID(info)
- return rc, n, err
+ if err != nil {
+ return nil, 0, err
+ }
+ defer rc.Close()
+
+ tmpFile, err := ioutil.TempFile(tmpdir.TemporaryDirectoryForBigFiles(s.systemContext), "")
+ if err != nil {
+ return nil, 0, err
+ }
+
+ if _, err := io.Copy(tmpFile, rc); err != nil {
+ return nil, 0, err
+ }
+
+ if _, err := tmpFile.Seek(0, 0); err != nil {
+ return nil, 0, err
+ }
+
+ wrapper := ioutils.NewReadCloserWrapper(tmpFile, func() error {
+ defer os.Remove(tmpFile.Name())
+ return tmpFile.Close()
+ })
+
+ return wrapper, n, err
}
// getBlobAndLayer reads the data blob or filesystem layer which matches the digest and size, if given.
@@ -347,14 +391,16 @@ func newImageDestination(sys *types.SystemContext, imageRef storageReference) (*
return nil, errors.Wrapf(err, "error creating a temporary directory")
}
image := &storageImageDestination{
- imageRef: imageRef,
- directory: directory,
- signatureses: make(map[digest.Digest][]byte),
- blobDiffIDs: make(map[digest.Digest]digest.Digest),
- fileSizes: make(map[digest.Digest]int64),
- filenames: make(map[digest.Digest]string),
- SignatureSizes: []int{},
- SignaturesSizes: make(map[digest.Digest][]int),
+ imageRef: imageRef,
+ directory: directory,
+ signatureses: make(map[digest.Digest][]byte),
+ blobDiffIDs: make(map[digest.Digest]digest.Digest),
+ fileSizes: make(map[digest.Digest]int64),
+ filenames: make(map[digest.Digest]string),
+ SignatureSizes: []int{},
+ SignaturesSizes: make(map[digest.Digest][]int),
+ indexToStorageID: make(map[int]*string),
+ indexToPulledBlob: make(map[int]*types.BlobInfo),
}
return image, nil
}
@@ -381,6 +427,26 @@ func (s *storageImageDestination) computeNextBlobCacheFile() string {
return filepath.Join(s.directory, fmt.Sprintf("%d", atomic.AddInt32(&s.nextTempFileID, 1)))
}
+// PutBlobWithOptions is a wrapper around PutBlob. If options.LayerIndex is
+// set, the blob will be committed directly. Either by the calling goroutine
+// or by another goroutine already committing layers.
+//
+// Please not that TryReusingBlobWithOptions and PutBlobWithOptions *must* be
+// used the together. Mixing the two with non "WithOptions" functions is not
+// supported.
+func (s *storageImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, blobinfo types.BlobInfo, options internalTypes.PutBlobOptions) (types.BlobInfo, error) {
+ info, err := s.PutBlob(ctx, stream, blobinfo, options.Cache, options.IsConfig)
+ if err != nil {
+ return info, err
+ }
+
+ if options.IsConfig || options.LayerIndex == nil {
+ return info, nil
+ }
+
+ return info, s.queueOrCommit(ctx, info, *options.LayerIndex)
+}
+
// HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently.
func (s *storageImageDestination) HasThreadSafePutBlob() bool {
return true
@@ -436,11 +502,11 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader,
return errorBlobInfo, errors.WithStack(ErrBlobSizeMismatch)
}
// Record information about the blob.
- s.putBlobMutex.Lock()
+ s.lock.Lock()
s.blobDiffIDs[hasher.Digest()] = diffID.Digest()
s.fileSizes[hasher.Digest()] = counter.Count
s.filenames[hasher.Digest()] = filename
- s.putBlobMutex.Unlock()
+ s.lock.Unlock()
blobDigest := blobinfo.Digest
if blobDigest.Validate() != nil {
blobDigest = hasher.Digest()
@@ -458,6 +524,22 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader,
}, nil
}
+// TryReusingBlobWithOptions is a wrapper around TryReusingBlob. If
+// options.LayerIndex is set, the reused blob will be recoreded as already
+// pulled.
+//
+// Please not that TryReusingBlobWithOptions and PutBlobWithOptions *must* be
+// used the together. Mixing the two with the non "WithOptions" functions
+// is not supported.
+func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context, blobinfo types.BlobInfo, options internalTypes.TryReusingBlobOptions) (bool, types.BlobInfo, error) {
+ reused, info, err := s.TryReusingBlob(ctx, blobinfo, options.Cache, options.CanSubstitute)
+ if err != nil || !reused || options.LayerIndex == nil {
+ return reused, info, err
+ }
+
+ return reused, info, s.queueOrCommit(ctx, info, *options.LayerIndex)
+}
+
// TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination
// (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree).
// info.Digest must not be empty.
@@ -469,8 +551,8 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader,
// May use and/or update cache.
func (s *storageImageDestination) TryReusingBlob(ctx context.Context, blobinfo types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) {
// lock the entire method as it executes fairly quickly
- s.putBlobMutex.Lock()
- defer s.putBlobMutex.Unlock()
+ s.lock.Lock()
+ defer s.lock.Unlock()
if blobinfo.Digest == "" {
return false, types.BlobInfo{}, errors.Errorf(`Can not check for a blob with unknown digest`)
}
@@ -607,6 +689,192 @@ func (s *storageImageDestination) getConfigBlob(info types.BlobInfo) ([]byte, er
return nil, errors.New("blob not found")
}
+// queueOrCommit queues in the specified blob to be committed to the storage.
+// If no other goroutine is already committing layers, the layer and all
+// subsequent layers (if already queued) will be committed to the storage.
+func (s *storageImageDestination) queueOrCommit(ctx context.Context, blob types.BlobInfo, index int) error {
+ // NOTE: whenever the code below is touched, make sure that all code
+ // paths unlock the lock and to unlock it exactly once.
+ //
+ // Conceptually, the code is divided in two stages:
+ //
+ // 1) Queue in work by marking the layer as ready to be committed.
+ // If at least one previous/parent layer with a lower index has
+ // not yet been committed, return early.
+ //
+ // 2) Process the queued-in work by committing the "ready" layers
+ // in sequence. Make sure that more items can be queued-in
+ // during the comparatively I/O expensive task of committing a
+ // layer.
+ //
+ // The conceptual benefit of this design is that caller can continue
+ // pulling layers after an early return. At any given time, only one
+ // caller is the "worker" routine comitting layers. All other routines
+ // can continue pulling and queuing in layers.
+ s.lock.Lock()
+ s.indexToPulledBlob[index] = &blob
+
+ // We're still waiting for at least one previous/parent layer to be
+ // committed, so there's nothing to do.
+ if index != s.currentIndex {
+ s.lock.Unlock()
+ return nil
+ }
+
+ for info := s.indexToPulledBlob[index]; info != nil; info = s.indexToPulledBlob[index] {
+ s.lock.Unlock()
+ layerInfo := manifest.LayerInfo{
+ BlobInfo: *info,
+ EmptyLayer: info.Digest == image.GzippedEmptyLayerDigest,
+ }
+ // Note: commitLayer locks on-demand.
+ if err := s.commitLayer(ctx, layerInfo, index); err != nil {
+ return err
+ }
+ s.lock.Lock()
+ index++
+ }
+
+ // Set the index at the very end to make sure that only one routine
+ // enters stage 2).
+ s.currentIndex = index
+ s.lock.Unlock()
+ return nil
+}
+
+// commitLayer commits the specified blob with the given index to the storage.
+// Note that the previous layer is expected to already be committed.
+//
+// Caution: this function must be called without holding `s.lock`. Callers
+// must guarantee that, at any given time, at most one goroutine may execute
+// `commitLayer()`.
+func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest.LayerInfo, index int) error {
+ // Already commited? Return early.
+ if _, alreadyCommitted := s.indexToStorageID[index]; alreadyCommitted {
+ return nil
+ }
+
+ // Start with an empty string or the previous layer ID. Note that
+ // `s.indexToStorageID` can only be accessed by *one* goroutine at any
+ // given time. Hence, we don't need to lock accesses.
+ var lastLayer string
+ if prev := s.indexToStorageID[index-1]; prev != nil {
+ lastLayer = *prev
+ }
+
+ // Carry over the previous ID for empty non-base layers.
+ if blob.EmptyLayer && index > 0 {
+ s.indexToStorageID[index] = &lastLayer
+ return nil
+ }
+
+ // Check if there's already a layer with the ID that we'd give to the result of applying
+ // this layer blob to its parent, if it has one, or the blob's hex value otherwise.
+ s.lock.Lock()
+ diffID, haveDiffID := s.blobDiffIDs[blob.Digest]
+ s.lock.Unlock()
+ if !haveDiffID {
+ // Check if it's elsewhere and the caller just forgot to pass it to us in a PutBlob(),
+ // or to even check if we had it.
+ // Use none.NoCache to avoid a repeated DiffID lookup in the BlobInfoCache; a caller
+ // that relies on using a blob digest that has never been seen by the store had better call
+ // TryReusingBlob; not calling PutBlob already violates the documented API, so there’s only
+ // so far we are going to accommodate that (if we should be doing that at all).
+ logrus.Debugf("looking for diffID for blob %+v", blob.Digest)
+ // NOTE: use `TryReusingBlob` to prevent recursion.
+ has, _, err := s.TryReusingBlob(ctx, blob.BlobInfo, none.NoCache, false)
+ if err != nil {
+ return errors.Wrapf(err, "error checking for a layer based on blob %q", blob.Digest.String())
+ }
+ if !has {
+ return errors.Errorf("error determining uncompressed digest for blob %q", blob.Digest.String())
+ }
+ diffID, haveDiffID = s.blobDiffIDs[blob.Digest]
+ if !haveDiffID {
+ return errors.Errorf("we have blob %q, but don't know its uncompressed digest", blob.Digest.String())
+ }
+ }
+ id := diffID.Hex()
+ if lastLayer != "" {
+ id = digest.Canonical.FromBytes([]byte(lastLayer + "+" + diffID.Hex())).Hex()
+ }
+ if layer, err2 := s.imageRef.transport.store.Layer(id); layer != nil && err2 == nil {
+ // There's already a layer that should have the right contents, just reuse it.
+ lastLayer = layer.ID
+ s.indexToStorageID[index] = &lastLayer
+ return nil
+ }
+ // Check if we previously cached a file with that blob's contents. If we didn't,
+ // then we need to read the desired contents from a layer.
+ s.lock.Lock()
+ filename, ok := s.filenames[blob.Digest]
+ s.lock.Unlock()
+ if !ok {
+ // Try to find the layer with contents matching that blobsum.
+ layer := ""
+ layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(diffID)
+ if err2 == nil && len(layers) > 0 {
+ layer = layers[0].ID
+ } else {
+ layers, err2 = s.imageRef.transport.store.LayersByCompressedDigest(blob.Digest)
+ if err2 == nil && len(layers) > 0 {
+ layer = layers[0].ID
+ }
+ }
+ if layer == "" {
+ return errors.Wrapf(err2, "error locating layer for blob %q", blob.Digest)
+ }
+ // Read the layer's contents.
+ noCompression := archive.Uncompressed
+ diffOptions := &storage.DiffOptions{
+ Compression: &noCompression,
+ }
+ diff, err2 := s.imageRef.transport.store.Diff("", layer, diffOptions)
+ if err2 != nil {
+ return errors.Wrapf(err2, "error reading layer %q for blob %q", layer, blob.Digest)
+ }
+ // Copy the layer diff to a file. Diff() takes a lock that it holds
+ // until the ReadCloser that it returns is closed, and PutLayer() wants
+ // the same lock, so the diff can't just be directly streamed from one
+ // to the other.
+ filename = s.computeNextBlobCacheFile()
+ file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600)
+ if err != nil {
+ diff.Close()
+ return errors.Wrapf(err, "error creating temporary file %q", filename)
+ }
+ // Copy the data to the file.
+ // TODO: This can take quite some time, and should ideally be cancellable using
+ // ctx.Done().
+ _, err = io.Copy(file, diff)
+ diff.Close()
+ file.Close()
+ if err != nil {
+ return errors.Wrapf(err, "error storing blob to file %q", filename)
+ }
+ // Make sure that we can find this file later, should we need the layer's
+ // contents again.
+ s.lock.Lock()
+ s.filenames[blob.Digest] = filename
+ s.lock.Unlock()
+ }
+ // Read the cached blob and use it as a diff.
+ file, err := os.Open(filename)
+ if err != nil {
+ return errors.Wrapf(err, "error opening file %q", filename)
+ }
+ defer file.Close()
+ // Build the new layer using the diff, regardless of where it came from.
+ // TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
+ layer, _, err := s.imageRef.transport.store.PutLayer(id, lastLayer, nil, "", false, nil, file)
+ if err != nil && errors.Cause(err) != storage.ErrDuplicateID {
+ return errors.Wrapf(err, "error adding layer with blob %q", blob.Digest)
+ }
+
+ s.indexToStorageID[index] = &layer.ID
+ return nil
+}
+
func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error {
if len(s.manifest) == 0 {
return errors.New("Internal error: storageImageDestination.Commit() called without PutManifest()")
@@ -644,108 +912,19 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t
return errors.Wrapf(err, "error parsing manifest")
}
layerBlobs := man.LayerInfos()
- // Extract or find the layers.
- lastLayer := ""
- for _, blob := range layerBlobs {
- if blob.EmptyLayer {
- continue
- }
-
- // Check if there's already a layer with the ID that we'd give to the result of applying
- // this layer blob to its parent, if it has one, or the blob's hex value otherwise.
- diffID, haveDiffID := s.blobDiffIDs[blob.Digest]
- if !haveDiffID {
- // Check if it's elsewhere and the caller just forgot to pass it to us in a PutBlob(),
- // or to even check if we had it.
- // Use none.NoCache to avoid a repeated DiffID lookup in the BlobInfoCache; a caller
- // that relies on using a blob digest that has never been seen by the store had better call
- // TryReusingBlob; not calling PutBlob already violates the documented API, so there’s only
- // so far we are going to accommodate that (if we should be doing that at all).
- logrus.Debugf("looking for diffID for blob %+v", blob.Digest)
- has, _, err := s.TryReusingBlob(ctx, blob.BlobInfo, none.NoCache, false)
- if err != nil {
- return errors.Wrapf(err, "error checking for a layer based on blob %q", blob.Digest.String())
- }
- if !has {
- return errors.Errorf("error determining uncompressed digest for blob %q", blob.Digest.String())
- }
- diffID, haveDiffID = s.blobDiffIDs[blob.Digest]
- if !haveDiffID {
- return errors.Errorf("we have blob %q, but don't know its uncompressed digest", blob.Digest.String())
- }
- }
- id := diffID.Hex()
- if lastLayer != "" {
- id = digest.Canonical.FromBytes([]byte(lastLayer + "+" + diffID.Hex())).Hex()
- }
- if layer, err2 := s.imageRef.transport.store.Layer(id); layer != nil && err2 == nil {
- // There's already a layer that should have the right contents, just reuse it.
- lastLayer = layer.ID
- continue
- }
- // Check if we previously cached a file with that blob's contents. If we didn't,
- // then we need to read the desired contents from a layer.
- filename, ok := s.filenames[blob.Digest]
- if !ok {
- // Try to find the layer with contents matching that blobsum.
- layer := ""
- layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(diffID)
- if err2 == nil && len(layers) > 0 {
- layer = layers[0].ID
- } else {
- layers, err2 = s.imageRef.transport.store.LayersByCompressedDigest(blob.Digest)
- if err2 == nil && len(layers) > 0 {
- layer = layers[0].ID
- }
- }
- if layer == "" {
- return errors.Wrapf(err2, "error locating layer for blob %q", blob.Digest)
- }
- // Read the layer's contents.
- noCompression := archive.Uncompressed
- diffOptions := &storage.DiffOptions{
- Compression: &noCompression,
- }
- diff, err2 := s.imageRef.transport.store.Diff("", layer, diffOptions)
- if err2 != nil {
- return errors.Wrapf(err2, "error reading layer %q for blob %q", layer, blob.Digest)
- }
- // Copy the layer diff to a file. Diff() takes a lock that it holds
- // until the ReadCloser that it returns is closed, and PutLayer() wants
- // the same lock, so the diff can't just be directly streamed from one
- // to the other.
- filename = s.computeNextBlobCacheFile()
- file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600)
- if err != nil {
- diff.Close()
- return errors.Wrapf(err, "error creating temporary file %q", filename)
- }
- // Copy the data to the file.
- // TODO: This can take quite some time, and should ideally be cancellable using
- // ctx.Done().
- _, err = io.Copy(file, diff)
- diff.Close()
- file.Close()
- if err != nil {
- return errors.Wrapf(err, "error storing blob to file %q", filename)
- }
- // Make sure that we can find this file later, should we need the layer's
- // contents again.
- s.filenames[blob.Digest] = filename
- }
- // Read the cached blob and use it as a diff.
- file, err := os.Open(filename)
- if err != nil {
- return errors.Wrapf(err, "error opening file %q", filename)
+ // Extract, commit, or find the layers.
+ for i, blob := range layerBlobs {
+ if err := s.commitLayer(ctx, blob, i); err != nil {
+ return err
}
- defer file.Close()
- // Build the new layer using the diff, regardless of where it came from.
- // TODO: This can take quite some time, and should ideally be cancellable using ctx.Done().
- layer, _, err := s.imageRef.transport.store.PutLayer(id, lastLayer, nil, "", false, nil, file)
- if err != nil && errors.Cause(err) != storage.ErrDuplicateID {
- return errors.Wrapf(err, "error adding layer with blob %q", blob.Digest)
+ }
+ var lastLayer string
+ if len(layerBlobs) > 0 { // Can happen when using caches
+ prev := s.indexToStorageID[len(layerBlobs)-1]
+ if prev == nil {
+ return errors.Errorf("Internal error: StorageImageDestination.Commit(): previous layer %d hasn't been commited (lastLayer == nil)", len(layerBlobs)-1)
}
- lastLayer = layer.ID
+ lastLayer = *prev
}
// If one of those blobs was a configuration blob, then we can try to dig out the date when the image
diff --git a/vendor/github.com/containers/image/v5/types/types.go b/vendor/github.com/containers/image/v5/types/types.go
index 8655ca443..3e7abd34d 100644
--- a/vendor/github.com/containers/image/v5/types/types.go
+++ b/vendor/github.com/containers/image/v5/types/types.go
@@ -147,7 +147,7 @@ type BlobInfo struct {
}
// BICTransportScope encapsulates transport-dependent representation of a “scope” where blobs are or are not present.
-// BlobInfocache.RecordKnownLocations / BlobInfocache.CandidateLocations record data aboud blobs keyed by (scope, digest).
+// BlobInfocache.RecordKnownLocations / BlobInfocache.CandidateLocations record data about blobs keyed by (scope, digest).
// The scope will typically be similar to an ImageReference, or a superset of it within which blobs are reusable.
//
// NOTE: The contents of this structure may be recorded in a persistent file, possibly shared across different
@@ -179,7 +179,7 @@ type BICReplacementCandidate struct {
// It records two kinds of data:
// - Sets of corresponding digest vs. uncompressed digest ("DiffID") pairs:
// One of the two digests is known to be uncompressed, and a single uncompressed digest may correspond to more than one compressed digest.
-// This allows matching compressed layer blobs to existing local uncompressed layers (to avoid unnecessary download and decompresssion),
+// This allows matching compressed layer blobs to existing local uncompressed layers (to avoid unnecessary download and decompression),
// or uncompressed layer blobs to existing remote compressed layers (to avoid unnecessary compression and upload)/
//
// It is allowed to record an (uncompressed digest, the same uncompressed digest) correspondence, to express that the digest is known
@@ -219,7 +219,7 @@ type BlobInfoCache interface {
// CandidateLocations returns a prioritized, limited, number of blobs and their locations that could possibly be reused
// within the specified (transport scope) (if they still exist, which is not guaranteed).
//
- // If !canSubstitute, the returned cadidates will match the submitted digest exactly; if canSubstitute,
+ // If !canSubstitute, the returned candidates will match the submitted digest exactly; if canSubstitute,
// data from previous RecordDigestUncompressedPair calls is used to also look up variants of the blob which have the same
// uncompressed digest.
CandidateLocations(transport ImageTransport, scope BICTransportScope, digest digest.Digest, canSubstitute bool) []BICReplacementCandidate
@@ -582,7 +582,7 @@ type SystemContext struct {
// === OCI.Transport overrides ===
// If not "", a directory containing a CA certificate (ending with ".crt"),
- // a client certificate (ending with ".cert") and a client ceritificate key
+ // a client certificate (ending with ".cert") and a client certificate key
// (ending with ".key") used when downloading OCI image layers.
OCICertPath string
// Allow downloading OCI image layers over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections.
@@ -594,7 +594,7 @@ type SystemContext struct {
// === docker.Transport overrides ===
// If not "", a directory containing a CA certificate (ending with ".crt"),
- // a client certificate (ending with ".cert") and a client ceritificate key
+ // a client certificate (ending with ".cert") and a client certificate key
// (ending with ".key") used when talking to a Docker Registry.
DockerCertPath string
// If not "", overrides the system’s default path for a directory containing host[:port] subdirectories with the same structure as DockerCertPath above.
diff --git a/vendor/github.com/containers/image/v5/version/version.go b/vendor/github.com/containers/image/v5/version/version.go
index fb7230241..3e9f09aab 100644
--- a/vendor/github.com/containers/image/v5/version/version.go
+++ b/vendor/github.com/containers/image/v5/version/version.go
@@ -6,9 +6,9 @@ const (
// VersionMajor is for an API incompatible changes
VersionMajor = 5
// VersionMinor is for functionality in a backwards-compatible manner
- VersionMinor = 10
+ VersionMinor = 11
// VersionPatch is for backwards-compatible bug fixes
- VersionPatch = 5
+ VersionPatch = 0
// VersionDev indicates development branch. Releases will be empty string.
VersionDev = ""
diff --git a/vendor/github.com/klauspost/compress/flate/deflate.go b/vendor/github.com/klauspost/compress/flate/deflate.go
index 25dbe3e15..3f428d8b5 100644
--- a/vendor/github.com/klauspost/compress/flate/deflate.go
+++ b/vendor/github.com/klauspost/compress/flate/deflate.go
@@ -645,15 +645,15 @@ func (d *compressor) init(w io.Writer, level int) (err error) {
d.fill = (*compressor).fillBlock
d.step = (*compressor).store
case level == ConstantCompression:
- d.w.logNewTablePenalty = 4
- d.window = make([]byte, maxStoreBlockSize)
+ d.w.logNewTablePenalty = 8
+ d.window = make([]byte, 32<<10)
d.fill = (*compressor).fillBlock
d.step = (*compressor).storeHuff
case level == DefaultCompression:
level = 5
fallthrough
case level >= 1 && level <= 6:
- d.w.logNewTablePenalty = 6
+ d.w.logNewTablePenalty = 8
d.fast = newFastEnc(level)
d.window = make([]byte, maxStoreBlockSize)
d.fill = (*compressor).fillBlock
diff --git a/vendor/github.com/klauspost/compress/flate/fast_encoder.go b/vendor/github.com/klauspost/compress/flate/fast_encoder.go
index 4a73e1bdd..678f08105 100644
--- a/vendor/github.com/klauspost/compress/flate/fast_encoder.go
+++ b/vendor/github.com/klauspost/compress/flate/fast_encoder.go
@@ -6,6 +6,7 @@
package flate
import (
+ "encoding/binary"
"fmt"
"math/bits"
)
@@ -65,26 +66,15 @@ func load32(b []byte, i int) uint32 {
}
func load64(b []byte, i int) uint64 {
- // Help the compiler eliminate bounds checks on the read so it can be done in a single read.
- b = b[i:]
- b = b[:8]
- return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
- uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
+ return binary.LittleEndian.Uint64(b[i:])
}
func load3232(b []byte, i int32) uint32 {
- // Help the compiler eliminate bounds checks on the read so it can be done in a single read.
- b = b[i:]
- b = b[:4]
- return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24
+ return binary.LittleEndian.Uint32(b[i:])
}
func load6432(b []byte, i int32) uint64 {
- // Help the compiler eliminate bounds checks on the read so it can be done in a single read.
- b = b[i:]
- b = b[:8]
- return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 |
- uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56
+ return binary.LittleEndian.Uint64(b[i:])
}
func hash(u uint32) uint32 {
@@ -225,9 +215,9 @@ func (e *fastGen) Reset() {
func matchLen(a, b []byte) int {
b = b[:len(a)]
var checked int
- if len(a) > 4 {
+ if len(a) >= 4 {
// Try 4 bytes first
- if diff := load32(a, 0) ^ load32(b, 0); diff != 0 {
+ if diff := binary.LittleEndian.Uint32(a) ^ binary.LittleEndian.Uint32(b); diff != 0 {
return bits.TrailingZeros32(diff) >> 3
}
// Switch to 8 byte matching.
@@ -236,7 +226,7 @@ func matchLen(a, b []byte) int {
b = b[4:]
for len(a) >= 8 {
b = b[:len(a)]
- if diff := load64(a, 0) ^ load64(b, 0); diff != 0 {
+ if diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b); diff != 0 {
return checked + (bits.TrailingZeros64(diff) >> 3)
}
checked += 8
@@ -247,7 +237,7 @@ func matchLen(a, b []byte) int {
b = b[:len(a)]
for i := range a {
if a[i] != b[i] {
- return int(i) + checked
+ return i + checked
}
}
return len(a) + checked
diff --git a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go
index 208d66711..db54be139 100644
--- a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go
+++ b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go
@@ -5,6 +5,7 @@
package flate
import (
+ "encoding/binary"
"io"
)
@@ -206,7 +207,7 @@ func (w *huffmanBitWriter) write(b []byte) {
}
func (w *huffmanBitWriter) writeBits(b int32, nb uint16) {
- w.bits |= uint64(b) << (w.nbits & reg16SizeMask64)
+ w.bits |= uint64(b) << w.nbits
w.nbits += nb
if w.nbits >= 48 {
w.writeOutBits()
@@ -420,13 +421,11 @@ func (w *huffmanBitWriter) writeOutBits() {
w.bits >>= 48
w.nbits -= 48
n := w.nbytes
- w.bytes[n] = byte(bits)
- w.bytes[n+1] = byte(bits >> 8)
- w.bytes[n+2] = byte(bits >> 16)
- w.bytes[n+3] = byte(bits >> 24)
- w.bytes[n+4] = byte(bits >> 32)
- w.bytes[n+5] = byte(bits >> 40)
+
+ // We over-write, but faster...
+ binary.LittleEndian.PutUint64(w.bytes[n:], bits)
n += 6
+
if n >= bufferFlushSize {
if w.err != nil {
n = 0
@@ -435,6 +434,7 @@ func (w *huffmanBitWriter) writeOutBits() {
w.write(w.bytes[:n])
n = 0
}
+
w.nbytes = n
}
@@ -759,7 +759,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
} else {
// inlined
c := lengths[lengthCode&31]
- w.bits |= uint64(c.code) << (w.nbits & reg16SizeMask64)
+ w.bits |= uint64(c.code) << w.nbits
w.nbits += c.len
if w.nbits >= 48 {
w.writeOutBits()
@@ -779,7 +779,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode)
} else {
// inlined
c := offs[offsetCode&31]
- w.bits |= uint64(c.code) << (w.nbits & reg16SizeMask64)
+ w.bits |= uint64(c.code) << w.nbits
w.nbits += c.len
if w.nbits >= 48 {
w.writeOutBits()
@@ -830,8 +830,8 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
// Assume header is around 70 bytes:
// https://stackoverflow.com/a/25454430
const guessHeaderSizeBits = 70 * 8
- estBits, estExtra := histogramSize(input, w.literalFreq[:], !eof && !sync)
- estBits += w.lastHeader + 15
+ estBits := histogramSize(input, w.literalFreq[:], !eof && !sync)
+ estBits += w.lastHeader + len(input)/32
if w.lastHeader == 0 {
estBits += guessHeaderSizeBits
}
@@ -845,9 +845,9 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
return
}
+ reuseSize := 0
if w.lastHeader > 0 {
- reuseSize := w.literalEncoding.bitLength(w.literalFreq[:256])
- estBits += estExtra
+ reuseSize = w.literalEncoding.bitLength(w.literalFreq[:256])
if estBits < reuseSize {
// We owe an EOB
@@ -859,6 +859,10 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
const numLiterals = endBlockMarker + 1
const numOffsets = 1
if w.lastHeader == 0 {
+ if !eof && !sync {
+ // Generate a slightly suboptimal tree that can be used for all.
+ fillHist(w.literalFreq[:numLiterals])
+ }
w.literalFreq[endBlockMarker] = 1
w.literalEncoding.generate(w.literalFreq[:numLiterals], 15)
@@ -878,19 +882,14 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) {
for _, t := range input {
// Bitwriting inlined, ~30% speedup
c := encoding[t]
- w.bits |= uint64(c.code) << ((w.nbits) & reg16SizeMask64)
+ w.bits |= uint64(c.code) << w.nbits
w.nbits += c.len
if w.nbits >= 48 {
bits := w.bits
w.bits >>= 48
w.nbits -= 48
n := w.nbytes
- w.bytes[n] = byte(bits)
- w.bytes[n+1] = byte(bits >> 8)
- w.bytes[n+2] = byte(bits >> 16)
- w.bytes[n+3] = byte(bits >> 24)
- w.bytes[n+4] = byte(bits >> 32)
- w.bytes[n+5] = byte(bits >> 40)
+ binary.LittleEndian.PutUint64(w.bytes[n:], bits)
n += 6
if n >= bufferFlushSize {
if w.err != nil {
diff --git a/vendor/github.com/klauspost/compress/flate/huffman_code.go b/vendor/github.com/klauspost/compress/flate/huffman_code.go
index 4c39a3018..0d3445a1c 100644
--- a/vendor/github.com/klauspost/compress/flate/huffman_code.go
+++ b/vendor/github.com/klauspost/compress/flate/huffman_code.go
@@ -122,6 +122,16 @@ func (h *huffmanEncoder) bitLength(freq []uint16) int {
return total
}
+func (h *huffmanEncoder) bitLengthRaw(b []byte) int {
+ var total int
+ for _, f := range b {
+ if f != 0 {
+ total += int(h.codes[f].len)
+ }
+ }
+ return total
+}
+
// Return the number of literals assigned to each bit size in the Huffman encoding
//
// This method is only called when list.length >= 3
@@ -327,37 +337,40 @@ func atLeastOne(v float32) float32 {
return v
}
+// Unassigned values are assigned '1' in the histogram.
+func fillHist(b []uint16) {
+ for i, v := range b {
+ if v == 0 {
+ b[i] = 1
+ }
+ }
+}
+
// histogramSize accumulates a histogram of b in h.
// An estimated size in bits is returned.
-// Unassigned values are assigned '1' in the histogram.
// len(h) must be >= 256, and h's elements must be all zeroes.
-func histogramSize(b []byte, h []uint16, fill bool) (int, int) {
+func histogramSize(b []byte, h []uint16, fill bool) (bits int) {
h = h[:256]
for _, t := range b {
h[t]++
}
- invTotal := 1.0 / float32(len(b))
- shannon := float32(0.0)
- var extra float32
+ total := len(b)
if fill {
- oneBits := atLeastOne(-mFastLog2(invTotal))
- for i, v := range h[:] {
- if v > 0 {
- n := float32(v)
- shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
- } else {
- h[i] = 1
- extra += oneBits
+ for _, v := range h {
+ if v == 0 {
+ total++
}
}
- } else {
- for _, v := range h[:] {
- if v > 0 {
- n := float32(v)
- shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
- }
+ }
+
+ invTotal := 1.0 / float32(total)
+ shannon := float32(0.0)
+ for _, v := range h {
+ if v > 0 {
+ n := float32(v)
+ shannon += atLeastOne(-mFastLog2(n*invTotal)) * n
}
}
- return int(shannon + 0.99), int(extra + 0.99)
+ return int(shannon + 0.99)
}
diff --git a/vendor/github.com/klauspost/compress/flate/level2.go b/vendor/github.com/klauspost/compress/flate/level2.go
index 5b986a194..234c4389a 100644
--- a/vendor/github.com/klauspost/compress/flate/level2.go
+++ b/vendor/github.com/klauspost/compress/flate/level2.go
@@ -155,7 +155,7 @@ func (e *fastEncL2) Encode(dst *tokens, src []byte) {
// Store every second hash in-between, but offset by 1.
for i := s - l + 2; i < s-5; i += 7 {
- x := load6432(src, int32(i))
+ x := load6432(src, i)
nextHash := hash4u(uint32(x), bTableBits)
e.table[nextHash] = tableEntry{offset: e.cur + i}
// Skip one
diff --git a/vendor/github.com/klauspost/compress/fse/compress.go b/vendor/github.com/klauspost/compress/fse/compress.go
index b69237c9b..0d31f5ebc 100644
--- a/vendor/github.com/klauspost/compress/fse/compress.go
+++ b/vendor/github.com/klauspost/compress/fse/compress.go
@@ -301,7 +301,7 @@ func (s *Scratch) writeCount() error {
out[outP+1] = byte(bitStream >> 8)
outP += (bitCount + 7) / 8
- if uint16(charnum) > s.symbolLen {
+ if charnum > s.symbolLen {
return errors.New("internal error: charnum > s.symbolLen")
}
s.Out = out[:outP]
@@ -331,7 +331,7 @@ type cTable struct {
func (s *Scratch) allocCtable() {
tableSize := 1 << s.actualTableLog
// get tableSymbol that is big enough.
- if cap(s.ct.tableSymbol) < int(tableSize) {
+ if cap(s.ct.tableSymbol) < tableSize {
s.ct.tableSymbol = make([]byte, tableSize)
}
s.ct.tableSymbol = s.ct.tableSymbol[:tableSize]
@@ -565,8 +565,8 @@ func (s *Scratch) normalizeCount2() error {
distributed uint32
total = uint32(s.br.remain())
tableLog = s.actualTableLog
- lowThreshold = uint32(total >> tableLog)
- lowOne = uint32((total * 3) >> (tableLog + 1))
+ lowThreshold = total >> tableLog
+ lowOne = (total * 3) >> (tableLog + 1)
)
for i, cnt := range s.count[:s.symbolLen] {
if cnt == 0 {
@@ -591,7 +591,7 @@ func (s *Scratch) normalizeCount2() error {
if (total / toDistribute) > lowOne {
// risk of rounding to zero
- lowOne = uint32((total * 3) / (toDistribute * 2))
+ lowOne = (total * 3) / (toDistribute * 2)
for i, cnt := range s.count[:s.symbolLen] {
if (s.norm[i] == notYetAssigned) && (cnt <= lowOne) {
s.norm[i] = 1
diff --git a/vendor/github.com/klauspost/compress/fse/decompress.go b/vendor/github.com/klauspost/compress/fse/decompress.go
index 413ec3b3c..926f5f153 100644
--- a/vendor/github.com/klauspost/compress/fse/decompress.go
+++ b/vendor/github.com/klauspost/compress/fse/decompress.go
@@ -172,7 +172,7 @@ type decSymbol struct {
// allocDtable will allocate decoding tables if they are not big enough.
func (s *Scratch) allocDtable() {
tableSize := 1 << s.actualTableLog
- if cap(s.decTable) < int(tableSize) {
+ if cap(s.decTable) < tableSize {
s.decTable = make([]decSymbol, tableSize)
}
s.decTable = s.decTable[:tableSize]
@@ -340,7 +340,7 @@ type decoder struct {
func (d *decoder) init(in *bitReader, dt []decSymbol, tableLog uint8) {
d.dt = dt
d.br = in
- d.state = uint16(in.getBits(tableLog))
+ d.state = in.getBits(tableLog)
}
// next returns the next symbol and sets the next state.
diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go
index f9ed5f830..dea115b33 100644
--- a/vendor/github.com/klauspost/compress/huff0/compress.go
+++ b/vendor/github.com/klauspost/compress/huff0/compress.go
@@ -403,7 +403,7 @@ func (s *Scratch) buildCTable() error {
var startNode = int16(s.symbolLen)
nonNullRank := s.symbolLen - 1
- nodeNb := int16(startNode)
+ nodeNb := startNode
huffNode := s.nodes[1 : huffNodesLen+1]
// This overlays the slice above, but allows "-1" index lookups.
@@ -580,7 +580,7 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 {
// Get pos of last (smallest) symbol per rank
{
- currentNbBits := uint8(maxNbBits)
+ currentNbBits := maxNbBits
for pos := int(n); pos >= 0; pos-- {
if huffNode[pos].nbBits >= currentNbBits {
continue
diff --git a/vendor/github.com/klauspost/compress/snappy/snappy.go b/vendor/github.com/klauspost/compress/snappy/snappy.go
index 74a36689e..ea58ced88 100644
--- a/vendor/github.com/klauspost/compress/snappy/snappy.go
+++ b/vendor/github.com/klauspost/compress/snappy/snappy.go
@@ -94,5 +94,5 @@ var crcTable = crc32.MakeTable(crc32.Castagnoli)
// https://github.com/google/snappy/blob/master/framing_format.txt
func crc(b []byte) uint32 {
c := crc32.Update(0, crcTable, b)
- return uint32(c>>15|c<<17) + 0xa282ead8
+ return c>>15 | c<<17 + 0xa282ead8
}
diff --git a/vendor/github.com/klauspost/compress/zstd/blockenc.go b/vendor/github.com/klauspost/compress/zstd/blockenc.go
index c85c40255..9647c64e5 100644
--- a/vendor/github.com/klauspost/compress/zstd/blockenc.go
+++ b/vendor/github.com/klauspost/compress/zstd/blockenc.go
@@ -22,28 +22,44 @@ type blockEnc struct {
dictLitEnc *huff0.Scratch
wr bitWriter
- extraLits int
- last bool
-
+ extraLits int
output []byte
recentOffsets [3]uint32
prevRecentOffsets [3]uint32
+
+ last bool
+ lowMem bool
}
// init should be used once the block has been created.
// If called more than once, the effect is the same as calling reset.
func (b *blockEnc) init() {
- if cap(b.literals) < maxCompressedLiteralSize {
- b.literals = make([]byte, 0, maxCompressedLiteralSize)
- }
- const defSeqs = 200
- b.literals = b.literals[:0]
- if cap(b.sequences) < defSeqs {
- b.sequences = make([]seq, 0, defSeqs)
- }
- if cap(b.output) < maxCompressedBlockSize {
- b.output = make([]byte, 0, maxCompressedBlockSize)
+ if b.lowMem {
+ // 1K literals
+ if cap(b.literals) < 1<<10 {
+ b.literals = make([]byte, 0, 1<<10)
+ }
+ const defSeqs = 20
+ if cap(b.sequences) < defSeqs {
+ b.sequences = make([]seq, 0, defSeqs)
+ }
+ // 1K
+ if cap(b.output) < 1<<10 {
+ b.output = make([]byte, 0, 1<<10)
+ }
+ } else {
+ if cap(b.literals) < maxCompressedBlockSize {
+ b.literals = make([]byte, 0, maxCompressedBlockSize)
+ }
+ const defSeqs = 200
+ if cap(b.sequences) < defSeqs {
+ b.sequences = make([]seq, 0, defSeqs)
+ }
+ if cap(b.output) < maxCompressedBlockSize {
+ b.output = make([]byte, 0, maxCompressedBlockSize)
+ }
}
+
if b.coders.mlEnc == nil {
b.coders.mlEnc = &fseEncoder{}
b.coders.mlPrev = &fseEncoder{}
diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go
index b1b7c6e6a..2d4d893eb 100644
--- a/vendor/github.com/klauspost/compress/zstd/enc_base.go
+++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go
@@ -7,6 +7,10 @@ import (
"github.com/klauspost/compress/zstd/internal/xxhash"
)
+const (
+ dictShardBits = 6
+)
+
type fastBase struct {
// cur is the offset at the start of hist
cur int32
@@ -17,6 +21,7 @@ type fastBase struct {
tmp [8]byte
blk *blockEnc
lastDictID uint32
+ lowMem bool
}
// CRC returns the underlying CRC writer.
@@ -57,15 +62,10 @@ func (e *fastBase) addBlock(src []byte) int32 {
// check if we have space already
if len(e.hist)+len(src) > cap(e.hist) {
if cap(e.hist) == 0 {
- l := e.maxMatchOff * 2
- // Make it at least 1MB.
- if l < 1<<20 {
- l = 1 << 20
- }
- e.hist = make([]byte, 0, l)
+ e.ensureHist(len(src))
} else {
- if cap(e.hist) < int(e.maxMatchOff*2) {
- panic("unexpected buffer size")
+ if cap(e.hist) < int(e.maxMatchOff+maxCompressedBlockSize) {
+ panic(fmt.Errorf("unexpected buffer cap %d, want at least %d with window %d", cap(e.hist), e.maxMatchOff+maxCompressedBlockSize, e.maxMatchOff))
}
// Move down
offset := int32(len(e.hist)) - e.maxMatchOff
@@ -79,6 +79,28 @@ func (e *fastBase) addBlock(src []byte) int32 {
return s
}
+// ensureHist will ensure that history can keep at least this many bytes.
+func (e *fastBase) ensureHist(n int) {
+ if cap(e.hist) >= n {
+ return
+ }
+ l := e.maxMatchOff
+ if (e.lowMem && e.maxMatchOff > maxCompressedBlockSize) || e.maxMatchOff <= maxCompressedBlockSize {
+ l += maxCompressedBlockSize
+ } else {
+ l += e.maxMatchOff
+ }
+ // Make it at least 1MB.
+ if l < 1<<20 && !e.lowMem {
+ l = 1 << 20
+ }
+ // Make it at least the requested size.
+ if l < int32(n) {
+ l = int32(n)
+ }
+ e.hist = make([]byte, 0, l)
+}
+
// useBlock will replace the block with the provided one,
// but transfer recent offsets from the previous.
func (e *fastBase) UseBlock(enc *blockEnc) {
@@ -117,7 +139,7 @@ func (e *fastBase) matchlen(s, t int32, src []byte) int32 {
// Reset the encoding table.
func (e *fastBase) resetBase(d *dict, singleBlock bool) {
if e.blk == nil {
- e.blk = &blockEnc{}
+ e.blk = &blockEnc{lowMem: e.lowMem}
e.blk.init()
} else {
e.blk.reset(nil)
diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go
index bb71d1eea..fe3625c5f 100644
--- a/vendor/github.com/klauspost/compress/zstd/enc_best.go
+++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go
@@ -407,6 +407,7 @@ encodeLoop:
// Most notable difference is that src will not be copied for history and
// we do not need to check for max match length.
func (e *bestFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) {
+ e.ensureHist(len(src))
e.Encode(blk, src)
}
diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go
index 94a5343d0..c2ce4a2ba 100644
--- a/vendor/github.com/klauspost/compress/zstd/enc_better.go
+++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go
@@ -16,6 +16,12 @@ const (
// This greatly depends on the type of input.
betterShortTableBits = 13 // Bits used in the short match table
betterShortTableSize = 1 << betterShortTableBits // Size of the table
+
+ betterLongTableShardCnt = 1 << (betterLongTableBits - dictShardBits) // Number of shards in the table
+ betterLongTableShardSize = betterLongTableSize / betterLongTableShardCnt // Size of an individual shard
+
+ betterShortTableShardCnt = 1 << (betterShortTableBits - dictShardBits) // Number of shards in the table
+ betterShortTableShardSize = betterShortTableSize / betterShortTableShardCnt // Size of an individual shard
)
type prevEntry struct {
@@ -31,10 +37,17 @@ type prevEntry struct {
// and that it is longer (lazy matching).
type betterFastEncoder struct {
fastBase
- table [betterShortTableSize]tableEntry
- longTable [betterLongTableSize]prevEntry
- dictTable []tableEntry
- dictLongTable []prevEntry
+ table [betterShortTableSize]tableEntry
+ longTable [betterLongTableSize]prevEntry
+}
+
+type betterFastEncoderDict struct {
+ betterFastEncoder
+ dictTable []tableEntry
+ dictLongTable []prevEntry
+ shortTableShardDirty [betterShortTableShardCnt]bool
+ longTableShardDirty [betterLongTableShardCnt]bool
+ allDirty bool
}
// Encode improves compression...
@@ -516,12 +529,512 @@ encodeLoop:
// Most notable difference is that src will not be copied for history and
// we do not need to check for max match length.
func (e *betterFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) {
+ e.ensureHist(len(src))
e.Encode(blk, src)
}
+// Encode improves compression...
+func (e *betterFastEncoderDict) Encode(blk *blockEnc, src []byte) {
+ const (
+ // Input margin is the number of bytes we read (8)
+ // and the maximum we will read ahead (2)
+ inputMargin = 8 + 2
+ minNonLiteralBlockSize = 16
+ )
+
+ // Protect against e.cur wraparound.
+ for e.cur >= bufferReset {
+ if len(e.hist) == 0 {
+ for i := range e.table[:] {
+ e.table[i] = tableEntry{}
+ }
+ for i := range e.longTable[:] {
+ e.longTable[i] = prevEntry{}
+ }
+ e.cur = e.maxMatchOff
+ e.allDirty = true
+ break
+ }
+ // Shift down everything in the table that isn't already too far away.
+ minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff
+ for i := range e.table[:] {
+ v := e.table[i].offset
+ if v < minOff {
+ v = 0
+ } else {
+ v = v - e.cur + e.maxMatchOff
+ }
+ e.table[i].offset = v
+ }
+ for i := range e.longTable[:] {
+ v := e.longTable[i].offset
+ v2 := e.longTable[i].prev
+ if v < minOff {
+ v = 0
+ v2 = 0
+ } else {
+ v = v - e.cur + e.maxMatchOff
+ if v2 < minOff {
+ v2 = 0
+ } else {
+ v2 = v2 - e.cur + e.maxMatchOff
+ }
+ }
+ e.longTable[i] = prevEntry{
+ offset: v,
+ prev: v2,
+ }
+ }
+ e.allDirty = true
+ e.cur = e.maxMatchOff
+ break
+ }
+
+ s := e.addBlock(src)
+ blk.size = len(src)
+ if len(src) < minNonLiteralBlockSize {
+ blk.extraLits = len(src)
+ blk.literals = blk.literals[:len(src)]
+ copy(blk.literals, src)
+ return
+ }
+
+ // Override src
+ src = e.hist
+ sLimit := int32(len(src)) - inputMargin
+ // stepSize is the number of bytes to skip on every main loop iteration.
+ // It should be >= 1.
+ const stepSize = 1
+
+ const kSearchStrength = 9
+
+ // nextEmit is where in src the next emitLiteral should start from.
+ nextEmit := s
+ cv := load6432(src, s)
+
+ // Relative offsets
+ offset1 := int32(blk.recentOffsets[0])
+ offset2 := int32(blk.recentOffsets[1])
+
+ addLiterals := func(s *seq, until int32) {
+ if until == nextEmit {
+ return
+ }
+ blk.literals = append(blk.literals, src[nextEmit:until]...)
+ s.litLen = uint32(until - nextEmit)
+ }
+ if debug {
+ println("recent offsets:", blk.recentOffsets)
+ }
+
+encodeLoop:
+ for {
+ var t int32
+ // We allow the encoder to optionally turn off repeat offsets across blocks
+ canRepeat := len(blk.sequences) > 2
+ var matched int32
+
+ for {
+ if debugAsserts && canRepeat && offset1 == 0 {
+ panic("offset0 was 0")
+ }
+
+ nextHashS := hash5(cv, betterShortTableBits)
+ nextHashL := hash8(cv, betterLongTableBits)
+ candidateL := e.longTable[nextHashL]
+ candidateS := e.table[nextHashS]
+
+ const repOff = 1
+ repIndex := s - offset1 + repOff
+ off := s + e.cur
+ e.longTable[nextHashL] = prevEntry{offset: off, prev: candidateL.offset}
+ e.markLongShardDirty(nextHashL)
+ e.table[nextHashS] = tableEntry{offset: off, val: uint32(cv)}
+ e.markShortShardDirty(nextHashS)
+
+ if canRepeat {
+ if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) {
+ // Consider history as well.
+ var seq seq
+ lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src)
+
+ seq.matchLen = uint32(lenght - zstdMinMatch)
+
+ // We might be able to match backwards.
+ // Extend as long as we can.
+ start := s + repOff
+ // We end the search early, so we don't risk 0 literals
+ // and have to do special offset treatment.
+ startLimit := nextEmit + 1
+
+ tMin := s - e.maxMatchOff
+ if tMin < 0 {
+ tMin = 0
+ }
+ for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 {
+ repIndex--
+ start--
+ seq.matchLen++
+ }
+ addLiterals(&seq, start)
+
+ // rep 0
+ seq.offset = 1
+ if debugSequences {
+ println("repeat sequence", seq, "next s:", s)
+ }
+ blk.sequences = append(blk.sequences, seq)
+
+ // Index match start+1 (long) -> s - 1
+ index0 := s + repOff
+ s += lenght + repOff
+
+ nextEmit = s
+ if s >= sLimit {
+ if debug {
+ println("repeat ended", s, lenght)
+
+ }
+ break encodeLoop
+ }
+ // Index skipped...
+ for index0 < s-1 {
+ cv0 := load6432(src, index0)
+ cv1 := cv0 >> 8
+ h0 := hash8(cv0, betterLongTableBits)
+ off := index0 + e.cur
+ e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset}
+ e.markLongShardDirty(h0)
+ h1 := hash5(cv1, betterShortTableBits)
+ e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)}
+ e.markShortShardDirty(h1)
+ index0 += 2
+ }
+ cv = load6432(src, s)
+ continue
+ }
+ const repOff2 = 1
+
+ // We deviate from the reference encoder and also check offset 2.
+ // Still slower and not much better, so disabled.
+ // repIndex = s - offset2 + repOff2
+ if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) {
+ // Consider history as well.
+ var seq seq
+ lenght := 8 + e.matchlen(s+8+repOff2, repIndex+8, src)
+
+ seq.matchLen = uint32(lenght - zstdMinMatch)
+
+ // We might be able to match backwards.
+ // Extend as long as we can.
+ start := s + repOff2
+ // We end the search early, so we don't risk 0 literals
+ // and have to do special offset treatment.
+ startLimit := nextEmit + 1
+
+ tMin := s - e.maxMatchOff
+ if tMin < 0 {
+ tMin = 0
+ }
+ for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 {
+ repIndex--
+ start--
+ seq.matchLen++
+ }
+ addLiterals(&seq, start)
+
+ // rep 2
+ seq.offset = 2
+ if debugSequences {
+ println("repeat sequence 2", seq, "next s:", s)
+ }
+ blk.sequences = append(blk.sequences, seq)
+
+ index0 := s + repOff2
+ s += lenght + repOff2
+ nextEmit = s
+ if s >= sLimit {
+ if debug {
+ println("repeat ended", s, lenght)
+
+ }
+ break encodeLoop
+ }
+
+ // Index skipped...
+ for index0 < s-1 {
+ cv0 := load6432(src, index0)
+ cv1 := cv0 >> 8
+ h0 := hash8(cv0, betterLongTableBits)
+ off := index0 + e.cur
+ e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset}
+ e.markLongShardDirty(h0)
+ h1 := hash5(cv1, betterShortTableBits)
+ e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)}
+ e.markShortShardDirty(h1)
+ index0 += 2
+ }
+ cv = load6432(src, s)
+ // Swap offsets
+ offset1, offset2 = offset2, offset1
+ continue
+ }
+ }
+ // Find the offsets of our two matches.
+ coffsetL := candidateL.offset - e.cur
+ coffsetLP := candidateL.prev - e.cur
+
+ // Check if we have a long match.
+ if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) {
+ // Found a long match, at least 8 bytes.
+ matched = e.matchlen(s+8, coffsetL+8, src) + 8
+ t = coffsetL
+ if debugAsserts && s <= t {
+ panic(fmt.Sprintf("s (%d) <= t (%d)", s, t))
+ }
+ if debugAsserts && s-t > e.maxMatchOff {
+ panic("s - t >e.maxMatchOff")
+ }
+ if debugMatches {
+ println("long match")
+ }
+
+ if s-coffsetLP < e.maxMatchOff && cv == load6432(src, coffsetLP) {
+ // Found a long match, at least 8 bytes.
+ prevMatch := e.matchlen(s+8, coffsetLP+8, src) + 8
+ if prevMatch > matched {
+ matched = prevMatch
+ t = coffsetLP
+ }
+ if debugAsserts && s <= t {
+ panic(fmt.Sprintf("s (%d) <= t (%d)", s, t))
+ }
+ if debugAsserts && s-t > e.maxMatchOff {
+ panic("s - t >e.maxMatchOff")
+ }
+ if debugMatches {
+ println("long match")
+ }
+ }
+ break
+ }
+
+ // Check if we have a long match on prev.
+ if s-coffsetLP < e.maxMatchOff && cv == load6432(src, coffsetLP) {
+ // Found a long match, at least 8 bytes.
+ matched = e.matchlen(s+8, coffsetLP+8, src) + 8
+ t = coffsetLP
+ if debugAsserts && s <= t {
+ panic(fmt.Sprintf("s (%d) <= t (%d)", s, t))
+ }
+ if debugAsserts && s-t > e.maxMatchOff {
+ panic("s - t >e.maxMatchOff")
+ }
+ if debugMatches {
+ println("long match")
+ }
+ break
+ }
+
+ coffsetS := candidateS.offset - e.cur
+
+ // Check if we have a short match.
+ if s-coffsetS < e.maxMatchOff && uint32(cv) == candidateS.val {
+ // found a regular match
+ matched = e.matchlen(s+4, coffsetS+4, src) + 4
+
+ // See if we can find a long match at s+1
+ const checkAt = 1
+ cv := load6432(src, s+checkAt)
+ nextHashL = hash8(cv, betterLongTableBits)
+ candidateL = e.longTable[nextHashL]
+ coffsetL = candidateL.offset - e.cur
+
+ // We can store it, since we have at least a 4 byte match.
+ e.longTable[nextHashL] = prevEntry{offset: s + checkAt + e.cur, prev: candidateL.offset}
+ e.markLongShardDirty(nextHashL)
+ if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) {
+ // Found a long match, at least 8 bytes.
+ matchedNext := e.matchlen(s+8+checkAt, coffsetL+8, src) + 8
+ if matchedNext > matched {
+ t = coffsetL
+ s += checkAt
+ matched = matchedNext
+ if debugMatches {
+ println("long match (after short)")
+ }
+ break
+ }
+ }
+
+ // Check prev long...
+ coffsetL = candidateL.prev - e.cur
+ if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) {
+ // Found a long match, at least 8 bytes.
+ matchedNext := e.matchlen(s+8+checkAt, coffsetL+8, src) + 8
+ if matchedNext > matched {
+ t = coffsetL
+ s += checkAt
+ matched = matchedNext
+ if debugMatches {
+ println("prev long match (after short)")
+ }
+ break
+ }
+ }
+ t = coffsetS
+ if debugAsserts && s <= t {
+ panic(fmt.Sprintf("s (%d) <= t (%d)", s, t))
+ }
+ if debugAsserts && s-t > e.maxMatchOff {
+ panic("s - t >e.maxMatchOff")
+ }
+ if debugAsserts && t < 0 {
+ panic("t<0")
+ }
+ if debugMatches {
+ println("short match")
+ }
+ break
+ }
+
+ // No match found, move forward in input.
+ s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1))
+ if s >= sLimit {
+ break encodeLoop
+ }
+ cv = load6432(src, s)
+ }
+
+ // A 4-byte match has been found. Update recent offsets.
+ // We'll later see if more than 4 bytes.
+ offset2 = offset1
+ offset1 = s - t
+
+ if debugAsserts && s <= t {
+ panic(fmt.Sprintf("s (%d) <= t (%d)", s, t))
+ }
+
+ if debugAsserts && canRepeat && int(offset1) > len(src) {
+ panic("invalid offset")
+ }
+
+ // Extend the n-byte match as long as possible.
+ l := matched
+
+ // Extend backwards
+ tMin := s - e.maxMatchOff
+ if tMin < 0 {
+ tMin = 0
+ }
+ for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength {
+ s--
+ t--
+ l++
+ }
+
+ // Write our sequence
+ var seq seq
+ seq.litLen = uint32(s - nextEmit)
+ seq.matchLen = uint32(l - zstdMinMatch)
+ if seq.litLen > 0 {
+ blk.literals = append(blk.literals, src[nextEmit:s]...)
+ }
+ seq.offset = uint32(s-t) + 3
+ s += l
+ if debugSequences {
+ println("sequence", seq, "next s:", s)
+ }
+ blk.sequences = append(blk.sequences, seq)
+ nextEmit = s
+ if s >= sLimit {
+ break encodeLoop
+ }
+
+ // Index match start+1 (long) -> s - 1
+ index0 := s - l + 1
+ for index0 < s-1 {
+ cv0 := load6432(src, index0)
+ cv1 := cv0 >> 8
+ h0 := hash8(cv0, betterLongTableBits)
+ off := index0 + e.cur
+ e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset}
+ e.markLongShardDirty(h0)
+ h1 := hash5(cv1, betterShortTableBits)
+ e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)}
+ e.markShortShardDirty(h1)
+ index0 += 2
+ }
+
+ cv = load6432(src, s)
+ if !canRepeat {
+ continue
+ }
+
+ // Check offset 2
+ for {
+ o2 := s - offset2
+ if load3232(src, o2) != uint32(cv) {
+ // Do regular search
+ break
+ }
+
+ // Store this, since we have it.
+ nextHashS := hash5(cv, betterShortTableBits)
+ nextHashL := hash8(cv, betterLongTableBits)
+
+ // We have at least 4 byte match.
+ // No need to check backwards. We come straight from a match
+ l := 4 + e.matchlen(s+4, o2+4, src)
+
+ e.longTable[nextHashL] = prevEntry{offset: s + e.cur, prev: e.longTable[nextHashL].offset}
+ e.markLongShardDirty(nextHashL)
+ e.table[nextHashS] = tableEntry{offset: s + e.cur, val: uint32(cv)}
+ e.markShortShardDirty(nextHashS)
+ seq.matchLen = uint32(l) - zstdMinMatch
+ seq.litLen = 0
+
+ // Since litlen is always 0, this is offset 1.
+ seq.offset = 1
+ s += l
+ nextEmit = s
+ if debugSequences {
+ println("sequence", seq, "next s:", s)
+ }
+ blk.sequences = append(blk.sequences, seq)
+
+ // Swap offset 1 and 2.
+ offset1, offset2 = offset2, offset1
+ if s >= sLimit {
+ // Finished
+ break encodeLoop
+ }
+ cv = load6432(src, s)
+ }
+ }
+
+ if int(nextEmit) < len(src) {
+ blk.literals = append(blk.literals, src[nextEmit:]...)
+ blk.extraLits = len(src) - int(nextEmit)
+ }
+ blk.recentOffsets[0] = uint32(offset1)
+ blk.recentOffsets[1] = uint32(offset2)
+ if debug {
+ println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits)
+ }
+}
+
// ResetDict will reset and set a dictionary if not nil
func (e *betterFastEncoder) Reset(d *dict, singleBlock bool) {
e.resetBase(d, singleBlock)
+ if d != nil {
+ panic("betterFastEncoder: Reset with dict")
+ }
+}
+
+// ResetDict will reset and set a dictionary if not nil
+func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) {
+ e.resetBase(d, singleBlock)
if d == nil {
return
}
@@ -557,6 +1070,7 @@ func (e *betterFastEncoder) Reset(d *dict, singleBlock bool) {
}
}
e.lastDictID = d.id
+ e.allDirty = true
}
// Init or copy dict table
@@ -585,11 +1099,72 @@ func (e *betterFastEncoder) Reset(d *dict, singleBlock bool) {
}
}
e.lastDictID = d.id
+ e.allDirty = true
}
+
// Reset table to initial state
- copy(e.longTable[:], e.dictLongTable)
+ {
+ dirtyShardCnt := 0
+ if !e.allDirty {
+ for i := range e.shortTableShardDirty {
+ if e.shortTableShardDirty[i] {
+ dirtyShardCnt++
+ }
+ }
+ }
+ const shardCnt = betterShortTableShardCnt
+ const shardSize = betterShortTableShardSize
+ if e.allDirty || dirtyShardCnt > shardCnt*4/6 {
+ copy(e.table[:], e.dictTable)
+ for i := range e.shortTableShardDirty {
+ e.shortTableShardDirty[i] = false
+ }
+ } else {
+ for i := range e.shortTableShardDirty {
+ if !e.shortTableShardDirty[i] {
+ continue
+ }
+ copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize])
+ e.shortTableShardDirty[i] = false
+ }
+ }
+ }
+ {
+ dirtyShardCnt := 0
+ if !e.allDirty {
+ for i := range e.shortTableShardDirty {
+ if e.shortTableShardDirty[i] {
+ dirtyShardCnt++
+ }
+ }
+ }
+ const shardCnt = betterLongTableShardCnt
+ const shardSize = betterLongTableShardSize
+ if e.allDirty || dirtyShardCnt > shardCnt*4/6 {
+ copy(e.longTable[:], e.dictLongTable)
+ for i := range e.longTableShardDirty {
+ e.longTableShardDirty[i] = false
+ }
+ } else {
+ for i := range e.longTableShardDirty {
+ if !e.longTableShardDirty[i] {
+ continue
+ }
+
+ copy(e.longTable[i*shardSize:(i+1)*shardSize], e.dictLongTable[i*shardSize:(i+1)*shardSize])
+ e.longTableShardDirty[i] = false
+ }
+ }
+ }
e.cur = e.maxMatchOff
- // Reset table to initial state
- copy(e.table[:], e.dictTable)
+ e.allDirty = false
+}
+
+func (e *betterFastEncoderDict) markLongShardDirty(entryNum uint32) {
+ e.longTableShardDirty[entryNum/betterLongTableShardSize] = true
+}
+
+func (e *betterFastEncoderDict) markShortShardDirty(entryNum uint32) {
+ e.shortTableShardDirty[entryNum/betterShortTableShardSize] = true
}
diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go
index 19eebf66e..8629d43d8 100644
--- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go
+++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go
@@ -11,6 +11,9 @@ const (
dFastLongTableSize = 1 << dFastLongTableBits // Size of the table
dFastLongTableMask = dFastLongTableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks.
+ dLongTableShardCnt = 1 << (dFastLongTableBits - dictShardBits) // Number of shards in the table
+ dLongTableShardSize = dFastLongTableSize / tableShardCnt // Size of an individual shard
+
dFastShortTableBits = tableBits // Bits used in the short match table
dFastShortTableSize = 1 << dFastShortTableBits // Size of the table
dFastShortTableMask = dFastShortTableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks.
@@ -18,8 +21,14 @@ const (
type doubleFastEncoder struct {
fastEncoder
- longTable [dFastLongTableSize]tableEntry
- dictLongTable []tableEntry
+ longTable [dFastLongTableSize]tableEntry
+}
+
+type doubleFastEncoderDict struct {
+ fastEncoderDict
+ longTable [dFastLongTableSize]tableEntry
+ dictLongTable []tableEntry
+ longTableShardDirty [dLongTableShardCnt]bool
}
// Encode mimmics functionality in zstd_dfast.c
@@ -678,9 +687,379 @@ encodeLoop:
}
}
+// Encode will encode the content, with a dictionary if initialized for it.
+func (e *doubleFastEncoderDict) Encode(blk *blockEnc, src []byte) {
+ const (
+ // Input margin is the number of bytes we read (8)
+ // and the maximum we will read ahead (2)
+ inputMargin = 8 + 2
+ minNonLiteralBlockSize = 16
+ )
+
+ // Protect against e.cur wraparound.
+ for e.cur >= bufferReset {
+ if len(e.hist) == 0 {
+ for i := range e.table[:] {
+ e.table[i] = tableEntry{}
+ }
+ for i := range e.longTable[:] {
+ e.longTable[i] = tableEntry{}
+ }
+ e.markAllShardsDirty()
+ e.cur = e.maxMatchOff
+ break
+ }
+ // Shift down everything in the table that isn't already too far away.
+ minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff
+ for i := range e.table[:] {
+ v := e.table[i].offset
+ if v < minOff {
+ v = 0
+ } else {
+ v = v - e.cur + e.maxMatchOff
+ }
+ e.table[i].offset = v
+ }
+ for i := range e.longTable[:] {
+ v := e.longTable[i].offset
+ if v < minOff {
+ v = 0
+ } else {
+ v = v - e.cur + e.maxMatchOff
+ }
+ e.longTable[i].offset = v
+ }
+ e.markAllShardsDirty()
+ e.cur = e.maxMatchOff
+ break
+ }
+
+ s := e.addBlock(src)
+ blk.size = len(src)
+ if len(src) < minNonLiteralBlockSize {
+ blk.extraLits = len(src)
+ blk.literals = blk.literals[:len(src)]
+ copy(blk.literals, src)
+ return
+ }
+
+ // Override src
+ src = e.hist
+ sLimit := int32(len(src)) - inputMargin
+ // stepSize is the number of bytes to skip on every main loop iteration.
+ // It should be >= 1.
+ const stepSize = 1
+
+ const kSearchStrength = 8
+
+ // nextEmit is where in src the next emitLiteral should start from.
+ nextEmit := s
+ cv := load6432(src, s)
+
+ // Relative offsets
+ offset1 := int32(blk.recentOffsets[0])
+ offset2 := int32(blk.recentOffsets[1])
+
+ addLiterals := func(s *seq, until int32) {
+ if until == nextEmit {
+ return
+ }
+ blk.literals = append(blk.literals, src[nextEmit:until]...)
+ s.litLen = uint32(until - nextEmit)
+ }
+ if debug {
+ println("recent offsets:", blk.recentOffsets)
+ }
+
+encodeLoop:
+ for {
+ var t int32
+ // We allow the encoder to optionally turn off repeat offsets across blocks
+ canRepeat := len(blk.sequences) > 2
+
+ for {
+ if debugAsserts && canRepeat && offset1 == 0 {
+ panic("offset0 was 0")
+ }
+
+ nextHashS := hash5(cv, dFastShortTableBits)
+ nextHashL := hash8(cv, dFastLongTableBits)
+ candidateL := e.longTable[nextHashL]
+ candidateS := e.table[nextHashS]
+
+ const repOff = 1
+ repIndex := s - offset1 + repOff
+ entry := tableEntry{offset: s + e.cur, val: uint32(cv)}
+ e.longTable[nextHashL] = entry
+ e.markLongShardDirty(nextHashL)
+ e.table[nextHashS] = entry
+ e.markShardDirty(nextHashS)
+
+ if canRepeat {
+ if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) {
+ // Consider history as well.
+ var seq seq
+ lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src)
+
+ seq.matchLen = uint32(lenght - zstdMinMatch)
+
+ // We might be able to match backwards.
+ // Extend as long as we can.
+ start := s + repOff
+ // We end the search early, so we don't risk 0 literals
+ // and have to do special offset treatment.
+ startLimit := nextEmit + 1
+
+ tMin := s - e.maxMatchOff
+ if tMin < 0 {
+ tMin = 0
+ }
+ for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 {
+ repIndex--
+ start--
+ seq.matchLen++
+ }
+ addLiterals(&seq, start)
+
+ // rep 0
+ seq.offset = 1
+ if debugSequences {
+ println("repeat sequence", seq, "next s:", s)
+ }
+ blk.sequences = append(blk.sequences, seq)
+ s += lenght + repOff
+ nextEmit = s
+ if s >= sLimit {
+ if debug {
+ println("repeat ended", s, lenght)
+
+ }
+ break encodeLoop
+ }
+ cv = load6432(src, s)
+ continue
+ }
+ }
+ // Find the offsets of our two matches.
+ coffsetL := s - (candidateL.offset - e.cur)
+ coffsetS := s - (candidateS.offset - e.cur)
+
+ // Check if we have a long match.
+ if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val {
+ // Found a long match, likely at least 8 bytes.
+ // Reference encoder checks all 8 bytes, we only check 4,
+ // but the likelihood of both the first 4 bytes and the hash matching should be enough.
+ t = candidateL.offset - e.cur
+ if debugAsserts && s <= t {
+ panic(fmt.Sprintf("s (%d) <= t (%d)", s, t))
+ }
+ if debugAsserts && s-t > e.maxMatchOff {
+ panic("s - t >e.maxMatchOff")
+ }
+ if debugMatches {
+ println("long match")
+ }
+ break
+ }
+
+ // Check if we have a short match.
+ if coffsetS < e.maxMatchOff && uint32(cv) == candidateS.val {
+ // found a regular match
+ // See if we can find a long match at s+1
+ const checkAt = 1
+ cv := load6432(src, s+checkAt)
+ nextHashL = hash8(cv, dFastLongTableBits)
+ candidateL = e.longTable[nextHashL]
+ coffsetL = s - (candidateL.offset - e.cur) + checkAt
+
+ // We can store it, since we have at least a 4 byte match.
+ e.longTable[nextHashL] = tableEntry{offset: s + checkAt + e.cur, val: uint32(cv)}
+ e.markLongShardDirty(nextHashL)
+ if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val {
+ // Found a long match, likely at least 8 bytes.
+ // Reference encoder checks all 8 bytes, we only check 4,
+ // but the likelihood of both the first 4 bytes and the hash matching should be enough.
+ t = candidateL.offset - e.cur
+ s += checkAt
+ if debugMatches {
+ println("long match (after short)")
+ }
+ break
+ }
+
+ t = candidateS.offset - e.cur
+ if debugAsserts && s <= t {
+ panic(fmt.Sprintf("s (%d) <= t (%d)", s, t))
+ }
+ if debugAsserts && s-t > e.maxMatchOff {
+ panic("s - t >e.maxMatchOff")
+ }
+ if debugAsserts && t < 0 {
+ panic("t<0")
+ }
+ if debugMatches {
+ println("short match")
+ }
+ break
+ }
+
+ // No match found, move forward in input.
+ s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1))
+ if s >= sLimit {
+ break encodeLoop
+ }
+ cv = load6432(src, s)
+ }
+
+ // A 4-byte match has been found. Update recent offsets.
+ // We'll later see if more than 4 bytes.
+ offset2 = offset1
+ offset1 = s - t
+
+ if debugAsserts && s <= t {
+ panic(fmt.Sprintf("s (%d) <= t (%d)", s, t))
+ }
+
+ if debugAsserts && canRepeat && int(offset1) > len(src) {
+ panic("invalid offset")
+ }
+
+ // Extend the 4-byte match as long as possible.
+ l := e.matchlen(s+4, t+4, src) + 4
+
+ // Extend backwards
+ tMin := s - e.maxMatchOff
+ if tMin < 0 {
+ tMin = 0
+ }
+ for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength {
+ s--
+ t--
+ l++
+ }
+
+ // Write our sequence
+ var seq seq
+ seq.litLen = uint32(s - nextEmit)
+ seq.matchLen = uint32(l - zstdMinMatch)
+ if seq.litLen > 0 {
+ blk.literals = append(blk.literals, src[nextEmit:s]...)
+ }
+ seq.offset = uint32(s-t) + 3
+ s += l
+ if debugSequences {
+ println("sequence", seq, "next s:", s)
+ }
+ blk.sequences = append(blk.sequences, seq)
+ nextEmit = s
+ if s >= sLimit {
+ break encodeLoop
+ }
+
+ // Index match start+1 (long) and start+2 (short)
+ index0 := s - l + 1
+ // Index match end-2 (long) and end-1 (short)
+ index1 := s - 2
+
+ cv0 := load6432(src, index0)
+ cv1 := load6432(src, index1)
+ te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)}
+ te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)}
+ longHash1 := hash8(cv0, dFastLongTableBits)
+ longHash2 := hash8(cv0, dFastLongTableBits)
+ e.longTable[longHash1] = te0
+ e.longTable[longHash2] = te1
+ e.markLongShardDirty(longHash1)
+ e.markLongShardDirty(longHash2)
+ cv0 >>= 8
+ cv1 >>= 8
+ te0.offset++
+ te1.offset++
+ te0.val = uint32(cv0)
+ te1.val = uint32(cv1)
+ hashVal1 := hash5(cv0, dFastShortTableBits)
+ hashVal2 := hash5(cv1, dFastShortTableBits)
+ e.table[hashVal1] = te0
+ e.markShardDirty(hashVal1)
+ e.table[hashVal2] = te1
+ e.markShardDirty(hashVal2)
+
+ cv = load6432(src, s)
+
+ if !canRepeat {
+ continue
+ }
+
+ // Check offset 2
+ for {
+ o2 := s - offset2
+ if load3232(src, o2) != uint32(cv) {
+ // Do regular search
+ break
+ }
+
+ // Store this, since we have it.
+ nextHashS := hash5(cv, dFastShortTableBits)
+ nextHashL := hash8(cv, dFastLongTableBits)
+
+ // We have at least 4 byte match.
+ // No need to check backwards. We come straight from a match
+ l := 4 + e.matchlen(s+4, o2+4, src)
+
+ entry := tableEntry{offset: s + e.cur, val: uint32(cv)}
+ e.longTable[nextHashL] = entry
+ e.markLongShardDirty(nextHashL)
+ e.table[nextHashS] = entry
+ e.markShardDirty(nextHashS)
+ seq.matchLen = uint32(l) - zstdMinMatch
+ seq.litLen = 0
+
+ // Since litlen is always 0, this is offset 1.
+ seq.offset = 1
+ s += l
+ nextEmit = s
+ if debugSequences {
+ println("sequence", seq, "next s:", s)
+ }
+ blk.sequences = append(blk.sequences, seq)
+
+ // Swap offset 1 and 2.
+ offset1, offset2 = offset2, offset1
+ if s >= sLimit {
+ // Finished
+ break encodeLoop
+ }
+ cv = load6432(src, s)
+ }
+ }
+
+ if int(nextEmit) < len(src) {
+ blk.literals = append(blk.literals, src[nextEmit:]...)
+ blk.extraLits = len(src) - int(nextEmit)
+ }
+ blk.recentOffsets[0] = uint32(offset1)
+ blk.recentOffsets[1] = uint32(offset2)
+ if debug {
+ println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits)
+ }
+ // If we encoded more than 64K mark all dirty.
+ if len(src) > 64<<10 {
+ e.markAllShardsDirty()
+ }
+}
+
// ResetDict will reset and set a dictionary if not nil
func (e *doubleFastEncoder) Reset(d *dict, singleBlock bool) {
e.fastEncoder.Reset(d, singleBlock)
+ if d != nil {
+ panic("doubleFastEncoder: Reset with dict not supported")
+ }
+}
+
+// ResetDict will reset and set a dictionary if not nil
+func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) {
+ allDirty := e.allDirty
+ e.fastEncoderDict.Reset(d, singleBlock)
if d == nil {
return
}
@@ -706,8 +1085,37 @@ func (e *doubleFastEncoder) Reset(d *dict, singleBlock bool) {
}
}
e.lastDictID = d.id
+ e.allDirty = true
}
// Reset table to initial state
e.cur = e.maxMatchOff
- copy(e.longTable[:], e.dictLongTable)
+
+ dirtyShardCnt := 0
+ if !allDirty {
+ for i := range e.longTableShardDirty {
+ if e.longTableShardDirty[i] {
+ dirtyShardCnt++
+ }
+ }
+ }
+
+ if allDirty || dirtyShardCnt > dLongTableShardCnt/2 {
+ copy(e.longTable[:], e.dictLongTable)
+ for i := range e.longTableShardDirty {
+ e.longTableShardDirty[i] = false
+ }
+ return
+ }
+ for i := range e.longTableShardDirty {
+ if !e.longTableShardDirty[i] {
+ continue
+ }
+
+ copy(e.longTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize], e.dictLongTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize])
+ e.longTableShardDirty[i] = false
+ }
+}
+
+func (e *doubleFastEncoderDict) markLongShardDirty(entryNum uint32) {
+ e.longTableShardDirty[entryNum/dLongTableShardSize] = true
}
diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go
index 0045016d9..ba4a17e10 100644
--- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go
+++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go
@@ -11,9 +11,11 @@ import (
)
const (
- tableBits = 15 // Bits used in the table
- tableSize = 1 << tableBits // Size of the table
- tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks.
+ tableBits = 15 // Bits used in the table
+ tableSize = 1 << tableBits // Size of the table
+ tableShardCnt = 1 << (tableBits - dictShardBits) // Number of shards in the table
+ tableShardSize = tableSize / tableShardCnt // Size of an individual shard
+ tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks.
maxMatchLength = 131074
)
@@ -24,8 +26,14 @@ type tableEntry struct {
type fastEncoder struct {
fastBase
- table [tableSize]tableEntry
- dictTable []tableEntry
+ table [tableSize]tableEntry
+}
+
+type fastEncoderDict struct {
+ fastEncoder
+ dictTable []tableEntry
+ tableShardDirty [tableShardCnt]bool
+ allDirty bool
}
// Encode mimmics functionality in zstd_fast.c
@@ -617,9 +625,323 @@ encodeLoop:
}
}
+// Encode will encode the content, with a dictionary if initialized for it.
+func (e *fastEncoderDict) Encode(blk *blockEnc, src []byte) {
+ const (
+ inputMargin = 8
+ minNonLiteralBlockSize = 1 + 1 + inputMargin
+ )
+ if e.allDirty || len(src) > 32<<10 {
+ e.fastEncoder.Encode(blk, src)
+ e.allDirty = true
+ return
+ }
+ // Protect against e.cur wraparound.
+ for e.cur >= bufferReset {
+ if len(e.hist) == 0 {
+ for i := range e.table[:] {
+ e.table[i] = tableEntry{}
+ }
+ e.cur = e.maxMatchOff
+ break
+ }
+ // Shift down everything in the table that isn't already too far away.
+ minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff
+ for i := range e.table[:] {
+ v := e.table[i].offset
+ if v < minOff {
+ v = 0
+ } else {
+ v = v - e.cur + e.maxMatchOff
+ }
+ e.table[i].offset = v
+ }
+ e.cur = e.maxMatchOff
+ break
+ }
+
+ s := e.addBlock(src)
+ blk.size = len(src)
+ if len(src) < minNonLiteralBlockSize {
+ blk.extraLits = len(src)
+ blk.literals = blk.literals[:len(src)]
+ copy(blk.literals, src)
+ return
+ }
+
+ // Override src
+ src = e.hist
+ sLimit := int32(len(src)) - inputMargin
+ // stepSize is the number of bytes to skip on every main loop iteration.
+ // It should be >= 2.
+ const stepSize = 2
+
+ // TEMPLATE
+ const hashLog = tableBits
+ // seems global, but would be nice to tweak.
+ const kSearchStrength = 7
+
+ // nextEmit is where in src the next emitLiteral should start from.
+ nextEmit := s
+ cv := load6432(src, s)
+
+ // Relative offsets
+ offset1 := int32(blk.recentOffsets[0])
+ offset2 := int32(blk.recentOffsets[1])
+
+ addLiterals := func(s *seq, until int32) {
+ if until == nextEmit {
+ return
+ }
+ blk.literals = append(blk.literals, src[nextEmit:until]...)
+ s.litLen = uint32(until - nextEmit)
+ }
+ if debug {
+ println("recent offsets:", blk.recentOffsets)
+ }
+
+encodeLoop:
+ for {
+ // t will contain the match offset when we find one.
+ // When existing the search loop, we have already checked 4 bytes.
+ var t int32
+
+ // We will not use repeat offsets across blocks.
+ // By not using them for the first 3 matches
+ canRepeat := len(blk.sequences) > 2
+
+ for {
+ if debugAsserts && canRepeat && offset1 == 0 {
+ panic("offset0 was 0")
+ }
+
+ nextHash := hash6(cv, hashLog)
+ nextHash2 := hash6(cv>>8, hashLog)
+ candidate := e.table[nextHash]
+ candidate2 := e.table[nextHash2]
+ repIndex := s - offset1 + 2
+
+ e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)}
+ e.markShardDirty(nextHash)
+ e.table[nextHash2] = tableEntry{offset: s + e.cur + 1, val: uint32(cv >> 8)}
+ e.markShardDirty(nextHash2)
+
+ if canRepeat && repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>16) {
+ // Consider history as well.
+ var seq seq
+ var length int32
+ // length = 4 + e.matchlen(s+6, repIndex+4, src)
+ {
+ a := src[s+6:]
+ b := src[repIndex+4:]
+ endI := len(a) & (math.MaxInt32 - 7)
+ length = int32(endI) + 4
+ for i := 0; i < endI; i += 8 {
+ if diff := load64(a, i) ^ load64(b, i); diff != 0 {
+ length = int32(i+bits.TrailingZeros64(diff)>>3) + 4
+ break
+ }
+ }
+ }
+
+ seq.matchLen = uint32(length - zstdMinMatch)
+
+ // We might be able to match backwards.
+ // Extend as long as we can.
+ start := s + 2
+ // We end the search early, so we don't risk 0 literals
+ // and have to do special offset treatment.
+ startLimit := nextEmit + 1
+
+ sMin := s - e.maxMatchOff
+ if sMin < 0 {
+ sMin = 0
+ }
+ for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch {
+ repIndex--
+ start--
+ seq.matchLen++
+ }
+ addLiterals(&seq, start)
+
+ // rep 0
+ seq.offset = 1
+ if debugSequences {
+ println("repeat sequence", seq, "next s:", s)
+ }
+ blk.sequences = append(blk.sequences, seq)
+ s += length + 2
+ nextEmit = s
+ if s >= sLimit {
+ if debug {
+ println("repeat ended", s, length)
+
+ }
+ break encodeLoop
+ }
+ cv = load6432(src, s)
+ continue
+ }
+ coffset0 := s - (candidate.offset - e.cur)
+ coffset1 := s - (candidate2.offset - e.cur) + 1
+ if coffset0 < e.maxMatchOff && uint32(cv) == candidate.val {
+ // found a regular match
+ t = candidate.offset - e.cur
+ if debugAsserts && s <= t {
+ panic(fmt.Sprintf("s (%d) <= t (%d)", s, t))
+ }
+ if debugAsserts && s-t > e.maxMatchOff {
+ panic("s - t >e.maxMatchOff")
+ }
+ break
+ }
+
+ if coffset1 < e.maxMatchOff && uint32(cv>>8) == candidate2.val {
+ // found a regular match
+ t = candidate2.offset - e.cur
+ s++
+ if debugAsserts && s <= t {
+ panic(fmt.Sprintf("s (%d) <= t (%d)", s, t))
+ }
+ if debugAsserts && s-t > e.maxMatchOff {
+ panic("s - t >e.maxMatchOff")
+ }
+ if debugAsserts && t < 0 {
+ panic("t<0")
+ }
+ break
+ }
+ s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1))
+ if s >= sLimit {
+ break encodeLoop
+ }
+ cv = load6432(src, s)
+ }
+ // A 4-byte match has been found. We'll later see if more than 4 bytes.
+ offset2 = offset1
+ offset1 = s - t
+
+ if debugAsserts && s <= t {
+ panic(fmt.Sprintf("s (%d) <= t (%d)", s, t))
+ }
+
+ if debugAsserts && canRepeat && int(offset1) > len(src) {
+ panic("invalid offset")
+ }
+
+ // Extend the 4-byte match as long as possible.
+ //l := e.matchlen(s+4, t+4, src) + 4
+ var l int32
+ {
+ a := src[s+4:]
+ b := src[t+4:]
+ endI := len(a) & (math.MaxInt32 - 7)
+ l = int32(endI) + 4
+ for i := 0; i < endI; i += 8 {
+ if diff := load64(a, i) ^ load64(b, i); diff != 0 {
+ l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
+ break
+ }
+ }
+ }
+
+ // Extend backwards
+ tMin := s - e.maxMatchOff
+ if tMin < 0 {
+ tMin = 0
+ }
+ for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength {
+ s--
+ t--
+ l++
+ }
+
+ // Write our sequence.
+ var seq seq
+ seq.litLen = uint32(s - nextEmit)
+ seq.matchLen = uint32(l - zstdMinMatch)
+ if seq.litLen > 0 {
+ blk.literals = append(blk.literals, src[nextEmit:s]...)
+ }
+ // Don't use repeat offsets
+ seq.offset = uint32(s-t) + 3
+ s += l
+ if debugSequences {
+ println("sequence", seq, "next s:", s)
+ }
+ blk.sequences = append(blk.sequences, seq)
+ nextEmit = s
+ if s >= sLimit {
+ break encodeLoop
+ }
+ cv = load6432(src, s)
+
+ // Check offset 2
+ if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) {
+ // We have at least 4 byte match.
+ // No need to check backwards. We come straight from a match
+ //l := 4 + e.matchlen(s+4, o2+4, src)
+ var l int32
+ {
+ a := src[s+4:]
+ b := src[o2+4:]
+ endI := len(a) & (math.MaxInt32 - 7)
+ l = int32(endI) + 4
+ for i := 0; i < endI; i += 8 {
+ if diff := load64(a, i) ^ load64(b, i); diff != 0 {
+ l = int32(i+bits.TrailingZeros64(diff)>>3) + 4
+ break
+ }
+ }
+ }
+
+ // Store this, since we have it.
+ nextHash := hash6(cv, hashLog)
+ e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)}
+ e.markShardDirty(nextHash)
+ seq.matchLen = uint32(l) - zstdMinMatch
+ seq.litLen = 0
+ // Since litlen is always 0, this is offset 1.
+ seq.offset = 1
+ s += l
+ nextEmit = s
+ if debugSequences {
+ println("sequence", seq, "next s:", s)
+ }
+ blk.sequences = append(blk.sequences, seq)
+
+ // Swap offset 1 and 2.
+ offset1, offset2 = offset2, offset1
+ if s >= sLimit {
+ break encodeLoop
+ }
+ // Prepare next loop.
+ cv = load6432(src, s)
+ }
+ }
+
+ if int(nextEmit) < len(src) {
+ blk.literals = append(blk.literals, src[nextEmit:]...)
+ blk.extraLits = len(src) - int(nextEmit)
+ }
+ blk.recentOffsets[0] = uint32(offset1)
+ blk.recentOffsets[1] = uint32(offset2)
+ if debug {
+ println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits)
+ }
+}
+
// ResetDict will reset and set a dictionary if not nil
func (e *fastEncoder) Reset(d *dict, singleBlock bool) {
e.resetBase(d, singleBlock)
+ if d != nil {
+ panic("fastEncoder: Reset with dict")
+ }
+}
+
+// ResetDict will reset and set a dictionary if not nil
+func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) {
+ e.resetBase(d, singleBlock)
if d == nil {
return
}
@@ -653,9 +975,44 @@ func (e *fastEncoder) Reset(d *dict, singleBlock bool) {
}
}
e.lastDictID = d.id
+ e.allDirty = true
}
e.cur = e.maxMatchOff
- // Reset table to initial state
- copy(e.table[:], e.dictTable)
+ dirtyShardCnt := 0
+ if !e.allDirty {
+ for i := range e.tableShardDirty {
+ if e.tableShardDirty[i] {
+ dirtyShardCnt++
+ }
+ }
+ }
+
+ const shardCnt = tableShardCnt
+ const shardSize = tableShardSize
+ if e.allDirty || dirtyShardCnt > shardCnt*4/6 {
+ copy(e.table[:], e.dictTable)
+ for i := range e.tableShardDirty {
+ e.tableShardDirty[i] = false
+ }
+ e.allDirty = false
+ return
+ }
+ for i := range e.tableShardDirty {
+ if !e.tableShardDirty[i] {
+ continue
+ }
+
+ copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize])
+ e.tableShardDirty[i] = false
+ }
+ e.allDirty = false
+}
+
+func (e *fastEncoderDict) markAllShardsDirty() {
+ e.allDirty = true
+}
+
+func (e *fastEncoderDict) markShardDirty(entryNum uint32) {
+ e.tableShardDirty[entryNum/tableShardSize] = true
}
diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go
index f5759211d..6f0265099 100644
--- a/vendor/github.com/klauspost/compress/zstd/encoder.go
+++ b/vendor/github.com/klauspost/compress/zstd/encoder.go
@@ -106,7 +106,7 @@ func (e *Encoder) Reset(w io.Writer) {
s.encoder = e.o.encoder()
}
if s.writing == nil {
- s.writing = &blockEnc{}
+ s.writing = &blockEnc{lowMem: e.o.lowMem}
s.writing.init()
}
s.writing.initNewEncode()
@@ -176,6 +176,12 @@ func (e *Encoder) nextBlock(final bool) error {
}
if !s.headerWritten {
// If we have a single block encode, do a sync compression.
+ if final && len(s.filling) == 0 && !e.o.fullZero {
+ s.headerWritten = true
+ s.fullFrameWritten = true
+ s.eofWritten = true
+ return nil
+ }
if final && len(s.filling) > 0 {
s.current = e.EncodeAll(s.filling, s.current[:0])
var n2 int
@@ -471,7 +477,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte {
}
// If less than 1MB, allocate a buffer up front.
- if len(dst) == 0 && cap(dst) == 0 && len(src) < 1<<20 {
+ if len(dst) == 0 && cap(dst) == 0 && len(src) < 1<<20 && !e.o.lowMem {
dst = make([]byte, 0, len(src))
}
dst, err := fh.appendTo(dst)
diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go
index a7312f42a..18a47eb03 100644
--- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go
+++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go
@@ -24,12 +24,12 @@ type encoderOptions struct {
allLitEntropy bool
customWindow bool
customALEntropy bool
+ lowMem bool
dict *dict
}
func (o *encoderOptions) setDefault() {
*o = encoderOptions{
- // use less ram: true for now, but may change.
concurrent: runtime.GOMAXPROCS(0),
crc: true,
single: nil,
@@ -37,20 +37,31 @@ func (o *encoderOptions) setDefault() {
windowSize: 8 << 20,
level: SpeedDefault,
allLitEntropy: true,
+ lowMem: false,
}
}
// encoder returns an encoder with the selected options.
func (o encoderOptions) encoder() encoder {
switch o.level {
+ case SpeedFastest:
+ if o.dict != nil {
+ return &fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}
+ }
+ return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}
+
case SpeedDefault:
- return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}}
+ if o.dict != nil {
+ return &doubleFastEncoderDict{fastEncoderDict: fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}}
+ }
+ return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}
case SpeedBetterCompression:
- return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}
+ if o.dict != nil {
+ return &betterFastEncoderDict{betterFastEncoder: betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}
+ }
+ return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}
case SpeedBestCompression:
- return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}
- case SpeedFastest:
- return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}
+ return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}
}
panic("unknown compression level")
}
@@ -276,6 +287,17 @@ func WithSingleSegment(b bool) EOption {
}
}
+// WithLowerEncoderMem will trade in some memory cases trade less memory usage for
+// slower encoding speed.
+// This will not change the window size which is the primary function for reducing
+// memory usage. See WithWindowSize.
+func WithLowerEncoderMem(b bool) EOption {
+ return func(o *encoderOptions) error {
+ o.lowMem = b
+ return nil
+ }
+}
+
// WithEncoderDict allows to register a dictionary that will be used for the encode.
// The encoder *may* choose to use no dictionary instead for certain payloads.
func WithEncoderDict(dict []byte) EOption {
diff --git a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go
index aa9eba88b..b80709d5e 100644
--- a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go
+++ b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go
@@ -97,7 +97,7 @@ func (s *fseEncoder) prepare() (*fseEncoder, error) {
func (s *fseEncoder) allocCtable() {
tableSize := 1 << s.actualTableLog
// get tableSymbol that is big enough.
- if cap(s.ct.tableSymbol) < int(tableSize) {
+ if cap(s.ct.tableSymbol) < tableSize {
s.ct.tableSymbol = make([]byte, tableSize)
}
s.ct.tableSymbol = s.ct.tableSymbol[:tableSize]
@@ -202,13 +202,13 @@ func (s *fseEncoder) buildCTable() error {
case 0:
case -1, 1:
symbolTT[i].deltaNbBits = tl
- symbolTT[i].deltaFindState = int16(total - 1)
+ symbolTT[i].deltaFindState = total - 1
total++
default:
maxBitsOut := uint32(tableLog) - highBit(uint32(v-1))
minStatePlus := uint32(v) << maxBitsOut
symbolTT[i].deltaNbBits = (maxBitsOut << 16) - minStatePlus
- symbolTT[i].deltaFindState = int16(total - v)
+ symbolTT[i].deltaFindState = total - v
total += v
}
}
@@ -353,8 +353,8 @@ func (s *fseEncoder) normalizeCount2(length int) error {
distributed uint32
total = uint32(length)
tableLog = s.actualTableLog
- lowThreshold = uint32(total >> tableLog)
- lowOne = uint32((total * 3) >> (tableLog + 1))
+ lowThreshold = total >> tableLog
+ lowOne = (total * 3) >> (tableLog + 1)
)
for i, cnt := range s.count[:s.symbolLen] {
if cnt == 0 {
@@ -379,7 +379,7 @@ func (s *fseEncoder) normalizeCount2(length int) error {
if (total / toDistribute) > lowOne {
// risk of rounding to zero
- lowOne = uint32((total * 3) / (toDistribute * 2))
+ lowOne = (total * 3) / (toDistribute * 2)
for i, cnt := range s.count[:s.symbolLen] {
if (s.norm[i] == notYetAssigned) && (cnt <= lowOne) {
s.norm[i] = 1
diff --git a/vendor/github.com/klauspost/compress/zstd/snappy.go b/vendor/github.com/klauspost/compress/zstd/snappy.go
index 841fd95ac..c95fe5111 100644
--- a/vendor/github.com/klauspost/compress/zstd/snappy.go
+++ b/vendor/github.com/klauspost/compress/zstd/snappy.go
@@ -417,7 +417,7 @@ var crcTable = crc32.MakeTable(crc32.Castagnoli)
// https://github.com/google/snappy/blob/master/framing_format.txt
func snappyCRC(b []byte) uint32 {
c := crc32.Update(0, crcTable, b)
- return uint32(c>>15|c<<17) + 0xa282ead8
+ return c>>15 | c<<17 + 0xa282ead8
}
// snappyDecodedLen returns the length of the decoded block and the number of bytes
diff --git a/vendor/github.com/vbauerster/mpb/v5/.gitignore b/vendor/github.com/vbauerster/mpb/v5/.gitignore
deleted file mode 100644
index 63bd91672..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/.gitignore
+++ /dev/null
@@ -1,5 +0,0 @@
-# Test binary, build with `go test -c`
-*.test
-
-# Output of the go coverage tool, specifically when used with LiteIDE
-*.out
diff --git a/vendor/github.com/vbauerster/mpb/v5/.travis.yml b/vendor/github.com/vbauerster/mpb/v5/.travis.yml
deleted file mode 100644
index 9a203a67d..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/.travis.yml
+++ /dev/null
@@ -1,11 +0,0 @@
-language: go
-arch:
- - amd64
- - ppc64le
-
-go:
- - 1.14.x
-
-script:
- - go test -race ./...
- - for i in _examples/*/; do go build $i/*.go || exit 1; done
diff --git a/vendor/github.com/vbauerster/mpb/v5/README.md b/vendor/github.com/vbauerster/mpb/v5/README.md
deleted file mode 100644
index bfb0c4d18..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/README.md
+++ /dev/null
@@ -1,118 +0,0 @@
-# Multi Progress Bar
-
-[![GoDoc](https://godoc.org/github.com/vbauerster/mpb?status.svg)](https://godoc.org/github.com/vbauerster/mpb)
-[![Build Status](https://travis-ci.org/vbauerster/mpb.svg?branch=master)](https://travis-ci.org/vbauerster/mpb)
-[![Go Report Card](https://goreportcard.com/badge/github.com/vbauerster/mpb)](https://goreportcard.com/report/github.com/vbauerster/mpb)
-
-**mpb** is a Go lib for rendering progress bars in terminal applications.
-
-## Features
-
-* __Multiple Bars__: Multiple progress bars are supported
-* __Dynamic Total__: Set total while bar is running
-* __Dynamic Add/Remove__: Dynamically add or remove bars
-* __Cancellation__: Cancel whole rendering process
-* __Predefined Decorators__: Elapsed time, [ewma](https://github.com/VividCortex/ewma) based ETA, Percentage, Bytes counter
-* __Decorator's width sync__: Synchronized decorator's width among multiple bars
-
-## Usage
-
-#### [Rendering single bar](_examples/singleBar/main.go)
-```go
-package main
-
-import (
- "math/rand"
- "time"
-
- "github.com/vbauerster/mpb/v5"
- "github.com/vbauerster/mpb/v5/decor"
-)
-
-func main() {
- // initialize progress container, with custom width
- p := mpb.New(mpb.WithWidth(64))
-
- total := 100
- name := "Single Bar:"
- // adding a single bar, which will inherit container's width
- bar := p.AddBar(int64(total),
- // override DefaultBarStyle, which is "[=>-]<+"
- mpb.BarStyle("╢▌▌░╟"),
- mpb.PrependDecorators(
- // display our name with one space on the right
- decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight}),
- // replace ETA decorator with "done" message, OnComplete event
- decor.OnComplete(
- decor.AverageETA(decor.ET_STYLE_GO, decor.WC{W: 4}), "done",
- ),
- ),
- mpb.AppendDecorators(decor.Percentage()),
- )
- // simulating some work
- max := 100 * time.Millisecond
- for i := 0; i < total; i++ {
- time.Sleep(time.Duration(rand.Intn(10)+1) * max / 10)
- bar.Increment()
- }
- // wait for our bar to complete and flush
- p.Wait()
-}
-```
-
-#### [Rendering multiple bars](_examples/multiBars/main.go)
-```go
- var wg sync.WaitGroup
- // pass &wg (optional), so p will wait for it eventually
- p := mpb.New(mpb.WithWaitGroup(&wg))
- total, numBars := 100, 3
- wg.Add(numBars)
-
- for i := 0; i < numBars; i++ {
- name := fmt.Sprintf("Bar#%d:", i)
- bar := p.AddBar(int64(total),
- mpb.PrependDecorators(
- // simple name decorator
- decor.Name(name),
- // decor.DSyncWidth bit enables column width synchronization
- decor.Percentage(decor.WCSyncSpace),
- ),
- mpb.AppendDecorators(
- // replace ETA decorator with "done" message, OnComplete event
- decor.OnComplete(
- // ETA decorator with ewma age of 60
- decor.EwmaETA(decor.ET_STYLE_GO, 60), "done",
- ),
- ),
- )
- // simulating some work
- go func() {
- defer wg.Done()
- rng := rand.New(rand.NewSource(time.Now().UnixNano()))
- max := 100 * time.Millisecond
- for i := 0; i < total; i++ {
- // start variable is solely for EWMA calculation
- // EWMA's unit of measure is an iteration's duration
- start := time.Now()
- time.Sleep(time.Duration(rng.Intn(10)+1) * max / 10)
- bar.Increment()
- // we need to call DecoratorEwmaUpdate to fulfill ewma decorator's contract
- bar.DecoratorEwmaUpdate(time.Since(start))
- }
- }()
- }
- // Waiting for passed &wg and for all bars to complete and flush
- p.Wait()
-```
-
-#### [Dynamic total](_examples/dynTotal/main.go)
-
-![dynamic total](_svg/godEMrCZmJkHYH1X9dN4Nm0U7.svg)
-
-#### [Complex example](_examples/complex/main.go)
-
-![complex](_svg/wHzf1M7sd7B3zVa2scBMnjqRf.svg)
-
-#### [Bytes counters](_examples/io/main.go)
-
-![byte counters](_svg/hIpTa3A5rQz65ssiVuRJu87X6.svg)
diff --git a/vendor/github.com/vbauerster/mpb/v5/UNLICENSE b/vendor/github.com/vbauerster/mpb/v5/UNLICENSE
deleted file mode 100644
index 68a49daad..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/UNLICENSE
+++ /dev/null
@@ -1,24 +0,0 @@
-This is free and unencumbered software released into the public domain.
-
-Anyone is free to copy, modify, publish, use, compile, sell, or
-distribute this software, either in source code form or as a compiled
-binary, for any purpose, commercial or non-commercial, and by any
-means.
-
-In jurisdictions that recognize copyright laws, the author or authors
-of this software dedicate any and all copyright interest in the
-software to the public domain. We make this dedication for the benefit
-of the public at large and to the detriment of our heirs and
-successors. We intend this dedication to be an overt act of
-relinquishment in perpetuity of all present and future rights to this
-software under copyright law.
-
-THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
-EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
-MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
-IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR
-OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
-ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
-OTHER DEALINGS IN THE SOFTWARE.
-
-For more information, please refer to <http://unlicense.org/>
diff --git a/vendor/github.com/vbauerster/mpb/v5/bar.go b/vendor/github.com/vbauerster/mpb/v5/bar.go
deleted file mode 100644
index 358cb048d..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/bar.go
+++ /dev/null
@@ -1,490 +0,0 @@
-package mpb
-
-import (
- "bytes"
- "context"
- "fmt"
- "io"
- "log"
- "runtime/debug"
- "strings"
- "time"
-
- "github.com/acarl005/stripansi"
- "github.com/mattn/go-runewidth"
- "github.com/vbauerster/mpb/v5/decor"
-)
-
-// Bar represents a progress Bar.
-type Bar struct {
- priority int // used by heap
- index int // used by heap
-
- extendedLines int
- toShutdown bool
- toDrop bool
- noPop bool
- hasEwmaDecorators bool
- operateState chan func(*bState)
- frameCh chan io.Reader
- syncTableCh chan [][]chan int
- completed chan bool
-
- // cancel is called either by user or on complete event
- cancel func()
- // done is closed after cacheState is assigned
- done chan struct{}
- // cacheState is populated, right after close(shutdown)
- cacheState *bState
-
- container *Progress
- dlogger *log.Logger
- recoveredPanic interface{}
-}
-
-type extFunc func(in io.Reader, reqWidth int, st decor.Statistics) (out io.Reader, lines int)
-
-type bState struct {
- id int
- priority int
- reqWidth int
- total int64
- current int64
- refill int64
- lastN int64
- iterated bool
- trimSpace bool
- toComplete bool
- completeFlushed bool
- ignoreComplete bool
- dropOnComplete bool
- noPop bool
- aDecorators []decor.Decorator
- pDecorators []decor.Decorator
- averageDecorators []decor.AverageDecorator
- ewmaDecorators []decor.EwmaDecorator
- shutdownListeners []decor.ShutdownListener
- bufP, bufB, bufA *bytes.Buffer
- filler BarFiller
- middleware func(BarFiller) BarFiller
- extender extFunc
-
- // runningBar is a key for *pState.parkedBars
- runningBar *Bar
-
- debugOut io.Writer
-}
-
-func newBar(container *Progress, bs *bState) *Bar {
- logPrefix := fmt.Sprintf("%sbar#%02d ", container.dlogger.Prefix(), bs.id)
- ctx, cancel := context.WithCancel(container.ctx)
-
- bar := &Bar{
- container: container,
- priority: bs.priority,
- toDrop: bs.dropOnComplete,
- noPop: bs.noPop,
- operateState: make(chan func(*bState)),
- frameCh: make(chan io.Reader, 1),
- syncTableCh: make(chan [][]chan int, 1),
- completed: make(chan bool, 1),
- done: make(chan struct{}),
- cancel: cancel,
- dlogger: log.New(bs.debugOut, logPrefix, log.Lshortfile),
- }
-
- go bar.serve(ctx, bs)
- return bar
-}
-
-// ProxyReader wraps r with metrics required for progress tracking.
-// Panics if r is nil.
-func (b *Bar) ProxyReader(r io.Reader) io.ReadCloser {
- if r == nil {
- panic("expected non nil io.Reader")
- }
- return newProxyReader(r, b)
-}
-
-// ID returs id of the bar.
-func (b *Bar) ID() int {
- result := make(chan int)
- select {
- case b.operateState <- func(s *bState) { result <- s.id }:
- return <-result
- case <-b.done:
- return b.cacheState.id
- }
-}
-
-// Current returns bar's current number, in other words sum of all increments.
-func (b *Bar) Current() int64 {
- result := make(chan int64)
- select {
- case b.operateState <- func(s *bState) { result <- s.current }:
- return <-result
- case <-b.done:
- return b.cacheState.current
- }
-}
-
-// SetRefill fills bar with refill rune up to amount argument.
-// Given default bar style is "[=>-]<+", refill rune is '+'.
-// To set bar style use mpb.BarStyle(string) BarOption.
-func (b *Bar) SetRefill(amount int64) {
- select {
- case b.operateState <- func(s *bState) {
- s.refill = amount
- }:
- case <-b.done:
- }
-}
-
-// TraverseDecorators traverses all available decorators and calls cb func on each.
-func (b *Bar) TraverseDecorators(cb func(decor.Decorator)) {
- select {
- case b.operateState <- func(s *bState) {
- for _, decorators := range [...][]decor.Decorator{
- s.pDecorators,
- s.aDecorators,
- } {
- for _, d := range decorators {
- cb(extractBaseDecorator(d))
- }
- }
- }:
- case <-b.done:
- }
-}
-
-// SetTotal sets total dynamically.
-// If total is less than or equal to zero it takes progress' current value.
-// A complete flag enables or disables complete event on `current >= total`.
-func (b *Bar) SetTotal(total int64, complete bool) {
- select {
- case b.operateState <- func(s *bState) {
- s.ignoreComplete = !complete
- if total <= 0 {
- s.total = s.current
- } else {
- s.total = total
- }
- if !s.ignoreComplete && !s.toComplete {
- s.current = s.total
- s.toComplete = true
- go b.refreshTillShutdown()
- }
- }:
- case <-b.done:
- }
-}
-
-// SetCurrent sets progress' current to an arbitrary value.
-// Setting a negative value will cause a panic.
-func (b *Bar) SetCurrent(current int64) {
- select {
- case b.operateState <- func(s *bState) {
- s.iterated = true
- s.lastN = current - s.current
- s.current = current
- if !s.ignoreComplete && s.current >= s.total {
- s.current = s.total
- s.toComplete = true
- go b.refreshTillShutdown()
- }
- }:
- case <-b.done:
- }
-}
-
-// Increment is a shorthand for b.IncrInt64(1).
-func (b *Bar) Increment() {
- b.IncrInt64(1)
-}
-
-// IncrBy is a shorthand for b.IncrInt64(int64(n)).
-func (b *Bar) IncrBy(n int) {
- b.IncrInt64(int64(n))
-}
-
-// IncrInt64 increments progress by amount of n.
-func (b *Bar) IncrInt64(n int64) {
- select {
- case b.operateState <- func(s *bState) {
- s.iterated = true
- s.lastN = n
- s.current += n
- if !s.ignoreComplete && s.current >= s.total {
- s.current = s.total
- s.toComplete = true
- go b.refreshTillShutdown()
- }
- }:
- case <-b.done:
- }
-}
-
-// DecoratorEwmaUpdate updates all EWMA based decorators. Should be
-// called on each iteration, because EWMA's unit of measure is an
-// iteration's duration. Panics if called before *Bar.Incr... family
-// methods.
-func (b *Bar) DecoratorEwmaUpdate(dur time.Duration) {
- select {
- case b.operateState <- func(s *bState) {
- ewmaIterationUpdate(false, s, dur)
- }:
- case <-b.done:
- ewmaIterationUpdate(true, b.cacheState, dur)
- }
-}
-
-// DecoratorAverageAdjust adjusts all average based decorators. Call
-// if you need to adjust start time of all average based decorators
-// or after progress resume.
-func (b *Bar) DecoratorAverageAdjust(start time.Time) {
- select {
- case b.operateState <- func(s *bState) {
- for _, d := range s.averageDecorators {
- d.AverageAdjust(start)
- }
- }:
- case <-b.done:
- }
-}
-
-// SetPriority changes bar's order among multiple bars. Zero is highest
-// priority, i.e. bar will be on top. If you don't need to set priority
-// dynamically, better use BarPriority option.
-func (b *Bar) SetPriority(priority int) {
- select {
- case <-b.done:
- default:
- b.container.setBarPriority(b, priority)
- }
-}
-
-// Abort interrupts bar's running goroutine. Call this, if you'd like
-// to stop/remove bar before completion event. It has no effect after
-// completion event. If drop is true bar will be removed as well.
-func (b *Bar) Abort(drop bool) {
- select {
- case <-b.done:
- default:
- if drop {
- b.container.dropBar(b)
- }
- b.cancel()
- }
-}
-
-// Completed reports whether the bar is in completed state.
-func (b *Bar) Completed() bool {
- select {
- case b.operateState <- func(s *bState) { b.completed <- s.toComplete }:
- return <-b.completed
- case <-b.done:
- return true
- }
-}
-
-func (b *Bar) serve(ctx context.Context, s *bState) {
- defer b.container.bwg.Done()
- for {
- select {
- case op := <-b.operateState:
- op(s)
- case <-ctx.Done():
- b.cacheState = s
- close(b.done)
- // Notifying decorators about shutdown event
- for _, sl := range s.shutdownListeners {
- sl.Shutdown()
- }
- return
- }
- }
-}
-
-func (b *Bar) render(tw int) {
- select {
- case b.operateState <- func(s *bState) {
- stat := newStatistics(tw, s)
- defer func() {
- // recovering if user defined decorator panics for example
- if p := recover(); p != nil {
- if b.recoveredPanic == nil {
- s.extender = makePanicExtender(p)
- b.toShutdown = !b.toShutdown
- b.recoveredPanic = p
- }
- frame, lines := s.extender(nil, s.reqWidth, stat)
- b.extendedLines = lines
- b.frameCh <- frame
- b.dlogger.Println(p)
- }
- s.completeFlushed = s.toComplete
- }()
- frame, lines := s.extender(s.draw(stat), s.reqWidth, stat)
- b.extendedLines = lines
- b.toShutdown = s.toComplete && !s.completeFlushed
- b.frameCh <- frame
- }:
- case <-b.done:
- s := b.cacheState
- stat := newStatistics(tw, s)
- var r io.Reader
- if b.recoveredPanic == nil {
- r = s.draw(stat)
- }
- frame, lines := s.extender(r, s.reqWidth, stat)
- b.extendedLines = lines
- b.frameCh <- frame
- }
-}
-
-func (b *Bar) subscribeDecorators() {
- var averageDecorators []decor.AverageDecorator
- var ewmaDecorators []decor.EwmaDecorator
- var shutdownListeners []decor.ShutdownListener
- b.TraverseDecorators(func(d decor.Decorator) {
- if d, ok := d.(decor.AverageDecorator); ok {
- averageDecorators = append(averageDecorators, d)
- }
- if d, ok := d.(decor.EwmaDecorator); ok {
- ewmaDecorators = append(ewmaDecorators, d)
- }
- if d, ok := d.(decor.ShutdownListener); ok {
- shutdownListeners = append(shutdownListeners, d)
- }
- })
- select {
- case b.operateState <- func(s *bState) {
- s.averageDecorators = averageDecorators
- s.ewmaDecorators = ewmaDecorators
- s.shutdownListeners = shutdownListeners
- }:
- b.hasEwmaDecorators = len(ewmaDecorators) != 0
- case <-b.done:
- }
-}
-
-func (b *Bar) refreshTillShutdown() {
- for {
- select {
- case b.container.refreshCh <- time.Now():
- case <-b.done:
- return
- }
- }
-}
-
-func (b *Bar) wSyncTable() [][]chan int {
- select {
- case b.operateState <- func(s *bState) { b.syncTableCh <- s.wSyncTable() }:
- return <-b.syncTableCh
- case <-b.done:
- return b.cacheState.wSyncTable()
- }
-}
-
-func (s *bState) draw(stat decor.Statistics) io.Reader {
- if !s.trimSpace {
- stat.AvailableWidth -= 2
- s.bufB.WriteByte(' ')
- defer s.bufB.WriteByte(' ')
- }
-
- nlr := strings.NewReader("\n")
- tw := stat.AvailableWidth
- for _, d := range s.pDecorators {
- str := d.Decor(stat)
- stat.AvailableWidth -= runewidth.StringWidth(stripansi.Strip(str))
- s.bufP.WriteString(str)
- }
- if stat.AvailableWidth <= 0 {
- trunc := strings.NewReader(runewidth.Truncate(stripansi.Strip(s.bufP.String()), tw, "…"))
- s.bufP.Reset()
- return io.MultiReader(trunc, s.bufB, nlr)
- }
-
- tw = stat.AvailableWidth
- for _, d := range s.aDecorators {
- str := d.Decor(stat)
- stat.AvailableWidth -= runewidth.StringWidth(stripansi.Strip(str))
- s.bufA.WriteString(str)
- }
- if stat.AvailableWidth <= 0 {
- trunc := strings.NewReader(runewidth.Truncate(stripansi.Strip(s.bufA.String()), tw, "…"))
- s.bufA.Reset()
- return io.MultiReader(s.bufP, s.bufB, trunc, nlr)
- }
-
- s.filler.Fill(s.bufB, s.reqWidth, stat)
-
- return io.MultiReader(s.bufP, s.bufB, s.bufA, nlr)
-}
-
-func (s *bState) wSyncTable() [][]chan int {
- columns := make([]chan int, 0, len(s.pDecorators)+len(s.aDecorators))
- var pCount int
- for _, d := range s.pDecorators {
- if ch, ok := d.Sync(); ok {
- columns = append(columns, ch)
- pCount++
- }
- }
- var aCount int
- for _, d := range s.aDecorators {
- if ch, ok := d.Sync(); ok {
- columns = append(columns, ch)
- aCount++
- }
- }
- table := make([][]chan int, 2)
- table[0] = columns[0:pCount]
- table[1] = columns[pCount : pCount+aCount : pCount+aCount]
- return table
-}
-
-func newStatistics(tw int, s *bState) decor.Statistics {
- return decor.Statistics{
- ID: s.id,
- AvailableWidth: tw,
- Total: s.total,
- Current: s.current,
- Refill: s.refill,
- Completed: s.completeFlushed,
- }
-}
-
-func extractBaseDecorator(d decor.Decorator) decor.Decorator {
- if d, ok := d.(decor.Wrapper); ok {
- return extractBaseDecorator(d.Base())
- }
- return d
-}
-
-func ewmaIterationUpdate(done bool, s *bState, dur time.Duration) {
- if !done && !s.iterated {
- panic("increment required before ewma iteration update")
- } else {
- s.iterated = false
- }
- for _, d := range s.ewmaDecorators {
- d.EwmaUpdate(s.lastN, dur)
- }
-}
-
-func makePanicExtender(p interface{}) extFunc {
- pstr := fmt.Sprint(p)
- stack := debug.Stack()
- stackLines := bytes.Count(stack, []byte("\n"))
- return func(_ io.Reader, _ int, st decor.Statistics) (io.Reader, int) {
- mr := io.MultiReader(
- strings.NewReader(runewidth.Truncate(pstr, st.AvailableWidth, "…")),
- strings.NewReader(fmt.Sprintf("\n%#v\n", st)),
- bytes.NewReader(stack),
- )
- return mr, stackLines + 1
- }
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/bar_filler.go b/vendor/github.com/vbauerster/mpb/v5/bar_filler.go
deleted file mode 100644
index 07148bffb..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/bar_filler.go
+++ /dev/null
@@ -1,30 +0,0 @@
-package mpb
-
-import (
- "io"
-
- "github.com/vbauerster/mpb/v5/decor"
-)
-
-// BarFiller interface.
-// Bar (without decorators) renders itself by calling BarFiller's Fill method.
-//
-// `reqWidth` is requested width, which is set via:
-// func WithWidth(width int) ContainerOption
-// func BarWidth(width int) BarOption
-//
-// Default implementations can be obtained via:
-//
-// func NewBarFiller(style string, reverse bool) BarFiller
-// func NewSpinnerFiller(style []string, alignment SpinnerAlignment) BarFiller
-//
-type BarFiller interface {
- Fill(w io.Writer, reqWidth int, stat decor.Statistics)
-}
-
-// BarFillerFunc is function type adapter to convert function into BarFiller.
-type BarFillerFunc func(w io.Writer, reqWidth int, stat decor.Statistics)
-
-func (f BarFillerFunc) Fill(w io.Writer, reqWidth int, stat decor.Statistics) {
- f(w, reqWidth, stat)
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/bar_filler_bar.go b/vendor/github.com/vbauerster/mpb/v5/bar_filler_bar.go
deleted file mode 100644
index 637bd88ca..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/bar_filler_bar.go
+++ /dev/null
@@ -1,173 +0,0 @@
-package mpb
-
-import (
- "bytes"
- "io"
- "unicode/utf8"
-
- "github.com/mattn/go-runewidth"
- "github.com/vbauerster/mpb/v5/decor"
- "github.com/vbauerster/mpb/v5/internal"
-)
-
-const (
- rLeft = iota
- rFill
- rTip
- rSpace
- rRight
- rRevTip
- rRefill
-)
-
-// DefaultBarStyle is a string containing 7 runes.
-// Each rune is a building block of a progress bar.
-//
-// '1st rune' stands for left boundary rune
-//
-// '2nd rune' stands for fill rune
-//
-// '3rd rune' stands for tip rune
-//
-// '4th rune' stands for space rune
-//
-// '5th rune' stands for right boundary rune
-//
-// '6th rune' stands for reverse tip rune
-//
-// '7th rune' stands for refill rune
-//
-const DefaultBarStyle string = "[=>-]<+"
-
-type barFiller struct {
- format [][]byte
- rwidth []int
- tip []byte
- refill int64
- reverse bool
- flush func(io.Writer, *space, [][]byte)
-}
-
-type space struct {
- space []byte
- rwidth int
- count int
-}
-
-// NewBarFiller constucts mpb.BarFiller, to be used with *Progress.Add(...) *Bar method.
-func NewBarFiller(style string, reverse bool) BarFiller {
- bf := &barFiller{
- format: make([][]byte, len(DefaultBarStyle)),
- rwidth: make([]int, len(DefaultBarStyle)),
- reverse: reverse,
- }
- bf.SetStyle(style)
- return bf
-}
-
-func (s *barFiller) SetStyle(style string) {
- if !utf8.ValidString(style) {
- panic("invalid bar style")
- }
- if style == "" {
- style = DefaultBarStyle
- }
- src := make([][]byte, utf8.RuneCountInString(style))
- i := 0
- for _, r := range style {
- s.rwidth[i] = runewidth.RuneWidth(r)
- src[i] = []byte(string(r))
- i++
- }
- copy(s.format, src)
- s.SetReverse(s.reverse)
-}
-
-func (s *barFiller) SetReverse(reverse bool) {
- if reverse {
- s.tip = s.format[rRevTip]
- s.flush = reverseFlush
- } else {
- s.tip = s.format[rTip]
- s.flush = regularFlush
- }
- s.reverse = reverse
-}
-
-func (s *barFiller) Fill(w io.Writer, reqWidth int, stat decor.Statistics) {
- width := internal.WidthForBarFiller(reqWidth, stat.AvailableWidth)
-
- if brackets := s.rwidth[rLeft] + s.rwidth[rRight]; width < brackets {
- return
- } else {
- // don't count brackets as progress
- width -= brackets
- }
- w.Write(s.format[rLeft])
- defer w.Write(s.format[rRight])
-
- cwidth := int(internal.PercentageRound(stat.Total, stat.Current, width))
- space := &space{
- space: s.format[rSpace],
- rwidth: s.rwidth[rSpace],
- count: width - cwidth,
- }
-
- index, refill := 0, 0
- bb := make([][]byte, cwidth)
-
- if cwidth > 0 && cwidth != width {
- bb[index] = s.tip
- cwidth -= s.rwidth[rTip]
- index++
- }
-
- if stat.Refill > 0 {
- refill = int(internal.PercentageRound(stat.Total, int64(stat.Refill), width))
- if refill > cwidth {
- refill = cwidth
- }
- cwidth -= refill
- }
-
- for cwidth > 0 {
- bb[index] = s.format[rFill]
- cwidth -= s.rwidth[rFill]
- index++
- }
-
- for refill > 0 {
- bb[index] = s.format[rRefill]
- refill -= s.rwidth[rRefill]
- index++
- }
-
- if cwidth+refill < 0 || space.rwidth > 1 {
- buf := new(bytes.Buffer)
- s.flush(buf, space, bb[:index])
- io.WriteString(w, runewidth.Truncate(buf.String(), width, "…"))
- return
- }
-
- s.flush(w, space, bb)
-}
-
-func regularFlush(w io.Writer, space *space, bb [][]byte) {
- for i := len(bb) - 1; i >= 0; i-- {
- w.Write(bb[i])
- }
- for space.count > 0 {
- w.Write(space.space)
- space.count -= space.rwidth
- }
-}
-
-func reverseFlush(w io.Writer, space *space, bb [][]byte) {
- for space.count > 0 {
- w.Write(space.space)
- space.count -= space.rwidth
- }
- for i := 0; i < len(bb); i++ {
- w.Write(bb[i])
- }
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/bar_filler_spinner.go b/vendor/github.com/vbauerster/mpb/v5/bar_filler_spinner.go
deleted file mode 100644
index d2cb2b726..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/bar_filler_spinner.go
+++ /dev/null
@@ -1,63 +0,0 @@
-package mpb
-
-import (
- "io"
- "strings"
- "unicode/utf8"
-
- "github.com/vbauerster/mpb/v5/decor"
- "github.com/vbauerster/mpb/v5/internal"
-)
-
-// SpinnerAlignment enum.
-type SpinnerAlignment int
-
-// SpinnerAlignment kinds.
-const (
- SpinnerOnLeft SpinnerAlignment = iota
- SpinnerOnMiddle
- SpinnerOnRight
-)
-
-// DefaultSpinnerStyle is a slice of strings, which makes a spinner.
-var DefaultSpinnerStyle = []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"}
-
-type spinnerFiller struct {
- frames []string
- count uint
- alignment SpinnerAlignment
-}
-
-// NewSpinnerFiller constucts mpb.BarFiller, to be used with *Progress.Add(...) *Bar method.
-func NewSpinnerFiller(style []string, alignment SpinnerAlignment) BarFiller {
- if len(style) == 0 {
- style = DefaultSpinnerStyle
- }
- filler := &spinnerFiller{
- frames: style,
- alignment: alignment,
- }
- return filler
-}
-
-func (s *spinnerFiller) Fill(w io.Writer, reqWidth int, stat decor.Statistics) {
- width := internal.WidthForBarFiller(reqWidth, stat.AvailableWidth)
-
- frame := s.frames[s.count%uint(len(s.frames))]
- frameWidth := utf8.RuneCountInString(frame)
-
- if width < frameWidth {
- return
- }
-
- switch rest := width - frameWidth; s.alignment {
- case SpinnerOnLeft:
- io.WriteString(w, frame+strings.Repeat(" ", rest))
- case SpinnerOnMiddle:
- str := strings.Repeat(" ", rest/2) + frame + strings.Repeat(" ", rest/2+rest%2)
- io.WriteString(w, str)
- case SpinnerOnRight:
- io.WriteString(w, strings.Repeat(" ", rest)+frame)
- }
- s.count++
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/bar_option.go b/vendor/github.com/vbauerster/mpb/v5/bar_option.go
deleted file mode 100644
index e7d2e41f9..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/bar_option.go
+++ /dev/null
@@ -1,213 +0,0 @@
-package mpb
-
-import (
- "bytes"
- "io"
-
- "github.com/vbauerster/mpb/v5/decor"
-)
-
-// BarOption is a function option which changes the default behavior of a bar.
-type BarOption func(*bState)
-
-func (s *bState) addDecorators(dest *[]decor.Decorator, decorators ...decor.Decorator) {
- type mergeWrapper interface {
- MergeUnwrap() []decor.Decorator
- }
- for _, decorator := range decorators {
- if mw, ok := decorator.(mergeWrapper); ok {
- *dest = append(*dest, mw.MergeUnwrap()...)
- }
- *dest = append(*dest, decorator)
- }
-}
-
-// AppendDecorators let you inject decorators to the bar's right side.
-func AppendDecorators(decorators ...decor.Decorator) BarOption {
- return func(s *bState) {
- s.addDecorators(&s.aDecorators, decorators...)
- }
-}
-
-// PrependDecorators let you inject decorators to the bar's left side.
-func PrependDecorators(decorators ...decor.Decorator) BarOption {
- return func(s *bState) {
- s.addDecorators(&s.pDecorators, decorators...)
- }
-}
-
-// BarID sets bar id.
-func BarID(id int) BarOption {
- return func(s *bState) {
- s.id = id
- }
-}
-
-// BarWidth sets bar width independent of the container.
-func BarWidth(width int) BarOption {
- return func(s *bState) {
- s.reqWidth = width
- }
-}
-
-// BarQueueAfter queues this (being constructed) bar to relplace
-// runningBar after it has been completed.
-func BarQueueAfter(runningBar *Bar) BarOption {
- if runningBar == nil {
- return nil
- }
- return func(s *bState) {
- s.runningBar = runningBar
- }
-}
-
-// BarRemoveOnComplete removes both bar's filler and its decorators
-// on complete event.
-func BarRemoveOnComplete() BarOption {
- return func(s *bState) {
- s.dropOnComplete = true
- }
-}
-
-// BarFillerClearOnComplete clears bar's filler on complete event.
-// It's shortcut for BarFillerOnComplete("").
-func BarFillerClearOnComplete() BarOption {
- return BarFillerOnComplete("")
-}
-
-// BarFillerOnComplete replaces bar's filler with message, on complete event.
-func BarFillerOnComplete(message string) BarOption {
- return BarFillerMiddleware(func(base BarFiller) BarFiller {
- return BarFillerFunc(func(w io.Writer, reqWidth int, st decor.Statistics) {
- if st.Completed {
- io.WriteString(w, message)
- } else {
- base.Fill(w, reqWidth, st)
- }
- })
- })
-}
-
-// BarFillerMiddleware provides a way to augment default BarFiller.
-func BarFillerMiddleware(middle func(BarFiller) BarFiller) BarOption {
- return func(s *bState) {
- s.middleware = middle
- }
-}
-
-// BarPriority sets bar's priority. Zero is highest priority, i.e. bar
-// will be on top. If `BarReplaceOnComplete` option is supplied, this
-// option is ignored.
-func BarPriority(priority int) BarOption {
- return func(s *bState) {
- s.priority = priority
- }
-}
-
-// BarExtender is an option to extend bar to the next new line, with
-// arbitrary output.
-func BarExtender(filler BarFiller) BarOption {
- if filler == nil {
- return nil
- }
- return func(s *bState) {
- s.extender = makeExtFunc(filler)
- }
-}
-
-func makeExtFunc(filler BarFiller) extFunc {
- buf := new(bytes.Buffer)
- return func(r io.Reader, reqWidth int, st decor.Statistics) (io.Reader, int) {
- filler.Fill(buf, reqWidth, st)
- return io.MultiReader(r, buf), bytes.Count(buf.Bytes(), []byte("\n"))
- }
-}
-
-// BarFillerTrim bar filler is rendered with leading and trailing space
-// like ' [===] ' by default. With this option leading and trailing
-// space will be removed.
-func BarFillerTrim() BarOption {
- return func(s *bState) {
- s.trimSpace = true
- }
-}
-
-// TrimSpace is an alias to BarFillerTrim.
-func TrimSpace() BarOption {
- return BarFillerTrim()
-}
-
-// BarStyle overrides mpb.DefaultBarStyle which is "[=>-]<+".
-// It's ok to pass string containing just 5 runes, for example "╢▌▌░╟",
-// if you don't need to override '<' (reverse tip) and '+' (refill rune).
-func BarStyle(style string) BarOption {
- if style == "" {
- return nil
- }
- type styleSetter interface {
- SetStyle(string)
- }
- return func(s *bState) {
- if t, ok := s.filler.(styleSetter); ok {
- t.SetStyle(style)
- }
- }
-}
-
-// BarNoPop disables bar pop out of container. Effective when
-// PopCompletedMode of container is enabled.
-func BarNoPop() BarOption {
- return func(s *bState) {
- s.noPop = true
- }
-}
-
-// BarReverse reverse mode, bar will progress from right to left.
-func BarReverse() BarOption {
- type revSetter interface {
- SetReverse(bool)
- }
- return func(s *bState) {
- if t, ok := s.filler.(revSetter); ok {
- t.SetReverse(true)
- }
- }
-}
-
-// SpinnerStyle sets custom spinner style.
-// Effective when Filler type is spinner.
-func SpinnerStyle(frames []string) BarOption {
- if len(frames) == 0 {
- return nil
- }
- chk := func(filler BarFiller) (interface{}, bool) {
- t, ok := filler.(*spinnerFiller)
- return t, ok
- }
- cb := func(t interface{}) {
- t.(*spinnerFiller).frames = frames
- }
- return MakeFillerTypeSpecificBarOption(chk, cb)
-}
-
-// MakeFillerTypeSpecificBarOption makes BarOption specific to Filler's
-// actual type. If you implement your own Filler, so most probably
-// you'll need this. See BarStyle or SpinnerStyle for example.
-func MakeFillerTypeSpecificBarOption(
- typeChecker func(BarFiller) (interface{}, bool),
- cb func(interface{}),
-) BarOption {
- return func(s *bState) {
- if t, ok := typeChecker(s.filler); ok {
- cb(t)
- }
- }
-}
-
-// BarOptOn returns option when condition evaluates to true.
-func BarOptOn(option BarOption, condition func() bool) BarOption {
- if condition() {
- return option
- }
- return nil
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/container_option.go b/vendor/github.com/vbauerster/mpb/v5/container_option.go
deleted file mode 100644
index fac59e436..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/container_option.go
+++ /dev/null
@@ -1,102 +0,0 @@
-package mpb
-
-import (
- "io"
- "io/ioutil"
- "sync"
- "time"
-)
-
-// ContainerOption is a function option which changes the default
-// behavior of progress container, if passed to mpb.New(...ContainerOption).
-type ContainerOption func(*pState)
-
-// WithWaitGroup provides means to have a single joint point. If
-// *sync.WaitGroup is provided, you can safely call just p.Wait()
-// without calling Wait() on provided *sync.WaitGroup. Makes sense
-// when there are more than one bar to render.
-func WithWaitGroup(wg *sync.WaitGroup) ContainerOption {
- return func(s *pState) {
- s.uwg = wg
- }
-}
-
-// WithWidth sets container width. If not set underlying bars will
-// occupy whole term width.
-func WithWidth(width int) ContainerOption {
- return func(s *pState) {
- s.reqWidth = width
- }
-}
-
-// WithRefreshRate overrides default 120ms refresh rate.
-func WithRefreshRate(d time.Duration) ContainerOption {
- return func(s *pState) {
- s.rr = d
- }
-}
-
-// WithManualRefresh disables internal auto refresh time.Ticker.
-// Refresh will occur upon receive value from provided ch.
-func WithManualRefresh(ch <-chan time.Time) ContainerOption {
- return func(s *pState) {
- s.refreshSrc = ch
- }
-}
-
-// WithRenderDelay delays rendering. By default rendering starts as
-// soon as bar is added, with this option it's possible to delay
-// rendering process by keeping provided chan unclosed. In other words
-// rendering will start as soon as provided chan is closed.
-func WithRenderDelay(ch <-chan struct{}) ContainerOption {
- return func(s *pState) {
- s.renderDelay = ch
- }
-}
-
-// WithShutdownNotifier provided chanel will be closed, after all bars
-// have been rendered.
-func WithShutdownNotifier(ch chan struct{}) ContainerOption {
- return func(s *pState) {
- s.shutdownNotifier = ch
- }
-}
-
-// WithOutput overrides default os.Stdout output. Setting it to nil
-// will effectively disable auto refresh rate and discard any output,
-// useful if you want to disable progress bars with little overhead.
-func WithOutput(w io.Writer) ContainerOption {
- return func(s *pState) {
- if w == nil {
- s.refreshSrc = make(chan time.Time)
- s.output = ioutil.Discard
- return
- }
- s.output = w
- }
-}
-
-// WithDebugOutput sets debug output.
-func WithDebugOutput(w io.Writer) ContainerOption {
- if w == nil {
- return nil
- }
- return func(s *pState) {
- s.debugOut = w
- }
-}
-
-// PopCompletedMode will pop and stop rendering completed bars.
-func PopCompletedMode() ContainerOption {
- return func(s *pState) {
- s.popCompleted = true
- }
-}
-
-// ContainerOptOn returns option when condition evaluates to true.
-func ContainerOptOn(option ContainerOption, condition func() bool) ContainerOption {
- if condition() {
- return option
- }
- return nil
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/util_bsd.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/util_bsd.go
deleted file mode 100644
index 4e3564ece..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/cwriter/util_bsd.go
+++ /dev/null
@@ -1,7 +0,0 @@
-// +build darwin dragonfly freebsd netbsd openbsd
-
-package cwriter
-
-import "golang.org/x/sys/unix"
-
-const ioctlReadTermios = unix.TIOCGETA
diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/util_linux.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/util_linux.go
deleted file mode 100644
index 253f12dd2..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/cwriter/util_linux.go
+++ /dev/null
@@ -1,7 +0,0 @@
-// +build aix linux
-
-package cwriter
-
-import "golang.org/x/sys/unix"
-
-const ioctlReadTermios = unix.TCGETS
diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/util_solaris.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/util_solaris.go
deleted file mode 100644
index 4b29ff5c0..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/cwriter/util_solaris.go
+++ /dev/null
@@ -1,7 +0,0 @@
-// +build solaris
-
-package cwriter
-
-import "golang.org/x/sys/unix"
-
-const ioctlReadTermios = unix.TCGETA
diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/writer.go
deleted file mode 100644
index 6f57875c6..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer.go
+++ /dev/null
@@ -1,84 +0,0 @@
-package cwriter
-
-import (
- "bytes"
- "errors"
- "io"
- "os"
- "strconv"
-)
-
-// NotATTY not a TeleTYpewriter error.
-var NotATTY = errors.New("not a terminal")
-
-// http://ascii-table.com/ansi-escape-sequences.php
-const (
- escOpen = "\x1b["
- cuuAndEd = "A\x1b[J"
-)
-
-// Writer is a buffered the writer that updates the terminal. The
-// contents of writer will be flushed when Flush is called.
-type Writer struct {
- out io.Writer
- buf bytes.Buffer
- lineCount int
- fd int
- isTerminal bool
-}
-
-// New returns a new Writer with defaults.
-func New(out io.Writer) *Writer {
- w := &Writer{out: out}
- if f, ok := out.(*os.File); ok {
- w.fd = int(f.Fd())
- w.isTerminal = IsTerminal(w.fd)
- }
- return w
-}
-
-// Flush flushes the underlying buffer.
-func (w *Writer) Flush(lineCount int) (err error) {
- // some terminals interpret clear 0 lines as clear 1
- if w.lineCount > 0 {
- err = w.clearLines()
- if err != nil {
- return
- }
- }
- w.lineCount = lineCount
- _, err = w.buf.WriteTo(w.out)
- return
-}
-
-// Write appends the contents of p to the underlying buffer.
-func (w *Writer) Write(p []byte) (n int, err error) {
- return w.buf.Write(p)
-}
-
-// WriteString writes string to the underlying buffer.
-func (w *Writer) WriteString(s string) (n int, err error) {
- return w.buf.WriteString(s)
-}
-
-// ReadFrom reads from the provided io.Reader and writes to the
-// underlying buffer.
-func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) {
- return w.buf.ReadFrom(r)
-}
-
-// GetWidth returns width of underlying terminal.
-func (w *Writer) GetWidth() (int, error) {
- if !w.isTerminal {
- return -1, NotATTY
- }
- tw, _, err := GetSize(w.fd)
- return tw, err
-}
-
-func (w *Writer) ansiCuuAndEd() (err error) {
- buf := make([]byte, 8)
- buf = strconv.AppendInt(buf[:copy(buf, escOpen)], int64(w.lineCount), 10)
- _, err = w.out.Write(append(buf, cuuAndEd...))
- return
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_posix.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_posix.go
deleted file mode 100644
index f54a5d06b..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_posix.go
+++ /dev/null
@@ -1,26 +0,0 @@
-// +build !windows
-
-package cwriter
-
-import (
- "golang.org/x/sys/unix"
-)
-
-func (w *Writer) clearLines() error {
- return w.ansiCuuAndEd()
-}
-
-// GetSize returns the dimensions of the given terminal.
-func GetSize(fd int) (width, height int, err error) {
- ws, err := unix.IoctlGetWinsize(fd, unix.TIOCGWINSZ)
- if err != nil {
- return -1, -1, err
- }
- return int(ws.Col), int(ws.Row), nil
-}
-
-// IsTerminal returns whether the given file descriptor is a terminal.
-func IsTerminal(fd int) bool {
- _, err := unix.IoctlGetTermios(fd, ioctlReadTermios)
- return err == nil
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_windows.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_windows.go
deleted file mode 100644
index 1a69c81ac..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_windows.go
+++ /dev/null
@@ -1,73 +0,0 @@
-// +build windows
-
-package cwriter
-
-import (
- "unsafe"
-
- "golang.org/x/sys/windows"
-)
-
-var kernel32 = windows.NewLazySystemDLL("kernel32.dll")
-
-var (
- procSetConsoleCursorPosition = kernel32.NewProc("SetConsoleCursorPosition")
- procFillConsoleOutputCharacter = kernel32.NewProc("FillConsoleOutputCharacterW")
-)
-
-func (w *Writer) clearLines() error {
- if !w.isTerminal {
- // hope it's cygwin or similar
- return w.ansiCuuAndEd()
- }
-
- var info windows.ConsoleScreenBufferInfo
- if err := windows.GetConsoleScreenBufferInfo(windows.Handle(w.fd), &info); err != nil {
- return err
- }
-
- info.CursorPosition.Y -= int16(w.lineCount)
- if info.CursorPosition.Y < 0 {
- info.CursorPosition.Y = 0
- }
- _, _, _ = procSetConsoleCursorPosition.Call(
- uintptr(w.fd),
- uintptr(uint32(uint16(info.CursorPosition.Y))<<16|uint32(uint16(info.CursorPosition.X))),
- )
-
- // clear the lines
- cursor := &windows.Coord{
- X: info.Window.Left,
- Y: info.CursorPosition.Y,
- }
- count := uint32(info.Size.X) * uint32(w.lineCount)
- _, _, _ = procFillConsoleOutputCharacter.Call(
- uintptr(w.fd),
- uintptr(' '),
- uintptr(count),
- *(*uintptr)(unsafe.Pointer(cursor)),
- uintptr(unsafe.Pointer(new(uint32))),
- )
- return nil
-}
-
-// GetSize returns the visible dimensions of the given terminal.
-//
-// These dimensions don't include any scrollback buffer height.
-func GetSize(fd int) (width, height int, err error) {
- var info windows.ConsoleScreenBufferInfo
- if err := windows.GetConsoleScreenBufferInfo(windows.Handle(fd), &info); err != nil {
- return 0, 0, err
- }
- // terminal.GetSize from crypto/ssh adds "+ 1" to both width and height:
- // https://go.googlesource.com/crypto/+/refs/heads/release-branch.go1.14/ssh/terminal/util_windows.go#75
- // but looks like this is a root cause of issue #66, so removing both "+ 1" have fixed it.
- return int(info.Window.Right - info.Window.Left), int(info.Window.Bottom - info.Window.Top), nil
-}
-
-// IsTerminal returns whether the given file descriptor is a terminal.
-func IsTerminal(fd int) bool {
- var st uint32
- err := windows.GetConsoleMode(windows.Handle(fd), &st)
- return err == nil
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/any.go b/vendor/github.com/vbauerster/mpb/v5/decor/any.go
deleted file mode 100644
index 39518f594..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/any.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package decor
-
-// Any decorator displays text, that can be changed during decorator's
-// lifetime via provided DecorFunc.
-//
-// `fn` DecorFunc callback
-//
-// `wcc` optional WC config
-//
-func Any(fn DecorFunc, wcc ...WC) Decorator {
- return &any{initWC(wcc...), fn}
-}
-
-type any struct {
- WC
- fn DecorFunc
-}
-
-func (d *any) Decor(s Statistics) string {
- return d.FormatMsg(d.fn(s))
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/counters.go b/vendor/github.com/vbauerster/mpb/v5/decor/counters.go
deleted file mode 100644
index 4a5343d41..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/counters.go
+++ /dev/null
@@ -1,243 +0,0 @@
-package decor
-
-import (
- "fmt"
- "strings"
-)
-
-const (
- _ = iota
- UnitKiB
- UnitKB
-)
-
-// CountersNoUnit is a wrapper around Counters with no unit param.
-func CountersNoUnit(pairFmt string, wcc ...WC) Decorator {
- return Counters(0, pairFmt, wcc...)
-}
-
-// CountersKibiByte is a wrapper around Counters with predefined unit
-// UnitKiB (bytes/1024).
-func CountersKibiByte(pairFmt string, wcc ...WC) Decorator {
- return Counters(UnitKiB, pairFmt, wcc...)
-}
-
-// CountersKiloByte is a wrapper around Counters with predefined unit
-// UnitKB (bytes/1000).
-func CountersKiloByte(pairFmt string, wcc ...WC) Decorator {
- return Counters(UnitKB, pairFmt, wcc...)
-}
-
-// Counters decorator with dynamic unit measure adjustment.
-//
-// `unit` one of [0|UnitKiB|UnitKB] zero for no unit
-//
-// `pairFmt` printf compatible verbs for current and total pair
-//
-// `wcc` optional WC config
-//
-// pairFmt example if unit=UnitKB:
-//
-// pairFmt="%.1f / %.1f" output: "1.0MB / 12.0MB"
-// pairFmt="% .1f / % .1f" output: "1.0 MB / 12.0 MB"
-// pairFmt="%d / %d" output: "1MB / 12MB"
-// pairFmt="% d / % d" output: "1 MB / 12 MB"
-//
-func Counters(unit int, pairFmt string, wcc ...WC) Decorator {
- producer := func(unit int, pairFmt string) DecorFunc {
- if pairFmt == "" {
- pairFmt = "%d / %d"
- } else if strings.Count(pairFmt, "%") != 2 {
- panic("expected pairFmt with exactly 2 verbs")
- }
- switch unit {
- case UnitKiB:
- return func(s Statistics) string {
- return fmt.Sprintf(pairFmt, SizeB1024(s.Current), SizeB1024(s.Total))
- }
- case UnitKB:
- return func(s Statistics) string {
- return fmt.Sprintf(pairFmt, SizeB1000(s.Current), SizeB1000(s.Total))
- }
- default:
- return func(s Statistics) string {
- return fmt.Sprintf(pairFmt, s.Current, s.Total)
- }
- }
- }
- return Any(producer(unit, pairFmt), wcc...)
-}
-
-// TotalNoUnit is a wrapper around Total with no unit param.
-func TotalNoUnit(format string, wcc ...WC) Decorator {
- return Total(0, format, wcc...)
-}
-
-// TotalKibiByte is a wrapper around Total with predefined unit
-// UnitKiB (bytes/1024).
-func TotalKibiByte(format string, wcc ...WC) Decorator {
- return Total(UnitKiB, format, wcc...)
-}
-
-// TotalKiloByte is a wrapper around Total with predefined unit
-// UnitKB (bytes/1000).
-func TotalKiloByte(format string, wcc ...WC) Decorator {
- return Total(UnitKB, format, wcc...)
-}
-
-// Total decorator with dynamic unit measure adjustment.
-//
-// `unit` one of [0|UnitKiB|UnitKB] zero for no unit
-//
-// `format` printf compatible verb for Total
-//
-// `wcc` optional WC config
-//
-// format example if unit=UnitKiB:
-//
-// format="%.1f" output: "12.0MiB"
-// format="% .1f" output: "12.0 MiB"
-// format="%d" output: "12MiB"
-// format="% d" output: "12 MiB"
-//
-func Total(unit int, format string, wcc ...WC) Decorator {
- producer := func(unit int, format string) DecorFunc {
- if format == "" {
- format = "%d"
- } else if strings.Count(format, "%") != 1 {
- panic("expected format with exactly 1 verb")
- }
-
- switch unit {
- case UnitKiB:
- return func(s Statistics) string {
- return fmt.Sprintf(format, SizeB1024(s.Total))
- }
- case UnitKB:
- return func(s Statistics) string {
- return fmt.Sprintf(format, SizeB1000(s.Total))
- }
- default:
- return func(s Statistics) string {
- return fmt.Sprintf(format, s.Total)
- }
- }
- }
- return Any(producer(unit, format), wcc...)
-}
-
-// CurrentNoUnit is a wrapper around Current with no unit param.
-func CurrentNoUnit(format string, wcc ...WC) Decorator {
- return Current(0, format, wcc...)
-}
-
-// CurrentKibiByte is a wrapper around Current with predefined unit
-// UnitKiB (bytes/1024).
-func CurrentKibiByte(format string, wcc ...WC) Decorator {
- return Current(UnitKiB, format, wcc...)
-}
-
-// CurrentKiloByte is a wrapper around Current with predefined unit
-// UnitKB (bytes/1000).
-func CurrentKiloByte(format string, wcc ...WC) Decorator {
- return Current(UnitKB, format, wcc...)
-}
-
-// Current decorator with dynamic unit measure adjustment.
-//
-// `unit` one of [0|UnitKiB|UnitKB] zero for no unit
-//
-// `format` printf compatible verb for Current
-//
-// `wcc` optional WC config
-//
-// format example if unit=UnitKiB:
-//
-// format="%.1f" output: "12.0MiB"
-// format="% .1f" output: "12.0 MiB"
-// format="%d" output: "12MiB"
-// format="% d" output: "12 MiB"
-//
-func Current(unit int, format string, wcc ...WC) Decorator {
- producer := func(unit int, format string) DecorFunc {
- if format == "" {
- format = "%d"
- } else if strings.Count(format, "%") != 1 {
- panic("expected format with exactly 1 verb")
- }
-
- switch unit {
- case UnitKiB:
- return func(s Statistics) string {
- return fmt.Sprintf(format, SizeB1024(s.Current))
- }
- case UnitKB:
- return func(s Statistics) string {
- return fmt.Sprintf(format, SizeB1000(s.Current))
- }
- default:
- return func(s Statistics) string {
- return fmt.Sprintf(format, s.Current)
- }
- }
- }
- return Any(producer(unit, format), wcc...)
-}
-
-// InvertedCurrentNoUnit is a wrapper around InvertedCurrent with no unit param.
-func InvertedCurrentNoUnit(format string, wcc ...WC) Decorator {
- return InvertedCurrent(0, format, wcc...)
-}
-
-// InvertedCurrentKibiByte is a wrapper around InvertedCurrent with predefined unit
-// UnitKiB (bytes/1024).
-func InvertedCurrentKibiByte(format string, wcc ...WC) Decorator {
- return InvertedCurrent(UnitKiB, format, wcc...)
-}
-
-// InvertedCurrentKiloByte is a wrapper around InvertedCurrent with predefined unit
-// UnitKB (bytes/1000).
-func InvertedCurrentKiloByte(format string, wcc ...WC) Decorator {
- return InvertedCurrent(UnitKB, format, wcc...)
-}
-
-// InvertedCurrent decorator with dynamic unit measure adjustment.
-//
-// `unit` one of [0|UnitKiB|UnitKB] zero for no unit
-//
-// `format` printf compatible verb for InvertedCurrent
-//
-// `wcc` optional WC config
-//
-// format example if unit=UnitKiB:
-//
-// format="%.1f" output: "12.0MiB"
-// format="% .1f" output: "12.0 MiB"
-// format="%d" output: "12MiB"
-// format="% d" output: "12 MiB"
-//
-func InvertedCurrent(unit int, format string, wcc ...WC) Decorator {
- producer := func(unit int, format string) DecorFunc {
- if format == "" {
- format = "%d"
- } else if strings.Count(format, "%") != 1 {
- panic("expected format with exactly 1 verb")
- }
-
- switch unit {
- case UnitKiB:
- return func(s Statistics) string {
- return fmt.Sprintf(format, SizeB1024(s.Total-s.Current))
- }
- case UnitKB:
- return func(s Statistics) string {
- return fmt.Sprintf(format, SizeB1000(s.Total-s.Current))
- }
- default:
- return func(s Statistics) string {
- return fmt.Sprintf(format, s.Total-s.Current)
- }
- }
- }
- return Any(producer(unit, format), wcc...)
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/decorator.go b/vendor/github.com/vbauerster/mpb/v5/decor/decorator.go
deleted file mode 100644
index e81fae367..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/decorator.go
+++ /dev/null
@@ -1,191 +0,0 @@
-package decor
-
-import (
- "fmt"
- "time"
-
- "github.com/acarl005/stripansi"
- "github.com/mattn/go-runewidth"
-)
-
-const (
- // DidentRight bit specifies identation direction.
- // |foo |b | With DidentRight
- // | foo| b| Without DidentRight
- DidentRight = 1 << iota
-
- // DextraSpace bit adds extra space, makes sense with DSyncWidth only.
- // When DidentRight bit set, the space will be added to the right,
- // otherwise to the left.
- DextraSpace
-
- // DSyncWidth bit enables same column width synchronization.
- // Effective with multiple bars only.
- DSyncWidth
-
- // DSyncWidthR is shortcut for DSyncWidth|DidentRight
- DSyncWidthR = DSyncWidth | DidentRight
-
- // DSyncSpace is shortcut for DSyncWidth|DextraSpace
- DSyncSpace = DSyncWidth | DextraSpace
-
- // DSyncSpaceR is shortcut for DSyncWidth|DextraSpace|DidentRight
- DSyncSpaceR = DSyncWidth | DextraSpace | DidentRight
-)
-
-// TimeStyle enum.
-type TimeStyle int
-
-// TimeStyle kinds.
-const (
- ET_STYLE_GO TimeStyle = iota
- ET_STYLE_HHMMSS
- ET_STYLE_HHMM
- ET_STYLE_MMSS
-)
-
-// Statistics consists of progress related statistics, that Decorator
-// may need.
-type Statistics struct {
- ID int
- AvailableWidth int
- Total int64
- Current int64
- Refill int64
- Completed bool
-}
-
-// Decorator interface.
-// Most of the time there is no need to implement this interface
-// manually, as decor package already provides a wide range of decorators
-// which implement this interface. If however built-in decorators don't
-// meet your needs, you're free to implement your own one by implementing
-// this particular interface. The easy way to go is to convert a
-// `DecorFunc` into a `Decorator` interface by using provided
-// `func Any(DecorFunc, ...WC) Decorator`.
-type Decorator interface {
- Configurator
- Synchronizer
- Decor(Statistics) string
-}
-
-// DecorFunc func type.
-// To be used with `func Any`(DecorFunc, ...WC) Decorator`.
-type DecorFunc func(Statistics) string
-
-// Synchronizer interface.
-// All decorators implement this interface implicitly. Its Sync
-// method exposes width sync channel, if DSyncWidth bit is set.
-type Synchronizer interface {
- Sync() (chan int, bool)
-}
-
-// Configurator interface.
-type Configurator interface {
- GetConf() WC
- SetConf(WC)
-}
-
-// Wrapper interface.
-// If you're implementing custom Decorator by wrapping a built-in one,
-// it is necessary to implement this interface to retain functionality
-// of built-in Decorator.
-type Wrapper interface {
- Base() Decorator
-}
-
-// EwmaDecorator interface.
-// EWMA based decorators should implement this one.
-type EwmaDecorator interface {
- EwmaUpdate(int64, time.Duration)
-}
-
-// AverageDecorator interface.
-// Average decorators should implement this interface to provide start
-// time adjustment facility, for resume-able tasks.
-type AverageDecorator interface {
- AverageAdjust(time.Time)
-}
-
-// ShutdownListener interface.
-// If decorator needs to be notified once upon bar shutdown event, so
-// this is the right interface to implement.
-type ShutdownListener interface {
- Shutdown()
-}
-
-// Global convenience instances of WC with sync width bit set.
-// To be used with multiple bars only, i.e. not effective for single bar usage.
-var (
- WCSyncWidth = WC{C: DSyncWidth}
- WCSyncWidthR = WC{C: DSyncWidthR}
- WCSyncSpace = WC{C: DSyncSpace}
- WCSyncSpaceR = WC{C: DSyncSpaceR}
-)
-
-// WC is a struct with two public fields W and C, both of int type.
-// W represents width and C represents bit set of width related config.
-// A decorator should embed WC, to enable width synchronization.
-type WC struct {
- W int
- C int
- fill func(s string, w int) string
- wsync chan int
-}
-
-// FormatMsg formats final message according to WC.W and WC.C.
-// Should be called by any Decorator implementation.
-func (wc *WC) FormatMsg(msg string) string {
- pureWidth := runewidth.StringWidth(msg)
- stripWidth := runewidth.StringWidth(stripansi.Strip(msg))
- maxCell := wc.W
- if (wc.C & DSyncWidth) != 0 {
- cellCount := stripWidth
- if (wc.C & DextraSpace) != 0 {
- cellCount++
- }
- wc.wsync <- cellCount
- maxCell = <-wc.wsync
- }
- return wc.fill(msg, maxCell+(pureWidth-stripWidth))
-}
-
-// Init initializes width related config.
-func (wc *WC) Init() WC {
- wc.fill = runewidth.FillLeft
- if (wc.C & DidentRight) != 0 {
- wc.fill = runewidth.FillRight
- }
- if (wc.C & DSyncWidth) != 0 {
- // it's deliberate choice to override wsync on each Init() call,
- // this way globals like WCSyncSpace can be reused
- wc.wsync = make(chan int)
- }
- return *wc
-}
-
-// Sync is implementation of Synchronizer interface.
-func (wc *WC) Sync() (chan int, bool) {
- if (wc.C&DSyncWidth) != 0 && wc.wsync == nil {
- panic(fmt.Sprintf("%T is not initialized", wc))
- }
- return wc.wsync, (wc.C & DSyncWidth) != 0
-}
-
-// GetConf is implementation of Configurator interface.
-func (wc *WC) GetConf() WC {
- return *wc
-}
-
-// SetConf is implementation of Configurator interface.
-func (wc *WC) SetConf(conf WC) {
- *wc = conf.Init()
-}
-
-func initWC(wcc ...WC) WC {
- var wc WC
- for _, nwc := range wcc {
- wc = nwc
- }
- return wc.Init()
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/doc.go b/vendor/github.com/vbauerster/mpb/v5/decor/doc.go
deleted file mode 100644
index 6d2614451..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/doc.go
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- Package decor provides common decorators for "github.com/vbauerster/mpb/v5" module.
-
- Some decorators returned by this package might have a closure state. It is ok to use
- decorators concurrently, unless you share the same decorator among multiple
- *mpb.Bar instances. To avoid data races, create new decorator per *mpb.Bar instance.
-
- Don't:
-
- p := mpb.New()
- name := decor.Name("bar")
- p.AddBar(100, mpb.AppendDecorators(name))
- p.AddBar(100, mpb.AppendDecorators(name))
-
- Do:
-
- p := mpb.New()
- p.AddBar(100, mpb.AppendDecorators(decor.Name("bar1")))
- p.AddBar(100, mpb.AppendDecorators(decor.Name("bar2")))
-*/
-package decor
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/elapsed.go b/vendor/github.com/vbauerster/mpb/v5/decor/elapsed.go
deleted file mode 100644
index e389f1581..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/elapsed.go
+++ /dev/null
@@ -1,35 +0,0 @@
-package decor
-
-import (
- "time"
-)
-
-// Elapsed decorator. It's wrapper of NewElapsed.
-//
-// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
-//
-// `wcc` optional WC config
-//
-func Elapsed(style TimeStyle, wcc ...WC) Decorator {
- return NewElapsed(style, time.Now(), wcc...)
-}
-
-// NewElapsed returns elapsed time decorator.
-//
-// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
-//
-// `startTime` start time
-//
-// `wcc` optional WC config
-//
-func NewElapsed(style TimeStyle, startTime time.Time, wcc ...WC) Decorator {
- var msg string
- producer := chooseTimeProducer(style)
- fn := func(s Statistics) string {
- if !s.Completed {
- msg = producer(time.Since(startTime))
- }
- return msg
- }
- return Any(fn, wcc...)
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/eta.go b/vendor/github.com/vbauerster/mpb/v5/decor/eta.go
deleted file mode 100644
index d03caa735..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/eta.go
+++ /dev/null
@@ -1,203 +0,0 @@
-package decor
-
-import (
- "fmt"
- "math"
- "time"
-
- "github.com/VividCortex/ewma"
-)
-
-// TimeNormalizer interface. Implementors could be passed into
-// MovingAverageETA, in order to affect i.e. normalize its output.
-type TimeNormalizer interface {
- Normalize(time.Duration) time.Duration
-}
-
-// TimeNormalizerFunc is function type adapter to convert function
-// into TimeNormalizer.
-type TimeNormalizerFunc func(time.Duration) time.Duration
-
-func (f TimeNormalizerFunc) Normalize(src time.Duration) time.Duration {
- return f(src)
-}
-
-// EwmaETA exponential-weighted-moving-average based ETA decorator.
-// For this decorator to work correctly you have to measure each
-// iteration's duration and pass it to the
-// *Bar.DecoratorEwmaUpdate(time.Duration) method after each increment.
-func EwmaETA(style TimeStyle, age float64, wcc ...WC) Decorator {
- var average ewma.MovingAverage
- if age == 0 {
- average = ewma.NewMovingAverage()
- } else {
- average = ewma.NewMovingAverage(age)
- }
- return MovingAverageETA(style, NewThreadSafeMovingAverage(average), nil, wcc...)
-}
-
-// MovingAverageETA decorator relies on MovingAverage implementation to calculate its average.
-//
-// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
-//
-// `average` implementation of MovingAverage interface
-//
-// `normalizer` available implementations are [FixedIntervalTimeNormalizer|MaxTolerateTimeNormalizer]
-//
-// `wcc` optional WC config
-//
-func MovingAverageETA(style TimeStyle, average ewma.MovingAverage, normalizer TimeNormalizer, wcc ...WC) Decorator {
- d := &movingAverageETA{
- WC: initWC(wcc...),
- average: average,
- normalizer: normalizer,
- producer: chooseTimeProducer(style),
- }
- return d
-}
-
-type movingAverageETA struct {
- WC
- average ewma.MovingAverage
- normalizer TimeNormalizer
- producer func(time.Duration) string
-}
-
-func (d *movingAverageETA) Decor(s Statistics) string {
- v := math.Round(d.average.Value())
- remaining := time.Duration((s.Total - s.Current) * int64(v))
- if d.normalizer != nil {
- remaining = d.normalizer.Normalize(remaining)
- }
- return d.FormatMsg(d.producer(remaining))
-}
-
-func (d *movingAverageETA) EwmaUpdate(n int64, dur time.Duration) {
- durPerItem := float64(dur) / float64(n)
- if math.IsInf(durPerItem, 0) || math.IsNaN(durPerItem) {
- return
- }
- d.average.Add(durPerItem)
-}
-
-// AverageETA decorator. It's wrapper of NewAverageETA.
-//
-// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
-//
-// `wcc` optional WC config
-//
-func AverageETA(style TimeStyle, wcc ...WC) Decorator {
- return NewAverageETA(style, time.Now(), nil, wcc...)
-}
-
-// NewAverageETA decorator with user provided start time.
-//
-// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS]
-//
-// `startTime` start time
-//
-// `normalizer` available implementations are [FixedIntervalTimeNormalizer|MaxTolerateTimeNormalizer]
-//
-// `wcc` optional WC config
-//
-func NewAverageETA(style TimeStyle, startTime time.Time, normalizer TimeNormalizer, wcc ...WC) Decorator {
- d := &averageETA{
- WC: initWC(wcc...),
- startTime: startTime,
- normalizer: normalizer,
- producer: chooseTimeProducer(style),
- }
- return d
-}
-
-type averageETA struct {
- WC
- startTime time.Time
- normalizer TimeNormalizer
- producer func(time.Duration) string
-}
-
-func (d *averageETA) Decor(s Statistics) string {
- var remaining time.Duration
- if s.Current != 0 {
- durPerItem := float64(time.Since(d.startTime)) / float64(s.Current)
- durPerItem = math.Round(durPerItem)
- remaining = time.Duration((s.Total - s.Current) * int64(durPerItem))
- if d.normalizer != nil {
- remaining = d.normalizer.Normalize(remaining)
- }
- }
- return d.FormatMsg(d.producer(remaining))
-}
-
-func (d *averageETA) AverageAdjust(startTime time.Time) {
- d.startTime = startTime
-}
-
-// MaxTolerateTimeNormalizer returns implementation of TimeNormalizer.
-func MaxTolerateTimeNormalizer(maxTolerate time.Duration) TimeNormalizer {
- var normalized time.Duration
- var lastCall time.Time
- return TimeNormalizerFunc(func(remaining time.Duration) time.Duration {
- if diff := normalized - remaining; diff <= 0 || diff > maxTolerate || remaining < time.Minute {
- normalized = remaining
- lastCall = time.Now()
- return remaining
- }
- normalized -= time.Since(lastCall)
- lastCall = time.Now()
- return normalized
- })
-}
-
-// FixedIntervalTimeNormalizer returns implementation of TimeNormalizer.
-func FixedIntervalTimeNormalizer(updInterval int) TimeNormalizer {
- var normalized time.Duration
- var lastCall time.Time
- var count int
- return TimeNormalizerFunc(func(remaining time.Duration) time.Duration {
- if count == 0 || remaining < time.Minute {
- count = updInterval
- normalized = remaining
- lastCall = time.Now()
- return remaining
- }
- count--
- normalized -= time.Since(lastCall)
- lastCall = time.Now()
- return normalized
- })
-}
-
-func chooseTimeProducer(style TimeStyle) func(time.Duration) string {
- switch style {
- case ET_STYLE_HHMMSS:
- return func(remaining time.Duration) string {
- hours := int64(remaining/time.Hour) % 60
- minutes := int64(remaining/time.Minute) % 60
- seconds := int64(remaining/time.Second) % 60
- return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
- }
- case ET_STYLE_HHMM:
- return func(remaining time.Duration) string {
- hours := int64(remaining/time.Hour) % 60
- minutes := int64(remaining/time.Minute) % 60
- return fmt.Sprintf("%02d:%02d", hours, minutes)
- }
- case ET_STYLE_MMSS:
- return func(remaining time.Duration) string {
- hours := int64(remaining/time.Hour) % 60
- minutes := int64(remaining/time.Minute) % 60
- seconds := int64(remaining/time.Second) % 60
- if hours > 0 {
- return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds)
- }
- return fmt.Sprintf("%02d:%02d", minutes, seconds)
- }
- default:
- return func(remaining time.Duration) string {
- // strip off nanoseconds
- return ((remaining / time.Second) * time.Second).String()
- }
- }
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/merge.go b/vendor/github.com/vbauerster/mpb/v5/decor/merge.go
deleted file mode 100644
index e41406a64..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/merge.go
+++ /dev/null
@@ -1,107 +0,0 @@
-package decor
-
-import (
- "strings"
-
- "github.com/acarl005/stripansi"
- "github.com/mattn/go-runewidth"
-)
-
-// Merge wraps its decorator argument with intention to sync width
-// with several decorators of another bar. Visual example:
-//
-// +----+--------+---------+--------+
-// | B1 | MERGE(D, P1, Pn) |
-// +----+--------+---------+--------+
-// | B2 | D0 | D1 | Dn |
-// +----+--------+---------+--------+
-//
-func Merge(decorator Decorator, placeholders ...WC) Decorator {
- if _, ok := decorator.Sync(); !ok || len(placeholders) == 0 {
- return decorator
- }
- md := &mergeDecorator{
- Decorator: decorator,
- wc: decorator.GetConf(),
- placeHolders: make([]*placeHolderDecorator, len(placeholders)),
- }
- decorator.SetConf(WC{})
- for i, wc := range placeholders {
- if (wc.C & DSyncWidth) == 0 {
- return decorator
- }
- md.placeHolders[i] = &placeHolderDecorator{wc.Init()}
- }
- return md
-}
-
-type mergeDecorator struct {
- Decorator
- wc WC
- placeHolders []*placeHolderDecorator
-}
-
-func (d *mergeDecorator) GetConf() WC {
- return d.wc
-}
-
-func (d *mergeDecorator) SetConf(conf WC) {
- d.wc = conf.Init()
-}
-
-func (d *mergeDecorator) MergeUnwrap() []Decorator {
- decorators := make([]Decorator, len(d.placeHolders))
- for i, ph := range d.placeHolders {
- decorators[i] = ph
- }
- return decorators
-}
-
-func (d *mergeDecorator) Sync() (chan int, bool) {
- return d.wc.Sync()
-}
-
-func (d *mergeDecorator) Base() Decorator {
- return d.Decorator
-}
-
-func (d *mergeDecorator) Decor(s Statistics) string {
- msg := d.Decorator.Decor(s)
- pureWidth := runewidth.StringWidth(msg)
- stripWidth := runewidth.StringWidth(stripansi.Strip(msg))
- cellCount := stripWidth
- if (d.wc.C & DextraSpace) != 0 {
- cellCount++
- }
-
- total := runewidth.StringWidth(d.placeHolders[0].FormatMsg(""))
- pw := (cellCount - total) / len(d.placeHolders)
- rem := (cellCount - total) % len(d.placeHolders)
-
- var diff int
- for i := 1; i < len(d.placeHolders); i++ {
- ph := d.placeHolders[i]
- width := pw - diff
- if (ph.WC.C & DextraSpace) != 0 {
- width--
- if width < 0 {
- width = 0
- }
- }
- max := runewidth.StringWidth(ph.FormatMsg(strings.Repeat(" ", width)))
- total += max
- diff = max - pw
- }
-
- d.wc.wsync <- pw + rem
- max := <-d.wc.wsync
- return d.wc.fill(msg, max+total+(pureWidth-stripWidth))
-}
-
-type placeHolderDecorator struct {
- WC
-}
-
-func (d *placeHolderDecorator) Decor(Statistics) string {
- return ""
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/moving_average.go b/vendor/github.com/vbauerster/mpb/v5/decor/moving_average.go
deleted file mode 100644
index 50ac9c393..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/moving_average.go
+++ /dev/null
@@ -1,68 +0,0 @@
-package decor
-
-import (
- "sort"
- "sync"
-
- "github.com/VividCortex/ewma"
-)
-
-type threadSafeMovingAverage struct {
- ewma.MovingAverage
- mu sync.Mutex
-}
-
-func (s *threadSafeMovingAverage) Add(value float64) {
- s.mu.Lock()
- s.MovingAverage.Add(value)
- s.mu.Unlock()
-}
-
-func (s *threadSafeMovingAverage) Value() float64 {
- s.mu.Lock()
- defer s.mu.Unlock()
- return s.MovingAverage.Value()
-}
-
-func (s *threadSafeMovingAverage) Set(value float64) {
- s.mu.Lock()
- s.MovingAverage.Set(value)
- s.mu.Unlock()
-}
-
-// NewThreadSafeMovingAverage converts provided ewma.MovingAverage
-// into thread safe ewma.MovingAverage.
-func NewThreadSafeMovingAverage(average ewma.MovingAverage) ewma.MovingAverage {
- if tsma, ok := average.(*threadSafeMovingAverage); ok {
- return tsma
- }
- return &threadSafeMovingAverage{MovingAverage: average}
-}
-
-type medianWindow [3]float64
-
-func (s *medianWindow) Len() int { return len(s) }
-func (s *medianWindow) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
-func (s *medianWindow) Less(i, j int) bool { return s[i] < s[j] }
-
-func (s *medianWindow) Add(value float64) {
- s[0], s[1] = s[1], s[2]
- s[2] = value
-}
-
-func (s *medianWindow) Value() float64 {
- tmp := *s
- sort.Sort(&tmp)
- return tmp[1]
-}
-
-func (s *medianWindow) Set(value float64) {
- for i := 0; i < len(s); i++ {
- s[i] = value
- }
-}
-
-// NewMedian is fixed last 3 samples median MovingAverage.
-func NewMedian() ewma.MovingAverage {
- return NewThreadSafeMovingAverage(new(medianWindow))
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/name.go b/vendor/github.com/vbauerster/mpb/v5/decor/name.go
deleted file mode 100644
index 3af311254..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/name.go
+++ /dev/null
@@ -1,12 +0,0 @@
-package decor
-
-// Name decorator displays text that is set once and can't be changed
-// during decorator's lifetime.
-//
-// `str` string to display
-//
-// `wcc` optional WC config
-//
-func Name(str string, wcc ...WC) Decorator {
- return Any(func(Statistics) string { return str }, wcc...)
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/on_complete.go b/vendor/github.com/vbauerster/mpb/v5/decor/on_complete.go
deleted file mode 100644
index f46b19aba..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/on_complete.go
+++ /dev/null
@@ -1,37 +0,0 @@
-package decor
-
-// OnComplete returns decorator, which wraps provided decorator, with
-// sole purpose to display provided message on complete event.
-//
-// `decorator` Decorator to wrap
-//
-// `message` message to display on complete event
-//
-func OnComplete(decorator Decorator, message string) Decorator {
- d := &onCompleteWrapper{
- Decorator: decorator,
- msg: message,
- }
- if md, ok := decorator.(*mergeDecorator); ok {
- d.Decorator, md.Decorator = md.Decorator, d
- return md
- }
- return d
-}
-
-type onCompleteWrapper struct {
- Decorator
- msg string
-}
-
-func (d *onCompleteWrapper) Decor(s Statistics) string {
- if s.Completed {
- wc := d.GetConf()
- return wc.FormatMsg(d.msg)
- }
- return d.Decorator.Decor(s)
-}
-
-func (d *onCompleteWrapper) Base() Decorator {
- return d.Decorator
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/percentage.go b/vendor/github.com/vbauerster/mpb/v5/decor/percentage.go
deleted file mode 100644
index d6314a619..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/percentage.go
+++ /dev/null
@@ -1,58 +0,0 @@
-package decor
-
-import (
- "fmt"
- "io"
- "strconv"
-
- "github.com/vbauerster/mpb/v5/internal"
-)
-
-type percentageType float64
-
-func (s percentageType) Format(st fmt.State, verb rune) {
- var prec int
- switch verb {
- case 'd':
- case 's':
- prec = -1
- default:
- if p, ok := st.Precision(); ok {
- prec = p
- } else {
- prec = 6
- }
- }
-
- io.WriteString(st, strconv.FormatFloat(float64(s), 'f', prec, 64))
-
- if st.Flag(' ') {
- io.WriteString(st, " ")
- }
- io.WriteString(st, "%")
-}
-
-// Percentage returns percentage decorator. It's a wrapper of NewPercentage.
-func Percentage(wcc ...WC) Decorator {
- return NewPercentage("% d", wcc...)
-}
-
-// NewPercentage percentage decorator with custom format string.
-//
-// format examples:
-//
-// format="%.1f" output: "1.0%"
-// format="% .1f" output: "1.0 %"
-// format="%d" output: "1%"
-// format="% d" output: "1 %"
-//
-func NewPercentage(format string, wcc ...WC) Decorator {
- if format == "" {
- format = "% d"
- }
- f := func(s Statistics) string {
- p := internal.Percentage(s.Total, s.Current, 100)
- return fmt.Sprintf(format, percentageType(p))
- }
- return Any(f, wcc...)
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/size_type.go b/vendor/github.com/vbauerster/mpb/v5/decor/size_type.go
deleted file mode 100644
index e4b974058..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/size_type.go
+++ /dev/null
@@ -1,109 +0,0 @@
-package decor
-
-import (
- "fmt"
- "io"
- "math"
- "strconv"
-)
-
-//go:generate stringer -type=SizeB1024 -trimprefix=_i
-//go:generate stringer -type=SizeB1000 -trimprefix=_
-
-const (
- _ib SizeB1024 = iota + 1
- _iKiB SizeB1024 = 1 << (iota * 10)
- _iMiB
- _iGiB
- _iTiB
-)
-
-// SizeB1024 named type, which implements fmt.Formatter interface. It
-// adjusts its value according to byte size multiple by 1024 and appends
-// appropriate size marker (KiB, MiB, GiB, TiB).
-type SizeB1024 int64
-
-func (self SizeB1024) Format(st fmt.State, verb rune) {
- var prec int
- switch verb {
- case 'd':
- case 's':
- prec = -1
- default:
- if p, ok := st.Precision(); ok {
- prec = p
- } else {
- prec = 6
- }
- }
-
- var unit SizeB1024
- switch {
- case self < _iKiB:
- unit = _ib
- case self < _iMiB:
- unit = _iKiB
- case self < _iGiB:
- unit = _iMiB
- case self < _iTiB:
- unit = _iGiB
- case self <= math.MaxInt64:
- unit = _iTiB
- }
-
- io.WriteString(st, strconv.FormatFloat(float64(self)/float64(unit), 'f', prec, 64))
-
- if st.Flag(' ') {
- io.WriteString(st, " ")
- }
- io.WriteString(st, unit.String())
-}
-
-const (
- _b SizeB1000 = 1
- _KB SizeB1000 = _b * 1000
- _MB SizeB1000 = _KB * 1000
- _GB SizeB1000 = _MB * 1000
- _TB SizeB1000 = _GB * 1000
-)
-
-// SizeB1000 named type, which implements fmt.Formatter interface. It
-// adjusts its value according to byte size multiple by 1000 and appends
-// appropriate size marker (KB, MB, GB, TB).
-type SizeB1000 int64
-
-func (self SizeB1000) Format(st fmt.State, verb rune) {
- var prec int
- switch verb {
- case 'd':
- case 's':
- prec = -1
- default:
- if p, ok := st.Precision(); ok {
- prec = p
- } else {
- prec = 6
- }
- }
-
- var unit SizeB1000
- switch {
- case self < _KB:
- unit = _b
- case self < _MB:
- unit = _KB
- case self < _GB:
- unit = _MB
- case self < _TB:
- unit = _GB
- case self <= math.MaxInt64:
- unit = _TB
- }
-
- io.WriteString(st, strconv.FormatFloat(float64(self)/float64(unit), 'f', prec, 64))
-
- if st.Flag(' ') {
- io.WriteString(st, " ")
- }
- io.WriteString(st, unit.String())
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/sizeb1000_string.go b/vendor/github.com/vbauerster/mpb/v5/decor/sizeb1000_string.go
deleted file mode 100644
index 3f32ef715..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/sizeb1000_string.go
+++ /dev/null
@@ -1,41 +0,0 @@
-// Code generated by "stringer -type=SizeB1000 -trimprefix=_"; DO NOT EDIT.
-
-package decor
-
-import "strconv"
-
-func _() {
- // An "invalid array index" compiler error signifies that the constant values have changed.
- // Re-run the stringer command to generate them again.
- var x [1]struct{}
- _ = x[_b-1]
- _ = x[_KB-1000]
- _ = x[_MB-1000000]
- _ = x[_GB-1000000000]
- _ = x[_TB-1000000000000]
-}
-
-const (
- _SizeB1000_name_0 = "b"
- _SizeB1000_name_1 = "KB"
- _SizeB1000_name_2 = "MB"
- _SizeB1000_name_3 = "GB"
- _SizeB1000_name_4 = "TB"
-)
-
-func (i SizeB1000) String() string {
- switch {
- case i == 1:
- return _SizeB1000_name_0
- case i == 1000:
- return _SizeB1000_name_1
- case i == 1000000:
- return _SizeB1000_name_2
- case i == 1000000000:
- return _SizeB1000_name_3
- case i == 1000000000000:
- return _SizeB1000_name_4
- default:
- return "SizeB1000(" + strconv.FormatInt(int64(i), 10) + ")"
- }
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/sizeb1024_string.go b/vendor/github.com/vbauerster/mpb/v5/decor/sizeb1024_string.go
deleted file mode 100644
index 9fca66cc7..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/sizeb1024_string.go
+++ /dev/null
@@ -1,41 +0,0 @@
-// Code generated by "stringer -type=SizeB1024 -trimprefix=_i"; DO NOT EDIT.
-
-package decor
-
-import "strconv"
-
-func _() {
- // An "invalid array index" compiler error signifies that the constant values have changed.
- // Re-run the stringer command to generate them again.
- var x [1]struct{}
- _ = x[_ib-1]
- _ = x[_iKiB-1024]
- _ = x[_iMiB-1048576]
- _ = x[_iGiB-1073741824]
- _ = x[_iTiB-1099511627776]
-}
-
-const (
- _SizeB1024_name_0 = "b"
- _SizeB1024_name_1 = "KiB"
- _SizeB1024_name_2 = "MiB"
- _SizeB1024_name_3 = "GiB"
- _SizeB1024_name_4 = "TiB"
-)
-
-func (i SizeB1024) String() string {
- switch {
- case i == 1:
- return _SizeB1024_name_0
- case i == 1024:
- return _SizeB1024_name_1
- case i == 1048576:
- return _SizeB1024_name_2
- case i == 1073741824:
- return _SizeB1024_name_3
- case i == 1099511627776:
- return _SizeB1024_name_4
- default:
- return "SizeB1024(" + strconv.FormatInt(int64(i), 10) + ")"
- }
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/speed.go b/vendor/github.com/vbauerster/mpb/v5/decor/speed.go
deleted file mode 100644
index 634edabfd..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/speed.go
+++ /dev/null
@@ -1,171 +0,0 @@
-package decor
-
-import (
- "fmt"
- "io"
- "math"
- "time"
-
- "github.com/VividCortex/ewma"
-)
-
-// FmtAsSpeed adds "/s" to the end of the input formatter. To be
-// used with SizeB1000 or SizeB1024 types, for example:
-//
-// fmt.Printf("%.1f", FmtAsSpeed(SizeB1024(2048)))
-//
-func FmtAsSpeed(input fmt.Formatter) fmt.Formatter {
- return &speedFormatter{input}
-}
-
-type speedFormatter struct {
- fmt.Formatter
-}
-
-func (self *speedFormatter) Format(st fmt.State, verb rune) {
- self.Formatter.Format(st, verb)
- io.WriteString(st, "/s")
-}
-
-// EwmaSpeed exponential-weighted-moving-average based speed decorator.
-// For this decorator to work correctly you have to measure each
-// iteration's duration and pass it to the
-// *Bar.DecoratorEwmaUpdate(time.Duration) method after each increment.
-func EwmaSpeed(unit int, format string, age float64, wcc ...WC) Decorator {
- var average ewma.MovingAverage
- if age == 0 {
- average = ewma.NewMovingAverage()
- } else {
- average = ewma.NewMovingAverage(age)
- }
- return MovingAverageSpeed(unit, format, NewThreadSafeMovingAverage(average), wcc...)
-}
-
-// MovingAverageSpeed decorator relies on MovingAverage implementation
-// to calculate its average.
-//
-// `unit` one of [0|UnitKiB|UnitKB] zero for no unit
-//
-// `format` printf compatible verb for value, like "%f" or "%d"
-//
-// `average` MovingAverage implementation
-//
-// `wcc` optional WC config
-//
-// format examples:
-//
-// unit=UnitKiB, format="%.1f" output: "1.0MiB/s"
-// unit=UnitKiB, format="% .1f" output: "1.0 MiB/s"
-// unit=UnitKB, format="%.1f" output: "1.0MB/s"
-// unit=UnitKB, format="% .1f" output: "1.0 MB/s"
-//
-func MovingAverageSpeed(unit int, format string, average ewma.MovingAverage, wcc ...WC) Decorator {
- if format == "" {
- format = "%.0f"
- }
- d := &movingAverageSpeed{
- WC: initWC(wcc...),
- average: average,
- producer: chooseSpeedProducer(unit, format),
- }
- return d
-}
-
-type movingAverageSpeed struct {
- WC
- producer func(float64) string
- average ewma.MovingAverage
- msg string
-}
-
-func (d *movingAverageSpeed) Decor(s Statistics) string {
- if !s.Completed {
- var speed float64
- if v := d.average.Value(); v > 0 {
- speed = 1 / v
- }
- d.msg = d.producer(speed * 1e9)
- }
- return d.FormatMsg(d.msg)
-}
-
-func (d *movingAverageSpeed) EwmaUpdate(n int64, dur time.Duration) {
- durPerByte := float64(dur) / float64(n)
- if math.IsInf(durPerByte, 0) || math.IsNaN(durPerByte) {
- return
- }
- d.average.Add(durPerByte)
-}
-
-// AverageSpeed decorator with dynamic unit measure adjustment. It's
-// a wrapper of NewAverageSpeed.
-func AverageSpeed(unit int, format string, wcc ...WC) Decorator {
- return NewAverageSpeed(unit, format, time.Now(), wcc...)
-}
-
-// NewAverageSpeed decorator with dynamic unit measure adjustment and
-// user provided start time.
-//
-// `unit` one of [0|UnitKiB|UnitKB] zero for no unit
-//
-// `format` printf compatible verb for value, like "%f" or "%d"
-//
-// `startTime` start time
-//
-// `wcc` optional WC config
-//
-// format examples:
-//
-// unit=UnitKiB, format="%.1f" output: "1.0MiB/s"
-// unit=UnitKiB, format="% .1f" output: "1.0 MiB/s"
-// unit=UnitKB, format="%.1f" output: "1.0MB/s"
-// unit=UnitKB, format="% .1f" output: "1.0 MB/s"
-//
-func NewAverageSpeed(unit int, format string, startTime time.Time, wcc ...WC) Decorator {
- if format == "" {
- format = "%.0f"
- }
- d := &averageSpeed{
- WC: initWC(wcc...),
- startTime: startTime,
- producer: chooseSpeedProducer(unit, format),
- }
- return d
-}
-
-type averageSpeed struct {
- WC
- startTime time.Time
- producer func(float64) string
- msg string
-}
-
-func (d *averageSpeed) Decor(s Statistics) string {
- if !s.Completed {
- speed := float64(s.Current) / float64(time.Since(d.startTime))
- d.msg = d.producer(speed * 1e9)
- }
-
- return d.FormatMsg(d.msg)
-}
-
-func (d *averageSpeed) AverageAdjust(startTime time.Time) {
- d.startTime = startTime
-}
-
-func chooseSpeedProducer(unit int, format string) func(float64) string {
- switch unit {
- case UnitKiB:
- return func(speed float64) string {
- return fmt.Sprintf(format, FmtAsSpeed(SizeB1024(math.Round(speed))))
- }
- case UnitKB:
- return func(speed float64) string {
- return fmt.Sprintf(format, FmtAsSpeed(SizeB1000(math.Round(speed))))
- }
- default:
- return func(speed float64) string {
- return fmt.Sprintf(format, speed)
- }
- }
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/spinner.go b/vendor/github.com/vbauerster/mpb/v5/decor/spinner.go
deleted file mode 100644
index 6871639db..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/decor/spinner.go
+++ /dev/null
@@ -1,21 +0,0 @@
-package decor
-
-var defaultSpinnerStyle = []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"}
-
-// Spinner returns spinner decorator.
-//
-// `frames` spinner frames, if nil or len==0, default is used
-//
-// `wcc` optional WC config
-func Spinner(frames []string, wcc ...WC) Decorator {
- if len(frames) == 0 {
- frames = defaultSpinnerStyle
- }
- var count uint
- f := func(s Statistics) string {
- frame := frames[count%uint(len(frames))]
- count++
- return frame
- }
- return Any(f, wcc...)
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/doc.go b/vendor/github.com/vbauerster/mpb/v5/doc.go
deleted file mode 100644
index 5ada71774..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/doc.go
+++ /dev/null
@@ -1,2 +0,0 @@
-// Package mpb is a library for rendering progress bars in terminal applications.
-package mpb
diff --git a/vendor/github.com/vbauerster/mpb/v5/go.mod b/vendor/github.com/vbauerster/mpb/v5/go.mod
deleted file mode 100644
index e80d1a10d..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/go.mod
+++ /dev/null
@@ -1,10 +0,0 @@
-module github.com/vbauerster/mpb/v5
-
-require (
- github.com/VividCortex/ewma v1.1.1
- github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d
- github.com/mattn/go-runewidth v0.0.9
- golang.org/x/sys v0.0.0-20201218084310-7d0127a74742
-)
-
-go 1.14
diff --git a/vendor/github.com/vbauerster/mpb/v5/go.sum b/vendor/github.com/vbauerster/mpb/v5/go.sum
deleted file mode 100644
index 62cc10af0..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/go.sum
+++ /dev/null
@@ -1,8 +0,0 @@
-github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM=
-github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA=
-github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8=
-github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo=
-github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0=
-github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI=
-golang.org/x/sys v0.0.0-20201218084310-7d0127a74742 h1:+CBz4km/0KPU3RGTwARGh/noP3bEwtHcq+0YcBQM2JQ=
-golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
diff --git a/vendor/github.com/vbauerster/mpb/v5/internal/percentage.go b/vendor/github.com/vbauerster/mpb/v5/internal/percentage.go
deleted file mode 100644
index e321e0a6b..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/internal/percentage.go
+++ /dev/null
@@ -1,18 +0,0 @@
-package internal
-
-import "math"
-
-// Percentage is a helper function, to calculate percentage.
-func Percentage(total, current int64, width int) float64 {
- if total <= 0 {
- return 0
- }
- if current >= total {
- return float64(width)
- }
- return float64(int64(width)*current) / float64(total)
-}
-
-func PercentageRound(total, current int64, width int) float64 {
- return math.Round(Percentage(total, current, width))
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/internal/width.go b/vendor/github.com/vbauerster/mpb/v5/internal/width.go
deleted file mode 100644
index 35d528983..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/internal/width.go
+++ /dev/null
@@ -1,8 +0,0 @@
-package internal
-
-func WidthForBarFiller(reqWidth, available int) int {
- if reqWidth <= 0 || reqWidth >= available {
- return available
- }
- return reqWidth
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/priority_queue.go b/vendor/github.com/vbauerster/mpb/v5/priority_queue.go
deleted file mode 100644
index 29d9bd5a8..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/priority_queue.go
+++ /dev/null
@@ -1,32 +0,0 @@
-package mpb
-
-// A priorityQueue implements heap.Interface
-type priorityQueue []*Bar
-
-func (pq priorityQueue) Len() int { return len(pq) }
-
-func (pq priorityQueue) Less(i, j int) bool {
- return pq[i].priority < pq[j].priority
-}
-
-func (pq priorityQueue) Swap(i, j int) {
- pq[i], pq[j] = pq[j], pq[i]
- pq[i].index = i
- pq[j].index = j
-}
-
-func (pq *priorityQueue) Push(x interface{}) {
- s := *pq
- bar := x.(*Bar)
- bar.index = len(s)
- s = append(s, bar)
- *pq = s
-}
-
-func (pq *priorityQueue) Pop() interface{} {
- s := *pq
- *pq = s[0 : len(s)-1]
- bar := s[len(s)-1]
- bar.index = -1 // for safety
- return bar
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/progress.go b/vendor/github.com/vbauerster/mpb/v5/progress.go
deleted file mode 100644
index fb66ce05d..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/progress.go
+++ /dev/null
@@ -1,390 +0,0 @@
-package mpb
-
-import (
- "bytes"
- "container/heap"
- "context"
- "fmt"
- "io"
- "io/ioutil"
- "log"
- "math"
- "os"
- "sync"
- "time"
-
- "github.com/vbauerster/mpb/v5/cwriter"
- "github.com/vbauerster/mpb/v5/decor"
-)
-
-const (
- // default RefreshRate
- prr = 120 * time.Millisecond
-)
-
-// Progress represents the container that renders Progress bars
-type Progress struct {
- ctx context.Context
- uwg *sync.WaitGroup
- cwg *sync.WaitGroup
- bwg *sync.WaitGroup
- operateState chan func(*pState)
- done chan struct{}
- refreshCh chan time.Time
- once sync.Once
- dlogger *log.Logger
-}
-
-type pState struct {
- bHeap priorityQueue
- heapUpdated bool
- pMatrix map[int][]chan int
- aMatrix map[int][]chan int
- barShutdownQueue []*Bar
-
- // following are provided/overrided by user
- idCount int
- reqWidth int
- popCompleted bool
- rr time.Duration
- uwg *sync.WaitGroup
- refreshSrc <-chan time.Time
- renderDelay <-chan struct{}
- shutdownNotifier chan struct{}
- parkedBars map[*Bar]*Bar
- output io.Writer
- debugOut io.Writer
-}
-
-// New creates new Progress container instance. It's not possible to
-// reuse instance after *Progress.Wait() method has been called.
-func New(options ...ContainerOption) *Progress {
- return NewWithContext(context.Background(), options...)
-}
-
-// NewWithContext creates new Progress container instance with provided
-// context. It's not possible to reuse instance after *Progress.Wait()
-// method has been called.
-func NewWithContext(ctx context.Context, options ...ContainerOption) *Progress {
- s := &pState{
- bHeap: priorityQueue{},
- rr: prr,
- parkedBars: make(map[*Bar]*Bar),
- output: os.Stdout,
- debugOut: ioutil.Discard,
- }
-
- for _, opt := range options {
- if opt != nil {
- opt(s)
- }
- }
-
- p := &Progress{
- ctx: ctx,
- uwg: s.uwg,
- cwg: new(sync.WaitGroup),
- bwg: new(sync.WaitGroup),
- operateState: make(chan func(*pState)),
- done: make(chan struct{}),
- dlogger: log.New(s.debugOut, "[mpb] ", log.Lshortfile),
- }
-
- p.cwg.Add(1)
- go p.serve(s, cwriter.New(s.output))
- return p
-}
-
-// AddBar creates a new progress bar and adds it to the rendering queue.
-func (p *Progress) AddBar(total int64, options ...BarOption) *Bar {
- return p.Add(total, NewBarFiller(DefaultBarStyle, false), options...)
-}
-
-// AddSpinner creates a new spinner bar and adds it to the rendering queue.
-func (p *Progress) AddSpinner(total int64, alignment SpinnerAlignment, options ...BarOption) *Bar {
- return p.Add(total, NewSpinnerFiller(DefaultSpinnerStyle, alignment), options...)
-}
-
-// Add creates a bar which renders itself by provided filler.
-// Set total to 0, if you plan to update it later.
-// Panics if *Progress instance is done, i.e. called after *Progress.Wait().
-func (p *Progress) Add(total int64, filler BarFiller, options ...BarOption) *Bar {
- if filler == nil {
- filler = BarFillerFunc(func(io.Writer, int, decor.Statistics) {})
- }
- p.bwg.Add(1)
- result := make(chan *Bar)
- select {
- case p.operateState <- func(ps *pState) {
- bs := ps.makeBarState(total, filler, options...)
- bar := newBar(p, bs)
- if bs.runningBar != nil {
- bs.runningBar.noPop = true
- ps.parkedBars[bs.runningBar] = bar
- } else {
- heap.Push(&ps.bHeap, bar)
- ps.heapUpdated = true
- }
- ps.idCount++
- result <- bar
- }:
- bar := <-result
- bar.subscribeDecorators()
- return bar
- case <-p.done:
- p.bwg.Done()
- panic(fmt.Sprintf("%T instance can't be reused after it's done!", p))
- }
-}
-
-func (p *Progress) dropBar(b *Bar) {
- select {
- case p.operateState <- func(s *pState) {
- if b.index < 0 {
- return
- }
- heap.Remove(&s.bHeap, b.index)
- s.heapUpdated = true
- }:
- case <-p.done:
- }
-}
-
-func (p *Progress) setBarPriority(b *Bar, priority int) {
- select {
- case p.operateState <- func(s *pState) {
- if b.index < 0 {
- return
- }
- b.priority = priority
- heap.Fix(&s.bHeap, b.index)
- }:
- case <-p.done:
- }
-}
-
-// UpdateBarPriority same as *Bar.SetPriority(int).
-func (p *Progress) UpdateBarPriority(b *Bar, priority int) {
- p.setBarPriority(b, priority)
-}
-
-// BarCount returns bars count
-func (p *Progress) BarCount() int {
- result := make(chan int, 1)
- select {
- case p.operateState <- func(s *pState) { result <- s.bHeap.Len() }:
- return <-result
- case <-p.done:
- return 0
- }
-}
-
-// Wait waits for all bars to complete and finally shutdowns container.
-// After this method has been called, there is no way to reuse *Progress
-// instance.
-func (p *Progress) Wait() {
- if p.uwg != nil {
- // wait for user wg
- p.uwg.Wait()
- }
-
- // wait for bars to quit, if any
- p.bwg.Wait()
-
- p.once.Do(p.shutdown)
-
- // wait for container to quit
- p.cwg.Wait()
-}
-
-func (p *Progress) shutdown() {
- close(p.done)
-}
-
-func (p *Progress) serve(s *pState, cw *cwriter.Writer) {
- defer p.cwg.Done()
-
- p.refreshCh = s.newTicker(p.done)
-
- for {
- select {
- case op := <-p.operateState:
- op(s)
- case <-p.refreshCh:
- if err := s.render(cw); err != nil {
- p.dlogger.Println(err)
- }
- case <-s.shutdownNotifier:
- if s.heapUpdated {
- if err := s.render(cw); err != nil {
- p.dlogger.Println(err)
- }
- }
- return
- }
- }
-}
-
-func (s *pState) newTicker(done <-chan struct{}) chan time.Time {
- ch := make(chan time.Time)
- if s.shutdownNotifier == nil {
- s.shutdownNotifier = make(chan struct{})
- }
- go func() {
- if s.renderDelay != nil {
- <-s.renderDelay
- }
- if s.refreshSrc == nil {
- ticker := time.NewTicker(s.rr)
- defer ticker.Stop()
- s.refreshSrc = ticker.C
- }
- for {
- select {
- case tick := <-s.refreshSrc:
- ch <- tick
- case <-done:
- close(s.shutdownNotifier)
- return
- }
- }
- }()
- return ch
-}
-
-func (s *pState) render(cw *cwriter.Writer) error {
- if s.heapUpdated {
- s.updateSyncMatrix()
- s.heapUpdated = false
- }
- syncWidth(s.pMatrix)
- syncWidth(s.aMatrix)
-
- tw, err := cw.GetWidth()
- if err != nil {
- tw = s.reqWidth
- }
- for i := 0; i < s.bHeap.Len(); i++ {
- bar := s.bHeap[i]
- go bar.render(tw)
- }
-
- return s.flush(cw)
-}
-
-func (s *pState) flush(cw *cwriter.Writer) error {
- var lineCount int
- bm := make(map[*Bar]struct{}, s.bHeap.Len())
- for s.bHeap.Len() > 0 {
- b := heap.Pop(&s.bHeap).(*Bar)
- cw.ReadFrom(<-b.frameCh)
- if b.toShutdown {
- if b.recoveredPanic != nil {
- s.barShutdownQueue = append(s.barShutdownQueue, b)
- b.toShutdown = false
- } else {
- // shutdown at next flush
- // this ensures no bar ends up with less than 100% rendered
- defer func() {
- s.barShutdownQueue = append(s.barShutdownQueue, b)
- }()
- }
- }
- lineCount += b.extendedLines + 1
- bm[b] = struct{}{}
- }
-
- for _, b := range s.barShutdownQueue {
- if parkedBar := s.parkedBars[b]; parkedBar != nil {
- parkedBar.priority = b.priority
- heap.Push(&s.bHeap, parkedBar)
- delete(s.parkedBars, b)
- b.toDrop = true
- }
- if s.popCompleted && !b.noPop {
- lineCount -= b.extendedLines + 1
- b.toDrop = true
- }
- if b.toDrop {
- delete(bm, b)
- s.heapUpdated = true
- }
- b.cancel()
- }
- s.barShutdownQueue = s.barShutdownQueue[0:0]
-
- for b := range bm {
- heap.Push(&s.bHeap, b)
- }
-
- return cw.Flush(lineCount)
-}
-
-func (s *pState) updateSyncMatrix() {
- s.pMatrix = make(map[int][]chan int)
- s.aMatrix = make(map[int][]chan int)
- for i := 0; i < s.bHeap.Len(); i++ {
- bar := s.bHeap[i]
- table := bar.wSyncTable()
- pRow, aRow := table[0], table[1]
-
- for i, ch := range pRow {
- s.pMatrix[i] = append(s.pMatrix[i], ch)
- }
-
- for i, ch := range aRow {
- s.aMatrix[i] = append(s.aMatrix[i], ch)
- }
- }
-}
-
-func (s *pState) makeBarState(total int64, filler BarFiller, options ...BarOption) *bState {
- bs := &bState{
- id: s.idCount,
- priority: s.idCount,
- reqWidth: s.reqWidth,
- total: total,
- filler: filler,
- extender: func(r io.Reader, _ int, _ decor.Statistics) (io.Reader, int) { return r, 0 },
- debugOut: s.debugOut,
- }
-
- for _, opt := range options {
- if opt != nil {
- opt(bs)
- }
- }
-
- if bs.middleware != nil {
- bs.filler = bs.middleware(filler)
- bs.middleware = nil
- }
-
- if s.popCompleted && !bs.noPop {
- bs.priority = -(math.MaxInt32 - s.idCount)
- }
-
- bs.bufP = bytes.NewBuffer(make([]byte, 0, 128))
- bs.bufB = bytes.NewBuffer(make([]byte, 0, 256))
- bs.bufA = bytes.NewBuffer(make([]byte, 0, 128))
-
- return bs
-}
-
-func syncWidth(matrix map[int][]chan int) {
- for _, column := range matrix {
- go maxWidthDistributor(column)
- }
-}
-
-var maxWidthDistributor = func(column []chan int) {
- var maxWidth int
- for _, ch := range column {
- if w := <-ch; w > maxWidth {
- maxWidth = w
- }
- }
- for _, ch := range column {
- ch <- maxWidth
- }
-}
diff --git a/vendor/github.com/vbauerster/mpb/v5/proxyreader.go b/vendor/github.com/vbauerster/mpb/v5/proxyreader.go
deleted file mode 100644
index 316f438d7..000000000
--- a/vendor/github.com/vbauerster/mpb/v5/proxyreader.go
+++ /dev/null
@@ -1,90 +0,0 @@
-package mpb
-
-import (
- "io"
- "io/ioutil"
- "time"
-)
-
-type proxyReader struct {
- io.ReadCloser
- bar *Bar
-}
-
-func (x *proxyReader) Read(p []byte) (int, error) {
- n, err := x.ReadCloser.Read(p)
- x.bar.IncrBy(n)
- if err == io.EOF {
- go x.bar.SetTotal(0, true)
- }
- return n, err
-}
-
-type proxyWriterTo struct {
- io.ReadCloser // *proxyReader
- wt io.WriterTo
- bar *Bar
-}
-
-func (x *proxyWriterTo) WriteTo(w io.Writer) (int64, error) {
- n, err := x.wt.WriteTo(w)
- x.bar.IncrInt64(n)
- if err == io.EOF {
- go x.bar.SetTotal(0, true)
- }
- return n, err
-}
-
-type ewmaProxyReader struct {
- io.ReadCloser // *proxyReader
- bar *Bar
- iT time.Time
-}
-
-func (x *ewmaProxyReader) Read(p []byte) (int, error) {
- n, err := x.ReadCloser.Read(p)
- if n > 0 {
- x.bar.DecoratorEwmaUpdate(time.Since(x.iT))
- x.iT = time.Now()
- }
- return n, err
-}
-
-type ewmaProxyWriterTo struct {
- io.ReadCloser // *ewmaProxyReader
- wt io.WriterTo // *proxyWriterTo
- bar *Bar
- iT time.Time
-}
-
-func (x *ewmaProxyWriterTo) WriteTo(w io.Writer) (int64, error) {
- n, err := x.wt.WriteTo(w)
- if n > 0 {
- x.bar.DecoratorEwmaUpdate(time.Since(x.iT))
- x.iT = time.Now()
- }
- return n, err
-}
-
-func newProxyReader(r io.Reader, bar *Bar) io.ReadCloser {
- rc := toReadCloser(r)
- rc = &proxyReader{rc, bar}
-
- if wt, isWriterTo := r.(io.WriterTo); bar.hasEwmaDecorators {
- now := time.Now()
- rc = &ewmaProxyReader{rc, bar, now}
- if isWriterTo {
- rc = &ewmaProxyWriterTo{rc, wt, bar, now}
- }
- } else if isWriterTo {
- rc = &proxyWriterTo{rc, wt, bar}
- }
- return rc
-}
-
-func toReadCloser(r io.Reader) io.ReadCloser {
- if rc, ok := r.(io.ReadCloser); ok {
- return rc
- }
- return ioutil.NopCloser(r)
-}