diff options
23 files changed, 567 insertions, 309 deletions
@@ -12,12 +12,12 @@ require ( github.com/containernetworking/cni v1.0.1 github.com/containernetworking/plugins v1.0.1 github.com/containers/buildah v1.23.1 - github.com/containers/common v0.46.1-0.20210928081721-32e20295f1c6 + github.com/containers/common v0.46.1-0.20211001143714-161e078e4c7f github.com/containers/conmon v2.0.20+incompatible github.com/containers/image/v5 v5.16.0 github.com/containers/ocicrypt v1.1.2 github.com/containers/psgo v1.7.1 - github.com/containers/storage v1.36.1-0.20210929132900-162a0bf730ce + github.com/containers/storage v1.37.0 github.com/coreos/go-systemd/v22 v22.3.2 github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3 github.com/cyphar/filepath-securejoin v0.2.3 @@ -222,8 +222,9 @@ github.com/containerd/imgcrypt v1.1.1/go.mod h1:xpLnwiQmEUJPvQoAapeb2SNCxz7Xr6PJ github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/nri v0.1.0/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= -github.com/containerd/stargz-snapshotter/estargz v0.8.0 h1:oA1wx8kTFfImfsT5bScbrZd8gK+WtQnn15q82Djvm0Y= github.com/containerd/stargz-snapshotter/estargz v0.8.0/go.mod h1:mwIwuwb+D8FX2t45Trwi0hmWmZm5VW7zPP/rekwhWQU= +github.com/containerd/stargz-snapshotter/estargz v0.9.0 h1:PkB6BSTfOKX23erT2GkoUKkJEcXfNcyKskIViK770v8= +github.com/containerd/stargz-snapshotter/estargz v0.9.0/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= @@ -250,8 +251,8 @@ github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNB github.com/containers/buildah v1.23.1 h1:Tpc9DsRuU+0Oofewpxb6OJVNQjCu7yloN/obUqzfDTY= github.com/containers/buildah v1.23.1/go.mod h1:4WnrN0yrA7ab0ppgunixu2WM1rlD2rG8QLJAKbEkZlQ= github.com/containers/common v0.44.2/go.mod h1:7sdP4vmI5Bm6FPFxb3lvAh1Iktb6tiO1MzjUzhxdoGo= -github.com/containers/common v0.46.1-0.20210928081721-32e20295f1c6 h1:DojkCc4a9f3WB25Fk0GDap1/OkKU9UmDLvPJyqw3TBc= -github.com/containers/common v0.46.1-0.20210928081721-32e20295f1c6/go.mod h1:L4+sJlqi+R7frlbiWBW0baPra/cH8u5ZYwbxkukw3Lk= +github.com/containers/common v0.46.1-0.20211001143714-161e078e4c7f h1:vVmx51AzWvB4/ao2zyR6s053a1leLTOh+zsOPVWQRgA= +github.com/containers/common v0.46.1-0.20211001143714-161e078e4c7f/go.mod h1:aml/OO4FmYfPbfT87rvWiCgkLzTdqO6PuZ/xXq6bPbk= github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg= github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= github.com/containers/image/v5 v5.16.0 h1:WQcNSzb7+ngS2cfynx0vUwhk+scpgiKlldVcsF8GPbI= @@ -268,8 +269,8 @@ github.com/containers/psgo v1.7.1/go.mod h1:mWGpFzW73qWFA+blhF6l7GuKzbrACkYgr/aj github.com/containers/storage v1.23.5/go.mod h1:ha26Q6ngehFNhf3AWoXldvAvwI4jFe3ETQAf/CeZPyM= github.com/containers/storage v1.35.0/go.mod h1:qzYhasQP2/V9D9XdO+vRwkHBhsBO0oznMLzzRDQ8s20= github.com/containers/storage v1.36.0/go.mod h1:vbd3SKVQNHdmU5qQI6hTEcKPxnZkGqydG4f6uwrI5a8= -github.com/containers/storage v1.36.1-0.20210929132900-162a0bf730ce h1:6YOfANEWtL7+Q4RmnAfloGLIJNtt17MEHjvlHXz0vVY= -github.com/containers/storage v1.36.1-0.20210929132900-162a0bf730ce/go.mod h1:b7OGxODIyB3XpvCSWR91lllT9fv9DXeC8yfnaUocWJU= +github.com/containers/storage v1.37.0 h1:HVhDsur6sx889ZIZ1d1kEiOzv3gsr5q0diX2VZmOdSg= +github.com/containers/storage v1.37.0/go.mod h1:kqeJeS0b7DO2ZT1nVWs0XufrmPFbgV3c+Q/45RlH6r4= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go index e997d9cce..3ef029116 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/estargz.go @@ -23,7 +23,6 @@ package estargz import ( - "archive/tar" "bufio" "bytes" "compress/gzip" @@ -42,6 +41,7 @@ import ( "github.com/containerd/stargz-snapshotter/estargz/errorutil" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" + "github.com/vbatts/tar-split/archive/tar" ) // A Reader permits random access reads from a stargz file. @@ -95,10 +95,10 @@ func WithTelemetry(telemetry *Telemetry) OpenOption { } } -// A func which takes start time and records the diff +// MeasureLatencyHook is a func which takes start time and records the diff type MeasureLatencyHook func(time.Time) -// A struct which defines telemetry hooks. By implementing these hooks you should be able to record +// Telemetry is a struct which defines telemetry hooks. By implementing these hooks you should be able to record // the latency metrics of the respective steps of estargz open operation. To be used with estargz.OpenWithTelemetry(...) type Telemetry struct { GetFooterLatency MeasureLatencyHook // measure time to get stargz footer (in milliseconds) @@ -146,7 +146,7 @@ func Open(sr *io.SectionReader, opt ...OpenOption) (*Reader, error) { fSize := d.FooterSize() fOffset := positive(int64(len(footer)) - fSize) maybeTocBytes := footer[:fOffset] - tocOffset, tocSize, err := d.ParseFooter(footer[fOffset:]) + _, tocOffset, tocSize, err := d.ParseFooter(footer[fOffset:]) if err != nil { allErr = append(allErr, err) continue @@ -187,7 +187,7 @@ func OpenFooter(sr *io.SectionReader) (tocOffset int64, footerSize int64, rErr e for _, d := range []Decompressor{new(GzipDecompressor), new(legacyGzipDecompressor)} { fSize := d.FooterSize() fOffset := positive(int64(len(footer)) - fSize) - tocOffset, _, err := d.ParseFooter(footer[fOffset:]) + _, tocOffset, _, err := d.ParseFooter(footer[fOffset:]) if err == nil { return tocOffset, fSize, err } @@ -326,6 +326,10 @@ func (r *Reader) getOrCreateDir(d string) *TOCEntry { return e } +func (r *Reader) TOCDigest() digest.Digest { + return r.tocDigest +} + // VerifyTOC checks that the TOC JSON in the passed blob matches the // passed digests and that the TOC JSON contains digests for all chunks // contained in the blob. If the verification succceeds, this function @@ -335,7 +339,12 @@ func (r *Reader) VerifyTOC(tocDigest digest.Digest) (TOCEntryVerifier, error) { if r.tocDigest != tocDigest { return nil, fmt.Errorf("invalid TOC JSON %q; want %q", r.tocDigest, tocDigest) } + return r.Verifiers() +} +// Verifiers returns TOCEntryVerifier of this chunk. Use VerifyTOC instead in most cases +// because this doesn't verify TOC. +func (r *Reader) Verifiers() (TOCEntryVerifier, error) { chunkDigestMap := make(map[int64]digest.Digest) // map from chunk offset to the chunk digest regDigestMap := make(map[int64]digest.Digest) // map from chunk offset to the reg file digest var chunkDigestMapIncomplete bool @@ -591,6 +600,11 @@ type currentCompressionWriter struct{ w *Writer } func (ccw currentCompressionWriter) Write(p []byte) (int, error) { ccw.w.diffHash.Write(p) + if ccw.w.gz == nil { + if err := ccw.w.condOpenGz(); err != nil { + return 0, err + } + } return ccw.w.gz.Write(p) } @@ -601,6 +615,25 @@ func (w *Writer) chunkSize() int { return w.ChunkSize } +// Unpack decompresses the given estargz blob and returns a ReadCloser of the tar blob. +// TOC JSON and footer are removed. +func Unpack(sr *io.SectionReader, c Decompressor) (io.ReadCloser, error) { + footerSize := c.FooterSize() + if sr.Size() < footerSize { + return nil, fmt.Errorf("blob is too small; %d < %d", sr.Size(), footerSize) + } + footerOffset := sr.Size() - footerSize + footer := make([]byte, footerSize) + if _, err := sr.ReadAt(footer, footerOffset); err != nil { + return nil, err + } + blobPayloadSize, _, _, err := c.ParseFooter(footer) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse footer") + } + return c.Reader(io.LimitReader(sr, blobPayloadSize)) +} + // NewWriter returns a new stargz writer (gzip-based) writing to w. // // The writer must be closed to write its trailing table of contents. @@ -616,7 +649,7 @@ func NewWriterLevel(w io.Writer, compressionLevel int) *Writer { return NewWriterWithCompressor(w, NewGzipCompressorWithLevel(compressionLevel)) } -// NewWriterLevel returns a new stargz writer writing to w. +// NewWriterWithCompressor returns a new stargz writer writing to w. // The compression method is configurable. // // The writer must be closed to write its trailing table of contents. @@ -696,29 +729,71 @@ func (w *Writer) condOpenGz() (err error) { // each of its contents to w. // // The input r can optionally be gzip compressed but the output will -// always be gzip compressed. +// always be compressed by the specified compressor. func (w *Writer) AppendTar(r io.Reader) error { + return w.appendTar(r, false) +} + +// AppendTarLossLess reads the tar or tar.gz file from r and appends +// each of its contents to w. +// +// The input r can optionally be gzip compressed but the output will +// always be compressed by the specified compressor. +// +// The difference of this func with AppendTar is that this writes +// the input tar stream into w without any modification (e.g. to header bytes). +// +// Note that if the input tar stream already contains TOC JSON, this returns +// error because w cannot overwrite the TOC JSON to the one generated by w without +// lossy modification. To avoid this error, if the input stream is known to be stargz/estargz, +// you shoud decompress it and remove TOC JSON in advance. +func (w *Writer) AppendTarLossLess(r io.Reader) error { + return w.appendTar(r, true) +} + +func (w *Writer) appendTar(r io.Reader, lossless bool) error { + var src io.Reader br := bufio.NewReader(r) - var tr *tar.Reader if isGzip(br) { - // NewReader can't fail if isGzip returned true. zr, _ := gzip.NewReader(br) - tr = tar.NewReader(zr) + src = zr } else { - tr = tar.NewReader(br) + src = io.Reader(br) + } + dst := currentCompressionWriter{w} + var tw *tar.Writer + if !lossless { + tw = tar.NewWriter(dst) // use tar writer only when this isn't lossless mode. + } + tr := tar.NewReader(src) + if lossless { + tr.RawAccounting = true } for { h, err := tr.Next() if err == io.EOF { + if lossless { + if remain := tr.RawBytes(); len(remain) > 0 { + // Collect the remaining null bytes. + // https://github.com/vbatts/tar-split/blob/80a436fd6164c557b131f7c59ed69bd81af69761/concept/main.go#L49-L53 + if _, err := dst.Write(remain); err != nil { + return err + } + } + } break } if err != nil { return fmt.Errorf("error reading from source tar: tar.Reader.Next: %v", err) } - if h.Name == TOCTarName { + if cleanEntryName(h.Name) == TOCTarName { // It is possible for a layer to be "stargzified" twice during the // distribution lifecycle. So we reserve "TOCTarName" here to avoid // duplicated entries in the resulting layer. + if lossless { + // We cannot handle this in lossless way. + return fmt.Errorf("existing TOC JSON is not allowed; decompress layer before append") + } continue } @@ -744,9 +819,14 @@ func (w *Writer) AppendTar(r io.Reader) error { if err := w.condOpenGz(); err != nil { return err } - tw := tar.NewWriter(currentCompressionWriter{w}) - if err := tw.WriteHeader(h); err != nil { - return err + if tw != nil { + if err := tw.WriteHeader(h); err != nil { + return err + } + } else { + if _, err := dst.Write(tr.RawBytes()); err != nil { + return err + } } switch h.Typeflag { case tar.TypeLink: @@ -808,7 +888,13 @@ func (w *Writer) AppendTar(r io.Reader) error { } teeChunk := io.TeeReader(tee, chunkDigest.Hash()) - if _, err := io.CopyN(tw, teeChunk, chunkSize); err != nil { + var out io.Writer + if tw != nil { + out = tw + } else { + out = dst + } + if _, err := io.CopyN(out, teeChunk, chunkSize); err != nil { return fmt.Errorf("error copying %q: %v", h.Name, err) } ent.ChunkDigest = chunkDigest.Digest().String() @@ -825,11 +911,18 @@ func (w *Writer) AppendTar(r io.Reader) error { if payloadDigest != nil { regFileEntry.Digest = payloadDigest.Digest().String() } - if err := tw.Flush(); err != nil { - return err + if tw != nil { + if err := tw.Flush(); err != nil { + return err + } } } - return nil + remainDest := ioutil.Discard + if lossless { + remainDest = dst // Preserve the remaining bytes in lossless mode + } + _, err := io.Copy(remainDest, src) + return err } // DiffID returns the SHA-256 of the uncompressed tar bytes. diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/go.mod b/vendor/github.com/containerd/stargz-snapshotter/estargz/go.mod index ee6b2e17f..144d022ba 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/go.mod +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/go.mod @@ -3,8 +3,9 @@ module github.com/containerd/stargz-snapshotter/estargz go 1.16 require ( - github.com/klauspost/compress v1.13.5 + github.com/klauspost/compress v1.13.6 github.com/opencontainers/go-digest v1.0.0 github.com/pkg/errors v0.9.1 + github.com/vbatts/tar-split v0.11.2 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a ) diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/go.sum b/vendor/github.com/containerd/stargz-snapshotter/estargz/go.sum index 66cd2d69c..d3c934ff8 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/go.sum +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/go.sum @@ -1,8 +1,22 @@ -github.com/klauspost/compress v1.13.5 h1:9O69jUPDcsT9fEm74W92rZL9FQY7rCdaXVneq+yyzl4= -github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= +github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= +github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go index efc435e09..88e1283d8 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/gzip.go @@ -124,31 +124,31 @@ func (gz *GzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst digest.Dig return parseTOCEStargz(r) } -func (gz *GzipDecompressor) ParseFooter(p []byte) (tocOffset, tocSize int64, err error) { +func (gz *GzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) { if len(p) != FooterSize { - return 0, 0, fmt.Errorf("invalid length %d cannot be parsed", len(p)) + return 0, 0, 0, fmt.Errorf("invalid length %d cannot be parsed", len(p)) } zr, err := gzip.NewReader(bytes.NewReader(p)) if err != nil { - return 0, 0, err + return 0, 0, 0, err } defer zr.Close() extra := zr.Header.Extra si1, si2, subfieldlen, subfield := extra[0], extra[1], extra[2:4], extra[4:] if si1 != 'S' || si2 != 'G' { - return 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2) + return 0, 0, 0, fmt.Errorf("invalid subfield IDs: %q, %q; want E, S", si1, si2) } if slen := binary.LittleEndian.Uint16(subfieldlen); slen != uint16(16+len("STARGZ")) { - return 0, 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ")) + return 0, 0, 0, fmt.Errorf("invalid length of subfield %d; want %d", slen, 16+len("STARGZ")) } if string(subfield[16:]) != "STARGZ" { - return 0, 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield") + return 0, 0, 0, fmt.Errorf("STARGZ magic string must be included in the footer subfield") } tocOffset, err = strconv.ParseInt(string(subfield[:16]), 16, 64) if err != nil { - return 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset") + return 0, 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset") } - return tocOffset, 0, nil + return tocOffset, tocOffset, 0, nil } func (gz *GzipDecompressor) FooterSize() int64 { @@ -165,27 +165,27 @@ func (gz *legacyGzipDecompressor) ParseTOC(r io.Reader) (toc *JTOC, tocDgst dige return parseTOCEStargz(r) } -func (gz *legacyGzipDecompressor) ParseFooter(p []byte) (tocOffset, tocSize int64, err error) { +func (gz *legacyGzipDecompressor) ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) { if len(p) != legacyFooterSize { - return 0, 0, fmt.Errorf("legacy: invalid length %d cannot be parsed", len(p)) + return 0, 0, 0, fmt.Errorf("legacy: invalid length %d cannot be parsed", len(p)) } zr, err := gzip.NewReader(bytes.NewReader(p)) if err != nil { - return 0, 0, errors.Wrapf(err, "legacy: failed to get footer gzip reader") + return 0, 0, 0, errors.Wrapf(err, "legacy: failed to get footer gzip reader") } defer zr.Close() extra := zr.Header.Extra if len(extra) != 16+len("STARGZ") { - return 0, 0, fmt.Errorf("legacy: invalid stargz's extra field size") + return 0, 0, 0, fmt.Errorf("legacy: invalid stargz's extra field size") } if string(extra[16:]) != "STARGZ" { - return 0, 0, fmt.Errorf("legacy: magic string STARGZ not found") + return 0, 0, 0, fmt.Errorf("legacy: magic string STARGZ not found") } tocOffset, err = strconv.ParseInt(string(extra[:16]), 16, 64) if err != nil { - return 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset") + return 0, 0, 0, errors.Wrapf(err, "legacy: failed to parse toc offset") } - return tocOffset, 0, nil + return tocOffset, tocOffset, 0, nil } func (gz *legacyGzipDecompressor) FooterSize() int64 { diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go index 020729b7e..9224e456d 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/testutil.go @@ -148,93 +148,96 @@ func testBuild(t *testing.T, controllers ...TestingController) { srcCompression := srcCompression for _, cl := range controllers { cl := cl - for _, prefix := range allowedPrefix { - prefix := prefix - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v-prefix=%q-src=%d", cl, prefix, srcCompression), func(t *testing.T) { - tarBlob := buildTarStatic(t, tt.in, prefix) - // Test divideEntries() - entries, err := sortEntries(tarBlob, nil, nil) // identical order - if err != nil { - t.Fatalf("faield to parse tar: %v", err) - } - var merged []*entry - for _, part := range divideEntries(entries, 4) { - merged = append(merged, part...) - } - if !reflect.DeepEqual(entries, merged) { - for _, e := range entries { - t.Logf("Original: %v", e.header) + for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { + srcTarFormat := srcTarFormat + for _, prefix := range allowedPrefix { + prefix := prefix + t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,src=%d,format=%s", cl, prefix, srcCompression, srcTarFormat), func(t *testing.T) { + tarBlob := buildTar(t, tt.in, prefix, srcTarFormat) + // Test divideEntries() + entries, err := sortEntries(tarBlob, nil, nil) // identical order + if err != nil { + t.Fatalf("failed to parse tar: %v", err) } - for _, e := range merged { - t.Logf("Merged: %v", e.header) + var merged []*entry + for _, part := range divideEntries(entries, 4) { + merged = append(merged, part...) + } + if !reflect.DeepEqual(entries, merged) { + for _, e := range entries { + t.Logf("Original: %v", e.header) + } + for _, e := range merged { + t.Logf("Merged: %v", e.header) + } + t.Errorf("divided entries couldn't be merged") + return } - t.Errorf("divided entries couldn't be merged") - return - } - // Prepare sample data - wantBuf := new(bytes.Buffer) - sw := NewWriterWithCompressor(wantBuf, cl) - sw.ChunkSize = tt.chunkSize - if err := sw.AppendTar(tarBlob); err != nil { - t.Fatalf("faield to append tar to want stargz: %v", err) - } - if _, err := sw.Close(); err != nil { - t.Fatalf("faield to prepare want stargz: %v", err) - } - wantData := wantBuf.Bytes() - want, err := Open(io.NewSectionReader( - bytes.NewReader(wantData), 0, int64(len(wantData))), - WithDecompressors(cl), - ) - if err != nil { - t.Fatalf("failed to parse the want stargz: %v", err) - } + // Prepare sample data + wantBuf := new(bytes.Buffer) + sw := NewWriterWithCompressor(wantBuf, cl) + sw.ChunkSize = tt.chunkSize + if err := sw.AppendTar(tarBlob); err != nil { + t.Fatalf("failed to append tar to want stargz: %v", err) + } + if _, err := sw.Close(); err != nil { + t.Fatalf("failed to prepare want stargz: %v", err) + } + wantData := wantBuf.Bytes() + want, err := Open(io.NewSectionReader( + bytes.NewReader(wantData), 0, int64(len(wantData))), + WithDecompressors(cl), + ) + if err != nil { + t.Fatalf("failed to parse the want stargz: %v", err) + } - // Prepare testing data - rc, err := Build(compressBlob(t, tarBlob, srcCompression), - WithChunkSize(tt.chunkSize), WithCompression(cl)) - if err != nil { - t.Fatalf("faield to build stargz: %v", err) - } - defer rc.Close() - gotBuf := new(bytes.Buffer) - if _, err := io.Copy(gotBuf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - gotData := gotBuf.Bytes() - got, err := Open(io.NewSectionReader( - bytes.NewReader(gotBuf.Bytes()), 0, int64(len(gotData))), - WithDecompressors(cl), - ) - if err != nil { - t.Fatalf("failed to parse the got stargz: %v", err) - } + // Prepare testing data + rc, err := Build(compressBlob(t, tarBlob, srcCompression), + WithChunkSize(tt.chunkSize), WithCompression(cl)) + if err != nil { + t.Fatalf("failed to build stargz: %v", err) + } + defer rc.Close() + gotBuf := new(bytes.Buffer) + if _, err := io.Copy(gotBuf, rc); err != nil { + t.Fatalf("failed to copy built stargz blob: %v", err) + } + gotData := gotBuf.Bytes() + got, err := Open(io.NewSectionReader( + bytes.NewReader(gotBuf.Bytes()), 0, int64(len(gotData))), + WithDecompressors(cl), + ) + if err != nil { + t.Fatalf("failed to parse the got stargz: %v", err) + } - // Check DiffID is properly calculated - rc.Close() - diffID := rc.DiffID() - wantDiffID := cl.DiffIDOf(t, gotData) - if diffID.String() != wantDiffID { - t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) - } + // Check DiffID is properly calculated + rc.Close() + diffID := rc.DiffID() + wantDiffID := cl.DiffIDOf(t, gotData) + if diffID.String() != wantDiffID { + t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) + } - // Compare as stargz - if !isSameVersion(t, cl, wantData, gotData) { - t.Errorf("built stargz hasn't same json") - return - } - if !isSameEntries(t, want, got) { - t.Errorf("built stargz isn't same as the original") - return - } + // Compare as stargz + if !isSameVersion(t, cl, wantData, gotData) { + t.Errorf("built stargz hasn't same json") + return + } + if !isSameEntries(t, want, got) { + t.Errorf("built stargz isn't same as the original") + return + } - // Compare as tar.gz - if !isSameTarGz(t, cl, wantData, gotData) { - t.Errorf("built stargz isn't same tar.gz") - return - } - }) + // Compare as tar.gz + if !isSameTarGz(t, cl, wantData, gotData) { + t.Errorf("built stargz isn't same tar.gz") + return + } + }) + } } } } @@ -526,7 +529,7 @@ func testDigestAndVerify(t *testing.T, controllers ...TestingController) { checks: []check{ checkStargzTOC, checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTarStatic(t, tarOf( + checkVerifyInvalidStargzFail(buildTar(t, tarOf( dir("test2/"), // modified ), allowedPrefix[0])), }, @@ -544,7 +547,7 @@ func testDigestAndVerify(t *testing.T, controllers ...TestingController) { checks: []check{ checkStargzTOC, checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTarStatic(t, tarOf( + checkVerifyInvalidStargzFail(buildTar(t, tarOf( file("baz.txt", ""), file("foo.txt", "M"), // modified dir("test/"), @@ -567,7 +570,7 @@ func testDigestAndVerify(t *testing.T, controllers ...TestingController) { checks: []check{ checkStargzTOC, checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTarStatic(t, tarOf( + checkVerifyInvalidStargzFail(buildTar(t, tarOf( file("baz.txt", "bazbazbazMMMbazbazbaz"), // modified file("foo.txt", "a"), dir("test/"), @@ -593,7 +596,7 @@ func testDigestAndVerify(t *testing.T, controllers ...TestingController) { checks: []check{ checkStargzTOC, checkVerifyTOC, - checkVerifyInvalidStargzFail(buildTarStatic(t, tarOf( + checkVerifyInvalidStargzFail(buildTar(t, tarOf( file("baz.txt", "bazbazbazbazbazbazbaz"), file("foo.txt", "a"), symlink("barlink", "test/bar.txt"), @@ -615,30 +618,33 @@ func testDigestAndVerify(t *testing.T, controllers ...TestingController) { cl := cl for _, prefix := range allowedPrefix { prefix := prefix - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v-prefix=%q", cl, prefix), func(t *testing.T) { - // Get original tar file and chunk digests - dgstMap := make(map[string]digest.Digest) - tarBlob := buildTarStatic(t, tt.tarInit(t, dgstMap), prefix) - - rc, err := Build(compressBlob(t, tarBlob, srcCompression), - WithChunkSize(chunkSize), WithCompression(cl)) - if err != nil { - t.Fatalf("failed to convert stargz: %v", err) - } - tocDigest := rc.TOCDigest() - defer rc.Close() - buf := new(bytes.Buffer) - if _, err := io.Copy(buf, rc); err != nil { - t.Fatalf("failed to copy built stargz blob: %v", err) - } - newStargz := buf.Bytes() - // NoPrefetchLandmark is added during `Bulid`, which is expected behaviour. - dgstMap[chunkID(NoPrefetchLandmark, 0, int64(len([]byte{landmarkContents})))] = digest.FromBytes([]byte{landmarkContents}) + for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { + srcTarFormat := srcTarFormat + t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,format=%s", cl, prefix, srcTarFormat), func(t *testing.T) { + // Get original tar file and chunk digests + dgstMap := make(map[string]digest.Digest) + tarBlob := buildTar(t, tt.tarInit(t, dgstMap), prefix, srcTarFormat) + + rc, err := Build(compressBlob(t, tarBlob, srcCompression), + WithChunkSize(chunkSize), WithCompression(cl)) + if err != nil { + t.Fatalf("failed to convert stargz: %v", err) + } + tocDigest := rc.TOCDigest() + defer rc.Close() + buf := new(bytes.Buffer) + if _, err := io.Copy(buf, rc); err != nil { + t.Fatalf("failed to copy built stargz blob: %v", err) + } + newStargz := buf.Bytes() + // NoPrefetchLandmark is added during `Bulid`, which is expected behaviour. + dgstMap[chunkID(NoPrefetchLandmark, 0, int64(len([]byte{landmarkContents})))] = digest.FromBytes([]byte{landmarkContents}) - for _, check := range tt.checks { - check(t, newStargz, tocDigest, dgstMap, cl) - } - }) + for _, check := range tt.checks { + check(t, newStargz, tocDigest, dgstMap, cl) + } + }) + } } } } @@ -1058,7 +1064,7 @@ func parseStargz(sgz *io.SectionReader, controller TestingController) (decodedJT if _, err := sgz.ReadAt(footer, sgz.Size()-fSize); err != nil { return nil, 0, errors.Wrap(err, "error reading footer") } - tocOffset, _, err := controller.ParseFooter(footer[positive(int64(len(footer))-fSize):]) + _, tocOffset, _, err := controller.ParseFooter(footer[positive(int64(len(footer))-fSize):]) if err != nil { return nil, 0, errors.Wrapf(err, "failed to parse footer") } @@ -1085,11 +1091,15 @@ func testWriteAndOpen(t *testing.T, controllers ...TestingController) { in []tarEntry want []stargzCheck wantNumGz int // expected number of streams + + wantNumGzLossLess int // expected number of streams (> 0) in lossless mode if it's different from wantNumGz + wantFailOnLossLess bool }{ { - name: "empty", - in: tarOf(), - wantNumGz: 2, // TOC + footer + name: "empty", + in: tarOf(), + wantNumGz: 2, // empty tar + TOC + footer + wantNumGzLossLess: 3, // empty tar + TOC + footer want: checks( numTOCEntries(0), ), @@ -1224,26 +1234,29 @@ func testWriteAndOpen(t *testing.T, controllers ...TestingController) { { name: "block_char_fifo", in: tarOf( - tarEntryFunc(func(w *tar.Writer, prefix string) error { + tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { return w.WriteHeader(&tar.Header{ Name: prefix + "b", Typeflag: tar.TypeBlock, Devmajor: 123, Devminor: 456, + Format: format, }) }), - tarEntryFunc(func(w *tar.Writer, prefix string) error { + tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { return w.WriteHeader(&tar.Header{ Name: prefix + "c", Typeflag: tar.TypeChar, Devmajor: 111, Devminor: 222, + Format: format, }) }), - tarEntryFunc(func(w *tar.Writer, prefix string) error { + tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { return w.WriteHeader(&tar.Header{ Name: prefix + "f", Typeflag: tar.TypeFifo, + Format: format, }) }), ), @@ -1278,6 +1291,29 @@ func testWriteAndOpen(t *testing.T, controllers ...TestingController) { hasMode("foo3/bar5", os.FileMode(0755)), ), }, + { + name: "lossy", + in: tarOf( + dir("bar/", sampleOwner), + dir("foo/", sampleOwner), + file("foo/bar.txt", content, sampleOwner), + file(TOCTarName, "dummy"), // ignored by the writer. (lossless write returns error) + ), + wantNumGz: 4, // both dirs, foo.txt alone, TOC, footer + want: checks( + numTOCEntries(3), + hasDir("bar/"), + hasDir("foo/"), + hasFileLen("foo/bar.txt", len(content)), + entryHasChildren("", "bar", "foo"), + entryHasChildren("foo", "bar.txt"), + hasChunkEntries("foo/bar.txt", 1), + hasEntryOwner("bar/", sampleOwner), + hasEntryOwner("foo/", sampleOwner), + hasEntryOwner("foo/bar.txt", sampleOwner), + ), + wantFailOnLossLess: true, + }, } for _, tt := range tests { @@ -1285,47 +1321,90 @@ func testWriteAndOpen(t *testing.T, controllers ...TestingController) { cl := cl for _, prefix := range allowedPrefix { prefix := prefix - t.Run(tt.name+"-"+fmt.Sprintf("compression=%v-prefix=%q", cl, prefix), func(t *testing.T) { - tr, cancel := buildTar(t, tt.in, prefix) - defer cancel() - var stargzBuf bytes.Buffer - w := NewWriterWithCompressor(&stargzBuf, cl) - w.ChunkSize = tt.chunkSize - if err := w.AppendTar(tr); err != nil { - t.Fatalf("Append: %v", err) - } - if _, err := w.Close(); err != nil { - t.Fatalf("Writer.Close: %v", err) - } - b := stargzBuf.Bytes() + for _, srcTarFormat := range []tar.Format{tar.FormatUSTAR, tar.FormatPAX, tar.FormatGNU} { + srcTarFormat := srcTarFormat + for _, lossless := range []bool{true, false} { + t.Run(tt.name+"-"+fmt.Sprintf("compression=%v,prefix=%q,lossless=%v,format=%s", cl, prefix, lossless, srcTarFormat), func(t *testing.T) { + var tr io.Reader = buildTar(t, tt.in, prefix, srcTarFormat) + origTarDgstr := digest.Canonical.Digester() + tr = io.TeeReader(tr, origTarDgstr.Hash()) + var stargzBuf bytes.Buffer + w := NewWriterWithCompressor(&stargzBuf, cl) + w.ChunkSize = tt.chunkSize + if lossless { + err := w.AppendTarLossLess(tr) + if tt.wantFailOnLossLess { + if err != nil { + return // expected to fail + } + t.Fatalf("Append wanted to fail on lossless") + } + if err != nil { + t.Fatalf("Append(lossless): %v", err) + } + } else { + if err := w.AppendTar(tr); err != nil { + t.Fatalf("Append: %v", err) + } + } + if _, err := w.Close(); err != nil { + t.Fatalf("Writer.Close: %v", err) + } + b := stargzBuf.Bytes() + + if lossless { + // Check if the result blob reserves original tar metadata + rc, err := Unpack(io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), cl) + if err != nil { + t.Errorf("failed to decompress blob: %v", err) + return + } + defer rc.Close() + resultDgstr := digest.Canonical.Digester() + if _, err := io.Copy(resultDgstr.Hash(), rc); err != nil { + t.Errorf("failed to read result decompressed blob: %v", err) + return + } + if resultDgstr.Digest() != origTarDgstr.Digest() { + t.Errorf("lossy compression occurred: digest=%v; want %v", + resultDgstr.Digest(), origTarDgstr.Digest()) + return + } + } - diffID := w.DiffID() - wantDiffID := cl.DiffIDOf(t, b) - if diffID != wantDiffID { - t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) - } + diffID := w.DiffID() + wantDiffID := cl.DiffIDOf(t, b) + if diffID != wantDiffID { + t.Errorf("DiffID = %q; want %q", diffID, wantDiffID) + } - got := cl.CountStreams(t, b) - if got != tt.wantNumGz { - t.Errorf("number of streams = %d; want %d", got, tt.wantNumGz) - } + got := cl.CountStreams(t, b) + wantNumGz := tt.wantNumGz + if lossless && tt.wantNumGzLossLess > 0 { + wantNumGz = tt.wantNumGzLossLess + } + if got != wantNumGz { + t.Errorf("number of streams = %d; want %d", got, wantNumGz) + } - telemetry, checkCalled := newCalledTelemetry() - r, err := Open( - io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), - WithDecompressors(cl), - WithTelemetry(telemetry), - ) - if err != nil { - t.Fatalf("stargz.Open: %v", err) - } - if err := checkCalled(); err != nil { - t.Errorf("telemetry failure: %v", err) - } - for _, want := range tt.want { - want.check(t, r) + telemetry, checkCalled := newCalledTelemetry() + r, err := Open( + io.NewSectionReader(bytes.NewReader(b), 0, int64(len(b))), + WithDecompressors(cl), + WithTelemetry(telemetry), + ) + if err != nil { + t.Fatalf("stargz.Open: %v", err) + } + if err := checkCalled(); err != nil { + t.Errorf("telemetry failure: %v", err) + } + for _, want := range tt.want { + want.check(t, r) + } + }) } - }) + } } } } @@ -1655,49 +1734,41 @@ func hasEntryOwner(entry string, owner owner) stargzCheck { func tarOf(s ...tarEntry) []tarEntry { return s } type tarEntry interface { - appendTar(tw *tar.Writer, prefix string) error + appendTar(tw *tar.Writer, prefix string, format tar.Format) error } -type tarEntryFunc func(*tar.Writer, string) error - -func (f tarEntryFunc) appendTar(tw *tar.Writer, prefix string) error { return f(tw, prefix) } +type tarEntryFunc func(*tar.Writer, string, tar.Format) error -func buildTar(t *testing.T, ents []tarEntry, prefix string) (r io.Reader, cancel func()) { - pr, pw := io.Pipe() - go func() { - tw := tar.NewWriter(pw) - for _, ent := range ents { - if err := ent.appendTar(tw, prefix); err != nil { - t.Errorf("building input tar: %v", err) - pw.Close() - return - } - } - if err := tw.Close(); err != nil { - t.Errorf("closing write of input tar: %v", err) - } - pw.Close() - }() - return pr, func() { go pr.Close(); go pw.Close() } +func (f tarEntryFunc) appendTar(tw *tar.Writer, prefix string, format tar.Format) error { + return f(tw, prefix, format) } -func buildTarStatic(t *testing.T, ents []tarEntry, prefix string) *io.SectionReader { +func buildTar(t *testing.T, ents []tarEntry, prefix string, opts ...interface{}) *io.SectionReader { + format := tar.FormatUnknown + for _, opt := range opts { + switch v := opt.(type) { + case tar.Format: + format = v + default: + panic(fmt.Errorf("unsupported opt for buildTar: %v", opt)) + } + } buf := new(bytes.Buffer) tw := tar.NewWriter(buf) for _, ent := range ents { - if err := ent.appendTar(tw, prefix); err != nil { + if err := ent.appendTar(tw, prefix, format); err != nil { t.Fatalf("building input tar: %v", err) } } if err := tw.Close(); err != nil { t.Errorf("closing write of input tar: %v", err) } - data := buf.Bytes() + data := append(buf.Bytes(), make([]byte, 100)...) // append empty bytes at the tail to see lossless works return io.NewSectionReader(bytes.NewReader(data), 0, int64(len(data))) } func dir(name string, opts ...interface{}) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string) error { + return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { var o owner mode := os.FileMode(0755) for _, opt := range opts { @@ -1723,6 +1794,7 @@ func dir(name string, opts ...interface{}) tarEntry { Mode: tm, Uid: o.uid, Gid: o.gid, + Format: format, }) }) } @@ -1737,7 +1809,7 @@ type owner struct { } func file(name, contents string, opts ...interface{}) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string) error { + return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { var xattrs xAttr var o owner mode := os.FileMode(0644) @@ -1760,6 +1832,9 @@ func file(name, contents string, opts ...interface{}) tarEntry { if err != nil { return err } + if len(xattrs) > 0 { + format = tar.FormatPAX // only PAX supports xattrs + } if err := tw.WriteHeader(&tar.Header{ Typeflag: tar.TypeReg, Name: prefix + name, @@ -1768,6 +1843,7 @@ func file(name, contents string, opts ...interface{}) tarEntry { Size: int64(len(contents)), Uid: o.uid, Gid: o.gid, + Format: format, }); err != nil { return err } @@ -1777,78 +1853,76 @@ func file(name, contents string, opts ...interface{}) tarEntry { } func symlink(name, target string) tarEntry { - return tarEntryFunc(func(tw *tar.Writer, prefix string) error { + return tarEntryFunc(func(tw *tar.Writer, prefix string, format tar.Format) error { return tw.WriteHeader(&tar.Header{ Typeflag: tar.TypeSymlink, Name: prefix + name, Linkname: target, Mode: 0644, + Format: format, }) }) } func link(name string, linkname string) tarEntry { now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string) error { + return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeLink, - Name: prefix + name, - Linkname: linkname, - ModTime: now, - AccessTime: now, - ChangeTime: now, + Typeflag: tar.TypeLink, + Name: prefix + name, + Linkname: linkname, + ModTime: now, + Format: format, }) }) } func chardev(name string, major, minor int64) tarEntry { now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string) error { + return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeChar, - Name: prefix + name, - Devmajor: major, - Devminor: minor, - ModTime: now, - AccessTime: now, - ChangeTime: now, + Typeflag: tar.TypeChar, + Name: prefix + name, + Devmajor: major, + Devminor: minor, + ModTime: now, + Format: format, }) }) } func blockdev(name string, major, minor int64) tarEntry { now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string) error { + return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeBlock, - Name: prefix + name, - Devmajor: major, - Devminor: minor, - ModTime: now, - AccessTime: now, - ChangeTime: now, + Typeflag: tar.TypeBlock, + Name: prefix + name, + Devmajor: major, + Devminor: minor, + ModTime: now, + Format: format, }) }) } func fifo(name string) tarEntry { now := time.Now() - return tarEntryFunc(func(w *tar.Writer, prefix string) error { + return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { return w.WriteHeader(&tar.Header{ - Typeflag: tar.TypeFifo, - Name: prefix + name, - ModTime: now, - AccessTime: now, - ChangeTime: now, + Typeflag: tar.TypeFifo, + Name: prefix + name, + ModTime: now, + Format: format, }) }) } func prefetchLandmark() tarEntry { - return tarEntryFunc(func(w *tar.Writer, prefix string) error { + return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { if err := w.WriteHeader(&tar.Header{ Name: PrefetchLandmark, Typeflag: tar.TypeReg, Size: int64(len([]byte{landmarkContents})), + Format: format, }); err != nil { return err } @@ -1861,11 +1935,12 @@ func prefetchLandmark() tarEntry { } func noPrefetchLandmark() tarEntry { - return tarEntryFunc(func(w *tar.Writer, prefix string) error { + return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { if err := w.WriteHeader(&tar.Header{ Name: NoPrefetchLandmark, Typeflag: tar.TypeReg, Size: int64(len([]byte{landmarkContents})), + Format: format, }); err != nil { return err } @@ -1899,11 +1974,12 @@ func regDigest(t *testing.T, name string, contentStr string, digestMap map[strin n += size } - return tarEntryFunc(func(w *tar.Writer, prefix string) error { + return tarEntryFunc(func(w *tar.Writer, prefix string, format tar.Format) error { if err := w.WriteHeader(&tar.Header{ Typeflag: tar.TypeReg, Name: prefix + name, Size: int64(len(content)), + Format: format, }); err != nil { return err } diff --git a/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go b/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go index 1b1075a64..384ff7fd7 100644 --- a/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go +++ b/vendor/github.com/containerd/stargz-snapshotter/estargz/types.go @@ -290,7 +290,7 @@ type Compressor interface { WriteTOCAndFooter(w io.Writer, off int64, toc *JTOC, diffHash hash.Hash) (tocDgst digest.Digest, err error) } -// Deompressor represents the helper mothods to be used for parsing eStargz. +// Decompressor represents the helper mothods to be used for parsing eStargz. type Decompressor interface { // Reader returns ReadCloser to be used for decompressing file payload. Reader(r io.Reader) (io.ReadCloser, error) @@ -299,10 +299,12 @@ type Decompressor interface { FooterSize() int64 // ParseFooter parses the footer and returns the offset and (compressed) size of TOC. + // payloadBlobSize is the (compressed) size of the blob payload (i.e. the size between + // the top until the TOC JSON). // // Here, tocSize is optional. If tocSize <= 0, it's by default the size of the range // from tocOffset until the beginning of the footer (blob size - tocOff - FooterSize). - ParseFooter(p []byte) (tocOffset, tocSize int64, err error) + ParseFooter(p []byte) (blobPayloadSize, tocOffset, tocSize int64, err error) // ParseTOC parses TOC from the passed reader. The reader provides the partial contents // of the underlying blob that has the range specified by ParseFooter method. diff --git a/vendor/github.com/containers/common/libimage/copier.go b/vendor/github.com/containers/common/libimage/copier.go index 636b97bfd..4f5c7d0a1 100644 --- a/vendor/github.com/containers/common/libimage/copier.go +++ b/vendor/github.com/containers/common/libimage/copier.go @@ -2,7 +2,6 @@ package libimage import ( "context" - "encoding/json" "io" "os" "strings" diff --git a/vendor/github.com/containers/common/libimage/image_config.go b/vendor/github.com/containers/common/libimage/image_config.go index b57121182..140202440 100644 --- a/vendor/github.com/containers/common/libimage/image_config.go +++ b/vendor/github.com/containers/common/libimage/image_config.go @@ -1,7 +1,6 @@ package libimage import ( - "encoding/json" "fmt" "path/filepath" "strconv" diff --git a/vendor/github.com/containers/common/libimage/inspect.go b/vendor/github.com/containers/common/libimage/inspect.go index 349709155..a872e5cf9 100644 --- a/vendor/github.com/containers/common/libimage/inspect.go +++ b/vendor/github.com/containers/common/libimage/inspect.go @@ -2,7 +2,6 @@ package libimage import ( "context" - "encoding/json" "time" "github.com/containers/image/v5/manifest" diff --git a/vendor/github.com/containers/common/libimage/load.go b/vendor/github.com/containers/common/libimage/load.go index f2b57c43a..74a1870e0 100644 --- a/vendor/github.com/containers/common/libimage/load.go +++ b/vendor/github.com/containers/common/libimage/load.go @@ -35,6 +35,17 @@ func (r *Runtime) Load(ctx context.Context, path string, options *LoadOptions) ( var loadErrors []error for _, f := range []func() ([]string, string, error){ + // DOCKER-ARCHIVE - must be first (see containers/podman/issues/10809) + func() ([]string, string, error) { + logrus.Debugf("-> Attempting to load %q as a Docker archive", path) + ref, err := dockerArchiveTransport.ParseReference(path) + if err != nil { + return nil, dockerArchiveTransport.Transport.Name(), err + } + images, err := r.loadMultiImageDockerArchive(ctx, ref, &options.CopyOptions) + return images, dockerArchiveTransport.Transport.Name(), err + }, + // OCI func() ([]string, string, error) { logrus.Debugf("-> Attempting to load %q as an OCI directory", path) @@ -67,17 +78,6 @@ func (r *Runtime) Load(ctx context.Context, path string, options *LoadOptions) ( images, err := r.copyFromDefault(ctx, ref, &options.CopyOptions) return images, dirTransport.Transport.Name(), err }, - - // DOCKER-ARCHIVE - func() ([]string, string, error) { - logrus.Debugf("-> Attempting to load %q as a Docker archive", path) - ref, err := dockerArchiveTransport.ParseReference(path) - if err != nil { - return nil, dockerArchiveTransport.Transport.Name(), err - } - images, err := r.loadMultiImageDockerArchive(ctx, ref, &options.CopyOptions) - return images, dockerArchiveTransport.Transport.Name(), err - }, } { loadedImages, transportName, err := f() if err == nil { diff --git a/vendor/github.com/containers/common/libimage/runtime.go b/vendor/github.com/containers/common/libimage/runtime.go index dabadbec0..7f25df200 100644 --- a/vendor/github.com/containers/common/libimage/runtime.go +++ b/vendor/github.com/containers/common/libimage/runtime.go @@ -13,10 +13,14 @@ import ( "github.com/containers/image/v5/types" "github.com/containers/storage" deepcopy "github.com/jinzhu/copier" + jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) +// Faster than the standard library, see https://github.com/json-iterator/go. +var json = jsoniter.ConfigCompatibleWithStandardLibrary + // RuntimeOptions allow for creating a customized Runtime. type RuntimeOptions struct { // The base system context of the runtime which will be used throughout diff --git a/vendor/github.com/containers/common/pkg/config/config.go b/vendor/github.com/containers/common/pkg/config/config.go index 3b4c7fa04..d5be77edd 100644 --- a/vendor/github.com/containers/common/pkg/config/config.go +++ b/vendor/github.com/containers/common/pkg/config/config.go @@ -140,7 +140,7 @@ type ContainersConfig struct { // will be truncated. It can be expressed as a human-friendly string // that is parsed to bytes. // Negative values indicate that the log file won't be truncated. - LogSizeMax int64 `toml:"log_size_max,omitempty"` + LogSizeMax int64 `toml:"log_size_max,omitempty,omitzero"` // Specifies default format tag for container log messages. // This is useful for creating a specific tag for container log messages. @@ -155,7 +155,7 @@ type ContainersConfig struct { // PidsLimit is the number of processes each container is restricted to // by the cgroup process number controller. - PidsLimit int64 `toml:"pids_limit,omitempty"` + PidsLimit int64 `toml:"pids_limit,omitempty,omitzero"` // PidNS indicates how to create a pid namespace for the container PidNS string `toml:"pidns,omitempty"` @@ -192,7 +192,7 @@ type ContainersConfig struct { UserNS string `toml:"userns,omitempty"` // UserNSSize how many UIDs to allocate for automatically created UserNS - UserNSSize int `toml:"userns_size,omitempty"` + UserNSSize int `toml:"userns_size,omitempty,omitzero"` } // EngineConfig contains configuration options used to set up a engine runtime @@ -259,7 +259,7 @@ type EngineConfig struct { // ImageParallelCopies indicates the maximum number of image layers // to be copied simultaneously. If this is zero, container engines // will fall back to containers/image defaults. - ImageParallelCopies uint `toml:"image_parallel_copies,omitempty"` + ImageParallelCopies uint `toml:"image_parallel_copies,omitempty,omitzero"` // ImageDefaultFormat specified the manifest Type (oci, v2s2, or v2s1) // to use when pulling, pushing, building container images. By default @@ -308,7 +308,7 @@ type EngineConfig struct { // NumLocks is the number of locks to make available for containers and // pods. - NumLocks uint32 `toml:"num_locks,omitempty"` + NumLocks uint32 `toml:"num_locks,omitempty,omitzero"` // OCIRuntime is the OCI runtime to use. OCIRuntime string `toml:"runtime,omitempty"` @@ -380,7 +380,7 @@ type EngineConfig struct { // ServiceTimeout is the number of seconds to wait without a connection // before the `podman system service` times out and exits - ServiceTimeout uint `toml:"service_timeout,omitempty"` + ServiceTimeout uint `toml:"service_timeout,omitempty,omitzero"` // StaticDir is the path to a persistent directory to store container // files. @@ -388,7 +388,7 @@ type EngineConfig struct { // StopTimeout is the number of seconds to wait for container to exit // before sending kill signal. - StopTimeout uint `toml:"stop_timeout,omitempty"` + StopTimeout uint `toml:"stop_timeout,omitempty,omitzero"` // ImageCopyTmpDir is the default location for storing temporary // container image content, Can be overridden with the TMPDIR @@ -413,7 +413,7 @@ type EngineConfig struct { // ChownCopiedFiles tells the container engine whether to chown files copied // into a container to the container's primary uid/gid. - ChownCopiedFiles bool `toml:"chown_copied_files"` + ChownCopiedFiles bool `toml:"chown_copied_files,omitempty"` } // SetOptions contains a subset of options in a Config. It's used to indicate if @@ -492,13 +492,13 @@ type SecretConfig struct { // MachineConfig represents the "machine" TOML config table type MachineConfig struct { // Number of CPU's a machine is created with. - CPUs uint64 `toml:"cpus,omitempty"` + CPUs uint64 `toml:"cpus,omitempty,omitzero"` // DiskSize is the size of the disk in GB created when init-ing a podman-machine VM - DiskSize uint64 `toml:"disk_size,omitempty"` + DiskSize uint64 `toml:"disk_size,omitempty,omitzero"` // MachineImage is the image used when init-ing a podman-machine VM Image string `toml:"image,omitempty"` // Memory in MB a machine is created with. - Memory uint64 `toml:"memory,omitempty"` + Memory uint64 `toml:"memory,omitempty,omitzero"` } // Destination represents destination for remote service @@ -1067,17 +1067,6 @@ func ReadCustomConfig() (*Config, error) { if err != nil { return nil, err } - // hack since Ommitempty does not seem to work with Write - c, err := Default() - if err != nil { - if os.IsNotExist(errors.Cause(err)) { - c, err = DefaultConfig() - } - if err != nil { - return nil, err - } - } - newConfig := &Config{} if _, err := os.Stat(path); err == nil { if err := readConfigFromFile(path, newConfig); err != nil { @@ -1088,11 +1077,6 @@ func ReadCustomConfig() (*Config, error) { return nil, err } } - newConfig.Containers.LogSizeMax = c.Containers.LogSizeMax - newConfig.Containers.PidsLimit = c.Containers.PidsLimit - newConfig.Containers.UserNSSize = c.Containers.UserNSSize - newConfig.Engine.NumLocks = c.Engine.NumLocks - newConfig.Engine.StopTimeout = c.Engine.StopTimeout return newConfig, nil } diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index afc132ffa..bf50e910e 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.36.0+dev +1.37.0 diff --git a/vendor/github.com/containers/storage/drivers/driver_linux.go b/vendor/github.com/containers/storage/drivers/driver_linux.go index dddf8a8b4..0fe3eea7a 100644 --- a/vendor/github.com/containers/storage/drivers/driver_linux.go +++ b/vendor/github.com/containers/storage/drivers/driver_linux.go @@ -50,6 +50,40 @@ const ( FsMagicOverlay = FsMagic(0x794C7630) // FsMagicFUSE filesystem id for FUSE FsMagicFUSE = FsMagic(0x65735546) + // FsMagicAcfs filesystem id for Acfs + FsMagicAcfs = FsMagic(0x61636673) + // FsMagicAfs filesystem id for Afs + FsMagicAfs = FsMagic(0x5346414f) + // FsMagicCephFs filesystem id for Ceph + FsMagicCephFs = FsMagic(0x00C36400) + // FsMagicCIFS filesystem id for CIFS + FsMagicCIFS = FsMagic(0xFF534D42) + // FsMagicFHGFS filesystem id for FHGFS + FsMagicFHGFSFs = FsMagic(0x19830326) + // FsMagicIBRIX filesystem id for IBRIX + FsMagicIBRIX = FsMagic(0x013111A8) + // FsMagicKAFS filesystem id for KAFS + FsMagicKAFS = FsMagic(0x6B414653) + // FsMagicLUSTRE filesystem id for LUSTRE + FsMagicLUSTRE = FsMagic(0x0BD00BD0) + // FsMagicNCP filesystem id for NCP + FsMagicNCP = FsMagic(0x564C) + // FsMagicNFSD filesystem id for NFSD + FsMagicNFSD = FsMagic(0x6E667364) + // FsMagicOCFS2 filesystem id for OCFS2 + FsMagicOCFS2 = FsMagic(0x7461636F) + // FsMagicPANFS filesystem id for PANFS + FsMagicPANFS = FsMagic(0xAAD7AAEA) + // FsMagicPRLFS filesystem id for PRLFS + FsMagicPRLFS = FsMagic(0x7C7C6673) + // FsMagicSMB2 filesystem id for SMB2 + FsMagicSMB2 = FsMagic(0xFE534D42) + // FsMagicSNFS filesystem id for SNFS + FsMagicSNFS = FsMagic(0xBEEFDEAD) + // FsMagicVBOXSF filesystem id for VBOXSF + FsMagicVBOXSF = FsMagic(0x786F4256) + // FsMagicVXFS filesystem id for VXFS + FsMagicVXFS = FsMagic(0xA501FCF5) ) var ( diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 62130c73e..1efe7316d 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -248,6 +248,23 @@ func (d *Driver) getSupportsVolatile() (bool, error) { return supportsVolatile, nil } +// isNetworkFileSystem checks if the specified file system is supported by native overlay +// as backing store when running in a user namespace. +func isNetworkFileSystem(fsMagic graphdriver.FsMagic) bool { + switch fsMagic { + // a bunch of network file systems... + case graphdriver.FsMagicNfsFs, graphdriver.FsMagicSmbFs, graphdriver.FsMagicAcfs, + graphdriver.FsMagicAfs, graphdriver.FsMagicCephFs, graphdriver.FsMagicCIFS, + graphdriver.FsMagicFHGFSFs, graphdriver.FsMagicGPFS, graphdriver.FsMagicIBRIX, + graphdriver.FsMagicKAFS, graphdriver.FsMagicLUSTRE, graphdriver.FsMagicNCP, + graphdriver.FsMagicNFSD, graphdriver.FsMagicOCFS2, graphdriver.FsMagicPANFS, + graphdriver.FsMagicPRLFS, graphdriver.FsMagicSMB2, graphdriver.FsMagicSNFS, + graphdriver.FsMagicVBOXSF, graphdriver.FsMagicVXFS: + return true + } + return false +} + // Init returns the a native diff driver for overlay filesystem. // If overlay filesystem is not supported on the host, a wrapped graphdriver.ErrNotSupported is returned as error. // If an overlay filesystem is not supported over an existing filesystem then a wrapped graphdriver.ErrIncompatibleFS is returned. @@ -266,18 +283,27 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) } if opts.mountProgram != "" { + if unshare.IsRootless() && isNetworkFileSystem(fsMagic) && opts.forceMask == nil { + m := os.FileMode(0700) + opts.forceMask = &m + logrus.Warnf("Network file system detected as backing store. Enforcing overlay option `force_mask=\"%o\"`. Add it to storage.conf to silence this warning", m) + } + if err := ioutil.WriteFile(getMountProgramFlagFile(home), []byte("true"), 0600); err != nil { return nil, err } } else { - // check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs if opts.forceMask != nil { return nil, errors.New("'force_mask' is supported only with 'mount_program'") } + // check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs switch fsMagic { case graphdriver.FsMagicAufs, graphdriver.FsMagicZfs, graphdriver.FsMagicOverlay, graphdriver.FsMagicEcryptfs: return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "'overlay' is not supported over %s, a mount_program is required", backingFs) } + if unshare.IsRootless() && isNetworkFileSystem(fsMagic) { + return nil, errors.Wrapf(graphdriver.ErrIncompatibleFS, "A network file system with user namespaces is not supported. Please use a mount_program") + } } rootUID, rootGID, err := idtools.GetRootUIDGID(options.UIDMaps, options.GIDMaps) @@ -1431,6 +1457,11 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO label = d.optsAppendMappings(label, options.UidMaps, options.GidMaps) } + // if forceMask is in place, tell fuse-overlayfs to write the permissions mask to an unprivileged xattr as well. + if d.options.forceMask != nil { + label = label + ",xattr_permissions=2" + } + mountProgram := exec.Command(d.options.mountProgram, "-o", label, target) mountProgram.Dir = d.home var b bytes.Buffer diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index ac7f45c3f..6cd809745 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -6,7 +6,7 @@ require ( github.com/BurntSushi/toml v0.4.1 github.com/Microsoft/go-winio v0.5.0 github.com/Microsoft/hcsshim v0.8.22 - github.com/containerd/stargz-snapshotter/estargz v0.8.0 + github.com/containerd/stargz-snapshotter/estargz v0.9.0 github.com/docker/go-units v0.4.0 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/google/go-intervals v0.0.2 diff --git a/vendor/github.com/containers/storage/go.sum b/vendor/github.com/containers/storage/go.sum index 3da64ce8c..1b602d484 100644 --- a/vendor/github.com/containers/storage/go.sum +++ b/vendor/github.com/containers/storage/go.sum @@ -31,8 +31,8 @@ github.com/containerd/containerd v1.4.9/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX github.com/containerd/continuity v0.1.0/go.mod h1:ICJu0PwR54nI0yPEnJ6jcS+J7CZAUXrLh8lPo2knzsM= github.com/containerd/fifo v1.0.0/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/go-runc v1.0.0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= -github.com/containerd/stargz-snapshotter/estargz v0.8.0 h1:oA1wx8kTFfImfsT5bScbrZd8gK+WtQnn15q82Djvm0Y= -github.com/containerd/stargz-snapshotter/estargz v0.8.0/go.mod h1:mwIwuwb+D8FX2t45Trwi0hmWmZm5VW7zPP/rekwhWQU= +github.com/containerd/stargz-snapshotter/estargz v0.9.0 h1:PkB6BSTfOKX23erT2GkoUKkJEcXfNcyKskIViK770v8= +github.com/containerd/stargz-snapshotter/estargz v0.9.0/go.mod h1:aE5PCyhFMwR8sbrErO5eM2GcvkyXTTJremG883D4qF0= github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= github.com/containerd/typeurl v1.0.2/go.mod h1:9trJWW2sRlGub4wZJRTW83VtbOLS6hwcDZXTn6oPz9s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= @@ -122,7 +122,6 @@ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvW github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.13.6 h1:P76CopJELS0TiO2mebmnzgWaajssP/EszplttgQxcgc= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= diff --git a/vendor/github.com/containers/storage/pkg/system/syscall_unix.go b/vendor/github.com/containers/storage/pkg/system/syscall_unix.go index 49dbdd378..1bb852d11 100644 --- a/vendor/github.com/containers/storage/pkg/system/syscall_unix.go +++ b/vendor/github.com/containers/storage/pkg/system/syscall_unix.go @@ -1,8 +1,11 @@ -// +build linux freebsd +// +build linux freebsd darwin package system -import "golang.org/x/sys/unix" +import ( + "github.com/pkg/errors" + "golang.org/x/sys/unix" +) // Unmount is a platform-specific helper function to call // the unmount syscall. @@ -15,3 +18,8 @@ func Unmount(dest string) error { func CommandLineToArgv(commandLine string) ([]string, error) { return []string{commandLine}, nil } + +// IsEBUSY checks if the specified error is EBUSY. +func IsEBUSY(err error) bool { + return errors.Is(err, unix.EBUSY) +} diff --git a/vendor/github.com/containers/storage/pkg/system/syscall_windows.go b/vendor/github.com/containers/storage/pkg/system/syscall_windows.go index 23e9b207c..f4d8692cd 100644 --- a/vendor/github.com/containers/storage/pkg/system/syscall_windows.go +++ b/vendor/github.com/containers/storage/pkg/system/syscall_windows.go @@ -120,3 +120,8 @@ func HasWin32KSupport() bool { // APIs. return ntuserApiset.Load() == nil } + +// IsEBUSY checks if the specified error is EBUSY. +func IsEBUSY(err error) bool { + return false +} diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 6f6f69807..169c7d151 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -23,6 +23,7 @@ import ( "github.com/containers/storage/pkg/parsers" "github.com/containers/storage/pkg/stringid" "github.com/containers/storage/pkg/stringutils" + "github.com/containers/storage/pkg/system" "github.com/containers/storage/types" "github.com/hashicorp/go-multierror" digest "github.com/opencontainers/go-digest" @@ -2498,7 +2499,15 @@ func (s *store) DeleteContainer(id string) error { gcpath := filepath.Join(s.GraphRoot(), middleDir, container.ID) wg.Add(1) go func() { - errChan <- os.RemoveAll(gcpath) + var err error + for attempts := 0; attempts < 50; attempts++ { + err = os.RemoveAll(gcpath) + if err == nil || !system.IsEBUSY(err) { + break + } + time.Sleep(time.Millisecond * 100) + } + errChan <- err wg.Done() }() diff --git a/vendor/modules.txt b/vendor/modules.txt index 3ad22d633..345a31ed3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -63,7 +63,7 @@ github.com/containerd/containerd/log github.com/containerd/containerd/pkg/userns github.com/containerd/containerd/platforms github.com/containerd/containerd/sys -# github.com/containerd/stargz-snapshotter/estargz v0.8.0 +# github.com/containerd/stargz-snapshotter/estargz v0.9.0 github.com/containerd/stargz-snapshotter/estargz github.com/containerd/stargz-snapshotter/estargz/errorutil # github.com/containernetworking/cni v1.0.1 @@ -97,7 +97,7 @@ github.com/containers/buildah/pkg/rusage github.com/containers/buildah/pkg/sshagent github.com/containers/buildah/pkg/util github.com/containers/buildah/util -# github.com/containers/common v0.46.1-0.20210928081721-32e20295f1c6 +# github.com/containers/common v0.46.1-0.20211001143714-161e078e4c7f github.com/containers/common/libimage github.com/containers/common/libimage/manifests github.com/containers/common/pkg/apparmor @@ -202,7 +202,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.36.1-0.20210929132900-162a0bf730ce +# github.com/containers/storage v1.37.0 github.com/containers/storage github.com/containers/storage/drivers github.com/containers/storage/drivers/aufs |