diff options
156 files changed, 4589 insertions, 4402 deletions
@@ -13,10 +13,10 @@ require ( github.com/containers/buildah v1.20.1-0.20210402144408-36a37402d0c8 github.com/containers/common v0.35.4 github.com/containers/conmon v2.0.20+incompatible - github.com/containers/image/v5 v5.10.5 + github.com/containers/image/v5 v5.11.0 github.com/containers/ocicrypt v1.1.0 github.com/containers/psgo v1.5.2 - github.com/containers/storage v1.28.1 + github.com/containers/storage v1.29.0 github.com/coreos/go-systemd/v22 v22.3.1 github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3 github.com/cri-o/ocicni v0.2.1-0.20210301205850-541cf7c703cf @@ -53,9 +53,11 @@ github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3h github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= -github.com/Microsoft/hcsshim v0.8.15 h1:Aof83YILRs2Vx3GhHqlvvfyx1asRJKMFIMeVlHsZKtI= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= +github.com/Microsoft/hcsshim v0.8.16 h1:8/auA4LFIZFTGrqfKhGBSXwM6/4X1fHa/xniyEHu8ac= +github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= +github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= @@ -119,13 +121,17 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= +github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= +github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= +github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102 h1:Qf4HiqfvmB7zS6scsmNgTLmByHbq8n9RTF39v+TzP7A= github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68 h1:hkGVFjz+plgr5UfxZUTPFbUFIF/Km6/s+RVRIRHLrrY= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= @@ -137,9 +143,12 @@ github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.5.0-beta.1 h1:IK6yirB4X7wpKyFSikWiT++nZsyIxGAAgNEv3fEGuls= github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= +github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= +github.com/containerd/containerd v1.5.0-beta.4 h1:zjz4MOAOFgdBlwid2nNUlJ3YLpVi/97L36lfMYJex60= +github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -151,12 +160,17 @@ github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= +github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= +github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= +github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= +github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= @@ -166,6 +180,8 @@ github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kw github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= +github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= +github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.1 h1:7zpDnQ3T3s4ucOuJ/ZCLrYBxzkg0AELFfII3Epo9TmI= @@ -180,8 +196,9 @@ github.com/containers/common v0.35.4 h1:szyWRncsHkBwCVpu1dkEOXUjkwCetlfcLmKJTwo1 github.com/containers/common v0.35.4/go.mod h1:rMzxgD7nMGw++cEbsp+NZv0UJO4rgXbm7F7IbJPTwIE= github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg= github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= -github.com/containers/image/v5 v5.10.5 h1:VK1UbsZMzjdw5Xqr3Im9h4iOqHWU0naFs+I78kavc7I= github.com/containers/image/v5 v5.10.5/go.mod h1:SgIbWEedCNBbn2FI5cH0/jed1Ecy2s8XK5zTxvJTzII= +github.com/containers/image/v5 v5.11.0 h1:SwxGucW1AZ8H/5KH9jW70lo9WyuOrtxafutyQ9RPPLw= +github.com/containers/image/v5 v5.11.0/go.mod h1:dCbUB4w6gmxIEOCsE0tZQppr8iBoXb4Evr74ZKlmwoI= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= @@ -193,8 +210,9 @@ github.com/containers/psgo v1.5.2/go.mod h1:2ubh0SsreMZjSXW1Hif58JrEcFudQyIy9EzP github.com/containers/storage v1.23.5/go.mod h1:ha26Q6ngehFNhf3AWoXldvAvwI4jFe3ETQAf/CeZPyM= github.com/containers/storage v1.24.8/go.mod h1:YC+2pY8SkfEAcZkwycxYbpK8EiRbx5soPPwz9dxe4IQ= github.com/containers/storage v1.28.0/go.mod h1:ixAwO7Bj31cigqPEG7aCz+PYmxkDxbIFdUFioYdxbzI= -github.com/containers/storage v1.28.1 h1:axYBD+c0N0YkHelDoqzdLQXfY3fgb8pqIMsRHqUNGts= github.com/containers/storage v1.28.1/go.mod h1:5bwiMh2LkrN3AWIfDFMH7A/xbVNLcve+oeXYvHvW8cc= +github.com/containers/storage v1.29.0 h1:l3Vh6+IiMKLGfQZ3rDkF84m+KF1Qb0XEcilWC+pYo2o= +github.com/containers/storage v1.29.0/go.mod h1:u84RU4CCufGeJBNTRNwMB+FoE+AiFeFw4SsMoqAOeCM= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= @@ -371,6 +389,8 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-intervals v0.0.2 h1:FGrVEiUnTRKR8yE04qzXYaJMtnIYqobR5QbblK3ixcM= +github.com/google/go-intervals v0.0.2/go.mod h1:MkaR3LNRfeKLPmqgJYs4E66z5InYjmCjbbr4TQlcT6Y= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= @@ -474,8 +494,9 @@ github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.12 h1:famVnQVu7QwryBN4jNseQdUKES71ZAOnB6UQQJPZvqk= github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -491,6 +512,7 @@ github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a h1:weJVJJRzAJBFRlAiJQROKQs8oC9vOxvm4rZmBBk0ONw= github.com/lunixbochs/vtclean v0.0.0-20180621232353-2d01aacdc34a/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= +github.com/magefile/mage v1.10.0/go.mod h1:z5UZb/iS3GoOSn0JgWuiw7dxlurVYTu+/jHXqQg881A= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -771,7 +793,6 @@ github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/vbatts/tar-split v0.11.1 h1:0Odu65rhcZ3JZaPHxl7tCI3V/C/Q9Zf82UFravl02dE= github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g= -github.com/vbauerster/mpb/v5 v5.4.0 h1:n8JPunifvQvh6P1D1HAl2Ur9YcmKT1tpoUuiea5mlmg= github.com/vbauerster/mpb/v5 v5.4.0/go.mod h1:fi4wVo7BVQ22QcvFObm+VwliQXlV1eBT8JDaKXR4JGI= github.com/vbauerster/mpb/v6 v6.0.3 h1:j+twHHhSUe8aXWaT/27E98G5cSBeqEuJSVCMjmLg0PI= github.com/vbauerster/mpb/v6 v6.0.3/go.mod h1:5luBx4rDLWxpA4t6I5sdeeQuZhqDxc+wr5Nqf35+tnM= @@ -978,6 +999,7 @@ golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200327173247-9dae0f8f5775/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/pkg/api/server/register_pods.go b/pkg/api/server/register_pods.go index 5944b98fd..3bcc50ba4 100644 --- a/pkg/api/server/register_pods.go +++ b/pkg/api/server/register_pods.go @@ -38,7 +38,7 @@ func (s *APIServer) registerPodsHandlers(r *mux.Router) error { // schema: // $ref: "#/definitions/PodSpecGenerator" // responses: - // 200: + // 201: // schema: // $ref: "#/definitions/IdResponse" // 400: diff --git a/test/apiv2/rest_api/test_rest_v2_0_0.py b/test/apiv2/rest_api/test_rest_v2_0_0.py index 97336cb71..bf0ee0603 100644 --- a/test/apiv2/rest_api/test_rest_v2_0_0.py +++ b/test/apiv2/rest_api/test_rest_v2_0_0.py @@ -735,5 +735,6 @@ class TestApi(unittest.TestCase): r = requests.get(_url("/system/df")) self.assertEqual(r.status_code, 200, r.text) + if __name__ == "__main__": unittest.main() diff --git a/vendor/github.com/Microsoft/hcsshim/.gometalinter.json b/vendor/github.com/Microsoft/hcsshim/.gometalinter.json deleted file mode 100644 index 00e9a6e2e..000000000 --- a/vendor/github.com/Microsoft/hcsshim/.gometalinter.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "Vendor": true, - "Deadline": "2m", - "Sort": [ - "linter", - "severity", - "path", - "line" - ], - "Skip": [ - "internal\\schema2" - ], - "EnableGC": true, - "Enable": [ - "gofmt" - ] -}
\ No newline at end of file diff --git a/vendor/github.com/Microsoft/hcsshim/Protobuild.toml b/vendor/github.com/Microsoft/hcsshim/Protobuild.toml index 47d7650fb..8bb920851 100644 --- a/vendor/github.com/Microsoft/hcsshim/Protobuild.toml +++ b/vendor/github.com/Microsoft/hcsshim/Protobuild.toml @@ -35,20 +35,10 @@ plugins = ["grpc", "fieldpath"] prefixes = ["github.com/Microsoft/hcsshim/internal/shimdiag"] plugins = ["ttrpc"] -# Lock down runhcs config - -[[descriptors]] -prefix = "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/options" -target = "cmd/containerd-shim-runhcs-v1/options/next.pb.txt" -ignore_files = [ - "google/protobuf/descriptor.proto", - "gogoproto/gogo.proto" -] +[[overrides]] +prefixes = ["github.com/Microsoft/hcsshim/internal/computeagent"] +plugins = ["ttrpc"] -[[descriptors]] -prefix = "github.com/Microsoft/hcsshim/cmd/containerd-shim-runhcs-v1/stats" -target = "cmd/containerd-shim-runhcs-v1/stats/next.pb.txt" -ignore_files = [ - "google/protobuf/descriptor.proto", - "gogoproto/gogo.proto" -]
\ No newline at end of file +[[overrides]] +prefixes = ["github.com/Microsoft/hcsshim/internal/ncproxyttrpc"] +plugins = ["ttrpc"]
\ No newline at end of file diff --git a/vendor/github.com/Microsoft/hcsshim/README.md b/vendor/github.com/Microsoft/hcsshim/README.md index d504f1889..95c300365 100644 --- a/vendor/github.com/Microsoft/hcsshim/README.md +++ b/vendor/github.com/Microsoft/hcsshim/README.md @@ -1,6 +1,6 @@ # hcsshim -[](https://ci.appveyor.com/project/WindowsVirtualization/hcsshim/branch/master) +[](https://github.com/microsoft/hcsshim/actions?query=branch%3Amaster) This package contains the Golang interface for using the Windows [Host Compute Service](https://techcommunity.microsoft.com/t5/containers/introducing-the-host-compute-service-hcs/ba-p/382332) (HCS) to launch and manage [Windows Containers](https://docs.microsoft.com/en-us/virtualization/windowscontainers/about/). It also contains other helpers and functions for managing Windows Containers such as the Golang interface for the Host Network Service (HNS). diff --git a/vendor/github.com/Microsoft/hcsshim/appveyor.yml b/vendor/github.com/Microsoft/hcsshim/appveyor.yml deleted file mode 100644 index b214a9626..000000000 --- a/vendor/github.com/Microsoft/hcsshim/appveyor.yml +++ /dev/null @@ -1,49 +0,0 @@ -version: 0.1.{build} - -image: Visual Studio 2019 - -clone_folder: c:\gopath\src\github.com\Microsoft\hcsshim - -environment: - GOPATH: c:\gopath - PATH: "%GOPATH%\\bin;C:\\gometalinter-2.0.12-windows-amd64;%PATH%" - GOPROXY: 'off' - GOFLAGS: '-mod=vendor' - -stack: go 1.15 - -build_script: - - appveyor DownloadFile https://github.com/alecthomas/gometalinter/releases/download/v2.0.12/gometalinter-2.0.12-windows-amd64.zip - - 7z x gometalinter-2.0.12-windows-amd64.zip -y -oC:\ > NUL - - gometalinter.exe --config .gometalinter.json ./... - - go build ./cmd/containerd-shim-runhcs-v1 - - go build ./cmd/runhcs - - go build ./cmd/tar2ext4 - - go build ./cmd/wclayer - - go build ./cmd/device-util - - go build ./internal/tools/grantvmgroupaccess - - go build ./internal/tools/uvmboot - - go build ./internal/tools/zapdir - - go test -gcflags=all=-d=checkptr -v ./... -tags admin - - cd test - - go test -gcflags=all=-d=checkptr -v ./internal -tags admin - - go test -gcflags=all=-d=checkptr -c ./containerd-shim-runhcs-v1/ -tags functional - - go test -gcflags=all=-d=checkptr -c ./cri-containerd/ -tags functional - - go test -gcflags=all=-d=checkptr -c ./functional/ -tags functional - - go test -gcflags=all=-d=checkptr -c ./runhcs/ -tags functional - - go build -o sample-logging-driver.exe ./cri-containerd/helpers/log.go - -artifacts: - - path: 'containerd-shim-runhcs-v1.exe' - - path: 'runhcs.exe' - - path: 'tar2ext4.exe' - - path: 'device-util.exe' - - path: 'wclayer.exe' - - path: 'grantvmgroupaccess.exe' - - path: 'uvmboot.exe' - - path: 'zapdir.exe' - - path: './test/containerd-shim-runhcs-v1.test.exe' - - path: './test/cri-containerd.test.exe' - - path: './test/functional.test.exe' - - path: './test/runhcs.test.exe' - - path: './test/sample-logging-driver.exe' diff --git a/vendor/github.com/Microsoft/hcsshim/computestorage/attach.go b/vendor/github.com/Microsoft/hcsshim/computestorage/attach.go index dcc61347c..7f1f2823d 100644 --- a/vendor/github.com/Microsoft/hcsshim/computestorage/attach.go +++ b/vendor/github.com/Microsoft/hcsshim/computestorage/attach.go @@ -18,7 +18,7 @@ import ( // `layerData` is the parent read-only layer data. func AttachLayerStorageFilter(ctx context.Context, layerPath string, layerData LayerData) (err error) { title := "hcsshim.AttachLayerStorageFilter" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes( diff --git a/vendor/github.com/Microsoft/hcsshim/computestorage/destroy.go b/vendor/github.com/Microsoft/hcsshim/computestorage/destroy.go index 39a9ba381..8e28e6c50 100644 --- a/vendor/github.com/Microsoft/hcsshim/computestorage/destroy.go +++ b/vendor/github.com/Microsoft/hcsshim/computestorage/destroy.go @@ -13,7 +13,7 @@ import ( // `layerPath` is a path to a directory containing the layer to export. func DestroyLayer(ctx context.Context, layerPath string) (err error) { title := "hcsshim.DestroyLayer" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("layerPath", layerPath)) diff --git a/vendor/github.com/Microsoft/hcsshim/computestorage/detach.go b/vendor/github.com/Microsoft/hcsshim/computestorage/detach.go index 9c144c066..435473257 100644 --- a/vendor/github.com/Microsoft/hcsshim/computestorage/detach.go +++ b/vendor/github.com/Microsoft/hcsshim/computestorage/detach.go @@ -13,7 +13,7 @@ import ( // `layerPath` is a path to a directory containing the layer to export. func DetachLayerStorageFilter(ctx context.Context, layerPath string) (err error) { title := "hcsshim.DetachLayerStorageFilter" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("layerPath", layerPath)) diff --git a/vendor/github.com/Microsoft/hcsshim/computestorage/export.go b/vendor/github.com/Microsoft/hcsshim/computestorage/export.go index 649b2602a..a1b12dd12 100644 --- a/vendor/github.com/Microsoft/hcsshim/computestorage/export.go +++ b/vendor/github.com/Microsoft/hcsshim/computestorage/export.go @@ -20,7 +20,7 @@ import ( // `options` are the export options applied to the exported layer. func ExportLayer(ctx context.Context, layerPath, exportFolderPath string, layerData LayerData, options ExportLayerOptions) (err error) { title := "hcsshim.ExportLayer" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes( diff --git a/vendor/github.com/Microsoft/hcsshim/computestorage/format.go b/vendor/github.com/Microsoft/hcsshim/computestorage/format.go index fe0861d2a..83c0fa33f 100644 --- a/vendor/github.com/Microsoft/hcsshim/computestorage/format.go +++ b/vendor/github.com/Microsoft/hcsshim/computestorage/format.go @@ -14,7 +14,7 @@ import ( // If the VHD is not mounted it will be temporarily mounted. func FormatWritableLayerVhd(ctx context.Context, vhdHandle windows.Handle) (err error) { title := "hcsshim.FormatWritableLayerVhd" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() diff --git a/vendor/github.com/Microsoft/hcsshim/computestorage/helpers.go b/vendor/github.com/Microsoft/hcsshim/computestorage/helpers.go index d31efd660..87fee452c 100644 --- a/vendor/github.com/Microsoft/hcsshim/computestorage/helpers.go +++ b/vendor/github.com/Microsoft/hcsshim/computestorage/helpers.go @@ -70,11 +70,9 @@ func SetupContainerBaseLayer(ctx context.Context, layerPath, baseVhdPath, diffVh defer func() { if err != nil { - syscall.CloseHandle(handle) + _ = syscall.CloseHandle(handle) os.RemoveAll(baseVhdPath) - if os.Stat(diffVhdPath); err == nil { - os.RemoveAll(diffVhdPath) - } + os.RemoveAll(diffVhdPath) } }() @@ -148,11 +146,9 @@ func SetupUtilityVMBaseLayer(ctx context.Context, uvmPath, baseVhdPath, diffVhdP defer func() { if err != nil { - syscall.CloseHandle(handle) + _ = syscall.CloseHandle(handle) os.RemoveAll(baseVhdPath) - if os.Stat(diffVhdPath); err == nil { - os.RemoveAll(diffVhdPath) - } + os.RemoveAll(diffVhdPath) } }() diff --git a/vendor/github.com/Microsoft/hcsshim/computestorage/import.go b/vendor/github.com/Microsoft/hcsshim/computestorage/import.go index 8ad2b085c..0c61dab32 100644 --- a/vendor/github.com/Microsoft/hcsshim/computestorage/import.go +++ b/vendor/github.com/Microsoft/hcsshim/computestorage/import.go @@ -20,7 +20,7 @@ import ( // `layerData` is the parent layer data. func ImportLayer(ctx context.Context, layerPath, sourceFolderPath string, layerData LayerData) (err error) { title := "hcsshim.ImportLayer" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes( diff --git a/vendor/github.com/Microsoft/hcsshim/computestorage/initialize.go b/vendor/github.com/Microsoft/hcsshim/computestorage/initialize.go index a50afd821..53ed8ea6e 100644 --- a/vendor/github.com/Microsoft/hcsshim/computestorage/initialize.go +++ b/vendor/github.com/Microsoft/hcsshim/computestorage/initialize.go @@ -17,7 +17,7 @@ import ( // `layerData` is the parent read-only layer data. func InitializeWritableLayer(ctx context.Context, layerPath string, layerData LayerData) (err error) { title := "hcsshim.InitializeWritableLayer" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes( diff --git a/vendor/github.com/Microsoft/hcsshim/computestorage/mount.go b/vendor/github.com/Microsoft/hcsshim/computestorage/mount.go index 1c16ff409..fcdbbef81 100644 --- a/vendor/github.com/Microsoft/hcsshim/computestorage/mount.go +++ b/vendor/github.com/Microsoft/hcsshim/computestorage/mount.go @@ -13,7 +13,7 @@ import ( // GetLayerVhdMountPath returns the volume path for a virtual disk of a writable container layer. func GetLayerVhdMountPath(ctx context.Context, vhdHandle windows.Handle) (path string, err error) { title := "hcsshim.GetLayerVhdMountPath" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() diff --git a/vendor/github.com/Microsoft/hcsshim/computestorage/setup.go b/vendor/github.com/Microsoft/hcsshim/computestorage/setup.go index 7506709ca..ca7b40ef6 100644 --- a/vendor/github.com/Microsoft/hcsshim/computestorage/setup.go +++ b/vendor/github.com/Microsoft/hcsshim/computestorage/setup.go @@ -22,7 +22,7 @@ import ( // `options` are the options applied while processing the layer. func SetupBaseOSLayer(ctx context.Context, layerPath string, vhdHandle windows.Handle, options OsLayerOptions) (err error) { title := "hcsshim.SetupBaseOSLayer" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes( @@ -53,7 +53,7 @@ func SetupBaseOSVolume(ctx context.Context, layerPath, volumePath string, option return errors.New("SetupBaseOSVolume is not present on builds older than 19645") } title := "hcsshim.SetupBaseOSVolume" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes( diff --git a/vendor/github.com/Microsoft/hcsshim/errors.go b/vendor/github.com/Microsoft/hcsshim/errors.go index 061727c67..794308673 100644 --- a/vendor/github.com/Microsoft/hcsshim/errors.go +++ b/vendor/github.com/Microsoft/hcsshim/errors.go @@ -83,7 +83,6 @@ type NetworkNotFoundError = hns.NetworkNotFoundError type ProcessError struct { Process *process Operation string - ExtraInfo string Err error Events []hcs.ErrorEvent } @@ -92,7 +91,6 @@ type ProcessError struct { type ContainerError struct { Container *container Operation string - ExtraInfo string Err error Events []hcs.ErrorEvent } @@ -125,22 +123,9 @@ func (e *ContainerError) Error() string { s += "\n" + ev.String() } - if e.ExtraInfo != "" { - s += " extra info: " + e.ExtraInfo - } - return s } -func makeContainerError(container *container, operation string, extraInfo string, err error) error { - // Don't double wrap errors - if _, ok := err.(*ContainerError); ok { - return err - } - containerError := &ContainerError{Container: container, Operation: operation, ExtraInfo: extraInfo, Err: err} - return containerError -} - func (e *ProcessError) Error() string { if e == nil { return "<nil>" @@ -171,15 +156,6 @@ func (e *ProcessError) Error() string { return s } -func makeProcessError(process *process, operation string, extraInfo string, err error) error { - // Don't double wrap errors - if _, ok := err.(*ProcessError); ok { - return err - } - processError := &ProcessError{Process: process, Operation: operation, ExtraInfo: extraInfo, Err: err} - return processError -} - // IsNotExist checks if an error is caused by the Container or Process not existing. // Note: Currently, ErrElementNotFound can mean that a Process has either // already exited, or does not exist. Both IsAlreadyStopped and IsNotExist @@ -256,7 +232,7 @@ func getInnerError(err error) error { func convertSystemError(err error, c *container) error { if serr, ok := err.(*hcs.SystemError); ok { - return &ContainerError{Container: c, Operation: serr.Op, ExtraInfo: serr.Extra, Err: serr.Err, Events: serr.Events} + return &ContainerError{Container: c, Operation: serr.Op, Err: serr.Err, Events: serr.Events} } return err } diff --git a/vendor/github.com/Microsoft/hcsshim/go.mod b/vendor/github.com/Microsoft/hcsshim/go.mod index a23d6e661..6e52f732b 100644 --- a/vendor/github.com/Microsoft/hcsshim/go.mod +++ b/vendor/github.com/Microsoft/hcsshim/go.mod @@ -4,10 +4,10 @@ go 1.13 require ( github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3 - github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102 + github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68 github.com/containerd/console v1.0.1 - github.com/containerd/containerd v1.5.0-beta.1 - github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328 + github.com/containerd/containerd v1.5.0-beta.4 + github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0 github.com/containerd/ttrpc v1.0.2 github.com/containerd/typeurl v1.0.1 github.com/gogo/protobuf v1.3.2 @@ -16,7 +16,7 @@ require ( github.com/sirupsen/logrus v1.7.0 github.com/urfave/cli v1.22.2 go.opencensus.io v0.22.3 - golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 - golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c - google.golang.org/grpc v1.30.0 + golang.org/x/sync v0.0.0-20201207232520-09787c993a3a + golang.org/x/sys v0.0.0-20210324051608-47abb6519492 + google.golang.org/grpc v1.33.2 ) diff --git a/vendor/github.com/Microsoft/hcsshim/go.sum b/vendor/github.com/Microsoft/hcsshim/go.sum index 57eb16762..545bb60b1 100644 --- a/vendor/github.com/Microsoft/hcsshim/go.sum +++ b/vendor/github.com/Microsoft/hcsshim/go.sum @@ -34,15 +34,12 @@ github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.15-0.20190919025122-fc70bd9a86b5/go.mod h1:tTuCMEN+UleMWgg9dVx4Hu52b1bJo+59jBh3ajtinzw= -github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331 h1:3YnB7Hpmh1lPecPE8doMOtYCrMdrpedZOvxfuNES/Vk= github.com/Microsoft/go-winio v0.4.16-0.20201130162521-d1ffc52c7331/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= -github.com/Microsoft/go-winio v0.4.16 h1:FtSW/jqD+l4ba5iPBj9CODVtgfYAD8w2wS923g/cFDk= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3 h1:mw6pDQqv38/WGF1cO/jF5t/jyAJ2yi7CmtFLLO5tGFI= github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= @@ -51,7 +48,9 @@ github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3h github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= +github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= +github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= @@ -90,15 +89,17 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= +github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= +github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= +github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= -github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59 h1:qWj4qVYZ95vLWwqyNJCQg7rDsG5wPdze0UaPolH7DUk= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102 h1:Qf4HiqfvmB7zS6scsmNgTLmByHbq8n9RTF39v+TzP7A= github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1 h1:uict5mhHFTzKLUCufdSLym7z/J0CbBJT59lYbP9wtbg= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68 h1:hkGVFjz+plgr5UfxZUTPFbUFIF/Km6/s+RVRIRHLrrY= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= @@ -108,49 +109,56 @@ github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtM github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.3.2 h1:ForxmXkA6tPIvffbrDAcPUIB32QgXkt2XFj+F0UxetA= github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.5.0-beta.1 h1:IK6yirB4X7wpKyFSikWiT++nZsyIxGAAgNEv3fEGuls= +github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= -github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc h1:TP+534wVlf61smEIq1nwLLAjQVEK2EADoW3CX9AuT+8= +github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= +github.com/containerd/containerd v1.5.0-beta.4 h1:zjz4MOAOFgdBlwid2nNUlJ3YLpVi/97L36lfMYJex60= +github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= -github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7 h1:6ejg6Lkk8dskcM7wQ28gONkukbQkM4qpj4RnYbpFzrI= github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e h1:6JKvHHt396/qabvMhnhUZvWaHZzfVfldxE60TK8YLhg= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= -github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448 h1:PUD50EuOMkXVcpBIA/R95d56duJR9VxhwncsFbNnxW4= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= -github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c h1:1c6xmkNiu6Jnr6AKGM91GGNsfU+nPNFvw9BZFSo0E+c= github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d h1:u6sWqdNGAy7+O8qG/r1dqdnZE7IdEjteK3WGuvbfreo= +github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= -github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3 h1:esQOJREg8nw8aXj6uCN5dfW5cKUBiEJ/+nni1Q/D/sw= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= -github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328 h1:PRTagVMbJcCezLcHXe8UJvR1oBzp2lG3CEumeFOLOds= github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= +github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0 h1:e+50zk22gvHLJKe8+d+xSMyA88PPQk/XfWuUw1BdnPA= +github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= +github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= +github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= -github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de h1:dlfGmNcE3jDAecLqwKPMNX6nk2qh1c1Vg1/YTzpOOF4= +github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= github.com/containerd/ttrpc v1.0.1/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= github.com/containerd/ttrpc v1.0.2 h1:2/O3oTZN36q2xRolk0a2WWGgh7/Vf/liElg5hFYLX9U= github.com/containerd/ttrpc v1.0.2/go.mod h1:UAxOpgT9ziI0gJrmKvgcZivgxOp8iFPSk8httJEt98Y= -github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd h1:JNn81o/xG+8NEo3bC/vx9pbi/g2WI8mtP2/nXzu297Y= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= github.com/containerd/typeurl v1.0.1 h1:PvuK4E3D5S5q6IqsPDCy928FhP0LUIGcmZ/Yhgp5Djw= github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= +github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= +github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= +github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -185,7 +193,6 @@ github.com/docker/go-events v0.0.0-20170721190031-9461782956ad/go.mod h1:Uw6Uezg github.com/docker/go-events v0.0.0-20190806004212-e31b211e4f1c/go.mod h1:Uw6UezgYA44ePAFQYUehOuCzmy5zmg/+nl2ZfMWGkpA= github.com/docker/go-metrics v0.0.0-20180209012529-399ea8c73916/go.mod h1:/u0gXw0Gay3ceNrsHubL3BtdOL2fHf93USgMTe0W5dI= github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= -github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= @@ -236,11 +243,9 @@ github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7a github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.2.2-0.20190723190241-65acae22fc9d/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.0/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= -github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= github.com/gogo/protobuf v1.3.1/go.mod h1:SlYgWuQ5SjCEi6WLHjHCa1yvBfUnHcTbrrZtXPKa29o= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= -github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -252,11 +257,8 @@ github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfb github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= -github.com/golang/protobuf v1.2.0 h1:P3YflyNX/ehuJFLhxviNdFxQPkGK5cDcApsge1SqnvM= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= -github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= @@ -273,14 +275,14 @@ github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= -github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -310,13 +312,13 @@ github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brv github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= -github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -334,13 +336,10 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/konsorten/go-windows-terminal-sequences v1.0.1 h1:mweAR1A6xJ3oS2pRaGiHgQ4OO8tzTaLawm8vnODuwDk= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -358,11 +357,13 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQZAeMln+1tSwduZz7+Af5oFlKirV/MSYes2A= -github.com/moby/sys/mountinfo v0.4.0 h1:1KInV3Huv18akCu58V7lzNlt+jFmqlu1EaErnEHE/VM= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= +github.com/moby/sys/mountinfo v0.4.1 h1:1O+1cHA1aujwEwwVMa2Xm2l+gIpUHyd3+D+d7LZh1kM= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -387,7 +388,6 @@ github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGV github.com/onsi/gomega v1.7.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= -github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2 h1:QhPf3A2AZW3tTGvHPg0TA+CR3oHbVLlXUhlghqISp1I= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1.0.20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -395,7 +395,6 @@ github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f h1:a969LJ4IQFwRHYqonHtUDMSh9i54WcKggeEkQ3fZMl4= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= @@ -404,7 +403,6 @@ github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84 github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d h1:pNa8metDkwZjb9g4T8s+krQ+HRgZAkqnXml+wNir/+s= github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -414,7 +412,6 @@ github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3 github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1-0.20171018195549-f15c970de5b7/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= -github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -435,7 +432,6 @@ github.com/prometheus/common v0.0.0-20180110214958-89604d197083/go.mod h1:daVV7q github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= -github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7 h1:hhvfGDVThBnd4kYisSFmYuHYeUhglxcwag7FhVPH9zM= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -444,8 +440,9 @@ github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= -github.com/prometheus/procfs v0.2.0 h1:wH4vA7pcjKuZzjF7lM8awk4fnuJO6idemZXoKnULUx4= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0 h1:mxy4L2jP6qMonqmq+aTtOx1ifVWUgG/TAmntgbh3xv4= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= @@ -458,9 +455,7 @@ github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeV github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= -github.com/sirupsen/logrus v1.4.1 h1:GL2rEmy6nsikmW0r8opw9JIRScdMF5hA8cOYLH7In1k= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0 h1:ShrD1U9pZB12TX0cVy0DtePoCH97K8EtX+mg7ZARUtM= @@ -476,15 +471,14 @@ github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bd github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= github.com/stretchr/testify v0.0.0-20180303142811-b89eecf5ca5d/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= -github.com/stretchr/testify v1.2.2 h1:bSDNvY7ZPG5RlJ8otE/7V6gMiyenm9RtJ7IUVIAoJ1w= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1 h1:hDPOHmpOpP40lSULcqw7IrRb/u7w6RpDC9399XyoNd0= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -517,8 +511,8 @@ github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= -go.opencensus.io v0.22.0 h1:C9hSCOW830chIVkdja34wa6Ky+IzWllkUinR+BtRZd4= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= @@ -537,6 +531,7 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -576,10 +571,8 @@ golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= -golang.org/x/net v0.0.0-20190311183353-d8887717615a h1:oWX7TPOiFAMXLq8o0ikBYfCJVlRHBcsciT5bXOrH628= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09 h1:KaQtG+aDELoNmXYas3TVkGNYRuq8JQ1aa7LJt8EXVyo= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= @@ -590,7 +583,6 @@ golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 h1:rjwSpXsdiK0dV8/Naq3kAw9ymfAeJIyd0upUIElB+lI= golang.org/x/net v0.0.0-20191004110552-13f9640d40b9/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191209160850-c0dbc17a3553/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -612,14 +604,13 @@ golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6 h1:bjcUS9ztw9kFmmIxJInhon/0Is3p+EHBKNgquIzo1OI= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -642,7 +633,6 @@ golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190812073006-9eafafc0a87e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3 h1:7TYNF4UdlohbFwpNH04CoPMp1cHUZgO1Ebq5r2hIjfo= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191001151750-bb3f8db39f24/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -655,7 +645,6 @@ golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191228213918-04cbcbbfeed8/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200106162015-b016eb3dc98e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200113162924-86b910548bc1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200120151820-655fe14d7479 h1:LhLiKguPgZL+Tglay4GhVtfF0kb8cvOJ0dHTCBO8YNI= golang.org/x/sys v0.0.0-20200120151820-655fe14d7479/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -665,25 +654,25 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3 h1:kzM6+9dur93BcC2kVlYl34cHU+TYZLanmpSJHVMmL64= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20201202213521-69691e467435 h1:25AvDqqB9PrNqj1FLf2/70I4W0L19qqoaFq3gjNwbKk= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.0 h1:g61tztE5qeGQ89tm6NTjjM9VPIm088od1l6aSorWRWg= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= -golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4 h1:0YWbFKbhXG/wIiuHDSKpS0Iy7FSA+u45VtBMfQcFTTc= @@ -752,13 +741,10 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= -google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8 h1:Nw54tB0rB7hY/N0NQvRW8DG4Yk3Q6T9cu9RcFQDu1tc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb h1:i1Ppqkc3WQXikh8bXiwHqAN5Rv3/qDCcRk0/Otx73BY= google.golang.org/genproto v0.0.0-20190425155659-357c62f0e4bb/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= -google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873 h1:nfPFGzJkUDX6uBmpN/pSw7MbOAWegH5QDQuoXFHedLg= google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190522204451-c2c4e71fbf69/go.mod h1:z3L6/3dTEVtUr6QSP8miRzeRqwQOioJ9I66odjN4I7s= google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= @@ -780,20 +766,19 @@ google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= -google.golang.org/grpc v1.20.1 h1:Hz2g2wirWK7H0qIIhGIqRGTuMwTE8HEKFnDZZ7lm9NU= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.0/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= -google.golang.org/grpc v1.23.1 h1:q4XQuHFC6I28BKZpo6IYyb3mNO+l7lSOxRuYTCiDfXk= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.24.0/go.mod h1:XDChyiUovWa60DnaeDeZmSW86xtLtjtZbwvSiRnRtcA= google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.30.0 h1:M5a8xTlYTxwMn5ZFkwhRabsygDY5G8TYLyQDBxJNAxE= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2 h1:EQyQC3sa8M+p6Ulc8yy9SWSS2GVwyRc83gAbG8lrl4o= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -809,7 +794,6 @@ gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4 gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -822,22 +806,23 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= -gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3 h1:4AuOwCGf4lLR9u3YOe2awrHygurzhO/HeQ6laiA6Sx0= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go b/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go index 09b3860a7..408312672 100644 --- a/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go +++ b/vendor/github.com/Microsoft/hcsshim/hnsendpoint.go @@ -40,6 +40,9 @@ func HNSListEndpointRequest() ([]HNSEndpoint, error) { // HotAttachEndpoint makes a HCS Call to attach the endpoint to the container func HotAttachEndpoint(containerID string, endpointID string) error { endpoint, err := GetHNSEndpointByID(endpointID) + if err != nil { + return err + } isAttached, err := endpoint.IsAttached(containerID) if isAttached { return err @@ -50,6 +53,9 @@ func HotAttachEndpoint(containerID string, endpointID string) error { // HotDetachEndpoint makes a HCS Call to detach the endpoint from the container func HotDetachEndpoint(containerID string, endpointID string) error { endpoint, err := GetHNSEndpointByID(endpointID) + if err != nil { + return err + } isAttached, err := endpoint.IsAttached(containerID) if !isAttached { return err diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go index bca824b8e..7696e4b48 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/hcs/errors.go @@ -171,7 +171,6 @@ type SystemError struct { ID string Op string Err error - Extra string Events []ErrorEvent } @@ -182,9 +181,6 @@ func (e *SystemError) Error() string { for _, ev := range e.Events { s += "\n" + ev.String() } - if e.Extra != "" { - s += "\n(extra info: " + e.Extra + ")" - } return s } @@ -198,7 +194,7 @@ func (e *SystemError) Timeout() bool { return ok && err.Timeout() } -func makeSystemError(system *System, op string, extra string, err error, events []ErrorEvent) error { +func makeSystemError(system *System, op string, err error, events []ErrorEvent) error { // Don't double wrap errors if _, ok := err.(*SystemError); ok { return err @@ -206,7 +202,6 @@ func makeSystemError(system *System, op string, extra string, err error, events return &SystemError{ ID: system.ID(), Op: op, - Extra: extra, Err: err, Events: events, } @@ -332,12 +327,3 @@ func getInnerError(err error) error { } return err } - -func getOperationLogResult(err error) (string, error) { - switch err { - case nil: - return "Success", nil - default: - return "Error", err - } -} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go index 2ad978f29..b74389eca 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/hcs/process.go @@ -64,11 +64,7 @@ type processStatus struct { LastWaitResult int32 } -const ( - stdIn string = "StdIn" - stdOut string = "StdOut" - stdErr string = "StdErr" -) +const stdIn string = "StdIn" const ( modifyConsoleSize string = "ConsoleSize" @@ -176,8 +172,10 @@ func (process *Process) waitBackground() { trace.Int64Attribute("pid", int64(process.processID))) var ( - err error - exitCode = -1 + err error + exitCode = -1 + propertiesJSON string + resultJSON string ) err = waitForNotification(ctx, process.callbackNumber, hcsNotificationProcessExited, nil) @@ -190,15 +188,15 @@ func (process *Process) waitBackground() { // Make sure we didnt race with Close() here if process.handle != 0 { - propertiesJSON, resultJSON, err := vmcompute.HcsGetProcessProperties(ctx, process.handle) + propertiesJSON, resultJSON, err = vmcompute.HcsGetProcessProperties(ctx, process.handle) events := processHcsResult(ctx, resultJSON) if err != nil { - err = makeProcessError(process, operation, err, events) + err = makeProcessError(process, operation, err, events) //nolint:ineffassign } else { properties := &processStatus{} err = json.Unmarshal([]byte(propertiesJSON), properties) if err != nil { - err = makeProcessError(process, operation, err, nil) + err = makeProcessError(process, operation, err, nil) //nolint:ineffassign } else { if properties.LastWaitResult != 0 { log.G(ctx).WithField("wait-result", properties.LastWaitResult).Warning("non-zero last wait result") @@ -468,7 +466,7 @@ func (process *Process) unregisterCallback(ctx context.Context) error { delete(callbackMap, callbackNumber) callbackMapLock.Unlock() - handle = 0 + handle = 0 //nolint:ineffassign return nil } diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go index bda393a6d..cea11dee7 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/hcs/system.go @@ -73,8 +73,8 @@ func CreateComputeSystem(ctx context.Context, id string, hcsDocumentInterface in if err = computeSystem.registerCallback(ctx); err != nil { // Terminate the compute system if it still exists. We're okay to // ignore a failure here. - computeSystem.Terminate(ctx) - return nil, makeSystemError(computeSystem, operation, "", err, nil) + _ = computeSystem.Terminate(ctx) + return nil, makeSystemError(computeSystem, operation, err, nil) } } @@ -83,9 +83,9 @@ func CreateComputeSystem(ctx context.Context, id string, hcsDocumentInterface in if err == ErrTimeout { // Terminate the compute system if it still exists. We're okay to // ignore a failure here. - computeSystem.Terminate(ctx) + _ = computeSystem.Terminate(ctx) } - return nil, makeSystemError(computeSystem, operation, hcsDocument, err, events) + return nil, makeSystemError(computeSystem, operation, err, events) } go computeSystem.waitBackground() if err = computeSystem.getCachedProperties(ctx); err != nil { @@ -102,7 +102,7 @@ func OpenComputeSystem(ctx context.Context, id string) (*System, error) { handle, resultJSON, err := vmcompute.HcsOpenComputeSystem(ctx, id) events := processHcsResult(ctx, resultJSON) if err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, events) + return nil, makeSystemError(computeSystem, operation, err, events) } computeSystem.handle = handle defer func() { @@ -111,7 +111,7 @@ func OpenComputeSystem(ctx context.Context, id string) (*System, error) { } }() if err = computeSystem.registerCallback(ctx); err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, nil) + return nil, makeSystemError(computeSystem, operation, err, nil) } go computeSystem.waitBackground() if err = computeSystem.getCachedProperties(ctx); err != nil { @@ -187,13 +187,13 @@ func (computeSystem *System) Start(ctx context.Context) (err error) { defer computeSystem.handleLock.RUnlock() if computeSystem.handle == 0 { - return makeSystemError(computeSystem, operation, "", ErrAlreadyClosed, nil) + return makeSystemError(computeSystem, operation, ErrAlreadyClosed, nil) } resultJSON, err := vmcompute.HcsStartComputeSystem(ctx, computeSystem.handle, "") events, err := processAsyncHcsResult(ctx, err, resultJSON, computeSystem.callbackNumber, hcsNotificationSystemStartCompleted, &timeout.SystemStart) if err != nil { - return makeSystemError(computeSystem, operation, "", err, events) + return makeSystemError(computeSystem, operation, err, events) } return nil @@ -220,7 +220,7 @@ func (computeSystem *System) Shutdown(ctx context.Context) error { switch err { case nil, ErrVmcomputeAlreadyStopped, ErrComputeSystemDoesNotExist, ErrVmcomputeOperationPending: default: - return makeSystemError(computeSystem, operation, "", err, events) + return makeSystemError(computeSystem, operation, err, events) } return nil } @@ -241,7 +241,7 @@ func (computeSystem *System) Terminate(ctx context.Context) error { switch err { case nil, ErrVmcomputeAlreadyStopped, ErrComputeSystemDoesNotExist, ErrVmcomputeOperationPending: default: - return makeSystemError(computeSystem, operation, "", err, events) + return makeSystemError(computeSystem, operation, err, events) } return nil } @@ -263,10 +263,10 @@ func (computeSystem *System) waitBackground() { log.G(ctx).Debug("system exited") case ErrVmcomputeUnexpectedExit: log.G(ctx).Debug("unexpected system exit") - computeSystem.exitError = makeSystemError(computeSystem, operation, "", err, nil) + computeSystem.exitError = makeSystemError(computeSystem, operation, err, nil) err = nil default: - err = makeSystemError(computeSystem, operation, "", err, nil) + err = makeSystemError(computeSystem, operation, err, nil) } computeSystem.closedWaitOnce.Do(func() { computeSystem.waitError = err @@ -304,13 +304,13 @@ func (computeSystem *System) Properties(ctx context.Context, types ...schema1.Pr queryBytes, err := json.Marshal(schema1.PropertyQuery{PropertyTypes: types}) if err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, nil) + return nil, makeSystemError(computeSystem, operation, err, nil) } propertiesJSON, resultJSON, err := vmcompute.HcsGetComputeSystemProperties(ctx, computeSystem.handle, string(queryBytes)) events := processHcsResult(ctx, resultJSON) if err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, events) + return nil, makeSystemError(computeSystem, operation, err, events) } if propertiesJSON == "" { @@ -318,7 +318,7 @@ func (computeSystem *System) Properties(ctx context.Context, types ...schema1.Pr } properties := &schema1.ContainerProperties{} if err := json.Unmarshal([]byte(propertiesJSON), properties); err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, nil) + return nil, makeSystemError(computeSystem, operation, err, nil) } return properties, nil @@ -333,13 +333,13 @@ func (computeSystem *System) PropertiesV2(ctx context.Context, types ...hcsschem queryBytes, err := json.Marshal(hcsschema.PropertyQuery{PropertyTypes: types}) if err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, nil) + return nil, makeSystemError(computeSystem, operation, err, nil) } propertiesJSON, resultJSON, err := vmcompute.HcsGetComputeSystemProperties(ctx, computeSystem.handle, string(queryBytes)) events := processHcsResult(ctx, resultJSON) if err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, events) + return nil, makeSystemError(computeSystem, operation, err, events) } if propertiesJSON == "" { @@ -347,7 +347,7 @@ func (computeSystem *System) PropertiesV2(ctx context.Context, types ...hcsschem } properties := &hcsschema.Properties{} if err := json.Unmarshal([]byte(propertiesJSON), properties); err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, nil) + return nil, makeSystemError(computeSystem, operation, err, nil) } return properties, nil @@ -368,13 +368,13 @@ func (computeSystem *System) Pause(ctx context.Context) (err error) { defer computeSystem.handleLock.RUnlock() if computeSystem.handle == 0 { - return makeSystemError(computeSystem, operation, "", ErrAlreadyClosed, nil) + return makeSystemError(computeSystem, operation, ErrAlreadyClosed, nil) } resultJSON, err := vmcompute.HcsPauseComputeSystem(ctx, computeSystem.handle, "") events, err := processAsyncHcsResult(ctx, err, resultJSON, computeSystem.callbackNumber, hcsNotificationSystemPauseCompleted, &timeout.SystemPause) if err != nil { - return makeSystemError(computeSystem, operation, "", err, events) + return makeSystemError(computeSystem, operation, err, events) } return nil @@ -395,13 +395,13 @@ func (computeSystem *System) Resume(ctx context.Context) (err error) { defer computeSystem.handleLock.RUnlock() if computeSystem.handle == 0 { - return makeSystemError(computeSystem, operation, "", ErrAlreadyClosed, nil) + return makeSystemError(computeSystem, operation, ErrAlreadyClosed, nil) } resultJSON, err := vmcompute.HcsResumeComputeSystem(ctx, computeSystem.handle, "") events, err := processAsyncHcsResult(ctx, err, resultJSON, computeSystem.callbackNumber, hcsNotificationSystemResumeCompleted, &timeout.SystemResume) if err != nil { - return makeSystemError(computeSystem, operation, "", err, events) + return makeSystemError(computeSystem, operation, err, events) } return nil @@ -427,13 +427,13 @@ func (computeSystem *System) Save(ctx context.Context, options interface{}) (err defer computeSystem.handleLock.RUnlock() if computeSystem.handle == 0 { - return makeSystemError(computeSystem, operation, "", ErrAlreadyClosed, nil) + return makeSystemError(computeSystem, operation, ErrAlreadyClosed, nil) } result, err := vmcompute.HcsSaveComputeSystem(ctx, computeSystem.handle, string(saveOptions)) events, err := processAsyncHcsResult(ctx, err, result, computeSystem.callbackNumber, hcsNotificationSystemSaveCompleted, &timeout.SystemSave) if err != nil { - return makeSystemError(computeSystem, operation, "", err, events) + return makeSystemError(computeSystem, operation, err, events) } return nil @@ -444,19 +444,19 @@ func (computeSystem *System) createProcess(ctx context.Context, operation string defer computeSystem.handleLock.RUnlock() if computeSystem.handle == 0 { - return nil, nil, makeSystemError(computeSystem, operation, "", ErrAlreadyClosed, nil) + return nil, nil, makeSystemError(computeSystem, operation, ErrAlreadyClosed, nil) } configurationb, err := json.Marshal(c) if err != nil { - return nil, nil, makeSystemError(computeSystem, operation, "", err, nil) + return nil, nil, makeSystemError(computeSystem, operation, err, nil) } configuration := string(configurationb) processInfo, processHandle, resultJSON, err := vmcompute.HcsCreateProcess(ctx, computeSystem.handle, configuration) events := processHcsResult(ctx, resultJSON) if err != nil { - return nil, nil, makeSystemError(computeSystem, operation, configuration, err, events) + return nil, nil, makeSystemError(computeSystem, operation, err, events) } log.G(ctx).WithField("pid", processInfo.ProcessId).Debug("created process pid") @@ -478,7 +478,7 @@ func (computeSystem *System) CreateProcess(ctx context.Context, c interface{}) ( pipes, err := makeOpenFiles([]syscall.Handle{processInfo.StdInput, processInfo.StdOutput, processInfo.StdError}) if err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, nil) + return nil, makeSystemError(computeSystem, operation, err, nil) } process.stdin = pipes[0] process.stdout = pipes[1] @@ -486,7 +486,7 @@ func (computeSystem *System) CreateProcess(ctx context.Context, c interface{}) ( process.hasCachedStdio = true if err = process.registerCallback(ctx); err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, nil) + return nil, makeSystemError(computeSystem, operation, err, nil) } go process.waitBackground() @@ -501,18 +501,18 @@ func (computeSystem *System) OpenProcess(ctx context.Context, pid int) (*Process operation := "hcsshim::System::OpenProcess" if computeSystem.handle == 0 { - return nil, makeSystemError(computeSystem, operation, "", ErrAlreadyClosed, nil) + return nil, makeSystemError(computeSystem, operation, ErrAlreadyClosed, nil) } processHandle, resultJSON, err := vmcompute.HcsOpenProcess(ctx, computeSystem.handle, uint32(pid)) events := processHcsResult(ctx, resultJSON) if err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, events) + return nil, makeSystemError(computeSystem, operation, err, events) } process := newProcess(processHandle, pid, computeSystem) if err = process.registerCallback(ctx); err != nil { - return nil, makeSystemError(computeSystem, operation, "", err, nil) + return nil, makeSystemError(computeSystem, operation, err, nil) } go process.waitBackground() @@ -536,12 +536,12 @@ func (computeSystem *System) Close() (err error) { } if err = computeSystem.unregisterCallback(ctx); err != nil { - return makeSystemError(computeSystem, operation, "", err, nil) + return makeSystemError(computeSystem, operation, err, nil) } err = vmcompute.HcsCloseComputeSystem(ctx, computeSystem.handle) if err != nil { - return makeSystemError(computeSystem, operation, "", err, nil) + return makeSystemError(computeSystem, operation, err, nil) } computeSystem.handle = 0 @@ -605,7 +605,7 @@ func (computeSystem *System) unregisterCallback(ctx context.Context) error { delete(callbackMap, callbackNumber) callbackMapLock.Unlock() - handle = 0 + handle = 0 //nolint:ineffassign return nil } @@ -618,7 +618,7 @@ func (computeSystem *System) Modify(ctx context.Context, config interface{}) err operation := "hcsshim::System::Modify" if computeSystem.handle == 0 { - return makeSystemError(computeSystem, operation, "", ErrAlreadyClosed, nil) + return makeSystemError(computeSystem, operation, ErrAlreadyClosed, nil) } requestBytes, err := json.Marshal(config) @@ -630,7 +630,7 @@ func (computeSystem *System) Modify(ctx context.Context, config interface{}) err resultJSON, err := vmcompute.HcsModifyComputeSystem(ctx, computeSystem.handle, requestJSON) events := processHcsResult(ctx, resultJSON) if err != nil { - return makeSystemError(computeSystem, operation, requestJSON, err, events) + return makeSystemError(computeSystem, operation, err, events) } return nil diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hcs/waithelper.go b/vendor/github.com/Microsoft/hcsshim/internal/hcs/waithelper.go index f07f532c1..db4e14fdf 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/hcs/waithelper.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/hcs/waithelper.go @@ -65,5 +65,4 @@ func waitForNotification(ctx context.Context, callbackNumber uintptr, expectedNo case <-c: return ErrTimeout } - return nil } diff --git a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsnetwork.go b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsnetwork.go index b7ae96fdd..f12d3ab04 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsnetwork.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/hns/hnsnetwork.go @@ -39,12 +39,6 @@ type HNSNetwork struct { AutomaticDNS bool `json:",omitempty"` } -type hnsNetworkResponse struct { - Success bool - Error string - Output HNSNetwork -} - type hnsResponse struct { Success bool Error string diff --git a/vendor/github.com/Microsoft/hcsshim/internal/safefile/safeopen.go b/vendor/github.com/Microsoft/hcsshim/internal/safefile/safeopen.go index 05f22f39d..66b8d7e03 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/safefile/safeopen.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/safefile/safeopen.go @@ -244,7 +244,7 @@ func RemoveRelative(path string, root *os.File) error { err = deleteOnClose(f) if err == syscall.ERROR_ACCESS_DENIED { // Maybe the file is marked readonly. Clear the bit and retry. - clearReadOnly(f) + _ = clearReadOnly(f) err = deleteOnClose(f) } } diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go b/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go index cd1ec84ab..b468ad636 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/schema1/schema1.go @@ -119,9 +119,9 @@ type PropertyType string const ( PropertyTypeStatistics PropertyType = "Statistics" // V1 and V2 - PropertyTypeProcessList = "ProcessList" // V1 and V2 - PropertyTypeMappedVirtualDisk = "MappedVirtualDisk" // Not supported in V2 schema call - PropertyTypeGuestConnection = "GuestConnection" // V1 and V2. Nil return from HCS before RS5 + PropertyTypeProcessList PropertyType = "ProcessList" // V1 and V2 + PropertyTypeMappedVirtualDisk PropertyType = "MappedVirtualDisk" // Not supported in V2 schema call + PropertyTypeGuestConnection PropertyType = "GuestConnection" // V1 and V2. Nil return from HCS before RS5 ) type PropertyQuery struct { diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/device.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/device.go index 0b9c0fbf7..107caddad 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/device.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/schema2/device.go @@ -13,8 +13,8 @@ type DeviceType string const ( ClassGUID DeviceType = "ClassGuid" - DeviceInstance = "DeviceInstance" - GPUMirror = "GpuMirror" + DeviceInstance DeviceType = "DeviceInstance" + GPUMirror DeviceType = "GpuMirror" ) type Device struct { diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/interrupt_moderation_mode.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/interrupt_moderation_mode.go new file mode 100644 index 000000000..a614d63bd --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/internal/schema2/interrupt_moderation_mode.go @@ -0,0 +1,42 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type InterruptModerationName string + +// The valid interrupt moderation modes for I/O virtualization (IOV) offloading. +const ( + DefaultName InterruptModerationName = "Default" + AdaptiveName InterruptModerationName = "Adaptive" + OffName InterruptModerationName = "Off" + LowName InterruptModerationName = "Low" + MediumName InterruptModerationName = "Medium" + HighName InterruptModerationName = "High" +) + +type InterruptModerationValue uint32 + +const ( + DefaultValue InterruptModerationValue = iota + AdaptiveValue + OffValue + LowValue InterruptModerationValue = 100 + MediumValue InterruptModerationValue = 200 + HighValue InterruptModerationValue = 300 +) + +var InterruptModerationValueToName = map[InterruptModerationValue]InterruptModerationName{ + DefaultValue: DefaultName, + AdaptiveValue: AdaptiveName, + OffValue: OffName, + LowValue: LowName, + MediumValue: MediumName, + HighValue: HighName, +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/iov_settings.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/iov_settings.go new file mode 100644 index 000000000..2a55cc37c --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/internal/schema2/iov_settings.go @@ -0,0 +1,22 @@ +/* + * HCS API + * + * No description provided (generated by Swagger Codegen https://github.com/swagger-api/swagger-codegen) + * + * API version: 2.4 + * Generated by: Swagger Codegen (https://github.com/swagger-api/swagger-codegen.git) + */ + +package hcsschema + +type IovSettings struct { + // The weight assigned to this port for I/O virtualization (IOV) offloading. + // Setting this to 0 disables IOV offloading. + OffloadWeight *uint32 `json:"OffloadWeight,omitempty"` + + // The number of queue pairs requested for this port for I/O virtualization (IOV) offloading. + QueuePairsRequested *uint32 `json:"QueuePairsRequested,omitempty"` + + // The interrupt moderation mode for I/O virtualization (IOV) offloading. + InterruptModeration *InterruptModerationName `json:"InterruptModeration,omitempty"` +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/logical_processor.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/logical_processor.go index 676ad300d..2e3aa5e17 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/logical_processor.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/schema2/logical_processor.go @@ -11,8 +11,8 @@ package hcsschema type LogicalProcessor struct { LpIndex uint32 `json:"LpIndex,omitempty"` - NodeNumber uint8 `json:"NodeNumber, omitempty"` - PackageId uint32 `json:"PackageId, omitempty"` - CoreId uint32 `json:"CoreId, omitempty"` - RootVpIndex int32 `json:"RootVpIndex, omitempty"` + NodeNumber uint8 `json:"NodeNumber,omitempty"` + PackageId uint32 `json:"PackageId,omitempty"` + CoreId uint32 `json:"CoreId,omitempty"` + RootVpIndex int32 `json:"RootVpIndex,omitempty"` } diff --git a/vendor/github.com/Microsoft/hcsshim/internal/schema2/network_adapter.go b/vendor/github.com/Microsoft/hcsshim/internal/schema2/network_adapter.go index a9c750b34..7408abd31 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/schema2/network_adapter.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/schema2/network_adapter.go @@ -11,6 +11,7 @@ package hcsschema type NetworkAdapter struct { EndpointId string `json:"EndpointId,omitempty"` - MacAddress string `json:"MacAddress,omitempty"` + // The I/O virtualization (IOV) offloading configuration. + IovSettings *IovSettings `json:"IovSettings,omitempty"` } diff --git a/vendor/github.com/Microsoft/hcsshim/internal/vmcompute/vmcompute.go b/vendor/github.com/Microsoft/hcsshim/internal/vmcompute/vmcompute.go index 32491f2c3..e7f114b67 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/vmcompute/vmcompute.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/vmcompute/vmcompute.go @@ -62,7 +62,7 @@ type HcsCallback syscall.Handle type HcsProcessInformation struct { // ProcessId is the pid of the created process. ProcessId uint32 - reserved uint32 + reserved uint32 //nolint:structcheck // StdInput is the handle associated with the stdin of the process. StdInput syscall.Handle // StdOutput is the handle associated with the stdout of the process. diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/activatelayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/activatelayer.go index 81e454956..ff81ac2c1 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/activatelayer.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/activatelayer.go @@ -14,7 +14,7 @@ import ( // An activated layer must later be deactivated via DeactivateLayer. func ActivateLayer(ctx context.Context, path string) (err error) { title := "hcsshim::ActivateLayer" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("path", path)) diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createlayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createlayer.go index 41e5e6731..ffee31ab1 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createlayer.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/createlayer.go @@ -12,7 +12,7 @@ import ( // the parent layer provided. func CreateLayer(ctx context.Context, path, parent string) (err error) { title := "hcsshim::CreateLayer" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes( diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/deactivatelayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/deactivatelayer.go index 70a711cf5..d5bf2f5bd 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/deactivatelayer.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/deactivatelayer.go @@ -11,7 +11,7 @@ import ( // DeactivateLayer will dismount a layer that was mounted via ActivateLayer. func DeactivateLayer(ctx context.Context, path string) (err error) { title := "hcsshim::DeactivateLayer" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("path", path)) diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/destroylayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/destroylayer.go index bf197e3b0..787054e79 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/destroylayer.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/destroylayer.go @@ -12,7 +12,7 @@ import ( // path, including that layer's containing folder, if any. func DestroyLayer(ctx context.Context, path string) (err error) { title := "hcsshim::DestroyLayer" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("path", path)) diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getlayermountpath.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getlayermountpath.go index 942e3bbf9..4d22d0ecf 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getlayermountpath.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getlayermountpath.go @@ -21,8 +21,7 @@ func GetLayerMountPath(ctx context.Context, path string) (_ string, err error) { defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("path", path)) - var mountPathLength uintptr - mountPathLength = 0 + var mountPathLength uintptr = 0 // Call the procedure itself. log.G(ctx).Debug("Calling proc (1)") diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getsharedbaseimages.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getsharedbaseimages.go index a50378f49..bcc8fbd42 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getsharedbaseimages.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/getsharedbaseimages.go @@ -14,7 +14,7 @@ import ( // of registering them with the graphdriver, graph, and tagstore. func GetSharedBaseImages(ctx context.Context) (_ string, err error) { title := "hcsshim::GetSharedBaseImages" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/grantvmaccess.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/grantvmaccess.go index aa7c8ae1f..3eaca2780 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/grantvmaccess.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/grantvmaccess.go @@ -11,7 +11,7 @@ import ( // GrantVmAccess adds access to a file for a given VM func GrantVmAccess(ctx context.Context, vmid string, filepath string) (err error) { title := "hcsshim::GrantVmAccess" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes( diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerexists.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerexists.go index 6dd6f2d57..c6999973c 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerexists.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/layerexists.go @@ -12,7 +12,7 @@ import ( // to the system. func LayerExists(ctx context.Context, path string) (_ bool, err error) { title := "hcsshim::LayerExists" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("path", path)) diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/legacy.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/legacy.go index dc3caf751..83ba72cfa 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/legacy.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/legacy.go @@ -390,7 +390,7 @@ func (w *legacyLayerWriter) CloseRoots() { w.destRoot = nil } for i := range w.parentRoots { - w.parentRoots[i].Close() + _ = w.parentRoots[i].Close() } w.parentRoots = nil } @@ -640,7 +640,7 @@ func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro defer func() { if f != nil { f.Close() - safefile.RemoveRelative(name, w.destRoot) + _ = safefile.RemoveRelative(name, w.destRoot) } }() @@ -676,7 +676,7 @@ func (w *legacyLayerWriter) Add(name string, fileInfo *winio.FileBasicInfo) erro defer func() { if f != nil { f.Close() - safefile.RemoveRelative(fname, w.root) + _ = safefile.RemoveRelative(fname, w.root) } }() diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/nametoguid.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/nametoguid.go index b732857b3..bcf39c6b8 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/nametoguid.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/nametoguid.go @@ -14,7 +14,7 @@ import ( // across all clients. func NameToGuid(ctx context.Context, name string) (_ guid.GUID, err error) { title := "hcsshim::NameToGuid" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("name", name)) diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/processimage.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/processimage.go index aabb31368..30bcdff5f 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/processimage.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/processimage.go @@ -12,7 +12,7 @@ import ( // The files should have been extracted to <path>\Files. func ProcessBaseLayer(ctx context.Context, path string) (err error) { title := "hcsshim::ProcessBaseLayer" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("path", path)) @@ -28,7 +28,7 @@ func ProcessBaseLayer(ctx context.Context, path string) (err error) { // The files should have been extracted to <path>\Files. func ProcessUtilityVMImage(ctx context.Context, path string) (err error) { title := "hcsshim::ProcessUtilityVMImage" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("path", path)) diff --git a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/unpreparelayer.go b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/unpreparelayer.go index 84f81848f..79fb98678 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/wclayer/unpreparelayer.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/wclayer/unpreparelayer.go @@ -12,7 +12,7 @@ import ( // the given id. func UnprepareLayer(ctx context.Context, path string) (err error) { title := "hcsshim::UnprepareLayer" - ctx, span := trace.StartSpan(ctx, title) + ctx, span := trace.StartSpan(ctx, title) //nolint:ineffassign,staticcheck defer span.End() defer func() { oc.SetSpanStatus(span, err) }() span.AddAttributes(trace.StringAttribute("path", path)) diff --git a/vendor/github.com/Microsoft/hcsshim/internal/winapi/memory.go b/vendor/github.com/Microsoft/hcsshim/internal/winapi/memory.go index ccaf5a624..83f704064 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/winapi/memory.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/winapi/memory.go @@ -9,3 +9,19 @@ package winapi //sys LocalAlloc(flags uint32, size int) (ptr uintptr) = kernel32.LocalAlloc //sys LocalFree(ptr uintptr) = kernel32.LocalFree + +// BOOL QueryWorkingSet( +// HANDLE hProcess, +// PVOID pv, +// DWORD cb +// ); +//sys QueryWorkingSet(handle windows.Handle, pv uintptr, cb uint32) (err error) = psapi.QueryWorkingSet + +type PSAPI_WORKING_SET_INFORMATION struct { + NumberOfEntries uintptr + WorkingSetInfo [1]PSAPI_WORKING_SET_BLOCK +} + +type PSAPI_WORKING_SET_BLOCK struct { + Flags uintptr +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/winapi/path.go b/vendor/github.com/Microsoft/hcsshim/internal/winapi/path.go index 0ae8f33ea..908920e87 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/winapi/path.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/winapi/path.go @@ -8,4 +8,4 @@ package winapi // LPWSTR lpBuffer, // LPWSTR *lpFilePart // ); -//sys SearchPath(lpPath *uint16, lpFileName *uint16, lpExtension *uint16, nBufferLength uint32, lpBuffer *uint16, lpFilePath **uint16) (size uint32, err error) = kernel32.SearchPathW +//sys SearchPath(lpPath *uint16, lpFileName *uint16, lpExtension *uint16, nBufferLength uint32, lpBuffer *uint16, lpFilePath *uint16) (size uint32, err error) = kernel32.SearchPathW diff --git a/vendor/github.com/Microsoft/hcsshim/internal/winapi/process.go b/vendor/github.com/Microsoft/hcsshim/internal/winapi/process.go index adf0168ea..b87068327 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/winapi/process.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/winapi/process.go @@ -1,3 +1,10 @@ package winapi const PROCESS_ALL_ACCESS uint32 = 2097151 + +// DWORD GetProcessImageFileNameW( +// HANDLE hProcess, +// LPWSTR lpImageFileName, +// DWORD nSize +// ); +//sys GetProcessImageFileName(hProcess windows.Handle, imageFileName *uint16, nSize uint32) (size uint32, err error) = kernel32.GetProcessImageFileNameW diff --git a/vendor/github.com/Microsoft/hcsshim/internal/winapi/system.go b/vendor/github.com/Microsoft/hcsshim/internal/winapi/system.go new file mode 100644 index 000000000..327f57d7c --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/internal/winapi/system.go @@ -0,0 +1,52 @@ +package winapi + +import "golang.org/x/sys/windows" + +const SystemProcessInformation = 5 + +const STATUS_INFO_LENGTH_MISMATCH = 0xC0000004 + +// __kernel_entry NTSTATUS NtQuerySystemInformation( +// SYSTEM_INFORMATION_CLASS SystemInformationClass, +// PVOID SystemInformation, +// ULONG SystemInformationLength, +// PULONG ReturnLength +// ); +//sys NtQuerySystemInformation(systemInfoClass int, systemInformation uintptr, systemInfoLength uint32, returnLength *uint32) (status uint32) = ntdll.NtQuerySystemInformation + +type SYSTEM_PROCESS_INFORMATION struct { + NextEntryOffset uint32 // ULONG + NumberOfThreads uint32 // ULONG + WorkingSetPrivateSize int64 // LARGE_INTEGER + HardFaultCount uint32 // ULONG + NumberOfThreadsHighWatermark uint32 // ULONG + CycleTime uint64 // ULONGLONG + CreateTime int64 // LARGE_INTEGER + UserTime int64 // LARGE_INTEGER + KernelTime int64 // LARGE_INTEGER + ImageName UnicodeString // UNICODE_STRING + BasePriority int32 // KPRIORITY + UniqueProcessID windows.Handle // HANDLE + InheritedFromUniqueProcessID windows.Handle // HANDLE + HandleCount uint32 // ULONG + SessionID uint32 // ULONG + UniqueProcessKey *uint32 // ULONG_PTR + PeakVirtualSize uintptr // SIZE_T + VirtualSize uintptr // SIZE_T + PageFaultCount uint32 // ULONG + PeakWorkingSetSize uintptr // SIZE_T + WorkingSetSize uintptr // SIZE_T + QuotaPeakPagedPoolUsage uintptr // SIZE_T + QuotaPagedPoolUsage uintptr // SIZE_T + QuotaPeakNonPagedPoolUsage uintptr // SIZE_T + QuotaNonPagedPoolUsage uintptr // SIZE_T + PagefileUsage uintptr // SIZE_T + PeakPagefileUsage uintptr // SIZE_T + PrivatePageCount uintptr // SIZE_T + ReadOperationCount int64 // LARGE_INTEGER + WriteOperationCount int64 // LARGE_INTEGER + OtherOperationCount int64 // LARGE_INTEGER + ReadTransferCount int64 // LARGE_INTEGER + WriteTransferCount int64 // LARGE_INTEGER + OtherTransferCount int64 // LARGE_INTEGER +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/winapi/thread.go b/vendor/github.com/Microsoft/hcsshim/internal/winapi/thread.go new file mode 100644 index 000000000..4724713e3 --- /dev/null +++ b/vendor/github.com/Microsoft/hcsshim/internal/winapi/thread.go @@ -0,0 +1,12 @@ +package winapi + +// HANDLE CreateRemoteThread( +// HANDLE hProcess, +// LPSECURITY_ATTRIBUTES lpThreadAttributes, +// SIZE_T dwStackSize, +// LPTHREAD_START_ROUTINE lpStartAddress, +// LPVOID lpParameter, +// DWORD dwCreationFlags, +// LPDWORD lpThreadId +// ); +//sys CreateRemoteThread(process windows.Handle, sa *windows.SecurityAttributes, stackSize uint32, startAddr uintptr, parameter uintptr, creationFlags uint32, threadID *uint32) (handle windows.Handle, err error) = kernel32.CreateRemoteThread diff --git a/vendor/github.com/Microsoft/hcsshim/internal/winapi/winapi.go b/vendor/github.com/Microsoft/hcsshim/internal/winapi/winapi.go index 77ea13e3e..ec88c0d21 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/winapi/winapi.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/winapi/winapi.go @@ -2,4 +2,4 @@ // be thought of as an extension to golang.org/x/sys/windows. package winapi -//go:generate go run ..\..\mksyscall_windows.go -output zsyscall_windows.go net.go iocp.go jobobject.go path.go logon.go memory.go processor.go devices.go filesystem.go errors.go +//go:generate go run ..\..\mksyscall_windows.go -output zsyscall_windows.go system.go net.go path.go thread.go iocp.go jobobject.go logon.go memory.go process.go processor.go devices.go filesystem.go errors.go diff --git a/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go index 3a54c1fa1..2941b0f98 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go @@ -37,13 +37,17 @@ func errnoErr(e syscall.Errno) error { } var ( + modntdll = windows.NewLazySystemDLL("ntdll.dll") modiphlpapi = windows.NewLazySystemDLL("iphlpapi.dll") modkernel32 = windows.NewLazySystemDLL("kernel32.dll") - modntdll = windows.NewLazySystemDLL("ntdll.dll") modadvapi32 = windows.NewLazySystemDLL("advapi32.dll") + modpsapi = windows.NewLazySystemDLL("psapi.dll") modcfgmgr32 = windows.NewLazySystemDLL("cfgmgr32.dll") + procNtQuerySystemInformation = modntdll.NewProc("NtQuerySystemInformation") procSetJobCompartmentId = modiphlpapi.NewProc("SetJobCompartmentId") + procSearchPathW = modkernel32.NewProc("SearchPathW") + procCreateRemoteThread = modkernel32.NewProc("CreateRemoteThread") procGetQueuedCompletionStatus = modkernel32.NewProc("GetQueuedCompletionStatus") procIsProcessInJob = modkernel32.NewProc("IsProcessInJob") procQueryInformationJobObject = modkernel32.NewProc("QueryInformationJobObject") @@ -52,11 +56,12 @@ var ( procQueryIoRateControlInformationJobObject = modkernel32.NewProc("QueryIoRateControlInformationJobObject") procNtOpenJobObject = modntdll.NewProc("NtOpenJobObject") procNtCreateJobObject = modntdll.NewProc("NtCreateJobObject") - procSearchPathW = modkernel32.NewProc("SearchPathW") procLogonUserW = modadvapi32.NewProc("LogonUserW") procRtlMoveMemory = modkernel32.NewProc("RtlMoveMemory") procLocalAlloc = modkernel32.NewProc("LocalAlloc") procLocalFree = modkernel32.NewProc("LocalFree") + procQueryWorkingSet = modpsapi.NewProc("QueryWorkingSet") + procGetProcessImageFileNameW = modkernel32.NewProc("GetProcessImageFileNameW") procGetActiveProcessorCount = modkernel32.NewProc("GetActiveProcessorCount") procCM_Get_Device_ID_List_SizeA = modcfgmgr32.NewProc("CM_Get_Device_ID_List_SizeA") procCM_Get_Device_ID_ListA = modcfgmgr32.NewProc("CM_Get_Device_ID_ListA") @@ -69,6 +74,12 @@ var ( procRtlNtStatusToDosError = modntdll.NewProc("RtlNtStatusToDosError") ) +func NtQuerySystemInformation(systemInfoClass int, systemInformation uintptr, systemInfoLength uint32, returnLength *uint32) (status uint32) { + r0, _, _ := syscall.Syscall6(procNtQuerySystemInformation.Addr(), 4, uintptr(systemInfoClass), uintptr(systemInformation), uintptr(systemInfoLength), uintptr(unsafe.Pointer(returnLength)), 0, 0) + status = uint32(r0) + return +} + func SetJobCompartmentId(handle windows.Handle, compartmentId uint32) (win32Err error) { r0, _, _ := syscall.Syscall(procSetJobCompartmentId.Addr(), 2, uintptr(handle), uintptr(compartmentId), 0) if r0 != 0 { @@ -77,6 +88,32 @@ func SetJobCompartmentId(handle windows.Handle, compartmentId uint32) (win32Err return } +func SearchPath(lpPath *uint16, lpFileName *uint16, lpExtension *uint16, nBufferLength uint32, lpBuffer *uint16, lpFilePath *uint16) (size uint32, err error) { + r0, _, e1 := syscall.Syscall6(procSearchPathW.Addr(), 6, uintptr(unsafe.Pointer(lpPath)), uintptr(unsafe.Pointer(lpFileName)), uintptr(unsafe.Pointer(lpExtension)), uintptr(nBufferLength), uintptr(unsafe.Pointer(lpBuffer)), uintptr(unsafe.Pointer(lpFilePath))) + size = uint32(r0) + if size == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func CreateRemoteThread(process windows.Handle, sa *windows.SecurityAttributes, stackSize uint32, startAddr uintptr, parameter uintptr, creationFlags uint32, threadID *uint32) (handle windows.Handle, err error) { + r0, _, e1 := syscall.Syscall9(procCreateRemoteThread.Addr(), 7, uintptr(process), uintptr(unsafe.Pointer(sa)), uintptr(stackSize), uintptr(startAddr), uintptr(parameter), uintptr(creationFlags), uintptr(unsafe.Pointer(threadID)), 0, 0) + handle = windows.Handle(r0) + if handle == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + func GetQueuedCompletionStatus(cphandle windows.Handle, qty *uint32, key *uintptr, overlapped **windows.Overlapped, timeout uint32) (err error) { r1, _, e1 := syscall.Syscall6(procGetQueuedCompletionStatus.Addr(), 5, uintptr(cphandle), uintptr(unsafe.Pointer(qty)), uintptr(unsafe.Pointer(key)), uintptr(unsafe.Pointer(overlapped)), uintptr(timeout), 0) if r1 == 0 { @@ -170,19 +207,6 @@ func NtCreateJobObject(jobHandle *windows.Handle, desiredAccess uint32, objAttri return } -func SearchPath(lpPath *uint16, lpFileName *uint16, lpExtension *uint16, nBufferLength uint32, lpBuffer *uint16, lpFilePath **uint16) (size uint32, err error) { - r0, _, e1 := syscall.Syscall6(procSearchPathW.Addr(), 6, uintptr(unsafe.Pointer(lpPath)), uintptr(unsafe.Pointer(lpFileName)), uintptr(unsafe.Pointer(lpExtension)), uintptr(nBufferLength), uintptr(unsafe.Pointer(lpBuffer)), uintptr(unsafe.Pointer(lpFilePath))) - size = uint32(r0) - if size == 0 { - if e1 != 0 { - err = errnoErr(e1) - } else { - err = syscall.EINVAL - } - } - return -} - func LogonUser(username *uint16, domain *uint16, password *uint16, logonType uint32, logonProvider uint32, token *windows.Token) (err error) { r1, _, e1 := syscall.Syscall6(procLogonUserW.Addr(), 6, uintptr(unsafe.Pointer(username)), uintptr(unsafe.Pointer(domain)), uintptr(unsafe.Pointer(password)), uintptr(logonType), uintptr(logonProvider), uintptr(unsafe.Pointer(token))) if r1 == 0 { @@ -218,6 +242,31 @@ func LocalFree(ptr uintptr) { return } +func QueryWorkingSet(handle windows.Handle, pv uintptr, cb uint32) (err error) { + r1, _, e1 := syscall.Syscall(procQueryWorkingSet.Addr(), 3, uintptr(handle), uintptr(pv), uintptr(cb)) + if r1 == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + +func GetProcessImageFileName(hProcess windows.Handle, imageFileName *uint16, nSize uint32) (size uint32, err error) { + r0, _, e1 := syscall.Syscall(procGetProcessImageFileNameW.Addr(), 3, uintptr(hProcess), uintptr(unsafe.Pointer(imageFileName)), uintptr(nSize)) + size = uint32(r0) + if size == 0 { + if e1 != 0 { + err = errnoErr(e1) + } else { + err = syscall.EINVAL + } + } + return +} + func GetActiveProcessorCount(groupNumber uint16) (amount uint32) { r0, _, _ := syscall.Syscall(procGetActiveProcessorCount.Addr(), 1, uintptr(groupNumber), 0, 0) amount = uint32(r0) diff --git a/vendor/github.com/Microsoft/hcsshim/osversion/osversion_windows.go b/vendor/github.com/Microsoft/hcsshim/osversion/osversion_windows.go index 477fe7078..42e58403d 100644 --- a/vendor/github.com/Microsoft/hcsshim/osversion/osversion_windows.go +++ b/vendor/github.com/Microsoft/hcsshim/osversion/osversion_windows.go @@ -15,21 +15,6 @@ type OSVersion struct { Build uint16 } -// https://msdn.microsoft.com/en-us/library/windows/desktop/ms724833(v=vs.85).aspx -type osVersionInfoEx struct { - OSVersionInfoSize uint32 - MajorVersion uint32 - MinorVersion uint32 - BuildNumber uint32 - PlatformID uint32 - CSDVersion [128]uint16 - ServicePackMajor uint16 - ServicePackMinor uint16 - SuiteMask uint16 - ProductType byte - Reserve byte -} - // Get gets the operating system version on Windows. // The calling application must be manifested to get the correct version information. func Get() OSVersion { diff --git a/vendor/github.com/containers/image/v5/copy/copy.go b/vendor/github.com/containers/image/v5/copy/copy.go index 3a2ee9a8f..165a8be4b 100644 --- a/vendor/github.com/containers/image/v5/copy/copy.go +++ b/vendor/github.com/containers/image/v5/copy/copy.go @@ -16,6 +16,7 @@ import ( "github.com/containers/image/v5/image" internalblobinfocache "github.com/containers/image/v5/internal/blobinfocache" "github.com/containers/image/v5/internal/pkg/platform" + internalTypes "github.com/containers/image/v5/internal/types" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/blobinfocache" "github.com/containers/image/v5/pkg/compression" @@ -28,8 +29,8 @@ import ( imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/vbauerster/mpb/v5" - "github.com/vbauerster/mpb/v5/decor" + "github.com/vbauerster/mpb/v6" + "github.com/vbauerster/mpb/v6/decor" "golang.org/x/crypto/ssh/terminal" "golang.org/x/sync/semaphore" ) @@ -46,7 +47,7 @@ var ( // ErrDecryptParamsMissing is returned if there is missing decryption parameters ErrDecryptParamsMissing = errors.New("Necessary DecryptParameters not present") - // maxParallelDownloads is used to limit the maxmimum number of parallel + // maxParallelDownloads is used to limit the maximum number of parallel // downloads. Let's follow Firefox by limiting it to 6. maxParallelDownloads = uint(6) ) @@ -108,19 +109,20 @@ func (d *digestingReader) Read(p []byte) (int, error) { // copier allows us to keep track of diffID values for blobs, and other // data shared across one or more images in a possible manifest list. type copier struct { - dest types.ImageDestination - rawSource types.ImageSource - reportWriter io.Writer - progressOutput io.Writer - progressInterval time.Duration - progress chan types.ProgressProperties - blobInfoCache internalblobinfocache.BlobInfoCache2 - copyInParallel bool - compressionFormat compression.Algorithm - compressionLevel *int - ociDecryptConfig *encconfig.DecryptConfig - ociEncryptConfig *encconfig.EncryptConfig - maxParallelDownloads uint + dest types.ImageDestination + rawSource types.ImageSource + reportWriter io.Writer + progressOutput io.Writer + progressInterval time.Duration + progress chan types.ProgressProperties + blobInfoCache internalblobinfocache.BlobInfoCache2 + copyInParallel bool + compressionFormat compression.Algorithm + compressionLevel *int + ociDecryptConfig *encconfig.DecryptConfig + ociEncryptConfig *encconfig.EncryptConfig + maxParallelDownloads uint + downloadForeignLayers bool } // imageCopier tracks state specific to a single image (possibly an item of a manifest list) @@ -194,6 +196,13 @@ type Options struct { OciDecryptConfig *encconfig.DecryptConfig // MaxParallelDownloads indicates the maximum layers to pull at the same time. A reasonable default is used if this is left as 0. MaxParallelDownloads uint + // When OptimizeDestinationImageAlreadyExists is set, optimize the copy assuming that the destination image already + // exists (and is equivalent). Making the eventual (no-op) copy more performant for this case. Enabling the option + // is slightly pessimistic if the destination image doesn't exist, or is not equivalent. + OptimizeDestinationImageAlreadyExists bool + // Download layer contents with "nondistributable" media types ("foreign" layers) and translate the layer media type + // to not indicate "nondistributable". + DownloadForeignLayers bool } // validateImageListSelection returns an error if the passed-in value is not one that we recognize as a valid ImageListSelection value @@ -269,10 +278,11 @@ func Image(ctx context.Context, policyContext *signature.PolicyContext, destRef, // FIXME? The cache is used for sources and destinations equally, but we only have a SourceCtx and DestinationCtx. // For now, use DestinationCtx (because blob reuse changes the behavior of the destination side more); eventually // we might want to add a separate CommonCtx — or would that be too confusing? - blobInfoCache: internalblobinfocache.FromBlobInfoCache(blobinfocache.DefaultCache(options.DestinationCtx)), - ociDecryptConfig: options.OciDecryptConfig, - ociEncryptConfig: options.OciEncryptConfig, - maxParallelDownloads: options.MaxParallelDownloads, + blobInfoCache: internalblobinfocache.FromBlobInfoCache(blobinfocache.DefaultCache(options.DestinationCtx)), + ociDecryptConfig: options.OciDecryptConfig, + ociEncryptConfig: options.OciEncryptConfig, + maxParallelDownloads: options.MaxParallelDownloads, + downloadForeignLayers: options.DownloadForeignLayers, } // Default to using gzip compression unless specified otherwise. if options.DestinationCtx == nil || options.DestinationCtx.CompressionFormat == nil { @@ -361,6 +371,45 @@ func supportsMultipleImages(dest types.ImageDestination) bool { return false } +// compareImageDestinationManifestEqual compares the `src` and `dest` image manifests (reading the manifest from the +// (possibly remote) destination). Returning true and the destination's manifest, type and digest if they compare equal. +func compareImageDestinationManifestEqual(ctx context.Context, options *Options, src types.Image, targetInstance *digest.Digest, dest types.ImageDestination) (bool, []byte, string, digest.Digest, error) { + srcManifest, _, err := src.Manifest(ctx) + if err != nil { + return false, nil, "", "", errors.Wrapf(err, "Error reading manifest from image") + } + + srcManifestDigest, err := manifest.Digest(srcManifest) + if err != nil { + return false, nil, "", "", errors.Wrapf(err, "Error calculating manifest digest") + } + + destImageSource, err := dest.Reference().NewImageSource(ctx, options.DestinationCtx) + if err != nil { + logrus.Debugf("Unable to create destination image %s source: %v", dest.Reference(), err) + return false, nil, "", "", nil + } + + destManifest, destManifestType, err := destImageSource.GetManifest(ctx, targetInstance) + if err != nil { + logrus.Debugf("Unable to get destination image %s/%s manifest: %v", destImageSource, targetInstance, err) + return false, nil, "", "", nil + } + + destManifestDigest, err := manifest.Digest(destManifest) + if err != nil { + return false, nil, "", "", errors.Wrapf(err, "Error calculating manifest digest") + } + + logrus.Debugf("Comparing source and destination manifest digests: %v vs. %v", srcManifestDigest, destManifestDigest) + if srcManifestDigest != destManifestDigest { + return false, nil, "", "", nil + } + + // Destination and source manifests, types and digests should all be equivalent + return true, destManifest, destManifestType, destManifestDigest, nil +} + // copyMultipleImages copies some or all of an image list's instances, using // policyContext to validate source image admissibility. func (c *copier) copyMultipleImages(ctx context.Context, policyContext *signature.PolicyContext, options *Options, unparsedToplevel *image.UnparsedImage) (copiedManifest []byte, copiedManifestType string, retErr error) { @@ -646,6 +695,26 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli // If encrypted and decryption keys provided, we should try to decrypt ic.diffIDsAreNeeded = ic.diffIDsAreNeeded || (isEncrypted(src) && ic.c.ociDecryptConfig != nil) || ic.c.ociEncryptConfig != nil + // If enabled, fetch and compare the destination's manifest. And as an optimization skip updating the destination iff equal + if options.OptimizeDestinationImageAlreadyExists { + shouldUpdateSigs := len(sigs) > 0 || options.SignBy != "" // TODO: Consider allowing signatures updates only and skipping the image's layers/manifest copy if possible + noPendingManifestUpdates := ic.noPendingManifestUpdates() + + logrus.Debugf("Checking if we can skip copying: has signatures=%t, OCI encryption=%t, no manifest updates=%t", shouldUpdateSigs, destRequiresOciEncryption, noPendingManifestUpdates) + if !shouldUpdateSigs && !destRequiresOciEncryption && noPendingManifestUpdates { + isSrcDestManifestEqual, retManifest, retManifestType, retManifestDigest, err := compareImageDestinationManifestEqual(ctx, options, src, targetInstance, c.dest) + if err != nil { + logrus.Warnf("Failed to compare destination image manifest: %v", err) + return nil, "", "", err + } + + if isSrcDestManifestEqual { + c.Printf("Skipping: image already present at destination\n") + return retManifest, retManifestType, retManifestDigest, nil + } + } + } + if err := ic.copyLayers(ctx); err != nil { return nil, "", "", err } @@ -702,6 +771,9 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli return nil, "", "", fmt.Errorf("Uploading manifest failed, attempted the following formats: %s", strings.Join(errs, ", ")) } } + if targetInstance != nil { + targetInstance = &retManifestDigest + } if options.SignBy != "" { newSig, err := c.createSignature(manifestBytes, options.SignBy) @@ -781,6 +853,10 @@ func (ic *imageCopier) updateEmbeddedDockerReference() error { return nil } +func (ic *imageCopier) noPendingManifestUpdates() bool { + return reflect.DeepEqual(*ic.manifestUpdates, types.ManifestUpdateOptions{InformationOnly: ic.manifestUpdates.InformationOnly}) +} + // isTTY returns true if the io.Writer is a file and a tty. func isTTY(w io.Writer) bool { if f, ok := w.(*os.File); ok { @@ -834,7 +910,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { defer copySemaphore.Release(1) defer copyGroup.Done() cld := copyLayerData{} - if ic.c.dest.AcceptsForeignLayerURLs() && len(srcLayer.URLs) != 0 { + if !ic.c.downloadForeignLayers && ic.c.dest.AcceptsForeignLayerURLs() && len(srcLayer.URLs) != 0 { // DiffIDs are, currently, needed only when converting from schema1. // In which case src.LayerInfos will not have URLs because schema1 // does not support them. @@ -845,7 +921,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { logrus.Debugf("Skipping foreign layer %q copy to %s", cld.destInfo.Digest, ic.c.dest.Reference().Transport().Name()) } } else { - cld.destInfo, cld.diffID, cld.err = ic.copyLayer(ctx, srcLayer, toEncrypt, pool) + cld.destInfo, cld.diffID, cld.err = ic.copyLayer(ctx, srcLayer, toEncrypt, pool, index) } data[index] = cld } @@ -901,6 +977,8 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { diffIDs[i] = cld.diffID } + // WARNING: If you are adding new reasons to change ic.manifestUpdates, also update the + // OptimizeDestinationImageAlreadyExists short-circuit conditions ic.manifestUpdates.InformationOnly.LayerInfos = destInfos if ic.diffIDsAreNeeded { ic.manifestUpdates.InformationOnly.LayerDiffIDs = diffIDs @@ -929,7 +1007,7 @@ func layerDigestsDiffer(a, b []types.BlobInfo) bool { // and its digest. func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, digest.Digest, error) { pendingImage := ic.src - if !reflect.DeepEqual(*ic.manifestUpdates, types.ManifestUpdateOptions{InformationOnly: ic.manifestUpdates.InformationOnly}) { + if !ic.noPendingManifestUpdates() { if !ic.canModifyManifest { return nil, "", errors.Errorf("Internal error: copy needs an updated manifest but that was known to be forbidden") } @@ -1012,10 +1090,9 @@ func (c *copier) createProgressBar(pool *mpb.Progress, info types.BlobInfo, kind ), ) } else { - bar = pool.AddSpinner(info.Size, - mpb.SpinnerOnLeft, + bar = pool.Add(0, + mpb.NewSpinnerFiller([]string{".", "..", "...", "....", ""}, mpb.SpinnerOnLeft), mpb.BarFillerClearOnComplete(), - mpb.SpinnerStyle([]string{".", "..", "...", "....", ""}), mpb.PrependDecorators( decor.OnComplete(decor.Name(prefix), onComplete), ), @@ -1040,7 +1117,7 @@ func (c *copier) copyConfig(ctx context.Context, src types.Image) error { progressPool, progressCleanup := c.newProgressPool(ctx) defer progressCleanup() bar := c.createProgressBar(progressPool, srcInfo, "config", "done") - destInfo, err := c.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, false, true, false, bar) + destInfo, err := c.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, false, true, false, bar, -1) if err != nil { return types.BlobInfo{}, err } @@ -1066,7 +1143,7 @@ type diffIDResult struct { // copyLayer copies a layer with srcInfo (with known Digest and Annotations and possibly known Size) in src to dest, perhaps (de/re/)compressing it, // and returns a complete blobInfo of the copied layer, and a value for LayerDiffIDs if diffIDIsNeeded -func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, toEncrypt bool, pool *mpb.Progress) (types.BlobInfo, digest.Digest, error) { +func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, toEncrypt bool, pool *mpb.Progress, layerIndex int) (types.BlobInfo, digest.Digest, error) { // If the srcInfo doesn't contain compression information, try to compute it from the // MediaType, which was either read from a manifest by way of LayerInfos() or constructed // by LayerInfosForCopy(), if it was supplied at all. If we succeed in copying the blob, @@ -1099,7 +1176,26 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to // a failure when we eventually try to update the manifest with the digest and MIME type of the reused blob. // Fixing that will probably require passing more information to TryReusingBlob() than the current version of // the ImageDestination interface lets us pass in. - reused, blobInfo, err := ic.c.dest.TryReusingBlob(ctx, srcInfo, ic.c.blobInfoCache, ic.canSubstituteBlobs) + var ( + blobInfo types.BlobInfo + reused bool + err error + ) + // Note: the storage destination optimizes the committing of + // layers which requires passing the index of the layer. + // Hence, we need to special case and cast. + dest, ok := ic.c.dest.(internalTypes.ImageDestinationWithOptions) + if ok { + options := internalTypes.TryReusingBlobOptions{ + Cache: ic.c.blobInfoCache, + CanSubstitute: ic.canSubstituteBlobs, + LayerIndex: &layerIndex, + } + reused, blobInfo, err = dest.TryReusingBlobWithOptions(ctx, srcInfo, options) + } else { + reused, blobInfo, err = ic.c.dest.TryReusingBlob(ctx, srcInfo, ic.c.blobInfoCache, ic.canSubstituteBlobs) + } + if err != nil { return types.BlobInfo{}, "", errors.Wrapf(err, "Error trying to reuse blob %s at destination", srcInfo.Digest) } @@ -1141,7 +1237,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to bar := ic.c.createProgressBar(pool, srcInfo, "blob", "done") - blobInfo, diffIDChan, err := ic.copyLayerFromStream(ctx, srcStream, types.BlobInfo{Digest: srcInfo.Digest, Size: srcBlobSize, MediaType: srcInfo.MediaType, Annotations: srcInfo.Annotations}, diffIDIsNeeded, toEncrypt, bar) + blobInfo, diffIDChan, err := ic.copyLayerFromStream(ctx, srcStream, types.BlobInfo{Digest: srcInfo.Digest, Size: srcBlobSize, MediaType: srcInfo.MediaType, Annotations: srcInfo.Annotations}, diffIDIsNeeded, toEncrypt, bar, layerIndex) if err != nil { return types.BlobInfo{}, "", err } @@ -1172,7 +1268,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to // perhaps (de/re/)compressing the stream, // and returns a complete blobInfo of the copied blob and perhaps a <-chan diffIDResult if diffIDIsNeeded, to be read by the caller. func (ic *imageCopier) copyLayerFromStream(ctx context.Context, srcStream io.Reader, srcInfo types.BlobInfo, - diffIDIsNeeded bool, toEncrypt bool, bar *mpb.Bar) (types.BlobInfo, <-chan diffIDResult, error) { + diffIDIsNeeded bool, toEncrypt bool, bar *mpb.Bar, layerIndex int) (types.BlobInfo, <-chan diffIDResult, error) { var getDiffIDRecorder func(compression.DecompressorFunc) io.Writer // = nil var diffIDChan chan diffIDResult @@ -1197,7 +1293,7 @@ func (ic *imageCopier) copyLayerFromStream(ctx context.Context, srcStream io.Rea } } - blobInfo, err := ic.c.copyBlobFromStream(ctx, srcStream, srcInfo, getDiffIDRecorder, ic.canModifyManifest, false, toEncrypt, bar) // Sets err to nil on success + blobInfo, err := ic.c.copyBlobFromStream(ctx, srcStream, srcInfo, getDiffIDRecorder, ic.canModifyManifest, false, toEncrypt, bar, layerIndex) // Sets err to nil on success return blobInfo, diffIDChan, err // We need the defer … pipeWriter.CloseWithError() to happen HERE so that the caller can block on reading from diffIDChan } @@ -1249,7 +1345,7 @@ func (r errorAnnotationReader) Read(b []byte) (n int, err error) { // and returns a complete blobInfo of the copied blob. func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, srcInfo types.BlobInfo, getOriginalLayerCopyWriter func(decompressor compression.DecompressorFunc) io.Writer, - canModifyBlob bool, isConfig bool, toEncrypt bool, bar *mpb.Bar) (types.BlobInfo, error) { + canModifyBlob bool, isConfig bool, toEncrypt bool, bar *mpb.Bar, layerIndex int) (types.BlobInfo, error) { if isConfig { // This is guaranteed by the caller, but set it here to be explicit. canModifyBlob = false } @@ -1267,8 +1363,9 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr if err != nil { return types.BlobInfo{}, errors.Wrapf(err, "Error preparing to verify blob %s", srcInfo.Digest) } - var destStream io.Reader = digestingReader + + // === Decrypt the stream, if required. var decrypted bool if isOciEncrypted(srcInfo.MediaType) && c.ociDecryptConfig != nil { newDesc := imgspecv1.Descriptor{ @@ -1298,12 +1395,13 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr return types.BlobInfo{}, errors.Wrapf(err, "Error reading blob %s", srcInfo.Digest) } isCompressed := decompressor != nil - destStream = bar.ProxyReader(destStream) - if expectedCompressionFormat, known := expectedCompressionFormats[srcInfo.MediaType]; known && isCompressed && compressionFormat.Name() != expectedCompressionFormat.Name() { logrus.Debugf("blob %s with type %s should be compressed with %s, but compressor appears to be %s", srcInfo.Digest.String(), srcInfo.MediaType, expectedCompressionFormat.Name(), compressionFormat.Name()) } + // === Update progress bars + destStream = bar.ProxyReader(destStream) + // === Send a copy of the original, uncompressed, stream, to a separate path if necessary. var originalLayerReader io.Reader // DO NOT USE this other than to drain the input if no other consumer in the pipeline has done so. if getOriginalLayerCopyWriter != nil { @@ -1312,6 +1410,8 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr } // === Deal with layer compression/decompression if necessary + // WARNING: If you are adding new reasons to change the blob, update also the OptimizeDestinationImageAlreadyExists + // short-circuit conditions var inputInfo types.BlobInfo var compressionOperation types.LayerCompression uploadCompressionFormat := &c.compressionFormat @@ -1393,7 +1493,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr } } - // Perform image encryption for valid mediatypes if ociEncryptConfig provided + // === Encrypt the stream for valid mediatypes if ociEncryptConfig provided var ( encrypted bool finalizer ocicrypt.EncryptLayerFinalizer @@ -1441,7 +1541,23 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr } // === Finally, send the layer stream to dest. - uploadedInfo, err := c.dest.PutBlob(ctx, &errorAnnotationReader{destStream}, inputInfo, c.blobInfoCache, isConfig) + var uploadedInfo types.BlobInfo + // Note: the storage destination optimizes the committing of layers + // which requires passing the index of the layer. Hence, we need to + // special case and cast. + dest, ok := c.dest.(internalTypes.ImageDestinationWithOptions) + if ok { + options := internalTypes.PutBlobOptions{ + Cache: c.blobInfoCache, + IsConfig: isConfig, + } + if !isConfig { + options.LayerIndex = &layerIndex + } + uploadedInfo, err = dest.PutBlobWithOptions(ctx, &errorAnnotationReader{destStream}, inputInfo, options) + } else { + uploadedInfo, err = c.dest.PutBlob(ctx, &errorAnnotationReader{destStream}, inputInfo, c.blobInfoCache, isConfig) + } if err != nil { return types.BlobInfo{}, errors.Wrap(err, "Error writing blob") } diff --git a/vendor/github.com/containers/image/v5/docker/docker_client.go b/vendor/github.com/containers/image/v5/docker/docker_client.go index be46508de..a9533ea39 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_client.go +++ b/vendor/github.com/containers/image/v5/docker/docker_client.go @@ -502,6 +502,8 @@ func (c *dockerClient) makeRequestToResolvedURL(ctx context.Context, method, url attempts == backoffNumIterations { return res, err } + // close response body before retry or context done + res.Body.Close() delay = parseRetryAfter(res, delay) if delay > backoffMaxDelay { diff --git a/vendor/github.com/containers/image/v5/docker/docker_image.go b/vendor/github.com/containers/image/v5/docker/docker_image.go index 0c1cee0d3..f9fe4e8a3 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_image.go +++ b/vendor/github.com/containers/image/v5/docker/docker_image.go @@ -139,6 +139,7 @@ func GetDigest(ctx context.Context, sys *types.SystemContext, ref types.ImageRef return "", err } + defer res.Body.Close() if res.StatusCode != http.StatusOK { return "", errors.Wrapf(registryHTTPResponseToError(res), "Error reading digest %s in %s", tagOrDigest, dr.ref.Name()) } diff --git a/vendor/github.com/containers/image/v5/docker/docker_image_dest.go b/vendor/github.com/containers/image/v5/docker/docker_image_dest.go index 842dcfba6..e11084dc8 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_image_dest.go +++ b/vendor/github.com/containers/image/v5/docker/docker_image_dest.go @@ -445,7 +445,7 @@ func successStatus(status int) bool { return status >= 200 && status <= 399 } -// isManifestInvalidError returns true iff err from client.HandleErrorReponse is a “manifest invalid” error. +// isManifestInvalidError returns true iff err from client.HandleErrorResponse is a “manifest invalid” error. func isManifestInvalidError(err error) bool { errors, ok := err.(errcode.Errors) if !ok || len(errors) == 0 { diff --git a/vendor/github.com/containers/image/v5/docker/docker_image_src.go b/vendor/github.com/containers/image/v5/docker/docker_image_src.go index bff950bb0..6916b7dad 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/v5/docker/docker_image_src.go @@ -251,6 +251,7 @@ func (s *dockerImageSource) getExternalBlob(ctx context.Context, urls []string) if resp.StatusCode != http.StatusOK { err = errors.Errorf("error fetching external blob from %q: %d (%s)", url, resp.StatusCode, http.StatusText(resp.StatusCode)) logrus.Debug(err) + resp.Body.Close() continue } break @@ -290,6 +291,7 @@ func (s *dockerImageSource) GetBlob(ctx context.Context, info types.BlobInfo, ca return nil, 0, err } if err := httpResponseToError(res, "Error fetching blob"); err != nil { + res.Body.Close() return nil, 0, err } cache.RecordKnownLocation(s.physicalRef.Transport(), bicTransportScope(s.physicalRef), info.Digest, newBICLocationReference(s.physicalRef)) diff --git a/vendor/github.com/containers/image/v5/internal/pkg/platform/platform_matcher.go b/vendor/github.com/containers/image/v5/internal/pkg/platform/platform_matcher.go index 5ea542bcf..3e81e06c0 100644 --- a/vendor/github.com/containers/image/v5/internal/pkg/platform/platform_matcher.go +++ b/vendor/github.com/containers/image/v5/internal/pkg/platform/platform_matcher.go @@ -123,14 +123,6 @@ var compatibility = map[string][]string{ "arm64": {"v8"}, } -// baseVariants contains, for a specified architecture, a variant that is known to be -// supported by _all_ machines using that architecture. -// Architectures that don’t have variants, or where there are possible versions without -// an established variant name, should not have an entry here. -var baseVariants = map[string]string{ - "arm64": "v8", -} - // WantedPlatforms returns all compatible platforms with the platform specifics possibly overridden by user, // the most compatible platform is first. // If some option (arch, os, variant) is not present, a value from current platform is detected. @@ -158,6 +150,8 @@ func WantedPlatforms(ctx *types.SystemContext) ([]imgspecv1.Platform, error) { var variants []string = nil if wantedVariant != "" { + // If the user requested a specific variant, we'll walk down + // the list from most to least compatible. if compatibility[wantedArch] != nil { variantOrder := compatibility[wantedArch] for i, v := range variantOrder { @@ -171,12 +165,14 @@ func WantedPlatforms(ctx *types.SystemContext) ([]imgspecv1.Platform, error) { // user wants a variant which we know nothing about - not even compatibility variants = []string{wantedVariant} } + // Make sure to have a candidate with an empty variant as well. variants = append(variants, "") } else { - variants = append(variants, "") // No variant specified, use a “no variant specified” image if present - if baseVariant, ok := baseVariants[wantedArch]; ok { - // But also accept an image with the “base” variant for the architecture, if it exists. - variants = append(variants, baseVariant) + // Make sure to have a candidate with an empty variant as well. + variants = append(variants, "") + // If available add the entire compatibility matrix for the specific architecture. + if possibleVariants, ok := compatibility[wantedArch]; ok { + variants = append(variants, possibleVariants...) } } diff --git a/vendor/github.com/containers/image/v5/internal/types/types.go b/vendor/github.com/containers/image/v5/internal/types/types.go new file mode 100644 index 000000000..9adf0d536 --- /dev/null +++ b/vendor/github.com/containers/image/v5/internal/types/types.go @@ -0,0 +1,53 @@ +package types + +import ( + "context" + "io" + + publicTypes "github.com/containers/image/v5/types" +) + +// ImageDestinationWithOptions is an internal extension to the ImageDestination +// interface. +type ImageDestinationWithOptions interface { + publicTypes.ImageDestination + + // PutBlobWithOptions is a wrapper around PutBlob. If + // options.LayerIndex is set, the blob will be committed directly. + // Either by the calling goroutine or by another goroutine already + // committing layers. + // + // Please note that TryReusingBlobWithOptions and PutBlobWithOptions + // *must* be used the together. Mixing the two with non "WithOptions" + // functions is not supported. + PutBlobWithOptions(ctx context.Context, stream io.Reader, blobinfo publicTypes.BlobInfo, options PutBlobOptions) (publicTypes.BlobInfo, error) + + // TryReusingBlobWithOptions is a wrapper around TryReusingBlob. If + // options.LayerIndex is set, the reused blob will be recoreded as + // already pulled. + // + // Please note that TryReusingBlobWithOptions and PutBlobWithOptions + // *must* be used the together. Mixing the two with non "WithOptions" + // functions is not supported. + TryReusingBlobWithOptions(ctx context.Context, blobinfo publicTypes.BlobInfo, options TryReusingBlobOptions) (bool, publicTypes.BlobInfo, error) +} + +// PutBlobOptions are used in PutBlobWithOptions. +type PutBlobOptions struct { + // Cache to look up blob infos. + Cache publicTypes.BlobInfoCache + // Denotes whether the blob is a config or not. + IsConfig bool + // The corresponding index in the layer slice. + LayerIndex *int +} + +// TryReusingBlobOptions are used in TryReusingBlobWithOptions. +type TryReusingBlobOptions struct { + // Cache to look up blob infos. + Cache publicTypes.BlobInfoCache + // Use an equivalent of the desired blob. + CanSubstitute bool + // The corresponding index in the layer slice. + LayerIndex *int +} diff --git a/vendor/github.com/containers/image/v5/manifest/manifest.go b/vendor/github.com/containers/image/v5/manifest/manifest.go index 7b0758873..32680e09d 100644 --- a/vendor/github.com/containers/image/v5/manifest/manifest.go +++ b/vendor/github.com/containers/image/v5/manifest/manifest.go @@ -30,7 +30,7 @@ const ( DockerV2ListMediaType = "application/vnd.docker.distribution.manifest.list.v2+json" // DockerV2Schema2ForeignLayerMediaType is the MIME type used for schema 2 foreign layers. DockerV2Schema2ForeignLayerMediaType = "application/vnd.docker.image.rootfs.foreign.diff.tar" - // DockerV2Schema2ForeignLayerMediaType is the MIME type used for gzippped schema 2 foreign layers. + // DockerV2Schema2ForeignLayerMediaType is the MIME type used for gzipped schema 2 foreign layers. DockerV2Schema2ForeignLayerMediaTypeGzip = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" ) @@ -132,9 +132,16 @@ func GuessMIMEType(manifest []byte) string { if err := json.Unmarshal(manifest, &ociMan); err != nil { return "" } - if ociMan.Config.MediaType == imgspecv1.MediaTypeImageConfig { + switch ociMan.Config.MediaType { + case imgspecv1.MediaTypeImageConfig: return imgspecv1.MediaTypeImageManifest + case DockerV2Schema2ConfigMediaType: + // This case should not happen since a Docker image + // must declare a top-level media type and + // `meta.MediaType` has already been checked. + return DockerV2Schema2MediaType } + // Maybe an image index or an OCI artifact. ociIndex := struct { Manifests []imgspecv1.Descriptor `json:"manifests"` }{} @@ -145,9 +152,13 @@ func GuessMIMEType(manifest []byte) string { if ociMan.Config.MediaType == "" { return imgspecv1.MediaTypeImageIndex } + // FIXME: this is mixing media types of manifests and configs. return ociMan.Config.MediaType } - return DockerV2Schema2MediaType + // It's most likely an OCI artifact with a custom config media + // type which is not (and cannot) be covered by the media-type + // checks cabove. + return imgspecv1.MediaTypeImageManifest } return "" } diff --git a/vendor/github.com/containers/image/v5/oci/layout/oci_src.go b/vendor/github.com/containers/image/v5/oci/layout/oci_src.go index 9925aeda7..6801c8432 100644 --- a/vendor/github.com/containers/image/v5/oci/layout/oci_src.go +++ b/vendor/github.com/containers/image/v5/oci/layout/oci_src.go @@ -15,6 +15,7 @@ import ( "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) type ociImageSource struct { @@ -94,6 +95,7 @@ func (s *ociImageSource) GetManifest(ctx context.Context, instanceDigest *digest m, err := ioutil.ReadFile(manifestPath) if err != nil { + logrus.Errorf("Error HERE") return nil, "", err } if mimeType == "" { diff --git a/vendor/github.com/containers/image/v5/openshift/openshift-copies.go b/vendor/github.com/containers/image/v5/openshift/openshift-copies.go index d87f748e6..ec88b4ebf 100644 --- a/vendor/github.com/containers/image/v5/openshift/openshift-copies.go +++ b/vendor/github.com/containers/image/v5/openshift/openshift-copies.go @@ -24,7 +24,7 @@ import ( "golang.org/x/net/http2" ) -// restTLSClientConfig is a modified copy of k8s.io/kubernets/pkg/client/restclient.TLSClientConfig. +// restTLSClientConfig is a modified copy of k8s.io/kubernetes/pkg/client/restclient.TLSClientConfig. // restTLSClientConfig contains settings to enable transport layer security type restTLSClientConfig struct { // Server requires TLS client certificate authentication @@ -45,7 +45,7 @@ type restTLSClientConfig struct { CAData []byte } -// restConfig is a modified copy of k8s.io/kubernets/pkg/client/restclient.Config. +// restConfig is a modified copy of k8s.io/kubernetes/pkg/client/restclient.Config. // Config holds the common attributes that can be passed to a Kubernetes client on // initialization. type restConfig struct { @@ -254,7 +254,7 @@ func getServerIdentificationPartialConfig(configAuthInfo clientcmdAuthInfo, conf // we want this order of precedence for user identification // 1. configAuthInfo minus auth-path (the final result of command line flags and merged .kubeconfig files) // 2. configAuthInfo.auth-path (this file can contain information that conflicts with #1, and we want #1 to win the priority) -// 3. if there is not enough information to idenfity the user, load try the ~/.kubernetes_auth file +// 3. if there is not enough information to identify the user, load try the ~/.kubernetes_auth file // 4. if there is not enough information to identify the user, prompt if possible func getUserIdentificationPartialConfig(configAuthInfo clientcmdAuthInfo) (*restConfig, error) { mergedConfig := &restConfig{} @@ -538,7 +538,7 @@ func (e errConfigurationInvalid) Error() string { // ClientConfigLoadingRules is an ExplicitPath and string slice of specific locations that are used for merging together a Config // Callers can put the chain together however they want, but we'd recommend: // EnvVarPathFiles if set (a list of files if set) OR the HomeDirectoryPath -// ExplicitPath is special, because if a user specifically requests a certain file be used and error is reported if thie file is not present +// ExplicitPath is special, because if a user specifically requests a certain file be used and error is reported if this file is not present type clientConfigLoadingRules struct { Precedence []string } @@ -741,7 +741,7 @@ func resolvePaths(refs []*string, base string) error { return nil } -// restClientFor is a modified copy of k8s.io/kubernets/pkg/client/restclient.RESTClientFor. +// restClientFor is a modified copy of k8s.io/kubernetes/pkg/client/restclient.RESTClientFor. // RESTClientFor returns a RESTClient that satisfies the requested attributes on a client Config // object. Note that a RESTClient may require fields that are optional when initializing a Client. // A RESTClient created by this method is generic - it expects to operate on an API that follows @@ -769,7 +769,7 @@ func restClientFor(config *restConfig) (*url.URL, *http.Client, error) { return baseURL, httpClient, nil } -// defaultServerURL is a modified copy of k8s.io/kubernets/pkg/client/restclient.DefaultServerURL. +// defaultServerURL is a modified copy of k8s.io/kubernetes/pkg/client/restclient.DefaultServerURL. // DefaultServerURL converts a host, host:port, or URL string to the default base server API path // to use with a Client at a given API version following the standard conventions for a // Kubernetes API. @@ -800,7 +800,7 @@ func defaultServerURL(host string, defaultTLS bool) (*url.URL, error) { return hostURL, nil } -// defaultServerURLFor is a modified copy of k8s.io/kubernets/pkg/client/restclient.defaultServerURLFor. +// defaultServerURLFor is a modified copy of k8s.io/kubernetes/pkg/client/restclient.defaultServerURLFor. // defaultServerUrlFor is shared between IsConfigTransportTLS and RESTClientFor. It // requires Host and Version to be set prior to being called. func defaultServerURLFor(config *restConfig) (*url.URL, error) { @@ -818,7 +818,7 @@ func defaultServerURLFor(config *restConfig) (*url.URL, error) { return defaultServerURL(host, defaultTLS) } -// transportFor is a modified copy of k8s.io/kubernets/pkg/client/restclient.transportFor. +// transportFor is a modified copy of k8s.io/kubernetes/pkg/client/restclient.transportFor. // TransportFor returns an http.RoundTripper that will provide the authentication // or transport level security defined by the provided Config. Will return the // default http.DefaultTransport if no special case behavior is needed. @@ -827,7 +827,7 @@ func transportFor(config *restConfig) (http.RoundTripper, error) { return transportNew(config) } -// isConfigTransportTLS is a modified copy of k8s.io/kubernets/pkg/client/restclient.IsConfigTransportTLS. +// isConfigTransportTLS is a modified copy of k8s.io/kubernetes/pkg/client/restclient.IsConfigTransportTLS. // IsConfigTransportTLS returns true if and only if the provided // config will result in a protected connection to the server when it // is passed to restclient.RESTClientFor(). Use to determine when to @@ -1055,11 +1055,11 @@ func (c *restConfig) HasCertAuth() bool { // Config holds the information needed to build connect to remote kubernetes clusters as a given user // IMPORTANT if you add fields to this struct, please update IsConfigEmpty() type clientcmdConfig struct { - // Clusters is a map of referencable names to cluster configs + // Clusters is a map of referenceable names to cluster configs Clusters clustersMap `json:"clusters"` - // AuthInfos is a map of referencable names to user configs + // AuthInfos is a map of referenceable names to user configs AuthInfos authInfosMap `json:"users"` - // Contexts is a map of referencable names to context configs + // Contexts is a map of referenceable names to context configs Contexts contextsMap `json:"contexts"` // CurrentContext is the name of the context that you would like to use by default CurrentContext string `json:"current-context"` diff --git a/vendor/github.com/containers/image/v5/pkg/blobinfocache/boltdb/boltdb.go b/vendor/github.com/containers/image/v5/pkg/blobinfocache/boltdb/boltdb.go index 2c211b8b8..a472efd95 100644 --- a/vendor/github.com/containers/image/v5/pkg/blobinfocache/boltdb/boltdb.go +++ b/vendor/github.com/containers/image/v5/pkg/blobinfocache/boltdb/boltdb.go @@ -282,7 +282,7 @@ func (bdc *cache) RecordKnownLocation(transport types.ImageTransport, scope type }) // FIXME? Log error (but throttle the log volume on repeated accesses)? } -// appendReplacementCandiates creates prioritize.CandidateWithTime values for digest in scopeBucket with corresponding compression info from compressionBucket (if compressionBucket is not nil), and returns the result of appending them to candidates. +// appendReplacementCandidates creates prioritize.CandidateWithTime values for digest in scopeBucket with corresponding compression info from compressionBucket (if compressionBucket is not nil), and returns the result of appending them to candidates. func (bdc *cache) appendReplacementCandidates(candidates []prioritize.CandidateWithTime, scopeBucket, compressionBucket *bolt.Bucket, digest digest.Digest, requireCompressionInfo bool) []prioritize.CandidateWithTime { digestKey := []byte(digest.String()) b := scopeBucket.Bucket(digestKey) @@ -321,7 +321,7 @@ func (bdc *cache) appendReplacementCandidates(candidates []prioritize.CandidateW // CandidateLocations2 returns a prioritized, limited, number of blobs and their locations that could possibly be reused // within the specified (transport scope) (if they still exist, which is not guaranteed). // -// If !canSubstitute, the returned cadidates will match the submitted digest exactly; if canSubstitute, +// If !canSubstitute, the returned candidates will match the submitted digest exactly; if canSubstitute, // data from previous RecordDigestUncompressedPair calls is used to also look up variants of the blob which have the same // uncompressed digest. func (bdc *cache) CandidateLocations2(transport types.ImageTransport, scope types.BICTransportScope, primaryDigest digest.Digest, canSubstitute bool) []blobinfocache.BICReplacementCandidate2 { diff --git a/vendor/github.com/containers/image/v5/pkg/blobinfocache/default.go b/vendor/github.com/containers/image/v5/pkg/blobinfocache/default.go index 01abb8d1e..83034b618 100644 --- a/vendor/github.com/containers/image/v5/pkg/blobinfocache/default.go +++ b/vendor/github.com/containers/image/v5/pkg/blobinfocache/default.go @@ -20,7 +20,7 @@ const ( systemBlobInfoCacheDir = "/var/lib/containers/cache" ) -// blobInfoCacheDir returns a path to a blob info cache appropripate for sys and euid. +// blobInfoCacheDir returns a path to a blob info cache appropriate for sys and euid. // euid is used so that (sudo …) does not write root-owned files into the unprivileged users’ home directory. func blobInfoCacheDir(sys *types.SystemContext, euid int) (string, error) { if sys != nil && sys.BlobInfoCacheDir != "" { diff --git a/vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go b/vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go index 3d598057e..426640366 100644 --- a/vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go +++ b/vendor/github.com/containers/image/v5/pkg/blobinfocache/memory/memory.go @@ -120,7 +120,7 @@ func (mem *cache) RecordDigestCompressorName(blobDigest digest.Digest, compresso mem.compressors[blobDigest] = compressorName } -// appendReplacementCandiates creates prioritize.CandidateWithTime values for (transport, scope, digest), and returns the result of appending them to candidates. +// appendReplacementCandidates creates prioritize.CandidateWithTime values for (transport, scope, digest), and returns the result of appending them to candidates. func (mem *cache) appendReplacementCandidates(candidates []prioritize.CandidateWithTime, transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, requireCompressionInfo bool) []prioritize.CandidateWithTime { locations := mem.knownLocations[locationKey{transport: transport.Name(), scope: scope, blobDigest: digest}] // nil if not present for l, t := range locations { @@ -146,7 +146,7 @@ func (mem *cache) appendReplacementCandidates(candidates []prioritize.CandidateW // CandidateLocations returns a prioritized, limited, number of blobs and their locations that could possibly be reused // within the specified (transport scope) (if they still exist, which is not guaranteed). // -// If !canSubstitute, the returned cadidates will match the submitted digest exactly; if canSubstitute, +// If !canSubstitute, the returned candidates will match the submitted digest exactly; if canSubstitute, // data from previous RecordDigestUncompressedPair calls is used to also look up variants of the blob which have the same // uncompressed digest. func (mem *cache) CandidateLocations(transport types.ImageTransport, scope types.BICTransportScope, primaryDigest digest.Digest, canSubstitute bool) []types.BICReplacementCandidate { diff --git a/vendor/github.com/containers/image/v5/pkg/blobinfocache/none/none.go b/vendor/github.com/containers/image/v5/pkg/blobinfocache/none/none.go index 2a54ff312..4b7122f92 100644 --- a/vendor/github.com/containers/image/v5/pkg/blobinfocache/none/none.go +++ b/vendor/github.com/containers/image/v5/pkg/blobinfocache/none/none.go @@ -42,7 +42,7 @@ func (noCache) RecordKnownLocation(transport types.ImageTransport, scope types.B // CandidateLocations returns a prioritized, limited, number of blobs and their locations that could possibly be reused // within the specified (transport scope) (if they still exist, which is not guaranteed). // -// If !canSubstitute, the returned cadidates will match the submitted digest exactly; if canSubstitute, +// If !canSubstitute, the returned candidates will match the submitted digest exactly; if canSubstitute, // data from previous RecordDigestUncompressedPair calls is used to also look up variants of the blob which have the same // uncompressed digest. func (noCache) CandidateLocations(transport types.ImageTransport, scope types.BICTransportScope, digest digest.Digest, canSubstitute bool) []types.BICReplacementCandidate { diff --git a/vendor/github.com/containers/image/v5/pkg/docker/config/config.go b/vendor/github.com/containers/image/v5/pkg/docker/config/config.go index 983df41d8..b84aac6e4 100644 --- a/vendor/github.com/containers/image/v5/pkg/docker/config/config.go +++ b/vendor/github.com/containers/image/v5/pkg/docker/config/config.go @@ -6,14 +6,17 @@ import ( "fmt" "io/ioutil" "os" + "os/exec" "path/filepath" "runtime" "strings" + "github.com/containers/image/v5/pkg/sysregistriesv2" "github.com/containers/image/v5/types" "github.com/containers/storage/pkg/homedir" helperclient "github.com/docker/docker-credential-helpers/client" "github.com/docker/docker-credential-helpers/credentials" + "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -41,12 +44,6 @@ var ( dockerLegacyHomePath = ".dockercfg" nonLinuxAuthFilePath = filepath.FromSlash(".config/containers/auth.json") - // Note that the keyring support has been disabled as it was causing - // regressions. Before enabling, please revisit TODO(keyring) comments - // which need to be addressed if the need remerged to support the - // kernel keyring. - enableKeyring = false - // ErrNotLoggedIn is returned for users not logged into a registry // that they are trying to logout of ErrNotLoggedIn = errors.New("not logged in") @@ -54,72 +51,114 @@ var ( ErrNotSupported = errors.New("not supported") ) -// SetAuthentication stores the username and password in the auth.json file +// SetAuthentication stores the username and password in the credential helper or file func SetAuthentication(sys *types.SystemContext, registry, username, password string) error { - return modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) { - if ch, exists := auths.CredHelpers[registry]; exists { - return false, setAuthToCredHelper(ch, registry, username, password) - } + helpers, err := sysregistriesv2.CredentialHelpers(sys) + if err != nil { + return err + } - // Set the credentials to kernel keyring if enableKeyring is true. - // The keyring might not work in all environments (e.g., missing capability) and isn't supported on all platforms. - // Hence, we want to fall-back to using the authfile in case the keyring failed. - // However, if the enableKeyring is false, we want adhere to the user specification and not use the keyring. - if enableKeyring { - err := setAuthToKernelKeyring(registry, username, password) - if err == nil { - logrus.Debugf("credentials for (%s, %s) were stored in the kernel keyring\n", registry, username) - return false, nil - } - logrus.Debugf("failed to authenticate with the kernel keyring, falling back to authfiles. %v", err) + // Make sure to collect all errors. + var multiErr error + for _, helper := range helpers { + var err error + switch helper { + // Special-case the built-in helpers for auth files. + case sysregistriesv2.AuthenticationFileHelper: + err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) { + if ch, exists := auths.CredHelpers[registry]; exists { + return false, setAuthToCredHelper(ch, registry, username, password) + } + creds := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) + newCreds := dockerAuthConfig{Auth: creds} + auths.AuthConfigs[registry] = newCreds + return true, nil + }) + // External helpers. + default: + err = setAuthToCredHelper(helper, registry, username, password) + } + if err != nil { + multiErr = multierror.Append(multiErr, err) + logrus.Debugf("Error storing credentials for %s in credential helper %s: %v", registry, helper, err) + continue } - creds := base64.StdEncoding.EncodeToString([]byte(username + ":" + password)) - newCreds := dockerAuthConfig{Auth: creds} - auths.AuthConfigs[registry] = newCreds - return true, nil - }) + logrus.Debugf("Stored credentials for %s in credential helper %s", registry, helper) + return nil + } + return multiErr } // GetAllCredentials returns the registry credentials for all registries stored -// in either the auth.json file or the docker/config.json. +// in any of the configured credential helpers. func GetAllCredentials(sys *types.SystemContext) (map[string]types.DockerAuthConfig, error) { - // Note: we need to read the auth files in the inverse order to prevent - // a priority inversion when writing to the map. - authConfigs := make(map[string]types.DockerAuthConfig) - paths := getAuthFilePaths(sys, homedir.Get()) - for i := len(paths) - 1; i >= 0; i-- { - path := paths[i] - // readJSONFile returns an empty map in case the path doesn't exist. - auths, err := readJSONFile(path.path, path.legacyFormat) - if err != nil { - return nil, errors.Wrapf(err, "error reading JSON file %q", path.path) - } + // To keep things simple, let's first extract all registries from all + // possible sources, and then call `GetCredentials` on them. That + // prevents us from having to reverse engineer the logic in + // `GetCredentials`. + allRegistries := make(map[string]bool) + addRegistry := func(s string) { + allRegistries[s] = true + } - for registry, data := range auths.AuthConfigs { - conf, err := decodeDockerAuth(data) - if err != nil { - return nil, err + helpers, err := sysregistriesv2.CredentialHelpers(sys) + if err != nil { + return nil, err + } + for _, helper := range helpers { + switch helper { + // Special-case the built-in helper for auth files. + case sysregistriesv2.AuthenticationFileHelper: + for _, path := range getAuthFilePaths(sys, homedir.Get()) { + // readJSONFile returns an empty map in case the path doesn't exist. + auths, err := readJSONFile(path.path, path.legacyFormat) + if err != nil { + return nil, errors.Wrapf(err, "error reading JSON file %q", path.path) + } + // Credential helpers in the auth file have a + // direct mapping to a registry, so we can just + // walk the map. + for registry := range auths.CredHelpers { + addRegistry(registry) + } + for registry := range auths.AuthConfigs { + addRegistry(registry) + } } - authConfigs[normalizeRegistry(registry)] = conf - } - - // Credential helpers may override credentials from the auth file. - for registry, credHelper := range auths.CredHelpers { - username, password, err := getAuthFromCredHelper(credHelper, registry) + // External helpers. + default: + creds, err := listAuthsFromCredHelper(helper) if err != nil { - if credentials.IsErrCredentialsNotFoundMessage(err.Error()) { - continue + logrus.Debugf("Error listing credentials stored in credential helper %s: %v", helper, err) + } + switch errors.Cause(err) { + case nil: + for registry := range creds { + addRegistry(registry) } + case exec.ErrNotFound: + // It's okay if the helper doesn't exist. + default: return nil, err } - - conf := types.DockerAuthConfig{Username: username, Password: password} - authConfigs[normalizeRegistry(registry)] = conf } } - // TODO(keyring): if we ever re-enable the keyring support, we had to - // query all credentials from the keyring here. + // Now use `GetCredentials` to the specific auth configs for each + // previously listed registry. + authConfigs := make(map[string]types.DockerAuthConfig) + for registry := range allRegistries { + authConf, err := GetCredentials(sys, registry) + if err != nil { + if credentials.IsErrCredentialsNotFoundMessage(err.Error()) { + // Ignore if the credentials could not be found (anymore). + continue + } + // Note: we rely on the logging in `GetCredentials`. + return nil, err + } + authConfigs[registry] = authConf + } return authConfigs, nil } @@ -159,7 +198,9 @@ func getAuthFilePaths(sys *types.SystemContext, homeDir string) []authPath { return paths } -// GetCredentials returns the registry credentials stored in either auth.json +// GetCredentials returns the registry credentials stored in the +// registry-specific credential helpers or in the default global credentials +// helpers with falling back to using either auth.json // file or .docker/config.json, including support for OAuth2 and IdentityToken. // If an entry is not found, an empty struct is returned. func GetCredentials(sys *types.SystemContext, registry string) (types.DockerAuthConfig, error) { @@ -170,41 +211,65 @@ func GetCredentials(sys *types.SystemContext, registry string) (types.DockerAuth // it exists only to allow testing it with an artificial home directory. func getCredentialsWithHomeDir(sys *types.SystemContext, registry, homeDir string) (types.DockerAuthConfig, error) { if sys != nil && sys.DockerAuthConfig != nil { - logrus.Debug("Returning credentials from DockerAuthConfig") + logrus.Debugf("Returning credentials for %s from DockerAuthConfig", registry) return *sys.DockerAuthConfig, nil } - if enableKeyring { - username, password, err := getAuthFromKernelKeyring(registry) - if err == nil { - logrus.Debug("returning credentials from kernel keyring") - return types.DockerAuthConfig{ - Username: username, - Password: password, - }, nil + // Anonymous function to query credentials from auth files. + getCredentialsFromAuthFiles := func() (types.DockerAuthConfig, error) { + for _, path := range getAuthFilePaths(sys, homeDir) { + authConfig, err := findAuthentication(registry, path.path, path.legacyFormat) + if err != nil { + return types.DockerAuthConfig{}, err + } + + if (authConfig.Username != "" && authConfig.Password != "") || authConfig.IdentityToken != "" { + return authConfig, nil + } } + return types.DockerAuthConfig{}, nil + } + + helpers, err := sysregistriesv2.CredentialHelpers(sys) + if err != nil { + return types.DockerAuthConfig{}, err } - for _, path := range getAuthFilePaths(sys, homeDir) { - authConfig, err := findAuthentication(registry, path.path, path.legacyFormat) + var multiErr error + for _, helper := range helpers { + var creds types.DockerAuthConfig + var err error + switch helper { + // Special-case the built-in helper for auth files. + case sysregistriesv2.AuthenticationFileHelper: + creds, err = getCredentialsFromAuthFiles() + // External helpers. + default: + creds, err = getAuthFromCredHelper(helper, registry) + } if err != nil { - logrus.Debugf("Credentials not found") - return types.DockerAuthConfig{}, err + logrus.Debugf("Error looking up credentials for %s in credential helper %s: %v", registry, helper, err) + multiErr = multierror.Append(multiErr, err) + continue } - - if (authConfig.Username != "" && authConfig.Password != "") || authConfig.IdentityToken != "" { - logrus.Debugf("Returning credentials from %s", path.path) - return authConfig, nil + if len(creds.Username)+len(creds.Password)+len(creds.IdentityToken) == 0 { + continue } + logrus.Debugf("Found credentials for %s in credential helper %s", registry, helper) + return creds, nil + } + if multiErr != nil { + return types.DockerAuthConfig{}, multiErr } - logrus.Debugf("Credentials not found") + logrus.Debugf("No credentials for %s found", registry) return types.DockerAuthConfig{}, nil } -// GetAuthentication returns the registry credentials stored in -// either auth.json file or .docker/config.json -// If an entry is not found empty strings are returned for the username and password +// GetAuthentication returns the registry credentials stored in the +// registry-specific credential helpers or in the default global credentials +// helpers with falling back to using either auth.json file or +// .docker/config.json // // Deprecated: This API only has support for username and password. To get the // support for oauth2 in docker registry authentication, we added the new @@ -227,53 +292,132 @@ func getAuthenticationWithHomeDir(sys *types.SystemContext, registry, homeDir st return auth.Username, auth.Password, nil } -// RemoveAuthentication deletes the credentials stored in auth.json +// RemoveAuthentication removes credentials for `registry` from all possible +// sources such as credential helpers and auth files. func RemoveAuthentication(sys *types.SystemContext, registry string) error { - return modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) { - // First try cred helpers. - if ch, exists := auths.CredHelpers[registry]; exists { - return false, deleteAuthFromCredHelper(ch, registry) + helpers, err := sysregistriesv2.CredentialHelpers(sys) + if err != nil { + return err + } + + var multiErr error + isLoggedIn := false + + removeFromCredHelper := func(helper string) { + err := deleteAuthFromCredHelper(helper, registry) + if err == nil { + logrus.Debugf("Credentials for %q were deleted from credential helper %s", registry, helper) + isLoggedIn = true + return + } + if credentials.IsErrCredentialsNotFoundMessage(err.Error()) { + logrus.Debugf("Not logged in to %s with credential helper %s", registry, helper) + return } + multiErr = multierror.Append(multiErr, errors.Wrapf(err, "error removing credentials for %s from credential helper %s", registry, helper)) + } - // Next if keyring is enabled try kernel keyring - if enableKeyring { - err := deleteAuthFromKernelKeyring(registry) - if err == nil { - logrus.Debugf("credentials for %s were deleted from the kernel keyring", registry) - return false, nil + for _, helper := range helpers { + var err error + switch helper { + // Special-case the built-in helper for auth files. + case sysregistriesv2.AuthenticationFileHelper: + err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) { + if innerHelper, exists := auths.CredHelpers[registry]; exists { + removeFromCredHelper(innerHelper) + } + if _, ok := auths.AuthConfigs[registry]; ok { + isLoggedIn = true + delete(auths.AuthConfigs, registry) + } else if _, ok := auths.AuthConfigs[normalizeRegistry(registry)]; ok { + isLoggedIn = true + delete(auths.AuthConfigs, normalizeRegistry(registry)) + } + return true, multiErr + }) + if err != nil { + multiErr = multierror.Append(multiErr, err) } - logrus.Debugf("failed to delete credentials from the kernel keyring, falling back to authfiles") + // External helpers. + default: + removeFromCredHelper(helper) } + } - if _, ok := auths.AuthConfigs[registry]; ok { - delete(auths.AuthConfigs, registry) - } else if _, ok := auths.AuthConfigs[normalizeRegistry(registry)]; ok { - delete(auths.AuthConfigs, normalizeRegistry(registry)) - } else { - return false, ErrNotLoggedIn - } - return true, nil - }) + if multiErr != nil { + return multiErr + } + if !isLoggedIn { + return ErrNotLoggedIn + } + + return nil } -// RemoveAllAuthentication deletes all the credentials stored in auth.json and kernel keyring +// RemoveAllAuthentication deletes all the credentials stored in credential +// helpers and auth files. func RemoveAllAuthentication(sys *types.SystemContext) error { - return modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) { - if enableKeyring { - err := removeAllAuthFromKernelKeyring() - if err == nil { - logrus.Debugf("removing all credentials from kernel keyring") - return false, nil + helpers, err := sysregistriesv2.CredentialHelpers(sys) + if err != nil { + return err + } + + var multiErr error + for _, helper := range helpers { + var err error + switch helper { + // Special-case the built-in helper for auth files. + case sysregistriesv2.AuthenticationFileHelper: + err = modifyJSON(sys, func(auths *dockerConfigFile) (bool, error) { + for registry, helper := range auths.CredHelpers { + // Helpers in auth files are expected + // to exist, so no special treatment + // for them. + if err := deleteAuthFromCredHelper(helper, registry); err != nil { + return false, err + } + } + auths.CredHelpers = make(map[string]string) + auths.AuthConfigs = make(map[string]dockerAuthConfig) + return true, nil + }) + // External helpers. + default: + var creds map[string]string + creds, err = listAuthsFromCredHelper(helper) + switch errors.Cause(err) { + case nil: + for registry := range creds { + err = deleteAuthFromCredHelper(helper, registry) + if err != nil { + break + } + } + case exec.ErrNotFound: + // It's okay if the helper doesn't exist. + continue + default: + // fall through } - logrus.Debugf("error removing credentials from kernel keyring") } - auths.CredHelpers = make(map[string]string) - auths.AuthConfigs = make(map[string]dockerAuthConfig) - return true, nil - }) + if err != nil { + logrus.Debugf("Error removing credentials from credential helper %s: %v", helper, err) + multiErr = multierror.Append(multiErr, err) + continue + } + logrus.Debugf("All credentials removed from credential helper %s", helper) + } + + return multiErr +} + +func listAuthsFromCredHelper(credHelper string) (map[string]string, error) { + helperName := fmt.Sprintf("docker-credential-%s", credHelper) + p := helperclient.NewShellProgramFunc(helperName) + return helperclient.List(p) } -// getPathToAuth gets the path of the auth.json file used for reading and writing credentials +// getPathToAuth gets the path of the auth.json file used for reading and writting credentials // returns the path, and a bool specifies whether the file is in legacy format func getPathToAuth(sys *types.SystemContext) (string, bool, error) { return getPathToAuthWithOS(sys, runtime.GOOS) @@ -387,14 +531,17 @@ func modifyJSON(sys *types.SystemContext, editor func(auths *dockerConfigFile) ( return nil } -func getAuthFromCredHelper(credHelper, registry string) (string, string, error) { +func getAuthFromCredHelper(credHelper, registry string) (types.DockerAuthConfig, error) { helperName := fmt.Sprintf("docker-credential-%s", credHelper) p := helperclient.NewShellProgramFunc(helperName) creds, err := helperclient.Get(p, registry) if err != nil { - return "", "", err + return types.DockerAuthConfig{}, err } - return creds.Username, creds.Secret, nil + return types.DockerAuthConfig{ + Username: creds.Username, + Password: creds.Secret, + }, nil } func setAuthToCredHelper(credHelper, registry, username, password string) error { @@ -423,15 +570,7 @@ func findAuthentication(registry, path string, legacyFormat bool) (types.DockerA // First try cred helpers. They should always be normalized. if ch, exists := auths.CredHelpers[registry]; exists { - username, password, err := getAuthFromCredHelper(ch, registry) - if err != nil { - return types.DockerAuthConfig{}, err - } - - return types.DockerAuthConfig{ - Username: username, - Password: password, - }, nil + return getAuthFromCredHelper(ch, registry) } // I'm feeling lucky diff --git a/vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go b/vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go index 1531d6943..5bbfb450f 100644 --- a/vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go +++ b/vendor/github.com/containers/image/v5/pkg/docker/config/config_linux.go @@ -9,9 +9,13 @@ import ( "github.com/sirupsen/logrus" ) -const keyDescribePrefix = "container-registry-login:" +// NOTE: none of the functions here are currently used. If we ever want to +// reenable keyring support, we should introduce a similar built-in credential +// helpers as for `sysregistriesv2.AuthenticationFileHelper`. -func getAuthFromKernelKeyring(registry string) (string, string, error) { +const keyDescribePrefix = "container-registry-login:" // nolint + +func getAuthFromKernelKeyring(registry string) (string, string, error) { // nolint userkeyring, err := keyctl.UserKeyring() if err != nil { return "", "", err @@ -31,7 +35,7 @@ func getAuthFromKernelKeyring(registry string) (string, string, error) { return parts[0], parts[1], nil } -func deleteAuthFromKernelKeyring(registry string) error { +func deleteAuthFromKernelKeyring(registry string) error { // nolint userkeyring, err := keyctl.UserKeyring() if err != nil { @@ -44,7 +48,7 @@ func deleteAuthFromKernelKeyring(registry string) error { return key.Unlink() } -func removeAllAuthFromKernelKeyring() error { +func removeAllAuthFromKernelKeyring() error { // nolint keys, err := keyctl.ReadUserKeyring() if err != nil { return err @@ -77,7 +81,7 @@ func removeAllAuthFromKernelKeyring() error { return nil } -func setAuthToKernelKeyring(registry, username, password string) error { +func setAuthToKernelKeyring(registry, username, password string) error { // nolint keyring, err := keyctl.SessionKeyring() if err != nil { return err @@ -110,6 +114,6 @@ func setAuthToKernelKeyring(registry, username, password string) error { return nil } -func genDescription(registry string) string { +func genDescription(registry string) string { // nolint return fmt.Sprintf("%s%s", keyDescribePrefix, registry) } diff --git a/vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go b/vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go index f1e5c453e..b64f44674 100644 --- a/vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go +++ b/vendor/github.com/containers/image/v5/pkg/shortnames/shortnames.go @@ -313,7 +313,10 @@ func Resolve(ctx *types.SystemContext, name string) (*Resolved, error) { } // Error out if there's no matching alias and no search registries. if len(unqualifiedSearchRegistries) == 0 { - return nil, errors.Errorf("short-name %q did not resolve to an alias and no unqualified-search registries are defined in %q", name, usrConfig) + if usrConfig != "" { + return nil, errors.Errorf("short-name %q did not resolve to an alias and no unqualified-search registries are defined in %q", name, usrConfig) + } + return nil, errors.Errorf("short-name %q did not resolve to an alias and no containers-registries.conf(5) was found", name) } resolved.originDescription = usrConfig diff --git a/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go b/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go index 3312237ef..880f8c871 100644 --- a/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go +++ b/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go @@ -30,16 +30,24 @@ const builtinRegistriesConfPath = "/etc/containers/registries.conf" // systemRegistriesConfDirPath is the path to the system-wide registry // configuration directory and is used to add/subtract potential registries for // obtaining images. You can override this at build time with -// -ldflags '-X github.com/containers/image/v5/sysregistries.systemRegistriesConfDirecotyPath=$your_path' +// -ldflags '-X github.com/containers/image/v5/sysregistries.systemRegistriesConfDirectoryPath=$your_path' var systemRegistriesConfDirPath = builtinRegistriesConfDirPath // builtinRegistriesConfDirPath is the path to the registry configuration directory. // DO NOT change this, instead see systemRegistriesConfDirectoryPath above. const builtinRegistriesConfDirPath = "/etc/containers/registries.conf.d" +// AuthenticationFileHelper is a special key for credential helpers indicating +// the usage of consulting containers-auth.json files instead of a credential +// helper. +const AuthenticationFileHelper = "containers-auth.json" + // Endpoint describes a remote location of a registry. type Endpoint struct { - // The endpoint's remote location. + // The endpoint's remote location. Can be empty iff Prefix contains + // wildcard in the format: "*.example.com" for subdomain matching. + // Please refer to FindRegistry / PullSourcesFromReference instead + // of accessing/interpreting `Location` directly. Location string `toml:"location,omitempty"` // If true, certs verification will be skipped and HTTP (non-TLS) // connections will be allowed. @@ -57,11 +65,26 @@ var userRegistriesDir = filepath.FromSlash(".config/containers/registries.conf.d // The function errors if the newly created reference is not parsable. func (e *Endpoint) rewriteReference(ref reference.Named, prefix string) (reference.Named, error) { refString := ref.String() - if !refMatchesPrefix(refString, prefix) { + var newNamedRef string + // refMatchingPrefix returns the length of the match. Everything that + // follows the match gets appended to registries location. + prefixLen := refMatchingPrefix(refString, prefix) + if prefixLen == -1 { return nil, fmt.Errorf("invalid prefix '%v' for reference '%v'", prefix, refString) } - - newNamedRef := strings.Replace(refString, prefix, e.Location, 1) + // In the case of an empty `location` field, simply return the original + // input ref as-is. + // + // FIXME: already validated in postProcessRegistries, so check can probably + // be dropped. + // https://github.com/containers/image/pull/1191#discussion_r610621608 + if e.Location == "" { + if prefix[:2] != "*." { + return nil, fmt.Errorf("invalid prefix '%v' for empty location, should be in the format: *.example.com", prefix) + } + return ref, nil + } + newNamedRef = e.Location + refString[prefixLen:] newParsedRef, err := reference.ParseNamed(newNamedRef) if err != nil { return nil, errors.Wrapf(err, "error rewriting reference") @@ -77,6 +100,11 @@ type Registry struct { // and we pull from "example.com/bar/myimage:latest", the image will // effectively be pulled from "example.com/foo/bar/myimage:latest". // If no Prefix is specified, it defaults to the specified location. + // Prefix can also be in the format: "*.example.com" for matching + // subdomains. The wildcard should only be in the beginning and should also + // not contain any namespaces or special characters: "/", "@" or ":". + // Please refer to FindRegistry / PullSourcesFromReference instead + // of accessing/interpreting `Prefix` directly. Prefix string `toml:"prefix"` // A registry is an Endpoint too Endpoint @@ -154,6 +182,14 @@ type V2RegistriesConf struct { Registries []Registry `toml:"registry"` // An array of host[:port] (not prefix!) entries to use for resolving unqualified image references UnqualifiedSearchRegistries []string `toml:"unqualified-search-registries"` + // An array of global credential helpers to use for authentication + // (e.g., ["pass", "secretservice"]). The helpers are consulted in the + // specified order. Note that "containers-auth.json" is a reserved + // value for consulting auth files as specified in + // containers-auth.json(5). + // + // If empty, CredentialHelpers defaults to ["containers-auth.json"]. + CredentialHelpers []string `toml:"credential-helpers"` // ShortNameMode defines how short-name resolution should be handled by // _consumers_ of this package. Depending on the mode, the user should @@ -177,7 +213,7 @@ func (config *V2RegistriesConf) Nonempty() bool { // parsedConfig is the result of parsing, and possibly merging, configuration files; // it is the boundary between the process of reading+ingesting the files, and -// later interpreting the configuraiton based on caller’s requests. +// later interpreting the configuration based on caller’s requests. type parsedConfig struct { // NOTE: Update also parsedConfig.updateWithConfigurationFrom! @@ -212,9 +248,15 @@ func (e *InvalidRegistries) Error() string { func parseLocation(input string) (string, error) { trimmed := strings.TrimRight(input, "/") - if trimmed == "" { - return "", &InvalidRegistries{s: "invalid location: cannot be empty"} - } + // FIXME: This check needs to exist but fails for empty Location field with + // wildcarded prefix. Removal of this check "only" allows invalid input in, + // and does not prevent correct operation. + // https://github.com/containers/image/pull/1191#discussion_r610122617 + // + // if trimmed == "" { + // return "", &InvalidRegistries{s: "invalid location: cannot be empty"} + // } + // if strings.HasPrefix(trimmed, "http://") || strings.HasPrefix(trimmed, "https://") { msg := fmt.Sprintf("invalid location '%s': URI schemes are not supported", input) @@ -293,12 +335,20 @@ func (config *V2RegistriesConf) postProcessRegistries() error { } if reg.Prefix == "" { + if reg.Location == "" { + return &InvalidRegistries{s: "invalid condition: both location and prefix are unset"} + } reg.Prefix = reg.Location } else { reg.Prefix, err = parseLocation(reg.Prefix) if err != nil { return err } + // FIXME: allow config authors to always use Prefix. + // https://github.com/containers/image/pull/1191#discussion_r610622495 + if reg.Prefix[:2] != "*." && reg.Location == "" { + return &InvalidRegistries{s: "invalid condition: location is unset and prefix is not in the format: *.example.com"} + } } // make sure mirrors are valid @@ -307,8 +357,19 @@ func (config *V2RegistriesConf) postProcessRegistries() error { if err != nil { return err } + + //FIXME: unqualifiedSearchRegistries now also accepts empty values + //and shouldn't + // https://github.com/containers/image/pull/1191#discussion_r610623216 + if mir.Location == "" { + return &InvalidRegistries{s: "invalid condition: mirror location is unset"} + } + } + if reg.Location == "" { + regMap[reg.Prefix] = append(regMap[reg.Prefix], reg) + } else { + regMap[reg.Location] = append(regMap[reg.Location], reg) } - regMap[reg.Location] = append(regMap[reg.Location], reg) } // Given a registry can be mentioned multiple times (e.g., to have @@ -318,7 +379,13 @@ func (config *V2RegistriesConf) postProcessRegistries() error { // Note: we need to iterate over the registries array to ensure a // deterministic behavior which is not guaranteed by maps. for _, reg := range config.Registries { - others, ok := regMap[reg.Location] + var others []*Registry + var ok bool + if reg.Location == "" { + others, ok = regMap[reg.Prefix] + } else { + others, ok = regMap[reg.Location] + } if !ok { return fmt.Errorf("Internal error in V2RegistriesConf.PostProcess: entry in regMap is missing") } @@ -450,7 +517,7 @@ func newConfigWrapperWithHomeDir(ctx *types.SystemContext, homeDir string) confi return wrapper } -// ConfigurationSourceDescription returns a string containres paths of registries.conf and registries.conf.d +// ConfigurationSourceDescription returns a string containers paths of registries.conf and registries.conf.d func ConfigurationSourceDescription(ctx *types.SystemContext) string { wrapper := newConfigWrapper(ctx) configSources := []string{wrapper.configPath} @@ -601,11 +668,17 @@ func tryUpdatingCache(ctx *types.SystemContext, wrapper configWrapper) (*parsedC config.shortNameMode = defaultShortNameMode } + if len(config.partialV2.CredentialHelpers) == 0 { + config.partialV2.CredentialHelpers = []string{AuthenticationFileHelper} + } + // populate the cache configCache[wrapper] = config return config, nil } +// GetRegistries has been deprecated. Use FindRegistry instead. +// // GetRegistries loads and returns the registries specified in the config. // Note the parsed content of registry config files is cached. For reloading, // use `InvalidateCache` and re-call `GetRegistries`. @@ -663,27 +736,72 @@ func GetShortNameMode(ctx *types.SystemContext) (types.ShortNameMode, error) { return config.shortNameMode, err } -// refMatchesPrefix returns true iff ref, +// CredentialHelpers returns the global top-level credential helpers. +func CredentialHelpers(sys *types.SystemContext) ([]string, error) { + config, err := getConfig(sys) + if err != nil { + return nil, err + } + return config.partialV2.CredentialHelpers, nil +} + +// refMatchingSubdomainPrefix returns the length of ref +// iff ref, which is a registry, repository namespace, repository or image reference (as formatted by +// reference.Domain(), reference.Named.Name() or reference.Reference.String() +// — note that this requires the name to start with an explicit hostname!), +// matches a Registry.Prefix value containing wildcarded subdomains in the +// format: *.example.com. Wildcards are only accepted at the beginning, so +// other formats like example.*.com will not work. Wildcarded prefixes also +// cannot contain port numbers or namespaces in them. +func refMatchingSubdomainPrefix(ref, prefix string) int { + index := strings.Index(ref, prefix[1:]) + if index == -1 { + return -1 + } + if strings.Contains(ref[:index], "/") { + return -1 + } + index += len(prefix[1:]) + if index == len(ref) { + return index + } + switch ref[index] { + case ':', '/', '@': + return index + default: + return -1 + } +} + +// refMatchingPrefix returns the length of the prefix iff ref, // which is a registry, repository namespace, repository or image reference (as formatted by // reference.Domain(), reference.Named.Name() or reference.Reference.String() // — note that this requires the name to start with an explicit hostname!), // matches a Registry.Prefix value. // (This is split from the caller primarily to make testing easier.) -func refMatchesPrefix(ref, prefix string) bool { +func refMatchingPrefix(ref, prefix string) int { switch { + case prefix[0:2] == "*.": + return refMatchingSubdomainPrefix(ref, prefix) case len(ref) < len(prefix): - return false + return -1 case len(ref) == len(prefix): - return ref == prefix + if ref == prefix { + return len(prefix) + } + return -1 case len(ref) > len(prefix): if !strings.HasPrefix(ref, prefix) { - return false + return -1 } c := ref[len(prefix)] // This allows "example.com:5000" to match "example.com", // which is unintended; that will get fixed eventually, DON'T RELY // ON THE CURRENT BEHAVIOR. - return c == ':' || c == '/' || c == '@' + if c == ':' || c == '/' || c == '@' { + return len(prefix) + } + return -1 default: panic("Internal error: impossible comparison outcome") } @@ -700,10 +818,16 @@ func FindRegistry(ctx *types.SystemContext, ref string) (*Registry, error) { return nil, err } + return findRegistryWithParsedConfig(config, ref) +} + +// findRegistryWithParsedConfig implements `FindRegistry` with a pre-loaded +// parseConfig. +func findRegistryWithParsedConfig(config *parsedConfig, ref string) (*Registry, error) { reg := Registry{} prefixLen := 0 for _, r := range config.partialV2.Registries { - if refMatchesPrefix(ref, r.Prefix) { + if refMatchingPrefix(ref, r.Prefix) != -1 { length := len(r.Prefix) if length > prefixLen { reg = r @@ -772,6 +896,17 @@ func loadConfigFile(path string, forceV2 bool) (*parsedConfig, error) { res.shortNameMode = types.ShortNameModeInvalid } + // Valid wildcarded prefixes must be in the format: *.example.com + // FIXME: Move to postProcessRegistries + // https://github.com/containers/image/pull/1191#discussion_r610623829 + for i := range res.partialV2.Registries { + prefix := res.partialV2.Registries[i].Prefix + if prefix[:2] == "*." && strings.ContainsAny(prefix, "/@:") { + msg := fmt.Sprintf("Wildcarded prefix should be in the format: *.example.com. Current prefix %q is incorrectly formatted", prefix) + return nil, &InvalidRegistries{s: msg} + } + } + // Parse and validate short-name aliases. cache, err := newShortNameAliasCache(path, &res.partialV2.shortNameAliasConf) if err != nil { @@ -825,6 +960,11 @@ func (c *parsedConfig) updateWithConfigurationFrom(updates *parsedConfig) { c.unqualifiedSearchRegistriesOrigin = updates.unqualifiedSearchRegistriesOrigin } + // == Merge credential helpers: + if updates.partialV2.CredentialHelpers != nil { + c.partialV2.CredentialHelpers = updates.partialV2.CredentialHelpers + } + // == Merge shortNameMode: // We don’t maintain c.partialV2.ShortNameMode. if updates.shortNameMode != types.ShortNameModeInvalid { diff --git a/vendor/github.com/containers/image/v5/storage/storage_image.go b/vendor/github.com/containers/image/v5/storage/storage_image.go index 5340690bc..ae020dd66 100644 --- a/vendor/github.com/containers/image/v5/storage/storage_image.go +++ b/vendor/github.com/containers/image/v5/storage/storage_image.go @@ -18,6 +18,7 @@ import ( "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/image" "github.com/containers/image/v5/internal/tmpdir" + internalTypes "github.com/containers/image/v5/internal/types" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/blobinfocache/none" "github.com/containers/image/v5/types" @@ -45,6 +46,7 @@ var ( type storageImageSource struct { imageRef storageReference image *storage.Image + systemContext *types.SystemContext // SystemContext used in GetBlob() to create temporary files layerPosition map[digest.Digest]int // Where we are in reading a blob's layers cachedManifest []byte // A cached copy of the manifest, if already known, or nil getBlobMutex sync.Mutex // Mutex to sync state for parallel GetBlob executions @@ -54,17 +56,31 @@ type storageImageSource struct { type storageImageDestination struct { imageRef storageReference - directory string // Temporary directory where we store blobs until Commit() time - nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs - manifest []byte // Manifest contents, temporary - signatures []byte // Signature contents, temporary - signatureses map[digest.Digest][]byte // Instance signature contents, temporary - putBlobMutex sync.Mutex // Mutex to sync state for parallel PutBlob executions - blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs - fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes - filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them - SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice - SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // Sizes of each manifest's signature slice + directory string // Temporary directory where we store blobs until Commit() time + nextTempFileID int32 // A counter that we use for computing filenames to assign to blobs + manifest []byte // Manifest contents, temporary + signatures []byte // Signature contents, temporary + signatureses map[digest.Digest][]byte // Instance signature contents, temporary + SignatureSizes []int `json:"signature-sizes,omitempty"` // List of sizes of each signature slice + SignaturesSizes map[digest.Digest][]int `json:"signatures-sizes,omitempty"` // Sizes of each manifest's signature slice + + // A storage destination may be used concurrently. Accesses are + // serialized via a mutex. Please refer to the individual comments + // below for details. + lock sync.Mutex + // Mapping from layer (by index) to the associated ID in the storage. + // It's protected *implicitly* since `commitLayer()`, at any given + // time, can only be executed by *one* goroutine. Please refer to + // `queueOrCommit()` for further details on how the single-caller + // guarantee is implemented. + indexToStorageID map[int]*string + // All accesses to below data are protected by `lock` which is made + // *explicit* in the code. + blobDiffIDs map[digest.Digest]digest.Digest // Mapping from layer blobsums to their corresponding DiffIDs + fileSizes map[digest.Digest]int64 // Mapping from layer blobsums to their sizes + filenames map[digest.Digest]string // Mapping from layer blobsums to names of files we used to hold them + currentIndex int // The index of the layer to be committed (i.e., lower indices have already been committed) + indexToPulledBlob map[int]*types.BlobInfo // Mapping from layer (by index) to pulled down blob } type storageImageCloser struct { @@ -96,6 +112,7 @@ func newImageSource(ctx context.Context, sys *types.SystemContext, imageRef stor // Build the reader object. image := &storageImageSource{ imageRef: imageRef, + systemContext: sys, image: img, layerPosition: make(map[digest.Digest]int), SignatureSizes: []int{}, @@ -131,8 +148,35 @@ func (s *storageImageSource) GetBlob(ctx context.Context, info types.BlobInfo, c if info.Digest == image.GzippedEmptyLayerDigest { return ioutil.NopCloser(bytes.NewReader(image.GzippedEmptyLayer)), int64(len(image.GzippedEmptyLayer)), nil } + + // NOTE: the blob is first written to a temporary file and subsequently + // closed. The intention is to keep the time we own the storage lock + // as short as possible to allow other processes to access the storage. rc, n, _, err = s.getBlobAndLayerID(info) - return rc, n, err + if err != nil { + return nil, 0, err + } + defer rc.Close() + + tmpFile, err := ioutil.TempFile(tmpdir.TemporaryDirectoryForBigFiles(s.systemContext), "") + if err != nil { + return nil, 0, err + } + + if _, err := io.Copy(tmpFile, rc); err != nil { + return nil, 0, err + } + + if _, err := tmpFile.Seek(0, 0); err != nil { + return nil, 0, err + } + + wrapper := ioutils.NewReadCloserWrapper(tmpFile, func() error { + defer os.Remove(tmpFile.Name()) + return tmpFile.Close() + }) + + return wrapper, n, err } // getBlobAndLayer reads the data blob or filesystem layer which matches the digest and size, if given. @@ -347,14 +391,16 @@ func newImageDestination(sys *types.SystemContext, imageRef storageReference) (* return nil, errors.Wrapf(err, "error creating a temporary directory") } image := &storageImageDestination{ - imageRef: imageRef, - directory: directory, - signatureses: make(map[digest.Digest][]byte), - blobDiffIDs: make(map[digest.Digest]digest.Digest), - fileSizes: make(map[digest.Digest]int64), - filenames: make(map[digest.Digest]string), - SignatureSizes: []int{}, - SignaturesSizes: make(map[digest.Digest][]int), + imageRef: imageRef, + directory: directory, + signatureses: make(map[digest.Digest][]byte), + blobDiffIDs: make(map[digest.Digest]digest.Digest), + fileSizes: make(map[digest.Digest]int64), + filenames: make(map[digest.Digest]string), + SignatureSizes: []int{}, + SignaturesSizes: make(map[digest.Digest][]int), + indexToStorageID: make(map[int]*string), + indexToPulledBlob: make(map[int]*types.BlobInfo), } return image, nil } @@ -381,6 +427,26 @@ func (s *storageImageDestination) computeNextBlobCacheFile() string { return filepath.Join(s.directory, fmt.Sprintf("%d", atomic.AddInt32(&s.nextTempFileID, 1))) } +// PutBlobWithOptions is a wrapper around PutBlob. If options.LayerIndex is +// set, the blob will be committed directly. Either by the calling goroutine +// or by another goroutine already committing layers. +// +// Please not that TryReusingBlobWithOptions and PutBlobWithOptions *must* be +// used the together. Mixing the two with non "WithOptions" functions is not +// supported. +func (s *storageImageDestination) PutBlobWithOptions(ctx context.Context, stream io.Reader, blobinfo types.BlobInfo, options internalTypes.PutBlobOptions) (types.BlobInfo, error) { + info, err := s.PutBlob(ctx, stream, blobinfo, options.Cache, options.IsConfig) + if err != nil { + return info, err + } + + if options.IsConfig || options.LayerIndex == nil { + return info, nil + } + + return info, s.queueOrCommit(ctx, info, *options.LayerIndex) +} + // HasThreadSafePutBlob indicates whether PutBlob can be executed concurrently. func (s *storageImageDestination) HasThreadSafePutBlob() bool { return true @@ -436,11 +502,11 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader, return errorBlobInfo, errors.WithStack(ErrBlobSizeMismatch) } // Record information about the blob. - s.putBlobMutex.Lock() + s.lock.Lock() s.blobDiffIDs[hasher.Digest()] = diffID.Digest() s.fileSizes[hasher.Digest()] = counter.Count s.filenames[hasher.Digest()] = filename - s.putBlobMutex.Unlock() + s.lock.Unlock() blobDigest := blobinfo.Digest if blobDigest.Validate() != nil { blobDigest = hasher.Digest() @@ -458,6 +524,22 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader, }, nil } +// TryReusingBlobWithOptions is a wrapper around TryReusingBlob. If +// options.LayerIndex is set, the reused blob will be recoreded as already +// pulled. +// +// Please not that TryReusingBlobWithOptions and PutBlobWithOptions *must* be +// used the together. Mixing the two with the non "WithOptions" functions +// is not supported. +func (s *storageImageDestination) TryReusingBlobWithOptions(ctx context.Context, blobinfo types.BlobInfo, options internalTypes.TryReusingBlobOptions) (bool, types.BlobInfo, error) { + reused, info, err := s.TryReusingBlob(ctx, blobinfo, options.Cache, options.CanSubstitute) + if err != nil || !reused || options.LayerIndex == nil { + return reused, info, err + } + + return reused, info, s.queueOrCommit(ctx, info, *options.LayerIndex) +} + // TryReusingBlob checks whether the transport already contains, or can efficiently reuse, a blob, and if so, applies it to the current destination // (e.g. if the blob is a filesystem layer, this signifies that the changes it describes need to be applied again when composing a filesystem tree). // info.Digest must not be empty. @@ -469,8 +551,8 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader, // May use and/or update cache. func (s *storageImageDestination) TryReusingBlob(ctx context.Context, blobinfo types.BlobInfo, cache types.BlobInfoCache, canSubstitute bool) (bool, types.BlobInfo, error) { // lock the entire method as it executes fairly quickly - s.putBlobMutex.Lock() - defer s.putBlobMutex.Unlock() + s.lock.Lock() + defer s.lock.Unlock() if blobinfo.Digest == "" { return false, types.BlobInfo{}, errors.Errorf(`Can not check for a blob with unknown digest`) } @@ -607,6 +689,192 @@ func (s *storageImageDestination) getConfigBlob(info types.BlobInfo) ([]byte, er return nil, errors.New("blob not found") } +// queueOrCommit queues in the specified blob to be committed to the storage. +// If no other goroutine is already committing layers, the layer and all +// subsequent layers (if already queued) will be committed to the storage. +func (s *storageImageDestination) queueOrCommit(ctx context.Context, blob types.BlobInfo, index int) error { + // NOTE: whenever the code below is touched, make sure that all code + // paths unlock the lock and to unlock it exactly once. + // + // Conceptually, the code is divided in two stages: + // + // 1) Queue in work by marking the layer as ready to be committed. + // If at least one previous/parent layer with a lower index has + // not yet been committed, return early. + // + // 2) Process the queued-in work by committing the "ready" layers + // in sequence. Make sure that more items can be queued-in + // during the comparatively I/O expensive task of committing a + // layer. + // + // The conceptual benefit of this design is that caller can continue + // pulling layers after an early return. At any given time, only one + // caller is the "worker" routine comitting layers. All other routines + // can continue pulling and queuing in layers. + s.lock.Lock() + s.indexToPulledBlob[index] = &blob + + // We're still waiting for at least one previous/parent layer to be + // committed, so there's nothing to do. + if index != s.currentIndex { + s.lock.Unlock() + return nil + } + + for info := s.indexToPulledBlob[index]; info != nil; info = s.indexToPulledBlob[index] { + s.lock.Unlock() + layerInfo := manifest.LayerInfo{ + BlobInfo: *info, + EmptyLayer: info.Digest == image.GzippedEmptyLayerDigest, + } + // Note: commitLayer locks on-demand. + if err := s.commitLayer(ctx, layerInfo, index); err != nil { + return err + } + s.lock.Lock() + index++ + } + + // Set the index at the very end to make sure that only one routine + // enters stage 2). + s.currentIndex = index + s.lock.Unlock() + return nil +} + +// commitLayer commits the specified blob with the given index to the storage. +// Note that the previous layer is expected to already be committed. +// +// Caution: this function must be called without holding `s.lock`. Callers +// must guarantee that, at any given time, at most one goroutine may execute +// `commitLayer()`. +func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest.LayerInfo, index int) error { + // Already commited? Return early. + if _, alreadyCommitted := s.indexToStorageID[index]; alreadyCommitted { + return nil + } + + // Start with an empty string or the previous layer ID. Note that + // `s.indexToStorageID` can only be accessed by *one* goroutine at any + // given time. Hence, we don't need to lock accesses. + var lastLayer string + if prev := s.indexToStorageID[index-1]; prev != nil { + lastLayer = *prev + } + + // Carry over the previous ID for empty non-base layers. + if blob.EmptyLayer && index > 0 { + s.indexToStorageID[index] = &lastLayer + return nil + } + + // Check if there's already a layer with the ID that we'd give to the result of applying + // this layer blob to its parent, if it has one, or the blob's hex value otherwise. + s.lock.Lock() + diffID, haveDiffID := s.blobDiffIDs[blob.Digest] + s.lock.Unlock() + if !haveDiffID { + // Check if it's elsewhere and the caller just forgot to pass it to us in a PutBlob(), + // or to even check if we had it. + // Use none.NoCache to avoid a repeated DiffID lookup in the BlobInfoCache; a caller + // that relies on using a blob digest that has never been seen by the store had better call + // TryReusingBlob; not calling PutBlob already violates the documented API, so there’s only + // so far we are going to accommodate that (if we should be doing that at all). + logrus.Debugf("looking for diffID for blob %+v", blob.Digest) + // NOTE: use `TryReusingBlob` to prevent recursion. + has, _, err := s.TryReusingBlob(ctx, blob.BlobInfo, none.NoCache, false) + if err != nil { + return errors.Wrapf(err, "error checking for a layer based on blob %q", blob.Digest.String()) + } + if !has { + return errors.Errorf("error determining uncompressed digest for blob %q", blob.Digest.String()) + } + diffID, haveDiffID = s.blobDiffIDs[blob.Digest] + if !haveDiffID { + return errors.Errorf("we have blob %q, but don't know its uncompressed digest", blob.Digest.String()) + } + } + id := diffID.Hex() + if lastLayer != "" { + id = digest.Canonical.FromBytes([]byte(lastLayer + "+" + diffID.Hex())).Hex() + } + if layer, err2 := s.imageRef.transport.store.Layer(id); layer != nil && err2 == nil { + // There's already a layer that should have the right contents, just reuse it. + lastLayer = layer.ID + s.indexToStorageID[index] = &lastLayer + return nil + } + // Check if we previously cached a file with that blob's contents. If we didn't, + // then we need to read the desired contents from a layer. + s.lock.Lock() + filename, ok := s.filenames[blob.Digest] + s.lock.Unlock() + if !ok { + // Try to find the layer with contents matching that blobsum. + layer := "" + layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(diffID) + if err2 == nil && len(layers) > 0 { + layer = layers[0].ID + } else { + layers, err2 = s.imageRef.transport.store.LayersByCompressedDigest(blob.Digest) + if err2 == nil && len(layers) > 0 { + layer = layers[0].ID + } + } + if layer == "" { + return errors.Wrapf(err2, "error locating layer for blob %q", blob.Digest) + } + // Read the layer's contents. + noCompression := archive.Uncompressed + diffOptions := &storage.DiffOptions{ + Compression: &noCompression, + } + diff, err2 := s.imageRef.transport.store.Diff("", layer, diffOptions) + if err2 != nil { + return errors.Wrapf(err2, "error reading layer %q for blob %q", layer, blob.Digest) + } + // Copy the layer diff to a file. Diff() takes a lock that it holds + // until the ReadCloser that it returns is closed, and PutLayer() wants + // the same lock, so the diff can't just be directly streamed from one + // to the other. + filename = s.computeNextBlobCacheFile() + file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600) + if err != nil { + diff.Close() + return errors.Wrapf(err, "error creating temporary file %q", filename) + } + // Copy the data to the file. + // TODO: This can take quite some time, and should ideally be cancellable using + // ctx.Done(). + _, err = io.Copy(file, diff) + diff.Close() + file.Close() + if err != nil { + return errors.Wrapf(err, "error storing blob to file %q", filename) + } + // Make sure that we can find this file later, should we need the layer's + // contents again. + s.lock.Lock() + s.filenames[blob.Digest] = filename + s.lock.Unlock() + } + // Read the cached blob and use it as a diff. + file, err := os.Open(filename) + if err != nil { + return errors.Wrapf(err, "error opening file %q", filename) + } + defer file.Close() + // Build the new layer using the diff, regardless of where it came from. + // TODO: This can take quite some time, and should ideally be cancellable using ctx.Done(). + layer, _, err := s.imageRef.transport.store.PutLayer(id, lastLayer, nil, "", false, nil, file) + if err != nil && errors.Cause(err) != storage.ErrDuplicateID { + return errors.Wrapf(err, "error adding layer with blob %q", blob.Digest) + } + + s.indexToStorageID[index] = &layer.ID + return nil +} + func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel types.UnparsedImage) error { if len(s.manifest) == 0 { return errors.New("Internal error: storageImageDestination.Commit() called without PutManifest()") @@ -644,108 +912,19 @@ func (s *storageImageDestination) Commit(ctx context.Context, unparsedToplevel t return errors.Wrapf(err, "error parsing manifest") } layerBlobs := man.LayerInfos() - // Extract or find the layers. - lastLayer := "" - for _, blob := range layerBlobs { - if blob.EmptyLayer { - continue - } - - // Check if there's already a layer with the ID that we'd give to the result of applying - // this layer blob to its parent, if it has one, or the blob's hex value otherwise. - diffID, haveDiffID := s.blobDiffIDs[blob.Digest] - if !haveDiffID { - // Check if it's elsewhere and the caller just forgot to pass it to us in a PutBlob(), - // or to even check if we had it. - // Use none.NoCache to avoid a repeated DiffID lookup in the BlobInfoCache; a caller - // that relies on using a blob digest that has never been seen by the store had better call - // TryReusingBlob; not calling PutBlob already violates the documented API, so there’s only - // so far we are going to accommodate that (if we should be doing that at all). - logrus.Debugf("looking for diffID for blob %+v", blob.Digest) - has, _, err := s.TryReusingBlob(ctx, blob.BlobInfo, none.NoCache, false) - if err != nil { - return errors.Wrapf(err, "error checking for a layer based on blob %q", blob.Digest.String()) - } - if !has { - return errors.Errorf("error determining uncompressed digest for blob %q", blob.Digest.String()) - } - diffID, haveDiffID = s.blobDiffIDs[blob.Digest] - if !haveDiffID { - return errors.Errorf("we have blob %q, but don't know its uncompressed digest", blob.Digest.String()) - } - } - id := diffID.Hex() - if lastLayer != "" { - id = digest.Canonical.FromBytes([]byte(lastLayer + "+" + diffID.Hex())).Hex() - } - if layer, err2 := s.imageRef.transport.store.Layer(id); layer != nil && err2 == nil { - // There's already a layer that should have the right contents, just reuse it. - lastLayer = layer.ID - continue - } - // Check if we previously cached a file with that blob's contents. If we didn't, - // then we need to read the desired contents from a layer. - filename, ok := s.filenames[blob.Digest] - if !ok { - // Try to find the layer with contents matching that blobsum. - layer := "" - layers, err2 := s.imageRef.transport.store.LayersByUncompressedDigest(diffID) - if err2 == nil && len(layers) > 0 { - layer = layers[0].ID - } else { - layers, err2 = s.imageRef.transport.store.LayersByCompressedDigest(blob.Digest) - if err2 == nil && len(layers) > 0 { - layer = layers[0].ID - } - } - if layer == "" { - return errors.Wrapf(err2, "error locating layer for blob %q", blob.Digest) - } - // Read the layer's contents. - noCompression := archive.Uncompressed - diffOptions := &storage.DiffOptions{ - Compression: &noCompression, - } - diff, err2 := s.imageRef.transport.store.Diff("", layer, diffOptions) - if err2 != nil { - return errors.Wrapf(err2, "error reading layer %q for blob %q", layer, blob.Digest) - } - // Copy the layer diff to a file. Diff() takes a lock that it holds - // until the ReadCloser that it returns is closed, and PutLayer() wants - // the same lock, so the diff can't just be directly streamed from one - // to the other. - filename = s.computeNextBlobCacheFile() - file, err := os.OpenFile(filename, os.O_CREATE|os.O_TRUNC|os.O_WRONLY|os.O_EXCL, 0600) - if err != nil { - diff.Close() - return errors.Wrapf(err, "error creating temporary file %q", filename) - } - // Copy the data to the file. - // TODO: This can take quite some time, and should ideally be cancellable using - // ctx.Done(). - _, err = io.Copy(file, diff) - diff.Close() - file.Close() - if err != nil { - return errors.Wrapf(err, "error storing blob to file %q", filename) - } - // Make sure that we can find this file later, should we need the layer's - // contents again. - s.filenames[blob.Digest] = filename - } - // Read the cached blob and use it as a diff. - file, err := os.Open(filename) - if err != nil { - return errors.Wrapf(err, "error opening file %q", filename) + // Extract, commit, or find the layers. + for i, blob := range layerBlobs { + if err := s.commitLayer(ctx, blob, i); err != nil { + return err } - defer file.Close() - // Build the new layer using the diff, regardless of where it came from. - // TODO: This can take quite some time, and should ideally be cancellable using ctx.Done(). - layer, _, err := s.imageRef.transport.store.PutLayer(id, lastLayer, nil, "", false, nil, file) - if err != nil && errors.Cause(err) != storage.ErrDuplicateID { - return errors.Wrapf(err, "error adding layer with blob %q", blob.Digest) + } + var lastLayer string + if len(layerBlobs) > 0 { // Can happen when using caches + prev := s.indexToStorageID[len(layerBlobs)-1] + if prev == nil { + return errors.Errorf("Internal error: StorageImageDestination.Commit(): previous layer %d hasn't been commited (lastLayer == nil)", len(layerBlobs)-1) } - lastLayer = layer.ID + lastLayer = *prev } // If one of those blobs was a configuration blob, then we can try to dig out the date when the image diff --git a/vendor/github.com/containers/image/v5/types/types.go b/vendor/github.com/containers/image/v5/types/types.go index 8655ca443..3e7abd34d 100644 --- a/vendor/github.com/containers/image/v5/types/types.go +++ b/vendor/github.com/containers/image/v5/types/types.go @@ -147,7 +147,7 @@ type BlobInfo struct { } // BICTransportScope encapsulates transport-dependent representation of a “scope” where blobs are or are not present. -// BlobInfocache.RecordKnownLocations / BlobInfocache.CandidateLocations record data aboud blobs keyed by (scope, digest). +// BlobInfocache.RecordKnownLocations / BlobInfocache.CandidateLocations record data about blobs keyed by (scope, digest). // The scope will typically be similar to an ImageReference, or a superset of it within which blobs are reusable. // // NOTE: The contents of this structure may be recorded in a persistent file, possibly shared across different @@ -179,7 +179,7 @@ type BICReplacementCandidate struct { // It records two kinds of data: // - Sets of corresponding digest vs. uncompressed digest ("DiffID") pairs: // One of the two digests is known to be uncompressed, and a single uncompressed digest may correspond to more than one compressed digest. -// This allows matching compressed layer blobs to existing local uncompressed layers (to avoid unnecessary download and decompresssion), +// This allows matching compressed layer blobs to existing local uncompressed layers (to avoid unnecessary download and decompression), // or uncompressed layer blobs to existing remote compressed layers (to avoid unnecessary compression and upload)/ // // It is allowed to record an (uncompressed digest, the same uncompressed digest) correspondence, to express that the digest is known @@ -219,7 +219,7 @@ type BlobInfoCache interface { // CandidateLocations returns a prioritized, limited, number of blobs and their locations that could possibly be reused // within the specified (transport scope) (if they still exist, which is not guaranteed). // - // If !canSubstitute, the returned cadidates will match the submitted digest exactly; if canSubstitute, + // If !canSubstitute, the returned candidates will match the submitted digest exactly; if canSubstitute, // data from previous RecordDigestUncompressedPair calls is used to also look up variants of the blob which have the same // uncompressed digest. CandidateLocations(transport ImageTransport, scope BICTransportScope, digest digest.Digest, canSubstitute bool) []BICReplacementCandidate @@ -582,7 +582,7 @@ type SystemContext struct { // === OCI.Transport overrides === // If not "", a directory containing a CA certificate (ending with ".crt"), - // a client certificate (ending with ".cert") and a client ceritificate key + // a client certificate (ending with ".cert") and a client certificate key // (ending with ".key") used when downloading OCI image layers. OCICertPath string // Allow downloading OCI image layers over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections. @@ -594,7 +594,7 @@ type SystemContext struct { // === docker.Transport overrides === // If not "", a directory containing a CA certificate (ending with ".crt"), - // a client certificate (ending with ".cert") and a client ceritificate key + // a client certificate (ending with ".cert") and a client certificate key // (ending with ".key") used when talking to a Docker Registry. DockerCertPath string // If not "", overrides the system’s default path for a directory containing host[:port] subdirectories with the same structure as DockerCertPath above. diff --git a/vendor/github.com/containers/image/v5/version/version.go b/vendor/github.com/containers/image/v5/version/version.go index fb7230241..3e9f09aab 100644 --- a/vendor/github.com/containers/image/v5/version/version.go +++ b/vendor/github.com/containers/image/v5/version/version.go @@ -6,9 +6,9 @@ const ( // VersionMajor is for an API incompatible changes VersionMajor = 5 // VersionMinor is for functionality in a backwards-compatible manner - VersionMinor = 10 + VersionMinor = 11 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 5 + VersionPatch = 0 // VersionDev indicates development branch. Releases will be empty string. VersionDev = "" diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index 450a687b2..5e57fb895 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.28.1 +1.29.0 diff --git a/vendor/github.com/containers/storage/drivers/copy/copy_linux.go b/vendor/github.com/containers/storage/drivers/copy/copy_linux.go index 1cd16a501..c2156861f 100644 --- a/vendor/github.com/containers/storage/drivers/copy/copy_linux.go +++ b/vendor/github.com/containers/storage/drivers/copy/copy_linux.go @@ -25,6 +25,7 @@ import ( "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/pools" "github.com/containers/storage/pkg/system" + "github.com/containers/storage/pkg/unshare" rsystem "github.com/opencontainers/runc/libcontainer/system" "golang.org/x/sys/unix" ) @@ -293,6 +294,10 @@ func doCopyXattrs(srcPath, dstPath string) error { } } + if unshare.IsRootless() { + return nil + } + // We need to copy this attribute if it appears in an overlay upper layer, as // this function is used to copy those. It is set by overlay if a directory // is removed and then re-created and should not inherit anything from the diff --git a/vendor/github.com/containers/storage/drivers/driver.go b/vendor/github.com/containers/storage/drivers/driver.go index 1a450278a..9aa407168 100644 --- a/vendor/github.com/containers/storage/drivers/driver.go +++ b/vendor/github.com/containers/storage/drivers/driver.go @@ -14,6 +14,7 @@ import ( "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/directory" "github.com/containers/storage/pkg/idtools" + digest "github.com/opencontainers/go-digest" ) // FsMagic unsigned id of the filesystem in use. @@ -33,7 +34,9 @@ var ( // ErrPrerequisites returned when driver does not meet prerequisites. ErrPrerequisites = errors.New("prerequisites for driver not satisfied (wrong filesystem?)") // ErrIncompatibleFS returned when file system is not supported. - ErrIncompatibleFS = fmt.Errorf("backing file system is unsupported for this graph driver") + ErrIncompatibleFS = errors.New("backing file system is unsupported for this graph driver") + // ErrLayerUnknown returned when the specified layer is unknown by the driver. + ErrLayerUnknown = errors.New("unknown layer") ) //CreateOpts contains optional arguments for Create() and CreateReadWrite() @@ -117,6 +120,7 @@ type ProtoDriver interface { // known to this driver. Cleanup() error // AdditionalImageStores returns additional image stores supported by the driver + // This API is experimental and can be changed without bumping the major version number. AdditionalImageStores() []string } @@ -180,6 +184,30 @@ type CapabilityDriver interface { Capabilities() Capabilities } +// AdditionalLayer reprents a layer that is stored in the additional layer store +// This API is experimental and can be changed without bumping the major version number. +type AdditionalLayer interface { + // CreateAs creates a new layer from this additional layer + CreateAs(id, parent string) error + + // Info returns arbitrary information stored along with this layer (i.e. `info` file) + Info() (io.ReadCloser, error) + + // Release tells the additional layer store that we don't use this handler. + Release() +} + +// AdditionalLayerStoreDriver is the interface for driver that supports +// additional layer store functionality. +// This API is experimental and can be changed without bumping the major version number. +type AdditionalLayerStoreDriver interface { + Driver + + // LookupAdditionalLayer looks up additional layer store by the specified + // digest and ref and returns an object representing that layer. + LookupAdditionalLayer(d digest.Digest, ref string) (AdditionalLayer, error) +} + // DiffGetterDriver is the interface for layered file system drivers that // provide a specialized function for getting file contents for tar-split. type DiffGetterDriver interface { diff --git a/vendor/github.com/containers/storage/drivers/overlay/check.go b/vendor/github.com/containers/storage/drivers/overlay/check.go index 9a62e7d75..67287b492 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/check.go +++ b/vendor/github.com/containers/storage/drivers/overlay/check.go @@ -10,6 +10,7 @@ import ( "path/filepath" "syscall" + "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/ioutils" "github.com/containers/storage/pkg/mount" "github.com/containers/storage/pkg/system" @@ -54,7 +55,7 @@ func doesSupportNativeDiff(d, mountOpts string) error { } // Mark l2/d as opaque - if err := system.Lsetxattr(filepath.Join(td, "l2", "d"), "trusted.overlay.opaque", []byte("y"), 0); err != nil { + if err := system.Lsetxattr(filepath.Join(td, "l2", "d"), archive.GetOverlayXattrName("opaque"), []byte("y"), 0); err != nil { return errors.Wrap(err, "failed to set opaque flag on middle layer") } @@ -78,7 +79,7 @@ func doesSupportNativeDiff(d, mountOpts string) error { } // Check l3/d does not have opaque flag - xattrOpaque, err := system.Lgetxattr(filepath.Join(td, "l3", "d"), "trusted.overlay.opaque") + xattrOpaque, err := system.Lgetxattr(filepath.Join(td, "l3", "d"), archive.GetOverlayXattrName("opaque")) if err != nil { return errors.Wrap(err, "failed to read opaque flag on upper layer") } @@ -95,7 +96,7 @@ func doesSupportNativeDiff(d, mountOpts string) error { return errors.Wrap(err, "failed to rename dir in merged directory") } // get the xattr of "d2" - xattrRedirect, err := system.Lgetxattr(filepath.Join(td, "l3", "d2"), "trusted.overlay.redirect") + xattrRedirect, err := system.Lgetxattr(filepath.Join(td, "l3", "d2"), archive.GetOverlayXattrName("redirect")) if err != nil { return errors.Wrap(err, "failed to read redirect flag on upper layer") } @@ -161,7 +162,7 @@ func doesMetacopy(d, mountOpts string) (bool, error) { if err := os.Chmod(filepath.Join(td, "merged", "f"), 0600); err != nil { return false, errors.Wrap(err, "error changing permissions on file for metacopy check") } - metacopy, err := system.Lgetxattr(filepath.Join(td, "l2", "f"), "trusted.overlay.metacopy") + metacopy, err := system.Lgetxattr(filepath.Join(td, "l2", "f"), archive.GetOverlayXattrName("metacopy")) if err != nil { return false, errors.Wrap(err, "metacopy flag was not set on file in upper layer") } diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 864da844b..cbf31d353 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -4,6 +4,7 @@ package overlay import ( "bytes" + "encoding/base64" "fmt" "io" "io/ioutil" @@ -31,6 +32,7 @@ import ( "github.com/containers/storage/pkg/unshare" units "github.com/docker/go-units" "github.com/hashicorp/go-multierror" + digest "github.com/opencontainers/go-digest" rsystem "github.com/opencontainers/runc/libcontainer/system" "github.com/opencontainers/selinux/go-selinux" "github.com/opencontainers/selinux/go-selinux/label" @@ -94,6 +96,7 @@ const ( type overlayOptions struct { imageStores []string + layerStores []additionalLayerStore quota quota.Quota mountProgram string skipMountHome bool @@ -119,6 +122,17 @@ type Driver struct { locker *locker.Locker } +type additionalLayerStore struct { + + // path is the directory where this store is available on the host. + path string + + // withReference is true when the store contains image reference information (base64-encoded) + // in its layer search path so the path to the diff will be + // <path>/base64(reference)/<layerdigest>/ + withReference bool +} + var ( backingFs = "<unknown>" projectQuotaSupported = false @@ -397,6 +411,42 @@ func parseOptions(options []string) (*overlayOptions, error) { } o.imageStores = append(o.imageStores, store) } + case "additionallayerstore": + logrus.Debugf("overlay: additionallayerstore=%s", val) + // Additional read only layer stores to use for lower paths + if val == "" { + continue + } + for _, lstore := range strings.Split(val, ",") { + elems := strings.Split(lstore, ":") + lstore = filepath.Clean(elems[0]) + if !filepath.IsAbs(lstore) { + return nil, fmt.Errorf("overlay: additionallayerstore path %q is not absolute. Can not be relative", lstore) + } + st, err := os.Stat(lstore) + if err != nil { + return nil, errors.Wrap(err, "overlay: can't stat additionallayerstore dir") + } + if !st.IsDir() { + return nil, fmt.Errorf("overlay: additionallayerstore path %q must be a directory", lstore) + } + var withReference bool + for _, e := range elems[1:] { + switch e { + case "ref": + if withReference { + return nil, fmt.Errorf("overlay: additionallayerstore config of %q contains %q option twice", lstore, e) + } + withReference = true + default: + return nil, fmt.Errorf("overlay: additionallayerstore config %q contains unknown option %q", lstore, e) + } + } + o.layerStores = append(o.layerStores, additionalLayerStore{ + path: lstore, + withReference: withReference, + }) + } case "mount_program": logrus.Debugf("overlay: mount_program=%s", val) if val != "" { @@ -550,6 +600,10 @@ func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGI // Check that overlay supports selinux labels as well. flags = label.FormatMountLabel(flags, selinuxLabelTest) } + if unshare.IsRootless() { + flags = fmt.Sprintf("%s,userxattr", flags) + } + if len(flags) < unix.Getpagesize() { err := unix.Mount("overlay", mergedDir, "overlay", 0, flags) if err == nil { @@ -653,6 +707,24 @@ func (d *Driver) Cleanup() error { return mount.Unmount(d.home) } +// LookupAdditionalLayer looks up additional layer store by the specified +// digest and ref and returns an object representing that layer. +// This API is experimental and can be changed without bumping the major version number. +func (d *Driver) LookupAdditionalLayer(dgst digest.Digest, ref string) (graphdriver.AdditionalLayer, error) { + l, err := d.getAdditionalLayerPath(dgst, ref) + if err != nil { + return nil, err + } + // Tell the additional layer store that we use this layer. + // This will increase reference counter on the store's side. + // This will be decreased on Release() method. + notifyUseAdditionalLayer(l) + return &additionalLayer{ + path: l, + d: d, + }, nil +} + // CreateFromTemplate creates a layer with the same contents and parent as another layer. func (d *Driver) CreateFromTemplate(id, template string, templateIDMappings *idtools.IDMappings, parent string, parentIDMappings *idtools.IDMappings, opts *graphdriver.CreateOpts, readWrite bool) error { if readWrite { @@ -955,6 +1027,8 @@ func (d *Driver) Remove(id string) error { } } + d.releaseAdditionalLayerByID(id) + if err := system.EnsureRemoveAll(dir); err != nil && !os.IsNotExist(err) { return err } @@ -1418,7 +1492,10 @@ func (f fileGetNilCloser) Close() error { // DiffGetter returns a FileGetCloser that can read files from the directory that // contains files for the layer differences. Used for direct access for tar-split. func (d *Driver) DiffGetter(id string) (graphdriver.FileGetCloser, error) { - p := d.getDiffPath(id) + p, err := d.getDiffPath(id) + if err != nil { + return nil, err + } return fileGetNilCloser{storage.NewPathFileGetter(p)}, nil } @@ -1440,7 +1517,10 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) idMappings = &idtools.IDMappings{} } - applyDir := d.getDiffPath(id) + applyDir, err := d.getDiffPath(id) + if err != nil { + return 0, err + } logrus.Debugf("Applying tar in %s", applyDir) // Overlay doesn't need the parent id to apply the diff @@ -1458,10 +1538,23 @@ func (d *Driver) ApplyDiff(id, parent string, options graphdriver.ApplyDiffOpts) return directory.Size(applyDir) } -func (d *Driver) getDiffPath(id string) string { +func (d *Driver) getDiffPath(id string) (string, error) { dir := d.dir(id) + return redirectDiffIfAdditionalLayer(path.Join(dir, "diff")) +} - return path.Join(dir, "diff") +func (d *Driver) getLowerDiffPaths(id string) ([]string, error) { + layers, err := d.getLowerDirs(id) + if err != nil { + return nil, err + } + for i, l := range layers { + layers[i], err = redirectDiffIfAdditionalLayer(l) + if err != nil { + return nil, err + } + } + return layers, nil } // DiffSize calculates the changes between the specified id @@ -1472,7 +1565,11 @@ func (d *Driver) DiffSize(id string, idMappings *idtools.IDMappings, parent stri return d.naiveDiff.DiffSize(id, idMappings, parent, parentMappings, mountLabel) } - return directory.Size(d.getDiffPath(id)) + p, err := d.getDiffPath(id) + if err != nil { + return 0, err + } + return directory.Size(p) } // Diff produces an archive of the changes between the specified @@ -1486,12 +1583,15 @@ func (d *Driver) Diff(id string, idMappings *idtools.IDMappings, parent string, idMappings = &idtools.IDMappings{} } - lowerDirs, err := d.getLowerDirs(id) + lowerDirs, err := d.getLowerDiffPaths(id) if err != nil { return nil, err } - diffPath := d.getDiffPath(id) + diffPath, err := d.getDiffPath(id) + if err != nil { + return nil, err + } logrus.Debugf("Tar with options on %s", diffPath) return archive.TarWithOptions(diffPath, &archive.TarOptions{ Compression: archive.Uncompressed, @@ -1510,8 +1610,11 @@ func (d *Driver) Changes(id string, idMappings *idtools.IDMappings, parent strin } // Overlay doesn't have snapshots, so we need to get changes from all parent // layers. - diffPath := d.getDiffPath(id) - layers, err := d.getLowerDirs(id) + diffPath, err := d.getDiffPath(id) + if err != nil { + return nil, err + } + layers, err := d.getLowerDiffPaths(id) if err != nil { return nil, err } @@ -1623,3 +1726,139 @@ func nameWithSuffix(name string, number int) string { } return fmt.Sprintf("%s%d", name, number) } + +func (d *Driver) getAdditionalLayerPath(dgst digest.Digest, ref string) (string, error) { + refElem := base64.StdEncoding.EncodeToString([]byte(ref)) + for _, ls := range d.options.layerStores { + ref := "" + if ls.withReference { + ref = refElem + } + target := path.Join(ls.path, ref, dgst.String()) + // Check if all necessary files exist + for _, p := range []string{ + filepath.Join(target, "diff"), + filepath.Join(target, "info"), + // TODO(ktock): We should have an API to expose the stream data of this layer + // to enable the client to retrieve the entire contents of this + // layer when it exports this layer. + } { + if _, err := os.Stat(p); err != nil { + return "", errors.Wrapf(graphdriver.ErrLayerUnknown, + "failed to stat additional layer %q: %v", p, err) + } + } + return target, nil + } + + return "", errors.Wrapf(graphdriver.ErrLayerUnknown, + "additional layer (%q, %q) not found", dgst, ref) +} + +func (d *Driver) releaseAdditionalLayerByID(id string) { + if al, err := ioutil.ReadFile(path.Join(d.dir(id), "additionallayer")); err == nil { + notifyReleaseAdditionalLayer(string(al)) + } else if !os.IsNotExist(err) { + logrus.Warnf("unexpected error on reading Additional Layer Store pointer %v", err) + } +} + +// additionalLayer represents a layer in Additional Layer Store. +type additionalLayer struct { + path string + d *Driver + releaseOnce sync.Once +} + +// Info returns arbitrary information stored along with this layer (i.e. `info` file). +// This API is experimental and can be changed without bumping the major version number. +func (al *additionalLayer) Info() (io.ReadCloser, error) { + return os.Open(filepath.Join(al.path, "info")) +} + +// CreateAs creates a new layer from this additional layer. +// This API is experimental and can be changed without bumping the major version number. +func (al *additionalLayer) CreateAs(id, parent string) error { + // TODO: support opts + if err := al.d.Create(id, parent, nil); err != nil { + return err + } + dir := al.d.dir(id) + diffDir := path.Join(dir, "diff") + if err := os.RemoveAll(diffDir); err != nil { + return err + } + // tell the additional layer store that we use this layer. + // mark this layer as "additional layer" + if err := ioutil.WriteFile(path.Join(dir, "additionallayer"), []byte(al.path), 0644); err != nil { + return err + } + notifyUseAdditionalLayer(al.path) + return os.Symlink(filepath.Join(al.path, "diff"), diffDir) +} + +// Release tells the additional layer store that we don't use this handler. +// This API is experimental and can be changed without bumping the major version number. +func (al *additionalLayer) Release() { + // Tell the additional layer store that we don't use this layer handler. + // This will decrease the reference counter on the store's side, which was + // increased in LookupAdditionalLayer (so this must be called only once). + al.releaseOnce.Do(func() { + notifyReleaseAdditionalLayer(al.path) + }) +} + +// notifyUseAdditionalLayer notifies Additional Layer Store that we use the specified layer. +// This is done by creating "use" file in the layer directory. This is useful for +// Additional Layer Store to consider when to perform GC. Notification-aware Additional +// Layer Store must return ENOENT. +func notifyUseAdditionalLayer(al string) { + if !path.IsAbs(al) { + logrus.Warnf("additionallayer must be absolute (got: %v)", al) + return + } + useFile := path.Join(al, "use") + f, err := os.Create(useFile) + if os.IsNotExist(err) { + return + } else if err == nil { + f.Close() + if err := os.Remove(useFile); err != nil { + logrus.Warnf("failed to remove use file") + } + } + logrus.Warnf("unexpected error by Additional Layer Store %v during use; GC doesn't seem to be supported", err) +} + +// notifyReleaseAdditionalLayer notifies Additional Layer Store that we don't use the specified +// layer anymore. This is done by rmdir-ing the layer directory. This is useful for +// Additional Layer Store to consider when to perform GC. Notification-aware Additional +// Layer Store must return ENOENT. +func notifyReleaseAdditionalLayer(al string) { + if !path.IsAbs(al) { + logrus.Warnf("additionallayer must be absolute (got: %v)", al) + return + } + // tell the additional layer store that we don't use this layer anymore. + err := unix.Rmdir(al) + if os.IsNotExist(err) { + return + } + logrus.Warnf("unexpected error by Additional Layer Store %v during release; GC doesn't seem to be supported", err) +} + +// redirectDiffIfAdditionalLayer checks if the passed diff path is Additional Layer and +// returns the redirected path. If the passed diff is not the one in Additional Layer +// Store, it returns the original path without changes. +func redirectDiffIfAdditionalLayer(diffPath string) (string, error) { + if ld, err := os.Readlink(diffPath); err == nil { + // diff is the link to Additional Layer Store + if !path.IsAbs(ld) { + return "", fmt.Errorf("linkpath must be absolute (got: %q)", ld) + } + diffPath = ld + } else if err.(*os.PathError).Err != syscall.EINVAL { + return "", err + } + return diffPath, nil +} diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index 1f25390e0..3a455653a 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -5,10 +5,11 @@ module github.com/containers/storage require ( github.com/BurntSushi/toml v0.3.1 github.com/Microsoft/go-winio v0.4.17-0.20210211115548-6eac466e5fa3 - github.com/Microsoft/hcsshim v0.8.15 + github.com/Microsoft/hcsshim v0.8.16 github.com/docker/go-units v0.4.0 + github.com/google/go-intervals v0.0.2 github.com/hashicorp/go-multierror v1.1.1 - github.com/klauspost/compress v1.11.12 + github.com/klauspost/compress v1.11.13 github.com/klauspost/pgzip v1.2.5 github.com/mattn/go-shellwords v1.0.11 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible @@ -26,6 +27,6 @@ require ( github.com/ulikunitz/xz v0.5.10 github.com/vbatts/tar-split v0.11.1 golang.org/x/net v0.0.0-20201224014010-6772e930b67b - golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c + golang.org/x/sys v0.0.0-20210324051608-47abb6519492 gotest.tools v2.2.0+incompatible ) diff --git a/vendor/github.com/containers/storage/go.sum b/vendor/github.com/containers/storage/go.sum index d5f2c41fa..1de8a9825 100644 --- a/vendor/github.com/containers/storage/go.sum +++ b/vendor/github.com/containers/storage/go.sum @@ -49,9 +49,11 @@ github.com/Microsoft/hcsshim v0.8.7-0.20190325164909-8abdbb8205e4/go.mod h1:Op3h github.com/Microsoft/hcsshim v0.8.7/go.mod h1:OHd7sQqRFrYd3RmSgbgji+ctCwkbq2wbEYNSzOYtcBQ= github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg38RRsjT5y8= github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2ow3VK6a9Lg= -github.com/Microsoft/hcsshim v0.8.15 h1:Aof83YILRs2Vx3GhHqlvvfyx1asRJKMFIMeVlHsZKtI= github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= +github.com/Microsoft/hcsshim v0.8.16 h1:8/auA4LFIZFTGrqfKhGBSXwM6/4X1fHa/xniyEHu8ac= +github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= +github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= github.com/PuerkitoBio/purell v1.1.1/go.mod h1:c11w/QuzBsJSee3cPx9rAFu61PvFxuPbtSwDGJws/X0= github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdkoq3SwY9Y+13GIhn11/XLaGBb4BfwItxLd5jeuXE= @@ -90,13 +92,17 @@ github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDk github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= +github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= +github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= github.com/containerd/btrfs v0.0.0-20201111183144-404b9149801e/go.mod h1:jg2QkJcsabfHugurUvvPhS3E08Oxiuh5W/g1ybB4e0E= +github.com/containerd/btrfs v0.0.0-20210316141732-918d888fb676/go.mod h1:zMcX3qkXTAi9GI50+0HOeuV8LU2ryCE/V2vG/ZBiTss= github.com/containerd/cgroups v0.0.0-20190717030353-c4b9ac5c7601/go.mod h1:X9rLEHIqSf/wfK8NsPqxJmeZgW4pcfzdXITDrUSJ6uI= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= github.com/containerd/cgroups v0.0.0-20200710171044-318312a37340/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= -github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102 h1:Qf4HiqfvmB7zS6scsmNgTLmByHbq8n9RTF39v+TzP7A= github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102/go.mod h1:s5q4SojHctfxANBDvMeIaIovkq29IP48TKAxnhYRxvo= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68 h1:hkGVFjz+plgr5UfxZUTPFbUFIF/Km6/s+RVRIRHLrrY= +github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20181022165439-0650fd9eeb50/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/console v0.0.0-20191206165004-02ecf6a7291e/go.mod h1:8Pf4gM6VEbTNRIT26AyyU7hxdQU3MvAvxVI0sc00XBE= @@ -107,22 +113,32 @@ github.com/containerd/containerd v1.3.0/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMX github.com/containerd/containerd v1.3.1-0.20191213020239-082f7e3aed57/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.4.0-beta.2.0.20200729163537-40b22ef07410/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.1/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/containerd v1.4.3/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.5.0-beta.1/go.mod h1:5HfvG1V2FsKesEGQ17k5/T7V960Tmcumvqn8Mc+pCYQ= +github.com/containerd/containerd v1.5.0-beta.3/go.mod h1:/wr9AVtEM7x9c+n0+stptlo/uBBoBORwEx6ardVcmKU= +github.com/containerd/containerd v1.5.0-beta.4/go.mod h1:GmdgZd2zA2GYIBZ0w09ZvgqEq8EfBp/m3lcVZIvPHhI= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20200710164510-efbc4488d8fe/go.mod h1:cECdGN1O8G9bgKTlLhuPJimka6Xb/Gg7vYzCTNVxhvo= github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= +github.com/containerd/continuity v0.0.0-20210208174643-50096c924a4e/go.mod h1:EXlVlkqNba9rJe3j7w3Xa924itAMLgZH4UD/Q4PExuQ= github.com/containerd/fifo v0.0.0-20180307165137-3d5202aec260/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv3Kuk8dQz2ZQXpnv/UZZUHUCL7pnLehbXgQI= github.com/containerd/fifo v0.0.0-20200410184934-f15a3290365b/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= github.com/containerd/fifo v0.0.0-20201026212402-0724c46b320c/go.mod h1:jPQ2IAeZRCYxpS/Cm1495vGFww6ecHmMk1YJH2Q5ln0= +github.com/containerd/fifo v0.0.0-20210316144830-115abcc95a1d/go.mod h1:ocF/ME1SX5b1AOlWi9r677YJmCPSwwWnQ9O123vzpE4= github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZHtSlv++smU= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20200220073739-7016d3ce2328/go.mod h1:PpyHrqVs8FTi9vpyHwPwiNEGaACDxT/N/pLcvMSRA9g= +github.com/containerd/go-runc v0.0.0-20201020171139-16b287bc67d0/go.mod h1:cNU0ZbCgCQVZK4lgG3P+9tn9/PaJNmoDXPpoJhDR+Ok= github.com/containerd/imgcrypt v1.0.1/go.mod h1:mdd8cEPW7TPgNG4FpuP3sGBiQ7Yi/zak9TYCG3juvb0= +github.com/containerd/imgcrypt v1.0.4-0.20210301171431-0ae5c75f59ba/go.mod h1:6TNsg0ctmizkrOgXRNQjAPFWpMYRWuiB6dSF4Pfa5SA= +github.com/containerd/imgcrypt v1.1.1-0.20210312161619-7ed62a527887/go.mod h1:5AZJNI6sLHJljKuI9IHnw1pWqo/F0nGDOuR9zgTs7ow= github.com/containerd/nri v0.0.0-20201007170849-eb1350a75164/go.mod h1:+2wGSDGFYfE5+So4M5syatU0N0f0LbWpuqyMi4/BE8c= +github.com/containerd/nri v0.0.0-20210316161719-dbaa18c31c14/go.mod h1:lmxnXF6oMkbqs39FiCt1s0R2HSMhcLel9vNL3m4AaeY= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20190828172938-92c8520ef9f8/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/ttrpc v0.0.0-20191028202541-4f1b8fe65a5c/go.mod h1:LPm1u0xBw8r8NOKoOdNMeVHSawSsltak+Ihv+etqsE8= @@ -132,10 +148,13 @@ github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kw github.com/containerd/typeurl v0.0.0-20190911142611-5eb25027c9fd/go.mod h1:GeKYzf2pQcqv7tJ0AoCuuhtnqhva5LNU3U+OyKxxJpk= github.com/containerd/typeurl v1.0.1/go.mod h1:TB1hUtrpaiO88KEK56ijojHS1+NeF0izUACaJW2mdXg= github.com/containerd/zfs v0.0.0-20200918131355-0a33824f23a2/go.mod h1:8IgZOBdv8fAgXddBT4dBXJPtxyRsejFIpXoklgxgEjw= +github.com/containerd/zfs v0.0.0-20210301145711-11e8f1707f62/go.mod h1:A9zfAbMlQwE+/is6hi0Xw8ktpL+6glmqZYtevJgaB8Y= +github.com/containerd/zfs v0.0.0-20210315114300-dde8f0fda960/go.mod h1:m+m51S1DvAP6r3FcmYCp54bQ34pyOwTieQDNRIRHsFY= github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= +github.com/containers/ocicrypt v1.1.0/go.mod h1:b8AOe0YR67uU8OqfVNcznfFpAzu3rdgUV4GP9qXPfu4= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -256,8 +275,11 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.2 h1:X2ev0eStA3AbceY54o37/0PQ/UWqKEiiO2dKL5OPaFM= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4 h1:L8R9j+yAqZuZjsqh/z+F1NCffTKKLShY6zXTItVIZ8M= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-intervals v0.0.2 h1:FGrVEiUnTRKR8yE04qzXYaJMtnIYqobR5QbblK3ixcM= +github.com/google/go-intervals v0.0.2/go.mod h1:MkaR3LNRfeKLPmqgJYs4E66z5InYjmCjbbr4TQlcT6Y= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -297,6 +319,7 @@ github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1: github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.10/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= +github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/jmespath/go-jmespath v0.0.0-20160202185014-0b12d6b521d8/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= @@ -314,8 +337,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.12 h1:famVnQVu7QwryBN4jNseQdUKES71ZAOnB6UQQJPZvqk= -github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= +github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -341,6 +364,7 @@ github.com/mattn/go-shellwords v1.0.11 h1:vCoR9VPpsk/TZFW2JwK5I9S0xdrtUq2bph6/Yj github.com/mattn/go-shellwords v1.0.11/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible h1:aKW/4cBs+yK6gpqU3K/oIwk9Q/XICqd3zOX/UFuvqmk= github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible/go.mod h1:8AuVvqP/mXw1px98n46wfvcGfQ4ci2FwoAjKYxuo3Z4= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -429,6 +453,7 @@ github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDa github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= +github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -456,6 +481,7 @@ github.com/spf13/pflag v1.0.1-0.20171106142849-4c012f6dcd95/go.mod h1:DYY7MBk1bd github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980/go.mod h1:AO3tvPzVZ/ayst6UlUKUv6rcPQInYe3IknH3jYhAKu8= github.com/stretchr/objx v0.0.0-20180129172003-8a3f7159479f/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -503,6 +529,7 @@ github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.5/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg= +go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1/go.mod h1:SNgMg+EgDFwmvSmLRTNKC5fegJjB7v23qTQ0XLGUNHk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= @@ -522,6 +549,7 @@ golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200728195943-123391ffb6de/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -599,6 +627,7 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -642,7 +671,9 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200622214017-ed371f2e16b4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200728102440-3e129f6d46b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200817155316-9781c653f443/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200916030750-2334cc1a136f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200922070232-aee5d888a860/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -651,8 +682,9 @@ golang.org/x/sys v0.0.0-20201112073958-5cba982894dd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201202213521-69691e467435/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492 h1:Paq34FxTluEPvVyayQqMPgHm+vTOrIifmcYxFBx9TLg= +golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -759,6 +791,7 @@ google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -785,6 +818,7 @@ gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.2.2/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= +gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -793,11 +827,13 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= +gotest.tools/v3 v3.0.3/go.mod h1:Z7Lb0S5l+klDB31fvDQX8ss/FlKDxtlFlw3Oa8Ymbl8= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/containers/storage/idset.go b/vendor/github.com/containers/storage/idset.go new file mode 100644 index 000000000..f870b9cee --- /dev/null +++ b/vendor/github.com/containers/storage/idset.go @@ -0,0 +1,220 @@ +package storage + +import ( + "github.com/containers/storage/pkg/idtools" + "github.com/google/go-intervals/intervalset" + "github.com/pkg/errors" +) + +// idSet represents a set of integer IDs. It is stored as an ordered set of intervals. +type idSet struct { + set *intervalset.ImmutableSet +} + +func newIDSet(intervals []interval) *idSet { + s := intervalset.Empty() + for _, i := range intervals { + s.Add(intervalset.NewSet([]intervalset.Interval{i})) + } + return &idSet{set: s.ImmutableSet()} +} + +// getHostIDs returns all the host ids in the id map. +func getHostIDs(idMaps []idtools.IDMap) *idSet { + var intervals []interval + for _, m := range idMaps { + intervals = append(intervals, interval{start: m.HostID, end: m.HostID + m.Size}) + } + return newIDSet(intervals) +} + +// getContainerIDs returns all the container ids in the id map. +func getContainerIDs(idMaps []idtools.IDMap) *idSet { + var intervals []interval + for _, m := range idMaps { + intervals = append(intervals, interval{start: m.ContainerID, end: m.ContainerID + m.Size}) + } + return newIDSet(intervals) +} + +// subtract returns the subtraction of `s` and `t`. `s` and `t` are unchanged. +func (s *idSet) subtract(t *idSet) *idSet { + if s == nil || t == nil { + return s + } + return &idSet{set: s.set.Sub(t.set)} +} + +// union returns the union of `s` and `t`. `s` and `t` are unchanged. +func (s *idSet) union(t *idSet) *idSet { + if s == nil { + return t + } else if t == nil { + return s + } + return &idSet{set: s.set.Union(t.set)} +} + +// Methods to iterate over the intervals of the idSet. intervalset doesn't provide one :-( + +// iterator to idSet. Returns nil if iteration finishes. +type iteratorFn func() *interval + +// cancelFn must be called exactly once unless iteratorFn returns nil, otherwise go routine might +// leak. +type cancelFn func() + +func (s *idSet) iterator() (iteratorFn, cancelFn) { + if s == nil { + return func() *interval { return nil }, func() {} + } + cancelCh := make(chan byte) + dataCh := make(chan interval) + go func() { + s.set.Intervals(func(ii intervalset.Interval) bool { + select { + case <-cancelCh: + return false + case dataCh <- ii.(interval): + return true + } + }) + close(dataCh) + }() + iterator := func() *interval { + i, ok := <-dataCh + if !ok { + return nil + } + return &i + } + return iterator, func() { close(cancelCh) } +} + +// size returns the total number of ids in the ID set. +func (s *idSet) size() int { + var size int + iterator, cancel := s.iterator() + defer cancel() + for i := iterator(); i != nil; i = iterator() { + size += i.length() + } + return size +} + +// findAvailable finds the `n` ids from `s`. +func (s *idSet) findAvailable(n int) (*idSet, error) { + var intervals []intervalset.Interval + iterator, cancel := s.iterator() + defer cancel() + for i := iterator(); n > 0 && i != nil; i = iterator() { + i.end = minInt(i.end, i.start+n) + intervals = append(intervals, *i) + n -= i.length() + } + if n > 0 { + return nil, errors.New("could not find enough available IDs") + } + return &idSet{set: intervalset.NewImmutableSet(intervals)}, nil +} + +// zip creates an id map from `s` (host ids) and container ids. +func (s *idSet) zip(container *idSet) []idtools.IDMap { + hostIterator, hostCancel := s.iterator() + defer hostCancel() + containerIterator, containerCancel := container.iterator() + defer containerCancel() + var out []idtools.IDMap + for h, c := hostIterator(), containerIterator(); h != nil && c != nil; { + if n := minInt(h.length(), c.length()); n > 0 { + out = append(out, idtools.IDMap{ + ContainerID: c.start, + HostID: h.start, + Size: n, + }) + h.start += n + c.start += n + } + if h.IsZero() { + h = hostIterator() + } + if c.IsZero() { + c = containerIterator() + } + } + return out +} + +// interval represents an interval of integers [start, end). Note it is allowed to have +// start >= end, in which case it is treated as an empty interval. It implements interface +// intervalset.Interval. +type interval struct { + // Start of the interval (inclusive). + start int + // End of the interval (exclusive). + end int +} + +func (i interval) length() int { + return maxInt(0, i.end-i.start) +} + +func (i interval) Intersect(other intervalset.Interval) intervalset.Interval { + j := other.(interval) + return interval{start: maxInt(i.start, j.start), end: minInt(i.end, j.end)} +} + +func (i interval) Before(other intervalset.Interval) bool { + j := other.(interval) + return !i.IsZero() && !j.IsZero() && i.end < j.start +} + +func (i interval) IsZero() bool { + return i.length() <= 0 +} + +func (i interval) Bisect(other intervalset.Interval) (intervalset.Interval, intervalset.Interval) { + j := other.(interval) + if j.IsZero() { + return i, interval{} + } + // Subtracting [j.start, j.end) is equivalent to the union of intersecting (-inf, j.start) and + // [j.end, +inf). + left := interval{start: i.start, end: minInt(i.end, j.start)} + right := interval{start: maxInt(i.start, j.end), end: i.end} + return left, right +} + +func (i interval) Adjoin(other intervalset.Interval) intervalset.Interval { + j := other.(interval) + if !i.IsZero() && !j.IsZero() && (i.end == j.start || j.end == i.start) { + return interval{start: minInt(i.start, j.start), end: maxInt(i.end, j.end)} + } + return interval{} +} + +func (i interval) Encompass(other intervalset.Interval) intervalset.Interval { + j := other.(interval) + switch { + case i.IsZero(): + return j + case j.IsZero(): + return i + default: + return interval{start: minInt(i.start, j.start), end: maxInt(i.end, j.end)} + } +} + +func minInt(a, b int) int { + if a < b { + return a + } + return b +} + +func maxInt(a, b int) int { + if a < b { + return b + } + return a +} diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index ce059318d..d398a3ff9 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -250,6 +250,11 @@ type LayerStore interface { // LoadLocked wraps Load in a locked state. This means it loads the store // and cleans-up invalid layers if needed. LoadLocked() error + + // PutAdditionalLayer creates a layer using the diff contained in the additional layer + // store. + // This API is experimental and can be changed without bumping the major version number. + PutAdditionalLayer(id string, parentLayer *Layer, names []string, aLayer drivers.AdditionalLayer) (layer *Layer, err error) } type layerStore struct { @@ -610,6 +615,58 @@ func (r *layerStore) Status() ([][2]string, error) { return r.driver.Status(), nil } +func (r *layerStore) PutAdditionalLayer(id string, parentLayer *Layer, names []string, aLayer drivers.AdditionalLayer) (layer *Layer, err error) { + if duplicateLayer, idInUse := r.byid[id]; idInUse { + return duplicateLayer, ErrDuplicateID + } + for _, name := range names { + if _, nameInUse := r.byname[name]; nameInUse { + return nil, ErrDuplicateName + } + } + + parent := "" + if parentLayer != nil { + parent = parentLayer.ID + } + + info, err := aLayer.Info() + if err != nil { + return nil, err + } + defer info.Close() + layer = &Layer{} + if err := json.NewDecoder(info).Decode(layer); err != nil { + return nil, err + } + layer.ID = id + layer.Parent = parent + layer.Created = time.Now().UTC() + + if err := aLayer.CreateAs(id, parent); err != nil { + return nil, err + } + + // TODO: check if necessary fields are filled + r.layers = append(r.layers, layer) + r.idindex.Add(id) + r.byid[id] = layer + for _, name := range names { // names got from the additional layer store won't be used + r.byname[name] = layer + } + if layer.CompressedDigest != "" { + r.bycompressedsum[layer.CompressedDigest] = append(r.bycompressedsum[layer.CompressedDigest], layer.ID) + } + if layer.UncompressedDigest != "" { + r.byuncompressedsum[layer.CompressedDigest] = append(r.byuncompressedsum[layer.CompressedDigest], layer.ID) + } + if err := r.Save(); err != nil { + r.driver.Remove(id) + return nil, err + } + return copyLayer(layer), nil +} + func (r *layerStore) Put(id string, parentLayer *Layer, names []string, mountLabel string, options map[string]string, moreOptions *LayerOptions, writeable bool, flags map[string]interface{}, diff io.Reader) (layer *Layer, size int64, err error) { if !r.IsReadWrite() { return nil, -1, errors.Wrapf(ErrStoreIsReadOnly, "not allowed to create new layers at %q", r.layerspath()) diff --git a/vendor/github.com/containers/storage/pkg/archive/archive.go b/vendor/github.com/containers/storage/pkg/archive/archive.go index 1d21471eb..aa6689593 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive.go @@ -20,6 +20,7 @@ import ( "github.com/containers/storage/pkg/pools" "github.com/containers/storage/pkg/promise" "github.com/containers/storage/pkg/system" + "github.com/containers/storage/pkg/unshare" gzip "github.com/klauspost/pgzip" rsystem "github.com/opencontainers/runc/libcontainer/system" "github.com/pkg/errors" @@ -1489,3 +1490,14 @@ func TarPath(uidmap []idtools.IDMap, gidmap []idtools.IDMap) func(path string) ( }) } } + +// GetOverlayXattrName returns the xattr used by the overlay driver with the +// given name. +// It uses the trusted.overlay prefix when running as root, and user.overlay +// in rootless mode. +func GetOverlayXattrName(name string) string { + if unshare.IsRootless() { + return fmt.Sprintf("user.overlay.%s", name) + } + return fmt.Sprintf("trusted.overlay.%s", name) +} diff --git a/vendor/github.com/containers/storage/pkg/archive/archive_linux.go b/vendor/github.com/containers/storage/pkg/archive/archive_linux.go index 4efd5d3d9..f5c69d1c2 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive_linux.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive_linux.go @@ -12,6 +12,10 @@ import ( "golang.org/x/sys/unix" ) +func getOverlayOpaqueXattrName() string { + return GetOverlayXattrName("opaque") +} + func GetWhiteoutConverter(format WhiteoutFormat, data interface{}) TarWhiteoutConverter { if format == OverlayWhiteoutFormat { if rolayers, ok := data.([]string); ok && len(rolayers) > 0 { @@ -39,13 +43,13 @@ func (o overlayWhiteoutConverter) ConvertWrite(hdr *tar.Header, path string, fi if fi.Mode()&os.ModeDir != 0 { // convert opaque dirs to AUFS format by writing an empty file with the whiteout prefix - opaque, err := system.Lgetxattr(path, "trusted.overlay.opaque") + opaque, err := system.Lgetxattr(path, getOverlayOpaqueXattrName()) if err != nil { return nil, err } if len(opaque) == 1 && opaque[0] == 'y' { if hdr.Xattrs != nil { - delete(hdr.Xattrs, "trusted.overlay.opaque") + delete(hdr.Xattrs, getOverlayOpaqueXattrName()) } // If there are no lower layers, then it can't have been deleted in this layer. if len(o.rolayers) == 0 { @@ -114,7 +118,7 @@ func (overlayWhiteoutConverter) ConvertReadWithHandler(hdr *tar.Header, path str // if a directory is marked as opaque by the AUFS special file, we need to translate that to overlay if base == WhiteoutOpaqueDir { - err := handler.Setxattr(dir, "trusted.overlay.opaque", []byte{'y'}) + err := handler.Setxattr(dir, getOverlayOpaqueXattrName(), []byte{'y'}) // don't write the file itself return false, err } diff --git a/vendor/github.com/containers/storage/pkg/archive/changes_linux.go b/vendor/github.com/containers/storage/pkg/archive/changes_linux.go index ea1dae052..a3addebe6 100644 --- a/vendor/github.com/containers/storage/pkg/archive/changes_linux.go +++ b/vendor/github.com/containers/storage/pkg/archive/changes_linux.go @@ -349,7 +349,7 @@ func overlayDeletedFile(layers []string, root, path string, fi os.FileInfo) (str return "", nil } // If the directory isn't marked as opaque, then it's just a normal directory. - opaque, err := system.Lgetxattr(filepath.Join(root, path), "trusted.overlay.opaque") + opaque, err := system.Lgetxattr(filepath.Join(root, path), getOverlayOpaqueXattrName()) if err != nil { return "", err } diff --git a/vendor/github.com/containers/storage/pkg/config/config.go b/vendor/github.com/containers/storage/pkg/config/config.go index 7c9ac6ad6..2d2470722 100644 --- a/vendor/github.com/containers/storage/pkg/config/config.go +++ b/vendor/github.com/containers/storage/pkg/config/config.go @@ -122,6 +122,13 @@ type OptionsConfig struct { // for shared image content AdditionalImageStores []string `toml:"additionalimagestores"` + // AdditionalLayerStores is the location of additional read/only + // Layer stores. Usually used to access Networked File System + // for shared image content + // This API is experimental and can be changed without bumping the + // major version number. + AdditionalLayerStores []string `toml:"additionallayerstores"` + // Size Size string `toml:"size"` diff --git a/vendor/github.com/containers/storage/storage.conf b/vendor/github.com/containers/storage/storage.conf index 3ff708e20..e70f4f018 100644 --- a/vendor/github.com/containers/storage/storage.conf +++ b/vendor/github.com/containers/storage/storage.conf @@ -5,7 +5,7 @@ [storage] # Default Storage Driver, Must be set for proper operation. -driver = "" +driver = "overlay" # Temporary storage location runroot = "/run/containers/storage" diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 9c08eda69..feb931133 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -2,6 +2,7 @@ package storage import ( "encoding/base64" + "encoding/json" "fmt" "io" "io/ioutil" @@ -489,6 +490,30 @@ type Store interface { // GetDigestLock returns digest-specific Locker. GetDigestLock(digest.Digest) (Locker, error) + + // LayerFromAdditionalLayerStore searches layers from the additional layer store and + // returns the object for handling this. Note that this hasn't been stored to this store + // yet so this needs to be done through PutAs method. + // Releasing AdditionalLayer handler is caller's responsibility. + // This API is experimental and can be changed without bumping the major version number. + LookupAdditionalLayer(d digest.Digest, imageref string) (AdditionalLayer, error) +} + +// AdditionalLayer reprents a layer that is contained in the additional layer store +// This API is experimental and can be changed without bumping the major version number. +type AdditionalLayer interface { + // PutAs creates layer based on this handler, using diff contents from the additional + // layer store. + PutAs(id, parent string, names []string) (*Layer, error) + + // UncompressedDigest returns the uncompressed digest of this layer + UncompressedDigest() digest.Digest + + // CompressedSize returns the compressed size of this layer + CompressedSize() int64 + + // Release tells the additional layer store that we don't use this handler. + Release() } type AutoUserNsOptions = types.AutoUserNsOptions @@ -541,8 +566,8 @@ type store struct { uidMap []idtools.IDMap gidMap []idtools.IDMap autoUsernsUser string - autoUIDMap []idtools.IDMap // Set by getAvailableMappings() - autoGIDMap []idtools.IDMap // Set by getAvailableMappings() + additionalUIDs *idSet // Set by getAvailableIDs() + additionalGIDs *idSet // Set by getAvailableIDs() autoNsMinSize uint32 autoNsMaxSize uint32 graphDriver drivers.Driver @@ -648,8 +673,8 @@ func GetStore(options types.StoreOptions) (Store, error) { autoUsernsUser: options.RootAutoNsUser, autoNsMinSize: autoNsMinSize, autoNsMaxSize: autoNsMaxSize, - autoUIDMap: nil, - autoGIDMap: nil, + additionalUIDs: nil, + additionalGIDs: nil, usernsLock: usernsLock, } if err := s.load(); err != nil { @@ -3134,6 +3159,91 @@ func (s *store) Layer(id string) (*Layer, error) { return nil, ErrLayerUnknown } +func (s *store) LookupAdditionalLayer(d digest.Digest, imageref string) (AdditionalLayer, error) { + adriver, ok := s.graphDriver.(drivers.AdditionalLayerStoreDriver) + if !ok { + return nil, ErrLayerUnknown + } + + al, err := adriver.LookupAdditionalLayer(d, imageref) + if err != nil { + if errors.Is(err, drivers.ErrLayerUnknown) { + return nil, ErrLayerUnknown + } + return nil, err + } + info, err := al.Info() + if err != nil { + return nil, err + } + defer info.Close() + var layer Layer + if err := json.NewDecoder(info).Decode(&layer); err != nil { + return nil, err + } + return &additionalLayer{&layer, al, s}, nil +} + +type additionalLayer struct { + layer *Layer + handler drivers.AdditionalLayer + s *store +} + +func (al *additionalLayer) UncompressedDigest() digest.Digest { + return al.layer.UncompressedDigest +} + +func (al *additionalLayer) CompressedSize() int64 { + return al.layer.CompressedSize +} + +func (al *additionalLayer) PutAs(id, parent string, names []string) (*Layer, error) { + rlstore, err := al.s.LayerStore() + if err != nil { + return nil, err + } + rlstore.Lock() + defer rlstore.Unlock() + if modified, err := rlstore.Modified(); modified || err != nil { + if err = rlstore.Load(); err != nil { + return nil, err + } + } + rlstores, err := al.s.ROLayerStores() + if err != nil { + return nil, err + } + + var parentLayer *Layer + if parent != "" { + for _, lstore := range append([]ROLayerStore{rlstore}, rlstores...) { + if lstore != rlstore { + lstore.RLock() + defer lstore.Unlock() + if modified, err := lstore.Modified(); modified || err != nil { + if err = lstore.Load(); err != nil { + return nil, err + } + } + } + parentLayer, err = lstore.Get(parent) + if err == nil { + break + } + } + if parentLayer == nil { + return nil, ErrLayerUnknown + } + } + + return rlstore.PutAdditionalLayer(id, parentLayer, names, al.handler) +} + +func (al *additionalLayer) Release() { + al.handler.Release() +} + func (s *store) Image(id string) (*Image, error) { istore, err := s.ImageStore() if err != nil { diff --git a/vendor/github.com/containers/storage/types/options.go b/vendor/github.com/containers/storage/types/options.go index 007c5288b..fb80b18c5 100644 --- a/vendor/github.com/containers/storage/types/options.go +++ b/vendor/github.com/containers/storage/types/options.go @@ -278,6 +278,9 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) { fmt.Printf("Failed to parse %s %v\n", configFile, err.Error()) return } + + // Clear storeOptions of previos settings + *storeOptions = StoreOptions{} if config.Storage.Driver != "" { storeOptions.GraphDriverName = config.Storage.Driver } @@ -300,6 +303,9 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) { for _, s := range config.Storage.Options.AdditionalImageStores { storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.imagestore=%s", config.Storage.Driver, s)) } + for _, s := range config.Storage.Options.AdditionalLayerStores { + storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.additionallayerstore=%s", config.Storage.Driver, s)) + } if config.Storage.Options.Size != "" { storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.size=%s", config.Storage.Driver, config.Storage.Options.Size)) } @@ -338,13 +344,13 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) { if err != nil { fmt.Print(err) } else { - storeOptions.UIDMap = append(storeOptions.UIDMap, uidmap...) + storeOptions.UIDMap = uidmap } gidmap, err := idtools.ParseIDMap([]string{config.Storage.Options.RemapGIDs}, "remap-gids") if err != nil { fmt.Print(err) } else { - storeOptions.GIDMap = append(storeOptions.GIDMap, gidmap...) + storeOptions.GIDMap = gidmap } storeOptions.RootAutoNsUser = config.Storage.Options.RootAutoUsernsUser if config.Storage.Options.AutoUsernsMinSize > 0 { @@ -356,8 +362,8 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) { storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, cfg.GetGraphDriverOptions(storeOptions.GraphDriverName, config.Storage.Options)...) - if os.Getenv("STORAGE_OPTS") != "" { - storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, strings.Split(os.Getenv("STORAGE_OPTS"), ",")...) + if opts, ok := os.LookupEnv("STORAGE_OPTS"); ok { + storeOptions.GraphDriverOptions = strings.Split(opts, ",") } if len(storeOptions.GraphDriverOptions) == 1 && storeOptions.GraphDriverOptions[0] == "" { storeOptions.GraphDriverOptions = nil diff --git a/vendor/github.com/containers/storage/userns.go b/vendor/github.com/containers/storage/userns.go index 3b0f24fcd..3ada41f73 100644 --- a/vendor/github.com/containers/storage/userns.go +++ b/vendor/github.com/containers/storage/userns.go @@ -21,8 +21,8 @@ import ( // possible to use an arbitrary entry in /etc/sub*id. // Differently, if the username is not specified for root users, a // default name is used. -func getAdditionalSubIDs(username string) ([]idtools.IDMap, []idtools.IDMap, error) { - var uids, gids []idtools.IDMap +func getAdditionalSubIDs(username string) (*idSet, *idSet, error) { + var uids, gids *idSet if unshare.IsRootless() { username = os.Getenv("USER") @@ -45,48 +45,36 @@ func getAdditionalSubIDs(username string) ([]idtools.IDMap, []idtools.IDMap, err if err != nil { logrus.Errorf("cannot find mappings for user %q: %v", username, err) } else { - uids = mappings.UIDs() - gids = mappings.GIDs() + uids = getHostIDs(mappings.UIDs()) + gids = getHostIDs(mappings.GIDs()) } return uids, gids, nil } -// getAvailableMappings returns the list of ranges that are usable by the current user. +// getAvailableIDs returns the list of ranges that are usable by the current user. // When running as root, it looks up the additional IDs assigned to the specified user. // When running as rootless, the mappings assigned to the unprivileged user are converted // to the IDs inside of the initial rootless user namespace. -func (s *store) getAvailableMappings() ([]idtools.IDMap, []idtools.IDMap, error) { - if s.autoUIDMap == nil { +func (s *store) getAvailableIDs() (*idSet, *idSet, error) { + if s.additionalUIDs == nil { uids, gids, err := getAdditionalSubIDs(s.autoUsernsUser) if err != nil { return nil, nil, err } // Store the result so we don't need to look it up again next time - s.autoUIDMap, s.autoGIDMap = uids, gids + s.additionalUIDs, s.additionalGIDs = uids, gids } - uids := s.autoUIDMap - gids := s.autoGIDMap - if !unshare.IsRootless() { // No mapping to inner namespace needed - return copyIDMap(uids), copyIDMap(gids), nil + return s.additionalUIDs, s.additionalGIDs, nil } // We are already inside of the rootless user namespace. // We need to remap the configured mappings to what is available // inside of the rootless userns. - totaluid := 0 - totalgid := 0 - for _, u := range uids { - totaluid += u.Size - } - for _, g := range gids { - totalgid += g.Size - } - - u := []idtools.IDMap{{ContainerID: 0, HostID: 1, Size: totaluid}} - g := []idtools.IDMap{{ContainerID: 0, HostID: 1, Size: totalgid}} + u := newIDSet([]interval{{start: 1, end: s.additionalUIDs.size() + 1}}) + g := newIDSet([]interval{{start: 1, end: s.additionalGIDs.size() + 1}}) return u, g, nil } @@ -222,141 +210,6 @@ outer: return size, nil } -func minInt(a, b int) int { - if a < b { - return a - } - return b -} - -func maxInt(a, b int) int { - if a < b { - return b - } - return a -} - -// subtractHostIDs return the subtraction of the range USED from AVAIL. The range is specified -// by [HostID, HostID+Size). -// ContainerID is ignored. -func subtractHostIDs(avail idtools.IDMap, used idtools.IDMap) []idtools.IDMap { - var out []idtools.IDMap - availEnd := avail.HostID + avail.Size - usedEnd := used.HostID + used.Size - // Intersection of [avail.HostID, availEnd) and (-inf, used.HostID) is [avail.HostID, newEnd). - if newEnd := minInt(availEnd, used.HostID); newEnd > avail.HostID { - out = append(out, idtools.IDMap{ - ContainerID: avail.ContainerID, - HostID: avail.HostID, - Size: newEnd - avail.HostID, - }) - } - // Intersection of [avail.HostID, availEnd) and [usedEnd, +inf) is [newStart, availEnd). - if newStart := maxInt(avail.HostID, usedEnd); newStart < availEnd { - out = append(out, idtools.IDMap{ - ContainerID: newStart + avail.ContainerID - avail.HostID, - HostID: newStart, - Size: availEnd - newStart, - }) - } - return out -} - -// subtractContainerIDs return the subtraction of the range USED from AVAIL. The range is specified -// by [ContainerID, ContainerID+Size). -// HostID is ignored. -func subtractContainerIDs(avail idtools.IDMap, used idtools.IDMap) []idtools.IDMap { - var out []idtools.IDMap - availEnd := avail.ContainerID + avail.Size - usedEnd := used.ContainerID + used.Size - // Intersection of [avail.ContainerID, availEnd) and (-inf, used.ContainerID) is - // [avail.ContainerID, newEnd). - if newEnd := minInt(availEnd, used.ContainerID); newEnd > avail.ContainerID { - out = append(out, idtools.IDMap{ - ContainerID: avail.ContainerID, - HostID: avail.HostID, - Size: newEnd - avail.ContainerID, - }) - } - // Intersection of [avail.ContainerID, availEnd) and [usedEnd, +inf) is [newStart, availEnd). - if newStart := maxInt(avail.ContainerID, usedEnd); newStart < availEnd { - out = append(out, idtools.IDMap{ - ContainerID: newStart, - HostID: newStart + avail.HostID - avail.ContainerID, - Size: availEnd - newStart, - }) - } - return out -} - -// subtractAll subtracts all usedIDs from the available IDs. -func subtractAll(availableIDs, usedIDs []idtools.IDMap, host bool) []idtools.IDMap { - for _, u := range usedIDs { - var newAvailableIDs []idtools.IDMap - for _, cur := range availableIDs { - var newRanges []idtools.IDMap - if host { - newRanges = subtractHostIDs(cur, u) - } else { - newRanges = subtractContainerIDs(cur, u) - } - newAvailableIDs = append(newAvailableIDs, newRanges...) - } - availableIDs = newAvailableIDs - } - return availableIDs -} - -// findAvailableIDRange returns the list of IDs that are not used by existing containers. -// This function is used to lookup both UIDs and GIDs. -func findAvailableIDRange(size uint32, availableIDs, usedIDs []idtools.IDMap) ([]idtools.IDMap, error) { - var avail []idtools.IDMap - - // ContainerID will be adjusted later. - for _, i := range availableIDs { - n := idtools.IDMap{ - ContainerID: 0, - HostID: i.HostID, - Size: i.Size, - } - avail = append(avail, n) - } - avail = subtractAll(avail, usedIDs, true) - - currentID := 0 - remaining := size - // We know the size for each intervals, let's adjust the ContainerID for each - // of them. - for i := 0; i < len(avail); i++ { - avail[i].ContainerID = currentID - if uint32(avail[i].Size) >= remaining { - avail[i].Size = int(remaining) - return avail[:i+1], nil - } - remaining -= uint32(avail[i].Size) - currentID += avail[i].Size - } - - return nil, errors.New("could not find enough available IDs") -} - -// findAvailableRange returns both the list of UIDs and GIDs ranges that are not -// currently used by other containers. -// It is a wrapper for findAvailableIDRange. -func findAvailableRange(sizeUID, sizeGID uint32, availableUIDs, availableGIDs, usedUIDs, usedGIDs []idtools.IDMap) ([]idtools.IDMap, []idtools.IDMap, error) { - UIDMap, err := findAvailableIDRange(sizeUID, availableUIDs, usedUIDs) - if err != nil { - return nil, nil, err - } - - GIDMap, err := findAvailableIDRange(sizeGID, availableGIDs, usedGIDs) - if err != nil { - return nil, nil, err - } - - return UIDMap, GIDMap, nil -} - // getAutoUserNS creates an automatic user namespace func (s *store) getAutoUserNS(id string, options *types.AutoUserNsOptions, image *Image) ([]idtools.IDMap, []idtools.IDMap, error) { requestedSize := uint32(0) @@ -368,7 +221,7 @@ func (s *store) getAutoUserNS(id string, options *types.AutoUserNsOptions, image initialSize = options.InitialSize } - availableUIDs, availableGIDs, err := s.getAvailableMappings() + availableUIDs, availableGIDs, err := s.getAvailableIDs() if err != nil { return nil, nil, errors.Wrapf(err, "cannot read mappings") } @@ -409,22 +262,41 @@ func (s *store) getAutoUserNS(id string, options *types.AutoUserNsOptions, image return nil, nil, errors.Errorf("the container needs a user namespace with size %q that is bigger than the maximum value allowed with userns=auto %q", size, s.autoNsMaxSize) } } + + return getAutoUserNSIDMappings( + int(size), + availableUIDs, availableGIDs, + usedUIDs, usedGIDs, + options.AdditionalUIDMappings, options.AdditionalGIDMappings, + ) +} + +// getAutoUserNSIDMappings computes the user/group id mappings for the automatic user namespace. +func getAutoUserNSIDMappings( + size int, + availableUIDs, availableGIDs *idSet, + usedUIDMappings, usedGIDMappings, additionalUIDMappings, additionalGIDMappings []idtools.IDMap, +) ([]idtools.IDMap, []idtools.IDMap, error) { + usedUIDs := getHostIDs(append(usedUIDMappings, additionalUIDMappings...)) + usedGIDs := getHostIDs(append(usedGIDMappings, additionalGIDMappings...)) + + // Exclude additional uids and gids from requested range. + targetIDs := newIDSet([]interval{{start: 0, end: size}}) + requestedContainerUIDs := targetIDs.subtract(getContainerIDs(additionalUIDMappings)) + requestedContainerGIDs := targetIDs.subtract(getContainerIDs(additionalGIDMappings)) + // Make sure the specified additional IDs are not used as part of the automatic // mapping - usedUIDs = append(usedUIDs, options.AdditionalUIDMappings...) - usedGIDs = append(usedGIDs, options.AdditionalGIDMappings...) - availableUIDs, availableGIDs, err = findAvailableRange(size, size, availableUIDs, availableGIDs, usedUIDs, usedGIDs) + availableUIDs, err := availableUIDs.subtract(usedUIDs).findAvailable(requestedContainerUIDs.size()) if err != nil { return nil, nil, err } - - // We need to make sure the specified container IDs are also dropped from the automatic - // namespaces we have found. - if len(options.AdditionalUIDMappings) > 0 { - availableUIDs = subtractAll(availableUIDs, options.AdditionalUIDMappings, false) - } - if len(options.AdditionalGIDMappings) > 0 { - availableGIDs = subtractAll(availableGIDs, options.AdditionalGIDMappings, false) + availableGIDs, err = availableGIDs.subtract(usedGIDs).findAvailable(requestedContainerGIDs.size()) + if err != nil { + return nil, nil, err } - return append(availableUIDs, options.AdditionalUIDMappings...), append(availableGIDs, options.AdditionalGIDMappings...), nil + + uidMap := append(availableUIDs.zip(requestedContainerUIDs), additionalUIDMappings...) + gidMap := append(availableGIDs.zip(requestedContainerGIDs), additionalGIDMappings...) + return uidMap, gidMap, nil } diff --git a/vendor/github.com/google/go-intervals/LICENSE b/vendor/github.com/google/go-intervals/LICENSE new file mode 100644 index 000000000..d64569567 --- /dev/null +++ b/vendor/github.com/google/go-intervals/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/google/go-intervals/intervalset/intervalset.go b/vendor/github.com/google/go-intervals/intervalset/intervalset.go new file mode 100644 index 000000000..6a33db63c --- /dev/null +++ b/vendor/github.com/google/go-intervals/intervalset/intervalset.go @@ -0,0 +1,545 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package intervalset provides an abtraction for dealing with sets of +// 1-dimensional spans, such as sets of time ranges. The Set type provides set +// arithmetic and enumeration methods based on an Interval interface. +// +// DISCLAIMER: This library is not yet stable, so expect breaking changes. +package intervalset + +import ( + "fmt" + "sort" + "strings" +) + +// Interval is the interface for a continuous or discrete span. The interval is +// assumed to be inclusive of the starting point and exclusive of the ending +// point. +// +// All methods in the interface are non-destructive: Calls to the methods should +// not modify the interval. Furthermore, the implementation assumes an interval +// will not be mutated by user code, either. +type Interval interface { + // Intersect returns the intersection of an interval with another + // interval. The function may panic if the other interval is incompatible. + Intersect(Interval) Interval + + // Before returns true if the interval is completely before another interval. + Before(Interval) bool + + // IsZero returns true for the zero value of an interval. + IsZero() bool + + // Bisect returns two intervals, one on the lower side of x and one on the + // upper side of x, corresponding to the subtraction of x from the original + // interval. The returned intervals are always within the range of the + // original interval. + Bisect(x Interval) (Interval, Interval) + + // Adjoin returns the union of two intervals, if the intervals are exactly + // adjacent, or the zero interval if they are not. + Adjoin(Interval) Interval + + // Encompass returns an interval that covers the exact extents of two + // intervals. + Encompass(Interval) Interval +} + +// Set is a set of interval objects used for +type Set struct { + //non-overlapping intervals + intervals []Interval + // factory is needed when the extents of the empty set are needed. + factory intervalFactory +} + +// SetInput is an interface implemented by Set and ImmutableSet. It is used when +// one of these types type must take a set as an argument. +type SetInput interface { + // Extent returns the Interval defined by the minimum and maximum values of + // the set. + Extent() Interval + + // IntervalsBetween iterates over the intervals within extents set and calls f + // with each. If f returns false, iteration ceases. + // + // Any interval within the set that overlaps partially with extents is truncated + // before being passed to f. + IntervalsBetween(extents Interval, f IntervalReceiver) +} + +// NewSet returns a new set given a sorted slice of intervals. This function +// panics if the intervals are not sorted. +func NewSet(intervals []Interval) *Set { + return NewSetV1(intervals, oldBehaviorFactory.makeZero) +} + +// NewSetV1 returns a new set given a sorted slice of intervals. This function +// panics if the intervals are not sorted. +// +// NewSetV1 will be renamed and will replace NewSet in the v1 release. +func NewSetV1(intervals []Interval, makeZero func() Interval) *Set { + if err := CheckSorted(intervals); err != nil { + panic(err) + } + return &Set{intervals, makeIntervalFactor(makeZero)} +} + +// CheckSorted checks that interval[i+1] is not before interval[i] for all +// relevant elements of the input slice. Nil is returned when len(intervals) is +// 0 or 1. +func CheckSorted(intervals []Interval) error { + for i := 0; i < len(intervals)-1; i++ { + if !intervals[i].Before(intervals[i+1]) { + return fmt.Errorf("!intervals[%d].Before(intervals[%d]) for %s, %s", i, i+1, intervals[i], intervals[i+1]) + } + } + return nil +} + +// Empty returns a new, empty set of intervals. +func Empty() *Set { + return EmptyV1(oldBehaviorFactory.makeZero) +} + +// EmptyV1 returns a new, empty set of intervals using the semantics of the V1 +// API, which will require a factory method for construction of an empty interval. +func EmptyV1(makeZero func() Interval) *Set { + return &Set{nil, makeIntervalFactor(makeZero)} +} + +// Copy returns a copy of a set that may be mutated without affecting the original. +func (s *Set) Copy() *Set { + return &Set{append([]Interval(nil), s.intervals...), s.factory} +} + +// String returns a human-friendly representation of the set. +func (s *Set) String() string { + var strs []string + for _, x := range s.intervals { + strs = append(strs, fmt.Sprintf("%s", x)) + } + return fmt.Sprintf("{%s}", strings.Join(strs, ", ")) +} + +// Extent returns the Interval defined by the minimum and maximum values of the +// set. +func (s *Set) Extent() Interval { + if len(s.intervals) == 0 { + return s.factory.makeZero() + } + return s.intervals[0].Encompass(s.intervals[len(s.intervals)-1]) +} + +// Add adds all the elements of another set to this set. +func (s *Set) Add(b SetInput) { + // Deal with nil extent. See https://github.com/google/go-intervals/issues/6. + bExtent := b.Extent() + if bExtent == nil { + return // no changes needed + } + + // Loop through the intervals of x + b.IntervalsBetween(bExtent, func(x Interval) bool { + s.insert(x) + return true + }) +} + +// Contains reports whether an interval is entirely contained by the set. +func (s *Set) Contains(ival Interval) bool { + // Loop through the intervals of x + next := s.iterator(ival, true) + for setInterval := next(); setInterval != nil; setInterval = next() { + left, right := ival.Bisect(setInterval) + if !left.IsZero() { + return false + } + ival = right + } + return ival.IsZero() +} + +// adjoinOrAppend adds an interval to the end of intervals unless that value +// directly adjoins the last element of intervals, in which case the last +// element will be replaced by the adjoined interval. +func adjoinOrAppend(intervals []Interval, x Interval) []Interval { + lastIndex := len(intervals) - 1 + if lastIndex == -1 { + return append(intervals, x) + } + adjoined := intervals[lastIndex].Adjoin(x) + if adjoined.IsZero() { + return append(intervals, x) + } + intervals[lastIndex] = adjoined + return intervals +} + +func (s *Set) insert(insertion Interval) { + if s.Contains(insertion) { + return + } + // TODO(reddaly): Something like Java's ArrayList would allow both O(log(n)) + // insertion and O(log(n)) lookup. For now, we have O(log(n)) lookup and O(n) + // insertion. + var newIntervals []Interval + push := func(x Interval) { + newIntervals = adjoinOrAppend(newIntervals, x) + } + inserted := false + for _, x := range s.intervals { + if inserted { + push(x) + continue + } + if insertion.Before(x) { + push(insertion) + push(x) + inserted = true + continue + } + // [===left===)[==x===)[===right===) + left, right := insertion.Bisect(x) + if !left.IsZero() { + push(left) + } + push(x) + // Replace the interval being inserted with the remaining portion of the + // interval to be inserted. + if right.IsZero() { + inserted = true + } else { + insertion = right + } + } + if !inserted { + push(insertion) + } + s.intervals = newIntervals +} + +// Sub destructively modifies the set by subtracting b. +func (s *Set) Sub(b SetInput) { + extent := s.Extent() + // Deal with nil extent. See https://github.com/google/go-intervals/issues/6. + if extent == nil { + // Set is already empty, no changes necessary. + return + } + var newIntervals []Interval + push := func(x Interval) { + newIntervals = adjoinOrAppend(newIntervals, x) + } + nextX := s.iterator(extent, true) + nextY, cancel := setIntervalIterator(b, extent) + defer cancel() + + x := nextX() + y := nextY() + for x != nil { + // If y == nil, all of the remaining intervals in A are to the right of B, + // so just yield them. + if y == nil { + push(x) + x = nextX() + continue + } + // Split x into parts left and right of y. + // The diagrams below show the bisection results for various situations. + // if left.IsZero() && !right.IsZero() + // xxx + // y1y1 y2y2 y3 y4y4 + // xxx + // or + // xxxxxxxxxxxx + // y1y1 y2y2 y3 y4y4 + // + // if !left.IsZero() && !right.IsZero() + // x1x1x1x1x1 + // y1 y2 + // + // if left.IsZero() && right.IsZero() + // x1x1x1x1 x2x2x2 + // y1y1y1y1y1y1y1 + // + // if !left.IsZero() && right.IsZero() + // x1x1 x2 + // y1y1y1y1 + left, right := x.Bisect(y) + + // If the left side of x is non-zero, it can definitely be pushed to the + // resulting interval set since no subsequent y value will intersect it. + // The sequences look something like + // x1x1x1x1x1 OR x1x1x1 x2 + // y1 y2 y1y1y1 + // left = x1x1 x1x1x1 + // right = x1x1 {zero} + if !left.IsZero() { + push(left) + } + + if !right.IsZero() { + // If the right side of x is non-zero: + // 1) Right is the remaining portion of x that needs to be pushed. + x = right + // 2) It's not possible for current y to intersect it, so advance y. It's + // possible nextY() will intersect it, so don't push yet. + y = nextY() + } else { + // There's nothing left of x to push, so advance x. + x = nextX() + } + } + + // Setting s.intervals is the only side effect in this function. + s.intervals = newIntervals +} + +// intersectionIterator returns a function that yields intervals that are +// members of the intersection of s and b, in increasing order. +func (s *Set) intersectionIterator(b SetInput) (iter func() Interval, cancel func()) { + return intervalMapperToIterator(func(f IntervalReceiver) { + sExtent, bExtent := s.Extent(), b.Extent() + // Deal with nil extent. See https://github.com/google/go-intervals/issues/6. + if sExtent == nil || bExtent == nil { + // IF either set is already empty, the intersection is empty. This + // voids a panic below where a valid Interval is needed for each + // extent. + return + } + nextX := s.iterator(bExtent, true) + nextY, cancel := setIntervalIterator(b, sExtent) + defer cancel() + + x := nextX() + y := nextY() + // Loop through corresponding intervals of S and B. + // If y == nil, all of the remaining intervals in S are to the right of B. + // If x == nil, all of the remaining intervals in B are to the right of S. + for x != nil && y != nil { + if x.Before(y) { + x = nextX() + continue + } + if y.Before(x) { + y = nextY() + continue + } + xyIntersect := x.Intersect(y) + if !xyIntersect.IsZero() { + if !f(xyIntersect) { + return + } + _, right := x.Bisect(y) + if !right.IsZero() { + x = right + } else { + x = nextX() + } + } + } + }) +} + +// Intersect destructively modifies the set by intersectin it with b. +func (s *Set) Intersect(b SetInput) { + iter, cancel := s.intersectionIterator(b) + defer cancel() + var newIntervals []Interval + for x := iter(); x != nil; x = iter() { + newIntervals = append(newIntervals, x) + } + s.intervals = newIntervals +} + +// searchLow returns the first index in s.intervals that is not before x. +func (s *Set) searchLow(x Interval) int { + return sort.Search(len(s.intervals), func(i int) bool { + return !s.intervals[i].Before(x) + }) +} + +// searchLow returns the index of the first interval in s.intervals that is +// entirely after x. +func (s *Set) searchHigh(x Interval) int { + return sort.Search(len(s.intervals), func(i int) bool { + return x.Before(s.intervals[i]) + }) +} + +// iterator returns a function that yields elements of the set in order. +// +// The function returned will return nil when finished iterating. +func (s *Set) iterator(extents Interval, forward bool) func() Interval { + low, high := s.searchLow(extents), s.searchHigh(extents) + + i, stride := low, 1 + if !forward { + i, stride = high-1, -1 + } + + return func() Interval { + if i < 0 || i >= len(s.intervals) { + return nil + } + x := s.intervals[i] + i += stride + return x + } +} + +// IntervalReceiver is a function used for iterating over a set of intervals. It +// takes the start and end times and returns true if the iteration should +// continue. +type IntervalReceiver func(Interval) bool + +// IntervalsBetween iterates over the intervals within extents set and calls f +// with each. If f returns false, iteration ceases. +// +// Any interval within the set that overlaps partially with extents is truncated +// before being passed to f. +func (s *Set) IntervalsBetween(extents Interval, f IntervalReceiver) { + // Begin = first index in s.intervals that is not before extents. + begin := sort.Search(len(s.intervals), func(i int) bool { + return !s.intervals[i].Before(extents) + }) + + // TODO(reddaly): Optimize this by performing a binary search for the ending + // point. + for _, interval := range s.intervals[begin:] { + // If the interval is after the extents, there will be no more overlap, so + // break out of the loop. + if extents.Before(interval) { + break + } + portionOfInterval := extents.Intersect(interval) + if portionOfInterval.IsZero() { + continue + } + + if !f(portionOfInterval) { + return + } + } +} + +// Intervals iterates over all the intervals within the set and calls f with +// each one. If f returns false, iteration ceases. +func (s *Set) Intervals(f IntervalReceiver) { + for _, interval := range s.intervals { + if !f(interval) { + return + } + } +} + +// AllIntervals returns an ordered slice of all the intervals in the set. +func (s *Set) AllIntervals() []Interval { + return append(make([]Interval, 0, len(s.intervals)), s.intervals...) +} + +// ImmutableSet returns an immutable copy of this set. +func (s *Set) ImmutableSet() *ImmutableSet { + return NewImmutableSet(s.AllIntervals()) +} + +// mapFn reports true if an iteration should continue. It is called on values of +// a collection. +type mapFn func(interface{}) bool + +// mapFn calls mapFn for each member of a collection. +type mapperFn func(mapFn) + +// iteratorFn returns the next item in an iteration or the zero value. The +// second return value indicates whether the first return value is a member of +// the collection. +type iteratorFn func() (interface{}, bool) + +// generatorFn returns an iterator. +type generatorFn func() iteratorFn + +// cancelFn should be called to clean up the goroutine that would otherwise leak. +type cancelFn func() + +// mapperToIterator returns an iteratorFn version of a mappingFn. The second +// return value must be called at the end of iteration, or the underlying +// goroutine will leak. +func mapperToIterator(m mapperFn) (iteratorFn, cancelFn) { + generatedValues := make(chan interface{}, 1) + stopCh := make(chan interface{}, 1) + go func() { + m(func(obj interface{}) bool { + select { + case <-stopCh: + return false + case generatedValues <- obj: + return true + } + }) + close(generatedValues) + }() + iter := func() (interface{}, bool) { + value, ok := <-generatedValues + return value, ok + } + return iter, func() { + stopCh <- nil + } +} + +func intervalMapperToIterator(mapper func(IntervalReceiver)) (iter func() Interval, cancel func()) { + genericMapper := func(m mapFn) { + mapper(func(ival Interval) bool { + return m(ival) + }) + } + + genericIter, cancel := mapperToIterator(genericMapper) + return func() Interval { + genericVal, iterationEnded := genericIter() + if !iterationEnded { + return nil + } + ival, ok := genericVal.(Interval) + if !ok { + panic("unexpected value type, internal error") + } + return ival + }, cancel +} + +func setIntervalIterator(s SetInput, extent Interval) (iter func() Interval, cancel func()) { + return intervalMapperToIterator(func(f IntervalReceiver) { + s.IntervalsBetween(extent, f) + }) +} + +// oldBehaviorFactory returns a nil interval. This was used before +// construction of a Set/ImmutableSet required passing in a factory method for +// creating a zero interval object. +var oldBehaviorFactory = makeIntervalFactor(func() Interval { return nil }) + +// intervalFactory is used to construct a zero-value interval. The zero value +// interval may be different for different types of intervals, so a factory is +// sometimes needed to write generic algorithms about intervals. +type intervalFactory struct { + makeZero func() Interval +} + +func makeIntervalFactor(makeZero func() Interval) intervalFactory { + return intervalFactory{makeZero} +} diff --git a/vendor/github.com/google/go-intervals/intervalset/intervalset_immutable.go b/vendor/github.com/google/go-intervals/intervalset/intervalset_immutable.go new file mode 100644 index 000000000..f6d5cbb52 --- /dev/null +++ b/vendor/github.com/google/go-intervals/intervalset/intervalset_immutable.go @@ -0,0 +1,85 @@ +// Copyright 2017 Google Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package intervalset + +// ImmutableSet is a set of interval objects. It provides various set theory +// operations. +type ImmutableSet struct { + set *Set +} + +// NewImmutableSet returns a new set given a sorted slice of intervals. This +// function panics if the intervals are not sorted. +func NewImmutableSet(intervals []Interval) *ImmutableSet { + return NewImmutableSetV1(intervals, oldBehaviorFactory.makeZero) +} + +// NewImmutableSetV1 returns a new set given a sorted slice of intervals. This +// function panics if the intervals are not sorted. +func NewImmutableSetV1(intervals []Interval, makeZero func() Interval) *ImmutableSet { + return &ImmutableSet{NewSetV1(intervals, makeZero)} +} + +// String returns a human-friendly representation of the set. +func (s *ImmutableSet) String() string { + return s.set.String() +} + +// Extent returns the Interval defined by the minimum and maximum values of the +// set. +func (s *ImmutableSet) Extent() Interval { + return s.set.Extent() +} + +// Contains reports whether an interval is entirely contained by the set. +func (s *ImmutableSet) Contains(ival Interval) bool { + return s.set.Contains(ival) +} + +// Union returns a set with the contents of this set and another set. +func (s *ImmutableSet) Union(b SetInput) *ImmutableSet { + union := s.set.Copy() + union.Add(b) + return &ImmutableSet{union} +} + +// Sub returns a set without the intervals of another set. +func (s *ImmutableSet) Sub(b SetInput) *ImmutableSet { + x := s.set.Copy() + x.Sub(b) + return &ImmutableSet{x} +} + +// Intersect returns the intersection of two sets. +func (s *ImmutableSet) Intersect(b SetInput) *ImmutableSet { + x := s.set.Copy() + x.Intersect(b) + return &ImmutableSet{x} +} + +// IntervalsBetween iterates over the intervals within extents set and calls f +// with each. If f returns false, iteration ceases. +// +// Any interval within the set that overlaps partially with extents is truncated +// before being passed to f. +func (s *ImmutableSet) IntervalsBetween(extents Interval, f IntervalReceiver) { + s.set.IntervalsBetween(extents, f) +} + +// Intervals iterates over all the intervals within the set and calls f with +// each one. If f returns false, iteration ceases. +func (s *ImmutableSet) Intervals(f IntervalReceiver) { + s.set.Intervals(f) +} diff --git a/vendor/github.com/klauspost/compress/flate/deflate.go b/vendor/github.com/klauspost/compress/flate/deflate.go index 25dbe3e15..3f428d8b5 100644 --- a/vendor/github.com/klauspost/compress/flate/deflate.go +++ b/vendor/github.com/klauspost/compress/flate/deflate.go @@ -645,15 +645,15 @@ func (d *compressor) init(w io.Writer, level int) (err error) { d.fill = (*compressor).fillBlock d.step = (*compressor).store case level == ConstantCompression: - d.w.logNewTablePenalty = 4 - d.window = make([]byte, maxStoreBlockSize) + d.w.logNewTablePenalty = 8 + d.window = make([]byte, 32<<10) d.fill = (*compressor).fillBlock d.step = (*compressor).storeHuff case level == DefaultCompression: level = 5 fallthrough case level >= 1 && level <= 6: - d.w.logNewTablePenalty = 6 + d.w.logNewTablePenalty = 8 d.fast = newFastEnc(level) d.window = make([]byte, maxStoreBlockSize) d.fill = (*compressor).fillBlock diff --git a/vendor/github.com/klauspost/compress/flate/fast_encoder.go b/vendor/github.com/klauspost/compress/flate/fast_encoder.go index 4a73e1bdd..678f08105 100644 --- a/vendor/github.com/klauspost/compress/flate/fast_encoder.go +++ b/vendor/github.com/klauspost/compress/flate/fast_encoder.go @@ -6,6 +6,7 @@ package flate import ( + "encoding/binary" "fmt" "math/bits" ) @@ -65,26 +66,15 @@ func load32(b []byte, i int) uint32 { } func load64(b []byte, i int) uint64 { - // Help the compiler eliminate bounds checks on the read so it can be done in a single read. - b = b[i:] - b = b[:8] - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + return binary.LittleEndian.Uint64(b[i:]) } func load3232(b []byte, i int32) uint32 { - // Help the compiler eliminate bounds checks on the read so it can be done in a single read. - b = b[i:] - b = b[:4] - return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 + return binary.LittleEndian.Uint32(b[i:]) } func load6432(b []byte, i int32) uint64 { - // Help the compiler eliminate bounds checks on the read so it can be done in a single read. - b = b[i:] - b = b[:8] - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + return binary.LittleEndian.Uint64(b[i:]) } func hash(u uint32) uint32 { @@ -225,9 +215,9 @@ func (e *fastGen) Reset() { func matchLen(a, b []byte) int { b = b[:len(a)] var checked int - if len(a) > 4 { + if len(a) >= 4 { // Try 4 bytes first - if diff := load32(a, 0) ^ load32(b, 0); diff != 0 { + if diff := binary.LittleEndian.Uint32(a) ^ binary.LittleEndian.Uint32(b); diff != 0 { return bits.TrailingZeros32(diff) >> 3 } // Switch to 8 byte matching. @@ -236,7 +226,7 @@ func matchLen(a, b []byte) int { b = b[4:] for len(a) >= 8 { b = b[:len(a)] - if diff := load64(a, 0) ^ load64(b, 0); diff != 0 { + if diff := binary.LittleEndian.Uint64(a) ^ binary.LittleEndian.Uint64(b); diff != 0 { return checked + (bits.TrailingZeros64(diff) >> 3) } checked += 8 @@ -247,7 +237,7 @@ func matchLen(a, b []byte) int { b = b[:len(a)] for i := range a { if a[i] != b[i] { - return int(i) + checked + return i + checked } } return len(a) + checked diff --git a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go index 208d66711..db54be139 100644 --- a/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go +++ b/vendor/github.com/klauspost/compress/flate/huffman_bit_writer.go @@ -5,6 +5,7 @@ package flate import ( + "encoding/binary" "io" ) @@ -206,7 +207,7 @@ func (w *huffmanBitWriter) write(b []byte) { } func (w *huffmanBitWriter) writeBits(b int32, nb uint16) { - w.bits |= uint64(b) << (w.nbits & reg16SizeMask64) + w.bits |= uint64(b) << w.nbits w.nbits += nb if w.nbits >= 48 { w.writeOutBits() @@ -420,13 +421,11 @@ func (w *huffmanBitWriter) writeOutBits() { w.bits >>= 48 w.nbits -= 48 n := w.nbytes - w.bytes[n] = byte(bits) - w.bytes[n+1] = byte(bits >> 8) - w.bytes[n+2] = byte(bits >> 16) - w.bytes[n+3] = byte(bits >> 24) - w.bytes[n+4] = byte(bits >> 32) - w.bytes[n+5] = byte(bits >> 40) + + // We over-write, but faster... + binary.LittleEndian.PutUint64(w.bytes[n:], bits) n += 6 + if n >= bufferFlushSize { if w.err != nil { n = 0 @@ -435,6 +434,7 @@ func (w *huffmanBitWriter) writeOutBits() { w.write(w.bytes[:n]) n = 0 } + w.nbytes = n } @@ -759,7 +759,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) } else { // inlined c := lengths[lengthCode&31] - w.bits |= uint64(c.code) << (w.nbits & reg16SizeMask64) + w.bits |= uint64(c.code) << w.nbits w.nbits += c.len if w.nbits >= 48 { w.writeOutBits() @@ -779,7 +779,7 @@ func (w *huffmanBitWriter) writeTokens(tokens []token, leCodes, oeCodes []hcode) } else { // inlined c := offs[offsetCode&31] - w.bits |= uint64(c.code) << (w.nbits & reg16SizeMask64) + w.bits |= uint64(c.code) << w.nbits w.nbits += c.len if w.nbits >= 48 { w.writeOutBits() @@ -830,8 +830,8 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { // Assume header is around 70 bytes: // https://stackoverflow.com/a/25454430 const guessHeaderSizeBits = 70 * 8 - estBits, estExtra := histogramSize(input, w.literalFreq[:], !eof && !sync) - estBits += w.lastHeader + 15 + estBits := histogramSize(input, w.literalFreq[:], !eof && !sync) + estBits += w.lastHeader + len(input)/32 if w.lastHeader == 0 { estBits += guessHeaderSizeBits } @@ -845,9 +845,9 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { return } + reuseSize := 0 if w.lastHeader > 0 { - reuseSize := w.literalEncoding.bitLength(w.literalFreq[:256]) - estBits += estExtra + reuseSize = w.literalEncoding.bitLength(w.literalFreq[:256]) if estBits < reuseSize { // We owe an EOB @@ -859,6 +859,10 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { const numLiterals = endBlockMarker + 1 const numOffsets = 1 if w.lastHeader == 0 { + if !eof && !sync { + // Generate a slightly suboptimal tree that can be used for all. + fillHist(w.literalFreq[:numLiterals]) + } w.literalFreq[endBlockMarker] = 1 w.literalEncoding.generate(w.literalFreq[:numLiterals], 15) @@ -878,19 +882,14 @@ func (w *huffmanBitWriter) writeBlockHuff(eof bool, input []byte, sync bool) { for _, t := range input { // Bitwriting inlined, ~30% speedup c := encoding[t] - w.bits |= uint64(c.code) << ((w.nbits) & reg16SizeMask64) + w.bits |= uint64(c.code) << w.nbits w.nbits += c.len if w.nbits >= 48 { bits := w.bits w.bits >>= 48 w.nbits -= 48 n := w.nbytes - w.bytes[n] = byte(bits) - w.bytes[n+1] = byte(bits >> 8) - w.bytes[n+2] = byte(bits >> 16) - w.bytes[n+3] = byte(bits >> 24) - w.bytes[n+4] = byte(bits >> 32) - w.bytes[n+5] = byte(bits >> 40) + binary.LittleEndian.PutUint64(w.bytes[n:], bits) n += 6 if n >= bufferFlushSize { if w.err != nil { diff --git a/vendor/github.com/klauspost/compress/flate/huffman_code.go b/vendor/github.com/klauspost/compress/flate/huffman_code.go index 4c39a3018..0d3445a1c 100644 --- a/vendor/github.com/klauspost/compress/flate/huffman_code.go +++ b/vendor/github.com/klauspost/compress/flate/huffman_code.go @@ -122,6 +122,16 @@ func (h *huffmanEncoder) bitLength(freq []uint16) int { return total } +func (h *huffmanEncoder) bitLengthRaw(b []byte) int { + var total int + for _, f := range b { + if f != 0 { + total += int(h.codes[f].len) + } + } + return total +} + // Return the number of literals assigned to each bit size in the Huffman encoding // // This method is only called when list.length >= 3 @@ -327,37 +337,40 @@ func atLeastOne(v float32) float32 { return v } +// Unassigned values are assigned '1' in the histogram. +func fillHist(b []uint16) { + for i, v := range b { + if v == 0 { + b[i] = 1 + } + } +} + // histogramSize accumulates a histogram of b in h. // An estimated size in bits is returned. -// Unassigned values are assigned '1' in the histogram. // len(h) must be >= 256, and h's elements must be all zeroes. -func histogramSize(b []byte, h []uint16, fill bool) (int, int) { +func histogramSize(b []byte, h []uint16, fill bool) (bits int) { h = h[:256] for _, t := range b { h[t]++ } - invTotal := 1.0 / float32(len(b)) - shannon := float32(0.0) - var extra float32 + total := len(b) if fill { - oneBits := atLeastOne(-mFastLog2(invTotal)) - for i, v := range h[:] { - if v > 0 { - n := float32(v) - shannon += atLeastOne(-mFastLog2(n*invTotal)) * n - } else { - h[i] = 1 - extra += oneBits + for _, v := range h { + if v == 0 { + total++ } } - } else { - for _, v := range h[:] { - if v > 0 { - n := float32(v) - shannon += atLeastOne(-mFastLog2(n*invTotal)) * n - } + } + + invTotal := 1.0 / float32(total) + shannon := float32(0.0) + for _, v := range h { + if v > 0 { + n := float32(v) + shannon += atLeastOne(-mFastLog2(n*invTotal)) * n } } - return int(shannon + 0.99), int(extra + 0.99) + return int(shannon + 0.99) } diff --git a/vendor/github.com/klauspost/compress/flate/level2.go b/vendor/github.com/klauspost/compress/flate/level2.go index 5b986a194..234c4389a 100644 --- a/vendor/github.com/klauspost/compress/flate/level2.go +++ b/vendor/github.com/klauspost/compress/flate/level2.go @@ -155,7 +155,7 @@ func (e *fastEncL2) Encode(dst *tokens, src []byte) { // Store every second hash in-between, but offset by 1. for i := s - l + 2; i < s-5; i += 7 { - x := load6432(src, int32(i)) + x := load6432(src, i) nextHash := hash4u(uint32(x), bTableBits) e.table[nextHash] = tableEntry{offset: e.cur + i} // Skip one diff --git a/vendor/github.com/klauspost/compress/fse/compress.go b/vendor/github.com/klauspost/compress/fse/compress.go index b69237c9b..0d31f5ebc 100644 --- a/vendor/github.com/klauspost/compress/fse/compress.go +++ b/vendor/github.com/klauspost/compress/fse/compress.go @@ -301,7 +301,7 @@ func (s *Scratch) writeCount() error { out[outP+1] = byte(bitStream >> 8) outP += (bitCount + 7) / 8 - if uint16(charnum) > s.symbolLen { + if charnum > s.symbolLen { return errors.New("internal error: charnum > s.symbolLen") } s.Out = out[:outP] @@ -331,7 +331,7 @@ type cTable struct { func (s *Scratch) allocCtable() { tableSize := 1 << s.actualTableLog // get tableSymbol that is big enough. - if cap(s.ct.tableSymbol) < int(tableSize) { + if cap(s.ct.tableSymbol) < tableSize { s.ct.tableSymbol = make([]byte, tableSize) } s.ct.tableSymbol = s.ct.tableSymbol[:tableSize] @@ -565,8 +565,8 @@ func (s *Scratch) normalizeCount2() error { distributed uint32 total = uint32(s.br.remain()) tableLog = s.actualTableLog - lowThreshold = uint32(total >> tableLog) - lowOne = uint32((total * 3) >> (tableLog + 1)) + lowThreshold = total >> tableLog + lowOne = (total * 3) >> (tableLog + 1) ) for i, cnt := range s.count[:s.symbolLen] { if cnt == 0 { @@ -591,7 +591,7 @@ func (s *Scratch) normalizeCount2() error { if (total / toDistribute) > lowOne { // risk of rounding to zero - lowOne = uint32((total * 3) / (toDistribute * 2)) + lowOne = (total * 3) / (toDistribute * 2) for i, cnt := range s.count[:s.symbolLen] { if (s.norm[i] == notYetAssigned) && (cnt <= lowOne) { s.norm[i] = 1 diff --git a/vendor/github.com/klauspost/compress/fse/decompress.go b/vendor/github.com/klauspost/compress/fse/decompress.go index 413ec3b3c..926f5f153 100644 --- a/vendor/github.com/klauspost/compress/fse/decompress.go +++ b/vendor/github.com/klauspost/compress/fse/decompress.go @@ -172,7 +172,7 @@ type decSymbol struct { // allocDtable will allocate decoding tables if they are not big enough. func (s *Scratch) allocDtable() { tableSize := 1 << s.actualTableLog - if cap(s.decTable) < int(tableSize) { + if cap(s.decTable) < tableSize { s.decTable = make([]decSymbol, tableSize) } s.decTable = s.decTable[:tableSize] @@ -340,7 +340,7 @@ type decoder struct { func (d *decoder) init(in *bitReader, dt []decSymbol, tableLog uint8) { d.dt = dt d.br = in - d.state = uint16(in.getBits(tableLog)) + d.state = in.getBits(tableLog) } // next returns the next symbol and sets the next state. diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go index f9ed5f830..dea115b33 100644 --- a/vendor/github.com/klauspost/compress/huff0/compress.go +++ b/vendor/github.com/klauspost/compress/huff0/compress.go @@ -403,7 +403,7 @@ func (s *Scratch) buildCTable() error { var startNode = int16(s.symbolLen) nonNullRank := s.symbolLen - 1 - nodeNb := int16(startNode) + nodeNb := startNode huffNode := s.nodes[1 : huffNodesLen+1] // This overlays the slice above, but allows "-1" index lookups. @@ -580,7 +580,7 @@ func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { // Get pos of last (smallest) symbol per rank { - currentNbBits := uint8(maxNbBits) + currentNbBits := maxNbBits for pos := int(n); pos >= 0; pos-- { if huffNode[pos].nbBits >= currentNbBits { continue diff --git a/vendor/github.com/klauspost/compress/snappy/snappy.go b/vendor/github.com/klauspost/compress/snappy/snappy.go index 74a36689e..ea58ced88 100644 --- a/vendor/github.com/klauspost/compress/snappy/snappy.go +++ b/vendor/github.com/klauspost/compress/snappy/snappy.go @@ -94,5 +94,5 @@ var crcTable = crc32.MakeTable(crc32.Castagnoli) // https://github.com/google/snappy/blob/master/framing_format.txt func crc(b []byte) uint32 { c := crc32.Update(0, crcTable, b) - return uint32(c>>15|c<<17) + 0xa282ead8 + return c>>15 | c<<17 + 0xa282ead8 } diff --git a/vendor/github.com/klauspost/compress/zstd/blockenc.go b/vendor/github.com/klauspost/compress/zstd/blockenc.go index c85c40255..9647c64e5 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockenc.go +++ b/vendor/github.com/klauspost/compress/zstd/blockenc.go @@ -22,28 +22,44 @@ type blockEnc struct { dictLitEnc *huff0.Scratch wr bitWriter - extraLits int - last bool - + extraLits int output []byte recentOffsets [3]uint32 prevRecentOffsets [3]uint32 + + last bool + lowMem bool } // init should be used once the block has been created. // If called more than once, the effect is the same as calling reset. func (b *blockEnc) init() { - if cap(b.literals) < maxCompressedLiteralSize { - b.literals = make([]byte, 0, maxCompressedLiteralSize) - } - const defSeqs = 200 - b.literals = b.literals[:0] - if cap(b.sequences) < defSeqs { - b.sequences = make([]seq, 0, defSeqs) - } - if cap(b.output) < maxCompressedBlockSize { - b.output = make([]byte, 0, maxCompressedBlockSize) + if b.lowMem { + // 1K literals + if cap(b.literals) < 1<<10 { + b.literals = make([]byte, 0, 1<<10) + } + const defSeqs = 20 + if cap(b.sequences) < defSeqs { + b.sequences = make([]seq, 0, defSeqs) + } + // 1K + if cap(b.output) < 1<<10 { + b.output = make([]byte, 0, 1<<10) + } + } else { + if cap(b.literals) < maxCompressedBlockSize { + b.literals = make([]byte, 0, maxCompressedBlockSize) + } + const defSeqs = 200 + if cap(b.sequences) < defSeqs { + b.sequences = make([]seq, 0, defSeqs) + } + if cap(b.output) < maxCompressedBlockSize { + b.output = make([]byte, 0, maxCompressedBlockSize) + } } + if b.coders.mlEnc == nil { b.coders.mlEnc = &fseEncoder{} b.coders.mlPrev = &fseEncoder{} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go index b1b7c6e6a..2d4d893eb 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_base.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go @@ -7,6 +7,10 @@ import ( "github.com/klauspost/compress/zstd/internal/xxhash" ) +const ( + dictShardBits = 6 +) + type fastBase struct { // cur is the offset at the start of hist cur int32 @@ -17,6 +21,7 @@ type fastBase struct { tmp [8]byte blk *blockEnc lastDictID uint32 + lowMem bool } // CRC returns the underlying CRC writer. @@ -57,15 +62,10 @@ func (e *fastBase) addBlock(src []byte) int32 { // check if we have space already if len(e.hist)+len(src) > cap(e.hist) { if cap(e.hist) == 0 { - l := e.maxMatchOff * 2 - // Make it at least 1MB. - if l < 1<<20 { - l = 1 << 20 - } - e.hist = make([]byte, 0, l) + e.ensureHist(len(src)) } else { - if cap(e.hist) < int(e.maxMatchOff*2) { - panic("unexpected buffer size") + if cap(e.hist) < int(e.maxMatchOff+maxCompressedBlockSize) { + panic(fmt.Errorf("unexpected buffer cap %d, want at least %d with window %d", cap(e.hist), e.maxMatchOff+maxCompressedBlockSize, e.maxMatchOff)) } // Move down offset := int32(len(e.hist)) - e.maxMatchOff @@ -79,6 +79,28 @@ func (e *fastBase) addBlock(src []byte) int32 { return s } +// ensureHist will ensure that history can keep at least this many bytes. +func (e *fastBase) ensureHist(n int) { + if cap(e.hist) >= n { + return + } + l := e.maxMatchOff + if (e.lowMem && e.maxMatchOff > maxCompressedBlockSize) || e.maxMatchOff <= maxCompressedBlockSize { + l += maxCompressedBlockSize + } else { + l += e.maxMatchOff + } + // Make it at least 1MB. + if l < 1<<20 && !e.lowMem { + l = 1 << 20 + } + // Make it at least the requested size. + if l < int32(n) { + l = int32(n) + } + e.hist = make([]byte, 0, l) +} + // useBlock will replace the block with the provided one, // but transfer recent offsets from the previous. func (e *fastBase) UseBlock(enc *blockEnc) { @@ -117,7 +139,7 @@ func (e *fastBase) matchlen(s, t int32, src []byte) int32 { // Reset the encoding table. func (e *fastBase) resetBase(d *dict, singleBlock bool) { if e.blk == nil { - e.blk = &blockEnc{} + e.blk = &blockEnc{lowMem: e.lowMem} e.blk.init() } else { e.blk.reset(nil) diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go index bb71d1eea..fe3625c5f 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_best.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go @@ -407,6 +407,7 @@ encodeLoop: // Most notable difference is that src will not be copied for history and // we do not need to check for max match length. func (e *bestFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { + e.ensureHist(len(src)) e.Encode(blk, src) } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index 94a5343d0..c2ce4a2ba 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -16,6 +16,12 @@ const ( // This greatly depends on the type of input. betterShortTableBits = 13 // Bits used in the short match table betterShortTableSize = 1 << betterShortTableBits // Size of the table + + betterLongTableShardCnt = 1 << (betterLongTableBits - dictShardBits) // Number of shards in the table + betterLongTableShardSize = betterLongTableSize / betterLongTableShardCnt // Size of an individual shard + + betterShortTableShardCnt = 1 << (betterShortTableBits - dictShardBits) // Number of shards in the table + betterShortTableShardSize = betterShortTableSize / betterShortTableShardCnt // Size of an individual shard ) type prevEntry struct { @@ -31,10 +37,17 @@ type prevEntry struct { // and that it is longer (lazy matching). type betterFastEncoder struct { fastBase - table [betterShortTableSize]tableEntry - longTable [betterLongTableSize]prevEntry - dictTable []tableEntry - dictLongTable []prevEntry + table [betterShortTableSize]tableEntry + longTable [betterLongTableSize]prevEntry +} + +type betterFastEncoderDict struct { + betterFastEncoder + dictTable []tableEntry + dictLongTable []prevEntry + shortTableShardDirty [betterShortTableShardCnt]bool + longTableShardDirty [betterLongTableShardCnt]bool + allDirty bool } // Encode improves compression... @@ -516,12 +529,512 @@ encodeLoop: // Most notable difference is that src will not be copied for history and // we do not need to check for max match length. func (e *betterFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { + e.ensureHist(len(src)) e.Encode(blk, src) } +// Encode improves compression... +func (e *betterFastEncoderDict) Encode(blk *blockEnc, src []byte) { + const ( + // Input margin is the number of bytes we read (8) + // and the maximum we will read ahead (2) + inputMargin = 8 + 2 + minNonLiteralBlockSize = 16 + ) + + // Protect against e.cur wraparound. + for e.cur >= bufferReset { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + for i := range e.longTable[:] { + e.longTable[i] = prevEntry{} + } + e.cur = e.maxMatchOff + e.allDirty = true + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff + for i := range e.table[:] { + v := e.table[i].offset + if v < minOff { + v = 0 + } else { + v = v - e.cur + e.maxMatchOff + } + e.table[i].offset = v + } + for i := range e.longTable[:] { + v := e.longTable[i].offset + v2 := e.longTable[i].prev + if v < minOff { + v = 0 + v2 = 0 + } else { + v = v - e.cur + e.maxMatchOff + if v2 < minOff { + v2 = 0 + } else { + v2 = v2 - e.cur + e.maxMatchOff + } + } + e.longTable[i] = prevEntry{ + offset: v, + prev: v2, + } + } + e.allDirty = true + e.cur = e.maxMatchOff + break + } + + s := e.addBlock(src) + blk.size = len(src) + if len(src) < minNonLiteralBlockSize { + blk.extraLits = len(src) + blk.literals = blk.literals[:len(src)] + copy(blk.literals, src) + return + } + + // Override src + src = e.hist + sLimit := int32(len(src)) - inputMargin + // stepSize is the number of bytes to skip on every main loop iteration. + // It should be >= 1. + const stepSize = 1 + + const kSearchStrength = 9 + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := s + cv := load6432(src, s) + + // Relative offsets + offset1 := int32(blk.recentOffsets[0]) + offset2 := int32(blk.recentOffsets[1]) + + addLiterals := func(s *seq, until int32) { + if until == nextEmit { + return + } + blk.literals = append(blk.literals, src[nextEmit:until]...) + s.litLen = uint32(until - nextEmit) + } + if debug { + println("recent offsets:", blk.recentOffsets) + } + +encodeLoop: + for { + var t int32 + // We allow the encoder to optionally turn off repeat offsets across blocks + canRepeat := len(blk.sequences) > 2 + var matched int32 + + for { + if debugAsserts && canRepeat && offset1 == 0 { + panic("offset0 was 0") + } + + nextHashS := hash5(cv, betterShortTableBits) + nextHashL := hash8(cv, betterLongTableBits) + candidateL := e.longTable[nextHashL] + candidateS := e.table[nextHashS] + + const repOff = 1 + repIndex := s - offset1 + repOff + off := s + e.cur + e.longTable[nextHashL] = prevEntry{offset: off, prev: candidateL.offset} + e.markLongShardDirty(nextHashL) + e.table[nextHashS] = tableEntry{offset: off, val: uint32(cv)} + e.markShortShardDirty(nextHashS) + + if canRepeat { + if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { + // Consider history as well. + var seq seq + lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + + seq.matchLen = uint32(lenght - zstdMinMatch) + + // We might be able to match backwards. + // Extend as long as we can. + start := s + repOff + // We end the search early, so we don't risk 0 literals + // and have to do special offset treatment. + startLimit := nextEmit + 1 + + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { + repIndex-- + start-- + seq.matchLen++ + } + addLiterals(&seq, start) + + // rep 0 + seq.offset = 1 + if debugSequences { + println("repeat sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + // Index match start+1 (long) -> s - 1 + index0 := s + repOff + s += lenght + repOff + + nextEmit = s + if s >= sLimit { + if debug { + println("repeat ended", s, lenght) + + } + break encodeLoop + } + // Index skipped... + for index0 < s-1 { + cv0 := load6432(src, index0) + cv1 := cv0 >> 8 + h0 := hash8(cv0, betterLongTableBits) + off := index0 + e.cur + e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} + e.markLongShardDirty(h0) + h1 := hash5(cv1, betterShortTableBits) + e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} + e.markShortShardDirty(h1) + index0 += 2 + } + cv = load6432(src, s) + continue + } + const repOff2 = 1 + + // We deviate from the reference encoder and also check offset 2. + // Still slower and not much better, so disabled. + // repIndex = s - offset2 + repOff2 + if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) { + // Consider history as well. + var seq seq + lenght := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) + + seq.matchLen = uint32(lenght - zstdMinMatch) + + // We might be able to match backwards. + // Extend as long as we can. + start := s + repOff2 + // We end the search early, so we don't risk 0 literals + // and have to do special offset treatment. + startLimit := nextEmit + 1 + + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { + repIndex-- + start-- + seq.matchLen++ + } + addLiterals(&seq, start) + + // rep 2 + seq.offset = 2 + if debugSequences { + println("repeat sequence 2", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + index0 := s + repOff2 + s += lenght + repOff2 + nextEmit = s + if s >= sLimit { + if debug { + println("repeat ended", s, lenght) + + } + break encodeLoop + } + + // Index skipped... + for index0 < s-1 { + cv0 := load6432(src, index0) + cv1 := cv0 >> 8 + h0 := hash8(cv0, betterLongTableBits) + off := index0 + e.cur + e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} + e.markLongShardDirty(h0) + h1 := hash5(cv1, betterShortTableBits) + e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} + e.markShortShardDirty(h1) + index0 += 2 + } + cv = load6432(src, s) + // Swap offsets + offset1, offset2 = offset2, offset1 + continue + } + } + // Find the offsets of our two matches. + coffsetL := candidateL.offset - e.cur + coffsetLP := candidateL.prev - e.cur + + // Check if we have a long match. + if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { + // Found a long match, at least 8 bytes. + matched = e.matchlen(s+8, coffsetL+8, src) + 8 + t = coffsetL + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + if debugAsserts && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debugMatches { + println("long match") + } + + if s-coffsetLP < e.maxMatchOff && cv == load6432(src, coffsetLP) { + // Found a long match, at least 8 bytes. + prevMatch := e.matchlen(s+8, coffsetLP+8, src) + 8 + if prevMatch > matched { + matched = prevMatch + t = coffsetLP + } + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + if debugAsserts && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debugMatches { + println("long match") + } + } + break + } + + // Check if we have a long match on prev. + if s-coffsetLP < e.maxMatchOff && cv == load6432(src, coffsetLP) { + // Found a long match, at least 8 bytes. + matched = e.matchlen(s+8, coffsetLP+8, src) + 8 + t = coffsetLP + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + if debugAsserts && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debugMatches { + println("long match") + } + break + } + + coffsetS := candidateS.offset - e.cur + + // Check if we have a short match. + if s-coffsetS < e.maxMatchOff && uint32(cv) == candidateS.val { + // found a regular match + matched = e.matchlen(s+4, coffsetS+4, src) + 4 + + // See if we can find a long match at s+1 + const checkAt = 1 + cv := load6432(src, s+checkAt) + nextHashL = hash8(cv, betterLongTableBits) + candidateL = e.longTable[nextHashL] + coffsetL = candidateL.offset - e.cur + + // We can store it, since we have at least a 4 byte match. + e.longTable[nextHashL] = prevEntry{offset: s + checkAt + e.cur, prev: candidateL.offset} + e.markLongShardDirty(nextHashL) + if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { + // Found a long match, at least 8 bytes. + matchedNext := e.matchlen(s+8+checkAt, coffsetL+8, src) + 8 + if matchedNext > matched { + t = coffsetL + s += checkAt + matched = matchedNext + if debugMatches { + println("long match (after short)") + } + break + } + } + + // Check prev long... + coffsetL = candidateL.prev - e.cur + if s-coffsetL < e.maxMatchOff && cv == load6432(src, coffsetL) { + // Found a long match, at least 8 bytes. + matchedNext := e.matchlen(s+8+checkAt, coffsetL+8, src) + 8 + if matchedNext > matched { + t = coffsetL + s += checkAt + matched = matchedNext + if debugMatches { + println("prev long match (after short)") + } + break + } + } + t = coffsetS + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + if debugAsserts && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debugAsserts && t < 0 { + panic("t<0") + } + if debugMatches { + println("short match") + } + break + } + + // No match found, move forward in input. + s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) + if s >= sLimit { + break encodeLoop + } + cv = load6432(src, s) + } + + // A 4-byte match has been found. Update recent offsets. + // We'll later see if more than 4 bytes. + offset2 = offset1 + offset1 = s - t + + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + + if debugAsserts && canRepeat && int(offset1) > len(src) { + panic("invalid offset") + } + + // Extend the n-byte match as long as possible. + l := matched + + // Extend backwards + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { + s-- + t-- + l++ + } + + // Write our sequence + var seq seq + seq.litLen = uint32(s - nextEmit) + seq.matchLen = uint32(l - zstdMinMatch) + if seq.litLen > 0 { + blk.literals = append(blk.literals, src[nextEmit:s]...) + } + seq.offset = uint32(s-t) + 3 + s += l + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + nextEmit = s + if s >= sLimit { + break encodeLoop + } + + // Index match start+1 (long) -> s - 1 + index0 := s - l + 1 + for index0 < s-1 { + cv0 := load6432(src, index0) + cv1 := cv0 >> 8 + h0 := hash8(cv0, betterLongTableBits) + off := index0 + e.cur + e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} + e.markLongShardDirty(h0) + h1 := hash5(cv1, betterShortTableBits) + e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} + e.markShortShardDirty(h1) + index0 += 2 + } + + cv = load6432(src, s) + if !canRepeat { + continue + } + + // Check offset 2 + for { + o2 := s - offset2 + if load3232(src, o2) != uint32(cv) { + // Do regular search + break + } + + // Store this, since we have it. + nextHashS := hash5(cv, betterShortTableBits) + nextHashL := hash8(cv, betterLongTableBits) + + // We have at least 4 byte match. + // No need to check backwards. We come straight from a match + l := 4 + e.matchlen(s+4, o2+4, src) + + e.longTable[nextHashL] = prevEntry{offset: s + e.cur, prev: e.longTable[nextHashL].offset} + e.markLongShardDirty(nextHashL) + e.table[nextHashS] = tableEntry{offset: s + e.cur, val: uint32(cv)} + e.markShortShardDirty(nextHashS) + seq.matchLen = uint32(l) - zstdMinMatch + seq.litLen = 0 + + // Since litlen is always 0, this is offset 1. + seq.offset = 1 + s += l + nextEmit = s + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + // Swap offset 1 and 2. + offset1, offset2 = offset2, offset1 + if s >= sLimit { + // Finished + break encodeLoop + } + cv = load6432(src, s) + } + } + + if int(nextEmit) < len(src) { + blk.literals = append(blk.literals, src[nextEmit:]...) + blk.extraLits = len(src) - int(nextEmit) + } + blk.recentOffsets[0] = uint32(offset1) + blk.recentOffsets[1] = uint32(offset2) + if debug { + println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) + } +} + // ResetDict will reset and set a dictionary if not nil func (e *betterFastEncoder) Reset(d *dict, singleBlock bool) { e.resetBase(d, singleBlock) + if d != nil { + panic("betterFastEncoder: Reset with dict") + } +} + +// ResetDict will reset and set a dictionary if not nil +func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { + e.resetBase(d, singleBlock) if d == nil { return } @@ -557,6 +1070,7 @@ func (e *betterFastEncoder) Reset(d *dict, singleBlock bool) { } } e.lastDictID = d.id + e.allDirty = true } // Init or copy dict table @@ -585,11 +1099,72 @@ func (e *betterFastEncoder) Reset(d *dict, singleBlock bool) { } } e.lastDictID = d.id + e.allDirty = true } + // Reset table to initial state - copy(e.longTable[:], e.dictLongTable) + { + dirtyShardCnt := 0 + if !e.allDirty { + for i := range e.shortTableShardDirty { + if e.shortTableShardDirty[i] { + dirtyShardCnt++ + } + } + } + const shardCnt = betterShortTableShardCnt + const shardSize = betterShortTableShardSize + if e.allDirty || dirtyShardCnt > shardCnt*4/6 { + copy(e.table[:], e.dictTable) + for i := range e.shortTableShardDirty { + e.shortTableShardDirty[i] = false + } + } else { + for i := range e.shortTableShardDirty { + if !e.shortTableShardDirty[i] { + continue + } + copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize]) + e.shortTableShardDirty[i] = false + } + } + } + { + dirtyShardCnt := 0 + if !e.allDirty { + for i := range e.shortTableShardDirty { + if e.shortTableShardDirty[i] { + dirtyShardCnt++ + } + } + } + const shardCnt = betterLongTableShardCnt + const shardSize = betterLongTableShardSize + if e.allDirty || dirtyShardCnt > shardCnt*4/6 { + copy(e.longTable[:], e.dictLongTable) + for i := range e.longTableShardDirty { + e.longTableShardDirty[i] = false + } + } else { + for i := range e.longTableShardDirty { + if !e.longTableShardDirty[i] { + continue + } + + copy(e.longTable[i*shardSize:(i+1)*shardSize], e.dictLongTable[i*shardSize:(i+1)*shardSize]) + e.longTableShardDirty[i] = false + } + } + } e.cur = e.maxMatchOff - // Reset table to initial state - copy(e.table[:], e.dictTable) + e.allDirty = false +} + +func (e *betterFastEncoderDict) markLongShardDirty(entryNum uint32) { + e.longTableShardDirty[entryNum/betterLongTableShardSize] = true +} + +func (e *betterFastEncoderDict) markShortShardDirty(entryNum uint32) { + e.shortTableShardDirty[entryNum/betterShortTableShardSize] = true } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index 19eebf66e..8629d43d8 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -11,6 +11,9 @@ const ( dFastLongTableSize = 1 << dFastLongTableBits // Size of the table dFastLongTableMask = dFastLongTableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. + dLongTableShardCnt = 1 << (dFastLongTableBits - dictShardBits) // Number of shards in the table + dLongTableShardSize = dFastLongTableSize / tableShardCnt // Size of an individual shard + dFastShortTableBits = tableBits // Bits used in the short match table dFastShortTableSize = 1 << dFastShortTableBits // Size of the table dFastShortTableMask = dFastShortTableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. @@ -18,8 +21,14 @@ const ( type doubleFastEncoder struct { fastEncoder - longTable [dFastLongTableSize]tableEntry - dictLongTable []tableEntry + longTable [dFastLongTableSize]tableEntry +} + +type doubleFastEncoderDict struct { + fastEncoderDict + longTable [dFastLongTableSize]tableEntry + dictLongTable []tableEntry + longTableShardDirty [dLongTableShardCnt]bool } // Encode mimmics functionality in zstd_dfast.c @@ -678,9 +687,379 @@ encodeLoop: } } +// Encode will encode the content, with a dictionary if initialized for it. +func (e *doubleFastEncoderDict) Encode(blk *blockEnc, src []byte) { + const ( + // Input margin is the number of bytes we read (8) + // and the maximum we will read ahead (2) + inputMargin = 8 + 2 + minNonLiteralBlockSize = 16 + ) + + // Protect against e.cur wraparound. + for e.cur >= bufferReset { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + for i := range e.longTable[:] { + e.longTable[i] = tableEntry{} + } + e.markAllShardsDirty() + e.cur = e.maxMatchOff + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff + for i := range e.table[:] { + v := e.table[i].offset + if v < minOff { + v = 0 + } else { + v = v - e.cur + e.maxMatchOff + } + e.table[i].offset = v + } + for i := range e.longTable[:] { + v := e.longTable[i].offset + if v < minOff { + v = 0 + } else { + v = v - e.cur + e.maxMatchOff + } + e.longTable[i].offset = v + } + e.markAllShardsDirty() + e.cur = e.maxMatchOff + break + } + + s := e.addBlock(src) + blk.size = len(src) + if len(src) < minNonLiteralBlockSize { + blk.extraLits = len(src) + blk.literals = blk.literals[:len(src)] + copy(blk.literals, src) + return + } + + // Override src + src = e.hist + sLimit := int32(len(src)) - inputMargin + // stepSize is the number of bytes to skip on every main loop iteration. + // It should be >= 1. + const stepSize = 1 + + const kSearchStrength = 8 + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := s + cv := load6432(src, s) + + // Relative offsets + offset1 := int32(blk.recentOffsets[0]) + offset2 := int32(blk.recentOffsets[1]) + + addLiterals := func(s *seq, until int32) { + if until == nextEmit { + return + } + blk.literals = append(blk.literals, src[nextEmit:until]...) + s.litLen = uint32(until - nextEmit) + } + if debug { + println("recent offsets:", blk.recentOffsets) + } + +encodeLoop: + for { + var t int32 + // We allow the encoder to optionally turn off repeat offsets across blocks + canRepeat := len(blk.sequences) > 2 + + for { + if debugAsserts && canRepeat && offset1 == 0 { + panic("offset0 was 0") + } + + nextHashS := hash5(cv, dFastShortTableBits) + nextHashL := hash8(cv, dFastLongTableBits) + candidateL := e.longTable[nextHashL] + candidateS := e.table[nextHashS] + + const repOff = 1 + repIndex := s - offset1 + repOff + entry := tableEntry{offset: s + e.cur, val: uint32(cv)} + e.longTable[nextHashL] = entry + e.markLongShardDirty(nextHashL) + e.table[nextHashS] = entry + e.markShardDirty(nextHashS) + + if canRepeat { + if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { + // Consider history as well. + var seq seq + lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + + seq.matchLen = uint32(lenght - zstdMinMatch) + + // We might be able to match backwards. + // Extend as long as we can. + start := s + repOff + // We end the search early, so we don't risk 0 literals + // and have to do special offset treatment. + startLimit := nextEmit + 1 + + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for repIndex > tMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch-1 { + repIndex-- + start-- + seq.matchLen++ + } + addLiterals(&seq, start) + + // rep 0 + seq.offset = 1 + if debugSequences { + println("repeat sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + s += lenght + repOff + nextEmit = s + if s >= sLimit { + if debug { + println("repeat ended", s, lenght) + + } + break encodeLoop + } + cv = load6432(src, s) + continue + } + } + // Find the offsets of our two matches. + coffsetL := s - (candidateL.offset - e.cur) + coffsetS := s - (candidateS.offset - e.cur) + + // Check if we have a long match. + if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val { + // Found a long match, likely at least 8 bytes. + // Reference encoder checks all 8 bytes, we only check 4, + // but the likelihood of both the first 4 bytes and the hash matching should be enough. + t = candidateL.offset - e.cur + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + if debugAsserts && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debugMatches { + println("long match") + } + break + } + + // Check if we have a short match. + if coffsetS < e.maxMatchOff && uint32(cv) == candidateS.val { + // found a regular match + // See if we can find a long match at s+1 + const checkAt = 1 + cv := load6432(src, s+checkAt) + nextHashL = hash8(cv, dFastLongTableBits) + candidateL = e.longTable[nextHashL] + coffsetL = s - (candidateL.offset - e.cur) + checkAt + + // We can store it, since we have at least a 4 byte match. + e.longTable[nextHashL] = tableEntry{offset: s + checkAt + e.cur, val: uint32(cv)} + e.markLongShardDirty(nextHashL) + if coffsetL < e.maxMatchOff && uint32(cv) == candidateL.val { + // Found a long match, likely at least 8 bytes. + // Reference encoder checks all 8 bytes, we only check 4, + // but the likelihood of both the first 4 bytes and the hash matching should be enough. + t = candidateL.offset - e.cur + s += checkAt + if debugMatches { + println("long match (after short)") + } + break + } + + t = candidateS.offset - e.cur + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + if debugAsserts && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debugAsserts && t < 0 { + panic("t<0") + } + if debugMatches { + println("short match") + } + break + } + + // No match found, move forward in input. + s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) + if s >= sLimit { + break encodeLoop + } + cv = load6432(src, s) + } + + // A 4-byte match has been found. Update recent offsets. + // We'll later see if more than 4 bytes. + offset2 = offset1 + offset1 = s - t + + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + + if debugAsserts && canRepeat && int(offset1) > len(src) { + panic("invalid offset") + } + + // Extend the 4-byte match as long as possible. + l := e.matchlen(s+4, t+4, src) + 4 + + // Extend backwards + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { + s-- + t-- + l++ + } + + // Write our sequence + var seq seq + seq.litLen = uint32(s - nextEmit) + seq.matchLen = uint32(l - zstdMinMatch) + if seq.litLen > 0 { + blk.literals = append(blk.literals, src[nextEmit:s]...) + } + seq.offset = uint32(s-t) + 3 + s += l + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + nextEmit = s + if s >= sLimit { + break encodeLoop + } + + // Index match start+1 (long) and start+2 (short) + index0 := s - l + 1 + // Index match end-2 (long) and end-1 (short) + index1 := s - 2 + + cv0 := load6432(src, index0) + cv1 := load6432(src, index1) + te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} + te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} + longHash1 := hash8(cv0, dFastLongTableBits) + longHash2 := hash8(cv0, dFastLongTableBits) + e.longTable[longHash1] = te0 + e.longTable[longHash2] = te1 + e.markLongShardDirty(longHash1) + e.markLongShardDirty(longHash2) + cv0 >>= 8 + cv1 >>= 8 + te0.offset++ + te1.offset++ + te0.val = uint32(cv0) + te1.val = uint32(cv1) + hashVal1 := hash5(cv0, dFastShortTableBits) + hashVal2 := hash5(cv1, dFastShortTableBits) + e.table[hashVal1] = te0 + e.markShardDirty(hashVal1) + e.table[hashVal2] = te1 + e.markShardDirty(hashVal2) + + cv = load6432(src, s) + + if !canRepeat { + continue + } + + // Check offset 2 + for { + o2 := s - offset2 + if load3232(src, o2) != uint32(cv) { + // Do regular search + break + } + + // Store this, since we have it. + nextHashS := hash5(cv, dFastShortTableBits) + nextHashL := hash8(cv, dFastLongTableBits) + + // We have at least 4 byte match. + // No need to check backwards. We come straight from a match + l := 4 + e.matchlen(s+4, o2+4, src) + + entry := tableEntry{offset: s + e.cur, val: uint32(cv)} + e.longTable[nextHashL] = entry + e.markLongShardDirty(nextHashL) + e.table[nextHashS] = entry + e.markShardDirty(nextHashS) + seq.matchLen = uint32(l) - zstdMinMatch + seq.litLen = 0 + + // Since litlen is always 0, this is offset 1. + seq.offset = 1 + s += l + nextEmit = s + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + // Swap offset 1 and 2. + offset1, offset2 = offset2, offset1 + if s >= sLimit { + // Finished + break encodeLoop + } + cv = load6432(src, s) + } + } + + if int(nextEmit) < len(src) { + blk.literals = append(blk.literals, src[nextEmit:]...) + blk.extraLits = len(src) - int(nextEmit) + } + blk.recentOffsets[0] = uint32(offset1) + blk.recentOffsets[1] = uint32(offset2) + if debug { + println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) + } + // If we encoded more than 64K mark all dirty. + if len(src) > 64<<10 { + e.markAllShardsDirty() + } +} + // ResetDict will reset and set a dictionary if not nil func (e *doubleFastEncoder) Reset(d *dict, singleBlock bool) { e.fastEncoder.Reset(d, singleBlock) + if d != nil { + panic("doubleFastEncoder: Reset with dict not supported") + } +} + +// ResetDict will reset and set a dictionary if not nil +func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { + allDirty := e.allDirty + e.fastEncoderDict.Reset(d, singleBlock) if d == nil { return } @@ -706,8 +1085,37 @@ func (e *doubleFastEncoder) Reset(d *dict, singleBlock bool) { } } e.lastDictID = d.id + e.allDirty = true } // Reset table to initial state e.cur = e.maxMatchOff - copy(e.longTable[:], e.dictLongTable) + + dirtyShardCnt := 0 + if !allDirty { + for i := range e.longTableShardDirty { + if e.longTableShardDirty[i] { + dirtyShardCnt++ + } + } + } + + if allDirty || dirtyShardCnt > dLongTableShardCnt/2 { + copy(e.longTable[:], e.dictLongTable) + for i := range e.longTableShardDirty { + e.longTableShardDirty[i] = false + } + return + } + for i := range e.longTableShardDirty { + if !e.longTableShardDirty[i] { + continue + } + + copy(e.longTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize], e.dictLongTable[i*dLongTableShardSize:(i+1)*dLongTableShardSize]) + e.longTableShardDirty[i] = false + } +} + +func (e *doubleFastEncoderDict) markLongShardDirty(entryNum uint32) { + e.longTableShardDirty[entryNum/dLongTableShardSize] = true } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index 0045016d9..ba4a17e10 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -11,9 +11,11 @@ import ( ) const ( - tableBits = 15 // Bits used in the table - tableSize = 1 << tableBits // Size of the table - tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. + tableBits = 15 // Bits used in the table + tableSize = 1 << tableBits // Size of the table + tableShardCnt = 1 << (tableBits - dictShardBits) // Number of shards in the table + tableShardSize = tableSize / tableShardCnt // Size of an individual shard + tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. maxMatchLength = 131074 ) @@ -24,8 +26,14 @@ type tableEntry struct { type fastEncoder struct { fastBase - table [tableSize]tableEntry - dictTable []tableEntry + table [tableSize]tableEntry +} + +type fastEncoderDict struct { + fastEncoder + dictTable []tableEntry + tableShardDirty [tableShardCnt]bool + allDirty bool } // Encode mimmics functionality in zstd_fast.c @@ -617,9 +625,323 @@ encodeLoop: } } +// Encode will encode the content, with a dictionary if initialized for it. +func (e *fastEncoderDict) Encode(blk *blockEnc, src []byte) { + const ( + inputMargin = 8 + minNonLiteralBlockSize = 1 + 1 + inputMargin + ) + if e.allDirty || len(src) > 32<<10 { + e.fastEncoder.Encode(blk, src) + e.allDirty = true + return + } + // Protect against e.cur wraparound. + for e.cur >= bufferReset { + if len(e.hist) == 0 { + for i := range e.table[:] { + e.table[i] = tableEntry{} + } + e.cur = e.maxMatchOff + break + } + // Shift down everything in the table that isn't already too far away. + minOff := e.cur + int32(len(e.hist)) - e.maxMatchOff + for i := range e.table[:] { + v := e.table[i].offset + if v < minOff { + v = 0 + } else { + v = v - e.cur + e.maxMatchOff + } + e.table[i].offset = v + } + e.cur = e.maxMatchOff + break + } + + s := e.addBlock(src) + blk.size = len(src) + if len(src) < minNonLiteralBlockSize { + blk.extraLits = len(src) + blk.literals = blk.literals[:len(src)] + copy(blk.literals, src) + return + } + + // Override src + src = e.hist + sLimit := int32(len(src)) - inputMargin + // stepSize is the number of bytes to skip on every main loop iteration. + // It should be >= 2. + const stepSize = 2 + + // TEMPLATE + const hashLog = tableBits + // seems global, but would be nice to tweak. + const kSearchStrength = 7 + + // nextEmit is where in src the next emitLiteral should start from. + nextEmit := s + cv := load6432(src, s) + + // Relative offsets + offset1 := int32(blk.recentOffsets[0]) + offset2 := int32(blk.recentOffsets[1]) + + addLiterals := func(s *seq, until int32) { + if until == nextEmit { + return + } + blk.literals = append(blk.literals, src[nextEmit:until]...) + s.litLen = uint32(until - nextEmit) + } + if debug { + println("recent offsets:", blk.recentOffsets) + } + +encodeLoop: + for { + // t will contain the match offset when we find one. + // When existing the search loop, we have already checked 4 bytes. + var t int32 + + // We will not use repeat offsets across blocks. + // By not using them for the first 3 matches + canRepeat := len(blk.sequences) > 2 + + for { + if debugAsserts && canRepeat && offset1 == 0 { + panic("offset0 was 0") + } + + nextHash := hash6(cv, hashLog) + nextHash2 := hash6(cv>>8, hashLog) + candidate := e.table[nextHash] + candidate2 := e.table[nextHash2] + repIndex := s - offset1 + 2 + + e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} + e.markShardDirty(nextHash) + e.table[nextHash2] = tableEntry{offset: s + e.cur + 1, val: uint32(cv >> 8)} + e.markShardDirty(nextHash2) + + if canRepeat && repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>16) { + // Consider history as well. + var seq seq + var length int32 + // length = 4 + e.matchlen(s+6, repIndex+4, src) + { + a := src[s+6:] + b := src[repIndex+4:] + endI := len(a) & (math.MaxInt32 - 7) + length = int32(endI) + 4 + for i := 0; i < endI; i += 8 { + if diff := load64(a, i) ^ load64(b, i); diff != 0 { + length = int32(i+bits.TrailingZeros64(diff)>>3) + 4 + break + } + } + } + + seq.matchLen = uint32(length - zstdMinMatch) + + // We might be able to match backwards. + // Extend as long as we can. + start := s + 2 + // We end the search early, so we don't risk 0 literals + // and have to do special offset treatment. + startLimit := nextEmit + 1 + + sMin := s - e.maxMatchOff + if sMin < 0 { + sMin = 0 + } + for repIndex > sMin && start > startLimit && src[repIndex-1] == src[start-1] && seq.matchLen < maxMatchLength-zstdMinMatch { + repIndex-- + start-- + seq.matchLen++ + } + addLiterals(&seq, start) + + // rep 0 + seq.offset = 1 + if debugSequences { + println("repeat sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + s += length + 2 + nextEmit = s + if s >= sLimit { + if debug { + println("repeat ended", s, length) + + } + break encodeLoop + } + cv = load6432(src, s) + continue + } + coffset0 := s - (candidate.offset - e.cur) + coffset1 := s - (candidate2.offset - e.cur) + 1 + if coffset0 < e.maxMatchOff && uint32(cv) == candidate.val { + // found a regular match + t = candidate.offset - e.cur + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + if debugAsserts && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + break + } + + if coffset1 < e.maxMatchOff && uint32(cv>>8) == candidate2.val { + // found a regular match + t = candidate2.offset - e.cur + s++ + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + if debugAsserts && s-t > e.maxMatchOff { + panic("s - t >e.maxMatchOff") + } + if debugAsserts && t < 0 { + panic("t<0") + } + break + } + s += stepSize + ((s - nextEmit) >> (kSearchStrength - 1)) + if s >= sLimit { + break encodeLoop + } + cv = load6432(src, s) + } + // A 4-byte match has been found. We'll later see if more than 4 bytes. + offset2 = offset1 + offset1 = s - t + + if debugAsserts && s <= t { + panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) + } + + if debugAsserts && canRepeat && int(offset1) > len(src) { + panic("invalid offset") + } + + // Extend the 4-byte match as long as possible. + //l := e.matchlen(s+4, t+4, src) + 4 + var l int32 + { + a := src[s+4:] + b := src[t+4:] + endI := len(a) & (math.MaxInt32 - 7) + l = int32(endI) + 4 + for i := 0; i < endI; i += 8 { + if diff := load64(a, i) ^ load64(b, i); diff != 0 { + l = int32(i+bits.TrailingZeros64(diff)>>3) + 4 + break + } + } + } + + // Extend backwards + tMin := s - e.maxMatchOff + if tMin < 0 { + tMin = 0 + } + for t > tMin && s > nextEmit && src[t-1] == src[s-1] && l < maxMatchLength { + s-- + t-- + l++ + } + + // Write our sequence. + var seq seq + seq.litLen = uint32(s - nextEmit) + seq.matchLen = uint32(l - zstdMinMatch) + if seq.litLen > 0 { + blk.literals = append(blk.literals, src[nextEmit:s]...) + } + // Don't use repeat offsets + seq.offset = uint32(s-t) + 3 + s += l + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + nextEmit = s + if s >= sLimit { + break encodeLoop + } + cv = load6432(src, s) + + // Check offset 2 + if o2 := s - offset2; canRepeat && load3232(src, o2) == uint32(cv) { + // We have at least 4 byte match. + // No need to check backwards. We come straight from a match + //l := 4 + e.matchlen(s+4, o2+4, src) + var l int32 + { + a := src[s+4:] + b := src[o2+4:] + endI := len(a) & (math.MaxInt32 - 7) + l = int32(endI) + 4 + for i := 0; i < endI; i += 8 { + if diff := load64(a, i) ^ load64(b, i); diff != 0 { + l = int32(i+bits.TrailingZeros64(diff)>>3) + 4 + break + } + } + } + + // Store this, since we have it. + nextHash := hash6(cv, hashLog) + e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} + e.markShardDirty(nextHash) + seq.matchLen = uint32(l) - zstdMinMatch + seq.litLen = 0 + // Since litlen is always 0, this is offset 1. + seq.offset = 1 + s += l + nextEmit = s + if debugSequences { + println("sequence", seq, "next s:", s) + } + blk.sequences = append(blk.sequences, seq) + + // Swap offset 1 and 2. + offset1, offset2 = offset2, offset1 + if s >= sLimit { + break encodeLoop + } + // Prepare next loop. + cv = load6432(src, s) + } + } + + if int(nextEmit) < len(src) { + blk.literals = append(blk.literals, src[nextEmit:]...) + blk.extraLits = len(src) - int(nextEmit) + } + blk.recentOffsets[0] = uint32(offset1) + blk.recentOffsets[1] = uint32(offset2) + if debug { + println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) + } +} + // ResetDict will reset and set a dictionary if not nil func (e *fastEncoder) Reset(d *dict, singleBlock bool) { e.resetBase(d, singleBlock) + if d != nil { + panic("fastEncoder: Reset with dict") + } +} + +// ResetDict will reset and set a dictionary if not nil +func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { + e.resetBase(d, singleBlock) if d == nil { return } @@ -653,9 +975,44 @@ func (e *fastEncoder) Reset(d *dict, singleBlock bool) { } } e.lastDictID = d.id + e.allDirty = true } e.cur = e.maxMatchOff - // Reset table to initial state - copy(e.table[:], e.dictTable) + dirtyShardCnt := 0 + if !e.allDirty { + for i := range e.tableShardDirty { + if e.tableShardDirty[i] { + dirtyShardCnt++ + } + } + } + + const shardCnt = tableShardCnt + const shardSize = tableShardSize + if e.allDirty || dirtyShardCnt > shardCnt*4/6 { + copy(e.table[:], e.dictTable) + for i := range e.tableShardDirty { + e.tableShardDirty[i] = false + } + e.allDirty = false + return + } + for i := range e.tableShardDirty { + if !e.tableShardDirty[i] { + continue + } + + copy(e.table[i*shardSize:(i+1)*shardSize], e.dictTable[i*shardSize:(i+1)*shardSize]) + e.tableShardDirty[i] = false + } + e.allDirty = false +} + +func (e *fastEncoderDict) markAllShardsDirty() { + e.allDirty = true +} + +func (e *fastEncoderDict) markShardDirty(entryNum uint32) { + e.tableShardDirty[entryNum/tableShardSize] = true } diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index f5759211d..6f0265099 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -106,7 +106,7 @@ func (e *Encoder) Reset(w io.Writer) { s.encoder = e.o.encoder() } if s.writing == nil { - s.writing = &blockEnc{} + s.writing = &blockEnc{lowMem: e.o.lowMem} s.writing.init() } s.writing.initNewEncode() @@ -176,6 +176,12 @@ func (e *Encoder) nextBlock(final bool) error { } if !s.headerWritten { // If we have a single block encode, do a sync compression. + if final && len(s.filling) == 0 && !e.o.fullZero { + s.headerWritten = true + s.fullFrameWritten = true + s.eofWritten = true + return nil + } if final && len(s.filling) > 0 { s.current = e.EncodeAll(s.filling, s.current[:0]) var n2 int @@ -471,7 +477,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } // If less than 1MB, allocate a buffer up front. - if len(dst) == 0 && cap(dst) == 0 && len(src) < 1<<20 { + if len(dst) == 0 && cap(dst) == 0 && len(src) < 1<<20 && !e.o.lowMem { dst = make([]byte, 0, len(src)) } dst, err := fh.appendTo(dst) diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index a7312f42a..18a47eb03 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -24,12 +24,12 @@ type encoderOptions struct { allLitEntropy bool customWindow bool customALEntropy bool + lowMem bool dict *dict } func (o *encoderOptions) setDefault() { *o = encoderOptions{ - // use less ram: true for now, but may change. concurrent: runtime.GOMAXPROCS(0), crc: true, single: nil, @@ -37,20 +37,31 @@ func (o *encoderOptions) setDefault() { windowSize: 8 << 20, level: SpeedDefault, allLitEntropy: true, + lowMem: false, } } // encoder returns an encoder with the selected options. func (o encoderOptions) encoder() encoder { switch o.level { + case SpeedFastest: + if o.dict != nil { + return &fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}} + } + return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}} + case SpeedDefault: - return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}}} + if o.dict != nil { + return &doubleFastEncoderDict{fastEncoderDict: fastEncoderDict{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}}} + } + return &doubleFastEncoder{fastEncoder: fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}} case SpeedBetterCompression: - return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}} + if o.dict != nil { + return &betterFastEncoderDict{betterFastEncoder: betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}}} + } + return &betterFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}} case SpeedBestCompression: - return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}} - case SpeedFastest: - return &fastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize)}} + return &bestFastEncoder{fastBase: fastBase{maxMatchOff: int32(o.windowSize), lowMem: o.lowMem}} } panic("unknown compression level") } @@ -276,6 +287,17 @@ func WithSingleSegment(b bool) EOption { } } +// WithLowerEncoderMem will trade in some memory cases trade less memory usage for +// slower encoding speed. +// This will not change the window size which is the primary function for reducing +// memory usage. See WithWindowSize. +func WithLowerEncoderMem(b bool) EOption { + return func(o *encoderOptions) error { + o.lowMem = b + return nil + } +} + // WithEncoderDict allows to register a dictionary that will be used for the encode. // The encoder *may* choose to use no dictionary instead for certain payloads. func WithEncoderDict(dict []byte) EOption { diff --git a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go index aa9eba88b..b80709d5e 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go @@ -97,7 +97,7 @@ func (s *fseEncoder) prepare() (*fseEncoder, error) { func (s *fseEncoder) allocCtable() { tableSize := 1 << s.actualTableLog // get tableSymbol that is big enough. - if cap(s.ct.tableSymbol) < int(tableSize) { + if cap(s.ct.tableSymbol) < tableSize { s.ct.tableSymbol = make([]byte, tableSize) } s.ct.tableSymbol = s.ct.tableSymbol[:tableSize] @@ -202,13 +202,13 @@ func (s *fseEncoder) buildCTable() error { case 0: case -1, 1: symbolTT[i].deltaNbBits = tl - symbolTT[i].deltaFindState = int16(total - 1) + symbolTT[i].deltaFindState = total - 1 total++ default: maxBitsOut := uint32(tableLog) - highBit(uint32(v-1)) minStatePlus := uint32(v) << maxBitsOut symbolTT[i].deltaNbBits = (maxBitsOut << 16) - minStatePlus - symbolTT[i].deltaFindState = int16(total - v) + symbolTT[i].deltaFindState = total - v total += v } } @@ -353,8 +353,8 @@ func (s *fseEncoder) normalizeCount2(length int) error { distributed uint32 total = uint32(length) tableLog = s.actualTableLog - lowThreshold = uint32(total >> tableLog) - lowOne = uint32((total * 3) >> (tableLog + 1)) + lowThreshold = total >> tableLog + lowOne = (total * 3) >> (tableLog + 1) ) for i, cnt := range s.count[:s.symbolLen] { if cnt == 0 { @@ -379,7 +379,7 @@ func (s *fseEncoder) normalizeCount2(length int) error { if (total / toDistribute) > lowOne { // risk of rounding to zero - lowOne = uint32((total * 3) / (toDistribute * 2)) + lowOne = (total * 3) / (toDistribute * 2) for i, cnt := range s.count[:s.symbolLen] { if (s.norm[i] == notYetAssigned) && (cnt <= lowOne) { s.norm[i] = 1 diff --git a/vendor/github.com/klauspost/compress/zstd/snappy.go b/vendor/github.com/klauspost/compress/zstd/snappy.go index 841fd95ac..c95fe5111 100644 --- a/vendor/github.com/klauspost/compress/zstd/snappy.go +++ b/vendor/github.com/klauspost/compress/zstd/snappy.go @@ -417,7 +417,7 @@ var crcTable = crc32.MakeTable(crc32.Castagnoli) // https://github.com/google/snappy/blob/master/framing_format.txt func snappyCRC(b []byte) uint32 { c := crc32.Update(0, crcTable, b) - return uint32(c>>15|c<<17) + 0xa282ead8 + return c>>15 | c<<17 + 0xa282ead8 } // snappyDecodedLen returns the length of the decoded block and the number of bytes diff --git a/vendor/github.com/vbauerster/mpb/v5/.gitignore b/vendor/github.com/vbauerster/mpb/v5/.gitignore deleted file mode 100644 index 63bd91672..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -# Test binary, build with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out diff --git a/vendor/github.com/vbauerster/mpb/v5/.travis.yml b/vendor/github.com/vbauerster/mpb/v5/.travis.yml deleted file mode 100644 index 9a203a67d..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/.travis.yml +++ /dev/null @@ -1,11 +0,0 @@ -language: go -arch: - - amd64 - - ppc64le - -go: - - 1.14.x - -script: - - go test -race ./... - - for i in _examples/*/; do go build $i/*.go || exit 1; done diff --git a/vendor/github.com/vbauerster/mpb/v5/README.md b/vendor/github.com/vbauerster/mpb/v5/README.md deleted file mode 100644 index bfb0c4d18..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/README.md +++ /dev/null @@ -1,118 +0,0 @@ -# Multi Progress Bar - -[](https://godoc.org/github.com/vbauerster/mpb) -[](https://travis-ci.org/vbauerster/mpb) -[](https://goreportcard.com/report/github.com/vbauerster/mpb) - -**mpb** is a Go lib for rendering progress bars in terminal applications. - -## Features - -* __Multiple Bars__: Multiple progress bars are supported -* __Dynamic Total__: Set total while bar is running -* __Dynamic Add/Remove__: Dynamically add or remove bars -* __Cancellation__: Cancel whole rendering process -* __Predefined Decorators__: Elapsed time, [ewma](https://github.com/VividCortex/ewma) based ETA, Percentage, Bytes counter -* __Decorator's width sync__: Synchronized decorator's width among multiple bars - -## Usage - -#### [Rendering single bar](_examples/singleBar/main.go) -```go -package main - -import ( - "math/rand" - "time" - - "github.com/vbauerster/mpb/v5" - "github.com/vbauerster/mpb/v5/decor" -) - -func main() { - // initialize progress container, with custom width - p := mpb.New(mpb.WithWidth(64)) - - total := 100 - name := "Single Bar:" - // adding a single bar, which will inherit container's width - bar := p.AddBar(int64(total), - // override DefaultBarStyle, which is "[=>-]<+" - mpb.BarStyle("╢▌▌░╟"), - mpb.PrependDecorators( - // display our name with one space on the right - decor.Name(name, decor.WC{W: len(name) + 1, C: decor.DidentRight}), - // replace ETA decorator with "done" message, OnComplete event - decor.OnComplete( - decor.AverageETA(decor.ET_STYLE_GO, decor.WC{W: 4}), "done", - ), - ), - mpb.AppendDecorators(decor.Percentage()), - ) - // simulating some work - max := 100 * time.Millisecond - for i := 0; i < total; i++ { - time.Sleep(time.Duration(rand.Intn(10)+1) * max / 10) - bar.Increment() - } - // wait for our bar to complete and flush - p.Wait() -} -``` - -#### [Rendering multiple bars](_examples/multiBars/main.go) -```go - var wg sync.WaitGroup - // pass &wg (optional), so p will wait for it eventually - p := mpb.New(mpb.WithWaitGroup(&wg)) - total, numBars := 100, 3 - wg.Add(numBars) - - for i := 0; i < numBars; i++ { - name := fmt.Sprintf("Bar#%d:", i) - bar := p.AddBar(int64(total), - mpb.PrependDecorators( - // simple name decorator - decor.Name(name), - // decor.DSyncWidth bit enables column width synchronization - decor.Percentage(decor.WCSyncSpace), - ), - mpb.AppendDecorators( - // replace ETA decorator with "done" message, OnComplete event - decor.OnComplete( - // ETA decorator with ewma age of 60 - decor.EwmaETA(decor.ET_STYLE_GO, 60), "done", - ), - ), - ) - // simulating some work - go func() { - defer wg.Done() - rng := rand.New(rand.NewSource(time.Now().UnixNano())) - max := 100 * time.Millisecond - for i := 0; i < total; i++ { - // start variable is solely for EWMA calculation - // EWMA's unit of measure is an iteration's duration - start := time.Now() - time.Sleep(time.Duration(rng.Intn(10)+1) * max / 10) - bar.Increment() - // we need to call DecoratorEwmaUpdate to fulfill ewma decorator's contract - bar.DecoratorEwmaUpdate(time.Since(start)) - } - }() - } - // Waiting for passed &wg and for all bars to complete and flush - p.Wait() -``` - -#### [Dynamic total](_examples/dynTotal/main.go) - - - -#### [Complex example](_examples/complex/main.go) - - - -#### [Bytes counters](_examples/io/main.go) - - diff --git a/vendor/github.com/vbauerster/mpb/v5/UNLICENSE b/vendor/github.com/vbauerster/mpb/v5/UNLICENSE deleted file mode 100644 index 68a49daad..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/UNLICENSE +++ /dev/null @@ -1,24 +0,0 @@ -This is free and unencumbered software released into the public domain. - -Anyone is free to copy, modify, publish, use, compile, sell, or -distribute this software, either in source code form or as a compiled -binary, for any purpose, commercial or non-commercial, and by any -means. - -In jurisdictions that recognize copyright laws, the author or authors -of this software dedicate any and all copyright interest in the -software to the public domain. We make this dedication for the benefit -of the public at large and to the detriment of our heirs and -successors. We intend this dedication to be an overt act of -relinquishment in perpetuity of all present and future rights to this -software under copyright law. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, -EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF -MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. -IN NO EVENT SHALL THE AUTHORS BE LIABLE FOR ANY CLAIM, DAMAGES OR -OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, -ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR -OTHER DEALINGS IN THE SOFTWARE. - -For more information, please refer to <http://unlicense.org/> diff --git a/vendor/github.com/vbauerster/mpb/v5/bar.go b/vendor/github.com/vbauerster/mpb/v5/bar.go deleted file mode 100644 index 358cb048d..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/bar.go +++ /dev/null @@ -1,490 +0,0 @@ -package mpb - -import ( - "bytes" - "context" - "fmt" - "io" - "log" - "runtime/debug" - "strings" - "time" - - "github.com/acarl005/stripansi" - "github.com/mattn/go-runewidth" - "github.com/vbauerster/mpb/v5/decor" -) - -// Bar represents a progress Bar. -type Bar struct { - priority int // used by heap - index int // used by heap - - extendedLines int - toShutdown bool - toDrop bool - noPop bool - hasEwmaDecorators bool - operateState chan func(*bState) - frameCh chan io.Reader - syncTableCh chan [][]chan int - completed chan bool - - // cancel is called either by user or on complete event - cancel func() - // done is closed after cacheState is assigned - done chan struct{} - // cacheState is populated, right after close(shutdown) - cacheState *bState - - container *Progress - dlogger *log.Logger - recoveredPanic interface{} -} - -type extFunc func(in io.Reader, reqWidth int, st decor.Statistics) (out io.Reader, lines int) - -type bState struct { - id int - priority int - reqWidth int - total int64 - current int64 - refill int64 - lastN int64 - iterated bool - trimSpace bool - toComplete bool - completeFlushed bool - ignoreComplete bool - dropOnComplete bool - noPop bool - aDecorators []decor.Decorator - pDecorators []decor.Decorator - averageDecorators []decor.AverageDecorator - ewmaDecorators []decor.EwmaDecorator - shutdownListeners []decor.ShutdownListener - bufP, bufB, bufA *bytes.Buffer - filler BarFiller - middleware func(BarFiller) BarFiller - extender extFunc - - // runningBar is a key for *pState.parkedBars - runningBar *Bar - - debugOut io.Writer -} - -func newBar(container *Progress, bs *bState) *Bar { - logPrefix := fmt.Sprintf("%sbar#%02d ", container.dlogger.Prefix(), bs.id) - ctx, cancel := context.WithCancel(container.ctx) - - bar := &Bar{ - container: container, - priority: bs.priority, - toDrop: bs.dropOnComplete, - noPop: bs.noPop, - operateState: make(chan func(*bState)), - frameCh: make(chan io.Reader, 1), - syncTableCh: make(chan [][]chan int, 1), - completed: make(chan bool, 1), - done: make(chan struct{}), - cancel: cancel, - dlogger: log.New(bs.debugOut, logPrefix, log.Lshortfile), - } - - go bar.serve(ctx, bs) - return bar -} - -// ProxyReader wraps r with metrics required for progress tracking. -// Panics if r is nil. -func (b *Bar) ProxyReader(r io.Reader) io.ReadCloser { - if r == nil { - panic("expected non nil io.Reader") - } - return newProxyReader(r, b) -} - -// ID returs id of the bar. -func (b *Bar) ID() int { - result := make(chan int) - select { - case b.operateState <- func(s *bState) { result <- s.id }: - return <-result - case <-b.done: - return b.cacheState.id - } -} - -// Current returns bar's current number, in other words sum of all increments. -func (b *Bar) Current() int64 { - result := make(chan int64) - select { - case b.operateState <- func(s *bState) { result <- s.current }: - return <-result - case <-b.done: - return b.cacheState.current - } -} - -// SetRefill fills bar with refill rune up to amount argument. -// Given default bar style is "[=>-]<+", refill rune is '+'. -// To set bar style use mpb.BarStyle(string) BarOption. -func (b *Bar) SetRefill(amount int64) { - select { - case b.operateState <- func(s *bState) { - s.refill = amount - }: - case <-b.done: - } -} - -// TraverseDecorators traverses all available decorators and calls cb func on each. -func (b *Bar) TraverseDecorators(cb func(decor.Decorator)) { - select { - case b.operateState <- func(s *bState) { - for _, decorators := range [...][]decor.Decorator{ - s.pDecorators, - s.aDecorators, - } { - for _, d := range decorators { - cb(extractBaseDecorator(d)) - } - } - }: - case <-b.done: - } -} - -// SetTotal sets total dynamically. -// If total is less than or equal to zero it takes progress' current value. -// A complete flag enables or disables complete event on `current >= total`. -func (b *Bar) SetTotal(total int64, complete bool) { - select { - case b.operateState <- func(s *bState) { - s.ignoreComplete = !complete - if total <= 0 { - s.total = s.current - } else { - s.total = total - } - if !s.ignoreComplete && !s.toComplete { - s.current = s.total - s.toComplete = true - go b.refreshTillShutdown() - } - }: - case <-b.done: - } -} - -// SetCurrent sets progress' current to an arbitrary value. -// Setting a negative value will cause a panic. -func (b *Bar) SetCurrent(current int64) { - select { - case b.operateState <- func(s *bState) { - s.iterated = true - s.lastN = current - s.current - s.current = current - if !s.ignoreComplete && s.current >= s.total { - s.current = s.total - s.toComplete = true - go b.refreshTillShutdown() - } - }: - case <-b.done: - } -} - -// Increment is a shorthand for b.IncrInt64(1). -func (b *Bar) Increment() { - b.IncrInt64(1) -} - -// IncrBy is a shorthand for b.IncrInt64(int64(n)). -func (b *Bar) IncrBy(n int) { - b.IncrInt64(int64(n)) -} - -// IncrInt64 increments progress by amount of n. -func (b *Bar) IncrInt64(n int64) { - select { - case b.operateState <- func(s *bState) { - s.iterated = true - s.lastN = n - s.current += n - if !s.ignoreComplete && s.current >= s.total { - s.current = s.total - s.toComplete = true - go b.refreshTillShutdown() - } - }: - case <-b.done: - } -} - -// DecoratorEwmaUpdate updates all EWMA based decorators. Should be -// called on each iteration, because EWMA's unit of measure is an -// iteration's duration. Panics if called before *Bar.Incr... family -// methods. -func (b *Bar) DecoratorEwmaUpdate(dur time.Duration) { - select { - case b.operateState <- func(s *bState) { - ewmaIterationUpdate(false, s, dur) - }: - case <-b.done: - ewmaIterationUpdate(true, b.cacheState, dur) - } -} - -// DecoratorAverageAdjust adjusts all average based decorators. Call -// if you need to adjust start time of all average based decorators -// or after progress resume. -func (b *Bar) DecoratorAverageAdjust(start time.Time) { - select { - case b.operateState <- func(s *bState) { - for _, d := range s.averageDecorators { - d.AverageAdjust(start) - } - }: - case <-b.done: - } -} - -// SetPriority changes bar's order among multiple bars. Zero is highest -// priority, i.e. bar will be on top. If you don't need to set priority -// dynamically, better use BarPriority option. -func (b *Bar) SetPriority(priority int) { - select { - case <-b.done: - default: - b.container.setBarPriority(b, priority) - } -} - -// Abort interrupts bar's running goroutine. Call this, if you'd like -// to stop/remove bar before completion event. It has no effect after -// completion event. If drop is true bar will be removed as well. -func (b *Bar) Abort(drop bool) { - select { - case <-b.done: - default: - if drop { - b.container.dropBar(b) - } - b.cancel() - } -} - -// Completed reports whether the bar is in completed state. -func (b *Bar) Completed() bool { - select { - case b.operateState <- func(s *bState) { b.completed <- s.toComplete }: - return <-b.completed - case <-b.done: - return true - } -} - -func (b *Bar) serve(ctx context.Context, s *bState) { - defer b.container.bwg.Done() - for { - select { - case op := <-b.operateState: - op(s) - case <-ctx.Done(): - b.cacheState = s - close(b.done) - // Notifying decorators about shutdown event - for _, sl := range s.shutdownListeners { - sl.Shutdown() - } - return - } - } -} - -func (b *Bar) render(tw int) { - select { - case b.operateState <- func(s *bState) { - stat := newStatistics(tw, s) - defer func() { - // recovering if user defined decorator panics for example - if p := recover(); p != nil { - if b.recoveredPanic == nil { - s.extender = makePanicExtender(p) - b.toShutdown = !b.toShutdown - b.recoveredPanic = p - } - frame, lines := s.extender(nil, s.reqWidth, stat) - b.extendedLines = lines - b.frameCh <- frame - b.dlogger.Println(p) - } - s.completeFlushed = s.toComplete - }() - frame, lines := s.extender(s.draw(stat), s.reqWidth, stat) - b.extendedLines = lines - b.toShutdown = s.toComplete && !s.completeFlushed - b.frameCh <- frame - }: - case <-b.done: - s := b.cacheState - stat := newStatistics(tw, s) - var r io.Reader - if b.recoveredPanic == nil { - r = s.draw(stat) - } - frame, lines := s.extender(r, s.reqWidth, stat) - b.extendedLines = lines - b.frameCh <- frame - } -} - -func (b *Bar) subscribeDecorators() { - var averageDecorators []decor.AverageDecorator - var ewmaDecorators []decor.EwmaDecorator - var shutdownListeners []decor.ShutdownListener - b.TraverseDecorators(func(d decor.Decorator) { - if d, ok := d.(decor.AverageDecorator); ok { - averageDecorators = append(averageDecorators, d) - } - if d, ok := d.(decor.EwmaDecorator); ok { - ewmaDecorators = append(ewmaDecorators, d) - } - if d, ok := d.(decor.ShutdownListener); ok { - shutdownListeners = append(shutdownListeners, d) - } - }) - select { - case b.operateState <- func(s *bState) { - s.averageDecorators = averageDecorators - s.ewmaDecorators = ewmaDecorators - s.shutdownListeners = shutdownListeners - }: - b.hasEwmaDecorators = len(ewmaDecorators) != 0 - case <-b.done: - } -} - -func (b *Bar) refreshTillShutdown() { - for { - select { - case b.container.refreshCh <- time.Now(): - case <-b.done: - return - } - } -} - -func (b *Bar) wSyncTable() [][]chan int { - select { - case b.operateState <- func(s *bState) { b.syncTableCh <- s.wSyncTable() }: - return <-b.syncTableCh - case <-b.done: - return b.cacheState.wSyncTable() - } -} - -func (s *bState) draw(stat decor.Statistics) io.Reader { - if !s.trimSpace { - stat.AvailableWidth -= 2 - s.bufB.WriteByte(' ') - defer s.bufB.WriteByte(' ') - } - - nlr := strings.NewReader("\n") - tw := stat.AvailableWidth - for _, d := range s.pDecorators { - str := d.Decor(stat) - stat.AvailableWidth -= runewidth.StringWidth(stripansi.Strip(str)) - s.bufP.WriteString(str) - } - if stat.AvailableWidth <= 0 { - trunc := strings.NewReader(runewidth.Truncate(stripansi.Strip(s.bufP.String()), tw, "…")) - s.bufP.Reset() - return io.MultiReader(trunc, s.bufB, nlr) - } - - tw = stat.AvailableWidth - for _, d := range s.aDecorators { - str := d.Decor(stat) - stat.AvailableWidth -= runewidth.StringWidth(stripansi.Strip(str)) - s.bufA.WriteString(str) - } - if stat.AvailableWidth <= 0 { - trunc := strings.NewReader(runewidth.Truncate(stripansi.Strip(s.bufA.String()), tw, "…")) - s.bufA.Reset() - return io.MultiReader(s.bufP, s.bufB, trunc, nlr) - } - - s.filler.Fill(s.bufB, s.reqWidth, stat) - - return io.MultiReader(s.bufP, s.bufB, s.bufA, nlr) -} - -func (s *bState) wSyncTable() [][]chan int { - columns := make([]chan int, 0, len(s.pDecorators)+len(s.aDecorators)) - var pCount int - for _, d := range s.pDecorators { - if ch, ok := d.Sync(); ok { - columns = append(columns, ch) - pCount++ - } - } - var aCount int - for _, d := range s.aDecorators { - if ch, ok := d.Sync(); ok { - columns = append(columns, ch) - aCount++ - } - } - table := make([][]chan int, 2) - table[0] = columns[0:pCount] - table[1] = columns[pCount : pCount+aCount : pCount+aCount] - return table -} - -func newStatistics(tw int, s *bState) decor.Statistics { - return decor.Statistics{ - ID: s.id, - AvailableWidth: tw, - Total: s.total, - Current: s.current, - Refill: s.refill, - Completed: s.completeFlushed, - } -} - -func extractBaseDecorator(d decor.Decorator) decor.Decorator { - if d, ok := d.(decor.Wrapper); ok { - return extractBaseDecorator(d.Base()) - } - return d -} - -func ewmaIterationUpdate(done bool, s *bState, dur time.Duration) { - if !done && !s.iterated { - panic("increment required before ewma iteration update") - } else { - s.iterated = false - } - for _, d := range s.ewmaDecorators { - d.EwmaUpdate(s.lastN, dur) - } -} - -func makePanicExtender(p interface{}) extFunc { - pstr := fmt.Sprint(p) - stack := debug.Stack() - stackLines := bytes.Count(stack, []byte("\n")) - return func(_ io.Reader, _ int, st decor.Statistics) (io.Reader, int) { - mr := io.MultiReader( - strings.NewReader(runewidth.Truncate(pstr, st.AvailableWidth, "…")), - strings.NewReader(fmt.Sprintf("\n%#v\n", st)), - bytes.NewReader(stack), - ) - return mr, stackLines + 1 - } -} diff --git a/vendor/github.com/vbauerster/mpb/v5/bar_filler.go b/vendor/github.com/vbauerster/mpb/v5/bar_filler.go deleted file mode 100644 index 07148bffb..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/bar_filler.go +++ /dev/null @@ -1,30 +0,0 @@ -package mpb - -import ( - "io" - - "github.com/vbauerster/mpb/v5/decor" -) - -// BarFiller interface. -// Bar (without decorators) renders itself by calling BarFiller's Fill method. -// -// `reqWidth` is requested width, which is set via: -// func WithWidth(width int) ContainerOption -// func BarWidth(width int) BarOption -// -// Default implementations can be obtained via: -// -// func NewBarFiller(style string, reverse bool) BarFiller -// func NewSpinnerFiller(style []string, alignment SpinnerAlignment) BarFiller -// -type BarFiller interface { - Fill(w io.Writer, reqWidth int, stat decor.Statistics) -} - -// BarFillerFunc is function type adapter to convert function into BarFiller. -type BarFillerFunc func(w io.Writer, reqWidth int, stat decor.Statistics) - -func (f BarFillerFunc) Fill(w io.Writer, reqWidth int, stat decor.Statistics) { - f(w, reqWidth, stat) -} diff --git a/vendor/github.com/vbauerster/mpb/v5/bar_filler_bar.go b/vendor/github.com/vbauerster/mpb/v5/bar_filler_bar.go deleted file mode 100644 index 637bd88ca..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/bar_filler_bar.go +++ /dev/null @@ -1,173 +0,0 @@ -package mpb - -import ( - "bytes" - "io" - "unicode/utf8" - - "github.com/mattn/go-runewidth" - "github.com/vbauerster/mpb/v5/decor" - "github.com/vbauerster/mpb/v5/internal" -) - -const ( - rLeft = iota - rFill - rTip - rSpace - rRight - rRevTip - rRefill -) - -// DefaultBarStyle is a string containing 7 runes. -// Each rune is a building block of a progress bar. -// -// '1st rune' stands for left boundary rune -// -// '2nd rune' stands for fill rune -// -// '3rd rune' stands for tip rune -// -// '4th rune' stands for space rune -// -// '5th rune' stands for right boundary rune -// -// '6th rune' stands for reverse tip rune -// -// '7th rune' stands for refill rune -// -const DefaultBarStyle string = "[=>-]<+" - -type barFiller struct { - format [][]byte - rwidth []int - tip []byte - refill int64 - reverse bool - flush func(io.Writer, *space, [][]byte) -} - -type space struct { - space []byte - rwidth int - count int -} - -// NewBarFiller constucts mpb.BarFiller, to be used with *Progress.Add(...) *Bar method. -func NewBarFiller(style string, reverse bool) BarFiller { - bf := &barFiller{ - format: make([][]byte, len(DefaultBarStyle)), - rwidth: make([]int, len(DefaultBarStyle)), - reverse: reverse, - } - bf.SetStyle(style) - return bf -} - -func (s *barFiller) SetStyle(style string) { - if !utf8.ValidString(style) { - panic("invalid bar style") - } - if style == "" { - style = DefaultBarStyle - } - src := make([][]byte, utf8.RuneCountInString(style)) - i := 0 - for _, r := range style { - s.rwidth[i] = runewidth.RuneWidth(r) - src[i] = []byte(string(r)) - i++ - } - copy(s.format, src) - s.SetReverse(s.reverse) -} - -func (s *barFiller) SetReverse(reverse bool) { - if reverse { - s.tip = s.format[rRevTip] - s.flush = reverseFlush - } else { - s.tip = s.format[rTip] - s.flush = regularFlush - } - s.reverse = reverse -} - -func (s *barFiller) Fill(w io.Writer, reqWidth int, stat decor.Statistics) { - width := internal.WidthForBarFiller(reqWidth, stat.AvailableWidth) - - if brackets := s.rwidth[rLeft] + s.rwidth[rRight]; width < brackets { - return - } else { - // don't count brackets as progress - width -= brackets - } - w.Write(s.format[rLeft]) - defer w.Write(s.format[rRight]) - - cwidth := int(internal.PercentageRound(stat.Total, stat.Current, width)) - space := &space{ - space: s.format[rSpace], - rwidth: s.rwidth[rSpace], - count: width - cwidth, - } - - index, refill := 0, 0 - bb := make([][]byte, cwidth) - - if cwidth > 0 && cwidth != width { - bb[index] = s.tip - cwidth -= s.rwidth[rTip] - index++ - } - - if stat.Refill > 0 { - refill = int(internal.PercentageRound(stat.Total, int64(stat.Refill), width)) - if refill > cwidth { - refill = cwidth - } - cwidth -= refill - } - - for cwidth > 0 { - bb[index] = s.format[rFill] - cwidth -= s.rwidth[rFill] - index++ - } - - for refill > 0 { - bb[index] = s.format[rRefill] - refill -= s.rwidth[rRefill] - index++ - } - - if cwidth+refill < 0 || space.rwidth > 1 { - buf := new(bytes.Buffer) - s.flush(buf, space, bb[:index]) - io.WriteString(w, runewidth.Truncate(buf.String(), width, "…")) - return - } - - s.flush(w, space, bb) -} - -func regularFlush(w io.Writer, space *space, bb [][]byte) { - for i := len(bb) - 1; i >= 0; i-- { - w.Write(bb[i]) - } - for space.count > 0 { - w.Write(space.space) - space.count -= space.rwidth - } -} - -func reverseFlush(w io.Writer, space *space, bb [][]byte) { - for space.count > 0 { - w.Write(space.space) - space.count -= space.rwidth - } - for i := 0; i < len(bb); i++ { - w.Write(bb[i]) - } -} diff --git a/vendor/github.com/vbauerster/mpb/v5/bar_filler_spinner.go b/vendor/github.com/vbauerster/mpb/v5/bar_filler_spinner.go deleted file mode 100644 index d2cb2b726..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/bar_filler_spinner.go +++ /dev/null @@ -1,63 +0,0 @@ -package mpb - -import ( - "io" - "strings" - "unicode/utf8" - - "github.com/vbauerster/mpb/v5/decor" - "github.com/vbauerster/mpb/v5/internal" -) - -// SpinnerAlignment enum. -type SpinnerAlignment int - -// SpinnerAlignment kinds. -const ( - SpinnerOnLeft SpinnerAlignment = iota - SpinnerOnMiddle - SpinnerOnRight -) - -// DefaultSpinnerStyle is a slice of strings, which makes a spinner. -var DefaultSpinnerStyle = []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"} - -type spinnerFiller struct { - frames []string - count uint - alignment SpinnerAlignment -} - -// NewSpinnerFiller constucts mpb.BarFiller, to be used with *Progress.Add(...) *Bar method. -func NewSpinnerFiller(style []string, alignment SpinnerAlignment) BarFiller { - if len(style) == 0 { - style = DefaultSpinnerStyle - } - filler := &spinnerFiller{ - frames: style, - alignment: alignment, - } - return filler -} - -func (s *spinnerFiller) Fill(w io.Writer, reqWidth int, stat decor.Statistics) { - width := internal.WidthForBarFiller(reqWidth, stat.AvailableWidth) - - frame := s.frames[s.count%uint(len(s.frames))] - frameWidth := utf8.RuneCountInString(frame) - - if width < frameWidth { - return - } - - switch rest := width - frameWidth; s.alignment { - case SpinnerOnLeft: - io.WriteString(w, frame+strings.Repeat(" ", rest)) - case SpinnerOnMiddle: - str := strings.Repeat(" ", rest/2) + frame + strings.Repeat(" ", rest/2+rest%2) - io.WriteString(w, str) - case SpinnerOnRight: - io.WriteString(w, strings.Repeat(" ", rest)+frame) - } - s.count++ -} diff --git a/vendor/github.com/vbauerster/mpb/v5/bar_option.go b/vendor/github.com/vbauerster/mpb/v5/bar_option.go deleted file mode 100644 index e7d2e41f9..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/bar_option.go +++ /dev/null @@ -1,213 +0,0 @@ -package mpb - -import ( - "bytes" - "io" - - "github.com/vbauerster/mpb/v5/decor" -) - -// BarOption is a function option which changes the default behavior of a bar. -type BarOption func(*bState) - -func (s *bState) addDecorators(dest *[]decor.Decorator, decorators ...decor.Decorator) { - type mergeWrapper interface { - MergeUnwrap() []decor.Decorator - } - for _, decorator := range decorators { - if mw, ok := decorator.(mergeWrapper); ok { - *dest = append(*dest, mw.MergeUnwrap()...) - } - *dest = append(*dest, decorator) - } -} - -// AppendDecorators let you inject decorators to the bar's right side. -func AppendDecorators(decorators ...decor.Decorator) BarOption { - return func(s *bState) { - s.addDecorators(&s.aDecorators, decorators...) - } -} - -// PrependDecorators let you inject decorators to the bar's left side. -func PrependDecorators(decorators ...decor.Decorator) BarOption { - return func(s *bState) { - s.addDecorators(&s.pDecorators, decorators...) - } -} - -// BarID sets bar id. -func BarID(id int) BarOption { - return func(s *bState) { - s.id = id - } -} - -// BarWidth sets bar width independent of the container. -func BarWidth(width int) BarOption { - return func(s *bState) { - s.reqWidth = width - } -} - -// BarQueueAfter queues this (being constructed) bar to relplace -// runningBar after it has been completed. -func BarQueueAfter(runningBar *Bar) BarOption { - if runningBar == nil { - return nil - } - return func(s *bState) { - s.runningBar = runningBar - } -} - -// BarRemoveOnComplete removes both bar's filler and its decorators -// on complete event. -func BarRemoveOnComplete() BarOption { - return func(s *bState) { - s.dropOnComplete = true - } -} - -// BarFillerClearOnComplete clears bar's filler on complete event. -// It's shortcut for BarFillerOnComplete(""). -func BarFillerClearOnComplete() BarOption { - return BarFillerOnComplete("") -} - -// BarFillerOnComplete replaces bar's filler with message, on complete event. -func BarFillerOnComplete(message string) BarOption { - return BarFillerMiddleware(func(base BarFiller) BarFiller { - return BarFillerFunc(func(w io.Writer, reqWidth int, st decor.Statistics) { - if st.Completed { - io.WriteString(w, message) - } else { - base.Fill(w, reqWidth, st) - } - }) - }) -} - -// BarFillerMiddleware provides a way to augment default BarFiller. -func BarFillerMiddleware(middle func(BarFiller) BarFiller) BarOption { - return func(s *bState) { - s.middleware = middle - } -} - -// BarPriority sets bar's priority. Zero is highest priority, i.e. bar -// will be on top. If `BarReplaceOnComplete` option is supplied, this -// option is ignored. -func BarPriority(priority int) BarOption { - return func(s *bState) { - s.priority = priority - } -} - -// BarExtender is an option to extend bar to the next new line, with -// arbitrary output. -func BarExtender(filler BarFiller) BarOption { - if filler == nil { - return nil - } - return func(s *bState) { - s.extender = makeExtFunc(filler) - } -} - -func makeExtFunc(filler BarFiller) extFunc { - buf := new(bytes.Buffer) - return func(r io.Reader, reqWidth int, st decor.Statistics) (io.Reader, int) { - filler.Fill(buf, reqWidth, st) - return io.MultiReader(r, buf), bytes.Count(buf.Bytes(), []byte("\n")) - } -} - -// BarFillerTrim bar filler is rendered with leading and trailing space -// like ' [===] ' by default. With this option leading and trailing -// space will be removed. -func BarFillerTrim() BarOption { - return func(s *bState) { - s.trimSpace = true - } -} - -// TrimSpace is an alias to BarFillerTrim. -func TrimSpace() BarOption { - return BarFillerTrim() -} - -// BarStyle overrides mpb.DefaultBarStyle which is "[=>-]<+". -// It's ok to pass string containing just 5 runes, for example "╢▌▌░╟", -// if you don't need to override '<' (reverse tip) and '+' (refill rune). -func BarStyle(style string) BarOption { - if style == "" { - return nil - } - type styleSetter interface { - SetStyle(string) - } - return func(s *bState) { - if t, ok := s.filler.(styleSetter); ok { - t.SetStyle(style) - } - } -} - -// BarNoPop disables bar pop out of container. Effective when -// PopCompletedMode of container is enabled. -func BarNoPop() BarOption { - return func(s *bState) { - s.noPop = true - } -} - -// BarReverse reverse mode, bar will progress from right to left. -func BarReverse() BarOption { - type revSetter interface { - SetReverse(bool) - } - return func(s *bState) { - if t, ok := s.filler.(revSetter); ok { - t.SetReverse(true) - } - } -} - -// SpinnerStyle sets custom spinner style. -// Effective when Filler type is spinner. -func SpinnerStyle(frames []string) BarOption { - if len(frames) == 0 { - return nil - } - chk := func(filler BarFiller) (interface{}, bool) { - t, ok := filler.(*spinnerFiller) - return t, ok - } - cb := func(t interface{}) { - t.(*spinnerFiller).frames = frames - } - return MakeFillerTypeSpecificBarOption(chk, cb) -} - -// MakeFillerTypeSpecificBarOption makes BarOption specific to Filler's -// actual type. If you implement your own Filler, so most probably -// you'll need this. See BarStyle or SpinnerStyle for example. -func MakeFillerTypeSpecificBarOption( - typeChecker func(BarFiller) (interface{}, bool), - cb func(interface{}), -) BarOption { - return func(s *bState) { - if t, ok := typeChecker(s.filler); ok { - cb(t) - } - } -} - -// BarOptOn returns option when condition evaluates to true. -func BarOptOn(option BarOption, condition func() bool) BarOption { - if condition() { - return option - } - return nil -} diff --git a/vendor/github.com/vbauerster/mpb/v5/container_option.go b/vendor/github.com/vbauerster/mpb/v5/container_option.go deleted file mode 100644 index fac59e436..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/container_option.go +++ /dev/null @@ -1,102 +0,0 @@ -package mpb - -import ( - "io" - "io/ioutil" - "sync" - "time" -) - -// ContainerOption is a function option which changes the default -// behavior of progress container, if passed to mpb.New(...ContainerOption). -type ContainerOption func(*pState) - -// WithWaitGroup provides means to have a single joint point. If -// *sync.WaitGroup is provided, you can safely call just p.Wait() -// without calling Wait() on provided *sync.WaitGroup. Makes sense -// when there are more than one bar to render. -func WithWaitGroup(wg *sync.WaitGroup) ContainerOption { - return func(s *pState) { - s.uwg = wg - } -} - -// WithWidth sets container width. If not set underlying bars will -// occupy whole term width. -func WithWidth(width int) ContainerOption { - return func(s *pState) { - s.reqWidth = width - } -} - -// WithRefreshRate overrides default 120ms refresh rate. -func WithRefreshRate(d time.Duration) ContainerOption { - return func(s *pState) { - s.rr = d - } -} - -// WithManualRefresh disables internal auto refresh time.Ticker. -// Refresh will occur upon receive value from provided ch. -func WithManualRefresh(ch <-chan time.Time) ContainerOption { - return func(s *pState) { - s.refreshSrc = ch - } -} - -// WithRenderDelay delays rendering. By default rendering starts as -// soon as bar is added, with this option it's possible to delay -// rendering process by keeping provided chan unclosed. In other words -// rendering will start as soon as provided chan is closed. -func WithRenderDelay(ch <-chan struct{}) ContainerOption { - return func(s *pState) { - s.renderDelay = ch - } -} - -// WithShutdownNotifier provided chanel will be closed, after all bars -// have been rendered. -func WithShutdownNotifier(ch chan struct{}) ContainerOption { - return func(s *pState) { - s.shutdownNotifier = ch - } -} - -// WithOutput overrides default os.Stdout output. Setting it to nil -// will effectively disable auto refresh rate and discard any output, -// useful if you want to disable progress bars with little overhead. -func WithOutput(w io.Writer) ContainerOption { - return func(s *pState) { - if w == nil { - s.refreshSrc = make(chan time.Time) - s.output = ioutil.Discard - return - } - s.output = w - } -} - -// WithDebugOutput sets debug output. -func WithDebugOutput(w io.Writer) ContainerOption { - if w == nil { - return nil - } - return func(s *pState) { - s.debugOut = w - } -} - -// PopCompletedMode will pop and stop rendering completed bars. -func PopCompletedMode() ContainerOption { - return func(s *pState) { - s.popCompleted = true - } -} - -// ContainerOptOn returns option when condition evaluates to true. -func ContainerOptOn(option ContainerOption, condition func() bool) ContainerOption { - if condition() { - return option - } - return nil -} diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/util_bsd.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/util_bsd.go deleted file mode 100644 index 4e3564ece..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/cwriter/util_bsd.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build darwin dragonfly freebsd netbsd openbsd - -package cwriter - -import "golang.org/x/sys/unix" - -const ioctlReadTermios = unix.TIOCGETA diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/util_linux.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/util_linux.go deleted file mode 100644 index 253f12dd2..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/cwriter/util_linux.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build aix linux - -package cwriter - -import "golang.org/x/sys/unix" - -const ioctlReadTermios = unix.TCGETS diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/util_solaris.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/util_solaris.go deleted file mode 100644 index 4b29ff5c0..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/cwriter/util_solaris.go +++ /dev/null @@ -1,7 +0,0 @@ -// +build solaris - -package cwriter - -import "golang.org/x/sys/unix" - -const ioctlReadTermios = unix.TCGETA diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/writer.go deleted file mode 100644 index 6f57875c6..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer.go +++ /dev/null @@ -1,84 +0,0 @@ -package cwriter - -import ( - "bytes" - "errors" - "io" - "os" - "strconv" -) - -// NotATTY not a TeleTYpewriter error. -var NotATTY = errors.New("not a terminal") - -// http://ascii-table.com/ansi-escape-sequences.php -const ( - escOpen = "\x1b[" - cuuAndEd = "A\x1b[J" -) - -// Writer is a buffered the writer that updates the terminal. The -// contents of writer will be flushed when Flush is called. -type Writer struct { - out io.Writer - buf bytes.Buffer - lineCount int - fd int - isTerminal bool -} - -// New returns a new Writer with defaults. -func New(out io.Writer) *Writer { - w := &Writer{out: out} - if f, ok := out.(*os.File); ok { - w.fd = int(f.Fd()) - w.isTerminal = IsTerminal(w.fd) - } - return w -} - -// Flush flushes the underlying buffer. -func (w *Writer) Flush(lineCount int) (err error) { - // some terminals interpret clear 0 lines as clear 1 - if w.lineCount > 0 { - err = w.clearLines() - if err != nil { - return - } - } - w.lineCount = lineCount - _, err = w.buf.WriteTo(w.out) - return -} - -// Write appends the contents of p to the underlying buffer. -func (w *Writer) Write(p []byte) (n int, err error) { - return w.buf.Write(p) -} - -// WriteString writes string to the underlying buffer. -func (w *Writer) WriteString(s string) (n int, err error) { - return w.buf.WriteString(s) -} - -// ReadFrom reads from the provided io.Reader and writes to the -// underlying buffer. -func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) { - return w.buf.ReadFrom(r) -} - -// GetWidth returns width of underlying terminal. -func (w *Writer) GetWidth() (int, error) { - if !w.isTerminal { - return -1, NotATTY - } - tw, _, err := GetSize(w.fd) - return tw, err -} - -func (w *Writer) ansiCuuAndEd() (err error) { - buf := make([]byte, 8) - buf = strconv.AppendInt(buf[:copy(buf, escOpen)], int64(w.lineCount), 10) - _, err = w.out.Write(append(buf, cuuAndEd...)) - return -} diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_posix.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_posix.go deleted file mode 100644 index f54a5d06b..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_posix.go +++ /dev/null @@ -1,26 +0,0 @@ -// +build !windows - -package cwriter - -import ( - "golang.org/x/sys/unix" -) - -func (w *Writer) clearLines() error { - return w.ansiCuuAndEd() -} - -// GetSize returns the dimensions of the given terminal. -func GetSize(fd int) (width, height int, err error) { - ws, err := unix.IoctlGetWinsize(fd, unix.TIOCGWINSZ) - if err != nil { - return -1, -1, err - } - return int(ws.Col), int(ws.Row), nil -} - -// IsTerminal returns whether the given file descriptor is a terminal. -func IsTerminal(fd int) bool { - _, err := unix.IoctlGetTermios(fd, ioctlReadTermios) - return err == nil -} diff --git a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_windows.go b/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_windows.go deleted file mode 100644 index 1a69c81ac..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/cwriter/writer_windows.go +++ /dev/null @@ -1,73 +0,0 @@ -// +build windows - -package cwriter - -import ( - "unsafe" - - "golang.org/x/sys/windows" -) - -var kernel32 = windows.NewLazySystemDLL("kernel32.dll") - -var ( - procSetConsoleCursorPosition = kernel32.NewProc("SetConsoleCursorPosition") - procFillConsoleOutputCharacter = kernel32.NewProc("FillConsoleOutputCharacterW") -) - -func (w *Writer) clearLines() error { - if !w.isTerminal { - // hope it's cygwin or similar - return w.ansiCuuAndEd() - } - - var info windows.ConsoleScreenBufferInfo - if err := windows.GetConsoleScreenBufferInfo(windows.Handle(w.fd), &info); err != nil { - return err - } - - info.CursorPosition.Y -= int16(w.lineCount) - if info.CursorPosition.Y < 0 { - info.CursorPosition.Y = 0 - } - _, _, _ = procSetConsoleCursorPosition.Call( - uintptr(w.fd), - uintptr(uint32(uint16(info.CursorPosition.Y))<<16|uint32(uint16(info.CursorPosition.X))), - ) - - // clear the lines - cursor := &windows.Coord{ - X: info.Window.Left, - Y: info.CursorPosition.Y, - } - count := uint32(info.Size.X) * uint32(w.lineCount) - _, _, _ = procFillConsoleOutputCharacter.Call( - uintptr(w.fd), - uintptr(' '), - uintptr(count), - *(*uintptr)(unsafe.Pointer(cursor)), - uintptr(unsafe.Pointer(new(uint32))), - ) - return nil -} - -// GetSize returns the visible dimensions of the given terminal. -// -// These dimensions don't include any scrollback buffer height. -func GetSize(fd int) (width, height int, err error) { - var info windows.ConsoleScreenBufferInfo - if err := windows.GetConsoleScreenBufferInfo(windows.Handle(fd), &info); err != nil { - return 0, 0, err - } - // terminal.GetSize from crypto/ssh adds "+ 1" to both width and height: - // https://go.googlesource.com/crypto/+/refs/heads/release-branch.go1.14/ssh/terminal/util_windows.go#75 - // but looks like this is a root cause of issue #66, so removing both "+ 1" have fixed it. - return int(info.Window.Right - info.Window.Left), int(info.Window.Bottom - info.Window.Top), nil -} - -// IsTerminal returns whether the given file descriptor is a terminal. -func IsTerminal(fd int) bool { - var st uint32 - err := windows.GetConsoleMode(windows.Handle(fd), &st) - return err == nil -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/any.go b/vendor/github.com/vbauerster/mpb/v5/decor/any.go deleted file mode 100644 index 39518f594..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/any.go +++ /dev/null @@ -1,21 +0,0 @@ -package decor - -// Any decorator displays text, that can be changed during decorator's -// lifetime via provided DecorFunc. -// -// `fn` DecorFunc callback -// -// `wcc` optional WC config -// -func Any(fn DecorFunc, wcc ...WC) Decorator { - return &any{initWC(wcc...), fn} -} - -type any struct { - WC - fn DecorFunc -} - -func (d *any) Decor(s Statistics) string { - return d.FormatMsg(d.fn(s)) -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/counters.go b/vendor/github.com/vbauerster/mpb/v5/decor/counters.go deleted file mode 100644 index 4a5343d41..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/counters.go +++ /dev/null @@ -1,243 +0,0 @@ -package decor - -import ( - "fmt" - "strings" -) - -const ( - _ = iota - UnitKiB - UnitKB -) - -// CountersNoUnit is a wrapper around Counters with no unit param. -func CountersNoUnit(pairFmt string, wcc ...WC) Decorator { - return Counters(0, pairFmt, wcc...) -} - -// CountersKibiByte is a wrapper around Counters with predefined unit -// UnitKiB (bytes/1024). -func CountersKibiByte(pairFmt string, wcc ...WC) Decorator { - return Counters(UnitKiB, pairFmt, wcc...) -} - -// CountersKiloByte is a wrapper around Counters with predefined unit -// UnitKB (bytes/1000). -func CountersKiloByte(pairFmt string, wcc ...WC) Decorator { - return Counters(UnitKB, pairFmt, wcc...) -} - -// Counters decorator with dynamic unit measure adjustment. -// -// `unit` one of [0|UnitKiB|UnitKB] zero for no unit -// -// `pairFmt` printf compatible verbs for current and total pair -// -// `wcc` optional WC config -// -// pairFmt example if unit=UnitKB: -// -// pairFmt="%.1f / %.1f" output: "1.0MB / 12.0MB" -// pairFmt="% .1f / % .1f" output: "1.0 MB / 12.0 MB" -// pairFmt="%d / %d" output: "1MB / 12MB" -// pairFmt="% d / % d" output: "1 MB / 12 MB" -// -func Counters(unit int, pairFmt string, wcc ...WC) Decorator { - producer := func(unit int, pairFmt string) DecorFunc { - if pairFmt == "" { - pairFmt = "%d / %d" - } else if strings.Count(pairFmt, "%") != 2 { - panic("expected pairFmt with exactly 2 verbs") - } - switch unit { - case UnitKiB: - return func(s Statistics) string { - return fmt.Sprintf(pairFmt, SizeB1024(s.Current), SizeB1024(s.Total)) - } - case UnitKB: - return func(s Statistics) string { - return fmt.Sprintf(pairFmt, SizeB1000(s.Current), SizeB1000(s.Total)) - } - default: - return func(s Statistics) string { - return fmt.Sprintf(pairFmt, s.Current, s.Total) - } - } - } - return Any(producer(unit, pairFmt), wcc...) -} - -// TotalNoUnit is a wrapper around Total with no unit param. -func TotalNoUnit(format string, wcc ...WC) Decorator { - return Total(0, format, wcc...) -} - -// TotalKibiByte is a wrapper around Total with predefined unit -// UnitKiB (bytes/1024). -func TotalKibiByte(format string, wcc ...WC) Decorator { - return Total(UnitKiB, format, wcc...) -} - -// TotalKiloByte is a wrapper around Total with predefined unit -// UnitKB (bytes/1000). -func TotalKiloByte(format string, wcc ...WC) Decorator { - return Total(UnitKB, format, wcc...) -} - -// Total decorator with dynamic unit measure adjustment. -// -// `unit` one of [0|UnitKiB|UnitKB] zero for no unit -// -// `format` printf compatible verb for Total -// -// `wcc` optional WC config -// -// format example if unit=UnitKiB: -// -// format="%.1f" output: "12.0MiB" -// format="% .1f" output: "12.0 MiB" -// format="%d" output: "12MiB" -// format="% d" output: "12 MiB" -// -func Total(unit int, format string, wcc ...WC) Decorator { - producer := func(unit int, format string) DecorFunc { - if format == "" { - format = "%d" - } else if strings.Count(format, "%") != 1 { - panic("expected format with exactly 1 verb") - } - - switch unit { - case UnitKiB: - return func(s Statistics) string { - return fmt.Sprintf(format, SizeB1024(s.Total)) - } - case UnitKB: - return func(s Statistics) string { - return fmt.Sprintf(format, SizeB1000(s.Total)) - } - default: - return func(s Statistics) string { - return fmt.Sprintf(format, s.Total) - } - } - } - return Any(producer(unit, format), wcc...) -} - -// CurrentNoUnit is a wrapper around Current with no unit param. -func CurrentNoUnit(format string, wcc ...WC) Decorator { - return Current(0, format, wcc...) -} - -// CurrentKibiByte is a wrapper around Current with predefined unit -// UnitKiB (bytes/1024). -func CurrentKibiByte(format string, wcc ...WC) Decorator { - return Current(UnitKiB, format, wcc...) -} - -// CurrentKiloByte is a wrapper around Current with predefined unit -// UnitKB (bytes/1000). -func CurrentKiloByte(format string, wcc ...WC) Decorator { - return Current(UnitKB, format, wcc...) -} - -// Current decorator with dynamic unit measure adjustment. -// -// `unit` one of [0|UnitKiB|UnitKB] zero for no unit -// -// `format` printf compatible verb for Current -// -// `wcc` optional WC config -// -// format example if unit=UnitKiB: -// -// format="%.1f" output: "12.0MiB" -// format="% .1f" output: "12.0 MiB" -// format="%d" output: "12MiB" -// format="% d" output: "12 MiB" -// -func Current(unit int, format string, wcc ...WC) Decorator { - producer := func(unit int, format string) DecorFunc { - if format == "" { - format = "%d" - } else if strings.Count(format, "%") != 1 { - panic("expected format with exactly 1 verb") - } - - switch unit { - case UnitKiB: - return func(s Statistics) string { - return fmt.Sprintf(format, SizeB1024(s.Current)) - } - case UnitKB: - return func(s Statistics) string { - return fmt.Sprintf(format, SizeB1000(s.Current)) - } - default: - return func(s Statistics) string { - return fmt.Sprintf(format, s.Current) - } - } - } - return Any(producer(unit, format), wcc...) -} - -// InvertedCurrentNoUnit is a wrapper around InvertedCurrent with no unit param. -func InvertedCurrentNoUnit(format string, wcc ...WC) Decorator { - return InvertedCurrent(0, format, wcc...) -} - -// InvertedCurrentKibiByte is a wrapper around InvertedCurrent with predefined unit -// UnitKiB (bytes/1024). -func InvertedCurrentKibiByte(format string, wcc ...WC) Decorator { - return InvertedCurrent(UnitKiB, format, wcc...) -} - -// InvertedCurrentKiloByte is a wrapper around InvertedCurrent with predefined unit -// UnitKB (bytes/1000). -func InvertedCurrentKiloByte(format string, wcc ...WC) Decorator { - return InvertedCurrent(UnitKB, format, wcc...) -} - -// InvertedCurrent decorator with dynamic unit measure adjustment. -// -// `unit` one of [0|UnitKiB|UnitKB] zero for no unit -// -// `format` printf compatible verb for InvertedCurrent -// -// `wcc` optional WC config -// -// format example if unit=UnitKiB: -// -// format="%.1f" output: "12.0MiB" -// format="% .1f" output: "12.0 MiB" -// format="%d" output: "12MiB" -// format="% d" output: "12 MiB" -// -func InvertedCurrent(unit int, format string, wcc ...WC) Decorator { - producer := func(unit int, format string) DecorFunc { - if format == "" { - format = "%d" - } else if strings.Count(format, "%") != 1 { - panic("expected format with exactly 1 verb") - } - - switch unit { - case UnitKiB: - return func(s Statistics) string { - return fmt.Sprintf(format, SizeB1024(s.Total-s.Current)) - } - case UnitKB: - return func(s Statistics) string { - return fmt.Sprintf(format, SizeB1000(s.Total-s.Current)) - } - default: - return func(s Statistics) string { - return fmt.Sprintf(format, s.Total-s.Current) - } - } - } - return Any(producer(unit, format), wcc...) -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/decorator.go b/vendor/github.com/vbauerster/mpb/v5/decor/decorator.go deleted file mode 100644 index e81fae367..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/decorator.go +++ /dev/null @@ -1,191 +0,0 @@ -package decor - -import ( - "fmt" - "time" - - "github.com/acarl005/stripansi" - "github.com/mattn/go-runewidth" -) - -const ( - // DidentRight bit specifies identation direction. - // |foo |b | With DidentRight - // | foo| b| Without DidentRight - DidentRight = 1 << iota - - // DextraSpace bit adds extra space, makes sense with DSyncWidth only. - // When DidentRight bit set, the space will be added to the right, - // otherwise to the left. - DextraSpace - - // DSyncWidth bit enables same column width synchronization. - // Effective with multiple bars only. - DSyncWidth - - // DSyncWidthR is shortcut for DSyncWidth|DidentRight - DSyncWidthR = DSyncWidth | DidentRight - - // DSyncSpace is shortcut for DSyncWidth|DextraSpace - DSyncSpace = DSyncWidth | DextraSpace - - // DSyncSpaceR is shortcut for DSyncWidth|DextraSpace|DidentRight - DSyncSpaceR = DSyncWidth | DextraSpace | DidentRight -) - -// TimeStyle enum. -type TimeStyle int - -// TimeStyle kinds. -const ( - ET_STYLE_GO TimeStyle = iota - ET_STYLE_HHMMSS - ET_STYLE_HHMM - ET_STYLE_MMSS -) - -// Statistics consists of progress related statistics, that Decorator -// may need. -type Statistics struct { - ID int - AvailableWidth int - Total int64 - Current int64 - Refill int64 - Completed bool -} - -// Decorator interface. -// Most of the time there is no need to implement this interface -// manually, as decor package already provides a wide range of decorators -// which implement this interface. If however built-in decorators don't -// meet your needs, you're free to implement your own one by implementing -// this particular interface. The easy way to go is to convert a -// `DecorFunc` into a `Decorator` interface by using provided -// `func Any(DecorFunc, ...WC) Decorator`. -type Decorator interface { - Configurator - Synchronizer - Decor(Statistics) string -} - -// DecorFunc func type. -// To be used with `func Any`(DecorFunc, ...WC) Decorator`. -type DecorFunc func(Statistics) string - -// Synchronizer interface. -// All decorators implement this interface implicitly. Its Sync -// method exposes width sync channel, if DSyncWidth bit is set. -type Synchronizer interface { - Sync() (chan int, bool) -} - -// Configurator interface. -type Configurator interface { - GetConf() WC - SetConf(WC) -} - -// Wrapper interface. -// If you're implementing custom Decorator by wrapping a built-in one, -// it is necessary to implement this interface to retain functionality -// of built-in Decorator. -type Wrapper interface { - Base() Decorator -} - -// EwmaDecorator interface. -// EWMA based decorators should implement this one. -type EwmaDecorator interface { - EwmaUpdate(int64, time.Duration) -} - -// AverageDecorator interface. -// Average decorators should implement this interface to provide start -// time adjustment facility, for resume-able tasks. -type AverageDecorator interface { - AverageAdjust(time.Time) -} - -// ShutdownListener interface. -// If decorator needs to be notified once upon bar shutdown event, so -// this is the right interface to implement. -type ShutdownListener interface { - Shutdown() -} - -// Global convenience instances of WC with sync width bit set. -// To be used with multiple bars only, i.e. not effective for single bar usage. -var ( - WCSyncWidth = WC{C: DSyncWidth} - WCSyncWidthR = WC{C: DSyncWidthR} - WCSyncSpace = WC{C: DSyncSpace} - WCSyncSpaceR = WC{C: DSyncSpaceR} -) - -// WC is a struct with two public fields W and C, both of int type. -// W represents width and C represents bit set of width related config. -// A decorator should embed WC, to enable width synchronization. -type WC struct { - W int - C int - fill func(s string, w int) string - wsync chan int -} - -// FormatMsg formats final message according to WC.W and WC.C. -// Should be called by any Decorator implementation. -func (wc *WC) FormatMsg(msg string) string { - pureWidth := runewidth.StringWidth(msg) - stripWidth := runewidth.StringWidth(stripansi.Strip(msg)) - maxCell := wc.W - if (wc.C & DSyncWidth) != 0 { - cellCount := stripWidth - if (wc.C & DextraSpace) != 0 { - cellCount++ - } - wc.wsync <- cellCount - maxCell = <-wc.wsync - } - return wc.fill(msg, maxCell+(pureWidth-stripWidth)) -} - -// Init initializes width related config. -func (wc *WC) Init() WC { - wc.fill = runewidth.FillLeft - if (wc.C & DidentRight) != 0 { - wc.fill = runewidth.FillRight - } - if (wc.C & DSyncWidth) != 0 { - // it's deliberate choice to override wsync on each Init() call, - // this way globals like WCSyncSpace can be reused - wc.wsync = make(chan int) - } - return *wc -} - -// Sync is implementation of Synchronizer interface. -func (wc *WC) Sync() (chan int, bool) { - if (wc.C&DSyncWidth) != 0 && wc.wsync == nil { - panic(fmt.Sprintf("%T is not initialized", wc)) - } - return wc.wsync, (wc.C & DSyncWidth) != 0 -} - -// GetConf is implementation of Configurator interface. -func (wc *WC) GetConf() WC { - return *wc -} - -// SetConf is implementation of Configurator interface. -func (wc *WC) SetConf(conf WC) { - *wc = conf.Init() -} - -func initWC(wcc ...WC) WC { - var wc WC - for _, nwc := range wcc { - wc = nwc - } - return wc.Init() -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/doc.go b/vendor/github.com/vbauerster/mpb/v5/decor/doc.go deleted file mode 100644 index 6d2614451..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/doc.go +++ /dev/null @@ -1,21 +0,0 @@ -/* - Package decor provides common decorators for "github.com/vbauerster/mpb/v5" module. - - Some decorators returned by this package might have a closure state. It is ok to use - decorators concurrently, unless you share the same decorator among multiple - *mpb.Bar instances. To avoid data races, create new decorator per *mpb.Bar instance. - - Don't: - - p := mpb.New() - name := decor.Name("bar") - p.AddBar(100, mpb.AppendDecorators(name)) - p.AddBar(100, mpb.AppendDecorators(name)) - - Do: - - p := mpb.New() - p.AddBar(100, mpb.AppendDecorators(decor.Name("bar1"))) - p.AddBar(100, mpb.AppendDecorators(decor.Name("bar2"))) -*/ -package decor diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/elapsed.go b/vendor/github.com/vbauerster/mpb/v5/decor/elapsed.go deleted file mode 100644 index e389f1581..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/elapsed.go +++ /dev/null @@ -1,35 +0,0 @@ -package decor - -import ( - "time" -) - -// Elapsed decorator. It's wrapper of NewElapsed. -// -// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] -// -// `wcc` optional WC config -// -func Elapsed(style TimeStyle, wcc ...WC) Decorator { - return NewElapsed(style, time.Now(), wcc...) -} - -// NewElapsed returns elapsed time decorator. -// -// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] -// -// `startTime` start time -// -// `wcc` optional WC config -// -func NewElapsed(style TimeStyle, startTime time.Time, wcc ...WC) Decorator { - var msg string - producer := chooseTimeProducer(style) - fn := func(s Statistics) string { - if !s.Completed { - msg = producer(time.Since(startTime)) - } - return msg - } - return Any(fn, wcc...) -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/eta.go b/vendor/github.com/vbauerster/mpb/v5/decor/eta.go deleted file mode 100644 index d03caa735..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/eta.go +++ /dev/null @@ -1,203 +0,0 @@ -package decor - -import ( - "fmt" - "math" - "time" - - "github.com/VividCortex/ewma" -) - -// TimeNormalizer interface. Implementors could be passed into -// MovingAverageETA, in order to affect i.e. normalize its output. -type TimeNormalizer interface { - Normalize(time.Duration) time.Duration -} - -// TimeNormalizerFunc is function type adapter to convert function -// into TimeNormalizer. -type TimeNormalizerFunc func(time.Duration) time.Duration - -func (f TimeNormalizerFunc) Normalize(src time.Duration) time.Duration { - return f(src) -} - -// EwmaETA exponential-weighted-moving-average based ETA decorator. -// For this decorator to work correctly you have to measure each -// iteration's duration and pass it to the -// *Bar.DecoratorEwmaUpdate(time.Duration) method after each increment. -func EwmaETA(style TimeStyle, age float64, wcc ...WC) Decorator { - var average ewma.MovingAverage - if age == 0 { - average = ewma.NewMovingAverage() - } else { - average = ewma.NewMovingAverage(age) - } - return MovingAverageETA(style, NewThreadSafeMovingAverage(average), nil, wcc...) -} - -// MovingAverageETA decorator relies on MovingAverage implementation to calculate its average. -// -// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] -// -// `average` implementation of MovingAverage interface -// -// `normalizer` available implementations are [FixedIntervalTimeNormalizer|MaxTolerateTimeNormalizer] -// -// `wcc` optional WC config -// -func MovingAverageETA(style TimeStyle, average ewma.MovingAverage, normalizer TimeNormalizer, wcc ...WC) Decorator { - d := &movingAverageETA{ - WC: initWC(wcc...), - average: average, - normalizer: normalizer, - producer: chooseTimeProducer(style), - } - return d -} - -type movingAverageETA struct { - WC - average ewma.MovingAverage - normalizer TimeNormalizer - producer func(time.Duration) string -} - -func (d *movingAverageETA) Decor(s Statistics) string { - v := math.Round(d.average.Value()) - remaining := time.Duration((s.Total - s.Current) * int64(v)) - if d.normalizer != nil { - remaining = d.normalizer.Normalize(remaining) - } - return d.FormatMsg(d.producer(remaining)) -} - -func (d *movingAverageETA) EwmaUpdate(n int64, dur time.Duration) { - durPerItem := float64(dur) / float64(n) - if math.IsInf(durPerItem, 0) || math.IsNaN(durPerItem) { - return - } - d.average.Add(durPerItem) -} - -// AverageETA decorator. It's wrapper of NewAverageETA. -// -// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] -// -// `wcc` optional WC config -// -func AverageETA(style TimeStyle, wcc ...WC) Decorator { - return NewAverageETA(style, time.Now(), nil, wcc...) -} - -// NewAverageETA decorator with user provided start time. -// -// `style` one of [ET_STYLE_GO|ET_STYLE_HHMMSS|ET_STYLE_HHMM|ET_STYLE_MMSS] -// -// `startTime` start time -// -// `normalizer` available implementations are [FixedIntervalTimeNormalizer|MaxTolerateTimeNormalizer] -// -// `wcc` optional WC config -// -func NewAverageETA(style TimeStyle, startTime time.Time, normalizer TimeNormalizer, wcc ...WC) Decorator { - d := &averageETA{ - WC: initWC(wcc...), - startTime: startTime, - normalizer: normalizer, - producer: chooseTimeProducer(style), - } - return d -} - -type averageETA struct { - WC - startTime time.Time - normalizer TimeNormalizer - producer func(time.Duration) string -} - -func (d *averageETA) Decor(s Statistics) string { - var remaining time.Duration - if s.Current != 0 { - durPerItem := float64(time.Since(d.startTime)) / float64(s.Current) - durPerItem = math.Round(durPerItem) - remaining = time.Duration((s.Total - s.Current) * int64(durPerItem)) - if d.normalizer != nil { - remaining = d.normalizer.Normalize(remaining) - } - } - return d.FormatMsg(d.producer(remaining)) -} - -func (d *averageETA) AverageAdjust(startTime time.Time) { - d.startTime = startTime -} - -// MaxTolerateTimeNormalizer returns implementation of TimeNormalizer. -func MaxTolerateTimeNormalizer(maxTolerate time.Duration) TimeNormalizer { - var normalized time.Duration - var lastCall time.Time - return TimeNormalizerFunc(func(remaining time.Duration) time.Duration { - if diff := normalized - remaining; diff <= 0 || diff > maxTolerate || remaining < time.Minute { - normalized = remaining - lastCall = time.Now() - return remaining - } - normalized -= time.Since(lastCall) - lastCall = time.Now() - return normalized - }) -} - -// FixedIntervalTimeNormalizer returns implementation of TimeNormalizer. -func FixedIntervalTimeNormalizer(updInterval int) TimeNormalizer { - var normalized time.Duration - var lastCall time.Time - var count int - return TimeNormalizerFunc(func(remaining time.Duration) time.Duration { - if count == 0 || remaining < time.Minute { - count = updInterval - normalized = remaining - lastCall = time.Now() - return remaining - } - count-- - normalized -= time.Since(lastCall) - lastCall = time.Now() - return normalized - }) -} - -func chooseTimeProducer(style TimeStyle) func(time.Duration) string { - switch style { - case ET_STYLE_HHMMSS: - return func(remaining time.Duration) string { - hours := int64(remaining/time.Hour) % 60 - minutes := int64(remaining/time.Minute) % 60 - seconds := int64(remaining/time.Second) % 60 - return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) - } - case ET_STYLE_HHMM: - return func(remaining time.Duration) string { - hours := int64(remaining/time.Hour) % 60 - minutes := int64(remaining/time.Minute) % 60 - return fmt.Sprintf("%02d:%02d", hours, minutes) - } - case ET_STYLE_MMSS: - return func(remaining time.Duration) string { - hours := int64(remaining/time.Hour) % 60 - minutes := int64(remaining/time.Minute) % 60 - seconds := int64(remaining/time.Second) % 60 - if hours > 0 { - return fmt.Sprintf("%02d:%02d:%02d", hours, minutes, seconds) - } - return fmt.Sprintf("%02d:%02d", minutes, seconds) - } - default: - return func(remaining time.Duration) string { - // strip off nanoseconds - return ((remaining / time.Second) * time.Second).String() - } - } -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/merge.go b/vendor/github.com/vbauerster/mpb/v5/decor/merge.go deleted file mode 100644 index e41406a64..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/merge.go +++ /dev/null @@ -1,107 +0,0 @@ -package decor - -import ( - "strings" - - "github.com/acarl005/stripansi" - "github.com/mattn/go-runewidth" -) - -// Merge wraps its decorator argument with intention to sync width -// with several decorators of another bar. Visual example: -// -// +----+--------+---------+--------+ -// | B1 | MERGE(D, P1, Pn) | -// +----+--------+---------+--------+ -// | B2 | D0 | D1 | Dn | -// +----+--------+---------+--------+ -// -func Merge(decorator Decorator, placeholders ...WC) Decorator { - if _, ok := decorator.Sync(); !ok || len(placeholders) == 0 { - return decorator - } - md := &mergeDecorator{ - Decorator: decorator, - wc: decorator.GetConf(), - placeHolders: make([]*placeHolderDecorator, len(placeholders)), - } - decorator.SetConf(WC{}) - for i, wc := range placeholders { - if (wc.C & DSyncWidth) == 0 { - return decorator - } - md.placeHolders[i] = &placeHolderDecorator{wc.Init()} - } - return md -} - -type mergeDecorator struct { - Decorator - wc WC - placeHolders []*placeHolderDecorator -} - -func (d *mergeDecorator) GetConf() WC { - return d.wc -} - -func (d *mergeDecorator) SetConf(conf WC) { - d.wc = conf.Init() -} - -func (d *mergeDecorator) MergeUnwrap() []Decorator { - decorators := make([]Decorator, len(d.placeHolders)) - for i, ph := range d.placeHolders { - decorators[i] = ph - } - return decorators -} - -func (d *mergeDecorator) Sync() (chan int, bool) { - return d.wc.Sync() -} - -func (d *mergeDecorator) Base() Decorator { - return d.Decorator -} - -func (d *mergeDecorator) Decor(s Statistics) string { - msg := d.Decorator.Decor(s) - pureWidth := runewidth.StringWidth(msg) - stripWidth := runewidth.StringWidth(stripansi.Strip(msg)) - cellCount := stripWidth - if (d.wc.C & DextraSpace) != 0 { - cellCount++ - } - - total := runewidth.StringWidth(d.placeHolders[0].FormatMsg("")) - pw := (cellCount - total) / len(d.placeHolders) - rem := (cellCount - total) % len(d.placeHolders) - - var diff int - for i := 1; i < len(d.placeHolders); i++ { - ph := d.placeHolders[i] - width := pw - diff - if (ph.WC.C & DextraSpace) != 0 { - width-- - if width < 0 { - width = 0 - } - } - max := runewidth.StringWidth(ph.FormatMsg(strings.Repeat(" ", width))) - total += max - diff = max - pw - } - - d.wc.wsync <- pw + rem - max := <-d.wc.wsync - return d.wc.fill(msg, max+total+(pureWidth-stripWidth)) -} - -type placeHolderDecorator struct { - WC -} - -func (d *placeHolderDecorator) Decor(Statistics) string { - return "" -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/moving_average.go b/vendor/github.com/vbauerster/mpb/v5/decor/moving_average.go deleted file mode 100644 index 50ac9c393..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/moving_average.go +++ /dev/null @@ -1,68 +0,0 @@ -package decor - -import ( - "sort" - "sync" - - "github.com/VividCortex/ewma" -) - -type threadSafeMovingAverage struct { - ewma.MovingAverage - mu sync.Mutex -} - -func (s *threadSafeMovingAverage) Add(value float64) { - s.mu.Lock() - s.MovingAverage.Add(value) - s.mu.Unlock() -} - -func (s *threadSafeMovingAverage) Value() float64 { - s.mu.Lock() - defer s.mu.Unlock() - return s.MovingAverage.Value() -} - -func (s *threadSafeMovingAverage) Set(value float64) { - s.mu.Lock() - s.MovingAverage.Set(value) - s.mu.Unlock() -} - -// NewThreadSafeMovingAverage converts provided ewma.MovingAverage -// into thread safe ewma.MovingAverage. -func NewThreadSafeMovingAverage(average ewma.MovingAverage) ewma.MovingAverage { - if tsma, ok := average.(*threadSafeMovingAverage); ok { - return tsma - } - return &threadSafeMovingAverage{MovingAverage: average} -} - -type medianWindow [3]float64 - -func (s *medianWindow) Len() int { return len(s) } -func (s *medianWindow) Swap(i, j int) { s[i], s[j] = s[j], s[i] } -func (s *medianWindow) Less(i, j int) bool { return s[i] < s[j] } - -func (s *medianWindow) Add(value float64) { - s[0], s[1] = s[1], s[2] - s[2] = value -} - -func (s *medianWindow) Value() float64 { - tmp := *s - sort.Sort(&tmp) - return tmp[1] -} - -func (s *medianWindow) Set(value float64) { - for i := 0; i < len(s); i++ { - s[i] = value - } -} - -// NewMedian is fixed last 3 samples median MovingAverage. -func NewMedian() ewma.MovingAverage { - return NewThreadSafeMovingAverage(new(medianWindow)) -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/name.go b/vendor/github.com/vbauerster/mpb/v5/decor/name.go deleted file mode 100644 index 3af311254..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/name.go +++ /dev/null @@ -1,12 +0,0 @@ -package decor - -// Name decorator displays text that is set once and can't be changed -// during decorator's lifetime. -// -// `str` string to display -// -// `wcc` optional WC config -// -func Name(str string, wcc ...WC) Decorator { - return Any(func(Statistics) string { return str }, wcc...) -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/on_complete.go b/vendor/github.com/vbauerster/mpb/v5/decor/on_complete.go deleted file mode 100644 index f46b19aba..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/on_complete.go +++ /dev/null @@ -1,37 +0,0 @@ -package decor - -// OnComplete returns decorator, which wraps provided decorator, with -// sole purpose to display provided message on complete event. -// -// `decorator` Decorator to wrap -// -// `message` message to display on complete event -// -func OnComplete(decorator Decorator, message string) Decorator { - d := &onCompleteWrapper{ - Decorator: decorator, - msg: message, - } - if md, ok := decorator.(*mergeDecorator); ok { - d.Decorator, md.Decorator = md.Decorator, d - return md - } - return d -} - -type onCompleteWrapper struct { - Decorator - msg string -} - -func (d *onCompleteWrapper) Decor(s Statistics) string { - if s.Completed { - wc := d.GetConf() - return wc.FormatMsg(d.msg) - } - return d.Decorator.Decor(s) -} - -func (d *onCompleteWrapper) Base() Decorator { - return d.Decorator -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/percentage.go b/vendor/github.com/vbauerster/mpb/v5/decor/percentage.go deleted file mode 100644 index d6314a619..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/percentage.go +++ /dev/null @@ -1,58 +0,0 @@ -package decor - -import ( - "fmt" - "io" - "strconv" - - "github.com/vbauerster/mpb/v5/internal" -) - -type percentageType float64 - -func (s percentageType) Format(st fmt.State, verb rune) { - var prec int - switch verb { - case 'd': - case 's': - prec = -1 - default: - if p, ok := st.Precision(); ok { - prec = p - } else { - prec = 6 - } - } - - io.WriteString(st, strconv.FormatFloat(float64(s), 'f', prec, 64)) - - if st.Flag(' ') { - io.WriteString(st, " ") - } - io.WriteString(st, "%") -} - -// Percentage returns percentage decorator. It's a wrapper of NewPercentage. -func Percentage(wcc ...WC) Decorator { - return NewPercentage("% d", wcc...) -} - -// NewPercentage percentage decorator with custom format string. -// -// format examples: -// -// format="%.1f" output: "1.0%" -// format="% .1f" output: "1.0 %" -// format="%d" output: "1%" -// format="% d" output: "1 %" -// -func NewPercentage(format string, wcc ...WC) Decorator { - if format == "" { - format = "% d" - } - f := func(s Statistics) string { - p := internal.Percentage(s.Total, s.Current, 100) - return fmt.Sprintf(format, percentageType(p)) - } - return Any(f, wcc...) -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/size_type.go b/vendor/github.com/vbauerster/mpb/v5/decor/size_type.go deleted file mode 100644 index e4b974058..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/size_type.go +++ /dev/null @@ -1,109 +0,0 @@ -package decor - -import ( - "fmt" - "io" - "math" - "strconv" -) - -//go:generate stringer -type=SizeB1024 -trimprefix=_i -//go:generate stringer -type=SizeB1000 -trimprefix=_ - -const ( - _ib SizeB1024 = iota + 1 - _iKiB SizeB1024 = 1 << (iota * 10) - _iMiB - _iGiB - _iTiB -) - -// SizeB1024 named type, which implements fmt.Formatter interface. It -// adjusts its value according to byte size multiple by 1024 and appends -// appropriate size marker (KiB, MiB, GiB, TiB). -type SizeB1024 int64 - -func (self SizeB1024) Format(st fmt.State, verb rune) { - var prec int - switch verb { - case 'd': - case 's': - prec = -1 - default: - if p, ok := st.Precision(); ok { - prec = p - } else { - prec = 6 - } - } - - var unit SizeB1024 - switch { - case self < _iKiB: - unit = _ib - case self < _iMiB: - unit = _iKiB - case self < _iGiB: - unit = _iMiB - case self < _iTiB: - unit = _iGiB - case self <= math.MaxInt64: - unit = _iTiB - } - - io.WriteString(st, strconv.FormatFloat(float64(self)/float64(unit), 'f', prec, 64)) - - if st.Flag(' ') { - io.WriteString(st, " ") - } - io.WriteString(st, unit.String()) -} - -const ( - _b SizeB1000 = 1 - _KB SizeB1000 = _b * 1000 - _MB SizeB1000 = _KB * 1000 - _GB SizeB1000 = _MB * 1000 - _TB SizeB1000 = _GB * 1000 -) - -// SizeB1000 named type, which implements fmt.Formatter interface. It -// adjusts its value according to byte size multiple by 1000 and appends -// appropriate size marker (KB, MB, GB, TB). -type SizeB1000 int64 - -func (self SizeB1000) Format(st fmt.State, verb rune) { - var prec int - switch verb { - case 'd': - case 's': - prec = -1 - default: - if p, ok := st.Precision(); ok { - prec = p - } else { - prec = 6 - } - } - - var unit SizeB1000 - switch { - case self < _KB: - unit = _b - case self < _MB: - unit = _KB - case self < _GB: - unit = _MB - case self < _TB: - unit = _GB - case self <= math.MaxInt64: - unit = _TB - } - - io.WriteString(st, strconv.FormatFloat(float64(self)/float64(unit), 'f', prec, 64)) - - if st.Flag(' ') { - io.WriteString(st, " ") - } - io.WriteString(st, unit.String()) -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/sizeb1000_string.go b/vendor/github.com/vbauerster/mpb/v5/decor/sizeb1000_string.go deleted file mode 100644 index 3f32ef715..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/sizeb1000_string.go +++ /dev/null @@ -1,41 +0,0 @@ -// Code generated by "stringer -type=SizeB1000 -trimprefix=_"; DO NOT EDIT. - -package decor - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[_b-1] - _ = x[_KB-1000] - _ = x[_MB-1000000] - _ = x[_GB-1000000000] - _ = x[_TB-1000000000000] -} - -const ( - _SizeB1000_name_0 = "b" - _SizeB1000_name_1 = "KB" - _SizeB1000_name_2 = "MB" - _SizeB1000_name_3 = "GB" - _SizeB1000_name_4 = "TB" -) - -func (i SizeB1000) String() string { - switch { - case i == 1: - return _SizeB1000_name_0 - case i == 1000: - return _SizeB1000_name_1 - case i == 1000000: - return _SizeB1000_name_2 - case i == 1000000000: - return _SizeB1000_name_3 - case i == 1000000000000: - return _SizeB1000_name_4 - default: - return "SizeB1000(" + strconv.FormatInt(int64(i), 10) + ")" - } -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/sizeb1024_string.go b/vendor/github.com/vbauerster/mpb/v5/decor/sizeb1024_string.go deleted file mode 100644 index 9fca66cc7..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/sizeb1024_string.go +++ /dev/null @@ -1,41 +0,0 @@ -// Code generated by "stringer -type=SizeB1024 -trimprefix=_i"; DO NOT EDIT. - -package decor - -import "strconv" - -func _() { - // An "invalid array index" compiler error signifies that the constant values have changed. - // Re-run the stringer command to generate them again. - var x [1]struct{} - _ = x[_ib-1] - _ = x[_iKiB-1024] - _ = x[_iMiB-1048576] - _ = x[_iGiB-1073741824] - _ = x[_iTiB-1099511627776] -} - -const ( - _SizeB1024_name_0 = "b" - _SizeB1024_name_1 = "KiB" - _SizeB1024_name_2 = "MiB" - _SizeB1024_name_3 = "GiB" - _SizeB1024_name_4 = "TiB" -) - -func (i SizeB1024) String() string { - switch { - case i == 1: - return _SizeB1024_name_0 - case i == 1024: - return _SizeB1024_name_1 - case i == 1048576: - return _SizeB1024_name_2 - case i == 1073741824: - return _SizeB1024_name_3 - case i == 1099511627776: - return _SizeB1024_name_4 - default: - return "SizeB1024(" + strconv.FormatInt(int64(i), 10) + ")" - } -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/speed.go b/vendor/github.com/vbauerster/mpb/v5/decor/speed.go deleted file mode 100644 index 634edabfd..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/speed.go +++ /dev/null @@ -1,171 +0,0 @@ -package decor - -import ( - "fmt" - "io" - "math" - "time" - - "github.com/VividCortex/ewma" -) - -// FmtAsSpeed adds "/s" to the end of the input formatter. To be -// used with SizeB1000 or SizeB1024 types, for example: -// -// fmt.Printf("%.1f", FmtAsSpeed(SizeB1024(2048))) -// -func FmtAsSpeed(input fmt.Formatter) fmt.Formatter { - return &speedFormatter{input} -} - -type speedFormatter struct { - fmt.Formatter -} - -func (self *speedFormatter) Format(st fmt.State, verb rune) { - self.Formatter.Format(st, verb) - io.WriteString(st, "/s") -} - -// EwmaSpeed exponential-weighted-moving-average based speed decorator. -// For this decorator to work correctly you have to measure each -// iteration's duration and pass it to the -// *Bar.DecoratorEwmaUpdate(time.Duration) method after each increment. -func EwmaSpeed(unit int, format string, age float64, wcc ...WC) Decorator { - var average ewma.MovingAverage - if age == 0 { - average = ewma.NewMovingAverage() - } else { - average = ewma.NewMovingAverage(age) - } - return MovingAverageSpeed(unit, format, NewThreadSafeMovingAverage(average), wcc...) -} - -// MovingAverageSpeed decorator relies on MovingAverage implementation -// to calculate its average. -// -// `unit` one of [0|UnitKiB|UnitKB] zero for no unit -// -// `format` printf compatible verb for value, like "%f" or "%d" -// -// `average` MovingAverage implementation -// -// `wcc` optional WC config -// -// format examples: -// -// unit=UnitKiB, format="%.1f" output: "1.0MiB/s" -// unit=UnitKiB, format="% .1f" output: "1.0 MiB/s" -// unit=UnitKB, format="%.1f" output: "1.0MB/s" -// unit=UnitKB, format="% .1f" output: "1.0 MB/s" -// -func MovingAverageSpeed(unit int, format string, average ewma.MovingAverage, wcc ...WC) Decorator { - if format == "" { - format = "%.0f" - } - d := &movingAverageSpeed{ - WC: initWC(wcc...), - average: average, - producer: chooseSpeedProducer(unit, format), - } - return d -} - -type movingAverageSpeed struct { - WC - producer func(float64) string - average ewma.MovingAverage - msg string -} - -func (d *movingAverageSpeed) Decor(s Statistics) string { - if !s.Completed { - var speed float64 - if v := d.average.Value(); v > 0 { - speed = 1 / v - } - d.msg = d.producer(speed * 1e9) - } - return d.FormatMsg(d.msg) -} - -func (d *movingAverageSpeed) EwmaUpdate(n int64, dur time.Duration) { - durPerByte := float64(dur) / float64(n) - if math.IsInf(durPerByte, 0) || math.IsNaN(durPerByte) { - return - } - d.average.Add(durPerByte) -} - -// AverageSpeed decorator with dynamic unit measure adjustment. It's -// a wrapper of NewAverageSpeed. -func AverageSpeed(unit int, format string, wcc ...WC) Decorator { - return NewAverageSpeed(unit, format, time.Now(), wcc...) -} - -// NewAverageSpeed decorator with dynamic unit measure adjustment and -// user provided start time. -// -// `unit` one of [0|UnitKiB|UnitKB] zero for no unit -// -// `format` printf compatible verb for value, like "%f" or "%d" -// -// `startTime` start time -// -// `wcc` optional WC config -// -// format examples: -// -// unit=UnitKiB, format="%.1f" output: "1.0MiB/s" -// unit=UnitKiB, format="% .1f" output: "1.0 MiB/s" -// unit=UnitKB, format="%.1f" output: "1.0MB/s" -// unit=UnitKB, format="% .1f" output: "1.0 MB/s" -// -func NewAverageSpeed(unit int, format string, startTime time.Time, wcc ...WC) Decorator { - if format == "" { - format = "%.0f" - } - d := &averageSpeed{ - WC: initWC(wcc...), - startTime: startTime, - producer: chooseSpeedProducer(unit, format), - } - return d -} - -type averageSpeed struct { - WC - startTime time.Time - producer func(float64) string - msg string -} - -func (d *averageSpeed) Decor(s Statistics) string { - if !s.Completed { - speed := float64(s.Current) / float64(time.Since(d.startTime)) - d.msg = d.producer(speed * 1e9) - } - - return d.FormatMsg(d.msg) -} - -func (d *averageSpeed) AverageAdjust(startTime time.Time) { - d.startTime = startTime -} - -func chooseSpeedProducer(unit int, format string) func(float64) string { - switch unit { - case UnitKiB: - return func(speed float64) string { - return fmt.Sprintf(format, FmtAsSpeed(SizeB1024(math.Round(speed)))) - } - case UnitKB: - return func(speed float64) string { - return fmt.Sprintf(format, FmtAsSpeed(SizeB1000(math.Round(speed)))) - } - default: - return func(speed float64) string { - return fmt.Sprintf(format, speed) - } - } -} diff --git a/vendor/github.com/vbauerster/mpb/v5/decor/spinner.go b/vendor/github.com/vbauerster/mpb/v5/decor/spinner.go deleted file mode 100644 index 6871639db..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/decor/spinner.go +++ /dev/null @@ -1,21 +0,0 @@ -package decor - -var defaultSpinnerStyle = []string{"⠋", "⠙", "⠹", "⠸", "⠼", "⠴", "⠦", "⠧", "⠇", "⠏"} - -// Spinner returns spinner decorator. -// -// `frames` spinner frames, if nil or len==0, default is used -// -// `wcc` optional WC config -func Spinner(frames []string, wcc ...WC) Decorator { - if len(frames) == 0 { - frames = defaultSpinnerStyle - } - var count uint - f := func(s Statistics) string { - frame := frames[count%uint(len(frames))] - count++ - return frame - } - return Any(f, wcc...) -} diff --git a/vendor/github.com/vbauerster/mpb/v5/doc.go b/vendor/github.com/vbauerster/mpb/v5/doc.go deleted file mode 100644 index 5ada71774..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/doc.go +++ /dev/null @@ -1,2 +0,0 @@ -// Package mpb is a library for rendering progress bars in terminal applications. -package mpb diff --git a/vendor/github.com/vbauerster/mpb/v5/go.mod b/vendor/github.com/vbauerster/mpb/v5/go.mod deleted file mode 100644 index e80d1a10d..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/go.mod +++ /dev/null @@ -1,10 +0,0 @@ -module github.com/vbauerster/mpb/v5 - -require ( - github.com/VividCortex/ewma v1.1.1 - github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d - github.com/mattn/go-runewidth v0.0.9 - golang.org/x/sys v0.0.0-20201218084310-7d0127a74742 -) - -go 1.14 diff --git a/vendor/github.com/vbauerster/mpb/v5/go.sum b/vendor/github.com/vbauerster/mpb/v5/go.sum deleted file mode 100644 index 62cc10af0..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/go.sum +++ /dev/null @@ -1,8 +0,0 @@ -github.com/VividCortex/ewma v1.1.1 h1:MnEK4VOv6n0RSY4vtRe3h11qjxL3+t0B8yOL8iMXdcM= -github.com/VividCortex/ewma v1.1.1/go.mod h1:2Tkkvm3sRDVXaiyucHiACn4cqf7DpdyLvmxzcbUokwA= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d h1:licZJFw2RwpHMqeKTCYkitsPqHNxTmd4SNR5r94FGM8= -github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d/go.mod h1:asat636LX7Bqt5lYEZ27JNDcqxfjdBQuJ/MM4CN/Lzo= -github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/QdE+0= -github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -golang.org/x/sys v0.0.0-20201218084310-7d0127a74742 h1:+CBz4km/0KPU3RGTwARGh/noP3bEwtHcq+0YcBQM2JQ= -golang.org/x/sys v0.0.0-20201218084310-7d0127a74742/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/vendor/github.com/vbauerster/mpb/v5/internal/percentage.go b/vendor/github.com/vbauerster/mpb/v5/internal/percentage.go deleted file mode 100644 index e321e0a6b..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/internal/percentage.go +++ /dev/null @@ -1,18 +0,0 @@ -package internal - -import "math" - -// Percentage is a helper function, to calculate percentage. -func Percentage(total, current int64, width int) float64 { - if total <= 0 { - return 0 - } - if current >= total { - return float64(width) - } - return float64(int64(width)*current) / float64(total) -} - -func PercentageRound(total, current int64, width int) float64 { - return math.Round(Percentage(total, current, width)) -} diff --git a/vendor/github.com/vbauerster/mpb/v5/internal/width.go b/vendor/github.com/vbauerster/mpb/v5/internal/width.go deleted file mode 100644 index 35d528983..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/internal/width.go +++ /dev/null @@ -1,8 +0,0 @@ -package internal - -func WidthForBarFiller(reqWidth, available int) int { - if reqWidth <= 0 || reqWidth >= available { - return available - } - return reqWidth -} diff --git a/vendor/github.com/vbauerster/mpb/v5/priority_queue.go b/vendor/github.com/vbauerster/mpb/v5/priority_queue.go deleted file mode 100644 index 29d9bd5a8..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/priority_queue.go +++ /dev/null @@ -1,32 +0,0 @@ -package mpb - -// A priorityQueue implements heap.Interface -type priorityQueue []*Bar - -func (pq priorityQueue) Len() int { return len(pq) } - -func (pq priorityQueue) Less(i, j int) bool { - return pq[i].priority < pq[j].priority -} - -func (pq priorityQueue) Swap(i, j int) { - pq[i], pq[j] = pq[j], pq[i] - pq[i].index = i - pq[j].index = j -} - -func (pq *priorityQueue) Push(x interface{}) { - s := *pq - bar := x.(*Bar) - bar.index = len(s) - s = append(s, bar) - *pq = s -} - -func (pq *priorityQueue) Pop() interface{} { - s := *pq - *pq = s[0 : len(s)-1] - bar := s[len(s)-1] - bar.index = -1 // for safety - return bar -} diff --git a/vendor/github.com/vbauerster/mpb/v5/progress.go b/vendor/github.com/vbauerster/mpb/v5/progress.go deleted file mode 100644 index fb66ce05d..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/progress.go +++ /dev/null @@ -1,390 +0,0 @@ -package mpb - -import ( - "bytes" - "container/heap" - "context" - "fmt" - "io" - "io/ioutil" - "log" - "math" - "os" - "sync" - "time" - - "github.com/vbauerster/mpb/v5/cwriter" - "github.com/vbauerster/mpb/v5/decor" -) - -const ( - // default RefreshRate - prr = 120 * time.Millisecond -) - -// Progress represents the container that renders Progress bars -type Progress struct { - ctx context.Context - uwg *sync.WaitGroup - cwg *sync.WaitGroup - bwg *sync.WaitGroup - operateState chan func(*pState) - done chan struct{} - refreshCh chan time.Time - once sync.Once - dlogger *log.Logger -} - -type pState struct { - bHeap priorityQueue - heapUpdated bool - pMatrix map[int][]chan int - aMatrix map[int][]chan int - barShutdownQueue []*Bar - - // following are provided/overrided by user - idCount int - reqWidth int - popCompleted bool - rr time.Duration - uwg *sync.WaitGroup - refreshSrc <-chan time.Time - renderDelay <-chan struct{} - shutdownNotifier chan struct{} - parkedBars map[*Bar]*Bar - output io.Writer - debugOut io.Writer -} - -// New creates new Progress container instance. It's not possible to -// reuse instance after *Progress.Wait() method has been called. -func New(options ...ContainerOption) *Progress { - return NewWithContext(context.Background(), options...) -} - -// NewWithContext creates new Progress container instance with provided -// context. It's not possible to reuse instance after *Progress.Wait() -// method has been called. -func NewWithContext(ctx context.Context, options ...ContainerOption) *Progress { - s := &pState{ - bHeap: priorityQueue{}, - rr: prr, - parkedBars: make(map[*Bar]*Bar), - output: os.Stdout, - debugOut: ioutil.Discard, - } - - for _, opt := range options { - if opt != nil { - opt(s) - } - } - - p := &Progress{ - ctx: ctx, - uwg: s.uwg, - cwg: new(sync.WaitGroup), - bwg: new(sync.WaitGroup), - operateState: make(chan func(*pState)), - done: make(chan struct{}), - dlogger: log.New(s.debugOut, "[mpb] ", log.Lshortfile), - } - - p.cwg.Add(1) - go p.serve(s, cwriter.New(s.output)) - return p -} - -// AddBar creates a new progress bar and adds it to the rendering queue. -func (p *Progress) AddBar(total int64, options ...BarOption) *Bar { - return p.Add(total, NewBarFiller(DefaultBarStyle, false), options...) -} - -// AddSpinner creates a new spinner bar and adds it to the rendering queue. -func (p *Progress) AddSpinner(total int64, alignment SpinnerAlignment, options ...BarOption) *Bar { - return p.Add(total, NewSpinnerFiller(DefaultSpinnerStyle, alignment), options...) -} - -// Add creates a bar which renders itself by provided filler. -// Set total to 0, if you plan to update it later. -// Panics if *Progress instance is done, i.e. called after *Progress.Wait(). -func (p *Progress) Add(total int64, filler BarFiller, options ...BarOption) *Bar { - if filler == nil { - filler = BarFillerFunc(func(io.Writer, int, decor.Statistics) {}) - } - p.bwg.Add(1) - result := make(chan *Bar) - select { - case p.operateState <- func(ps *pState) { - bs := ps.makeBarState(total, filler, options...) - bar := newBar(p, bs) - if bs.runningBar != nil { - bs.runningBar.noPop = true - ps.parkedBars[bs.runningBar] = bar - } else { - heap.Push(&ps.bHeap, bar) - ps.heapUpdated = true - } - ps.idCount++ - result <- bar - }: - bar := <-result - bar.subscribeDecorators() - return bar - case <-p.done: - p.bwg.Done() - panic(fmt.Sprintf("%T instance can't be reused after it's done!", p)) - } -} - -func (p *Progress) dropBar(b *Bar) { - select { - case p.operateState <- func(s *pState) { - if b.index < 0 { - return - } - heap.Remove(&s.bHeap, b.index) - s.heapUpdated = true - }: - case <-p.done: - } -} - -func (p *Progress) setBarPriority(b *Bar, priority int) { - select { - case p.operateState <- func(s *pState) { - if b.index < 0 { - return - } - b.priority = priority - heap.Fix(&s.bHeap, b.index) - }: - case <-p.done: - } -} - -// UpdateBarPriority same as *Bar.SetPriority(int). -func (p *Progress) UpdateBarPriority(b *Bar, priority int) { - p.setBarPriority(b, priority) -} - -// BarCount returns bars count -func (p *Progress) BarCount() int { - result := make(chan int, 1) - select { - case p.operateState <- func(s *pState) { result <- s.bHeap.Len() }: - return <-result - case <-p.done: - return 0 - } -} - -// Wait waits for all bars to complete and finally shutdowns container. -// After this method has been called, there is no way to reuse *Progress -// instance. -func (p *Progress) Wait() { - if p.uwg != nil { - // wait for user wg - p.uwg.Wait() - } - - // wait for bars to quit, if any - p.bwg.Wait() - - p.once.Do(p.shutdown) - - // wait for container to quit - p.cwg.Wait() -} - -func (p *Progress) shutdown() { - close(p.done) -} - -func (p *Progress) serve(s *pState, cw *cwriter.Writer) { - defer p.cwg.Done() - - p.refreshCh = s.newTicker(p.done) - - for { - select { - case op := <-p.operateState: - op(s) - case <-p.refreshCh: - if err := s.render(cw); err != nil { - p.dlogger.Println(err) - } - case <-s.shutdownNotifier: - if s.heapUpdated { - if err := s.render(cw); err != nil { - p.dlogger.Println(err) - } - } - return - } - } -} - -func (s *pState) newTicker(done <-chan struct{}) chan time.Time { - ch := make(chan time.Time) - if s.shutdownNotifier == nil { - s.shutdownNotifier = make(chan struct{}) - } - go func() { - if s.renderDelay != nil { - <-s.renderDelay - } - if s.refreshSrc == nil { - ticker := time.NewTicker(s.rr) - defer ticker.Stop() - s.refreshSrc = ticker.C - } - for { - select { - case tick := <-s.refreshSrc: - ch <- tick - case <-done: - close(s.shutdownNotifier) - return - } - } - }() - return ch -} - -func (s *pState) render(cw *cwriter.Writer) error { - if s.heapUpdated { - s.updateSyncMatrix() - s.heapUpdated = false - } - syncWidth(s.pMatrix) - syncWidth(s.aMatrix) - - tw, err := cw.GetWidth() - if err != nil { - tw = s.reqWidth - } - for i := 0; i < s.bHeap.Len(); i++ { - bar := s.bHeap[i] - go bar.render(tw) - } - - return s.flush(cw) -} - -func (s *pState) flush(cw *cwriter.Writer) error { - var lineCount int - bm := make(map[*Bar]struct{}, s.bHeap.Len()) - for s.bHeap.Len() > 0 { - b := heap.Pop(&s.bHeap).(*Bar) - cw.ReadFrom(<-b.frameCh) - if b.toShutdown { - if b.recoveredPanic != nil { - s.barShutdownQueue = append(s.barShutdownQueue, b) - b.toShutdown = false - } else { - // shutdown at next flush - // this ensures no bar ends up with less than 100% rendered - defer func() { - s.barShutdownQueue = append(s.barShutdownQueue, b) - }() - } - } - lineCount += b.extendedLines + 1 - bm[b] = struct{}{} - } - - for _, b := range s.barShutdownQueue { - if parkedBar := s.parkedBars[b]; parkedBar != nil { - parkedBar.priority = b.priority - heap.Push(&s.bHeap, parkedBar) - delete(s.parkedBars, b) - b.toDrop = true - } - if s.popCompleted && !b.noPop { - lineCount -= b.extendedLines + 1 - b.toDrop = true - } - if b.toDrop { - delete(bm, b) - s.heapUpdated = true - } - b.cancel() - } - s.barShutdownQueue = s.barShutdownQueue[0:0] - - for b := range bm { - heap.Push(&s.bHeap, b) - } - - return cw.Flush(lineCount) -} - -func (s *pState) updateSyncMatrix() { - s.pMatrix = make(map[int][]chan int) - s.aMatrix = make(map[int][]chan int) - for i := 0; i < s.bHeap.Len(); i++ { - bar := s.bHeap[i] - table := bar.wSyncTable() - pRow, aRow := table[0], table[1] - - for i, ch := range pRow { - s.pMatrix[i] = append(s.pMatrix[i], ch) - } - - for i, ch := range aRow { - s.aMatrix[i] = append(s.aMatrix[i], ch) - } - } -} - -func (s *pState) makeBarState(total int64, filler BarFiller, options ...BarOption) *bState { - bs := &bState{ - id: s.idCount, - priority: s.idCount, - reqWidth: s.reqWidth, - total: total, - filler: filler, - extender: func(r io.Reader, _ int, _ decor.Statistics) (io.Reader, int) { return r, 0 }, - debugOut: s.debugOut, - } - - for _, opt := range options { - if opt != nil { - opt(bs) - } - } - - if bs.middleware != nil { - bs.filler = bs.middleware(filler) - bs.middleware = nil - } - - if s.popCompleted && !bs.noPop { - bs.priority = -(math.MaxInt32 - s.idCount) - } - - bs.bufP = bytes.NewBuffer(make([]byte, 0, 128)) - bs.bufB = bytes.NewBuffer(make([]byte, 0, 256)) - bs.bufA = bytes.NewBuffer(make([]byte, 0, 128)) - - return bs -} - -func syncWidth(matrix map[int][]chan int) { - for _, column := range matrix { - go maxWidthDistributor(column) - } -} - -var maxWidthDistributor = func(column []chan int) { - var maxWidth int - for _, ch := range column { - if w := <-ch; w > maxWidth { - maxWidth = w - } - } - for _, ch := range column { - ch <- maxWidth - } -} diff --git a/vendor/github.com/vbauerster/mpb/v5/proxyreader.go b/vendor/github.com/vbauerster/mpb/v5/proxyreader.go deleted file mode 100644 index 316f438d7..000000000 --- a/vendor/github.com/vbauerster/mpb/v5/proxyreader.go +++ /dev/null @@ -1,90 +0,0 @@ -package mpb - -import ( - "io" - "io/ioutil" - "time" -) - -type proxyReader struct { - io.ReadCloser - bar *Bar -} - -func (x *proxyReader) Read(p []byte) (int, error) { - n, err := x.ReadCloser.Read(p) - x.bar.IncrBy(n) - if err == io.EOF { - go x.bar.SetTotal(0, true) - } - return n, err -} - -type proxyWriterTo struct { - io.ReadCloser // *proxyReader - wt io.WriterTo - bar *Bar -} - -func (x *proxyWriterTo) WriteTo(w io.Writer) (int64, error) { - n, err := x.wt.WriteTo(w) - x.bar.IncrInt64(n) - if err == io.EOF { - go x.bar.SetTotal(0, true) - } - return n, err -} - -type ewmaProxyReader struct { - io.ReadCloser // *proxyReader - bar *Bar - iT time.Time -} - -func (x *ewmaProxyReader) Read(p []byte) (int, error) { - n, err := x.ReadCloser.Read(p) - if n > 0 { - x.bar.DecoratorEwmaUpdate(time.Since(x.iT)) - x.iT = time.Now() - } - return n, err -} - -type ewmaProxyWriterTo struct { - io.ReadCloser // *ewmaProxyReader - wt io.WriterTo // *proxyWriterTo - bar *Bar - iT time.Time -} - -func (x *ewmaProxyWriterTo) WriteTo(w io.Writer) (int64, error) { - n, err := x.wt.WriteTo(w) - if n > 0 { - x.bar.DecoratorEwmaUpdate(time.Since(x.iT)) - x.iT = time.Now() - } - return n, err -} - -func newProxyReader(r io.Reader, bar *Bar) io.ReadCloser { - rc := toReadCloser(r) - rc = &proxyReader{rc, bar} - - if wt, isWriterTo := r.(io.WriterTo); bar.hasEwmaDecorators { - now := time.Now() - rc = &ewmaProxyReader{rc, bar, now} - if isWriterTo { - rc = &ewmaProxyWriterTo{rc, wt, bar, now} - } - } else if isWriterTo { - rc = &proxyWriterTo{rc, wt, bar} - } - return rc -} - -func toReadCloser(r io.Reader) io.ReadCloser { - if rc, ok := r.(io.ReadCloser); ok { - return rc - } - return ioutil.NopCloser(r) -} diff --git a/vendor/modules.txt b/vendor/modules.txt index d43ed6753..0f8b515e2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -9,7 +9,7 @@ github.com/Microsoft/go-winio/backuptar github.com/Microsoft/go-winio/pkg/guid github.com/Microsoft/go-winio/pkg/security github.com/Microsoft/go-winio/vhd -# github.com/Microsoft/hcsshim v0.8.15 +# github.com/Microsoft/hcsshim v0.8.16 github.com/Microsoft/hcsshim github.com/Microsoft/hcsshim/computestorage github.com/Microsoft/hcsshim/internal/cow @@ -49,9 +49,9 @@ github.com/checkpoint-restore/go-criu github.com/checkpoint-restore/go-criu/rpc # github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/chzyer/readline -# github.com/containerd/cgroups v0.0.0-20200824123100-0b889c03f102 +# github.com/containerd/cgroups v0.0.0-20210114181951-8a68de567b68 github.com/containerd/cgroups/stats/v1 -# github.com/containerd/containerd v1.5.0-beta.1 +# github.com/containerd/containerd v1.5.0-beta.4 github.com/containerd/containerd/errdefs github.com/containerd/containerd/log github.com/containerd/containerd/platforms @@ -115,7 +115,7 @@ github.com/containers/common/pkg/umask github.com/containers/common/version # github.com/containers/conmon v2.0.20+incompatible github.com/containers/conmon/runner/config -# github.com/containers/image/v5 v5.10.5 +# github.com/containers/image/v5 v5.11.0 github.com/containers/image/v5/copy github.com/containers/image/v5/directory github.com/containers/image/v5/directory/explicitfilepath @@ -133,6 +133,7 @@ github.com/containers/image/v5/internal/pkg/keyctl github.com/containers/image/v5/internal/pkg/platform github.com/containers/image/v5/internal/rootless github.com/containers/image/v5/internal/tmpdir +github.com/containers/image/v5/internal/types github.com/containers/image/v5/internal/uploadreader github.com/containers/image/v5/manifest github.com/containers/image/v5/oci/archive @@ -187,7 +188,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.28.1 +# github.com/containers/storage v1.29.0 github.com/containers/storage github.com/containers/storage/drivers github.com/containers/storage/drivers/aufs @@ -350,6 +351,8 @@ github.com/google/go-cmp/cmp/internal/diff github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function github.com/google/go-cmp/cmp/internal/value +# github.com/google/go-intervals v0.0.2 +github.com/google/go-intervals/intervalset # github.com/google/gofuzz v1.1.0 github.com/google/gofuzz # github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf @@ -381,7 +384,7 @@ github.com/json-iterator/go # github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a github.com/juju/ansiterm github.com/juju/ansiterm/tabwriter -# github.com/klauspost/compress v1.11.12 +# github.com/klauspost/compress v1.11.13 github.com/klauspost/compress/flate github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 @@ -588,11 +591,6 @@ github.com/ulikunitz/xz/lzma github.com/vbatts/tar-split/archive/tar github.com/vbatts/tar-split/tar/asm github.com/vbatts/tar-split/tar/storage -# github.com/vbauerster/mpb/v5 v5.4.0 -github.com/vbauerster/mpb/v5 -github.com/vbauerster/mpb/v5/cwriter -github.com/vbauerster/mpb/v5/decor -github.com/vbauerster/mpb/v5/internal # github.com/vbauerster/mpb/v6 v6.0.3 github.com/vbauerster/mpb/v6 github.com/vbauerster/mpb/v6/cwriter |