diff options
Diffstat (limited to 'vendor/github.com')
141 files changed, 3533 insertions, 5150 deletions
diff --git a/vendor/github.com/containers/buildah/.cirrus.yml b/vendor/github.com/containers/buildah/.cirrus.yml index 795c1746a..1b25b190c 100644 --- a/vendor/github.com/containers/buildah/.cirrus.yml +++ b/vendor/github.com/containers/buildah/.cirrus.yml @@ -119,7 +119,7 @@ vendor_task: # Runs within Cirrus's "community cluster" container: - image: docker.io/library/golang:1.16 + image: docker.io/library/golang:1.17 cpu: 1 memory: 1 diff --git a/vendor/github.com/containers/buildah/Makefile b/vendor/github.com/containers/buildah/Makefile index c2bde6d28..c9121cc87 100644 --- a/vendor/github.com/containers/buildah/Makefile +++ b/vendor/github.com/containers/buildah/Makefile @@ -26,7 +26,8 @@ export GO_TEST=$(GO) test endif RACEFLAGS := $(shell $(GO_TEST) -race ./pkg/dummy > /dev/null 2>&1 && echo -race) -GIT_COMMIT ?= $(if $(shell git rev-parse --short HEAD),$(shell git rev-parse --short HEAD),$(error "git failed")) +COMMIT_NO ?= $(shell git rev-parse HEAD 2> /dev/null || true) +GIT_COMMIT ?= $(if $(shell git status --porcelain --untracked-files=no),${COMMIT_NO}-dirty,${COMMIT_NO}) SOURCE_DATE_EPOCH ?= $(if $(shell date +%s),$(shell date +%s),$(error "date failed")) STATIC_STORAGETAGS = "containers_image_openpgp exclude_graphdriver_devicemapper $(STORAGE_TAGS)" @@ -177,7 +178,7 @@ test-unit: tests/testreport/testreport $(GO_TEST) -v -tags "$(STORAGETAGS) $(SECURITYTAGS)" -cover $(RACEFLAGS) ./cmd/buildah -args --root $$tmp/root --runroot $$tmp/runroot --storage-driver vfs --signature-policy $(shell pwd)/tests/policy.json --registries-conf $(shell pwd)/tests/registries.conf vendor-in-container: - podman run --privileged --rm --env HOME=/root -v `pwd`:/src -w /src docker.io/library/golang:1.16 make vendor + podman run --privileged --rm --env HOME=/root -v `pwd`:/src -w /src docker.io/library/golang:1.17 make vendor .PHONY: vendor vendor: diff --git a/vendor/github.com/containers/buildah/chroot/seccomp.go b/vendor/github.com/containers/buildah/chroot/seccomp.go index f130f7a22..f36359e34 100644 --- a/vendor/github.com/containers/buildah/chroot/seccomp.go +++ b/vendor/github.com/containers/buildah/chroot/seccomp.go @@ -1,3 +1,4 @@ +//go:build linux && seccomp // +build linux,seccomp package chroot @@ -21,7 +22,7 @@ func setSeccomp(spec *specs.Spec) error { mapAction := func(specAction specs.LinuxSeccompAction, errnoRet *uint) libseccomp.ScmpAction { switch specAction { case specs.ActKill: - return libseccomp.ActKill + return libseccomp.ActKillThread case specs.ActTrap: return libseccomp.ActTrap case specs.ActErrno: diff --git a/vendor/github.com/containers/buildah/go.mod b/vendor/github.com/containers/buildah/go.mod index 95bf7212b..1fc8c6016 100644 --- a/vendor/github.com/containers/buildah/go.mod +++ b/vendor/github.com/containers/buildah/go.mod @@ -1,35 +1,32 @@ module github.com/containers/buildah -go 1.16 +go 1.17 require ( github.com/containerd/containerd v1.6.6 github.com/containernetworking/cni v1.1.1 - github.com/containers/common v0.48.1-0.20220519181648-280c6f69fa82 + github.com/containers/common v0.48.1-0.20220608111710-dbecabbe82c9 github.com/containers/image/v5 v5.21.2-0.20220520105616-e594853d6471 github.com/containers/ocicrypt v1.1.4-0.20220428134531-566b808bdf6f - github.com/containers/storage v1.41.1-0.20220517121726-5019cd55275c + github.com/containers/storage v1.41.1-0.20220607143333-8951d0153bf6 github.com/docker/distribution v2.8.1+incompatible github.com/docker/docker v20.10.17+incompatible github.com/docker/go-units v0.4.0 - github.com/docker/libnetwork v0.8.0-dev.2.0.20190625141545-5a177b73e316 github.com/fsouza/go-dockerclient v1.8.1 github.com/ghodss/yaml v1.0.0 github.com/hashicorp/go-multierror v1.1.1 - github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee // indirect - github.com/konsorten/go-windows-terminal-sequences v1.0.3 // indirect github.com/mattn/go-shellwords v1.0.12 github.com/onsi/ginkgo v1.16.5 github.com/onsi/gomega v1.19.0 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.3-0.20211202193544-a5463b7f9c84 - github.com/opencontainers/runc v1.1.2 + github.com/opencontainers/runc v1.1.3 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 github.com/opencontainers/runtime-tools v0.9.0 github.com/opencontainers/selinux v1.10.1 github.com/openshift/imagebuilder v1.2.4-0.20220502172744-009dbc6cb805 github.com/pkg/errors v0.9.1 - github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921 + github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 @@ -42,6 +39,84 @@ require ( golang.org/x/term v0.0.0-20220526004731-065cf7ba2467 ) -replace github.com/sirupsen/logrus => github.com/sirupsen/logrus v1.4.2 +require ( + github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect + github.com/BurntSushi/toml v1.1.0 // indirect + github.com/Microsoft/go-winio v0.5.2 // indirect + github.com/Microsoft/hcsshim v0.9.3 // indirect + github.com/VividCortex/ewma v1.2.0 // indirect + github.com/acarl005/stripansi v0.0.0-20180116102854-5a71ef0e047d // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/blang/semver v3.5.1+incompatible // indirect + github.com/cespare/xxhash/v2 v2.1.2 // indirect + github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e // indirect + github.com/containerd/cgroups v1.0.3 // indirect + github.com/containerd/stargz-snapshotter/estargz v0.11.4 // indirect + github.com/containernetworking/plugins v1.1.1 // indirect + github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a // indirect + github.com/cyphar/filepath-securejoin v0.2.3 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/disiqueira/gotree/v3 v3.0.2 // indirect + github.com/docker/docker-credential-helpers v0.6.4 // indirect + github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 // indirect + github.com/docker/go-metrics v0.0.1 // indirect + github.com/fsnotify/fsnotify v1.4.9 // indirect + github.com/gogo/protobuf v1.3.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.2 // indirect + github.com/google/go-intervals v0.0.2 // indirect + github.com/google/uuid v1.3.0 // indirect + github.com/gorilla/mux v1.8.0 // indirect + github.com/hashicorp/errwrap v1.1.0 // indirect + github.com/imdario/mergo v0.3.12 // indirect + github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/jinzhu/copier v0.3.5 // indirect + github.com/json-iterator/go v1.1.12 // indirect + github.com/klauspost/compress v1.15.6 // indirect + github.com/klauspost/pgzip v1.2.5 // indirect + github.com/manifoldco/promptui v0.9.0 // indirect + github.com/mattn/go-runewidth v0.0.13 // indirect + github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect + github.com/miekg/pkcs11 v1.1.1 // indirect + github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible // indirect + github.com/moby/sys/mount v0.3.3 // indirect + github.com/moby/sys/mountinfo v0.6.2 // indirect + github.com/moby/term v0.0.0-20210619224110-3f7ff695adc6 // indirect + github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect + github.com/modern-go/reflect2 v1.0.2 // indirect + github.com/morikuni/aec v1.0.0 // indirect + github.com/nxadm/tail v1.4.8 // indirect + github.com/ostreedev/ostree-go v0.0.0-20210805093236-719684c64e4f // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect + github.com/proglottis/gpgme v0.1.2 // indirect + github.com/prometheus/client_golang v1.11.1 // indirect + github.com/prometheus/client_model v0.2.0 // indirect + github.com/prometheus/common v0.30.0 // indirect + github.com/prometheus/procfs v0.7.3 // indirect + github.com/rivo/uniseg v0.2.0 // indirect + github.com/stefanberger/go-pkcs11uri v0.0.0-20201008174630-78d3cae3a980 // indirect + github.com/sylabs/sif/v2 v2.7.0 // indirect + github.com/tchap/go-patricia v2.3.0+incompatible // indirect + github.com/ulikunitz/xz v0.5.10 // indirect + github.com/vbatts/tar-split v0.11.2 // indirect + github.com/vbauerster/mpb/v7 v7.4.1 // indirect + github.com/vishvananda/netlink v1.1.1-0.20210330154013-f5de75959ad5 // indirect + github.com/vishvananda/netns v0.0.0-20210104183010-2eb08e3e575f // indirect + github.com/xeipuuv/gojsonpointer v0.0.0-20190905194746-02993c407bfb // indirect + github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 // indirect + github.com/xeipuuv/gojsonschema v1.2.0 // indirect + go.mozilla.org/pkcs7 v0.0.0-20200128120323-432b2356ecb1 // indirect + go.opencensus.io v0.23.0 // indirect + golang.org/x/net v0.0.0-20220225172249-27dd8689420f // indirect + golang.org/x/text v0.3.7 // indirect + google.golang.org/genproto v0.0.0-20220304144024-325a89244dc8 // indirect + google.golang.org/grpc v1.44.0 // indirect + google.golang.org/protobuf v1.28.0 // indirect + gopkg.in/square/go-jose.v2 v2.5.1 // indirect + gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect + gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect + k8s.io/klog v1.0.0 // indirect +) replace github.com/opencontainers/image-spec => github.com/opencontainers/image-spec v1.0.2-0.20211123152302-43a7dee1ec31 diff --git a/vendor/github.com/containers/buildah/go.sum b/vendor/github.com/containers/buildah/go.sum index 255353fe2..61bffdb64 100644 --- a/vendor/github.com/containers/buildah/go.sum +++ b/vendor/github.com/containers/buildah/go.sum @@ -165,6 +165,7 @@ github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= +github.com/cilium/ebpf v0.9.0/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -218,7 +219,6 @@ github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0 github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE= -github.com/containerd/containerd v1.6.4/go.mod h1:oWOqbuJUZmOVafhA0lj2NAXbiO1u7F0K5l1bUgdyo94= github.com/containerd/containerd v1.6.6 h1:xJNPhbrmz8xAMDNoVjHy9YHtWwEQNS+CDkcIRh7t8Y0= github.com/containerd/containerd v1.6.6/go.mod h1:ZoP1geJldzCVY3Tonoz7b1IXk8rIX0Nltt5QE4OMNk0= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -240,7 +240,6 @@ github.com/containerd/go-cni v1.0.1/go.mod h1:+vUpYxKvAF72G9i1WoDOiPGRtQpqsNW/ZH github.com/containerd/go-cni v1.0.2/go.mod h1:nrNABBHzu0ZwCug9Ije8hL2xBCYh/pjfMb1aZGrrohk= github.com/containerd/go-cni v1.1.0/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= github.com/containerd/go-cni v1.1.3/go.mod h1:Rflh2EJ/++BA2/vY5ao3K6WJRR/bZKsX123aPk+kUtA= -github.com/containerd/go-cni v1.1.5/go.mod h1:Rf2ZrMycr1El589IyuRzn7RkfdRZVKaFGaxSDHVAjj0= github.com/containerd/go-cni v1.1.6/go.mod h1:BWtoWl5ghVymxu6MBjg79W9NZrCRyHIdUtk4cauMe34= github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/go-runc v0.0.0-20190911050354-e029b79d8cda/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= @@ -279,7 +278,6 @@ github.com/containernetworking/cni v0.7.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ github.com/containernetworking/cni v0.8.0/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v0.8.1/go.mod h1:LGwApLUm2FpoOfxTDEeq8T9ipbpZ61X79hmU3w8FmsY= github.com/containernetworking/cni v1.0.1/go.mod h1:AKuhXbN5EzmD4yTNtfSsX3tPcmtrBI6QcRV0NiNt15Y= -github.com/containernetworking/cni v1.1.0/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= github.com/containernetworking/cni v1.1.1 h1:ky20T7c0MvKvbMOwS/FrlbNwjEoqJEUUYfsL4b0mc4k= github.com/containernetworking/cni v1.1.1/go.mod h1:sDpYKmGVENF3s6uvMvGgldDWeG8dMxakj/u+i9ht9vw= github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHVlzhJpcY6TQxn/fUyDDM= @@ -287,8 +285,8 @@ github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRD github.com/containernetworking/plugins v1.0.1/go.mod h1:QHCfGpaTwYTbbH+nZXKVTxNBDZcxSOplJT5ico8/FLE= github.com/containernetworking/plugins v1.1.1 h1:+AGfFigZ5TiQH00vhR8qPeSatj53eNGz0C1d3wVYlHE= github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8= -github.com/containers/common v0.48.1-0.20220519181648-280c6f69fa82 h1:+FcjjNdCzhLp9jmkkZJ9wxqGwFtQVlKKDR/GWHwTOXY= -github.com/containers/common v0.48.1-0.20220519181648-280c6f69fa82/go.mod h1:Ru/JjL1CTHzlxghVMhchzcFUwHLvlIeR5/SUMw8VUOI= +github.com/containers/common v0.48.1-0.20220608111710-dbecabbe82c9 h1:sK+TNC8oUBkruZTIqwYJrENetSLQnk+goBVyLiqsJq8= +github.com/containers/common v0.48.1-0.20220608111710-dbecabbe82c9/go.mod h1:WBLwq+i7bicCpH54V70HM6s7jqDAESTlYnd05XXp0ac= github.com/containers/image/v5 v5.21.2-0.20220511203756-fe4fd4ed8be4/go.mod h1:OsX9sFexyGF0FCNAjfcVFv3IwMqDyLyV/WQY/roLPcE= github.com/containers/image/v5 v5.21.2-0.20220520105616-e594853d6471 h1:2mm1jEFATvpdFfp8lUB/yc237OqwruMvfIPiVn1Wpgg= github.com/containers/image/v5 v5.21.2-0.20220520105616-e594853d6471/go.mod h1:KntCBNQn3qOuZmQuJ38ORyTozmWXiuo05Vef2S0Sm5M= @@ -304,8 +302,8 @@ github.com/containers/ocicrypt v1.1.4-0.20220428134531-566b808bdf6f/go.mod h1:xp github.com/containers/storage v1.37.0/go.mod h1:kqeJeS0b7DO2ZT1nVWs0XufrmPFbgV3c+Q/45RlH6r4= github.com/containers/storage v1.40.2/go.mod h1:zUyPC3CFIGR1OhY1CKkffxgw9+LuH76PGvVcFj38dgs= github.com/containers/storage v1.41.0/go.mod h1:Pb0l5Sm/89kolX3o2KolKQ5cCHk5vPNpJrhNaLcdS5s= -github.com/containers/storage v1.41.1-0.20220517121726-5019cd55275c h1:DQVf7UhxndNUtZ2+BIS/GtEdzszxMxrdqe43DRKRV2w= -github.com/containers/storage v1.41.1-0.20220517121726-5019cd55275c/go.mod h1:HjV2DQuTFnjKYXDS3foE1EHODXu+dKHi7gT+uxT+kNk= +github.com/containers/storage v1.41.1-0.20220607143333-8951d0153bf6 h1:AWGEIiqWFIfzTIv4Q3k6vJt/EYyo8dh35ny7WhnOd0s= +github.com/containers/storage v1.41.1-0.20220607143333-8951d0153bf6/go.mod h1:6XQ68cEG8ojfP/m3HIupFV1rZsnqeFmaE8N1ctBP94Y= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -373,8 +371,6 @@ github.com/docker/go-metrics v0.0.1 h1:AgB/0SvBxihN0X8OR4SjsblXkbMvalQ8cjmtKQ2rQ github.com/docker/go-metrics v0.0.1/go.mod h1:cG1hvH2utMXtqgqqYE9plW6lDxS3/5ayHzueweSI3Vw= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= -github.com/docker/libnetwork v0.8.0-dev.2.0.20190625141545-5a177b73e316 h1:moehPjPiGUaWdwgOl92xRyFHJyaqXDHcCyW9M6nmCK4= -github.com/docker/libnetwork v0.8.0-dev.2.0.20190625141545-5a177b73e316/go.mod h1:93m0aTqz6z+g32wla4l4WxTrdtvBRmVzYRkYvasA5Z8= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7 h1:UhxFibDNY/bfvqU5CAUmr9zpesgbU6SWc8/B4mflAE4= github.com/docker/libtrust v0.0.0-20160708172513-aabc10ec26b7/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= @@ -401,6 +397,7 @@ github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSw github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/form3tech-oss/jwt-go v3.2.3+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -604,8 +601,6 @@ github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH github.com/inconshreveable/mousetrap v1.0.0 h1:Z8tu5sraLXCXIcARxBp/8cbvlwVa7Z1NHg9XEKhtSvM= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/intel/goresctrl v0.2.0/go.mod h1:+CZdzouYFn5EsxgqAQTEzMfwKwuc0fVdMrT9FCCAVRQ= -github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee h1:PAXLXk1heNZ5yokbMBpVLZQxo43wCZxRwl00mX+dd44= -github.com/ishidawataru/sctp v0.0.0-20210226210310-f2269e66cdee/go.mod h1:co9pwDoBCm1kGxawmb4sPq0cSIOOWNPT4KnHotMP1Zg= github.com/j-keck/arping v0.0.0-20160618110441-2cf9dc699c56/go.mod h1:ymszkNOg6tORTn+6F6j+Jc8TOr5osrynvN6ivFWZ2GA= github.com/j-keck/arping v1.0.2/go.mod h1:aJbELhR92bSk7tp79AWM/ftfc90EfEi2bQJrbBFOsPw= github.com/jinzhu/copier v0.3.5 h1:GlvfUwHk62RokgqVNvYsku0TATCF7bAHVwEXoBh3iJg= @@ -637,19 +632,20 @@ github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdY github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.2/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/compress v1.15.4 h1:1kn4/7MepF/CHmYub99/nNX8az0IJjfSOU/jbnTVfqQ= github.com/klauspost/compress v1.15.4/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY= +github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= -github.com/konsorten/go-windows-terminal-sequences v1.0.3 h1:CE8S1cTafDpPvMhIxNJKvHsGVBgn1xWYf1NbHQhywc8= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -781,9 +777,11 @@ github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84 github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runc v1.0.3/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= +github.com/opencontainers/runc v1.1.1-0.20220607072441-a7a45d7d2721/go.mod h1:QvA0UNe48mC1JxcXq0sENIR38+/LdJMLNxuAvtFBhxA= github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= -github.com/opencontainers/runc v1.1.2 h1:2VSZwLx5k/BfsBxMMipG/LYUnmqOD/BPkIVgQUcTlLw= github.com/opencontainers/runc v1.1.2/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= +github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w= +github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -866,6 +864,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -878,14 +878,22 @@ github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg github.com/sebdah/goldie/v2 v2.5.3 h1:9ES/mNN+HNUbNWpVAlrzuZ7jE+Nrczbj8uFRjM7624Y= github.com/sebdah/goldie/v2 v2.5.3/go.mod h1:oZ9fp0+se1eapSRjfYbsV/0Hqhbuu3bJVvKI/NNtssI= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921 h1:58EBmR2dMNL2n/FnbQewK3D14nXr0V9CObDSvMJLq+Y= github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646 h1:RpforrEYXWkmGwJHIGnLZ3tTWStkjVVstwzNGqxX2Ds= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= +github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= +github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= +github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= +github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= @@ -949,6 +957,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.9/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= github.com/vbauerster/mpb/v7 v7.4.1 h1:NhLMWQ3gNg2KJR8oeA9lO8Xvq+eNPmixDmB6JEQOUdA= @@ -1038,6 +1047,7 @@ go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9i go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= golang.org/x/crypto v0.0.0-20171113213409-9f005a07e0d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181009213950-7c1a557ab941/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -1180,6 +1190,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cO golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1502,8 +1513,9 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/vendor/github.com/containers/buildah/internal/util/util.go b/vendor/github.com/containers/buildah/internal/util/util.go index 20438051b..abaadc616 100644 --- a/vendor/github.com/containers/buildah/internal/util/util.go +++ b/vendor/github.com/containers/buildah/internal/util/util.go @@ -8,6 +8,8 @@ import ( "github.com/containers/buildah/define" "github.com/containers/common/libimage" "github.com/containers/image/v5/types" + encconfig "github.com/containers/ocicrypt/config" + enchelpers "github.com/containers/ocicrypt/helpers" "github.com/containers/storage" "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/chrootarchive" @@ -87,3 +89,49 @@ func ExportFromReader(input io.Reader, opts define.BuildOutputOption) error { } return nil } + +// DecryptConfig translates decryptionKeys into a DescriptionConfig structure +func DecryptConfig(decryptionKeys []string) (*encconfig.DecryptConfig, error) { + decryptConfig := &encconfig.DecryptConfig{} + if len(decryptionKeys) > 0 { + // decryption + dcc, err := enchelpers.CreateCryptoConfig([]string{}, decryptionKeys) + if err != nil { + return nil, errors.Wrapf(err, "invalid decryption keys") + } + cc := encconfig.CombineCryptoConfigs([]encconfig.CryptoConfig{dcc}) + decryptConfig = cc.DecryptConfig + } + + return decryptConfig, nil +} + +// EncryptConfig translates encryptionKeys into a EncriptionsConfig structure +func EncryptConfig(encryptionKeys []string, encryptLayers []int) (*encconfig.EncryptConfig, *[]int, error) { + var encLayers *[]int + var encConfig *encconfig.EncryptConfig + + if len(encryptionKeys) > 0 { + // encryption + encLayers = &encryptLayers + ecc, err := enchelpers.CreateCryptoConfig(encryptionKeys, []string{}) + if err != nil { + return nil, nil, errors.Wrapf(err, "invalid encryption keys") + } + cc := encconfig.CombineCryptoConfigs([]encconfig.CryptoConfig{ecc}) + encConfig = cc.EncryptConfig + } + return encConfig, encLayers, nil +} + +// GetFormat translates format string into either docker or OCI format constant +func GetFormat(format string) (string, error) { + switch format { + case define.OCI: + return define.OCIv1ImageManifest, nil + case define.DOCKER: + return define.Dockerv2ImageManifest, nil + default: + return "", errors.Errorf("unrecognized image type %q", format) + } +} diff --git a/vendor/github.com/containers/buildah/pkg/cli/build.go b/vendor/github.com/containers/buildah/pkg/cli/build.go index c9f7de5de..396a9e74e 100644 --- a/vendor/github.com/containers/buildah/pkg/cli/build.go +++ b/vendor/github.com/containers/buildah/pkg/cli/build.go @@ -14,11 +14,10 @@ import ( "time" "github.com/containers/buildah/define" + iutil "github.com/containers/buildah/internal/util" "github.com/containers/buildah/pkg/parse" "github.com/containers/buildah/pkg/util" "github.com/containers/common/pkg/auth" - encconfig "github.com/containers/ocicrypt/config" - enchelpers "github.com/containers/ocicrypt/helpers" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -125,7 +124,7 @@ func GenBuildOptions(c *cobra.Command, inputArgs []string, iopts BuildOptions) ( } containerfiles := getContainerfiles(iopts.File) - format, err := GetFormat(iopts.Format) + format, err := iutil.GetFormat(iopts.Format) if err != nil { return options, nil, nil, err } @@ -266,7 +265,7 @@ func GenBuildOptions(c *cobra.Command, inputArgs []string, iopts BuildOptions) ( return options, nil, nil, err } - decryptConfig, err := DecryptConfig(iopts.DecryptionKeys) + decryptConfig, err := iutil.DecryptConfig(iopts.DecryptionKeys) if err != nil { return options, nil, nil, errors.Wrapf(err, "unable to obtain decrypt config") } @@ -293,44 +292,51 @@ func GenBuildOptions(c *cobra.Command, inputArgs []string, iopts BuildOptions) ( } options = define.BuildOptions{ AddCapabilities: iopts.CapAdd, + AdditionalBuildContexts: additionalBuildContext, AdditionalTags: tags, AllPlatforms: iopts.AllPlatforms, Annotations: iopts.Annotation, Architecture: systemContext.ArchitectureChoice, Args: args, - AdditionalBuildContexts: additionalBuildContext, BlobDirectory: iopts.BlobCache, + BuildOutput: iopts.BuildOutput, CNIConfigDir: iopts.CNIConfigDir, CNIPluginPath: iopts.CNIPlugInPath, + CPPFlags: iopts.CPPFlags, CommonBuildOpts: commonOpts, Compression: compression, ConfigureNetwork: networkPolicy, ContextDirectory: contextDir, - CPPFlags: iopts.CPPFlags, Devices: iopts.Devices, DropCapabilities: iopts.CapDrop, + Envs: iopts.Envs, Err: stderr, + Excludes: excludes, ForceRmIntermediateCtrs: iopts.ForceRm, From: iopts.From, IDMappingOptions: idmappingOptions, IIDFile: iopts.Iidfile, + IgnoreFile: iopts.IgnoreFile, In: stdin, Isolation: isolation, - IgnoreFile: iopts.IgnoreFile, + Jobs: &iopts.Jobs, Labels: iopts.Label, Layers: layers, LogFile: iopts.Logfile, - LogSplitByPlatform: iopts.LogSplitByPlatform, LogRusage: iopts.LogRusage, + LogSplitByPlatform: iopts.LogSplitByPlatform, Manifest: iopts.Manifest, MaxPullPushRetries: MaxPullPushRetries, NamespaceOptions: namespaceOptions, NoCache: iopts.NoCache, OS: systemContext.OSChoice, + OSFeatures: iopts.OSFeatures, + OSVersion: iopts.OSVersion, + OciDecryptConfig: decryptConfig, Out: stdout, Output: output, - BuildOutput: iopts.BuildOutput, OutputFormat: format, + Platforms: platforms, PullPolicy: pullPolicy, PullPushRetryDelay: PullPushRetryDelay, Quiet: iopts.Quiet, @@ -344,16 +350,9 @@ func GenBuildOptions(c *cobra.Command, inputArgs []string, iopts BuildOptions) ( Squash: iopts.Squash, SystemContext: systemContext, Target: iopts.Target, - TransientMounts: iopts.Volumes, - OciDecryptConfig: decryptConfig, - Jobs: &iopts.Jobs, - Excludes: excludes, Timestamp: timestamp, - Platforms: platforms, + TransientMounts: iopts.Volumes, UnsetEnvs: iopts.UnsetEnvs, - Envs: iopts.Envs, - OSFeatures: iopts.OSFeatures, - OSVersion: iopts.OSVersion, } if iopts.Quiet { options.ReportWriter = ioutil.Discard @@ -372,49 +371,3 @@ func getContainerfiles(files []string) []string { } return containerfiles } - -// GetFormat translates format string into either docker or OCI format constant -func GetFormat(format string) (string, error) { - switch format { - case define.OCI: - return define.OCIv1ImageManifest, nil - case define.DOCKER: - return define.Dockerv2ImageManifest, nil - default: - return "", errors.Errorf("unrecognized image type %q", format) - } -} - -// DecryptConfig translates decryptionKeys into a DescriptionConfig structure -func DecryptConfig(decryptionKeys []string) (*encconfig.DecryptConfig, error) { - decryptConfig := &encconfig.DecryptConfig{} - if len(decryptionKeys) > 0 { - // decryption - dcc, err := enchelpers.CreateCryptoConfig([]string{}, decryptionKeys) - if err != nil { - return nil, errors.Wrapf(err, "invalid decryption keys") - } - cc := encconfig.CombineCryptoConfigs([]encconfig.CryptoConfig{dcc}) - decryptConfig = cc.DecryptConfig - } - - return decryptConfig, nil -} - -// EncryptConfig translates encryptionKeys into a EncriptionsConfig structure -func EncryptConfig(encryptionKeys []string, encryptLayers []int) (*encconfig.EncryptConfig, *[]int, error) { - var encLayers *[]int - var encConfig *encconfig.EncryptConfig - - if len(encryptionKeys) > 0 { - // encryption - encLayers = &encryptLayers - ecc, err := enchelpers.CreateCryptoConfig(encryptionKeys, []string{}) - if err != nil { - return nil, nil, errors.Wrapf(err, "invalid encryption keys") - } - cc := encconfig.CombineCryptoConfigs([]encconfig.CryptoConfig{ecc}) - encConfig = cc.EncryptConfig - } - return encConfig, encLayers, nil -} diff --git a/vendor/github.com/containers/buildah/run_linux.go b/vendor/github.com/containers/buildah/run_linux.go index 8a3cdc8d7..3d2a83f55 100644 --- a/vendor/github.com/containers/buildah/run_linux.go +++ b/vendor/github.com/containers/buildah/run_linux.go @@ -35,6 +35,7 @@ import ( "github.com/containers/buildah/util" "github.com/containers/common/libnetwork/etchosts" "github.com/containers/common/libnetwork/network" + "github.com/containers/common/libnetwork/resolvconf" nettypes "github.com/containers/common/libnetwork/types" "github.com/containers/common/pkg/capabilities" "github.com/containers/common/pkg/chown" @@ -50,8 +51,6 @@ import ( "github.com/containers/storage/pkg/unshare" storagetypes "github.com/containers/storage/types" "github.com/docker/go-units" - "github.com/docker/libnetwork/resolvconf" - "github.com/docker/libnetwork/types" "github.com/opencontainers/go-digest" "github.com/opencontainers/runtime-spec/specs-go" spec "github.com/opencontainers/runtime-spec/specs-go" @@ -250,7 +249,6 @@ func (b *Builder) Run(command []string, options RunOptions) error { } bindFiles := make(map[string]string) - namespaceOptions := append(b.NamespaceOptions, options.NamespaceOptions...) volumes := b.Volumes() // Figure out who owns files that will appear to be owned by UID/GID 0 in the container. @@ -281,15 +279,12 @@ func (b *Builder) Run(command []string, options RunOptions) error { } } - if !(contains(volumes, "/etc/resolv.conf") || (len(b.CommonBuildOpts.DNSServers) == 1 && strings.ToLower(b.CommonBuildOpts.DNSServers[0]) == "none")) { - resolvFile, err := b.addResolvConf(path, rootIDPair, b.CommonBuildOpts.DNSServers, b.CommonBuildOpts.DNSSearch, b.CommonBuildOpts.DNSOptions, namespaceOptions) + if !contains(volumes, resolvconf.DefaultResolvConf) && options.ConfigureNetwork != define.NetworkDisabled && !(len(b.CommonBuildOpts.DNSServers) == 1 && strings.ToLower(b.CommonBuildOpts.DNSServers[0]) == "none") { + resolvFile, err := b.addResolvConf(path, rootIDPair, b.CommonBuildOpts.DNSServers, b.CommonBuildOpts.DNSSearch, b.CommonBuildOpts.DNSOptions, spec.Linux.Namespaces) if err != nil { return err } - // Only bind /etc/resolv.conf if there's a network - if options.ConfigureNetwork != define.NetworkDisabled { - bindFiles["/etc/resolv.conf"] = resolvFile - } + bindFiles[resolvconf.DefaultResolvConf] = resolvFile } // Empty file, so no need to recreate if it exists if _, ok := bindFiles["/run/.containerenv"]; !ok { @@ -595,94 +590,52 @@ func cleanableDestinationListFromMounts(mounts []spec.Mount) []string { } // addResolvConf copies files from host and sets them up to bind mount into container -func (b *Builder) addResolvConf(rdir string, chownOpts *idtools.IDPair, dnsServers, dnsSearch, dnsOptions []string, namespaceOptions define.NamespaceOptions) (string, error) { - resolvConf := "/etc/resolv.conf" - - stat, err := os.Stat(resolvConf) +func (b *Builder) addResolvConf(rdir string, chownOpts *idtools.IDPair, dnsServers, dnsSearch, dnsOptions []string, namespaces []specs.LinuxNamespace) (string, error) { + defaultConfig, err := config.Default() if err != nil { - return "", err - } - contents, err := ioutil.ReadFile(resolvConf) - // resolv.conf doesn't have to exists - if err != nil && !os.IsNotExist(err) { - return "", err + return "", errors.Wrapf(err, "failed to get config") } - netns := false - ns := namespaceOptions.Find(string(spec.NetworkNamespace)) - if ns != nil && !ns.Host { - netns = true - } + nameservers := make([]string, 0, len(defaultConfig.Containers.DNSServers)+len(dnsServers)) + nameservers = append(nameservers, defaultConfig.Containers.DNSServers...) + nameservers = append(nameservers, dnsServers...) - nameservers := resolvconf.GetNameservers(contents, types.IPv4) - // check if systemd-resolved is used, assume it is used when 127.0.0.53 is the only nameserver - if len(nameservers) == 1 && nameservers[0] == "127.0.0.53" && netns { - // read the actual resolv.conf file for systemd-resolved - resolvedContents, err := ioutil.ReadFile("/run/systemd/resolve/resolv.conf") - if err != nil { - if !os.IsNotExist(err) { - return "", errors.Wrapf(err, "detected that systemd-resolved is in use, but could not locate real resolv.conf") + keepHostServers := false + // special check for slirp ip + if len(nameservers) == 0 && b.Isolation == IsolationOCIRootless { + for _, ns := range namespaces { + if ns.Type == specs.NetworkNamespace && ns.Path == "" { + keepHostServers = true + // if we are using slirp4netns, also add the built-in DNS server. + logrus.Debugf("adding slirp4netns 10.0.2.3 built-in DNS server") + nameservers = append([]string{"10.0.2.3"}, nameservers...) } - } else { - contents = resolvedContents } } - // Ensure that the container's /etc/resolv.conf is compatible with its - // network configuration. - if netns { - // FIXME handle IPv6 - resolve, err := resolvconf.FilterResolvDNS(contents, true) - if err != nil { - return "", errors.Wrapf(err, "error parsing host resolv.conf") - } - contents = resolve.Content - } - search := resolvconf.GetSearchDomains(contents) - nameservers = resolvconf.GetNameservers(contents, types.IP) - options := resolvconf.GetOptions(contents) + searches := make([]string, 0, len(defaultConfig.Containers.DNSSearches)+len(dnsSearch)) + searches = append(searches, defaultConfig.Containers.DNSSearches...) + searches = append(searches, dnsSearch...) - defaultContainerConfig, err := config.Default() - if err != nil { - return "", errors.Wrapf(err, "failed to get container config") - } - dnsSearch = append(defaultContainerConfig.Containers.DNSSearches, dnsSearch...) - if len(dnsSearch) > 0 { - search = dnsSearch - } + options := make([]string, 0, len(defaultConfig.Containers.DNSOptions)+len(dnsOptions)) + options = append(options, defaultConfig.Containers.DNSOptions...) + options = append(options, dnsOptions...) - if b.Isolation == IsolationOCIRootless { - if ns != nil && !ns.Host && ns.Path == "" { - // if we are using slirp4netns, also add the built-in DNS server. - logrus.Debugf("adding slirp4netns 10.0.2.3 built-in DNS server") - nameservers = append([]string{"10.0.2.3"}, nameservers...) - } - } - - dnsServers = append(defaultContainerConfig.Containers.DNSServers, dnsServers...) - if len(dnsServers) != 0 { - dns, err := getDNSIP(dnsServers) - if err != nil { - return "", errors.Wrapf(err, "error getting dns servers") - } - nameservers = []string{} - for _, server := range dns { - nameservers = append(nameservers, server.String()) - } - } - - dnsOptions = append(defaultContainerConfig.Containers.DNSOptions, dnsOptions...) - if len(dnsOptions) != 0 { - options = dnsOptions - } - - cfile := filepath.Join(rdir, filepath.Base(resolvConf)) - if _, err = resolvconf.Build(cfile, nameservers, search, options); err != nil { + cfile := filepath.Join(rdir, "resolv.conf") + if err := resolvconf.New(&resolvconf.Params{ + Path: cfile, + Namespaces: namespaces, + IPv6Enabled: true, // TODO we should check if we have ipv6 + KeepHostServers: keepHostServers, + Nameservers: nameservers, + Searches: searches, + Options: options, + }); err != nil { return "", errors.Wrapf(err, "error building resolv.conf for container %s", b.ContainerID) } - uid := int(stat.Sys().(*syscall.Stat_t).Uid) - gid := int(stat.Sys().(*syscall.Stat_t).Gid) + uid := 0 + gid := 0 if chownOpts != nil { uid = chownOpts.UID gid = chownOpts.GID @@ -2096,17 +2049,6 @@ func runLookupPath(g *generate.Generator, command []string) []string { return command } -func getDNSIP(dnsServers []string) (dns []net.IP, err error) { - for _, i := range dnsServers { - result := net.ParseIP(i) - if result == nil { - return dns, errors.Errorf("invalid IP address %s", i) - } - dns = append(dns, result) - } - return dns, nil -} - func (b *Builder) configureUIDGID(g *generate.Generator, mountPoint string, options RunOptions) (string, error) { // Set the user UID/GID/supplemental group list/capabilities lists. user, homeDir, err := b.userForRun(mountPoint, options.User) diff --git a/vendor/github.com/containers/buildah/run_unix.go b/vendor/github.com/containers/buildah/run_unix.go index 9e62691e8..280176dba 100644 --- a/vendor/github.com/containers/buildah/run_unix.go +++ b/vendor/github.com/containers/buildah/run_unix.go @@ -5,6 +5,7 @@ package buildah import ( "github.com/containers/buildah/define" nettypes "github.com/containers/common/libnetwork/types" + "github.com/opencontainers/runtime-spec/specs-go" "github.com/containers/storage" "github.com/pkg/errors" ) @@ -22,10 +23,19 @@ func (b *Builder) Run(command []string, options RunOptions) error { return errors.New("function not supported on non-linux systems") } func DefaultNamespaceOptions() (NamespaceOptions, error) { - return NamespaceOptions{}, errors.New("function not supported on non-linux systems") + options := NamespaceOptions{ + {Name: string(specs.CgroupNamespace), Host: false}, + {Name: string(specs.IPCNamespace), Host: false}, + {Name: string(specs.MountNamespace), Host: false}, + {Name: string(specs.NetworkNamespace), Host: false}, + {Name: string(specs.PIDNamespace), Host: false}, + {Name: string(specs.UserNamespace), Host: false}, + {Name: string(specs.UTSNamespace), Host: false}, + } + return options, nil } // getNetworkInterface creates the network interface func getNetworkInterface(store storage.Store, cniConfDir, cniPluginPath string) (nettypes.ContainerNetwork, error) { - return nil, errors.New("function not supported on non-linux systems") + return nil, nil } diff --git a/vendor/github.com/containers/common/libimage/copier.go b/vendor/github.com/containers/common/libimage/copier.go index 01cedc7ed..1cba29143 100644 --- a/vendor/github.com/containers/common/libimage/copier.go +++ b/vendor/github.com/containers/common/libimage/copier.go @@ -139,7 +139,7 @@ type CopyOptions struct { // copier is an internal helper to conveniently copy images. type copier struct { imageCopyOptions copy.Options - retryOptions retry.RetryOptions + retryOptions retry.Options systemContext *types.SystemContext policyContext *signature.PolicyContext @@ -370,7 +370,7 @@ func (c *copier) copy(ctx context.Context, source, destination types.ImageRefere } return err } - return returnManifest, retry.RetryIfNecessary(ctx, f, &c.retryOptions) + return returnManifest, retry.IfNecessary(ctx, f, &c.retryOptions) } // checkRegistrySourcesAllows checks the $BUILD_REGISTRY_SOURCES environment diff --git a/vendor/github.com/containers/common/libimage/inspect.go b/vendor/github.com/containers/common/libimage/inspect.go index ae06acd2c..5da8df1bf 100644 --- a/vendor/github.com/containers/common/libimage/inspect.go +++ b/vendor/github.com/containers/common/libimage/inspect.go @@ -216,7 +216,7 @@ func (i *Image) inspectInfo(ctx context.Context) (*types.ImageInspectInfo, error return nil, err } - img, err := ref.NewImage(ctx, i.runtime.systemContextCopy()) + img, err := ref.NewImage(ctx, &i.runtime.systemContext) if err != nil { return nil, err } diff --git a/vendor/github.com/containers/common/libimage/platform.go b/vendor/github.com/containers/common/libimage/platform.go new file mode 100644 index 000000000..8b78bce24 --- /dev/null +++ b/vendor/github.com/containers/common/libimage/platform.go @@ -0,0 +1,63 @@ +package libimage + +import ( + "context" + "fmt" + "runtime" +) + +// PlatformPolicy controls the behavior of image-platform matching. +type PlatformPolicy int + +const ( + // Only debug log if an image does not match the expected platform. + PlatformPolicyDefault PlatformPolicy = iota + // Warn if an image does not match the expected platform. + PlatformPolicyWarn +) + +func toPlatformString(architecture, os, variant string) string { + if variant == "" { + return fmt.Sprintf("%s/%s", os, architecture) + } + return fmt.Sprintf("%s/%s/%s", os, architecture, variant) +} + +// Checks whether the image matches the specified platform. +// Returns +// * 1) a matching error that can be used for logging (or returning) what does not match +// * 2) a bool indicating whether architecture, os or variant were set (some callers need that to decide whether they need to throw an error) +// * 3) a fatal error that occurred prior to check for matches (e.g., storage errors etc.) +func (i *Image) matchesPlatform(ctx context.Context, architecture, os, variant string) (error, bool, error) { + customPlatform := len(architecture)+len(os)+len(variant) != 0 + + if len(architecture) == 0 { + architecture = runtime.GOARCH + } + if len(os) == 0 { + os = runtime.GOOS + } + + inspectInfo, err := i.inspectInfo(ctx) + if err != nil { + return nil, customPlatform, fmt.Errorf("inspecting image: %w", err) + } + + matches := true + switch { + case architecture != inspectInfo.Architecture: + matches = false + case os != inspectInfo.Os: + matches = false + case variant != "" && variant != inspectInfo.Variant: + matches = false + } + + if matches { + return nil, customPlatform, nil + } + + imagePlatform := toPlatformString(inspectInfo.Architecture, inspectInfo.Os, inspectInfo.Variant) + expectedPlatform := toPlatformString(architecture, os, variant) + return fmt.Errorf("image platform (%s) does not match the expected platform (%s)", imagePlatform, expectedPlatform), customPlatform, nil +} diff --git a/vendor/github.com/containers/common/libimage/pull.go b/vendor/github.com/containers/common/libimage/pull.go index d204ef1c4..5e743574c 100644 --- a/vendor/github.com/containers/common/libimage/pull.go +++ b/vendor/github.com/containers/common/libimage/pull.go @@ -161,11 +161,30 @@ func (r *Runtime) Pull(ctx context.Context, name string, pullPolicy config.PullP localImages := []*Image{} for _, name := range pulledImages { - local, _, err := r.LookupImage(name, nil) + image, _, err := r.LookupImage(name, nil) if err != nil { return nil, errors.Wrapf(err, "error locating pulled image %q name in containers storage", name) } - localImages = append(localImages, local) + + // Note that we can ignore the 2nd return value here. Some + // images may ship with "wrong" platform, but we already warn + // about it. Throwing an error is not (yet) the plan. + matchError, _, err := image.matchesPlatform(ctx, options.Architecture, options.OS, options.Variant) + if err != nil { + return nil, fmt.Errorf("checking platform of image %s: %w", name, err) + } + + // If the image does not match the expected/requested platform, + // make sure to leave some breadcrumbs for the user. + if matchError != nil { + if options.Writer == nil { + logrus.Warnf("%v", matchError) + } else { + fmt.Fprintf(options.Writer, "WARNING: %v\n", matchError) + } + } + + localImages = append(localImages, image) } return localImages, pullError diff --git a/vendor/github.com/containers/common/libimage/runtime.go b/vendor/github.com/containers/common/libimage/runtime.go index 472482410..efae2238d 100644 --- a/vendor/github.com/containers/common/libimage/runtime.go +++ b/vendor/github.com/containers/common/libimage/runtime.go @@ -182,6 +182,9 @@ type LookupImageOptions struct { // Lookup an image matching the specified variant. Variant string + // Controls the behavior when checking the platform of an image. + PlatformPolicy PlatformPolicy + // If set, do not look for items/instances in the manifest list that // match the current platform but return the manifest list as is. // only check for manifest list, return ErrNotAManifestList if not found. @@ -378,21 +381,36 @@ func (r *Runtime) lookupImageInLocalStorage(name, candidate string, options *Loo image = instance } - matches, err := r.imageReferenceMatchesContext(ref, options) - if err != nil { - return nil, err - } - - // NOTE: if the user referenced by ID we must optimistically assume - // that they know what they're doing. Given, we already did the - // manifest limbo above, we may already have resolved it. - if !matches && !strings.HasPrefix(image.ID(), candidate) { - return nil, nil - } // Also print the string within the storage transport. That may aid in // debugging when using additional stores since we see explicitly where // the store is and which driver (options) are used. logrus.Debugf("Found image %q as %q in local containers storage (%s)", name, candidate, ref.StringWithinTransport()) + + // Do not perform any further platform checks if the image was + // requested by ID. In that case, we must assume that the user/tool + // know what they're doing. + if strings.HasPrefix(image.ID(), candidate) { + return image, nil + } + + // Ignore the (fatal) error since the image may be corrupted, which + // will bubble up at other places. During lookup, we just return it as + // is. + if matchError, customPlatform, _ := image.matchesPlatform(context.Background(), options.Architecture, options.OS, options.Variant); matchError != nil { + if customPlatform { + logrus.Debugf("%v", matchError) + // Return nil if the user clearly requested a custom + // platform and the located image does not match. + return nil, nil + } + switch options.PlatformPolicy { + case PlatformPolicyDefault: + logrus.Debugf("%v", matchError) + case PlatformPolicyWarn: + logrus.Warnf("%v", matchError) + } + } + return image, nil } @@ -497,40 +515,6 @@ func (r *Runtime) ResolveName(name string) (string, error) { return normalized.String(), nil } -// imageReferenceMatchesContext return true if the specified reference matches -// the platform (os, arch, variant) as specified by the lookup options. -func (r *Runtime) imageReferenceMatchesContext(ref types.ImageReference, options *LookupImageOptions) (bool, error) { - if options.Architecture+options.OS+options.Variant == "" { - return true, nil - } - - ctx := context.Background() - img, err := ref.NewImage(ctx, &r.systemContext) - if err != nil { - return false, err - } - defer img.Close() - data, err := img.Inspect(ctx) - if err != nil { - return false, err - } - - if options.Architecture != "" && options.Architecture != data.Architecture { - logrus.Debugf("architecture %q does not match architecture %q of image %s", options.Architecture, data.Architecture, ref) - return false, nil - } - if options.OS != "" && options.OS != data.Os { - logrus.Debugf("OS %q does not match OS %q of image %s", options.OS, data.Os, ref) - return false, nil - } - if options.Variant != "" && options.Variant != data.Variant { - logrus.Debugf("variant %q does not match variant %q of image %s", options.Variant, data.Variant, ref) - return false, nil - } - - return true, nil -} - // IsExternalContainerFunc allows for checking whether the specified container // is an external one. The definition of an external container can be set by // callers. diff --git a/vendor/github.com/containers/common/pkg/completion/command.go b/vendor/github.com/containers/common/pkg/completion/command.go deleted file mode 100644 index 2deb58757..000000000 --- a/vendor/github.com/containers/common/pkg/completion/command.go +++ /dev/null @@ -1,96 +0,0 @@ -package completion - -import ( - "fmt" - "io" - "os" - "strings" - - "github.com/spf13/cobra" -) - -const ( - completionDescription = `Generate shell autocompletions. -Valid arguments are bash, zsh, fish and powershell.` - - bash = "bash" - zsh = "zsh" - fish = "fish" - powershell = "powershell" -) - -var ( - file string - noDesc bool - shells = []string{bash, zsh, fish, powershell} -) - -// AddCompletionCommand adds the completion command to the given command which should be the root command. -// This command can be used the generate the cobra shell completion scripts for bash, zsh, fish and powershell. -func AddCompletionCommand(rootCmd *cobra.Command) { - completionCmd := &cobra.Command{ - Use: fmt.Sprintf("completion [options] {%s}", strings.Join(shells, "|")), - Short: "Generate shell autocompletions", - Long: completionDescription, - ValidArgs: shells, - Args: cobra.ExactValidArgs(1), - RunE: completion, - Example: fmt.Sprintf(`%[1]s completion bash - %[1]s completion zsh -f _%[1]s - %[1]s completion fish --no-desc`, rootCmd.Name()), - // don't show this command to users - Hidden: true, - } - - flags := completionCmd.Flags() - fileFlagName := "file" - flags.StringVarP(&file, fileFlagName, "f", "", "Output the completion to file rather than stdout.") - _ = completionCmd.RegisterFlagCompletionFunc(fileFlagName, AutocompleteDefault) - - flags.BoolVar(&noDesc, "no-desc", false, "Don't include descriptions in the completion output.") - - rootCmd.AddCommand(completionCmd) -} - -func completion(cmd *cobra.Command, args []string) error { - var w io.Writer - - if file != "" { - file, err := os.Create(file) - if err != nil { - return err - } - defer file.Close() - w = file - } else { - w = os.Stdout - } - - var err error - switch args[0] { - case bash: - err = cmd.Root().GenBashCompletionV2(w, !noDesc) - case zsh: - if noDesc { - err = cmd.Root().GenZshCompletionNoDesc(w) - } else { - err = cmd.Root().GenZshCompletion(w) - } - case fish: - err = cmd.Root().GenFishCompletion(w, !noDesc) - case powershell: - if noDesc { - err = cmd.Root().GenPowerShellCompletion(w) - } else { - err = cmd.Root().GenPowerShellCompletionWithDesc(w) - } - } - if err != nil { - return err - } - - _, err = io.WriteString(w, fmt.Sprintf( - "# This file is generated with %q; DO NOT EDIT!\n", cmd.CommandPath(), - )) - return err -} diff --git a/vendor/github.com/containers/common/pkg/retry/retry.go b/vendor/github.com/containers/common/pkg/retry/retry.go index 234fd3448..321131f69 100644 --- a/vendor/github.com/containers/common/pkg/retry/retry.go +++ b/vendor/github.com/containers/common/pkg/retry/retry.go @@ -16,26 +16,29 @@ import ( "github.com/sirupsen/logrus" ) -// RetryOptions defines the option to retry -// revive does not like the name because the package is already called retry -//nolint:revive -type RetryOptions struct { - MaxRetry int // The number of times to possibly retry - Delay time.Duration // The delay to use between retries, if set +// Options defines the option to retry. +type Options struct { + MaxRetry int // The number of times to possibly retry. + Delay time.Duration // The delay to use between retries, if set. } -// RetryIfNecessary retries the operation in exponential backoff with the retryOptions -// -// revive does not like the name because the package is already called retry -//nolint:revive -func RetryIfNecessary(ctx context.Context, operation func() error, retryOptions *RetryOptions) error { +// RetryOptions is deprecated, use Options. +type RetryOptions = Options // nolint:revive + +// RetryIfNecessary deprecated function use IfNecessary. +func RetryIfNecessary(ctx context.Context, operation func() error, options *Options) error { // nolint:revive + return IfNecessary(ctx, operation, options) +} + +// IfNecessary retries the operation in exponential backoff with the retry Options. +func IfNecessary(ctx context.Context, operation func() error, options *Options) error { err := operation() - for attempt := 0; err != nil && isRetryable(err) && attempt < retryOptions.MaxRetry; attempt++ { + for attempt := 0; err != nil && isRetryable(err) && attempt < options.MaxRetry; attempt++ { delay := time.Duration(int(math.Pow(2, float64(attempt)))) * time.Second - if retryOptions.Delay != 0 { - delay = retryOptions.Delay + if options.Delay != 0 { + delay = options.Delay } - logrus.Warnf("Failed, retrying in %s ... (%d/%d). Error: %v", delay, attempt+1, retryOptions.MaxRetry, err) + logrus.Warnf("Failed, retrying in %s ... (%d/%d). Error: %v", delay, attempt+1, options.MaxRetry, err) select { case <-time.After(delay): break @@ -96,6 +99,14 @@ func isRetryable(err error) bool { } } return true + case net.Error: + if e.Timeout() { + return true + } + if unwrappable, ok := e.(unwrapper); ok { + err = unwrappable.Unwrap() + return isRetryable(err) + } case unwrapper: // Test this last, because various error types might implement .Unwrap() err = e.Unwrap() return isRetryable(err) diff --git a/vendor/github.com/containers/image/v5/copy/blob.go b/vendor/github.com/containers/image/v5/copy/blob.go new file mode 100644 index 000000000..020e703e8 --- /dev/null +++ b/vendor/github.com/containers/image/v5/copy/blob.go @@ -0,0 +1,170 @@ +package copy + +import ( + "context" + "io" + + "github.com/containers/image/v5/internal/private" + compressiontypes "github.com/containers/image/v5/pkg/compression/types" + "github.com/containers/image/v5/types" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// copyBlobFromStream copies a blob with srcInfo (with known Digest and Annotations and possibly known Size) from srcReader to dest, +// perhaps sending a copy to an io.Writer if getOriginalLayerCopyWriter != nil, +// perhaps (de/re/)compressing it if canModifyBlob, +// and returns a complete blobInfo of the copied blob. +func (ic *imageCopier) copyBlobFromStream(ctx context.Context, srcReader io.Reader, srcInfo types.BlobInfo, + getOriginalLayerCopyWriter func(decompressor compressiontypes.DecompressorFunc) io.Writer, + isConfig bool, toEncrypt bool, bar *progressBar, layerIndex int, emptyLayer bool) (types.BlobInfo, error) { + // The copying happens through a pipeline of connected io.Readers; + // that pipeline is built by updating stream. + // === Input: srcReader + stream := sourceStream{ + reader: srcReader, + info: srcInfo, + } + + // === Process input through digestingReader to validate against the expected digest. + // Be paranoid; in case PutBlob somehow managed to ignore an error from digestingReader, + // use a separate validation failure indicator. + // Note that for this check we don't use the stronger "validationSucceeded" indicator, because + // dest.PutBlob may detect that the layer already exists, in which case we don't + // read stream to the end, and validation does not happen. + digestingReader, err := newDigestingReader(stream.reader, srcInfo.Digest) + if err != nil { + return types.BlobInfo{}, errors.Wrapf(err, "preparing to verify blob %s", srcInfo.Digest) + } + stream.reader = digestingReader + + // === Update progress bars + stream.reader = bar.ProxyReader(stream.reader) + + // === Decrypt the stream, if required. + decryptionStep, err := ic.c.blobPipelineDecryptionStep(&stream, srcInfo) + if err != nil { + return types.BlobInfo{}, err + } + + // === Detect compression of the input stream. + // This requires us to “peek ahead” into the stream to read the initial part, which requires us to chain through another io.Reader returned by DetectCompression. + detectedCompression, err := blobPipelineDetectCompressionStep(&stream, srcInfo) + if err != nil { + return types.BlobInfo{}, err + } + + // === Send a copy of the original, uncompressed, stream, to a separate path if necessary. + var originalLayerReader io.Reader // DO NOT USE this other than to drain the input if no other consumer in the pipeline has done so. + if getOriginalLayerCopyWriter != nil { + stream.reader = io.TeeReader(stream.reader, getOriginalLayerCopyWriter(detectedCompression.decompressor)) + originalLayerReader = stream.reader + } + + // WARNING: If you are adding new reasons to change the blob, update also the OptimizeDestinationImageAlreadyExists + // short-circuit conditions + canModifyBlob := !isConfig && ic.cannotModifyManifestReason == "" + // === Deal with layer compression/decompression if necessary + compressionStep, err := ic.blobPipelineCompressionStep(&stream, canModifyBlob, srcInfo, detectedCompression) + if err != nil { + return types.BlobInfo{}, err + } + defer compressionStep.close() + + // === Encrypt the stream for valid mediatypes if ociEncryptConfig provided + if decryptionStep.decrypting && toEncrypt { + // If nothing else, we can only set uploadedInfo.CryptoOperation to a single value. + // Before relaxing this, see the original pull request’s review if there are other reasons to reject this. + return types.BlobInfo{}, errors.New("Unable to support both decryption and encryption in the same copy") + } + encryptionStep, err := ic.c.blobPipelineEncryptionStep(&stream, toEncrypt, srcInfo, decryptionStep) + if err != nil { + return types.BlobInfo{}, err + } + + // === Report progress using the ic.c.progress channel, if required. + if ic.c.progress != nil && ic.c.progressInterval > 0 { + progressReader := newProgressReader( + stream.reader, + ic.c.progress, + ic.c.progressInterval, + srcInfo, + ) + defer progressReader.reportDone() + stream.reader = progressReader + } + + // === Finally, send the layer stream to dest. + options := private.PutBlobOptions{ + Cache: ic.c.blobInfoCache, + IsConfig: isConfig, + EmptyLayer: emptyLayer, + } + if !isConfig { + options.LayerIndex = &layerIndex + } + uploadedInfo, err := ic.c.dest.PutBlobWithOptions(ctx, &errorAnnotationReader{stream.reader}, stream.info, options) + if err != nil { + return types.BlobInfo{}, errors.Wrap(err, "writing blob") + } + + uploadedInfo.Annotations = stream.info.Annotations + + compressionStep.updateCompressionEdits(&uploadedInfo.CompressionOperation, &uploadedInfo.CompressionAlgorithm, &uploadedInfo.Annotations) + decryptionStep.updateCryptoOperation(&uploadedInfo.CryptoOperation) + if err := encryptionStep.updateCryptoOperationAndAnnotations(&uploadedInfo.CryptoOperation, &uploadedInfo.Annotations); err != nil { + return types.BlobInfo{}, err + } + + // This is fairly horrible: the writer from getOriginalLayerCopyWriter wants to consume + // all of the input (to compute DiffIDs), even if dest.PutBlob does not need it. + // So, read everything from originalLayerReader, which will cause the rest to be + // sent there if we are not already at EOF. + if getOriginalLayerCopyWriter != nil { + logrus.Debugf("Consuming rest of the original blob to satisfy getOriginalLayerCopyWriter") + _, err := io.Copy(io.Discard, originalLayerReader) + if err != nil { + return types.BlobInfo{}, errors.Wrapf(err, "reading input blob %s", srcInfo.Digest) + } + } + + if digestingReader.validationFailed { // Coverage: This should never happen. + return types.BlobInfo{}, errors.Errorf("Internal error writing blob %s, digest verification failed but was ignored", srcInfo.Digest) + } + if stream.info.Digest != "" && uploadedInfo.Digest != stream.info.Digest { + return types.BlobInfo{}, errors.Errorf("Internal error writing blob %s, blob with digest %s saved with digest %s", srcInfo.Digest, stream.info.Digest, uploadedInfo.Digest) + } + if digestingReader.validationSucceeded { + if err := compressionStep.recordValidatedDigestData(ic.c, uploadedInfo, srcInfo, encryptionStep, decryptionStep); err != nil { + return types.BlobInfo{}, err + } + } + + return uploadedInfo, nil +} + +// sourceStream encapsulates an input consumed by copyBlobFromStream, in progress of being built. +// This allows handles of individual aspects to build the copy pipeline without _too much_ +// specific cooperation by the caller. +// +// We are currently very far from a generalized plug-and-play API for building/consuming the pipeline +// without specific knowledge of various aspects in copyBlobFromStream; that may come one day. +type sourceStream struct { + reader io.Reader + info types.BlobInfo // corresponding to the data available in reader. +} + +// errorAnnotationReader wraps the io.Reader passed to PutBlob for annotating the error happened during read. +// These errors are reported as PutBlob errors, so we would otherwise misleadingly attribute them to the copy destination. +type errorAnnotationReader struct { + reader io.Reader +} + +// Read annotates the error happened during read +func (r errorAnnotationReader) Read(b []byte) (n int, err error) { + n, err = r.reader.Read(b) + if err != io.EOF { + return n, errors.Wrapf(err, "happened during read") + } + return n, err +} diff --git a/vendor/github.com/containers/image/v5/copy/compression.go b/vendor/github.com/containers/image/v5/copy/compression.go new file mode 100644 index 000000000..99305a039 --- /dev/null +++ b/vendor/github.com/containers/image/v5/copy/compression.go @@ -0,0 +1,320 @@ +package copy + +import ( + "io" + + internalblobinfocache "github.com/containers/image/v5/internal/blobinfocache" + "github.com/containers/image/v5/pkg/compression" + compressiontypes "github.com/containers/image/v5/pkg/compression/types" + "github.com/containers/image/v5/types" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// bpDetectCompressionStepData contains data that the copy pipeline needs about the “detect compression” step. +type bpDetectCompressionStepData struct { + isCompressed bool + format compressiontypes.Algorithm // Valid if isCompressed + decompressor compressiontypes.DecompressorFunc // Valid if isCompressed + srcCompressorName string // Compressor name to possibly record in the blob info cache for the source blob. +} + +// blobPipelineDetectCompressionStep updates *stream to detect its current compression format. +// srcInfo is only used for error messages. +// Returns data for other steps. +func blobPipelineDetectCompressionStep(stream *sourceStream, srcInfo types.BlobInfo) (bpDetectCompressionStepData, error) { + // This requires us to “peek ahead” into the stream to read the initial part, which requires us to chain through another io.Reader returned by DetectCompression. + format, decompressor, reader, err := compression.DetectCompressionFormat(stream.reader) // We could skip this in some cases, but let's keep the code path uniform + if err != nil { + return bpDetectCompressionStepData{}, errors.Wrapf(err, "reading blob %s", srcInfo.Digest) + } + stream.reader = reader + + res := bpDetectCompressionStepData{ + isCompressed: decompressor != nil, + format: format, + decompressor: decompressor, + } + if res.isCompressed { + res.srcCompressorName = format.Name() + } else { + res.srcCompressorName = internalblobinfocache.Uncompressed + } + + if expectedFormat, known := expectedCompressionFormats[stream.info.MediaType]; known && res.isCompressed && format.Name() != expectedFormat.Name() { + logrus.Debugf("blob %s with type %s should be compressed with %s, but compressor appears to be %s", srcInfo.Digest.String(), srcInfo.MediaType, expectedFormat.Name(), format.Name()) + } + return res, nil +} + +// bpCompressionStepData contains data that the copy pipeline needs about the compression step. +type bpCompressionStepData struct { + operation types.LayerCompression // Operation to use for updating the blob metadata. + uploadedAlgorithm *compressiontypes.Algorithm // An algorithm parameter for the compressionOperation edits. + uploadedAnnotations map[string]string // Annotations that should be set on the uploaded blob. WARNING: This is only set after the srcStream.reader is fully consumed. + srcCompressorName string // Compressor name to record in the blob info cache for the source blob. + uploadedCompressorName string // Compressor name to record in the blob info cache for the uploaded blob. + closers []io.Closer // Objects to close after the upload is done, if any. +} + +// blobPipelineCompressionStep updates *stream to compress and/or decompress it. +// srcInfo is primarily used for error messages. +// Returns data for other steps; the caller should eventually call updateCompressionEdits and perhaps recordValidatedBlobData, +// and must eventually call close. +func (ic *imageCopier) blobPipelineCompressionStep(stream *sourceStream, canModifyBlob bool, srcInfo types.BlobInfo, + detected bpDetectCompressionStepData) (*bpCompressionStepData, error) { + // WARNING: If you are adding new reasons to change the blob, update also the OptimizeDestinationImageAlreadyExists + // short-circuit conditions + layerCompressionChangeSupported := ic.src.CanChangeLayerCompression(stream.info.MediaType) + if !layerCompressionChangeSupported { + logrus.Debugf("Compression change for blob %s (%q) not supported", srcInfo.Digest, stream.info.MediaType) + } + if canModifyBlob && layerCompressionChangeSupported { + for _, fn := range []func(*sourceStream, bpDetectCompressionStepData) (*bpCompressionStepData, error){ + ic.bpcPreserveEncrypted, + ic.bpcCompressUncompressed, + ic.bpcRecompressCompressed, + ic.bpcDecompressCompressed, + } { + res, err := fn(stream, detected) + if err != nil { + return nil, err + } + if res != nil { + return res, nil + } + } + } + return ic.bpcPreserveOriginal(stream, detected, layerCompressionChangeSupported), nil +} + +// bpcPreserveEncrypted checks if the input is encrypted, and returns a *bpCompressionStepData if so. +func (ic *imageCopier) bpcPreserveEncrypted(stream *sourceStream, _ bpDetectCompressionStepData) (*bpCompressionStepData, error) { + if isOciEncrypted(stream.info.MediaType) { + logrus.Debugf("Using original blob without modification for encrypted blob") + // PreserveOriginal due to any compression not being able to be done on an encrypted blob unless decrypted + return &bpCompressionStepData{ + operation: types.PreserveOriginal, + uploadedAlgorithm: nil, + srcCompressorName: internalblobinfocache.UnknownCompression, + uploadedCompressorName: internalblobinfocache.UnknownCompression, + }, nil + } + return nil, nil +} + +// bpcCompressUncompressed checks if we should be compressing an uncompressed input, and returns a *bpCompressionStepData if so. +func (ic *imageCopier) bpcCompressUncompressed(stream *sourceStream, detected bpDetectCompressionStepData) (*bpCompressionStepData, error) { + if ic.c.dest.DesiredLayerCompression() == types.Compress && !detected.isCompressed { + logrus.Debugf("Compressing blob on the fly") + var uploadedAlgorithm *compressiontypes.Algorithm + if ic.c.compressionFormat != nil { + uploadedAlgorithm = ic.c.compressionFormat + } else { + uploadedAlgorithm = defaultCompressionFormat + } + + reader, annotations := ic.c.compressedStream(stream.reader, *uploadedAlgorithm) + // Note: reader must be closed on all return paths. + stream.reader = reader + stream.info = types.BlobInfo{ // FIXME? Should we preserve more data in src.info? + Digest: "", + Size: -1, + } + return &bpCompressionStepData{ + operation: types.Compress, + uploadedAlgorithm: uploadedAlgorithm, + uploadedAnnotations: annotations, + srcCompressorName: detected.srcCompressorName, + uploadedCompressorName: uploadedAlgorithm.Name(), + closers: []io.Closer{reader}, + }, nil + } + return nil, nil +} + +// bpcRecompressCompressed checks if we should be recompressing a compressed input to another format, and returns a *bpCompressionStepData if so. +func (ic *imageCopier) bpcRecompressCompressed(stream *sourceStream, detected bpDetectCompressionStepData) (*bpCompressionStepData, error) { + if ic.c.dest.DesiredLayerCompression() == types.Compress && detected.isCompressed && + ic.c.compressionFormat != nil && ic.c.compressionFormat.Name() != detected.format.Name() { + // When the blob is compressed, but the desired format is different, it first needs to be decompressed and finally + // re-compressed using the desired format. + logrus.Debugf("Blob will be converted") + + decompressed, err := detected.decompressor(stream.reader) + if err != nil { + return nil, err + } + succeeded := false + defer func() { + if !succeeded { + decompressed.Close() + } + }() + + recompressed, annotations := ic.c.compressedStream(decompressed, *ic.c.compressionFormat) + // Note: recompressed must be closed on all return paths. + stream.reader = recompressed + stream.info = types.BlobInfo{ // FIXME? Should we preserve more data in src.info? + Digest: "", + Size: -1, + } + succeeded = true + return &bpCompressionStepData{ + operation: types.PreserveOriginal, + uploadedAlgorithm: ic.c.compressionFormat, + uploadedAnnotations: annotations, + srcCompressorName: detected.srcCompressorName, + uploadedCompressorName: ic.c.compressionFormat.Name(), + closers: []io.Closer{decompressed, recompressed}, + }, nil + } + return nil, nil +} + +// bpcDecompressCompressed checks if we should be decompressing a compressed input, and returns a *bpCompressionStepData if so. +func (ic *imageCopier) bpcDecompressCompressed(stream *sourceStream, detected bpDetectCompressionStepData) (*bpCompressionStepData, error) { + if ic.c.dest.DesiredLayerCompression() == types.Decompress && detected.isCompressed { + logrus.Debugf("Blob will be decompressed") + s, err := detected.decompressor(stream.reader) + if err != nil { + return nil, err + } + // Note: s must be closed on all return paths. + stream.reader = s + stream.info = types.BlobInfo{ // FIXME? Should we preserve more data in src.info? + Digest: "", + Size: -1, + } + return &bpCompressionStepData{ + operation: types.Decompress, + uploadedAlgorithm: nil, + srcCompressorName: detected.srcCompressorName, + uploadedCompressorName: internalblobinfocache.Uncompressed, + closers: []io.Closer{s}, + }, nil + } + return nil, nil +} + +// bpcPreserveOriginal returns a *bpCompressionStepData for not changing the original blob. +func (ic *imageCopier) bpcPreserveOriginal(stream *sourceStream, detected bpDetectCompressionStepData, + layerCompressionChangeSupported bool) *bpCompressionStepData { + logrus.Debugf("Using original blob without modification") + // Remember if the original blob was compressed, and if so how, so that if + // LayerInfosForCopy() returned something that differs from what was in the + // source's manifest, and UpdatedImage() needs to call UpdateLayerInfos(), + // it will be able to correctly derive the MediaType for the copied blob. + // + // But don’t touch blobs in objects where we can’t change compression, + // so that src.UpdatedImage() doesn’t fail; assume that for such blobs + // LayerInfosForCopy() should not be making any changes in the first place. + var algorithm *compressiontypes.Algorithm + if layerCompressionChangeSupported && detected.isCompressed { + algorithm = &detected.format + } else { + algorithm = nil + } + return &bpCompressionStepData{ + operation: types.PreserveOriginal, + uploadedAlgorithm: algorithm, + srcCompressorName: detected.srcCompressorName, + uploadedCompressorName: detected.srcCompressorName, + } +} + +// updateCompressionEdits sets *operation, *algorithm and updates *annotations, if necessary. +func (d *bpCompressionStepData) updateCompressionEdits(operation *types.LayerCompression, algorithm **compressiontypes.Algorithm, annotations *map[string]string) { + *operation = d.operation + // If we can modify the layer's blob, set the desired algorithm for it to be set in the manifest. + *algorithm = d.uploadedAlgorithm + if *annotations == nil { + *annotations = map[string]string{} + } + for k, v := range d.uploadedAnnotations { + (*annotations)[k] = v + } +} + +// recordValidatedBlobData updates b.blobInfoCache with data about the created uploadedInfo adnd the original srcInfo. +// This must ONLY be called if all data has been validated by OUR code, and is not comming from third parties. +func (d *bpCompressionStepData) recordValidatedDigestData(c *copier, uploadedInfo types.BlobInfo, srcInfo types.BlobInfo, + encryptionStep *bpEncryptionStepData, decryptionStep *bpDecryptionStepData) error { + // Don’t record any associations that involve encrypted data. This is a bit crude, + // some blob substitutions (replacing pulls of encrypted data with local reuse of known decryption outcomes) + // might be safe, but it’s not trivially obvious, so let’s be conservative for now. + // This crude approach also means we don’t need to record whether a blob is encrypted + // in the blob info cache (which would probably be necessary for any more complex logic), + // and the simplicity is attractive. + if !encryptionStep.encrypting && !decryptionStep.decrypting { + // If d.operation != types.PreserveOriginal, we now have two reliable digest values: + // srcinfo.Digest describes the pre-d.operation input, verified by digestingReader + // uploadedInfo.Digest describes the post-d.operation output, computed by PutBlob + // (because stream.info.Digest == "", this must have been computed afresh). + switch d.operation { + case types.PreserveOriginal: + break // Do nothing, we have only one digest and we might not have even verified it. + case types.Compress: + c.blobInfoCache.RecordDigestUncompressedPair(uploadedInfo.Digest, srcInfo.Digest) + case types.Decompress: + c.blobInfoCache.RecordDigestUncompressedPair(srcInfo.Digest, uploadedInfo.Digest) + default: + return errors.Errorf("Internal error: Unexpected d.operation value %#v", d.operation) + } + } + if d.uploadedCompressorName != "" && d.uploadedCompressorName != internalblobinfocache.UnknownCompression { + c.blobInfoCache.RecordDigestCompressorName(uploadedInfo.Digest, d.uploadedCompressorName) + } + if srcInfo.Digest != "" && d.srcCompressorName != "" && d.srcCompressorName != internalblobinfocache.UnknownCompression { + c.blobInfoCache.RecordDigestCompressorName(srcInfo.Digest, d.srcCompressorName) + } + return nil +} + +// close closes objects that carry state throughout the compression/decompression operation. +func (d *bpCompressionStepData) close() { + for _, c := range d.closers { + c.Close() + } +} + +// doCompression reads all input from src and writes its compressed equivalent to dest. +func doCompression(dest io.Writer, src io.Reader, metadata map[string]string, compressionFormat compressiontypes.Algorithm, compressionLevel *int) error { + compressor, err := compression.CompressStreamWithMetadata(dest, metadata, compressionFormat, compressionLevel) + if err != nil { + return err + } + + buf := make([]byte, compressionBufferSize) + + _, err = io.CopyBuffer(compressor, src, buf) // Sets err to nil, i.e. causes dest.Close() + if err != nil { + compressor.Close() + return err + } + + return compressor.Close() +} + +// compressGoroutine reads all input from src and writes its compressed equivalent to dest. +func (c *copier) compressGoroutine(dest *io.PipeWriter, src io.Reader, metadata map[string]string, compressionFormat compressiontypes.Algorithm) { + err := errors.New("Internal error: unexpected panic in compressGoroutine") + defer func() { // Note that this is not the same as {defer dest.CloseWithError(err)}; we need err to be evaluated lazily. + _ = dest.CloseWithError(err) // CloseWithError(nil) is equivalent to Close(), always returns nil + }() + + err = doCompression(dest, src, metadata, compressionFormat, c.compressionLevel) +} + +// compressedStream returns a stream the input reader compressed using format, and a metadata map. +// The caller must close the returned reader. +// AFTER the stream is consumed, metadata will be updated with annotations to use on the data. +func (c *copier) compressedStream(reader io.Reader, algorithm compressiontypes.Algorithm) (io.ReadCloser, map[string]string) { + pipeReader, pipeWriter := io.Pipe() + annotations := map[string]string{} + // If this fails while writing data, it will do pipeWriter.CloseWithError(); if it fails otherwise, + // e.g. because we have exited and due to pipeReader.Close() above further writing to the pipe has failed, + // we don’t care. + go c.compressGoroutine(pipeWriter, reader, annotations, algorithm) // Closes pipeWriter + return pipeReader, annotations +} diff --git a/vendor/github.com/containers/image/v5/copy/copy.go b/vendor/github.com/containers/image/v5/copy/copy.go index 123c23e02..0df595237 100644 --- a/vendor/github.com/containers/image/v5/copy/copy.go +++ b/vendor/github.com/containers/image/v5/copy/copy.go @@ -12,8 +12,8 @@ import ( "time" "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/image" internalblobinfocache "github.com/containers/image/v5/internal/blobinfocache" + "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/internal/imagedestination" "github.com/containers/image/v5/internal/imagesource" "github.com/containers/image/v5/internal/pkg/platform" @@ -25,7 +25,6 @@ import ( "github.com/containers/image/v5/signature" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" - "github.com/containers/ocicrypt" encconfig "github.com/containers/ocicrypt/config" digest "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -82,7 +81,7 @@ type copier struct { type imageCopier struct { c *copier manifestUpdates *types.ManifestUpdateOptions - src types.Image + src *image.SourcedImage diffIDsAreNeeded bool cannotModifyManifestReason string // The reason the manifest cannot be modified, or an empty string if it can canSubstituteBlobs bool @@ -349,13 +348,8 @@ func supportsMultipleImages(dest types.ImageDestination) bool { // compareImageDestinationManifestEqual compares the `src` and `dest` image manifests (reading the manifest from the // (possibly remote) destination). Returning true and the destination's manifest, type and digest if they compare equal. -func compareImageDestinationManifestEqual(ctx context.Context, options *Options, src types.Image, targetInstance *digest.Digest, dest types.ImageDestination) (bool, []byte, string, digest.Digest, error) { - srcManifest, _, err := src.Manifest(ctx) - if err != nil { - return false, nil, "", "", errors.Wrapf(err, "reading manifest from image") - } - - srcManifestDigest, err := manifest.Digest(srcManifest) +func compareImageDestinationManifestEqual(ctx context.Context, options *Options, src *image.SourcedImage, targetInstance *digest.Digest, dest types.ImageDestination) (bool, []byte, string, digest.Digest, error) { + srcManifestDigest, err := manifest.Digest(src.ManifestBlob) if err != nil { return false, nil, "", "", errors.Wrapf(err, "calculating manifest digest") } @@ -620,11 +614,7 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli if named := c.dest.Reference().DockerReference(); named != nil { if digested, ok := named.(reference.Digested); ok { destIsDigestedReference = true - sourceManifest, _, err := src.Manifest(ctx) - if err != nil { - return nil, "", "", errors.Wrapf(err, "reading manifest from source image") - } - matches, err := manifest.MatchesDigest(sourceManifest, digested.Digest()) + matches, err := manifest.MatchesDigest(src.ManifestBlob, digested.Digest()) if err != nil { return nil, "", "", errors.Wrapf(err, "computing digest of source image's manifest") } @@ -688,12 +678,14 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli cannotModifyManifestReason: cannotModifyManifestReason, ociEncryptLayers: options.OciEncryptLayers, } - // Ensure _this_ copy sees exactly the intended data when either processing a signed image or signing it. - // This may be too conservative, but for now, better safe than sorry, _especially_ on the SignBy path: - // The signature makes the content non-repudiable, so it very much matters that the signature is made over exactly what the user intended. - // We do intend the RecordDigestUncompressedPair calls to only work with reliable data, but at least there’s a risk - // that the compressed version coming from a third party may be designed to attack some other decompressor implementation, - // and we would reuse and sign it. + // Decide whether we can substitute blobs with semantic equivalents: + // - Don’t do that if we can’t modify the manifest at all + // - Ensure _this_ copy sees exactly the intended data when either processing a signed image or signing it. + // This may be too conservative, but for now, better safe than sorry, _especially_ on the SignBy path: + // The signature makes the content non-repudiable, so it very much matters that the signature is made over exactly what the user intended. + // We do intend the RecordDigestUncompressedPair calls to only work with reliable data, but at least there’s a risk + // that the compressed version coming from a third party may be designed to attack some other decompressor implementation, + // and we would reuse and sign it. ic.canSubstituteBlobs = ic.cannotModifyManifestReason == "" && options.SignBy == "" if err := ic.updateEmbeddedDockerReference(); err != nil { @@ -702,12 +694,23 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli destRequiresOciEncryption := (isEncrypted(src) && ic.c.ociDecryptConfig != nil) || options.OciEncryptLayers != nil - // We compute preferredManifestMIMEType only to show it in error messages. - // Without having to add this context in an error message, we would be happy enough to know only that no conversion is needed. - preferredManifestMIMEType, otherManifestMIMETypeCandidates, err := ic.determineManifestConversion(ctx, c.dest.SupportedManifestMIMETypes(), options.ForceManifestMIMEType, destRequiresOciEncryption) + manifestConversionPlan, err := determineManifestConversion(determineManifestConversionInputs{ + srcMIMEType: ic.src.ManifestMIMEType, + destSupportedManifestMIMETypes: ic.c.dest.SupportedManifestMIMETypes(), + forceManifestMIMEType: options.ForceManifestMIMEType, + requiresOCIEncryption: destRequiresOciEncryption, + cannotModifyManifestReason: ic.cannotModifyManifestReason, + }) if err != nil { return nil, "", "", err } + // We set up this part of ic.manifestUpdates quite early, not just around the + // code that calls copyUpdatedConfigAndManifest, so that other parts of the copy code + // (e.g. the UpdatedImageNeedsLayerDiffIDs check just below) can make decisions based + // on the expected destination format. + if manifestConversionPlan.preferredMIMETypeNeedsConversion { + ic.manifestUpdates.ManifestMIMEType = manifestConversionPlan.preferredMIMEType + } // If src.UpdatedImageNeedsLayerDiffIDs(ic.manifestUpdates) will be true, it needs to be true by the time we get here. ic.diffIDsAreNeeded = src.UpdatedImageNeedsLayerDiffIDs(*ic.manifestUpdates) @@ -742,22 +745,22 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli // So, try the preferred manifest MIME type with possibly-updated blob digests, media types, and sizes if // we're altering how they're compressed. If the process succeeds, fine… manifestBytes, retManifestDigest, err := ic.copyUpdatedConfigAndManifest(ctx, targetInstance) - retManifestType = preferredManifestMIMEType + retManifestType = manifestConversionPlan.preferredMIMEType if err != nil { - logrus.Debugf("Writing manifest using preferred type %s failed: %v", preferredManifestMIMEType, err) + logrus.Debugf("Writing manifest using preferred type %s failed: %v", manifestConversionPlan.preferredMIMEType, err) // … if it fails, and the failure is either because the manifest is rejected by the registry, or // because we failed to create a manifest of the specified type because the specific manifest type // doesn't support the type of compression we're trying to use (e.g. docker v2s2 and zstd), we may // have other options available that could still succeed. _, isManifestRejected := errors.Cause(err).(types.ManifestTypeRejectedError) _, isCompressionIncompatible := errors.Cause(err).(manifest.ManifestLayerCompressionIncompatibilityError) - if (!isManifestRejected && !isCompressionIncompatible) || len(otherManifestMIMETypeCandidates) == 0 { + if (!isManifestRejected && !isCompressionIncompatible) || len(manifestConversionPlan.otherMIMETypeCandidates) == 0 { // We don’t have other options. // In principle the code below would handle this as well, but the resulting error message is fairly ugly. // Don’t bother the user with MIME types if we have no choice. return nil, "", "", err } - // If the original MIME type is acceptable, determineManifestConversion always uses it as preferredManifestMIMEType. + // If the original MIME type is acceptable, determineManifestConversion always uses it as manifestConversionPlan.preferredMIMEType. // So if we are here, we will definitely be trying to convert the manifest. // With ic.cannotModifyManifestReason != "", that would just be a string of repeated failures for the same reason, // so let’s bail out early and with a better error message. @@ -766,8 +769,8 @@ func (c *copier) copyOneImage(ctx context.Context, policyContext *signature.Poli } // errs is a list of errors when trying various manifest types. Also serves as an "upload succeeded" flag when set to nil. - errs := []string{fmt.Sprintf("%s(%v)", preferredManifestMIMEType, err)} - for _, manifestMIMEType := range otherManifestMIMETypeCandidates { + errs := []string{fmt.Sprintf("%s(%v)", manifestConversionPlan.preferredMIMEType, err)} + for _, manifestMIMEType := range manifestConversionPlan.otherMIMETypeCandidates { logrus.Debugf("Trying to use manifest type %s…", manifestMIMEType) ic.manifestUpdates.ManifestMIMEType = manifestMIMEType attemptedManifest, attemptedManifestDigest, err := ic.copyUpdatedConfigAndManifest(ctx, targetInstance) @@ -908,11 +911,7 @@ func (ic *imageCopier) copyLayers(ctx context.Context) error { // The manifest is used to extract the information whether a given // layer is empty. - manifestBlob, manifestType, err := ic.src.Manifest(ctx) - if err != nil { - return err - } - man, err := manifest.FromBlob(manifestBlob, manifestType) + man, err := manifest.FromBlob(ic.src.ManifestBlob, ic.src.ManifestMIMEType) if err != nil { return err } @@ -1022,7 +1021,7 @@ func layerDigestsDiffer(a, b []types.BlobInfo) bool { // stores the resulting config and manifest to the destination, and returns the stored manifest // and its digest. func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context, instanceDigest *digest.Digest) ([]byte, digest.Digest, error) { - pendingImage := ic.src + var pendingImage types.Image = ic.src if !ic.noPendingManifestUpdates() { if ic.cannotModifyManifestReason != "" { return nil, "", errors.Errorf("Internal error: copy needs an updated manifest but that was known to be forbidden: %q", ic.cannotModifyManifestReason) @@ -1047,7 +1046,7 @@ func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context, instanc return nil, "", errors.Wrap(err, "reading manifest") } - if err := ic.c.copyConfig(ctx, pendingImage); err != nil { + if err := ic.copyConfig(ctx, pendingImage); err != nil { return nil, "", err } @@ -1067,19 +1066,19 @@ func (ic *imageCopier) copyUpdatedConfigAndManifest(ctx context.Context, instanc } // copyConfig copies config.json, if any, from src to dest. -func (c *copier) copyConfig(ctx context.Context, src types.Image) error { +func (ic *imageCopier) copyConfig(ctx context.Context, src types.Image) error { srcInfo := src.ConfigInfo() if srcInfo.Digest != "" { - if err := c.concurrentBlobCopiesSemaphore.Acquire(ctx, 1); err != nil { + if err := ic.c.concurrentBlobCopiesSemaphore.Acquire(ctx, 1); err != nil { // This can only fail with ctx.Err(), so no need to blame acquiring the semaphore. return fmt.Errorf("copying config: %w", err) } - defer c.concurrentBlobCopiesSemaphore.Release(1) + defer ic.c.concurrentBlobCopiesSemaphore.Release(1) destInfo, err := func() (types.BlobInfo, error) { // A scope for defer - progressPool := c.newProgressPool() + progressPool := ic.c.newProgressPool() defer progressPool.Wait() - bar := c.createProgressBar(progressPool, false, srcInfo, "config", "done") + bar := ic.c.createProgressBar(progressPool, false, srcInfo, "config", "done") defer bar.Abort(false) configBlob, err := src.ConfigBlob(ctx) @@ -1087,7 +1086,7 @@ func (c *copier) copyConfig(ctx context.Context, src types.Image) error { return types.BlobInfo{}, errors.Wrapf(err, "reading config blob %s", srcInfo.Digest) } - destInfo, err := c.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, false, true, false, bar, -1, false) + destInfo, err := ic.copyBlobFromStream(ctx, bytes.NewReader(configBlob), srcInfo, nil, true, false, bar, -1, false) if err != nil { return types.BlobInfo{}, err } @@ -1146,6 +1145,10 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to // Don’t read the layer from the source if we already have the blob, and optimizations are acceptable. if canAvoidProcessingCompleteLayer { + canChangeLayerCompression := ic.src.CanChangeLayerCompression(srcInfo.MediaType) + logrus.Debugf("Checking if we can reuse blob %s: general substitution = %v, compression for MIME type %q = %v", + srcInfo.Digest, ic.canSubstituteBlobs, srcInfo.MediaType, canChangeLayerCompression) + canSubstitute := ic.canSubstituteBlobs && ic.src.CanChangeLayerCompression(srcInfo.MediaType) // TODO: at this point we don't know whether or not a blob we end up reusing is compressed using an algorithm // that is acceptable for use on layers in the manifest that we'll be writing later, so if we end up reusing // a blob that's compressed with e.g. zstd, but we're only allowed to write a v2s2 manifest, this will cause @@ -1154,7 +1157,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to // the ImageDestination interface lets us pass in. reused, blobInfo, err := ic.c.dest.TryReusingBlobWithOptions(ctx, srcInfo, private.TryReusingBlobOptions{ Cache: ic.c.blobInfoCache, - CanSubstitute: ic.canSubstituteBlobs, + CanSubstitute: canSubstitute, EmptyLayer: emptyLayer, LayerIndex: &layerIndex, SrcRef: srcRef, @@ -1303,7 +1306,7 @@ func (ic *imageCopier) copyLayerFromStream(ctx context.Context, srcStream io.Rea } } - blobInfo, err := ic.c.copyBlobFromStream(ctx, srcStream, srcInfo, getDiffIDRecorder, ic.cannotModifyManifestReason == "", false, toEncrypt, bar, layerIndex, emptyLayer) // Sets err to nil on success + blobInfo, err := ic.copyBlobFromStream(ctx, srcStream, srcInfo, getDiffIDRecorder, false, toEncrypt, bar, layerIndex, emptyLayer) // Sets err to nil on success return blobInfo, diffIDChan, err // We need the defer … pipeWriter.CloseWithError() to happen HERE so that the caller can block on reading from diffIDChan } @@ -1333,350 +1336,3 @@ func computeDiffID(stream io.Reader, decompressor compressiontypes.DecompressorF return digest.Canonical.FromReader(stream) } - -// errorAnnotationReader wraps the io.Reader passed to PutBlob for annotating the error happened during read. -// These errors are reported as PutBlob errors, so we would otherwise misleadingly attribute them to the copy destination. -type errorAnnotationReader struct { - reader io.Reader -} - -// Read annotates the error happened during read -func (r errorAnnotationReader) Read(b []byte) (n int, err error) { - n, err = r.reader.Read(b) - if err != io.EOF { - return n, errors.Wrapf(err, "happened during read") - } - return n, err -} - -// copyBlobFromStream copies a blob with srcInfo (with known Digest and Annotations and possibly known Size) from srcStream to dest, -// perhaps sending a copy to an io.Writer if getOriginalLayerCopyWriter != nil, -// perhaps (de/re/)compressing it if canModifyBlob, -// and returns a complete blobInfo of the copied blob. -func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, srcInfo types.BlobInfo, - getOriginalLayerCopyWriter func(decompressor compressiontypes.DecompressorFunc) io.Writer, - canModifyBlob bool, isConfig bool, toEncrypt bool, bar *progressBar, layerIndex int, emptyLayer bool) (types.BlobInfo, error) { - if isConfig { // This is guaranteed by the caller, but set it here to be explicit. - canModifyBlob = false - } - - // The copying happens through a pipeline of connected io.Readers. - // === Input: srcStream - - // === Process input through digestingReader to validate against the expected digest. - // Be paranoid; in case PutBlob somehow managed to ignore an error from digestingReader, - // use a separate validation failure indicator. - // Note that for this check we don't use the stronger "validationSucceeded" indicator, because - // dest.PutBlob may detect that the layer already exists, in which case we don't - // read stream to the end, and validation does not happen. - digestingReader, err := newDigestingReader(srcStream, srcInfo.Digest) - if err != nil { - return types.BlobInfo{}, errors.Wrapf(err, "preparing to verify blob %s", srcInfo.Digest) - } - var destStream io.Reader = digestingReader - - // === Update progress bars - destStream = bar.ProxyReader(destStream) - - // === Decrypt the stream, if required. - var decrypted bool - if isOciEncrypted(srcInfo.MediaType) && c.ociDecryptConfig != nil { - newDesc := imgspecv1.Descriptor{ - Annotations: srcInfo.Annotations, - } - - var d digest.Digest - destStream, d, err = ocicrypt.DecryptLayer(c.ociDecryptConfig, destStream, newDesc, false) - if err != nil { - return types.BlobInfo{}, errors.Wrapf(err, "decrypting layer %s", srcInfo.Digest) - } - - srcInfo.Digest = d - srcInfo.Size = -1 - for k := range srcInfo.Annotations { - if strings.HasPrefix(k, "org.opencontainers.image.enc") { - delete(srcInfo.Annotations, k) - } - } - decrypted = true - } - - // === Detect compression of the input stream. - // This requires us to “peek ahead” into the stream to read the initial part, which requires us to chain through another io.Reader returned by DetectCompression. - compressionFormat, decompressor, destStream, err := compression.DetectCompressionFormat(destStream) // We could skip this in some cases, but let's keep the code path uniform - if err != nil { - return types.BlobInfo{}, errors.Wrapf(err, "reading blob %s", srcInfo.Digest) - } - isCompressed := decompressor != nil - if expectedCompressionFormat, known := expectedCompressionFormats[srcInfo.MediaType]; known && isCompressed && compressionFormat.Name() != expectedCompressionFormat.Name() { - logrus.Debugf("blob %s with type %s should be compressed with %s, but compressor appears to be %s", srcInfo.Digest.String(), srcInfo.MediaType, expectedCompressionFormat.Name(), compressionFormat.Name()) - } - - // === Send a copy of the original, uncompressed, stream, to a separate path if necessary. - var originalLayerReader io.Reader // DO NOT USE this other than to drain the input if no other consumer in the pipeline has done so. - if getOriginalLayerCopyWriter != nil { - destStream = io.TeeReader(destStream, getOriginalLayerCopyWriter(decompressor)) - originalLayerReader = destStream - } - - compressionMetadata := map[string]string{} - // === Deal with layer compression/decompression if necessary - // WARNING: If you are adding new reasons to change the blob, update also the OptimizeDestinationImageAlreadyExists - // short-circuit conditions - var inputInfo types.BlobInfo - var compressionOperation types.LayerCompression - var uploadCompressionFormat *compressiontypes.Algorithm - srcCompressorName := internalblobinfocache.Uncompressed - if isCompressed { - srcCompressorName = compressionFormat.Name() - } - var uploadCompressorName string - if canModifyBlob && isOciEncrypted(srcInfo.MediaType) { - // PreserveOriginal due to any compression not being able to be done on an encrypted blob unless decrypted - logrus.Debugf("Using original blob without modification for encrypted blob") - compressionOperation = types.PreserveOriginal - inputInfo = srcInfo - srcCompressorName = internalblobinfocache.UnknownCompression - uploadCompressionFormat = nil - uploadCompressorName = internalblobinfocache.UnknownCompression - } else if canModifyBlob && c.dest.DesiredLayerCompression() == types.Compress && !isCompressed { - logrus.Debugf("Compressing blob on the fly") - compressionOperation = types.Compress - pipeReader, pipeWriter := io.Pipe() - defer pipeReader.Close() - - if c.compressionFormat != nil { - uploadCompressionFormat = c.compressionFormat - } else { - uploadCompressionFormat = defaultCompressionFormat - } - // If this fails while writing data, it will do pipeWriter.CloseWithError(); if it fails otherwise, - // e.g. because we have exited and due to pipeReader.Close() above further writing to the pipe has failed, - // we don’t care. - go c.compressGoroutine(pipeWriter, destStream, compressionMetadata, *uploadCompressionFormat) // Closes pipeWriter - destStream = pipeReader - inputInfo.Digest = "" - inputInfo.Size = -1 - uploadCompressorName = uploadCompressionFormat.Name() - } else if canModifyBlob && c.dest.DesiredLayerCompression() == types.Compress && isCompressed && - c.compressionFormat != nil && c.compressionFormat.Name() != compressionFormat.Name() { - // When the blob is compressed, but the desired format is different, it first needs to be decompressed and finally - // re-compressed using the desired format. - logrus.Debugf("Blob will be converted") - - compressionOperation = types.PreserveOriginal - s, err := decompressor(destStream) - if err != nil { - return types.BlobInfo{}, err - } - defer s.Close() - - pipeReader, pipeWriter := io.Pipe() - defer pipeReader.Close() - - uploadCompressionFormat = c.compressionFormat - go c.compressGoroutine(pipeWriter, s, compressionMetadata, *uploadCompressionFormat) // Closes pipeWriter - - destStream = pipeReader - inputInfo.Digest = "" - inputInfo.Size = -1 - uploadCompressorName = uploadCompressionFormat.Name() - } else if canModifyBlob && c.dest.DesiredLayerCompression() == types.Decompress && isCompressed { - logrus.Debugf("Blob will be decompressed") - compressionOperation = types.Decompress - s, err := decompressor(destStream) - if err != nil { - return types.BlobInfo{}, err - } - defer s.Close() - destStream = s - inputInfo.Digest = "" - inputInfo.Size = -1 - uploadCompressionFormat = nil - uploadCompressorName = internalblobinfocache.Uncompressed - } else { - // PreserveOriginal might also need to recompress the original blob if the desired compression format is different. - logrus.Debugf("Using original blob without modification") - compressionOperation = types.PreserveOriginal - inputInfo = srcInfo - // Remember if the original blob was compressed, and if so how, so that if - // LayerInfosForCopy() returned something that differs from what was in the - // source's manifest, and UpdatedImage() needs to call UpdateLayerInfos(), - // it will be able to correctly derive the MediaType for the copied blob. - if isCompressed { - uploadCompressionFormat = &compressionFormat - } else { - uploadCompressionFormat = nil - } - uploadCompressorName = srcCompressorName - } - - // === Encrypt the stream for valid mediatypes if ociEncryptConfig provided - var ( - encrypted bool - finalizer ocicrypt.EncryptLayerFinalizer - ) - if toEncrypt { - if decrypted { - return types.BlobInfo{}, errors.New("Unable to support both decryption and encryption in the same copy") - } - - if !isOciEncrypted(srcInfo.MediaType) && c.ociEncryptConfig != nil { - var annotations map[string]string - if !decrypted { - annotations = srcInfo.Annotations - } - desc := imgspecv1.Descriptor{ - MediaType: srcInfo.MediaType, - Digest: srcInfo.Digest, - Size: srcInfo.Size, - Annotations: annotations, - } - - s, fin, err := ocicrypt.EncryptLayer(c.ociEncryptConfig, destStream, desc) - if err != nil { - return types.BlobInfo{}, errors.Wrapf(err, "encrypting blob %s", srcInfo.Digest) - } - - destStream = s - finalizer = fin - inputInfo.Digest = "" - inputInfo.Size = -1 - encrypted = true - } - } - - // === Report progress using the c.progress channel, if required. - if c.progress != nil && c.progressInterval > 0 { - progressReader := newProgressReader( - destStream, - c.progress, - c.progressInterval, - srcInfo, - ) - defer progressReader.reportDone() - destStream = progressReader - } - - // === Finally, send the layer stream to dest. - options := private.PutBlobOptions{ - Cache: c.blobInfoCache, - IsConfig: isConfig, - EmptyLayer: emptyLayer, - } - if !isConfig { - options.LayerIndex = &layerIndex - } - uploadedInfo, err := c.dest.PutBlobWithOptions(ctx, &errorAnnotationReader{destStream}, inputInfo, options) - if err != nil { - return types.BlobInfo{}, errors.Wrap(err, "writing blob") - } - - uploadedInfo.Annotations = srcInfo.Annotations - - uploadedInfo.CompressionOperation = compressionOperation - // If we can modify the layer's blob, set the desired algorithm for it to be set in the manifest. - uploadedInfo.CompressionAlgorithm = uploadCompressionFormat - if decrypted { - uploadedInfo.CryptoOperation = types.Decrypt - } else if encrypted { - encryptAnnotations, err := finalizer() - if err != nil { - return types.BlobInfo{}, errors.Wrap(err, "Unable to finalize encryption") - } - uploadedInfo.CryptoOperation = types.Encrypt - if uploadedInfo.Annotations == nil { - uploadedInfo.Annotations = map[string]string{} - } - for k, v := range encryptAnnotations { - uploadedInfo.Annotations[k] = v - } - } - - // This is fairly horrible: the writer from getOriginalLayerCopyWriter wants to consume - // all of the input (to compute DiffIDs), even if dest.PutBlob does not need it. - // So, read everything from originalLayerReader, which will cause the rest to be - // sent there if we are not already at EOF. - if getOriginalLayerCopyWriter != nil { - logrus.Debugf("Consuming rest of the original blob to satisfy getOriginalLayerCopyWriter") - _, err := io.Copy(io.Discard, originalLayerReader) - if err != nil { - return types.BlobInfo{}, errors.Wrapf(err, "reading input blob %s", srcInfo.Digest) - } - } - - if digestingReader.validationFailed { // Coverage: This should never happen. - return types.BlobInfo{}, errors.Errorf("Internal error writing blob %s, digest verification failed but was ignored", srcInfo.Digest) - } - if inputInfo.Digest != "" && uploadedInfo.Digest != inputInfo.Digest { - return types.BlobInfo{}, errors.Errorf("Internal error writing blob %s, blob with digest %s saved with digest %s", srcInfo.Digest, inputInfo.Digest, uploadedInfo.Digest) - } - if digestingReader.validationSucceeded { - // Don’t record any associations that involve encrypted data. This is a bit crude, - // some blob substitutions (replacing pulls of encrypted data with local reuse of known decryption outcomes) - // might be safe, but it’s not trivially obvious, so let’s be conservative for now. - // This crude approach also means we don’t need to record whether a blob is encrypted - // in the blob info cache (which would probably be necessary for any more complex logic), - // and the simplicity is attractive. - if !encrypted && !decrypted { - // If compressionOperation != types.PreserveOriginal, we now have two reliable digest values: - // srcinfo.Digest describes the pre-compressionOperation input, verified by digestingReader - // uploadedInfo.Digest describes the post-compressionOperation output, computed by PutBlob - // (because inputInfo.Digest == "", this must have been computed afresh). - switch compressionOperation { - case types.PreserveOriginal: - break // Do nothing, we have only one digest and we might not have even verified it. - case types.Compress: - c.blobInfoCache.RecordDigestUncompressedPair(uploadedInfo.Digest, srcInfo.Digest) - case types.Decompress: - c.blobInfoCache.RecordDigestUncompressedPair(srcInfo.Digest, uploadedInfo.Digest) - default: - return types.BlobInfo{}, errors.Errorf("Internal error: Unexpected compressionOperation value %#v", compressionOperation) - } - } - if uploadCompressorName != "" && uploadCompressorName != internalblobinfocache.UnknownCompression { - c.blobInfoCache.RecordDigestCompressorName(uploadedInfo.Digest, uploadCompressorName) - } - if srcInfo.Digest != "" && srcCompressorName != "" && srcCompressorName != internalblobinfocache.UnknownCompression { - c.blobInfoCache.RecordDigestCompressorName(srcInfo.Digest, srcCompressorName) - } - } - - // Copy all the metadata generated by the compressor into the annotations. - if uploadedInfo.Annotations == nil { - uploadedInfo.Annotations = map[string]string{} - } - for k, v := range compressionMetadata { - uploadedInfo.Annotations[k] = v - } - - return uploadedInfo, nil -} - -// doCompression reads all input from src and writes its compressed equivalent to dest. -func doCompression(dest io.Writer, src io.Reader, metadata map[string]string, compressionFormat compressiontypes.Algorithm, compressionLevel *int) error { - compressor, err := compression.CompressStreamWithMetadata(dest, metadata, compressionFormat, compressionLevel) - if err != nil { - return err - } - - buf := make([]byte, compressionBufferSize) - - _, err = io.CopyBuffer(compressor, src, buf) // Sets err to nil, i.e. causes dest.Close() - if err != nil { - compressor.Close() - return err - } - - return compressor.Close() -} - -// compressGoroutine reads all input from src and writes its compressed equivalent to dest. -func (c *copier) compressGoroutine(dest *io.PipeWriter, src io.Reader, metadata map[string]string, compressionFormat compressiontypes.Algorithm) { - err := errors.New("Internal error: unexpected panic in compressGoroutine") - defer func() { // Note that this is not the same as {defer dest.CloseWithError(err)}; we need err to be evaluated lazily. - _ = dest.CloseWithError(err) // CloseWithError(nil) is equivalent to Close(), always returns nil - }() - - err = doCompression(dest, src, metadata, compressionFormat, c.compressionLevel) -} diff --git a/vendor/github.com/containers/image/v5/copy/encrypt.go b/vendor/github.com/containers/image/v5/copy/encrypt.go deleted file mode 100644 index a18d6f151..000000000 --- a/vendor/github.com/containers/image/v5/copy/encrypt.go +++ /dev/null @@ -1,24 +0,0 @@ -package copy - -import ( - "strings" - - "github.com/containers/image/v5/types" -) - -// isOciEncrypted returns a bool indicating if a mediatype is encrypted -// This function will be moved to be part of OCI spec when adopted. -func isOciEncrypted(mediatype string) bool { - return strings.HasSuffix(mediatype, "+encrypted") -} - -// isEncrypted checks if an image is encrypted -func isEncrypted(i types.Image) bool { - layers := i.LayerInfos() - for _, l := range layers { - if isOciEncrypted(l.MediaType) { - return true - } - } - return false -} diff --git a/vendor/github.com/containers/image/v5/copy/encryption.go b/vendor/github.com/containers/image/v5/copy/encryption.go new file mode 100644 index 000000000..ae0576da4 --- /dev/null +++ b/vendor/github.com/containers/image/v5/copy/encryption.go @@ -0,0 +1,129 @@ +package copy + +import ( + "strings" + + "github.com/containers/image/v5/types" + "github.com/containers/ocicrypt" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" +) + +// isOciEncrypted returns a bool indicating if a mediatype is encrypted +// This function will be moved to be part of OCI spec when adopted. +func isOciEncrypted(mediatype string) bool { + return strings.HasSuffix(mediatype, "+encrypted") +} + +// isEncrypted checks if an image is encrypted +func isEncrypted(i types.Image) bool { + layers := i.LayerInfos() + for _, l := range layers { + if isOciEncrypted(l.MediaType) { + return true + } + } + return false +} + +// bpDecryptionStepData contains data that the copy pipeline needs about the decryption step. +type bpDecryptionStepData struct { + decrypting bool // We are actually decrypting the stream +} + +// blobPipelineDecryptionStep updates *stream to decrypt if, it necessary. +// srcInfo is only used for error messages. +// Returns data for other steps; the caller should eventually use updateCryptoOperation. +func (c *copier) blobPipelineDecryptionStep(stream *sourceStream, srcInfo types.BlobInfo) (*bpDecryptionStepData, error) { + if isOciEncrypted(stream.info.MediaType) && c.ociDecryptConfig != nil { + desc := imgspecv1.Descriptor{ + Annotations: stream.info.Annotations, + } + reader, decryptedDigest, err := ocicrypt.DecryptLayer(c.ociDecryptConfig, stream.reader, desc, false) + if err != nil { + return nil, errors.Wrapf(err, "decrypting layer %s", srcInfo.Digest) + } + + stream.reader = reader + stream.info.Digest = decryptedDigest + stream.info.Size = -1 + for k := range stream.info.Annotations { + if strings.HasPrefix(k, "org.opencontainers.image.enc") { + delete(stream.info.Annotations, k) + } + } + return &bpDecryptionStepData{ + decrypting: true, + }, nil + } + return &bpDecryptionStepData{ + decrypting: false, + }, nil +} + +// updateCryptoOperation sets *operation, if necessary. +func (d *bpDecryptionStepData) updateCryptoOperation(operation *types.LayerCrypto) { + if d.decrypting { + *operation = types.Decrypt + } +} + +// bpdData contains data that the copy pipeline needs about the encryption step. +type bpEncryptionStepData struct { + encrypting bool // We are actually encrypting the stream + finalizer ocicrypt.EncryptLayerFinalizer +} + +// blobPipelineEncryptionStep updates *stream to encrypt if, it required by toEncrypt. +// srcInfo is primarily used for error messages. +// Returns data for other steps; the caller should eventually call updateCryptoOperationAndAnnotations. +func (c *copier) blobPipelineEncryptionStep(stream *sourceStream, toEncrypt bool, srcInfo types.BlobInfo, + decryptionStep *bpDecryptionStepData) (*bpEncryptionStepData, error) { + if toEncrypt && !isOciEncrypted(srcInfo.MediaType) && c.ociEncryptConfig != nil { + var annotations map[string]string + if !decryptionStep.decrypting { + annotations = srcInfo.Annotations + } + desc := imgspecv1.Descriptor{ + MediaType: srcInfo.MediaType, + Digest: srcInfo.Digest, + Size: srcInfo.Size, + Annotations: annotations, + } + reader, finalizer, err := ocicrypt.EncryptLayer(c.ociEncryptConfig, stream.reader, desc) + if err != nil { + return nil, errors.Wrapf(err, "encrypting blob %s", srcInfo.Digest) + } + + stream.reader = reader + stream.info.Digest = "" + stream.info.Size = -1 + return &bpEncryptionStepData{ + encrypting: true, + finalizer: finalizer, + }, nil + } + return &bpEncryptionStepData{ + encrypting: false, + }, nil +} + +// updateCryptoOperationAndAnnotations sets *operation and updates *annotations, if necessary. +func (d *bpEncryptionStepData) updateCryptoOperationAndAnnotations(operation *types.LayerCrypto, annotations *map[string]string) error { + if !d.encrypting { + return nil + } + + encryptAnnotations, err := d.finalizer() + if err != nil { + return errors.Wrap(err, "Unable to finalize encryption") + } + *operation = types.Encrypt + if *annotations == nil { + *annotations = map[string]string{} + } + for k, v := range encryptAnnotations { + (*annotations)[k] = v + } + return nil +} diff --git a/vendor/github.com/containers/image/v5/copy/manifest.go b/vendor/github.com/containers/image/v5/copy/manifest.go index 86ec8863a..b65459f8c 100644 --- a/vendor/github.com/containers/image/v5/copy/manifest.go +++ b/vendor/github.com/containers/image/v5/copy/manifest.go @@ -38,31 +38,50 @@ func (os *orderedSet) append(s string) { } } -// determineManifestConversion updates ic.manifestUpdates to convert manifest to a supported MIME type, if necessary and ic.canModifyManifest. -// Note that the conversion will only happen later, through ic.src.UpdatedImage -// Returns the preferred manifest MIME type (whether we are converting to it or using it unmodified), -// and a list of other possible alternatives, in order. -func (ic *imageCopier) determineManifestConversion(ctx context.Context, destSupportedManifestMIMETypes []string, forceManifestMIMEType string, requiresOciEncryption bool) (string, []string, error) { - _, srcType, err := ic.src.Manifest(ctx) - if err != nil { // This should have been cached?! - return "", nil, errors.Wrap(err, "reading manifest") - } +// determineManifestConversionInputs contains the inputs for determineManifestConversion. +type determineManifestConversionInputs struct { + srcMIMEType string // MIME type of the input manifest + + destSupportedManifestMIMETypes []string // MIME types supported by the destination, per types.ImageDestination.SupportedManifestMIMETypes() + + forceManifestMIMEType string // User’s choice of forced manifest MIME type + requiresOCIEncryption bool // Restrict to manifest formats that can support OCI encryption + cannotModifyManifestReason string // The reason the manifest cannot be modified, or an empty string if it can +} + +// manifestConversionPlan contains the decisions made by determineManifestConversion. +type manifestConversionPlan struct { + // The preferred manifest MIME type (whether we are converting to it or using it unmodified). + // We compute this only to show it in error messages; without having to add this context + // in an error message, we would be happy enough to know only that no conversion is needed. + preferredMIMEType string + preferredMIMETypeNeedsConversion bool // True if using preferredMIMEType requires a conversion step. + otherMIMETypeCandidates []string // Other possible alternatives, in order +} + +// determineManifestConversion returns a plan for what formats, and possibly conversions, to use based on in. +func determineManifestConversion(in determineManifestConversionInputs) (manifestConversionPlan, error) { + srcType := in.srcMIMEType normalizedSrcType := manifest.NormalizedMIMEType(srcType) if srcType != normalizedSrcType { logrus.Debugf("Source manifest MIME type %s, treating it as %s", srcType, normalizedSrcType) srcType = normalizedSrcType } - if forceManifestMIMEType != "" { - destSupportedManifestMIMETypes = []string{forceManifestMIMEType} + destSupportedManifestMIMETypes := in.destSupportedManifestMIMETypes + if in.forceManifestMIMEType != "" { + destSupportedManifestMIMETypes = []string{in.forceManifestMIMEType} } - if len(destSupportedManifestMIMETypes) == 0 && (!requiresOciEncryption || manifest.MIMETypeSupportsEncryption(srcType)) { - return srcType, []string{}, nil // Anything goes; just use the original as is, do not try any conversions. + if len(destSupportedManifestMIMETypes) == 0 && (!in.requiresOCIEncryption || manifest.MIMETypeSupportsEncryption(srcType)) { + return manifestConversionPlan{ // Anything goes; just use the original as is, do not try any conversions. + preferredMIMEType: srcType, + otherMIMETypeCandidates: []string{}, + }, nil } supportedByDest := map[string]struct{}{} for _, t := range destSupportedManifestMIMETypes { - if !requiresOciEncryption || manifest.MIMETypeSupportsEncryption(t) { + if !in.requiresOCIEncryption || manifest.MIMETypeSupportsEncryption(t) { supportedByDest[t] = struct{}{} } } @@ -79,13 +98,16 @@ func (ic *imageCopier) determineManifestConversion(ctx context.Context, destSupp if _, ok := supportedByDest[srcType]; ok { prioritizedTypes.append(srcType) } - if ic.cannotModifyManifestReason != "" { + if in.cannotModifyManifestReason != "" { // We could also drop this check and have the caller // make the choice; it is already doing that to an extent, to improve error // messages. But it is nice to hide the “if we can't modify, do no conversion” // special case in here; the caller can then worry (or not) only about a good UI. logrus.Debugf("We can't modify the manifest, hoping for the best...") - return srcType, []string{}, nil // Take our chances - FIXME? Or should we fail without trying? + return manifestConversionPlan{ // Take our chances - FIXME? Or should we fail without trying? + preferredMIMEType: srcType, + otherMIMETypeCandidates: []string{}, + }, nil } // Then use our list of preferred types. @@ -102,15 +124,17 @@ func (ic *imageCopier) determineManifestConversion(ctx context.Context, destSupp logrus.Debugf("Manifest has MIME type %s, ordered candidate list [%s]", srcType, strings.Join(prioritizedTypes.list, ", ")) if len(prioritizedTypes.list) == 0 { // Coverage: destSupportedManifestMIMETypes is not empty (or we would have exited in the “Anything goes” case above), so this should never happen. - return "", nil, errors.New("Internal error: no candidate MIME types") + return manifestConversionPlan{}, errors.New("Internal error: no candidate MIME types") } - preferredType := prioritizedTypes.list[0] - if preferredType != srcType { - ic.manifestUpdates.ManifestMIMEType = preferredType - } else { + res := manifestConversionPlan{ + preferredMIMEType: prioritizedTypes.list[0], + otherMIMETypeCandidates: prioritizedTypes.list[1:], + } + res.preferredMIMETypeNeedsConversion = res.preferredMIMEType != srcType + if !res.preferredMIMETypeNeedsConversion { logrus.Debugf("... will first try using the original manifest unmodified") } - return preferredType, prioritizedTypes.list[1:], nil + return res, nil } // isMultiImage returns true if img is a list of images diff --git a/vendor/github.com/containers/image/v5/directory/directory_transport.go b/vendor/github.com/containers/image/v5/directory/directory_transport.go index e542d888c..562404470 100644 --- a/vendor/github.com/containers/image/v5/directory/directory_transport.go +++ b/vendor/github.com/containers/image/v5/directory/directory_transport.go @@ -8,7 +8,7 @@ import ( "github.com/containers/image/v5/directory/explicitfilepath" "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/image" + "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" @@ -140,8 +140,7 @@ func (ref dirReference) PolicyConfigurationNamespaces() []string { // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. // WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. func (ref dirReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { - src := newImageSource(ref) - return image.FromSource(ctx, sys, src) + return image.FromReference(ctx, sys, ref) } // NewImageSource returns a types.ImageSource for this reference. diff --git a/vendor/github.com/containers/image/v5/docker/archive/transport.go b/vendor/github.com/containers/image/v5/docker/archive/transport.go index 9a48cb46c..f00b77930 100644 --- a/vendor/github.com/containers/image/v5/docker/archive/transport.go +++ b/vendor/github.com/containers/image/v5/docker/archive/transport.go @@ -8,7 +8,7 @@ import ( "github.com/containers/image/v5/docker/internal/tarfile" "github.com/containers/image/v5/docker/reference" - ctrImage "github.com/containers/image/v5/image" + ctrImage "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" "github.com/pkg/errors" @@ -185,11 +185,7 @@ func (ref archiveReference) PolicyConfigurationNamespaces() []string { // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. // WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. func (ref archiveReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { - src, err := newImageSource(ctx, sys, ref) - if err != nil { - return nil, err - } - return ctrImage.FromSource(ctx, sys, src) + return ctrImage.FromReference(ctx, sys, ref) } // NewImageSource returns a types.ImageSource for this reference. diff --git a/vendor/github.com/containers/image/v5/docker/daemon/daemon_transport.go b/vendor/github.com/containers/image/v5/docker/daemon/daemon_transport.go index 4e4ed6881..d75579784 100644 --- a/vendor/github.com/containers/image/v5/docker/daemon/daemon_transport.go +++ b/vendor/github.com/containers/image/v5/docker/daemon/daemon_transport.go @@ -6,7 +6,7 @@ import ( "github.com/containers/image/v5/docker/policyconfiguration" "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/image" + "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" @@ -195,11 +195,7 @@ func (ref daemonReference) PolicyConfigurationNamespaces() []string { // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. // WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. func (ref daemonReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { - src, err := newImageSource(ctx, sys, ref) - if err != nil { - return nil, err - } - return image.FromSource(ctx, sys, src) + return image.FromReference(ctx, sys, ref) } // NewImageSource returns a types.ImageSource for this reference. diff --git a/vendor/github.com/containers/image/v5/docker/docker_client.go b/vendor/github.com/containers/image/v5/docker/docker_client.go index daac45f87..29c256869 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_client.go +++ b/vendor/github.com/containers/image/v5/docker/docker_client.go @@ -163,9 +163,8 @@ func newBearerTokenFromJSONBlob(blob []byte) (*bearerToken, error) { func serverDefault() *tls.Config { return &tls.Config{ // Avoid fallback to SSL protocols < TLS1.0 - MinVersion: tls.VersionTLS10, - PreferServerCipherSuites: true, - CipherSuites: tlsconfig.DefaultServerAcceptedCiphers, + MinVersion: tls.VersionTLS10, + CipherSuites: tlsconfig.DefaultServerAcceptedCiphers, } } diff --git a/vendor/github.com/containers/image/v5/docker/docker_image.go b/vendor/github.com/containers/image/v5/docker/docker_image.go index c84bb37d2..73687e86f 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_image.go +++ b/vendor/github.com/containers/image/v5/docker/docker_image.go @@ -9,7 +9,7 @@ import ( "strings" "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/image" + "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" diff --git a/vendor/github.com/containers/image/v5/image/docker_schema2.go b/vendor/github.com/containers/image/v5/image/docker_schema2.go index b250a6b1d..e5a3b8991 100644 --- a/vendor/github.com/containers/image/v5/image/docker_schema2.go +++ b/vendor/github.com/containers/image/v5/image/docker_schema2.go @@ -1,400 +1,14 @@ package image import ( - "bytes" - "context" - "crypto/sha256" - "encoding/hex" - "encoding/json" - "fmt" - "strings" - - "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/internal/iolimits" - "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/pkg/blobinfocache/none" - "github.com/containers/image/v5/types" - "github.com/opencontainers/go-digest" - imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" + "github.com/containers/image/v5/internal/image" ) // GzippedEmptyLayer is a gzip-compressed version of an empty tar file (1024 NULL bytes) // This comes from github.com/docker/distribution/manifest/schema1/config_builder.go; there is // a non-zero embedded timestamp; we could zero that, but that would just waste storage space // in registries, so let’s use the same values. -var GzippedEmptyLayer = []byte{ - 31, 139, 8, 0, 0, 9, 110, 136, 0, 255, 98, 24, 5, 163, 96, 20, 140, 88, - 0, 8, 0, 0, 255, 255, 46, 175, 181, 239, 0, 4, 0, 0, -} +var GzippedEmptyLayer = image.GzippedEmptyLayer // GzippedEmptyLayerDigest is a digest of GzippedEmptyLayer -const GzippedEmptyLayerDigest = digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4") - -type manifestSchema2 struct { - src types.ImageSource // May be nil if configBlob is not nil - configBlob []byte // If set, corresponds to contents of ConfigDescriptor. - m *manifest.Schema2 -} - -func manifestSchema2FromManifest(src types.ImageSource, manifestBlob []byte) (genericManifest, error) { - m, err := manifest.Schema2FromManifest(manifestBlob) - if err != nil { - return nil, err - } - return &manifestSchema2{ - src: src, - m: m, - }, nil -} - -// manifestSchema2FromComponents builds a new manifestSchema2 from the supplied data: -func manifestSchema2FromComponents(config manifest.Schema2Descriptor, src types.ImageSource, configBlob []byte, layers []manifest.Schema2Descriptor) *manifestSchema2 { - return &manifestSchema2{ - src: src, - configBlob: configBlob, - m: manifest.Schema2FromComponents(config, layers), - } -} - -func (m *manifestSchema2) serialize() ([]byte, error) { - return m.m.Serialize() -} - -func (m *manifestSchema2) manifestMIMEType() string { - return m.m.MediaType -} - -// ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object. -// Note that the config object may not exist in the underlying storage in the return value of UpdatedImage! Use ConfigBlob() below. -func (m *manifestSchema2) ConfigInfo() types.BlobInfo { - return m.m.ConfigInfo() -} - -// OCIConfig returns the image configuration as per OCI v1 image-spec. Information about -// layers in the resulting configuration isn't guaranteed to be returned to due how -// old image manifests work (docker v2s1 especially). -func (m *manifestSchema2) OCIConfig(ctx context.Context) (*imgspecv1.Image, error) { - configBlob, err := m.ConfigBlob(ctx) - if err != nil { - return nil, err - } - // docker v2s2 and OCI v1 are mostly compatible but v2s2 contains more fields - // than OCI v1. This unmarshal makes sure we drop docker v2s2 - // fields that aren't needed in OCI v1. - configOCI := &imgspecv1.Image{} - if err := json.Unmarshal(configBlob, configOCI); err != nil { - return nil, err - } - return configOCI, nil -} - -// ConfigBlob returns the blob described by ConfigInfo, iff ConfigInfo().Digest != ""; nil otherwise. -// The result is cached; it is OK to call this however often you need. -func (m *manifestSchema2) ConfigBlob(ctx context.Context) ([]byte, error) { - if m.configBlob == nil { - if m.src == nil { - return nil, errors.Errorf("Internal error: neither src nor configBlob set in manifestSchema2") - } - stream, _, err := m.src.GetBlob(ctx, manifest.BlobInfoFromSchema2Descriptor(m.m.ConfigDescriptor), none.NoCache) - if err != nil { - return nil, err - } - defer stream.Close() - blob, err := iolimits.ReadAtMost(stream, iolimits.MaxConfigBodySize) - if err != nil { - return nil, err - } - computedDigest := digest.FromBytes(blob) - if computedDigest != m.m.ConfigDescriptor.Digest { - return nil, errors.Errorf("Download config.json digest %s does not match expected %s", computedDigest, m.m.ConfigDescriptor.Digest) - } - m.configBlob = blob - } - return m.configBlob, nil -} - -// LayerInfos returns a list of BlobInfos of layers referenced by this image, in order (the root layer first, and then successive layered layers). -// The Digest field is guaranteed to be provided; Size may be -1. -// WARNING: The list may contain duplicates, and they are semantically relevant. -func (m *manifestSchema2) LayerInfos() []types.BlobInfo { - return manifestLayerInfosToBlobInfos(m.m.LayerInfos()) -} - -// EmbeddedDockerReferenceConflicts whether a Docker reference embedded in the manifest, if any, conflicts with destination ref. -// It returns false if the manifest does not embed a Docker reference. -// (This embedding unfortunately happens for Docker schema1, please do not add support for this in any new formats.) -func (m *manifestSchema2) EmbeddedDockerReferenceConflicts(ref reference.Named) bool { - return false -} - -// Inspect returns various information for (skopeo inspect) parsed from the manifest and configuration. -func (m *manifestSchema2) Inspect(ctx context.Context) (*types.ImageInspectInfo, error) { - getter := func(info types.BlobInfo) ([]byte, error) { - if info.Digest != m.ConfigInfo().Digest { - // Shouldn't ever happen - return nil, errors.New("asked for a different config blob") - } - config, err := m.ConfigBlob(ctx) - if err != nil { - return nil, err - } - return config, nil - } - return m.m.Inspect(getter) -} - -// UpdatedImageNeedsLayerDiffIDs returns true iff UpdatedImage(options) needs InformationOnly.LayerDiffIDs. -// This is a horribly specific interface, but computing InformationOnly.LayerDiffIDs can be very expensive to compute -// (most importantly it forces us to download the full layers even if they are already present at the destination). -func (m *manifestSchema2) UpdatedImageNeedsLayerDiffIDs(options types.ManifestUpdateOptions) bool { - return false -} - -// UpdatedImage returns a types.Image modified according to options. -// This does not change the state of the original Image object. -// The returned error will be a manifest.ManifestLayerCompressionIncompatibilityError -// if the CompressionOperation and CompressionAlgorithm specified in one or more -// options.LayerInfos items is anything other than gzip. -func (m *manifestSchema2) UpdatedImage(ctx context.Context, options types.ManifestUpdateOptions) (types.Image, error) { - copy := manifestSchema2{ // NOTE: This is not a deep copy, it still shares slices etc. - src: m.src, - configBlob: m.configBlob, - m: manifest.Schema2Clone(m.m), - } - - converted, err := convertManifestIfRequiredWithUpdate(ctx, options, map[string]manifestConvertFn{ - manifest.DockerV2Schema1MediaType: copy.convertToManifestSchema1, - manifest.DockerV2Schema1SignedMediaType: copy.convertToManifestSchema1, - imgspecv1.MediaTypeImageManifest: copy.convertToManifestOCI1, - }) - if err != nil { - return nil, err - } - - if converted != nil { - return converted, nil - } - - // No conversion required, update manifest - if options.LayerInfos != nil { - if err := copy.m.UpdateLayerInfos(options.LayerInfos); err != nil { - return nil, err - } - } - // Ignore options.EmbeddedDockerReference: it may be set when converting from schema1 to schema2, but we really don't care. - - return memoryImageFromManifest(©), nil -} - -func oci1DescriptorFromSchema2Descriptor(d manifest.Schema2Descriptor) imgspecv1.Descriptor { - return imgspecv1.Descriptor{ - MediaType: d.MediaType, - Size: d.Size, - Digest: d.Digest, - URLs: d.URLs, - } -} - -// convertToManifestOCI1 returns a genericManifest implementation converted to imgspecv1.MediaTypeImageManifest. -// It may use options.InformationOnly and also adjust *options to be appropriate for editing the returned -// value. -// This does not change the state of the original manifestSchema2 object. -func (m *manifestSchema2) convertToManifestOCI1(ctx context.Context, _ *types.ManifestUpdateOptions) (genericManifest, error) { - configOCI, err := m.OCIConfig(ctx) - if err != nil { - return nil, err - } - configOCIBytes, err := json.Marshal(configOCI) - if err != nil { - return nil, err - } - - config := imgspecv1.Descriptor{ - MediaType: imgspecv1.MediaTypeImageConfig, - Size: int64(len(configOCIBytes)), - Digest: digest.FromBytes(configOCIBytes), - } - - layers := make([]imgspecv1.Descriptor, len(m.m.LayersDescriptors)) - for idx := range layers { - layers[idx] = oci1DescriptorFromSchema2Descriptor(m.m.LayersDescriptors[idx]) - switch m.m.LayersDescriptors[idx].MediaType { - case manifest.DockerV2Schema2ForeignLayerMediaType: - layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributable - case manifest.DockerV2Schema2ForeignLayerMediaTypeGzip: - layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributableGzip - case manifest.DockerV2SchemaLayerMediaTypeUncompressed: - layers[idx].MediaType = imgspecv1.MediaTypeImageLayer - case manifest.DockerV2Schema2LayerMediaType: - layers[idx].MediaType = imgspecv1.MediaTypeImageLayerGzip - default: - return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", m.m.LayersDescriptors[idx].MediaType) - } - } - - return manifestOCI1FromComponents(config, m.src, configOCIBytes, layers), nil -} - -// convertToManifestSchema1 returns a genericManifest implementation converted to manifest.DockerV2Schema1{Signed,}MediaType. -// It may use options.InformationOnly and also adjust *options to be appropriate for editing the returned -// value. -// This does not change the state of the original manifestSchema2 object. -// -// Based on docker/distribution/manifest/schema1/config_builder.go -func (m *manifestSchema2) convertToManifestSchema1(ctx context.Context, options *types.ManifestUpdateOptions) (genericManifest, error) { - dest := options.InformationOnly.Destination - - var convertedLayerUpdates []types.BlobInfo // Only used if options.LayerInfos != nil - if options.LayerInfos != nil { - if len(options.LayerInfos) != len(m.m.LayersDescriptors) { - return nil, fmt.Errorf("Error converting image: layer edits for %d layers vs %d existing layers", - len(options.LayerInfos), len(m.m.LayersDescriptors)) - } - convertedLayerUpdates = []types.BlobInfo{} - } - - configBytes, err := m.ConfigBlob(ctx) - if err != nil { - return nil, err - } - imageConfig := &manifest.Schema2Image{} - if err := json.Unmarshal(configBytes, imageConfig); err != nil { - return nil, err - } - - // Build fsLayers and History, discarding all configs. We will patch the top-level config in later. - fsLayers := make([]manifest.Schema1FSLayers, len(imageConfig.History)) - history := make([]manifest.Schema1History, len(imageConfig.History)) - nonemptyLayerIndex := 0 - var parentV1ID string // Set in the loop - v1ID := "" - haveGzippedEmptyLayer := false - if len(imageConfig.History) == 0 { - // What would this even mean?! Anyhow, the rest of the code depends on fsLayers[0] and history[0] existing. - return nil, errors.Errorf("Cannot convert an image with 0 history entries to %s", manifest.DockerV2Schema1SignedMediaType) - } - for v2Index, historyEntry := range imageConfig.History { - parentV1ID = v1ID - v1Index := len(imageConfig.History) - 1 - v2Index - - var blobDigest digest.Digest - if historyEntry.EmptyLayer { - emptyLayerBlobInfo := types.BlobInfo{Digest: GzippedEmptyLayerDigest, Size: int64(len(GzippedEmptyLayer))} - - if !haveGzippedEmptyLayer { - logrus.Debugf("Uploading empty layer during conversion to schema 1") - // Ideally we should update the relevant BlobInfoCache about this layer, but that would require passing it down here, - // and anyway this blob is so small that it’s easier to just copy it than to worry about figuring out another location where to get it. - info, err := dest.PutBlob(ctx, bytes.NewReader(GzippedEmptyLayer), emptyLayerBlobInfo, none.NoCache, false) - if err != nil { - return nil, errors.Wrap(err, "uploading empty layer") - } - if info.Digest != emptyLayerBlobInfo.Digest { - return nil, errors.Errorf("Internal error: Uploaded empty layer has digest %#v instead of %s", info.Digest, emptyLayerBlobInfo.Digest) - } - haveGzippedEmptyLayer = true - } - if options.LayerInfos != nil { - convertedLayerUpdates = append(convertedLayerUpdates, emptyLayerBlobInfo) - } - blobDigest = emptyLayerBlobInfo.Digest - } else { - if nonemptyLayerIndex >= len(m.m.LayersDescriptors) { - return nil, errors.Errorf("Invalid image configuration, needs more than the %d distributed layers", len(m.m.LayersDescriptors)) - } - if options.LayerInfos != nil { - convertedLayerUpdates = append(convertedLayerUpdates, options.LayerInfos[nonemptyLayerIndex]) - } - blobDigest = m.m.LayersDescriptors[nonemptyLayerIndex].Digest - nonemptyLayerIndex++ - } - - // AFAICT pull ignores these ID values, at least nowadays, so we could use anything unique, including a simple counter. Use what Docker uses for cargo-cult consistency. - v, err := v1IDFromBlobDigestAndComponents(blobDigest, parentV1ID) - if err != nil { - return nil, err - } - v1ID = v - - fakeImage := manifest.Schema1V1Compatibility{ - ID: v1ID, - Parent: parentV1ID, - Comment: historyEntry.Comment, - Created: historyEntry.Created, - Author: historyEntry.Author, - ThrowAway: historyEntry.EmptyLayer, - } - fakeImage.ContainerConfig.Cmd = []string{historyEntry.CreatedBy} - v1CompatibilityBytes, err := json.Marshal(&fakeImage) - if err != nil { - return nil, errors.Errorf("Internal error: Error creating v1compatibility for %#v", fakeImage) - } - - fsLayers[v1Index] = manifest.Schema1FSLayers{BlobSum: blobDigest} - history[v1Index] = manifest.Schema1History{V1Compatibility: string(v1CompatibilityBytes)} - // Note that parentV1ID of the top layer is preserved when exiting this loop - } - - // Now patch in real configuration for the top layer (v1Index == 0) - v1ID, err = v1IDFromBlobDigestAndComponents(fsLayers[0].BlobSum, parentV1ID, string(configBytes)) // See above WRT v1ID value generation and cargo-cult consistency. - if err != nil { - return nil, err - } - v1Config, err := v1ConfigFromConfigJSON(configBytes, v1ID, parentV1ID, imageConfig.History[len(imageConfig.History)-1].EmptyLayer) - if err != nil { - return nil, err - } - history[0].V1Compatibility = string(v1Config) - - if options.LayerInfos != nil { - options.LayerInfos = convertedLayerUpdates - } - m1, err := manifestSchema1FromComponents(dest.Reference().DockerReference(), fsLayers, history, imageConfig.Architecture) - if err != nil { - return nil, err // This should never happen, we should have created all the components correctly. - } - return m1, nil -} - -func v1IDFromBlobDigestAndComponents(blobDigest digest.Digest, others ...string) (string, error) { - if err := blobDigest.Validate(); err != nil { - return "", err - } - parts := append([]string{blobDigest.Hex()}, others...) - v1IDHash := sha256.Sum256([]byte(strings.Join(parts, " "))) - return hex.EncodeToString(v1IDHash[:]), nil -} - -func v1ConfigFromConfigJSON(configJSON []byte, v1ID, parentV1ID string, throwaway bool) ([]byte, error) { - // Preserve everything we don't specifically know about. - // (This must be a *json.RawMessage, even though *[]byte is fairly redundant, because only *RawMessage implements json.Marshaler.) - rawContents := map[string]*json.RawMessage{} - if err := json.Unmarshal(configJSON, &rawContents); err != nil { // We have already unmarshaled it before, using a more detailed schema?! - return nil, err - } - delete(rawContents, "rootfs") - delete(rawContents, "history") - - updates := map[string]interface{}{"id": v1ID} - if parentV1ID != "" { - updates["parent"] = parentV1ID - } - if throwaway { - updates["throwaway"] = throwaway - } - for field, value := range updates { - encoded, err := json.Marshal(value) - if err != nil { - return nil, err - } - rawContents[field] = (*json.RawMessage)(&encoded) - } - return json.Marshal(rawContents) -} - -// SupportsEncryption returns if encryption is supported for the manifest type -func (m *manifestSchema2) SupportsEncryption(context.Context) bool { - return false -} +const GzippedEmptyLayerDigest = image.GzippedEmptyLayerDigest diff --git a/vendor/github.com/containers/image/v5/image/sourced.go b/vendor/github.com/containers/image/v5/image/sourced.go index 3a016e1d0..2b7f6b144 100644 --- a/vendor/github.com/containers/image/v5/image/sourced.go +++ b/vendor/github.com/containers/image/v5/image/sourced.go @@ -6,17 +6,10 @@ package image import ( "context" + "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/types" ) -// imageCloser implements types.ImageCloser, perhaps allowing simple users -// to use a single object without having keep a reference to a types.ImageSource -// only to call types.ImageSource.Close(). -type imageCloser struct { - types.Image - src types.ImageSource -} - // FromSource returns a types.ImageCloser implementation for the default instance of source. // If source is a manifest list, .Manifest() still returns the manifest list, // but other methods transparently return data from an appropriate image instance. @@ -31,33 +24,7 @@ type imageCloser struct { // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage instead of calling this function. func FromSource(ctx context.Context, sys *types.SystemContext, src types.ImageSource) (types.ImageCloser, error) { - img, err := FromUnparsedImage(ctx, sys, UnparsedInstance(src, nil)) - if err != nil { - return nil, err - } - return &imageCloser{ - Image: img, - src: src, - }, nil -} - -func (ic *imageCloser) Close() error { - return ic.src.Close() -} - -// sourcedImage is a general set of utilities for working with container images, -// whatever is their underlying location (i.e. dockerImageSource-independent). -// Note the existence of skopeo/docker.Image: some instances of a `types.Image` -// may not be a `sourcedImage` directly. However, most users of `types.Image` -// do not care, and those who care about `skopeo/docker.Image` know they do. -type sourcedImage struct { - *UnparsedImage - manifestBlob []byte - manifestMIMEType string - // genericManifest contains data corresponding to manifestBlob. - // NOTE: The manifest may have been modified in the process; DO NOT reserialize and store genericManifest - // if you want to preserve the original manifest; use manifestBlob directly. - genericManifest + return image.FromSource(ctx, sys, src) } // FromUnparsedImage returns a types.Image implementation for unparsed. @@ -66,39 +33,5 @@ type sourcedImage struct { // // The Image must not be used after the underlying ImageSource is Close()d. func FromUnparsedImage(ctx context.Context, sys *types.SystemContext, unparsed *UnparsedImage) (types.Image, error) { - // Note that the input parameter above is specifically *image.UnparsedImage, not types.UnparsedImage: - // we want to be able to use unparsed.src. We could make that an explicit interface, but, well, - // this is the only UnparsedImage implementation around, anyway. - - // NOTE: It is essential for signature verification that all parsing done in this object happens on the same manifest which is returned by unparsed.Manifest(). - manifestBlob, manifestMIMEType, err := unparsed.Manifest(ctx) - if err != nil { - return nil, err - } - - parsedManifest, err := manifestInstanceFromBlob(ctx, sys, unparsed.src, manifestBlob, manifestMIMEType) - if err != nil { - return nil, err - } - - return &sourcedImage{ - UnparsedImage: unparsed, - manifestBlob: manifestBlob, - manifestMIMEType: manifestMIMEType, - genericManifest: parsedManifest, - }, nil -} - -// Size returns the size of the image as stored, if it's known, or -1 if it isn't. -func (i *sourcedImage) Size() (int64, error) { - return -1, nil -} - -// Manifest overrides the UnparsedImage.Manifest to always use the fields which we have already fetched. -func (i *sourcedImage) Manifest(ctx context.Context) ([]byte, string, error) { - return i.manifestBlob, i.manifestMIMEType, nil -} - -func (i *sourcedImage) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { - return i.UnparsedImage.src.LayerInfosForCopy(ctx, i.UnparsedImage.instanceDigest) + return image.FromUnparsedImage(ctx, sys, unparsed) } diff --git a/vendor/github.com/containers/image/v5/image/unparsed.go b/vendor/github.com/containers/image/v5/image/unparsed.go index c64852f72..123f6ce6f 100644 --- a/vendor/github.com/containers/image/v5/image/unparsed.go +++ b/vendor/github.com/containers/image/v5/image/unparsed.go @@ -1,95 +1,19 @@ package image import ( - "context" - - "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/types" "github.com/opencontainers/go-digest" - "github.com/pkg/errors" ) // UnparsedImage implements types.UnparsedImage . // An UnparsedImage is a pair of (ImageSource, instance digest); it can represent either a manifest list or a single image instance. -type UnparsedImage struct { - src types.ImageSource - instanceDigest *digest.Digest - cachedManifest []byte // A private cache for Manifest(); nil if not yet known. - // A private cache for Manifest(), may be the empty string if guessing failed. - // Valid iff cachedManifest is not nil. - cachedManifestMIMEType string - cachedSignatures [][]byte // A private cache for Signatures(); nil if not yet known. -} +type UnparsedImage = image.UnparsedImage // UnparsedInstance returns a types.UnparsedImage implementation for (source, instanceDigest). // If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list). // // The UnparsedImage must not be used after the underlying ImageSource is Close()d. func UnparsedInstance(src types.ImageSource, instanceDigest *digest.Digest) *UnparsedImage { - return &UnparsedImage{ - src: src, - instanceDigest: instanceDigest, - } -} - -// Reference returns the reference used to set up this source, _as specified by the user_ -// (not as the image itself, or its underlying storage, claims). This can be used e.g. to determine which public keys are trusted for this image. -func (i *UnparsedImage) Reference() types.ImageReference { - // Note that this does not depend on instanceDigest; e.g. all instances within a manifest list need to be signed with the manifest list identity. - return i.src.Reference() -} - -// Manifest is like ImageSource.GetManifest, but the result is cached; it is OK to call this however often you need. -func (i *UnparsedImage) Manifest(ctx context.Context) ([]byte, string, error) { - if i.cachedManifest == nil { - m, mt, err := i.src.GetManifest(ctx, i.instanceDigest) - if err != nil { - return nil, "", err - } - - // ImageSource.GetManifest does not do digest verification, but we do; - // this immediately protects also any user of types.Image. - if digest, haveDigest := i.expectedManifestDigest(); haveDigest { - matches, err := manifest.MatchesDigest(m, digest) - if err != nil { - return nil, "", errors.Wrap(err, "computing manifest digest") - } - if !matches { - return nil, "", errors.Errorf("Manifest does not match provided manifest digest %s", digest) - } - } - - i.cachedManifest = m - i.cachedManifestMIMEType = mt - } - return i.cachedManifest, i.cachedManifestMIMEType, nil -} - -// expectedManifestDigest returns a the expected value of the manifest digest, and an indicator whether it is known. -// The bool return value seems redundant with digest != ""; it is used explicitly -// to refuse (unexpected) situations when the digest exists but is "". -func (i *UnparsedImage) expectedManifestDigest() (digest.Digest, bool) { - if i.instanceDigest != nil { - return *i.instanceDigest, true - } - ref := i.Reference().DockerReference() - if ref != nil { - if canonical, ok := ref.(reference.Canonical); ok { - return canonical.Digest(), true - } - } - return "", false -} - -// Signatures is like ImageSource.GetSignatures, but the result is cached; it is OK to call this however often you need. -func (i *UnparsedImage) Signatures(ctx context.Context) ([][]byte, error) { - if i.cachedSignatures == nil { - sigs, err := i.src.GetSignatures(ctx, i.instanceDigest) - if err != nil { - return nil, err - } - i.cachedSignatures = sigs - } - return i.cachedSignatures, nil + return image.UnparsedInstance(src, instanceDigest) } diff --git a/vendor/github.com/containers/image/v5/image/docker_list.go b/vendor/github.com/containers/image/v5/internal/image/docker_list.go index af78ac1df..af78ac1df 100644 --- a/vendor/github.com/containers/image/v5/image/docker_list.go +++ b/vendor/github.com/containers/image/v5/internal/image/docker_list.go diff --git a/vendor/github.com/containers/image/v5/image/docker_schema1.go b/vendor/github.com/containers/image/v5/internal/image/docker_schema1.go index 5f24970c3..94f776224 100644 --- a/vendor/github.com/containers/image/v5/image/docker_schema1.go +++ b/vendor/github.com/containers/image/v5/internal/image/docker_schema1.go @@ -246,3 +246,12 @@ func (m *manifestSchema1) convertToManifestOCI1(ctx context.Context, options *ty func (m *manifestSchema1) SupportsEncryption(context.Context) bool { return false } + +// CanChangeLayerCompression returns true if we can compress/decompress layers with mimeType in the current image +// (and the code can handle that). +// NOTE: Even if this returns true, the relevant format might not accept all compression algorithms; the set of accepted +// algorithms depends not on the current format, but possibly on the target of a conversion (if UpdatedImage converts +// to a different manifest format). +func (m *manifestSchema1) CanChangeLayerCompression(mimeType string) bool { + return true // There are no MIME types in the manifest, so we must assume a valid image. +} diff --git a/vendor/github.com/containers/image/v5/internal/image/docker_schema2.go b/vendor/github.com/containers/image/v5/internal/image/docker_schema2.go new file mode 100644 index 000000000..7dfd3c5d8 --- /dev/null +++ b/vendor/github.com/containers/image/v5/internal/image/docker_schema2.go @@ -0,0 +1,413 @@ +package image + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "strings" + + "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/internal/iolimits" + "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/pkg/blobinfocache/none" + "github.com/containers/image/v5/types" + "github.com/opencontainers/go-digest" + imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// GzippedEmptyLayer is a gzip-compressed version of an empty tar file (1024 NULL bytes) +// This comes from github.com/docker/distribution/manifest/schema1/config_builder.go; there is +// a non-zero embedded timestamp; we could zero that, but that would just waste storage space +// in registries, so let’s use the same values. +// +// This is publicly visible as c/image/image.GzippedEmptyLayer. +var GzippedEmptyLayer = []byte{ + 31, 139, 8, 0, 0, 9, 110, 136, 0, 255, 98, 24, 5, 163, 96, 20, 140, 88, + 0, 8, 0, 0, 255, 255, 46, 175, 181, 239, 0, 4, 0, 0, +} + +// GzippedEmptyLayerDigest is a digest of GzippedEmptyLayer +// +// This is publicly visible as c/image/image.GzippedEmptyLayerDigest. +const GzippedEmptyLayerDigest = digest.Digest("sha256:a3ed95caeb02ffe68cdd9fd84406680ae93d633cb16422d00e8a7c22955b46d4") + +type manifestSchema2 struct { + src types.ImageSource // May be nil if configBlob is not nil + configBlob []byte // If set, corresponds to contents of ConfigDescriptor. + m *manifest.Schema2 +} + +func manifestSchema2FromManifest(src types.ImageSource, manifestBlob []byte) (genericManifest, error) { + m, err := manifest.Schema2FromManifest(manifestBlob) + if err != nil { + return nil, err + } + return &manifestSchema2{ + src: src, + m: m, + }, nil +} + +// manifestSchema2FromComponents builds a new manifestSchema2 from the supplied data: +func manifestSchema2FromComponents(config manifest.Schema2Descriptor, src types.ImageSource, configBlob []byte, layers []manifest.Schema2Descriptor) *manifestSchema2 { + return &manifestSchema2{ + src: src, + configBlob: configBlob, + m: manifest.Schema2FromComponents(config, layers), + } +} + +func (m *manifestSchema2) serialize() ([]byte, error) { + return m.m.Serialize() +} + +func (m *manifestSchema2) manifestMIMEType() string { + return m.m.MediaType +} + +// ConfigInfo returns a complete BlobInfo for the separate config object, or a BlobInfo{Digest:""} if there isn't a separate object. +// Note that the config object may not exist in the underlying storage in the return value of UpdatedImage! Use ConfigBlob() below. +func (m *manifestSchema2) ConfigInfo() types.BlobInfo { + return m.m.ConfigInfo() +} + +// OCIConfig returns the image configuration as per OCI v1 image-spec. Information about +// layers in the resulting configuration isn't guaranteed to be returned to due how +// old image manifests work (docker v2s1 especially). +func (m *manifestSchema2) OCIConfig(ctx context.Context) (*imgspecv1.Image, error) { + configBlob, err := m.ConfigBlob(ctx) + if err != nil { + return nil, err + } + // docker v2s2 and OCI v1 are mostly compatible but v2s2 contains more fields + // than OCI v1. This unmarshal makes sure we drop docker v2s2 + // fields that aren't needed in OCI v1. + configOCI := &imgspecv1.Image{} + if err := json.Unmarshal(configBlob, configOCI); err != nil { + return nil, err + } + return configOCI, nil +} + +// ConfigBlob returns the blob described by ConfigInfo, iff ConfigInfo().Digest != ""; nil otherwise. +// The result is cached; it is OK to call this however often you need. +func (m *manifestSchema2) ConfigBlob(ctx context.Context) ([]byte, error) { + if m.configBlob == nil { + if m.src == nil { + return nil, errors.Errorf("Internal error: neither src nor configBlob set in manifestSchema2") + } + stream, _, err := m.src.GetBlob(ctx, manifest.BlobInfoFromSchema2Descriptor(m.m.ConfigDescriptor), none.NoCache) + if err != nil { + return nil, err + } + defer stream.Close() + blob, err := iolimits.ReadAtMost(stream, iolimits.MaxConfigBodySize) + if err != nil { + return nil, err + } + computedDigest := digest.FromBytes(blob) + if computedDigest != m.m.ConfigDescriptor.Digest { + return nil, errors.Errorf("Download config.json digest %s does not match expected %s", computedDigest, m.m.ConfigDescriptor.Digest) + } + m.configBlob = blob + } + return m.configBlob, nil +} + +// LayerInfos returns a list of BlobInfos of layers referenced by this image, in order (the root layer first, and then successive layered layers). +// The Digest field is guaranteed to be provided; Size may be -1. +// WARNING: The list may contain duplicates, and they are semantically relevant. +func (m *manifestSchema2) LayerInfos() []types.BlobInfo { + return manifestLayerInfosToBlobInfos(m.m.LayerInfos()) +} + +// EmbeddedDockerReferenceConflicts whether a Docker reference embedded in the manifest, if any, conflicts with destination ref. +// It returns false if the manifest does not embed a Docker reference. +// (This embedding unfortunately happens for Docker schema1, please do not add support for this in any new formats.) +func (m *manifestSchema2) EmbeddedDockerReferenceConflicts(ref reference.Named) bool { + return false +} + +// Inspect returns various information for (skopeo inspect) parsed from the manifest and configuration. +func (m *manifestSchema2) Inspect(ctx context.Context) (*types.ImageInspectInfo, error) { + getter := func(info types.BlobInfo) ([]byte, error) { + if info.Digest != m.ConfigInfo().Digest { + // Shouldn't ever happen + return nil, errors.New("asked for a different config blob") + } + config, err := m.ConfigBlob(ctx) + if err != nil { + return nil, err + } + return config, nil + } + return m.m.Inspect(getter) +} + +// UpdatedImageNeedsLayerDiffIDs returns true iff UpdatedImage(options) needs InformationOnly.LayerDiffIDs. +// This is a horribly specific interface, but computing InformationOnly.LayerDiffIDs can be very expensive to compute +// (most importantly it forces us to download the full layers even if they are already present at the destination). +func (m *manifestSchema2) UpdatedImageNeedsLayerDiffIDs(options types.ManifestUpdateOptions) bool { + return false +} + +// UpdatedImage returns a types.Image modified according to options. +// This does not change the state of the original Image object. +// The returned error will be a manifest.ManifestLayerCompressionIncompatibilityError +// if the CompressionOperation and CompressionAlgorithm specified in one or more +// options.LayerInfos items is anything other than gzip. +func (m *manifestSchema2) UpdatedImage(ctx context.Context, options types.ManifestUpdateOptions) (types.Image, error) { + copy := manifestSchema2{ // NOTE: This is not a deep copy, it still shares slices etc. + src: m.src, + configBlob: m.configBlob, + m: manifest.Schema2Clone(m.m), + } + + converted, err := convertManifestIfRequiredWithUpdate(ctx, options, map[string]manifestConvertFn{ + manifest.DockerV2Schema1MediaType: copy.convertToManifestSchema1, + manifest.DockerV2Schema1SignedMediaType: copy.convertToManifestSchema1, + imgspecv1.MediaTypeImageManifest: copy.convertToManifestOCI1, + }) + if err != nil { + return nil, err + } + + if converted != nil { + return converted, nil + } + + // No conversion required, update manifest + if options.LayerInfos != nil { + if err := copy.m.UpdateLayerInfos(options.LayerInfos); err != nil { + return nil, err + } + } + // Ignore options.EmbeddedDockerReference: it may be set when converting from schema1 to schema2, but we really don't care. + + return memoryImageFromManifest(©), nil +} + +func oci1DescriptorFromSchema2Descriptor(d manifest.Schema2Descriptor) imgspecv1.Descriptor { + return imgspecv1.Descriptor{ + MediaType: d.MediaType, + Size: d.Size, + Digest: d.Digest, + URLs: d.URLs, + } +} + +// convertToManifestOCI1 returns a genericManifest implementation converted to imgspecv1.MediaTypeImageManifest. +// It may use options.InformationOnly and also adjust *options to be appropriate for editing the returned +// value. +// This does not change the state of the original manifestSchema2 object. +func (m *manifestSchema2) convertToManifestOCI1(ctx context.Context, _ *types.ManifestUpdateOptions) (genericManifest, error) { + configOCI, err := m.OCIConfig(ctx) + if err != nil { + return nil, err + } + configOCIBytes, err := json.Marshal(configOCI) + if err != nil { + return nil, err + } + + config := imgspecv1.Descriptor{ + MediaType: imgspecv1.MediaTypeImageConfig, + Size: int64(len(configOCIBytes)), + Digest: digest.FromBytes(configOCIBytes), + } + + layers := make([]imgspecv1.Descriptor, len(m.m.LayersDescriptors)) + for idx := range layers { + layers[idx] = oci1DescriptorFromSchema2Descriptor(m.m.LayersDescriptors[idx]) + switch m.m.LayersDescriptors[idx].MediaType { + case manifest.DockerV2Schema2ForeignLayerMediaType: + layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributable + case manifest.DockerV2Schema2ForeignLayerMediaTypeGzip: + layers[idx].MediaType = imgspecv1.MediaTypeImageLayerNonDistributableGzip + case manifest.DockerV2SchemaLayerMediaTypeUncompressed: + layers[idx].MediaType = imgspecv1.MediaTypeImageLayer + case manifest.DockerV2Schema2LayerMediaType: + layers[idx].MediaType = imgspecv1.MediaTypeImageLayerGzip + default: + return nil, fmt.Errorf("Unknown media type during manifest conversion: %q", m.m.LayersDescriptors[idx].MediaType) + } + } + + return manifestOCI1FromComponents(config, m.src, configOCIBytes, layers), nil +} + +// convertToManifestSchema1 returns a genericManifest implementation converted to manifest.DockerV2Schema1{Signed,}MediaType. +// It may use options.InformationOnly and also adjust *options to be appropriate for editing the returned +// value. +// This does not change the state of the original manifestSchema2 object. +// +// Based on docker/distribution/manifest/schema1/config_builder.go +func (m *manifestSchema2) convertToManifestSchema1(ctx context.Context, options *types.ManifestUpdateOptions) (genericManifest, error) { + dest := options.InformationOnly.Destination + + var convertedLayerUpdates []types.BlobInfo // Only used if options.LayerInfos != nil + if options.LayerInfos != nil { + if len(options.LayerInfos) != len(m.m.LayersDescriptors) { + return nil, fmt.Errorf("Error converting image: layer edits for %d layers vs %d existing layers", + len(options.LayerInfos), len(m.m.LayersDescriptors)) + } + convertedLayerUpdates = []types.BlobInfo{} + } + + configBytes, err := m.ConfigBlob(ctx) + if err != nil { + return nil, err + } + imageConfig := &manifest.Schema2Image{} + if err := json.Unmarshal(configBytes, imageConfig); err != nil { + return nil, err + } + + // Build fsLayers and History, discarding all configs. We will patch the top-level config in later. + fsLayers := make([]manifest.Schema1FSLayers, len(imageConfig.History)) + history := make([]manifest.Schema1History, len(imageConfig.History)) + nonemptyLayerIndex := 0 + var parentV1ID string // Set in the loop + v1ID := "" + haveGzippedEmptyLayer := false + if len(imageConfig.History) == 0 { + // What would this even mean?! Anyhow, the rest of the code depends on fsLayers[0] and history[0] existing. + return nil, errors.Errorf("Cannot convert an image with 0 history entries to %s", manifest.DockerV2Schema1SignedMediaType) + } + for v2Index, historyEntry := range imageConfig.History { + parentV1ID = v1ID + v1Index := len(imageConfig.History) - 1 - v2Index + + var blobDigest digest.Digest + if historyEntry.EmptyLayer { + emptyLayerBlobInfo := types.BlobInfo{Digest: GzippedEmptyLayerDigest, Size: int64(len(GzippedEmptyLayer))} + + if !haveGzippedEmptyLayer { + logrus.Debugf("Uploading empty layer during conversion to schema 1") + // Ideally we should update the relevant BlobInfoCache about this layer, but that would require passing it down here, + // and anyway this blob is so small that it’s easier to just copy it than to worry about figuring out another location where to get it. + info, err := dest.PutBlob(ctx, bytes.NewReader(GzippedEmptyLayer), emptyLayerBlobInfo, none.NoCache, false) + if err != nil { + return nil, errors.Wrap(err, "uploading empty layer") + } + if info.Digest != emptyLayerBlobInfo.Digest { + return nil, errors.Errorf("Internal error: Uploaded empty layer has digest %#v instead of %s", info.Digest, emptyLayerBlobInfo.Digest) + } + haveGzippedEmptyLayer = true + } + if options.LayerInfos != nil { + convertedLayerUpdates = append(convertedLayerUpdates, emptyLayerBlobInfo) + } + blobDigest = emptyLayerBlobInfo.Digest + } else { + if nonemptyLayerIndex >= len(m.m.LayersDescriptors) { + return nil, errors.Errorf("Invalid image configuration, needs more than the %d distributed layers", len(m.m.LayersDescriptors)) + } + if options.LayerInfos != nil { + convertedLayerUpdates = append(convertedLayerUpdates, options.LayerInfos[nonemptyLayerIndex]) + } + blobDigest = m.m.LayersDescriptors[nonemptyLayerIndex].Digest + nonemptyLayerIndex++ + } + + // AFAICT pull ignores these ID values, at least nowadays, so we could use anything unique, including a simple counter. Use what Docker uses for cargo-cult consistency. + v, err := v1IDFromBlobDigestAndComponents(blobDigest, parentV1ID) + if err != nil { + return nil, err + } + v1ID = v + + fakeImage := manifest.Schema1V1Compatibility{ + ID: v1ID, + Parent: parentV1ID, + Comment: historyEntry.Comment, + Created: historyEntry.Created, + Author: historyEntry.Author, + ThrowAway: historyEntry.EmptyLayer, + } + fakeImage.ContainerConfig.Cmd = []string{historyEntry.CreatedBy} + v1CompatibilityBytes, err := json.Marshal(&fakeImage) + if err != nil { + return nil, errors.Errorf("Internal error: Error creating v1compatibility for %#v", fakeImage) + } + + fsLayers[v1Index] = manifest.Schema1FSLayers{BlobSum: blobDigest} + history[v1Index] = manifest.Schema1History{V1Compatibility: string(v1CompatibilityBytes)} + // Note that parentV1ID of the top layer is preserved when exiting this loop + } + + // Now patch in real configuration for the top layer (v1Index == 0) + v1ID, err = v1IDFromBlobDigestAndComponents(fsLayers[0].BlobSum, parentV1ID, string(configBytes)) // See above WRT v1ID value generation and cargo-cult consistency. + if err != nil { + return nil, err + } + v1Config, err := v1ConfigFromConfigJSON(configBytes, v1ID, parentV1ID, imageConfig.History[len(imageConfig.History)-1].EmptyLayer) + if err != nil { + return nil, err + } + history[0].V1Compatibility = string(v1Config) + + if options.LayerInfos != nil { + options.LayerInfos = convertedLayerUpdates + } + m1, err := manifestSchema1FromComponents(dest.Reference().DockerReference(), fsLayers, history, imageConfig.Architecture) + if err != nil { + return nil, err // This should never happen, we should have created all the components correctly. + } + return m1, nil +} + +func v1IDFromBlobDigestAndComponents(blobDigest digest.Digest, others ...string) (string, error) { + if err := blobDigest.Validate(); err != nil { + return "", err + } + parts := append([]string{blobDigest.Hex()}, others...) + v1IDHash := sha256.Sum256([]byte(strings.Join(parts, " "))) + return hex.EncodeToString(v1IDHash[:]), nil +} + +func v1ConfigFromConfigJSON(configJSON []byte, v1ID, parentV1ID string, throwaway bool) ([]byte, error) { + // Preserve everything we don't specifically know about. + // (This must be a *json.RawMessage, even though *[]byte is fairly redundant, because only *RawMessage implements json.Marshaler.) + rawContents := map[string]*json.RawMessage{} + if err := json.Unmarshal(configJSON, &rawContents); err != nil { // We have already unmarshaled it before, using a more detailed schema?! + return nil, err + } + delete(rawContents, "rootfs") + delete(rawContents, "history") + + updates := map[string]interface{}{"id": v1ID} + if parentV1ID != "" { + updates["parent"] = parentV1ID + } + if throwaway { + updates["throwaway"] = throwaway + } + for field, value := range updates { + encoded, err := json.Marshal(value) + if err != nil { + return nil, err + } + rawContents[field] = (*json.RawMessage)(&encoded) + } + return json.Marshal(rawContents) +} + +// SupportsEncryption returns if encryption is supported for the manifest type +func (m *manifestSchema2) SupportsEncryption(context.Context) bool { + return false +} + +// CanChangeLayerCompression returns true if we can compress/decompress layers with mimeType in the current image +// (and the code can handle that). +// NOTE: Even if this returns true, the relevant format might not accept all compression algorithms; the set of accepted +// algorithms depends not on the current format, but possibly on the target of a conversion (if UpdatedImage converts +// to a different manifest format). +func (m *manifestSchema2) CanChangeLayerCompression(mimeType string) bool { + return m.m.CanChangeLayerCompression(mimeType) +} diff --git a/vendor/github.com/containers/image/v5/image/manifest.go b/vendor/github.com/containers/image/v5/internal/image/manifest.go index 36d70b5c2..6b5f34538 100644 --- a/vendor/github.com/containers/image/v5/image/manifest.go +++ b/vendor/github.com/containers/image/v5/internal/image/manifest.go @@ -12,9 +12,8 @@ import ( ) // genericManifest is an interface for parsing, modifying image manifests and related data. -// Note that the public methods are intended to be a subset of types.Image -// so that embedding a genericManifest into structs works. -// will support v1 one day... +// The public methods are related to types.Image so that embedding a genericManifest implements most of it, +// but there are also public methods that are only visible by packages that can import c/image/internal/image. type genericManifest interface { serialize() ([]byte, error) manifestMIMEType() string @@ -51,6 +50,16 @@ type genericManifest interface { // the process of updating a manifest between different manifest types was to update then convert. // This resulted in some fields in the update being lost. This has been fixed by: https://github.com/containers/image/pull/836 SupportsEncryption(ctx context.Context) bool + + // The following methods are not a part of types.Image: + // === + + // CanChangeLayerCompression returns true if we can compress/decompress layers with mimeType in the current image + // (and the code can handle that). + // NOTE: Even if this returns true, the relevant format might not accept all compression algorithms; the set of accepted + // algorithms depends not on the current format, but possibly on the target of a conversion (if UpdatedImage converts + // to a different manifest format). + CanChangeLayerCompression(mimeType string) bool } // manifestInstanceFromBlob returns a genericManifest implementation for (manblob, mt) in src. diff --git a/vendor/github.com/containers/image/v5/image/memory.go b/vendor/github.com/containers/image/v5/internal/image/memory.go index 4c96b37d8..4c96b37d8 100644 --- a/vendor/github.com/containers/image/v5/image/memory.go +++ b/vendor/github.com/containers/image/v5/internal/image/memory.go diff --git a/vendor/github.com/containers/image/v5/image/oci.go b/vendor/github.com/containers/image/v5/internal/image/oci.go index 58e9c03ba..af1a90e82 100644 --- a/vendor/github.com/containers/image/v5/image/oci.go +++ b/vendor/github.com/containers/image/v5/internal/image/oci.go @@ -7,6 +7,7 @@ import ( "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/internal/iolimits" + internalManifest "github.com/containers/image/v5/internal/manifest" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/blobinfocache/none" "github.com/containers/image/v5/types" @@ -84,6 +85,10 @@ func (m *manifestOCI1) ConfigBlob(ctx context.Context) ([]byte, error) { // layers in the resulting configuration isn't guaranteed to be returned to due how // old image manifests work (docker v2s1 especially). func (m *manifestOCI1) OCIConfig(ctx context.Context) (*imgspecv1.Image, error) { + if m.m.Config.MediaType != imgspecv1.MediaTypeImageConfig { + return nil, internalManifest.NewNonImageArtifactError(m.m.Config.MediaType) + } + cb, err := m.ConfigBlob(ctx) if err != nil { return nil, err @@ -194,10 +199,15 @@ func (m *manifestOCI1) convertToManifestSchema2Generic(ctx context.Context, opti // value. // This does not change the state of the original manifestOCI1 object. func (m *manifestOCI1) convertToManifestSchema2(_ context.Context, _ *types.ManifestUpdateOptions) (*manifestSchema2, error) { + if m.m.Config.MediaType != imgspecv1.MediaTypeImageConfig { + return nil, internalManifest.NewNonImageArtifactError(m.m.Config.MediaType) + } + // Create a copy of the descriptor. config := schema2DescriptorFromOCI1Descriptor(m.m.Config) - // The only difference between OCI and DockerSchema2 is the mediatypes. The + // Above, we have already checked that this manifest refers to an image, not an OCI artifact, + // so the only difference between OCI and DockerSchema2 is the mediatypes. The // media type of the manifest is handled by manifestSchema2FromComponents. config.MediaType = manifest.DockerV2Schema2ConfigMediaType @@ -233,7 +243,11 @@ func (m *manifestOCI1) convertToManifestSchema2(_ context.Context, _ *types.Mani // value. // This does not change the state of the original manifestOCI1 object. func (m *manifestOCI1) convertToManifestSchema1(ctx context.Context, options *types.ManifestUpdateOptions) (genericManifest, error) { - // We can't directly convert to V1, but we can transitively convert via a V2 image + if m.m.Config.MediaType != imgspecv1.MediaTypeImageConfig { + return nil, internalManifest.NewNonImageArtifactError(m.m.Config.MediaType) + } + + // We can't directly convert images to V1, but we can transitively convert via a V2 image m2, err := m.convertToManifestSchema2(ctx, options) if err != nil { return nil, err @@ -246,3 +260,12 @@ func (m *manifestOCI1) convertToManifestSchema1(ctx context.Context, options *ty func (m *manifestOCI1) SupportsEncryption(context.Context) bool { return true } + +// CanChangeLayerCompression returns true if we can compress/decompress layers with mimeType in the current image +// (and the code can handle that). +// NOTE: Even if this returns true, the relevant format might not accept all compression algorithms; the set of accepted +// algorithms depends not on the current format, but possibly on the target of a conversion (if UpdatedImage converts +// to a different manifest format). +func (m *manifestOCI1) CanChangeLayerCompression(mimeType string) bool { + return m.m.CanChangeLayerCompression(mimeType) +} diff --git a/vendor/github.com/containers/image/v5/image/oci_index.go b/vendor/github.com/containers/image/v5/internal/image/oci_index.go index d6e6685b1..d6e6685b1 100644 --- a/vendor/github.com/containers/image/v5/image/oci_index.go +++ b/vendor/github.com/containers/image/v5/internal/image/oci_index.go diff --git a/vendor/github.com/containers/image/v5/internal/image/sourced.go b/vendor/github.com/containers/image/v5/internal/image/sourced.go new file mode 100644 index 000000000..dc09a9e04 --- /dev/null +++ b/vendor/github.com/containers/image/v5/internal/image/sourced.go @@ -0,0 +1,134 @@ +// Package image consolidates knowledge about various container image formats +// (as opposed to image storage mechanisms, which are handled by types.ImageSource) +// and exposes all of them using an unified interface. +package image + +import ( + "context" + + "github.com/containers/image/v5/types" +) + +// FromReference returns a types.ImageCloser implementation for the default instance reading from reference. +// If reference poitns to a manifest list, .Manifest() still returns the manifest list, +// but other methods transparently return data from an appropriate image instance. +// +// The caller must call .Close() on the returned ImageCloser. +// +// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, +// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage instead of calling this function. +func FromReference(ctx context.Context, sys *types.SystemContext, ref types.ImageReference) (types.ImageCloser, error) { + src, err := ref.NewImageSource(ctx, sys) + if err != nil { + return nil, err + } + img, err := FromSource(ctx, sys, src) + if err != nil { + src.Close() + return nil, err + } + return img, nil +} + +// imageCloser implements types.ImageCloser, perhaps allowing simple users +// to use a single object without having keep a reference to a types.ImageSource +// only to call types.ImageSource.Close(). +type imageCloser struct { + types.Image + src types.ImageSource +} + +// FromSource returns a types.ImageCloser implementation for the default instance of source. +// If source is a manifest list, .Manifest() still returns the manifest list, +// but other methods transparently return data from an appropriate image instance. +// +// The caller must call .Close() on the returned ImageCloser. +// +// FromSource “takes ownership” of the input ImageSource and will call src.Close() +// when the image is closed. (This does not prevent callers from using both the +// Image and ImageSource objects simultaneously, but it means that they only need to +// the Image.) +// +// NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, +// verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage instead of calling this function. +// +// Most callers can use either FromUnparsedImage or FromReference instead. +// +// This is publicly visible as c/image/image.FromSource. +func FromSource(ctx context.Context, sys *types.SystemContext, src types.ImageSource) (types.ImageCloser, error) { + img, err := FromUnparsedImage(ctx, sys, UnparsedInstance(src, nil)) + if err != nil { + return nil, err + } + return &imageCloser{ + Image: img, + src: src, + }, nil +} + +func (ic *imageCloser) Close() error { + return ic.src.Close() +} + +// SourcedImage is a general set of utilities for working with container images, +// whatever is their underlying transport (i.e. ImageSource-independent). +// Note the existence of docker.Image and image.memoryImage: various instances +// of a types.Image may not be a SourcedImage directly. +// +// Most external users of `types.Image` do not care, and those who care about `docker.Image` know they do. +// +// Internal users may depend on methods available in SourcedImage but not (yet?) in types.Image. +type SourcedImage struct { + *UnparsedImage + ManifestBlob []byte // The manifest of the relevant instance + ManifestMIMEType string // MIME type of ManifestBlob + // genericManifest contains data corresponding to manifestBlob. + // NOTE: The manifest may have been modified in the process; DO NOT reserialize and store genericManifest + // if you want to preserve the original manifest; use manifestBlob directly. + genericManifest +} + +// FromUnparsedImage returns a types.Image implementation for unparsed. +// If unparsed represents a manifest list, .Manifest() still returns the manifest list, +// but other methods transparently return data from an appropriate single image. +// +// The Image must not be used after the underlying ImageSource is Close()d. +// +// This is publicly visible as c/image/image.FromUnparsedImage. +func FromUnparsedImage(ctx context.Context, sys *types.SystemContext, unparsed *UnparsedImage) (*SourcedImage, error) { + // Note that the input parameter above is specifically *image.UnparsedImage, not types.UnparsedImage: + // we want to be able to use unparsed.src. We could make that an explicit interface, but, well, + // this is the only UnparsedImage implementation around, anyway. + + // NOTE: It is essential for signature verification that all parsing done in this object happens on the same manifest which is returned by unparsed.Manifest(). + manifestBlob, manifestMIMEType, err := unparsed.Manifest(ctx) + if err != nil { + return nil, err + } + + parsedManifest, err := manifestInstanceFromBlob(ctx, sys, unparsed.src, manifestBlob, manifestMIMEType) + if err != nil { + return nil, err + } + + return &SourcedImage{ + UnparsedImage: unparsed, + ManifestBlob: manifestBlob, + ManifestMIMEType: manifestMIMEType, + genericManifest: parsedManifest, + }, nil +} + +// Size returns the size of the image as stored, if it's known, or -1 if it isn't. +func (i *SourcedImage) Size() (int64, error) { + return -1, nil +} + +// Manifest overrides the UnparsedImage.Manifest to always use the fields which we have already fetched. +func (i *SourcedImage) Manifest(ctx context.Context) ([]byte, string, error) { + return i.ManifestBlob, i.ManifestMIMEType, nil +} + +func (i *SourcedImage) LayerInfosForCopy(ctx context.Context) ([]types.BlobInfo, error) { + return i.UnparsedImage.src.LayerInfosForCopy(ctx, i.UnparsedImage.instanceDigest) +} diff --git a/vendor/github.com/containers/image/v5/internal/image/unparsed.go b/vendor/github.com/containers/image/v5/internal/image/unparsed.go new file mode 100644 index 000000000..8ea0f61b4 --- /dev/null +++ b/vendor/github.com/containers/image/v5/internal/image/unparsed.go @@ -0,0 +1,99 @@ +package image + +import ( + "context" + + "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/types" + "github.com/opencontainers/go-digest" + "github.com/pkg/errors" +) + +// UnparsedImage implements types.UnparsedImage . +// An UnparsedImage is a pair of (ImageSource, instance digest); it can represent either a manifest list or a single image instance. +// +// This is publicly visible as c/image/image.UnparsedImage. +type UnparsedImage struct { + src types.ImageSource + instanceDigest *digest.Digest + cachedManifest []byte // A private cache for Manifest(); nil if not yet known. + // A private cache for Manifest(), may be the empty string if guessing failed. + // Valid iff cachedManifest is not nil. + cachedManifestMIMEType string + cachedSignatures [][]byte // A private cache for Signatures(); nil if not yet known. +} + +// UnparsedInstance returns a types.UnparsedImage implementation for (source, instanceDigest). +// If instanceDigest is not nil, it contains a digest of the specific manifest instance to retrieve (when the primary manifest is a manifest list). +// +// The UnparsedImage must not be used after the underlying ImageSource is Close()d. +// +// This is publicly visible as c/image/image.UnparsedInstance. +func UnparsedInstance(src types.ImageSource, instanceDigest *digest.Digest) *UnparsedImage { + return &UnparsedImage{ + src: src, + instanceDigest: instanceDigest, + } +} + +// Reference returns the reference used to set up this source, _as specified by the user_ +// (not as the image itself, or its underlying storage, claims). This can be used e.g. to determine which public keys are trusted for this image. +func (i *UnparsedImage) Reference() types.ImageReference { + // Note that this does not depend on instanceDigest; e.g. all instances within a manifest list need to be signed with the manifest list identity. + return i.src.Reference() +} + +// Manifest is like ImageSource.GetManifest, but the result is cached; it is OK to call this however often you need. +func (i *UnparsedImage) Manifest(ctx context.Context) ([]byte, string, error) { + if i.cachedManifest == nil { + m, mt, err := i.src.GetManifest(ctx, i.instanceDigest) + if err != nil { + return nil, "", err + } + + // ImageSource.GetManifest does not do digest verification, but we do; + // this immediately protects also any user of types.Image. + if digest, haveDigest := i.expectedManifestDigest(); haveDigest { + matches, err := manifest.MatchesDigest(m, digest) + if err != nil { + return nil, "", errors.Wrap(err, "computing manifest digest") + } + if !matches { + return nil, "", errors.Errorf("Manifest does not match provided manifest digest %s", digest) + } + } + + i.cachedManifest = m + i.cachedManifestMIMEType = mt + } + return i.cachedManifest, i.cachedManifestMIMEType, nil +} + +// expectedManifestDigest returns a the expected value of the manifest digest, and an indicator whether it is known. +// The bool return value seems redundant with digest != ""; it is used explicitly +// to refuse (unexpected) situations when the digest exists but is "". +func (i *UnparsedImage) expectedManifestDigest() (digest.Digest, bool) { + if i.instanceDigest != nil { + return *i.instanceDigest, true + } + ref := i.Reference().DockerReference() + if ref != nil { + if canonical, ok := ref.(reference.Canonical); ok { + return canonical.Digest(), true + } + } + return "", false +} + +// Signatures is like ImageSource.GetSignatures, but the result is cached; it is OK to call this however often you need. +func (i *UnparsedImage) Signatures(ctx context.Context) ([][]byte, error) { + if i.cachedSignatures == nil { + sigs, err := i.src.GetSignatures(ctx, i.instanceDigest) + if err != nil { + return nil, err + } + i.cachedSignatures = sigs + } + return i.cachedSignatures, nil +} diff --git a/vendor/github.com/containers/image/v5/internal/manifest/errors.go b/vendor/github.com/containers/image/v5/internal/manifest/errors.go new file mode 100644 index 000000000..e5732a8c4 --- /dev/null +++ b/vendor/github.com/containers/image/v5/internal/manifest/errors.go @@ -0,0 +1,32 @@ +package manifest + +import "fmt" + +// NonImageArtifactError (detected via errors.As) is used when asking for an image-specific operation +// on an object which is not a “container image” in the standard sense (e.g. an OCI artifact) +// +// This is publicly visible as c/image/manifest.NonImageArtifactError (but we don’t provide a public constructor) +type NonImageArtifactError struct { + // Callers should not be blindly calling image-specific operations and only checking MIME types + // on failure; if they care about the artifact type, they should check before using it. + // If they blindly assume an image, they don’t really need this value; just a type check + // is sufficient for basic "we can only pull images" UI. + // + // Also, there are fairly widespread “artifacts” which nevertheless use imgspecv1.MediaTypeImageConfig, + // e.g. https://github.com/sigstore/cosign/blob/main/specs/SIGNATURE_SPEC.md , which could cause the callers + // to complain about a non-image artifact with the correct MIME type; we should probably add some other kind of + // type discrimination, _and_ somehow make it available in the API, if we expect API callers to make decisions + // based on that kind of data. + // + // So, let’s not expose this until a specific need is identified. + mimeType string +} + +// NewNonImageArtifactError returns a NonImageArtifactError about an artifact with mimeType. +func NewNonImageArtifactError(mimeType string) error { + return NonImageArtifactError{mimeType: mimeType} +} + +func (e NonImageArtifactError) Error() string { + return fmt.Sprintf("unsupported image-specific operation on artifact with type %q", e.mimeType) +} diff --git a/vendor/github.com/containers/image/v5/manifest/common.go b/vendor/github.com/containers/image/v5/manifest/common.go index 20955ab7f..9cf7dd3a9 100644 --- a/vendor/github.com/containers/image/v5/manifest/common.go +++ b/vendor/github.com/containers/image/v5/manifest/common.go @@ -118,6 +118,18 @@ type compressionMIMETypeSet map[string]string const mtsUncompressed = "" // A key in compressionMIMETypeSet for the uncompressed variant const mtsUnsupportedMIMEType = "" // A value in compressionMIMETypeSet that means “recognized but unsupported” +// findCompressionMIMETypeSet returns a pointer to a compressionMIMETypeSet in variantTable that contains a value of mimeType, or nil if not found +func findCompressionMIMETypeSet(variantTable []compressionMIMETypeSet, mimeType string) compressionMIMETypeSet { + for _, variants := range variantTable { + for _, mt := range variants { + if mt == mimeType { + return variants + } + } + } + return nil +} + // compressionVariantMIMEType returns a variant of mimeType for the specified algorithm (which may be nil // to mean "no compression"), based on variantTable. // The returned error will be a ManifestLayerCompressionIncompatibilityError if mimeType has variants @@ -130,29 +142,26 @@ func compressionVariantMIMEType(variantTable []compressionMIMETypeSet, mimeType if mimeType == mtsUnsupportedMIMEType { // Prevent matching against the {algo:mtsUnsupportedMIMEType} entries return "", fmt.Errorf("cannot update unknown MIME type") } - for _, variants := range variantTable { - for _, mt := range variants { - if mt == mimeType { // Found the variant - name := mtsUncompressed - if algorithm != nil { - name = algorithm.InternalUnstableUndocumentedMIMEQuestionMark() - } - if res, ok := variants[name]; ok { - if res != mtsUnsupportedMIMEType { - return res, nil - } - if name != mtsUncompressed { - return "", ManifestLayerCompressionIncompatibilityError{fmt.Sprintf("%s compression is not supported for type %q", name, mt)} - } - return "", ManifestLayerCompressionIncompatibilityError{fmt.Sprintf("uncompressed variant is not supported for type %q", mt)} - } - if name != mtsUncompressed { - return "", ManifestLayerCompressionIncompatibilityError{fmt.Sprintf("unknown compressed with algorithm %s variant for type %s", name, mt)} - } - // We can't very well say “the idea of no compression is unknown” - return "", ManifestLayerCompressionIncompatibilityError{fmt.Sprintf("uncompressed variant is not supported for type %q", mt)} + variants := findCompressionMIMETypeSet(variantTable, mimeType) + if variants != nil { + name := mtsUncompressed + if algorithm != nil { + name = algorithm.InternalUnstableUndocumentedMIMEQuestionMark() + } + if res, ok := variants[name]; ok { + if res != mtsUnsupportedMIMEType { + return res, nil } + if name != mtsUncompressed { + return "", ManifestLayerCompressionIncompatibilityError{fmt.Sprintf("%s compression is not supported for type %q", name, mimeType)} + } + return "", ManifestLayerCompressionIncompatibilityError{fmt.Sprintf("uncompressed variant is not supported for type %q", mimeType)} + } + if name != mtsUncompressed { + return "", ManifestLayerCompressionIncompatibilityError{fmt.Sprintf("unknown compressed with algorithm %s variant for type %s", name, mimeType)} } + // We can't very well say “the idea of no compression is unknown” + return "", ManifestLayerCompressionIncompatibilityError{fmt.Sprintf("uncompressed variant is not supported for type %q", mimeType)} } if algorithm != nil { return "", fmt.Errorf("unsupported MIME type for compression: %s", mimeType) @@ -209,3 +218,13 @@ type ManifestLayerCompressionIncompatibilityError struct { func (m ManifestLayerCompressionIncompatibilityError) Error() string { return m.text } + +// compressionVariantsRecognizeMIMEType returns true if variantTable contains data about compressing/decompressing layers with mimeType +// Note that the caller still needs to worry about a specific algorithm not being supported. +func compressionVariantsRecognizeMIMEType(variantTable []compressionMIMETypeSet, mimeType string) bool { + if mimeType == mtsUnsupportedMIMEType { // Prevent matching against the {algo:mtsUnsupportedMIMEType} entries + return false + } + variants := findCompressionMIMETypeSet(variantTable, mimeType) + return variants != nil // Alternatively, this could be len(variants) > 1, but really the caller should ask about a specific algorithm. +} diff --git a/vendor/github.com/containers/image/v5/manifest/docker_schema2.go b/vendor/github.com/containers/image/v5/manifest/docker_schema2.go index 1f4db54ee..8b3fbdd39 100644 --- a/vendor/github.com/containers/image/v5/manifest/docker_schema2.go +++ b/vendor/github.com/containers/image/v5/manifest/docker_schema2.go @@ -295,3 +295,11 @@ func (m *Schema2) ImageID([]digest.Digest) (string, error) { } return m.ConfigDescriptor.Digest.Hex(), nil } + +// CanChangeLayerCompression returns true if we can compress/decompress layers with mimeType in the current image +// (and the code can handle that). +// NOTE: Even if this returns true, the relevant format might not accept all compression algorithms; the set of accepted +// algorithms depends not on the current format, but possibly on the target of a conversion. +func (m *Schema2) CanChangeLayerCompression(mimeType string) bool { + return compressionVariantsRecognizeMIMEType(schema2CompressionMIMETypeSets, mimeType) +} diff --git a/vendor/github.com/containers/image/v5/manifest/manifest.go b/vendor/github.com/containers/image/v5/manifest/manifest.go index 2e3e5da15..53fc866a7 100644 --- a/vendor/github.com/containers/image/v5/manifest/manifest.go +++ b/vendor/github.com/containers/image/v5/manifest/manifest.go @@ -4,6 +4,7 @@ import ( "encoding/json" "fmt" + internalManifest "github.com/containers/image/v5/internal/manifest" "github.com/containers/image/v5/types" "github.com/containers/libtrust" digest "github.com/opencontainers/go-digest" @@ -34,6 +35,10 @@ const ( DockerV2Schema2ForeignLayerMediaTypeGzip = "application/vnd.docker.image.rootfs.foreign.diff.tar.gzip" ) +// NonImageArtifactError (detected via errors.As) is used when asking for an image-specific operation +// on an object which is not a “container image” in the standard sense (e.g. an OCI artifact) +type NonImageArtifactError = internalManifest.NonImageArtifactError + // SupportedSchema2MediaType checks if the specified string is a supported Docker v2s2 media type. func SupportedSchema2MediaType(m string) error { switch m { diff --git a/vendor/github.com/containers/image/v5/manifest/oci.go b/vendor/github.com/containers/image/v5/manifest/oci.go index 5892184df..11927ab5e 100644 --- a/vendor/github.com/containers/image/v5/manifest/oci.go +++ b/vendor/github.com/containers/image/v5/manifest/oci.go @@ -5,6 +5,7 @@ import ( "fmt" "strings" + internalManifest "github.com/containers/image/v5/internal/manifest" compressiontypes "github.com/containers/image/v5/pkg/compression/types" "github.com/containers/image/v5/types" ociencspec "github.com/containers/ocicrypt/spec" @@ -115,6 +116,12 @@ var oci1CompressionMIMETypeSets = []compressionMIMETypeSet{ // UpdateLayerInfos replaces the original layers with the specified BlobInfos (size+digest+urls+mediatype), in order (the root layer first, and then successive layered layers) // The returned error will be a manifest.ManifestLayerCompressionIncompatibilityError if any of the layerInfos includes a combination of CompressionOperation and // CompressionAlgorithm that isn't supported by OCI. +// +// It’s generally the caller’s responsibility to determine whether a particular edit is acceptable, rather than relying on +// failures of this function, because the layer is typically created _before_ UpdateLayerInfos is called, because UpdateLayerInfos needs +// to know the final digest). See OCI1.CanChangeLayerCompression for some help in determining this; other aspects like compression +// algorithms that might not be supported by a format, or the limited set of MIME types accepted for encryption, are not currently +// handled — that logic should eventually also be provided as OCI1 methods, not hard-coded in callers. func (m *OCI1) UpdateLayerInfos(layerInfos []types.BlobInfo) error { if len(m.Layers) != len(layerInfos) { return errors.Errorf("Error preparing updated manifest: layer count changed from %d to %d", len(m.Layers), len(layerInfos)) @@ -151,6 +158,33 @@ func (m *OCI1) UpdateLayerInfos(layerInfos []types.BlobInfo) error { return nil } +// getEncryptedMediaType will return the mediatype to its encrypted counterpart and return +// an error if the mediatype does not support encryption +func getEncryptedMediaType(mediatype string) (string, error) { + for _, s := range strings.Split(mediatype, "+")[1:] { + if s == "encrypted" { + return "", errors.Errorf("unsupportedmediatype: %v already encrypted", mediatype) + } + } + unsuffixedMediatype := strings.Split(mediatype, "+")[0] + switch unsuffixedMediatype { + case DockerV2Schema2LayerMediaType, imgspecv1.MediaTypeImageLayer, imgspecv1.MediaTypeImageLayerNonDistributable: + return mediatype + "+encrypted", nil + } + + return "", errors.Errorf("unsupported mediatype to encrypt: %v", mediatype) +} + +// getEncryptedMediaType will return the mediatype to its encrypted counterpart and return +// an error if the mediatype does not support decryption +func getDecryptedMediaType(mediatype string) (string, error) { + if !strings.HasSuffix(mediatype, "+encrypted") { + return "", errors.Errorf("unsupported mediatype to decrypt %v:", mediatype) + } + + return strings.TrimSuffix(mediatype, "+encrypted"), nil +} + // Serialize returns the manifest in a blob format. // NOTE: Serialize() does not in general reproduce the original blob if this object was loaded from one, even if no modifications were made! func (m *OCI1) Serialize() ([]byte, error) { @@ -159,6 +193,14 @@ func (m *OCI1) Serialize() ([]byte, error) { // Inspect returns various information for (skopeo inspect) parsed from the manifest and configuration. func (m *OCI1) Inspect(configGetter func(types.BlobInfo) ([]byte, error)) (*types.ImageInspectInfo, error) { + if m.Config.MediaType != imgspecv1.MediaTypeImageConfig { + // We could return at least the layers, but that’s already available in a better format via types.Image.LayerInfos. + // Most software calling this without human intervention is going to expect the values to be realistic and relevant, + // and is probably better served by failing; we can always re-visit that later if we fail now, but + // if we started returning some data for OCI artifacts now, we couldn’t start failing in this function later. + return nil, internalManifest.NewNonImageArtifactError(m.Config.MediaType) + } + config, err := configGetter(m.ConfigInfo()) if err != nil { return nil, err @@ -186,35 +228,39 @@ func (m *OCI1) Inspect(configGetter func(types.BlobInfo) ([]byte, error)) (*type // ImageID computes an ID which can uniquely identify this image by its contents. func (m *OCI1) ImageID([]digest.Digest) (string, error) { + // The way m.Config.Digest “uniquely identifies” an image is + // by containing RootFS.DiffIDs, which identify the layers of the image. + // For non-image artifacts, the we can’t expect the config to change + // any time the other layers (semantically) change, so this approach of + // distinguishing objects only by m.Config.Digest doesn’t work in general. + // + // Any caller of this method presumably wants to disambiguate the same + // images with a different representation, but doesn’t want to disambiguate + // representations (by using a manifest digest). So, submitting a non-image + // artifact to such a caller indicates an expectation mismatch. + // So, we just fail here instead of inventing some other ID value (e.g. + // by combining the config and blob layer digests). That still + // gives us the option to not fail, and return some value, in the future, + // without committing to that approach now. + // (The only known caller of ImageID is storage/storageImageDestination.computeID, + // which can’t work with non-image artifacts.) + if m.Config.MediaType != imgspecv1.MediaTypeImageConfig { + return "", internalManifest.NewNonImageArtifactError(m.Config.MediaType) + } + if err := m.Config.Digest.Validate(); err != nil { return "", err } return m.Config.Digest.Hex(), nil } -// getEncryptedMediaType will return the mediatype to its encrypted counterpart and return -// an error if the mediatype does not support encryption -func getEncryptedMediaType(mediatype string) (string, error) { - for _, s := range strings.Split(mediatype, "+")[1:] { - if s == "encrypted" { - return "", errors.Errorf("unsupportedmediatype: %v already encrypted", mediatype) - } - } - unsuffixedMediatype := strings.Split(mediatype, "+")[0] - switch unsuffixedMediatype { - case DockerV2Schema2LayerMediaType, imgspecv1.MediaTypeImageLayer, imgspecv1.MediaTypeImageLayerNonDistributable: - return mediatype + "+encrypted", nil - } - - return "", errors.Errorf("unsupported mediatype to encrypt: %v", mediatype) -} - -// getEncryptedMediaType will return the mediatype to its encrypted counterpart and return -// an error if the mediatype does not support decryption -func getDecryptedMediaType(mediatype string) (string, error) { - if !strings.HasSuffix(mediatype, "+encrypted") { - return "", errors.Errorf("unsupported mediatype to decrypt %v:", mediatype) +// CanChangeLayerCompression returns true if we can compress/decompress layers with mimeType in the current image +// (and the code can handle that). +// NOTE: Even if this returns true, the relevant format might not accept all compression algorithms; the set of accepted +// algorithms depends not on the current format, but possibly on the target of a conversion. +func (m *OCI1) CanChangeLayerCompression(mimeType string) bool { + if m.Config.MediaType != imgspecv1.MediaTypeImageConfig { + return false } - - return strings.TrimSuffix(mediatype, "+encrypted"), nil + return compressionVariantsRecognizeMIMEType(oci1CompressionMIMETypeSets, mimeType) } diff --git a/vendor/github.com/containers/image/v5/oci/archive/oci_transport.go b/vendor/github.com/containers/image/v5/oci/archive/oci_transport.go index 4fa912765..74fefbd4f 100644 --- a/vendor/github.com/containers/image/v5/oci/archive/oci_transport.go +++ b/vendor/github.com/containers/image/v5/oci/archive/oci_transport.go @@ -8,7 +8,7 @@ import ( "github.com/containers/image/v5/directory/explicitfilepath" "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/image" + "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/internal/tmpdir" "github.com/containers/image/v5/oci/internal" ocilayout "github.com/containers/image/v5/oci/layout" @@ -122,11 +122,7 @@ func (ref ociArchiveReference) PolicyConfigurationNamespaces() []string { // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. // WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. func (ref ociArchiveReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { - src, err := newImageSource(ctx, sys, ref) - if err != nil { - return nil, err - } - return image.FromSource(ctx, sys, src) + return image.FromReference(ctx, sys, ref) } // NewImageSource returns a types.ImageSource for this reference. diff --git a/vendor/github.com/containers/image/v5/oci/layout/oci_transport.go b/vendor/github.com/containers/image/v5/oci/layout/oci_transport.go index a99b63158..a9029a609 100644 --- a/vendor/github.com/containers/image/v5/oci/layout/oci_transport.go +++ b/vendor/github.com/containers/image/v5/oci/layout/oci_transport.go @@ -10,7 +10,7 @@ import ( "github.com/containers/image/v5/directory/explicitfilepath" "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/image" + "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/oci/internal" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" @@ -154,11 +154,7 @@ func (ref ociReference) PolicyConfigurationNamespaces() []string { // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. // WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. func (ref ociReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { - src, err := newImageSource(sys, ref) - if err != nil { - return nil, err - } - return image.FromSource(ctx, sys, src) + return image.FromReference(ctx, sys, ref) } // getIndex returns a pointer to the index references by this ociReference. If an error occurs opening an index nil is returned together diff --git a/vendor/github.com/containers/image/v5/openshift/openshift_transport.go b/vendor/github.com/containers/image/v5/openshift/openshift_transport.go index 6bbb43be2..c8d65c78a 100644 --- a/vendor/github.com/containers/image/v5/openshift/openshift_transport.go +++ b/vendor/github.com/containers/image/v5/openshift/openshift_transport.go @@ -8,7 +8,7 @@ import ( "github.com/containers/image/v5/docker/policyconfiguration" "github.com/containers/image/v5/docker/reference" - genericImage "github.com/containers/image/v5/image" + genericImage "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" "github.com/pkg/errors" @@ -132,11 +132,7 @@ func (ref openshiftReference) PolicyConfigurationNamespaces() []string { // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. // WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. func (ref openshiftReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { - src, err := newImageSource(sys, ref) - if err != nil { - return nil, err - } - return genericImage.FromSource(ctx, sys, src) + return genericImage.FromReference(ctx, sys, ref) } // NewImageSource returns a types.ImageSource for this reference. diff --git a/vendor/github.com/containers/image/v5/ostree/ostree_transport.go b/vendor/github.com/containers/image/v5/ostree/ostree_transport.go index 1e35ab605..6c4262368 100644 --- a/vendor/github.com/containers/image/v5/ostree/ostree_transport.go +++ b/vendor/github.com/containers/image/v5/ostree/ostree_transport.go @@ -14,7 +14,7 @@ import ( "github.com/containers/image/v5/directory/explicitfilepath" "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/image" + "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" "github.com/pkg/errors" @@ -184,17 +184,7 @@ func (s *ostreeImageCloser) Size() (int64, error) { // NOTE: If any kind of signature verification should happen, build an UnparsedImage from the value returned by NewImageSource, // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. func (ref ostreeReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { - var tmpDir string - if sys == nil || sys.OSTreeTmpDirPath == "" { - tmpDir = os.TempDir() - } else { - tmpDir = sys.OSTreeTmpDirPath - } - src, err := newImageSource(tmpDir, ref) - if err != nil { - return nil, err - } - return image.FromSource(ctx, sys, src) + return image.FromReference(ctx, sys, ref) } // NewImageSource returns a types.ImageSource for this reference. diff --git a/vendor/github.com/containers/image/v5/pkg/blobcache/blobcache.go b/vendor/github.com/containers/image/v5/pkg/blobcache/blobcache.go index 8b22733ac..c9971cbdc 100644 --- a/vendor/github.com/containers/image/v5/pkg/blobcache/blobcache.go +++ b/vendor/github.com/containers/image/v5/pkg/blobcache/blobcache.go @@ -9,7 +9,7 @@ import ( "sync" "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/image" + "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/pkg/compression" "github.com/containers/image/v5/transports" @@ -158,11 +158,7 @@ func (b *BlobCache) ClearCache() error { } func (b *BlobCache) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { - src, err := b.NewImageSource(ctx, sys) - if err != nil { - return nil, errors.Wrapf(err, "error creating new image %q", transports.ImageName(b.reference)) - } - return image.FromSource(ctx, sys, src) + return image.FromReference(ctx, sys, b) } func (b *BlobCache) NewImageSource(ctx context.Context, sys *types.SystemContext) (types.ImageSource, error) { diff --git a/vendor/github.com/containers/image/v5/sif/transport.go b/vendor/github.com/containers/image/v5/sif/transport.go index 18d894bc3..2037f2508 100644 --- a/vendor/github.com/containers/image/v5/sif/transport.go +++ b/vendor/github.com/containers/image/v5/sif/transport.go @@ -9,7 +9,7 @@ import ( "github.com/containers/image/v5/directory/explicitfilepath" "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/image" + "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/transports" "github.com/containers/image/v5/types" ) @@ -139,11 +139,7 @@ func (ref sifReference) PolicyConfigurationNamespaces() []string { // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. // WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. func (ref sifReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { - src, err := newImageSource(ctx, sys, ref) - if err != nil { - return nil, err - } - return image.FromSource(ctx, sys, src) + return image.FromReference(ctx, sys, ref) } // NewImageSource returns a types.ImageSource for this reference. diff --git a/vendor/github.com/containers/image/v5/storage/storage_image.go b/vendor/github.com/containers/image/v5/storage/storage_image.go index 8071e3b32..06f90d363 100644 --- a/vendor/github.com/containers/image/v5/storage/storage_image.go +++ b/vendor/github.com/containers/image/v5/storage/storage_image.go @@ -16,7 +16,7 @@ import ( "sync/atomic" "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/image" + "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/internal/private" "github.com/containers/image/v5/internal/putblobdigest" "github.com/containers/image/v5/internal/tmpdir" @@ -486,7 +486,7 @@ func (s *storageImageDestination) PutBlob(ctx context.Context, stream io.Reader, } // putBlobToPendingFile implements ImageDestination.PutBlobWithOptions, storing stream into an on-disk file. -// The caller must arrange the blob to be eventually commited using s.commitLayer(). +// The caller must arrange the blob to be eventually committed using s.commitLayer(). func (s *storageImageDestination) putBlobToPendingFile(ctx context.Context, stream io.Reader, blobinfo types.BlobInfo, options *private.PutBlobOptions) (types.BlobInfo, error) { // Stores a layer or data blob in our temporary directory, checking that any information // in the blobinfo matches the incoming data. @@ -641,7 +641,7 @@ func (s *storageImageDestination) TryReusingBlob(ctx context.Context, blobinfo t } // tryReusingBlobAsPending implements TryReusingBlobWithOptions, filling s.blobDiffIDs and other metadata. -// The caller must arrange the blob to be eventually commited using s.commitLayer(). +// The caller must arrange the blob to be eventually committed using s.commitLayer(). func (s *storageImageDestination) tryReusingBlobAsPending(ctx context.Context, blobinfo types.BlobInfo, options *private.TryReusingBlobOptions) (bool, types.BlobInfo, error) { // lock the entire method as it executes fairly quickly s.lock.Lock() @@ -941,7 +941,7 @@ func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest s.lock.Unlock() if ok { layer, err := al.PutAs(id, lastLayer, nil) - if err != nil { + if err != nil && errors.Cause(err) != storage.ErrDuplicateID { return errors.Wrapf(err, "failed to put layer from digest and labels") } lastLayer = layer.ID diff --git a/vendor/github.com/containers/image/v5/tarball/tarball_reference.go b/vendor/github.com/containers/image/v5/tarball/tarball_reference.go index 23f67c49e..690067ec3 100644 --- a/vendor/github.com/containers/image/v5/tarball/tarball_reference.go +++ b/vendor/github.com/containers/image/v5/tarball/tarball_reference.go @@ -7,7 +7,7 @@ import ( "strings" "github.com/containers/image/v5/docker/reference" - "github.com/containers/image/v5/image" + "github.com/containers/image/v5/internal/image" "github.com/containers/image/v5/types" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -67,16 +67,7 @@ func (r *tarballReference) PolicyConfigurationNamespaces() []string { // verify that UnparsedImage, and convert it into a real Image via image.FromUnparsedImage. // WARNING: This may not do the right thing for a manifest list, see image.FromSource for details. func (r *tarballReference) NewImage(ctx context.Context, sys *types.SystemContext) (types.ImageCloser, error) { - src, err := r.NewImageSource(ctx, sys) - if err != nil { - return nil, err - } - img, err := image.FromSource(ctx, sys, src) - if err != nil { - src.Close() - return nil, err - } - return img, nil + return image.FromReference(ctx, sys, r) } func (r *tarballReference) DeleteImage(ctx context.Context, sys *types.SystemContext) error { diff --git a/vendor/github.com/containers/ocicrypt/.travis.yml b/vendor/github.com/containers/ocicrypt/.travis.yml index e4dd4a402..1036c8d3f 100644 --- a/vendor/github.com/containers/ocicrypt/.travis.yml +++ b/vendor/github.com/containers/ocicrypt/.travis.yml @@ -21,7 +21,7 @@ addons: go_import_path: github.com/containers/ocicrypt install: - - curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.30.0 + - curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v1.46.2 script: - make diff --git a/vendor/github.com/containers/ocicrypt/config/constructors.go b/vendor/github.com/containers/ocicrypt/config/constructors.go index a789d052d..c537a20a0 100644 --- a/vendor/github.com/containers/ocicrypt/config/constructors.go +++ b/vendor/github.com/containers/ocicrypt/config/constructors.go @@ -21,7 +21,7 @@ import ( "strings" "github.com/pkg/errors" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // EncryptWithJwe returns a CryptoConfig to encrypt with jwe public keys diff --git a/vendor/github.com/containers/ocicrypt/config/pkcs11config/config.go b/vendor/github.com/containers/ocicrypt/config/pkcs11config/config.go index 76be34138..5a8bc022c 100644 --- a/vendor/github.com/containers/ocicrypt/config/pkcs11config/config.go +++ b/vendor/github.com/containers/ocicrypt/config/pkcs11config/config.go @@ -24,7 +24,7 @@ import ( "github.com/containers/ocicrypt/crypto/pkcs11" "github.com/pkg/errors" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // OcicryptConfig represents the format of an imgcrypt.conf config file diff --git a/vendor/github.com/containers/ocicrypt/crypto/pkcs11/common.go b/vendor/github.com/containers/ocicrypt/crypto/pkcs11/common.go index 7fcd2e3af..c6d47e830 100644 --- a/vendor/github.com/containers/ocicrypt/crypto/pkcs11/common.go +++ b/vendor/github.com/containers/ocicrypt/crypto/pkcs11/common.go @@ -17,7 +17,7 @@ import ( "fmt" "github.com/pkg/errors" pkcs11uri "github.com/stefanberger/go-pkcs11uri" - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v3" ) // Pkcs11KeyFile describes the format of the pkcs11 (private) key file. diff --git a/vendor/github.com/containers/ocicrypt/go.mod b/vendor/github.com/containers/ocicrypt/go.mod index 8837d288e..46ee2a289 100644 --- a/vendor/github.com/containers/ocicrypt/go.mod +++ b/vendor/github.com/containers/ocicrypt/go.mod @@ -17,5 +17,5 @@ require ( golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 google.golang.org/grpc v1.33.2 gopkg.in/square/go-jose.v2 v2.5.1 - gopkg.in/yaml.v2 v2.4.0 + gopkg.in/yaml.v3 v3.0.0 ) diff --git a/vendor/github.com/containers/ocicrypt/go.sum b/vendor/github.com/containers/ocicrypt/go.sum index a621a145c..86e36e768 100644 --- a/vendor/github.com/containers/ocicrypt/go.sum +++ b/vendor/github.com/containers/ocicrypt/go.sum @@ -111,7 +111,7 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/square/go-jose.v2 v2.5.1 h1:7odma5RETjNHWJnR32wx8t+Io4djHE1PqxCFx3iiZ2w= gopkg.in/square/go-jose.v2 v2.5.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= -gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= -gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= +gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/containers/storage/.cirrus.yml b/vendor/github.com/containers/storage/.cirrus.yml index fd3d31054..53b13cd33 100644 --- a/vendor/github.com/containers/storage/.cirrus.yml +++ b/vendor/github.com/containers/storage/.cirrus.yml @@ -17,14 +17,14 @@ env: #### #### Cache-image names to test with (double-quotes around names are critical) ### - FEDORA_NAME: "fedora-35" - PRIOR_FEDORA_NAME: "fedora-34" - UBUNTU_NAME: "ubuntu-2104" + FEDORA_NAME: "fedora-36" + PRIOR_FEDORA_NAME: "fedora-35" + UBUNTU_NAME: "ubuntu-2204" # GCE project where images live IMAGE_PROJECT: "libpod-218412" # VM Image built in containers/automation_images - IMAGE_SUFFIX: "c4512539143831552" + IMAGE_SUFFIX: "c5878804328480768" FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}" UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}" diff --git a/vendor/github.com/containers/storage/drivers/chown_darwin.go b/vendor/github.com/containers/storage/drivers/chown_darwin.go new file mode 100644 index 000000000..cf608d479 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/chown_darwin.go @@ -0,0 +1,109 @@ +//go:build darwin +// +build darwin + +package graphdriver + +import ( + "errors" + "fmt" + "os" + "sync" + "syscall" + + "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/pkg/system" +) + +type inode struct { + Dev uint64 + Ino uint64 +} + +type platformChowner struct { + mutex sync.Mutex + inodes map[inode]bool +} + +func newLChowner() *platformChowner { + return &platformChowner{ + inodes: make(map[inode]bool), + } +} + +func (c *platformChowner) LChown(path string, info os.FileInfo, toHost, toContainer *idtools.IDMappings) error { + st, ok := info.Sys().(*syscall.Stat_t) + if !ok { + return nil + } + + i := inode{ + Dev: uint64(st.Dev), + Ino: uint64(st.Ino), + } + c.mutex.Lock() + _, found := c.inodes[i] + if !found { + c.inodes[i] = true + } + c.mutex.Unlock() + + if found { + return nil + } + + // Map an on-disk UID/GID pair from host to container + // using the first map, then back to the host using the + // second map. Skip that first step if they're 0, to + // compensate for cases where a parent layer should + // have had a mapped value, but didn't. + uid, gid := int(st.Uid), int(st.Gid) + if toContainer != nil { + pair := idtools.IDPair{ + UID: uid, + GID: gid, + } + mappedUID, mappedGID, err := toContainer.ToContainer(pair) + if err != nil { + if (uid != 0) || (gid != 0) { + return fmt.Errorf("error mapping host ID pair %#v for %q to container: %v", pair, path, err) + } + mappedUID, mappedGID = uid, gid + } + uid, gid = mappedUID, mappedGID + } + if toHost != nil { + pair := idtools.IDPair{ + UID: uid, + GID: gid, + } + mappedPair, err := toHost.ToHostOverflow(pair) + if err != nil { + return fmt.Errorf("error mapping container ID pair %#v for %q to host: %v", pair, path, err) + } + uid, gid = mappedPair.UID, mappedPair.GID + } + if uid != int(st.Uid) || gid != int(st.Gid) { + cap, err := system.Lgetxattr(path, "security.capability") + if err != nil && !errors.Is(err, system.EOPNOTSUPP) && err != system.ErrNotSupportedPlatform { + return fmt.Errorf("%s: %v", os.Args[0], err) + } + + // Make the change. + if err := system.Lchown(path, uid, gid); err != nil { + return fmt.Errorf("%s: %v", os.Args[0], err) + } + // Restore the SUID and SGID bits if they were originally set. + if (info.Mode()&os.ModeSymlink == 0) && info.Mode()&(os.ModeSetuid|os.ModeSetgid) != 0 { + if err := system.Chmod(path, info.Mode()); err != nil { + return fmt.Errorf("%s: %v", os.Args[0], err) + } + } + if cap != nil { + if err := system.Lsetxattr(path, "security.capability", cap, 0); err != nil { + return fmt.Errorf("%s: %v", os.Args[0], err) + } + } + + } + return nil +} diff --git a/vendor/github.com/containers/storage/drivers/chown_unix.go b/vendor/github.com/containers/storage/drivers/chown_unix.go index c598b936d..65756d8ec 100644 --- a/vendor/github.com/containers/storage/drivers/chown_unix.go +++ b/vendor/github.com/containers/storage/drivers/chown_unix.go @@ -1,5 +1,5 @@ -//go:build !windows -// +build !windows +//go:build !windows && !darwin +// +build !windows,!darwin package graphdriver @@ -21,12 +21,12 @@ type inode struct { type platformChowner struct { mutex sync.Mutex - inodes map[inode]bool + inodes map[inode]string } func newLChowner() *platformChowner { return &platformChowner{ - inodes: make(map[inode]bool), + inodes: make(map[inode]string), } } @@ -40,15 +40,33 @@ func (c *platformChowner) LChown(path string, info os.FileInfo, toHost, toContai Dev: uint64(st.Dev), Ino: uint64(st.Ino), } + c.mutex.Lock() - _, found := c.inodes[i] + + oldTarget, found := c.inodes[i] if !found { - c.inodes[i] = true + c.inodes[i] = path + } + + // If we are dealing with a file with multiple links then keep the lock until the file is + // chowned to avoid a race where we link to the old version if the file is copied up. + if found || st.Nlink > 1 { + defer c.mutex.Unlock() + } else { + c.mutex.Unlock() } - c.mutex.Unlock() if found { - return nil + // If the dev/inode was already chowned then create a link to the old target instead + // of chowning it again. This is necessary when the underlying file system breaks + // inodes on copy-up (as it is with overlay with index=off) to maintain the original + // link and correct file ownership. + + // The target already exists so remove it before creating the link to the new target. + if err := os.Remove(path); err != nil { + return err + } + return os.Link(oldTarget, path) } // Map an on-disk UID/GID pair from host to container diff --git a/vendor/github.com/containers/storage/drivers/driver_darwin.go b/vendor/github.com/containers/storage/drivers/driver_darwin.go new file mode 100644 index 000000000..357851543 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/driver_darwin.go @@ -0,0 +1,14 @@ +package graphdriver + +var ( + // Slice of drivers that should be used in order + priority = []string{ + "vfs", + } +) + +// GetFSMagic returns the filesystem id given the path. +func GetFSMagic(rootpath string) (FsMagic, error) { + // Note it is OK to return FsMagicUnsupported on Windows. + return FsMagicUnsupported, nil +} diff --git a/vendor/github.com/containers/storage/drivers/driver_linux.go b/vendor/github.com/containers/storage/drivers/driver_linux.go index 0fe3eea7a..7c527d279 100644 --- a/vendor/github.com/containers/storage/drivers/driver_linux.go +++ b/vendor/github.com/containers/storage/drivers/driver_linux.go @@ -1,3 +1,4 @@ +//go:build linux // +build linux package graphdriver @@ -162,11 +163,32 @@ func (c *defaultChecker) IsMounted(path string) bool { return m } +// isMountPoint checks that the given path is a mount point +func isMountPoint(mountPath string) (bool, error) { + // it is already the root + if mountPath == "/" { + return true, nil + } + + var s1, s2 unix.Stat_t + if err := unix.Stat(mountPath, &s1); err != nil { + return true, err + } + if err := unix.Stat(filepath.Dir(mountPath), &s2); err != nil { + return true, err + } + return s1.Dev != s2.Dev, nil +} + // Mounted checks if the given path is mounted as the fs type func Mounted(fsType FsMagic, mountPath string) (bool, error) { var buf unix.Statfs_t + if err := unix.Statfs(mountPath, &buf); err != nil { return false, err } - return FsMagic(buf.Type) == fsType, nil + if FsMagic(buf.Type) != fsType { + return false, nil + } + return isMountPoint(mountPath) } diff --git a/vendor/github.com/containers/storage/drivers/driver_unsupported.go b/vendor/github.com/containers/storage/drivers/driver_unsupported.go index 4a875608b..3932c3ea5 100644 --- a/vendor/github.com/containers/storage/drivers/driver_unsupported.go +++ b/vendor/github.com/containers/storage/drivers/driver_unsupported.go @@ -1,4 +1,4 @@ -// +build !linux,!windows,!freebsd,!solaris +// +build !linux,!windows,!freebsd,!solaris,!darwin package graphdriver diff --git a/vendor/github.com/containers/storage/drivers/fsdiff.go b/vendor/github.com/containers/storage/drivers/fsdiff.go index b7e681ace..b619317e0 100644 --- a/vendor/github.com/containers/storage/drivers/fsdiff.go +++ b/vendor/github.com/containers/storage/drivers/fsdiff.go @@ -2,6 +2,8 @@ package graphdriver import ( "io" + "os" + "runtime" "time" "github.com/containers/storage/pkg/archive" @@ -170,9 +172,16 @@ func (gdw *NaiveDiffDriver) ApplyDiff(id, parent string, options ApplyDiffOpts) } defer driver.Put(id) + defaultForceMask := os.FileMode(0700) + var forceMask *os.FileMode = nil + if runtime.GOOS == "darwin" { + forceMask = &defaultForceMask + } + tarOptions := &archive.TarOptions{ InUserNS: userns.RunningInUserNS(), IgnoreChownErrors: options.IgnoreChownErrors, + ForceMask: forceMask, } if options.Mappings != nil { tarOptions.UIDMaps = options.Mappings.UIDs() diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index 1b58e2f63..b1073d55f 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -5,6 +5,7 @@ import ( "io" "os" "path/filepath" + "runtime" "strconv" "strings" @@ -170,6 +171,10 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, ro bool }() rootPerms := defaultPerms + if runtime.GOOS == "darwin" { + rootPerms = os.FileMode(0700) + } + if parent != "" { st, err := system.Stat(d.dir(parent)) if err != nil { diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index ea9de11db..7d8151b57 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -5,30 +5,30 @@ module github.com/containers/storage require ( github.com/BurntSushi/toml v1.1.0 github.com/Microsoft/go-winio v0.5.2 - github.com/Microsoft/hcsshim v0.9.2 + github.com/Microsoft/hcsshim v0.9.3 github.com/containerd/stargz-snapshotter/estargz v0.11.4 github.com/cyphar/filepath-securejoin v0.2.3 github.com/docker/go-units v0.4.0 github.com/google/go-intervals v0.0.2 github.com/hashicorp/go-multierror v1.1.1 github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.15.4 + github.com/klauspost/compress v1.15.6 github.com/klauspost/pgzip v1.2.5 github.com/mattn/go-shellwords v1.0.12 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible - github.com/moby/sys/mountinfo v0.6.1 + github.com/moby/sys/mountinfo v0.6.2 github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/runc v1.1.2 + github.com/opencontainers/runc v1.1.1-0.20220607072441-a7a45d7d2721 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 github.com/opencontainers/selinux v1.10.1 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 - github.com/stretchr/testify v1.7.1 + github.com/stretchr/testify v1.7.2 github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 github.com/tchap/go-patricia v2.3.0+incompatible github.com/ulikunitz/xz v0.5.10 github.com/vbatts/tar-split v0.11.2 golang.org/x/net v0.0.0-20210825183410-e898025ed96a - golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e + golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a gotest.tools v2.2.0+incompatible ) diff --git a/vendor/github.com/containers/storage/go.sum b/vendor/github.com/containers/storage/go.sum index 4776904ff..6fbca4e4f 100644 --- a/vendor/github.com/containers/storage/go.sum +++ b/vendor/github.com/containers/storage/go.sum @@ -57,8 +57,8 @@ github.com/Microsoft/hcsshim v0.8.14/go.mod h1:NtVKoYxQuTLx6gEq0L96c9Ju4JbRJ4nY2 github.com/Microsoft/hcsshim v0.8.15/go.mod h1:x38A4YbHbdxJtc0sF6oIz+RG0npwSCAvn69iY6URG00= github.com/Microsoft/hcsshim v0.8.16/go.mod h1:o5/SZqmR7x9JNKsW3pu+nqHm0MF8vbA+VxGOoXdC600= github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwTOcER2fw4I4= -github.com/Microsoft/hcsshim v0.9.2 h1:wB06W5aYFfUB3IvootYAY2WnOmIdgPGfqSI6tufQNnY= -github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= +github.com/Microsoft/hcsshim v0.9.3 h1:k371PzBuRrz2b+ebGuI2nVgVhgsVX60jMfSw80NECxo= +github.com/Microsoft/hcsshim v0.9.3/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -107,7 +107,7 @@ github.com/cilium/ebpf v0.0.0-20200702112145-1c8d4c9ef775/go.mod h1:7cR51M8ViRLI github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/cilium/ebpf v0.6.2/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= -github.com/cilium/ebpf v0.7.0/go.mod h1:/oI2+1shJiTGAMgl6/RgJr36Eo1jzrRcAWbcXO2usCA= +github.com/cilium/ebpf v0.9.0/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= @@ -266,6 +266,7 @@ github.com/evanphx/json-patch v4.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLi github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= @@ -299,7 +300,7 @@ github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblf github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= -github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -425,8 +426,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.4 h1:1kn4/7MepF/CHmYub99/nNX8az0IJjfSOU/jbnTVfqQ= -github.com/klauspost/compress v1.15.4/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.6 h1:6D9PcO8QWu0JyaQ2zUMmu16T1T+zjjEpP91guRsvDfY= +github.com/klauspost/compress v1.15.6/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -436,6 +437,7 @@ github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFB github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.5/go.mod h1:9r2w37qlBe7rQ6e1fg1S/9xpWHSnaqNdHD3WcMdbPDA= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -467,9 +469,8 @@ github.com/mitchellh/osext v0.0.0-20151018003038-5e2d6d41470f/go.mod h1:OkQIRizQ github.com/moby/locker v1.0.1/go.mod h1:S7SDdo5zpBK84bzzVlKr2V0hz+7x9hWbYC/kq7oQppc= github.com/moby/sys/mountinfo v0.4.0/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= -github.com/moby/sys/mountinfo v0.5.0/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= -github.com/moby/sys/mountinfo v0.6.1 h1:+H/KnGEAGRpTrEAqNVQ2AM3SiwMgJUt/TXj+Z8cmCIc= -github.com/moby/sys/mountinfo v0.6.1/go.mod h1:3bMD3Rg+zkqx8MRYPi7Pyb0Ie97QEBmdxbhnCLlSvSU= +github.com/moby/sys/mountinfo v0.6.2 h1:BzJjoreD5BMFNmD9Rus6gdd1pLuecOFPt8wC+Vygl78= +github.com/moby/sys/mountinfo v0.6.2/go.mod h1:IJb6JQeOklcdMU9F5xQ8ZALD+CUr5VlGpwtX+VE0rpI= github.com/moby/sys/symlink v0.1.0/go.mod h1:GGDODQmbFOjFsXvfLVn3+ZRxkch54RkSiGqsZeMYowQ= github.com/moby/term v0.0.0-20200312100748-672ec06f55cd/go.mod h1:DdlQx2hp0Ss5/fLikoLlEeIYiATotOjgB//nb973jeo= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -521,8 +522,8 @@ github.com/opencontainers/runc v1.0.0-rc8.0.20190926000215-3e425f80a8c9/go.mod h github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= -github.com/opencontainers/runc v1.1.2 h1:2VSZwLx5k/BfsBxMMipG/LYUnmqOD/BPkIVgQUcTlLw= -github.com/opencontainers/runc v1.1.2/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= +github.com/opencontainers/runc v1.1.1-0.20220607072441-a7a45d7d2721 h1:geG4wjkUPHyg+Ya/BBb8YlX1z4INWpVMdoUnmBxttqc= +github.com/opencontainers/runc v1.1.1-0.20220607072441-a7a45d7d2721/go.mod h1:QvA0UNe48mC1JxcXq0sENIR38+/LdJMLNxuAvtFBhxA= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -534,7 +535,6 @@ github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mo github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= -github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/opencontainers/selinux v1.10.1 h1:09LIPVRP3uuZGQvgR+SgMSNBd1Eb3vlRbGqQpoHsF8w= github.com/opencontainers/selinux v1.10.1/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -580,12 +580,13 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/safchain/ethtool v0.0.0-20190326074333-42ed695e3de8/go.mod h1:Z0q5wiBQGYcxhMZ6gUqHn6pYNLypFAvaL3UvgZLR0U4= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sclevine/spec v1.2.0/go.mod h1:W4J29eT/Kzv7/b9IWLB055Z+qvVC9vt0Arko24q7p+U= github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= -github.com/seccomp/libseccomp-golang v0.9.2-0.20210429002308-3879420cc921/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= +github.com/seccomp/libseccomp-golang v0.9.2-0.20220502022130-f33da4d89646/go.mod h1:JA8cRccbGaA1s33RQf7Y1+q9gHmZX1yB/z9WDN1C6fg= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.0.4-0.20170822132746-89742aefa4b2/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= github.com/sirupsen/logrus v1.0.6/go.mod h1:pMByvHTf9Beacp5x1UXfOR9xyW/9antXMhjMPG0dEzc= @@ -624,8 +625,8 @@ github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UV github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.7.1 h1:5TQK59W5E3v0r2duFAb7P95B6hEeOyEnHRa8MjYSMTY= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.7.2 h1:4jaiDzPyXQvSd7D0EjG45355tLlV3VOECpq10pLC+8s= +github.com/stretchr/testify v1.7.2/go.mod h1:R6va5+xMeoiuVRoj+gSkQ7d3FALtqAAGI1FQKckRals= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 h1:kdXcSzyDtseVEc4yCz2qF8ZrQvIDBJLl4S1c3GCXmoI= @@ -643,6 +644,7 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.4/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= +github.com/urfave/cli v1.22.9/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= @@ -848,10 +850,9 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210906170528-6f6e22806c34/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211116061358-0a5406a5449c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e h1:fLOSk5Q00efkSvAm+4xcoXD+RRmLmmulPn5I3Y9F2EM= golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a h1:dGzPydgVsqGcTRVwiLJ1jVbufYwmzD3LfVPLKsKg+0k= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -990,6 +991,7 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/airbrake/gobrake.v2 v2.0.9/go.mod h1:/h5ZAUhDkGaJfjzjKLSjv6zCL6O0LLBxU4K+aSYdM/U= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1018,8 +1020,9 @@ gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= diff --git a/vendor/github.com/containers/storage/pkg/archive/archive.go b/vendor/github.com/containers/storage/pkg/archive/archive.go index d4f129ee6..570000e82 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive.go @@ -12,6 +12,7 @@ import ( "os" "path/filepath" "runtime" + "strconv" "strings" "sync" "syscall" @@ -72,10 +73,10 @@ type ( ) const ( - tarExt = "tar" - solaris = "solaris" - windows = "windows" - containersOverrideXattr = "user.containers.override_stat" + tarExt = "tar" + solaris = "solaris" + windows = "windows" + darwin = "darwin" ) var xattrsToIgnore = map[string]interface{}{ @@ -698,9 +699,9 @@ func createTarFile(path, extractDir string, hdr *tar.Header, reader io.Reader, L return fmt.Errorf("unhandled tar header type %d", hdr.Typeflag) } - if forceMask != nil && hdr.Typeflag != tar.TypeSymlink { + if forceMask != nil && (hdr.Typeflag != tar.TypeSymlink || runtime.GOOS == "darwin") { value := fmt.Sprintf("%d:%d:0%o", hdr.Uid, hdr.Gid, hdrInfo.Mode()&07777) - if err := system.Lsetxattr(path, containersOverrideXattr, []byte(value), 0); err != nil { + if err := system.Lsetxattr(path, idtools.ContainersOverrideXattr, []byte(value), 0); err != nil { return err } } @@ -981,7 +982,7 @@ func Unpack(decompressedArchive io.Reader, dest string, options *TarOptions) err uid, gid, mode, err := GetFileOwner(dest) if err == nil { value := fmt.Sprintf("%d:%d:0%o", uid, gid, mode) - if err := system.Lsetxattr(dest, containersOverrideXattr, []byte(value), 0); err != nil { + if err := system.Lsetxattr(dest, idtools.ContainersOverrideXattr, []byte(value), 0); err != nil { return err } } @@ -1313,6 +1314,21 @@ func remapIDs(readIDMappings, writeIDMappings *idtools.IDMappings, chownOpts *id if err != nil { return err } + } else if runtime.GOOS == darwin { + uid, gid = hdr.Uid, hdr.Gid + if xstat, ok := hdr.Xattrs[idtools.ContainersOverrideXattr]; ok { + attrs := strings.Split(string(xstat), ":") + if len(attrs) == 3 { + val, err := strconv.ParseUint(attrs[0], 10, 32) + if err != nil { + uid = int(val) + } + val, err = strconv.ParseUint(attrs[1], 10, 32) + if err != nil { + gid = int(val) + } + } + } } else { uid, gid = hdr.Uid, hdr.Gid } diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_darwin.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_darwin.go new file mode 100644 index 000000000..d257cc8e9 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_darwin.go @@ -0,0 +1,21 @@ +package chrootarchive + +import ( + "io" + + "github.com/containers/storage/pkg/archive" +) + +func chroot(path string) error { + return nil +} + +func invokeUnpack(decompressedArchive io.ReadCloser, + dest string, + options *archive.TarOptions, root string) error { + return archive.Unpack(decompressedArchive, dest, options) +} + +func invokePack(srcPath string, options *archive.TarOptions, root string) (io.ReadCloser, error) { + return archive.TarWithOptions(srcPath, options) +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go index 9da10fe33..e4b45a454 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive_unix.go @@ -1,4 +1,4 @@ -// +build !windows +// +build !windows,!darwin package chrootarchive diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go index 83278ee50..d5aedd002 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_unix.go @@ -1,4 +1,4 @@ -// +build !windows,!linux +// +build !windows,!linux,!darwin package chrootarchive diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/diff_darwin.go b/vendor/github.com/containers/storage/pkg/chrootarchive/diff_darwin.go new file mode 100644 index 000000000..d6326c808 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/diff_darwin.go @@ -0,0 +1,41 @@ +package chrootarchive + +import ( + "fmt" + "io" + "io/ioutil" + "os" + "path/filepath" + + "github.com/containers/storage/pkg/archive" +) + +// applyLayerHandler parses a diff in the standard layer format from `layer`, and +// applies it to the directory `dest`. Returns the size in bytes of the +// contents of the layer. +func applyLayerHandler(dest string, layer io.Reader, options *archive.TarOptions, decompress bool) (size int64, err error) { + dest = filepath.Clean(dest) + + if decompress { + decompressed, err := archive.DecompressStream(layer) + if err != nil { + return 0, err + } + defer decompressed.Close() + + layer = decompressed + } + + tmpDir, err := ioutil.TempDir(os.Getenv("temp"), "temp-storage-extract") + if err != nil { + return 0, fmt.Errorf("ApplyLayer failed to create temp-storage-extract under %s. %s", dest, err) + } + + s, err := archive.UnpackLayer(dest, layer, options) + os.RemoveAll(tmpDir) + if err != nil { + return 0, fmt.Errorf("ApplyLayer %s failed UnpackLayer to %s: %s", layer, dest, err) + } + + return s, nil +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/diff_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/diff_unix.go index 84253c6aa..6dd5146cc 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/diff_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/diff_unix.go @@ -1,4 +1,4 @@ -//+build !windows +//+build !windows,!darwin package chrootarchive diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/init_darwin.go b/vendor/github.com/containers/storage/pkg/chrootarchive/init_darwin.go new file mode 100644 index 000000000..fa17c9bf8 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/init_darwin.go @@ -0,0 +1,4 @@ +package chrootarchive + +func init() { +} diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go b/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go index ea08135e4..45caec972 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/init_unix.go @@ -1,4 +1,4 @@ -// +build !windows +// +build !windows,!darwin package chrootarchive diff --git a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go index 9434499d2..7b6cd8fe4 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go @@ -272,14 +272,6 @@ func canDedupFileWithHardLink(file *internal.FileMetadata, fd int, s os.FileInfo return canDedupMetadataWithHardLink(file, &otherFile) } -func getFileDigest(f *os.File, buf []byte) (digest.Digest, error) { - digester := digest.Canonical.Digester() - if _, err := io.CopyBuffer(digester.Hash(), f, buf); err != nil { - return "", err - } - return digester.Digest(), nil -} - // findFileInOSTreeRepos checks whether the requested file already exist in one of the OSTree repo and copies the file content from there if possible. // file is the file to look for. // ostreeRepos is a list of OSTree repos. @@ -330,75 +322,6 @@ func findFileInOSTreeRepos(file *internal.FileMetadata, ostreeRepos []string, di return false, nil, 0, nil } -// findFileOnTheHost checks whether the requested file already exist on the host and copies the file content from there if possible. -// It is currently implemented to look only at the file with the same path. Ideally it can detect the same content also at different -// paths. -// file is the file to look for. -// dirfd is an open fd to the destination checkout. -// useHardLinks defines whether the deduplication can be performed using hard links. -func findFileOnTheHost(file *internal.FileMetadata, dirfd int, useHardLinks bool, buf []byte) (bool, *os.File, int64, error) { - sourceFile := filepath.Clean(filepath.Join("/", file.Name)) - if !strings.HasPrefix(sourceFile, "/usr/") { - // limit host deduplication to files under /usr. - return false, nil, 0, nil - } - - st, err := os.Stat(sourceFile) - if err != nil || !st.Mode().IsRegular() { - return false, nil, 0, nil - } - - if st.Size() != file.Size { - return false, nil, 0, nil - } - - fd, err := unix.Open(sourceFile, unix.O_RDONLY|unix.O_NONBLOCK, 0) - if err != nil { - return false, nil, 0, nil - } - - f := os.NewFile(uintptr(fd), "fd") - defer f.Close() - - manifestChecksum, err := digest.Parse(file.Digest) - if err != nil { - return false, nil, 0, err - } - - checksum, err := getFileDigest(f, buf) - if err != nil { - return false, nil, 0, err - } - - if checksum != manifestChecksum { - return false, nil, 0, nil - } - - // check if the open file can be deduplicated with hard links - useHardLinks = useHardLinks && canDedupFileWithHardLink(file, fd, st) - - dstFile, written, err := copyFileContent(fd, file.Name, dirfd, 0, useHardLinks) - if err != nil { - return false, nil, 0, nil - } - - // calculate the checksum again to make sure the file wasn't modified while it was copied - if _, err := f.Seek(0, 0); err != nil { - dstFile.Close() - return false, nil, 0, err - } - checksum, err = getFileDigest(f, buf) - if err != nil { - dstFile.Close() - return false, nil, 0, err - } - if checksum != manifestChecksum { - dstFile.Close() - return false, nil, 0, nil - } - return true, dstFile, written, nil -} - // findFileInOtherLayers finds the specified file in other layers. // cache is the layers cache to use. // file is the file to look for. @@ -1297,10 +1220,9 @@ func parseBooleanPullOption(storeOpts *storage.StoreOptions, name string, def bo } type findAndCopyFileOptions struct { - useHardLinks bool - enableHostDedup bool - ostreeRepos []string - options *archive.TarOptions + useHardLinks bool + ostreeRepos []string + options *archive.TarOptions } func (c *chunkedDiffer) findAndCopyFile(dirfd int, r *internal.FileMetadata, copyOptions *findAndCopyFileOptions, mode os.FileMode) (bool, error) { @@ -1336,18 +1258,6 @@ func (c *chunkedDiffer) findAndCopyFile(dirfd int, r *internal.FileMetadata, cop return true, nil } - if copyOptions.enableHostDedup { - found, dstFile, _, err = findFileOnTheHost(r, dirfd, copyOptions.useHardLinks, c.copyBuffer) - if err != nil { - return false, err - } - if found { - if err := finalizeFile(dstFile); err != nil { - return false, err - } - return true, nil - } - } return false, nil } @@ -1376,8 +1286,6 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions) (gra return output, errors.New("enable_partial_images not configured") } - enableHostDedup := parseBooleanPullOption(&storeOpts, "enable_host_deduplication", false) - // When the hard links deduplication is used, file attributes are ignored because setting them // modifies the source file as well. useHardLinks := parseBooleanPullOption(&storeOpts, "use_hard_links", false) @@ -1426,10 +1334,9 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions) (gra missingPartsSize, totalChunksSize := int64(0), int64(0) copyOptions := findAndCopyFileOptions{ - useHardLinks: useHardLinks, - enableHostDedup: enableHostDedup, - ostreeRepos: ostreeRepos, - options: options, + useHardLinks: useHardLinks, + ostreeRepos: ostreeRepos, + options: options, } type copyFileJob struct { diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools.go b/vendor/github.com/containers/storage/pkg/idtools/idtools.go index 7a8fec0ce..3ae2a1cd7 100644 --- a/vendor/github.com/containers/storage/pkg/idtools/idtools.go +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools.go @@ -6,6 +6,7 @@ import ( "io/ioutil" "os" "os/user" + "runtime" "sort" "strconv" "strings" @@ -38,8 +39,9 @@ func (e ranges) Swap(i, j int) { e[i], e[j] = e[j], e[i] } func (e ranges) Less(i, j int) bool { return e[i].Start < e[j].Start } const ( - subuidFileName string = "/etc/subuid" - subgidFileName string = "/etc/subgid" + subuidFileName string = "/etc/subuid" + subgidFileName string = "/etc/subgid" + ContainersOverrideXattr = "user.containers.override_stat" ) // MkdirAllAs creates a directory (include any along the path) and then modifies @@ -366,6 +368,25 @@ func checkChownErr(err error, name string, uid, gid int) error { } func SafeChown(name string, uid, gid int) error { + if runtime.GOOS == "darwin" { + var mode uint64 = 0o0700 + xstat, err := system.Lgetxattr(name, ContainersOverrideXattr) + if err == nil { + attrs := strings.Split(string(xstat), ":") + if len(attrs) == 3 { + val, err := strconv.ParseUint(attrs[2], 8, 32) + if err == nil { + mode = val + } + } + } + value := fmt.Sprintf("%d:%d:0%o", uid, gid, mode) + if err = system.Lsetxattr(name, ContainersOverrideXattr, []byte(value), 0); err != nil { + return err + } + uid = os.Getuid() + gid = os.Getgid() + } if stat, statErr := system.Stat(name); statErr == nil { if stat.UID() == uint32(uid) && stat.GID() == uint32(gid) { return nil @@ -375,6 +396,25 @@ func SafeChown(name string, uid, gid int) error { } func SafeLchown(name string, uid, gid int) error { + if runtime.GOOS == "darwin" { + var mode uint64 = 0o0700 + xstat, err := system.Lgetxattr(name, ContainersOverrideXattr) + if err == nil { + attrs := strings.Split(string(xstat), ":") + if len(attrs) == 3 { + val, err := strconv.ParseUint(attrs[2], 8, 32) + if err == nil { + mode = val + } + } + } + value := fmt.Sprintf("%d:%d:0%o", uid, gid, mode) + if err = system.Lsetxattr(name, ContainersOverrideXattr, []byte(value), 0); err != nil { + return err + } + uid = os.Getuid() + gid = os.Getgid() + } if stat, statErr := system.Lstat(name); statErr == nil { if stat.UID() == uint32(uid) && stat.GID() == uint32(gid) { return nil diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools_supported.go b/vendor/github.com/containers/storage/pkg/idtools/idtools_supported.go index 6e6e3b22b..c96465369 100644 --- a/vendor/github.com/containers/storage/pkg/idtools/idtools_supported.go +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools_supported.go @@ -1,8 +1,10 @@ +//go:build linux && cgo && libsubid // +build linux,cgo,libsubid package idtools import ( + "os/user" "unsafe" "github.com/pkg/errors" @@ -32,19 +34,34 @@ import "C" func readSubid(username string, isUser bool) (ranges, error) { var ret ranges + uidstr := "" + if username == "ALL" { return nil, errors.New("username ALL not supported") } + if u, err := user.Lookup(username); err == nil { + uidstr = u.Uid + } + cUsername := C.CString(username) defer C.free(unsafe.Pointer(cUsername)) + cuidstr := C.CString(uidstr) + defer C.free(unsafe.Pointer(cuidstr)) + var nRanges C.int var cRanges *C.struct_subid_range if isUser { nRanges = C.subid_get_uid_ranges(cUsername, &cRanges) + if nRanges <= 0 { + nRanges = C.subid_get_uid_ranges(cuidstr, &cRanges) + } } else { nRanges = C.subid_get_gid_ranges(cUsername, &cRanges) + if nRanges <= 0 { + nRanges = C.subid_get_gid_ranges(cuidstr, &cRanges) + } } if nRanges < 0 { return nil, errors.New("cannot read subids") diff --git a/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go b/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go index fc080acbe..aa843920f 100644 --- a/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go +++ b/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go @@ -1,15 +1,19 @@ +//go:build linux || solaris || darwin || freebsd // +build linux solaris darwin freebsd package lockfile import ( + "bytes" + cryptorand "crypto/rand" + "encoding/binary" "fmt" "os" "path/filepath" "sync" + "sync/atomic" "time" - "github.com/containers/storage/pkg/stringid" "github.com/containers/storage/pkg/system" "github.com/pkg/errors" "golang.org/x/sys/unix" @@ -23,13 +27,44 @@ type lockfile struct { counter int64 file string fd uintptr - lw string + lw []byte // "last writer"-unique value valid as of the last .Touch() or .Modified(), generated by newLastWriterID() locktype int16 locked bool ro bool recursive bool } +const lastWriterIDSize = 64 // This must be the same as len(stringid.GenerateRandomID) +var lastWriterIDCounter uint64 // Private state for newLastWriterID + +// newLastWriterID returns a new "last writer" ID. +// The value must be different on every call, and also differ from values +// generated by other processes. +func newLastWriterID() []byte { + // The ID is (PID, time, per-process counter, random) + // PID + time represents both a unique process across reboots, + // and a specific time within the process; the per-process counter + // is an extra safeguard for in-process concurrency. + // The random part disambiguates across process namespaces + // (where PID values might collide), serves as a general-purpose + // extra safety, _and_ is used to pad the output to lastWriterIDSize, + // because other versions of this code exist and they don't work + // efficiently if the size of the value changes. + pid := os.Getpid() + tm := time.Now().UnixNano() + counter := atomic.AddUint64(&lastWriterIDCounter, 1) + + res := make([]byte, lastWriterIDSize) + binary.LittleEndian.PutUint64(res[0:8], uint64(tm)) + binary.LittleEndian.PutUint64(res[8:16], counter) + binary.LittleEndian.PutUint32(res[16:20], uint32(pid)) + if n, err := cryptorand.Read(res[20:lastWriterIDSize]); err != nil || n != lastWriterIDSize-20 { + panic(err) // This shouldn't happen + } + + return res +} + // openLock opens the file at path and returns the corresponding file // descriptor. Note that the path is opened read-only when ro is set. If ro // is unset, openLock will open the path read-write and create the file if @@ -89,7 +124,7 @@ func createLockerForPath(path string, ro bool) (Locker, error) { stateMutex: &sync.Mutex{}, rwMutex: &sync.RWMutex{}, file: path, - lw: stringid.GenerateRandomID(), + lw: newLastWriterID(), locktype: int16(locktype), locked: false, ro: ro}, nil @@ -212,13 +247,12 @@ func (l *lockfile) Touch() error { panic("attempted to update last-writer in lockfile without the write lock") } defer l.stateMutex.Unlock() - l.lw = stringid.GenerateRandomID() - id := []byte(l.lw) - n, err := unix.Pwrite(int(l.fd), id, 0) + l.lw = newLastWriterID() + n, err := unix.Pwrite(int(l.fd), l.lw, 0) if err != nil { return err } - if n != len(id) { + if n != len(l.lw) { return unix.ENOSPC } return nil @@ -228,21 +262,21 @@ func (l *lockfile) Touch() error { // was loaded. func (l *lockfile) Modified() (bool, error) { l.stateMutex.Lock() - id := []byte(l.lw) if !l.locked { panic("attempted to check last-writer in lockfile without locking it first") } defer l.stateMutex.Unlock() - n, err := unix.Pread(int(l.fd), id, 0) + currentLW := make([]byte, len(l.lw)) + n, err := unix.Pread(int(l.fd), currentLW, 0) if err != nil { return true, err } - if n != len(id) { + if n != len(l.lw) { return true, nil } - lw := l.lw - l.lw = string(id) - return l.lw != lw, nil + oldLW := l.lw + l.lw = currentLW + return !bytes.Equal(currentLW, oldLW), nil } // IsReadWriteLock indicates if the lock file is a read-write lock. diff --git a/vendor/github.com/containers/storage/pkg/stringid/stringid.go b/vendor/github.com/containers/storage/pkg/stringid/stringid.go index a0c7c42a0..4c434f0e5 100644 --- a/vendor/github.com/containers/storage/pkg/stringid/stringid.go +++ b/vendor/github.com/containers/storage/pkg/stringid/stringid.go @@ -12,6 +12,7 @@ import ( "regexp" "strconv" "strings" + "sync" "time" ) @@ -20,6 +21,9 @@ const shortLen = 12 var ( validShortID = regexp.MustCompile("^[a-f0-9]{12}$") validHex = regexp.MustCompile(`^[a-f0-9]{64}$`) + + rngLock sync.Mutex + rng *rand.Rand // A RNG with seeding properties we control. It can only be accessed with randLock held. ) // IsShortID determines if an arbitrary string *looks like* a short ID. @@ -67,7 +71,9 @@ func GenerateRandomID() string { // secure sources of random. // It helps you to save entropy. func GenerateNonCryptoID() string { - return generateID(readerFunc(rand.Read)) + rngLock.Lock() + defer rngLock.Unlock() + return generateID(readerFunc(rng.Read)) } // ValidateID checks whether an ID string is a valid image ID. @@ -79,7 +85,7 @@ func ValidateID(id string) error { } func init() { - // safely set the seed globally so we generate random ids. Tries to use a + // Initialize a private RNG so we generate random ids. Tries to use a // crypto seed before falling back to time. var seed int64 if cryptoseed, err := cryptorand.Int(cryptorand.Reader, big.NewInt(math.MaxInt64)); err != nil { @@ -89,7 +95,7 @@ func init() { seed = cryptoseed.Int64() } - rand.Seed(seed) + rng = rand.New(rand.NewSource(seed)) } type readerFunc func(p []byte) (int, error) diff --git a/vendor/github.com/containers/storage/pkg/system/meminfo_unsupported.go b/vendor/github.com/containers/storage/pkg/system/meminfo_unsupported.go index 8d14fe9f8..0f9feb1d2 100644 --- a/vendor/github.com/containers/storage/pkg/system/meminfo_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/system/meminfo_unsupported.go @@ -1,5 +1,8 @@ -//go:build !linux && !windows && !solaris && !freebsd -// +build !linux,!windows,!solaris,!freebsd +//go:build !linux && !windows && !solaris && !(freebsd && cgo) +// +build !linux +// +build !windows +// +build !solaris +// +build !freebsd !cgo package system diff --git a/vendor/github.com/containers/storage/pkg/system/xattrs_darwin.go b/vendor/github.com/containers/storage/pkg/system/xattrs_darwin.go new file mode 100644 index 000000000..75275b964 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/system/xattrs_darwin.go @@ -0,0 +1,84 @@ +package system + +import ( + "bytes" + "os" + + "golang.org/x/sys/unix" +) + +const ( + // Value is larger than the maximum size allowed + E2BIG unix.Errno = unix.E2BIG + + // Operation not supported + EOPNOTSUPP unix.Errno = unix.EOPNOTSUPP +) + +// Lgetxattr retrieves the value of the extended attribute identified by attr +// and associated with the given path in the file system. +// Returns a []byte slice if the xattr is set and nil otherwise. +func Lgetxattr(path string, attr string) ([]byte, error) { + // Start with a 128 length byte array + dest := make([]byte, 128) + sz, errno := unix.Lgetxattr(path, attr, dest) + + for errno == unix.ERANGE { + // Buffer too small, use zero-sized buffer to get the actual size + sz, errno = unix.Lgetxattr(path, attr, []byte{}) + if errno != nil { + return nil, &os.PathError{Op: "lgetxattr", Path: path, Err: errno} + } + dest = make([]byte, sz) + sz, errno = unix.Lgetxattr(path, attr, dest) + } + + switch { + case errno == unix.ENOATTR: + return nil, nil + case errno != nil: + return nil, &os.PathError{Op: "lgetxattr", Path: path, Err: errno} + } + + return dest[:sz], nil +} + +// Lsetxattr sets the value of the extended attribute identified by attr +// and associated with the given path in the file system. +func Lsetxattr(path string, attr string, data []byte, flags int) error { + if err := unix.Lsetxattr(path, attr, data, flags); err != nil { + return &os.PathError{Op: "lsetxattr", Path: path, Err: err} + } + + return nil +} + +// Llistxattr lists extended attributes associated with the given path +// in the file system. +func Llistxattr(path string) ([]string, error) { + dest := make([]byte, 128) + sz, errno := unix.Llistxattr(path, dest) + + for errno == unix.ERANGE { + // Buffer too small, use zero-sized buffer to get the actual size + sz, errno = unix.Llistxattr(path, []byte{}) + if errno != nil { + return nil, &os.PathError{Op: "llistxattr", Path: path, Err: errno} + } + + dest = make([]byte, sz) + sz, errno = unix.Llistxattr(path, dest) + } + if errno != nil { + return nil, &os.PathError{Op: "llistxattr", Path: path, Err: errno} + } + + var attrs []string + for _, token := range bytes.Split(dest[:sz], []byte{0}) { + if len(token) > 0 { + attrs = append(attrs, string(token)) + } + } + + return attrs, nil +} diff --git a/vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go b/vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go index 3fc27f0b1..221eb78bc 100644 --- a/vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/system/xattrs_unsupported.go @@ -1,4 +1,4 @@ -// +build !linux +// +build !linux,!darwin package system diff --git a/vendor/github.com/containers/storage/pkg/unshare/unshare_darwin.go b/vendor/github.com/containers/storage/pkg/unshare/unshare_darwin.go new file mode 100644 index 000000000..01cf33bde --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/unshare/unshare_darwin.go @@ -0,0 +1,53 @@ +// +build darwin + +package unshare + +import ( + "os" + + "github.com/containers/storage/pkg/idtools" + "github.com/opencontainers/runtime-spec/specs-go" +) + +const ( + // UsernsEnvName is the environment variable, if set indicates in rootless mode + UsernsEnvName = "_CONTAINERS_USERNS_CONFIGURED" +) + +// IsRootless tells us if we are running in rootless mode +func IsRootless() bool { + return true +} + +// GetRootlessUID returns the UID of the user in the parent userNS +func GetRootlessUID() int { + return os.Getuid() +} + +// RootlessEnv returns the environment settings for the rootless containers +func RootlessEnv() []string { + return append(os.Environ(), UsernsEnvName+"=") +} + +// MaybeReexecUsingUserNamespace re-exec the process in a new namespace +func MaybeReexecUsingUserNamespace(evenForRoot bool) { +} + +// GetHostIDMappings reads mappings for the specified process (or the current +// process if pid is "self" or an empty string) from the kernel. +func GetHostIDMappings(pid string) ([]specs.LinuxIDMapping, []specs.LinuxIDMapping, error) { + return nil, nil, nil +} + +// ParseIDMappings parses mapping triples. +func ParseIDMappings(uidmap, gidmap []string) ([]idtools.IDMap, []idtools.IDMap, error) { + uid, err := idtools.ParseIDMap(uidmap, "userns-uid-map") + if err != nil { + return nil, nil, err + } + gid, err := idtools.ParseIDMap(gidmap, "userns-gid-map") + if err != nil { + return nil, nil, err + } + return uid, gid, nil +} diff --git a/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go b/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go index 8ee3ee125..16d14d2a9 100644 --- a/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go +++ b/vendor/github.com/containers/storage/pkg/unshare/unshare_linux.go @@ -78,7 +78,7 @@ func getRootlessGID() int { } // IsSetID checks if specified path has correct FileMode (Setuid|SETGID) or the -// matching file capabilitiy +// matching file capability func IsSetID(path string, modeid os.FileMode, capid capability.Cap) (bool, error) { info, err := os.Stat(path) if err != nil { diff --git a/vendor/github.com/containers/storage/pkg/unshare/unshare_unsupported.go b/vendor/github.com/containers/storage/pkg/unshare/unshare_unsupported.go index 166fa050b..66dd54596 100644 --- a/vendor/github.com/containers/storage/pkg/unshare/unshare_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/unshare/unshare_unsupported.go @@ -1,5 +1,5 @@ -//go:build !linux -// +build !linux +//go:build !linux && !darwin +// +build !linux,!darwin package unshare diff --git a/vendor/github.com/containers/storage/storage.conf b/vendor/github.com/containers/storage/storage.conf index c17dd6d37..e075bce13 100644 --- a/vendor/github.com/containers/storage/storage.conf +++ b/vendor/github.com/containers/storage/storage.conf @@ -40,6 +40,28 @@ graphroot = "/var/lib/containers/storage" additionalimagestores = [ ] +# Allows specification of how storage is populated when pulling images. This +# option can speed the pulling process of images compressed with format +# zstd:chunked. Containers/storage looks for files within images that are being +# pulled from a container registry that were previously pulled to the host. It +# can copy or create a hard link to the existing file when it finds them, +# eliminating the need to pull them from the container registry. These options +# can deduplicate pulling of content, disk storage of content and can allow the +# kernel to use less memory when running containers. + +# containers/storage supports four keys +# * enable_partial_images="true" | "false" +# Tells containers/storage to look for files previously pulled in storage +# rather then always pulling them from the container registry. +# * use_hard_links = "false" | "true" +# Tells containers/storage to use hard links rather then create new files in +# the image, if an identical file already existed in storage. +# * ostree_repos = "" +# Tells containers/storage where an ostree repository exists that might have +# previously pulled content which can be used when attempting to avoid +# pulling content from the container registry +pull_options = {enable_partial_images = "false", use_hard_links = "false", ostree_repos=""} + # Remap-UIDs/GIDs is the mapping from UIDs/GIDs as they should appear inside of # a container, to the UIDs/GIDs as they should appear outside of the container, # and the length of the range of UIDs/GIDs. Additional mapped sets can be diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 6bc104f19..4b074eea2 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -21,7 +21,6 @@ import ( "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/ioutils" "github.com/containers/storage/pkg/parsers" - "github.com/containers/storage/pkg/stringid" "github.com/containers/storage/pkg/stringutils" "github.com/containers/storage/pkg/system" "github.com/containers/storage/types" @@ -1016,9 +1015,6 @@ func (s *store) PutLayer(id, parent string, names []string, mountLabel string, w if err := rcstore.ReloadIfChanged(); err != nil { return nil, -1, err } - if id == "" { - id = stringid.GenerateRandomID() - } if options == nil { options = &LayerOptions{} } @@ -1097,10 +1093,6 @@ func (s *store) CreateLayer(id, parent string, names []string, mountLabel string } func (s *store) CreateImage(id string, names []string, layer, metadata string, options *ImageOptions) (*Image, error) { - if id == "" { - id = stringid.GenerateRandomID() - } - if layer != "" { lstore, err := s.LayerStore() if err != nil { @@ -1279,9 +1271,6 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat if err != nil { return nil, err } - if id == "" { - id = stringid.GenerateRandomID() - } var imageTopLayer *Layer imageID := "" @@ -1337,14 +1326,14 @@ func (s *store) CreateContainer(id string, names []string, image, layer, metadat } } if cimage == nil { - return nil, errors.Wrapf(ErrImageUnknown, "error locating image with ID %q", id) + return nil, errors.Wrapf(ErrImageUnknown, "error locating image with ID %q", image) } imageID = cimage.ID } if options.AutoUserNs { var err error - options.UIDMap, options.GIDMap, err = s.getAutoUserNS(id, &options.AutoUserNsOpts, cimage) + options.UIDMap, options.GIDMap, err = s.getAutoUserNS(&options.AutoUserNsOpts, cimage) if err != nil { return nil, err } diff --git a/vendor/github.com/containers/storage/types/options.go b/vendor/github.com/containers/storage/types/options.go index 38e780b44..a55bf62c3 100644 --- a/vendor/github.com/containers/storage/types/options.go +++ b/vendor/github.com/containers/storage/types/options.go @@ -26,15 +26,24 @@ type TomlConfig struct { } const ( - overlayDriver = "overlay" - overlay2 = "overlay2" + overlayDriver = "overlay" + overlay2 = "overlay2" + storageConfEnv = "CONTAINERS_STORAGE_CONF" ) -func init() { +var ( + defaultStoreOptionsOnce sync.Once +) + +func loaddefaultStoreOptions() { defaultStoreOptions.RunRoot = defaultRunRoot defaultStoreOptions.GraphRoot = defaultGraphRoot defaultStoreOptions.GraphDriverName = "" + if path, ok := os.LookupEnv(storageConfEnv); ok { + defaultOverrideConfigFile = path + } + if _, err := os.Stat(defaultOverrideConfigFile); err == nil { // The DefaultConfigFile(rootless) function returns the path // of the used storage.conf file, by returning defaultConfigFile @@ -64,6 +73,7 @@ func defaultStoreOptionsIsolated(rootless bool, rootlessUID int, storageConf str defaultRootlessGraphRoot string err error ) + defaultStoreOptionsOnce.Do(loaddefaultStoreOptions) storageOpts := defaultStoreOptions if rootless && rootlessUID != 0 { storageOpts, err = getRootlessStorageOpts(rootlessUID, storageOpts) @@ -391,6 +401,7 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) { } func Options() StoreOptions { + defaultStoreOptionsOnce.Do(loaddefaultStoreOptions) return defaultStoreOptions } diff --git a/vendor/github.com/containers/storage/types/utils.go b/vendor/github.com/containers/storage/types/utils.go index 4dd1a786e..c7f0d0fad 100644 --- a/vendor/github.com/containers/storage/types/utils.go +++ b/vendor/github.com/containers/storage/types/utils.go @@ -170,7 +170,7 @@ func DefaultConfigFile(rootless bool) (string, error) { return defaultConfigFile, nil } - if path, ok := os.LookupEnv("CONTAINERS_STORAGE_CONF"); ok { + if path, ok := os.LookupEnv(storageConfEnv); ok { return path, nil } if !rootless { diff --git a/vendor/github.com/containers/storage/userns.go b/vendor/github.com/containers/storage/userns.go index 523c92dc8..13ebd4021 100644 --- a/vendor/github.com/containers/storage/userns.go +++ b/vendor/github.com/containers/storage/userns.go @@ -124,7 +124,7 @@ func parseMountedFiles(containerMount, passwdFile, groupFile string) uint32 { // getMaxSizeFromImage returns the maximum ID used by the specified image. // The layer stores must be already locked. -func (s *store) getMaxSizeFromImage(id string, image *Image, passwdFile, groupFile string) (uint32, error) { +func (s *store) getMaxSizeFromImage(image *Image, passwdFile, groupFile string) (uint32, error) { lstore, err := s.LayerStore() if err != nil { return 0, err @@ -183,7 +183,7 @@ outer: // We need to create a temporary layer so we can mount it and lookup the // maximum IDs used. - clayer, err := rlstore.Create(id, topLayer, nil, "", nil, layerOptions, false) + clayer, err := rlstore.Create("", topLayer, nil, "", nil, layerOptions, false) if err != nil { return 0, err } @@ -211,7 +211,7 @@ outer: } // getAutoUserNS creates an automatic user namespace -func (s *store) getAutoUserNS(id string, options *types.AutoUserNsOptions, image *Image) ([]idtools.IDMap, []idtools.IDMap, error) { +func (s *store) getAutoUserNS(options *types.AutoUserNsOptions, image *Image) ([]idtools.IDMap, []idtools.IDMap, error) { requestedSize := uint32(0) initialSize := uint32(1) if options.Size > 0 { @@ -250,7 +250,7 @@ func (s *store) getAutoUserNS(id string, options *types.AutoUserNsOptions, image size = s.autoNsMinSize } if image != nil { - sizeFromImage, err := s.getMaxSizeFromImage(id, image, options.PasswdFile, options.GroupFile) + sizeFromImage, err := s.getMaxSizeFromImage(image, options.PasswdFile, options.GroupFile) if err != nil { return nil, nil, err } diff --git a/vendor/github.com/docker/libnetwork/resolvconf/README.md b/vendor/github.com/docker/libnetwork/resolvconf/README.md deleted file mode 100644 index cdda554ba..000000000 --- a/vendor/github.com/docker/libnetwork/resolvconf/README.md +++ /dev/null @@ -1 +0,0 @@ -Package resolvconf provides utility code to query and update DNS configuration in /etc/resolv.conf diff --git a/vendor/github.com/docker/libnetwork/resolvconf/dns/resolvconf.go b/vendor/github.com/docker/libnetwork/resolvconf/dns/resolvconf.go deleted file mode 100644 index e348bc57f..000000000 --- a/vendor/github.com/docker/libnetwork/resolvconf/dns/resolvconf.go +++ /dev/null @@ -1,26 +0,0 @@ -package dns - -import ( - "regexp" -) - -// IPLocalhost is a regex pattern for IPv4 or IPv6 loopback range. -const IPLocalhost = `((127\.([0-9]{1,3}\.){2}[0-9]{1,3})|(::1)$)` - -// IPv4Localhost is a regex pattern for IPv4 localhost address range. -const IPv4Localhost = `(127\.([0-9]{1,3}\.){2}[0-9]{1,3})` - -var localhostIPRegexp = regexp.MustCompile(IPLocalhost) -var localhostIPv4Regexp = regexp.MustCompile(IPv4Localhost) - -// IsLocalhost returns true if ip matches the localhost IP regular expression. -// Used for determining if nameserver settings are being passed which are -// localhost addresses -func IsLocalhost(ip string) bool { - return localhostIPRegexp.MatchString(ip) -} - -// IsIPv4Localhost returns true if ip matches the IPv4 localhost regular expression. -func IsIPv4Localhost(ip string) bool { - return localhostIPv4Regexp.MatchString(ip) -} diff --git a/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go b/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go deleted file mode 100644 index 946bb8712..000000000 --- a/vendor/github.com/docker/libnetwork/resolvconf/resolvconf.go +++ /dev/null @@ -1,285 +0,0 @@ -// Package resolvconf provides utility code to query and update DNS configuration in /etc/resolv.conf -package resolvconf - -import ( - "bytes" - "io/ioutil" - "regexp" - "strings" - "sync" - - "github.com/docker/docker/pkg/ioutils" - "github.com/docker/libnetwork/resolvconf/dns" - "github.com/docker/libnetwork/types" - "github.com/sirupsen/logrus" -) - -const ( - // defaultPath is the default path to the resolv.conf that contains information to resolve DNS. See Path(). - defaultPath = "/etc/resolv.conf" - // alternatePath is a path different from defaultPath, that may be used to resolve DNS. See Path(). - alternatePath = "/run/systemd/resolve/resolv.conf" -) - -var ( - detectSystemdResolvConfOnce sync.Once - pathAfterSystemdDetection = defaultPath -) - -// Path returns the path to the resolv.conf file that libnetwork should use. -// -// When /etc/resolv.conf contains 127.0.0.53 as the only nameserver, then -// it is assumed systemd-resolved manages DNS. Because inside the container 127.0.0.53 -// is not a valid DNS server, Path() returns /run/systemd/resolve/resolv.conf -// which is the resolv.conf that systemd-resolved generates and manages. -// Otherwise Path() returns /etc/resolv.conf. -// -// Errors are silenced as they will inevitably resurface at future open/read calls. -// -// More information at https://www.freedesktop.org/software/systemd/man/systemd-resolved.service.html#/etc/resolv.conf -func Path() string { - detectSystemdResolvConfOnce.Do(func() { - candidateResolvConf, err := ioutil.ReadFile(defaultPath) - if err != nil { - // silencing error as it will resurface at next calls trying to read defaultPath - return - } - ns := GetNameservers(candidateResolvConf, types.IP) - if len(ns) == 1 && ns[0] == "127.0.0.53" { - pathAfterSystemdDetection = alternatePath - logrus.Infof("detected 127.0.0.53 nameserver, assuming systemd-resolved, so using resolv.conf: %s", alternatePath) - } - }) - return pathAfterSystemdDetection -} - -var ( - // Note: the default IPv4 & IPv6 resolvers are set to Google's Public DNS - defaultIPv4Dns = []string{"nameserver 8.8.8.8", "nameserver 8.8.4.4"} - defaultIPv6Dns = []string{"nameserver 2001:4860:4860::8888", "nameserver 2001:4860:4860::8844"} - ipv4NumBlock = `(25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)` - ipv4Address = `(` + ipv4NumBlock + `\.){3}` + ipv4NumBlock - // This is not an IPv6 address verifier as it will accept a super-set of IPv6, and also - // will *not match* IPv4-Embedded IPv6 Addresses (RFC6052), but that and other variants - // -- e.g. other link-local types -- either won't work in containers or are unnecessary. - // For readability and sufficiency for Docker purposes this seemed more reasonable than a - // 1000+ character regexp with exact and complete IPv6 validation - ipv6Address = `([0-9A-Fa-f]{0,4}:){2,7}([0-9A-Fa-f]{0,4})(%\w+)?` - - localhostNSRegexp = regexp.MustCompile(`(?m)^nameserver\s+` + dns.IPLocalhost + `\s*\n*`) - nsIPv6Regexp = regexp.MustCompile(`(?m)^nameserver\s+` + ipv6Address + `\s*\n*`) - nsRegexp = regexp.MustCompile(`^\s*nameserver\s*((` + ipv4Address + `)|(` + ipv6Address + `))\s*$`) - nsIPv6Regexpmatch = regexp.MustCompile(`^\s*nameserver\s*((` + ipv6Address + `))\s*$`) - nsIPv4Regexpmatch = regexp.MustCompile(`^\s*nameserver\s*((` + ipv4Address + `))\s*$`) - searchRegexp = regexp.MustCompile(`^\s*search\s*(([^\s]+\s*)*)$`) - optionsRegexp = regexp.MustCompile(`^\s*options\s*(([^\s]+\s*)*)$`) -) - -var lastModified struct { - sync.Mutex - sha256 string - contents []byte -} - -// File contains the resolv.conf content and its hash -type File struct { - Content []byte - Hash string -} - -// Get returns the contents of /etc/resolv.conf and its hash -func Get() (*File, error) { - return GetSpecific(Path()) -} - -// GetSpecific returns the contents of the user specified resolv.conf file and its hash -func GetSpecific(path string) (*File, error) { - resolv, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - hash, err := ioutils.HashData(bytes.NewReader(resolv)) - if err != nil { - return nil, err - } - return &File{Content: resolv, Hash: hash}, nil -} - -// GetIfChanged retrieves the host /etc/resolv.conf file, checks against the last hash -// and, if modified since last check, returns the bytes and new hash. -// This feature is used by the resolv.conf updater for containers -func GetIfChanged() (*File, error) { - lastModified.Lock() - defer lastModified.Unlock() - - resolv, err := ioutil.ReadFile(Path()) - if err != nil { - return nil, err - } - newHash, err := ioutils.HashData(bytes.NewReader(resolv)) - if err != nil { - return nil, err - } - if lastModified.sha256 != newHash { - lastModified.sha256 = newHash - lastModified.contents = resolv - return &File{Content: resolv, Hash: newHash}, nil - } - // nothing changed, so return no data - return nil, nil -} - -// GetLastModified retrieves the last used contents and hash of the host resolv.conf. -// Used by containers updating on restart -func GetLastModified() *File { - lastModified.Lock() - defer lastModified.Unlock() - - return &File{Content: lastModified.contents, Hash: lastModified.sha256} -} - -// FilterResolvDNS cleans up the config in resolvConf. It has two main jobs: -// 1. It looks for localhost (127.*|::1) entries in the provided -// resolv.conf, removing local nameserver entries, and, if the resulting -// cleaned config has no defined nameservers left, adds default DNS entries -// 2. Given the caller provides the enable/disable state of IPv6, the filter -// code will remove all IPv6 nameservers if it is not enabled for containers -// -func FilterResolvDNS(resolvConf []byte, ipv6Enabled bool) (*File, error) { - cleanedResolvConf := localhostNSRegexp.ReplaceAll(resolvConf, []byte{}) - // if IPv6 is not enabled, also clean out any IPv6 address nameserver - if !ipv6Enabled { - cleanedResolvConf = nsIPv6Regexp.ReplaceAll(cleanedResolvConf, []byte{}) - } - // if the resulting resolvConf has no more nameservers defined, add appropriate - // default DNS servers for IPv4 and (optionally) IPv6 - if len(GetNameservers(cleanedResolvConf, types.IP)) == 0 { - logrus.Infof("No non-localhost DNS nameservers are left in resolv.conf. Using default external servers: %v", defaultIPv4Dns) - dns := defaultIPv4Dns - if ipv6Enabled { - logrus.Infof("IPv6 enabled; Adding default IPv6 external servers: %v", defaultIPv6Dns) - dns = append(dns, defaultIPv6Dns...) - } - cleanedResolvConf = append(cleanedResolvConf, []byte("\n"+strings.Join(dns, "\n"))...) - } - hash, err := ioutils.HashData(bytes.NewReader(cleanedResolvConf)) - if err != nil { - return nil, err - } - return &File{Content: cleanedResolvConf, Hash: hash}, nil -} - -// getLines parses input into lines and strips away comments. -func getLines(input []byte, commentMarker []byte) [][]byte { - lines := bytes.Split(input, []byte("\n")) - var output [][]byte - for _, currentLine := range lines { - var commentIndex = bytes.Index(currentLine, commentMarker) - if commentIndex == -1 { - output = append(output, currentLine) - } else { - output = append(output, currentLine[:commentIndex]) - } - } - return output -} - -// GetNameservers returns nameservers (if any) listed in /etc/resolv.conf -func GetNameservers(resolvConf []byte, kind int) []string { - nameservers := []string{} - for _, line := range getLines(resolvConf, []byte("#")) { - var ns [][]byte - if kind == types.IP { - ns = nsRegexp.FindSubmatch(line) - } else if kind == types.IPv4 { - ns = nsIPv4Regexpmatch.FindSubmatch(line) - } else if kind == types.IPv6 { - ns = nsIPv6Regexpmatch.FindSubmatch(line) - } - if len(ns) > 0 { - nameservers = append(nameservers, string(ns[1])) - } - } - return nameservers -} - -// GetNameserversAsCIDR returns nameservers (if any) listed in -// /etc/resolv.conf as CIDR blocks (e.g., "1.2.3.4/32") -// This function's output is intended for net.ParseCIDR -func GetNameserversAsCIDR(resolvConf []byte) []string { - nameservers := []string{} - for _, nameserver := range GetNameservers(resolvConf, types.IP) { - var address string - // If IPv6, strip zone if present - if strings.Contains(nameserver, ":") { - address = strings.Split(nameserver, "%")[0] + "/128" - } else { - address = nameserver + "/32" - } - nameservers = append(nameservers, address) - } - return nameservers -} - -// GetSearchDomains returns search domains (if any) listed in /etc/resolv.conf -// If more than one search line is encountered, only the contents of the last -// one is returned. -func GetSearchDomains(resolvConf []byte) []string { - domains := []string{} - for _, line := range getLines(resolvConf, []byte("#")) { - match := searchRegexp.FindSubmatch(line) - if match == nil { - continue - } - domains = strings.Fields(string(match[1])) - } - return domains -} - -// GetOptions returns options (if any) listed in /etc/resolv.conf -// If more than one options line is encountered, only the contents of the last -// one is returned. -func GetOptions(resolvConf []byte) []string { - options := []string{} - for _, line := range getLines(resolvConf, []byte("#")) { - match := optionsRegexp.FindSubmatch(line) - if match == nil { - continue - } - options = strings.Fields(string(match[1])) - } - return options -} - -// Build writes a configuration file to path containing a "nameserver" entry -// for every element in dns, a "search" entry for every element in -// dnsSearch, and an "options" entry for every element in dnsOptions. -func Build(path string, dns, dnsSearch, dnsOptions []string) (*File, error) { - content := bytes.NewBuffer(nil) - if len(dnsSearch) > 0 { - if searchString := strings.Join(dnsSearch, " "); strings.Trim(searchString, " ") != "." { - if _, err := content.WriteString("search " + searchString + "\n"); err != nil { - return nil, err - } - } - } - for _, dns := range dns { - if _, err := content.WriteString("nameserver " + dns + "\n"); err != nil { - return nil, err - } - } - if len(dnsOptions) > 0 { - if optsString := strings.Join(dnsOptions, " "); strings.Trim(optsString, " ") != "" { - if _, err := content.WriteString("options " + optsString + "\n"); err != nil { - return nil, err - } - } - } - - hash, err := ioutils.HashData(bytes.NewReader(content.Bytes())) - if err != nil { - return nil, err - } - - return &File{Content: content.Bytes(), Hash: hash}, ioutil.WriteFile(path, content.Bytes(), 0644) -} diff --git a/vendor/github.com/docker/libnetwork/types/types.go b/vendor/github.com/docker/libnetwork/types/types.go deleted file mode 100644 index db1960c10..000000000 --- a/vendor/github.com/docker/libnetwork/types/types.go +++ /dev/null @@ -1,653 +0,0 @@ -// Package types contains types that are common across libnetwork project -package types - -import ( - "bytes" - "fmt" - "net" - "strconv" - "strings" - - "github.com/ishidawataru/sctp" -) - -// constants for the IP address type -const ( - IP = iota // IPv4 and IPv6 - IPv4 - IPv6 -) - -// EncryptionKey is the libnetwork representation of the key distributed by the lead -// manager. -type EncryptionKey struct { - Subsystem string - Algorithm int32 - Key []byte - LamportTime uint64 -} - -// UUID represents a globally unique ID of various resources like network and endpoint -type UUID string - -// QosPolicy represents a quality of service policy on an endpoint -type QosPolicy struct { - MaxEgressBandwidth uint64 -} - -// TransportPort represents a local Layer 4 endpoint -type TransportPort struct { - Proto Protocol - Port uint16 -} - -// Equal checks if this instance of Transportport is equal to the passed one -func (t *TransportPort) Equal(o *TransportPort) bool { - if t == o { - return true - } - - if o == nil { - return false - } - - if t.Proto != o.Proto || t.Port != o.Port { - return false - } - - return true -} - -// GetCopy returns a copy of this TransportPort structure instance -func (t *TransportPort) GetCopy() TransportPort { - return TransportPort{Proto: t.Proto, Port: t.Port} -} - -// String returns the TransportPort structure in string form -func (t *TransportPort) String() string { - return fmt.Sprintf("%s/%d", t.Proto.String(), t.Port) -} - -// FromString reads the TransportPort structure from string -func (t *TransportPort) FromString(s string) error { - ps := strings.Split(s, "/") - if len(ps) == 2 { - t.Proto = ParseProtocol(ps[0]) - if p, err := strconv.ParseUint(ps[1], 10, 16); err == nil { - t.Port = uint16(p) - return nil - } - } - return BadRequestErrorf("invalid format for transport port: %s", s) -} - -// PortBinding represents a port binding between the container and the host -type PortBinding struct { - Proto Protocol - IP net.IP - Port uint16 - HostIP net.IP - HostPort uint16 - HostPortEnd uint16 -} - -// HostAddr returns the host side transport address -func (p PortBinding) HostAddr() (net.Addr, error) { - switch p.Proto { - case UDP: - return &net.UDPAddr{IP: p.HostIP, Port: int(p.HostPort)}, nil - case TCP: - return &net.TCPAddr{IP: p.HostIP, Port: int(p.HostPort)}, nil - case SCTP: - return &sctp.SCTPAddr{IPAddrs: []net.IPAddr{{IP: p.HostIP}}, Port: int(p.HostPort)}, nil - default: - return nil, ErrInvalidProtocolBinding(p.Proto.String()) - } -} - -// ContainerAddr returns the container side transport address -func (p PortBinding) ContainerAddr() (net.Addr, error) { - switch p.Proto { - case UDP: - return &net.UDPAddr{IP: p.IP, Port: int(p.Port)}, nil - case TCP: - return &net.TCPAddr{IP: p.IP, Port: int(p.Port)}, nil - case SCTP: - return &sctp.SCTPAddr{IPAddrs: []net.IPAddr{{IP: p.IP}}, Port: int(p.Port)}, nil - default: - return nil, ErrInvalidProtocolBinding(p.Proto.String()) - } -} - -// GetCopy returns a copy of this PortBinding structure instance -func (p *PortBinding) GetCopy() PortBinding { - return PortBinding{ - Proto: p.Proto, - IP: GetIPCopy(p.IP), - Port: p.Port, - HostIP: GetIPCopy(p.HostIP), - HostPort: p.HostPort, - HostPortEnd: p.HostPortEnd, - } -} - -// String returns the PortBinding structure in string form -func (p *PortBinding) String() string { - ret := fmt.Sprintf("%s/", p.Proto) - if p.IP != nil { - ret += p.IP.String() - } - ret = fmt.Sprintf("%s:%d/", ret, p.Port) - if p.HostIP != nil { - ret += p.HostIP.String() - } - ret = fmt.Sprintf("%s:%d", ret, p.HostPort) - return ret -} - -// FromString reads the PortBinding structure from string s. -// String s is a triple of "protocol/containerIP:port/hostIP:port" -// containerIP and hostIP can be in dotted decimal ("192.0.2.1") or IPv6 ("2001:db8::68") form. -// Zoned addresses ("169.254.0.23%eth0" or "fe80::1ff:fe23:4567:890a%eth0") are not supported. -// If string s is incorrectly formatted or the IP addresses or ports cannot be parsed, FromString -// returns an error. -func (p *PortBinding) FromString(s string) error { - ps := strings.Split(s, "/") - if len(ps) != 3 { - return BadRequestErrorf("invalid format for port binding: %s", s) - } - - p.Proto = ParseProtocol(ps[0]) - - var err error - if p.IP, p.Port, err = parseIPPort(ps[1]); err != nil { - return BadRequestErrorf("failed to parse Container IP/Port in port binding: %s", err.Error()) - } - - if p.HostIP, p.HostPort, err = parseIPPort(ps[2]); err != nil { - return BadRequestErrorf("failed to parse Host IP/Port in port binding: %s", err.Error()) - } - - return nil -} - -func parseIPPort(s string) (net.IP, uint16, error) { - hoststr, portstr, err := net.SplitHostPort(s) - if err != nil { - return nil, 0, err - } - - ip := net.ParseIP(hoststr) - if ip == nil { - return nil, 0, BadRequestErrorf("invalid ip: %s", hoststr) - } - - port, err := strconv.ParseUint(portstr, 10, 16) - if err != nil { - return nil, 0, BadRequestErrorf("invalid port: %s", portstr) - } - - return ip, uint16(port), nil -} - -// Equal checks if this instance of PortBinding is equal to the passed one -func (p *PortBinding) Equal(o *PortBinding) bool { - if p == o { - return true - } - - if o == nil { - return false - } - - if p.Proto != o.Proto || p.Port != o.Port || - p.HostPort != o.HostPort || p.HostPortEnd != o.HostPortEnd { - return false - } - - if p.IP != nil { - if !p.IP.Equal(o.IP) { - return false - } - } else { - if o.IP != nil { - return false - } - } - - if p.HostIP != nil { - if !p.HostIP.Equal(o.HostIP) { - return false - } - } else { - if o.HostIP != nil { - return false - } - } - - return true -} - -// ErrInvalidProtocolBinding is returned when the port binding protocol is not valid. -type ErrInvalidProtocolBinding string - -func (ipb ErrInvalidProtocolBinding) Error() string { - return fmt.Sprintf("invalid transport protocol: %s", string(ipb)) -} - -const ( - // ICMP is for the ICMP ip protocol - ICMP = 1 - // TCP is for the TCP ip protocol - TCP = 6 - // UDP is for the UDP ip protocol - UDP = 17 - // SCTP is for the SCTP ip protocol - SCTP = 132 -) - -// Protocol represents an IP protocol number -type Protocol uint8 - -func (p Protocol) String() string { - switch p { - case ICMP: - return "icmp" - case TCP: - return "tcp" - case UDP: - return "udp" - case SCTP: - return "sctp" - default: - return fmt.Sprintf("%d", p) - } -} - -// ParseProtocol returns the respective Protocol type for the passed string -func ParseProtocol(s string) Protocol { - switch strings.ToLower(s) { - case "icmp": - return ICMP - case "udp": - return UDP - case "tcp": - return TCP - case "sctp": - return SCTP - default: - return 0 - } -} - -// GetMacCopy returns a copy of the passed MAC address -func GetMacCopy(from net.HardwareAddr) net.HardwareAddr { - if from == nil { - return nil - } - to := make(net.HardwareAddr, len(from)) - copy(to, from) - return to -} - -// GetIPCopy returns a copy of the passed IP address -func GetIPCopy(from net.IP) net.IP { - if from == nil { - return nil - } - to := make(net.IP, len(from)) - copy(to, from) - return to -} - -// GetIPNetCopy returns a copy of the passed IP Network -func GetIPNetCopy(from *net.IPNet) *net.IPNet { - if from == nil { - return nil - } - bm := make(net.IPMask, len(from.Mask)) - copy(bm, from.Mask) - return &net.IPNet{IP: GetIPCopy(from.IP), Mask: bm} -} - -// GetIPNetCanonical returns the canonical form for the passed network -func GetIPNetCanonical(nw *net.IPNet) *net.IPNet { - if nw == nil { - return nil - } - c := GetIPNetCopy(nw) - c.IP = c.IP.Mask(nw.Mask) - return c -} - -// CompareIPNet returns equal if the two IP Networks are equal -func CompareIPNet(a, b *net.IPNet) bool { - if a == b { - return true - } - if a == nil || b == nil { - return false - } - return a.IP.Equal(b.IP) && bytes.Equal(a.Mask, b.Mask) -} - -// GetMinimalIP returns the address in its shortest form -// If ip contains an IPv4-mapped IPv6 address, the 4-octet form of the IPv4 address will be returned. -// Otherwise ip is returned unchanged. -func GetMinimalIP(ip net.IP) net.IP { - if ip != nil && ip.To4() != nil { - return ip.To4() - } - return ip -} - -// GetMinimalIPNet returns a copy of the passed IP Network with congruent ip and mask notation -func GetMinimalIPNet(nw *net.IPNet) *net.IPNet { - if nw == nil { - return nil - } - if len(nw.IP) == 16 && nw.IP.To4() != nil { - m := nw.Mask - if len(m) == 16 { - m = m[12:16] - } - return &net.IPNet{IP: nw.IP.To4(), Mask: m} - } - return nw -} - -// IsIPNetValid returns true if the ipnet is a valid network/mask -// combination. Otherwise returns false. -func IsIPNetValid(nw *net.IPNet) bool { - return nw.String() != "0.0.0.0/0" -} - -var v4inV6MaskPrefix = []byte{0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff} - -// compareIPMask checks if the passed ip and mask are semantically compatible. -// It returns the byte indexes for the address and mask so that caller can -// do bitwise operations without modifying address representation. -func compareIPMask(ip net.IP, mask net.IPMask) (is int, ms int, err error) { - // Find the effective starting of address and mask - if len(ip) == net.IPv6len && ip.To4() != nil { - is = 12 - } - if len(ip[is:]) == net.IPv4len && len(mask) == net.IPv6len && bytes.Equal(mask[:12], v4inV6MaskPrefix) { - ms = 12 - } - // Check if address and mask are semantically compatible - if len(ip[is:]) != len(mask[ms:]) { - err = fmt.Errorf("ip and mask are not compatible: (%#v, %#v)", ip, mask) - } - return -} - -// GetHostPartIP returns the host portion of the ip address identified by the mask. -// IP address representation is not modified. If address and mask are not compatible -// an error is returned. -func GetHostPartIP(ip net.IP, mask net.IPMask) (net.IP, error) { - // Find the effective starting of address and mask - is, ms, err := compareIPMask(ip, mask) - if err != nil { - return nil, fmt.Errorf("cannot compute host portion ip address because %s", err) - } - - // Compute host portion - out := GetIPCopy(ip) - for i := 0; i < len(mask[ms:]); i++ { - out[is+i] &= ^mask[ms+i] - } - - return out, nil -} - -// GetBroadcastIP returns the broadcast ip address for the passed network (ip and mask). -// IP address representation is not modified. If address and mask are not compatible -// an error is returned. -func GetBroadcastIP(ip net.IP, mask net.IPMask) (net.IP, error) { - // Find the effective starting of address and mask - is, ms, err := compareIPMask(ip, mask) - if err != nil { - return nil, fmt.Errorf("cannot compute broadcast ip address because %s", err) - } - - // Compute broadcast address - out := GetIPCopy(ip) - for i := 0; i < len(mask[ms:]); i++ { - out[is+i] |= ^mask[ms+i] - } - - return out, nil -} - -// ParseCIDR returns the *net.IPNet represented by the passed CIDR notation -func ParseCIDR(cidr string) (n *net.IPNet, e error) { - var i net.IP - if i, n, e = net.ParseCIDR(cidr); e == nil { - n.IP = i - } - return -} - -const ( - // NEXTHOP indicates a StaticRoute with an IP next hop. - NEXTHOP = iota - - // CONNECTED indicates a StaticRoute with an interface for directly connected peers. - CONNECTED -) - -// StaticRoute is a statically-provisioned IP route. -type StaticRoute struct { - Destination *net.IPNet - - RouteType int // NEXT_HOP or CONNECTED - - // NextHop will be resolved by the kernel (i.e. as a loose hop). - NextHop net.IP -} - -// GetCopy returns a copy of this StaticRoute structure -func (r *StaticRoute) GetCopy() *StaticRoute { - d := GetIPNetCopy(r.Destination) - nh := GetIPCopy(r.NextHop) - return &StaticRoute{Destination: d, - RouteType: r.RouteType, - NextHop: nh, - } -} - -// InterfaceStatistics represents the interface's statistics -type InterfaceStatistics struct { - RxBytes uint64 - RxPackets uint64 - RxErrors uint64 - RxDropped uint64 - TxBytes uint64 - TxPackets uint64 - TxErrors uint64 - TxDropped uint64 -} - -func (is *InterfaceStatistics) String() string { - return fmt.Sprintf("\nRxBytes: %d, RxPackets: %d, RxErrors: %d, RxDropped: %d, TxBytes: %d, TxPackets: %d, TxErrors: %d, TxDropped: %d", - is.RxBytes, is.RxPackets, is.RxErrors, is.RxDropped, is.TxBytes, is.TxPackets, is.TxErrors, is.TxDropped) -} - -/****************************** - * Well-known Error Interfaces - ******************************/ - -// MaskableError is an interface for errors which can be ignored by caller -type MaskableError interface { - // Maskable makes implementer into MaskableError type - Maskable() -} - -// RetryError is an interface for errors which might get resolved through retry -type RetryError interface { - // Retry makes implementer into RetryError type - Retry() -} - -// BadRequestError is an interface for errors originated by a bad request -type BadRequestError interface { - // BadRequest makes implementer into BadRequestError type - BadRequest() -} - -// NotFoundError is an interface for errors raised because a needed resource is not available -type NotFoundError interface { - // NotFound makes implementer into NotFoundError type - NotFound() -} - -// ForbiddenError is an interface for errors which denote a valid request that cannot be honored -type ForbiddenError interface { - // Forbidden makes implementer into ForbiddenError type - Forbidden() -} - -// NoServiceError is an interface for errors returned when the required service is not available -type NoServiceError interface { - // NoService makes implementer into NoServiceError type - NoService() -} - -// TimeoutError is an interface for errors raised because of timeout -type TimeoutError interface { - // Timeout makes implementer into TimeoutError type - Timeout() -} - -// NotImplementedError is an interface for errors raised because of requested functionality is not yet implemented -type NotImplementedError interface { - // NotImplemented makes implementer into NotImplementedError type - NotImplemented() -} - -// InternalError is an interface for errors raised because of an internal error -type InternalError interface { - // Internal makes implementer into InternalError type - Internal() -} - -/****************************** - * Well-known Error Formatters - ******************************/ - -// BadRequestErrorf creates an instance of BadRequestError -func BadRequestErrorf(format string, params ...interface{}) error { - return badRequest(fmt.Sprintf(format, params...)) -} - -// NotFoundErrorf creates an instance of NotFoundError -func NotFoundErrorf(format string, params ...interface{}) error { - return notFound(fmt.Sprintf(format, params...)) -} - -// ForbiddenErrorf creates an instance of ForbiddenError -func ForbiddenErrorf(format string, params ...interface{}) error { - return forbidden(fmt.Sprintf(format, params...)) -} - -// NoServiceErrorf creates an instance of NoServiceError -func NoServiceErrorf(format string, params ...interface{}) error { - return noService(fmt.Sprintf(format, params...)) -} - -// NotImplementedErrorf creates an instance of NotImplementedError -func NotImplementedErrorf(format string, params ...interface{}) error { - return notImpl(fmt.Sprintf(format, params...)) -} - -// TimeoutErrorf creates an instance of TimeoutError -func TimeoutErrorf(format string, params ...interface{}) error { - return timeout(fmt.Sprintf(format, params...)) -} - -// InternalErrorf creates an instance of InternalError -func InternalErrorf(format string, params ...interface{}) error { - return internal(fmt.Sprintf(format, params...)) -} - -// InternalMaskableErrorf creates an instance of InternalError and MaskableError -func InternalMaskableErrorf(format string, params ...interface{}) error { - return maskInternal(fmt.Sprintf(format, params...)) -} - -// RetryErrorf creates an instance of RetryError -func RetryErrorf(format string, params ...interface{}) error { - return retry(fmt.Sprintf(format, params...)) -} - -/*********************** - * Internal Error Types - ***********************/ -type badRequest string - -func (br badRequest) Error() string { - return string(br) -} -func (br badRequest) BadRequest() {} - -type maskBadRequest string - -type notFound string - -func (nf notFound) Error() string { - return string(nf) -} -func (nf notFound) NotFound() {} - -type forbidden string - -func (frb forbidden) Error() string { - return string(frb) -} -func (frb forbidden) Forbidden() {} - -type noService string - -func (ns noService) Error() string { - return string(ns) -} -func (ns noService) NoService() {} - -type maskNoService string - -type timeout string - -func (to timeout) Error() string { - return string(to) -} -func (to timeout) Timeout() {} - -type notImpl string - -func (ni notImpl) Error() string { - return string(ni) -} -func (ni notImpl) NotImplemented() {} - -type internal string - -func (nt internal) Error() string { - return string(nt) -} -func (nt internal) Internal() {} - -type maskInternal string - -func (mnt maskInternal) Error() string { - return string(mnt) -} -func (mnt maskInternal) Internal() {} -func (mnt maskInternal) Maskable() {} - -type retry string - -func (r retry) Error() string { - return string(r) -} -func (r retry) Retry() {} diff --git a/vendor/github.com/imdario/mergo/README.md b/vendor/github.com/imdario/mergo/README.md index aa8cbd7ce..7e6f7aeee 100644 --- a/vendor/github.com/imdario/mergo/README.md +++ b/vendor/github.com/imdario/mergo/README.md @@ -8,8 +8,7 @@ [![Coverage Status][9]][10] [![Sourcegraph][11]][12] [![FOSSA Status][13]][14] - -[![GoCenter Kudos][15]][16] +[![Become my sponsor][15]][16] [1]: https://travis-ci.org/imdario/mergo.png [2]: https://travis-ci.org/imdario/mergo @@ -25,8 +24,8 @@ [12]: https://sourcegraph.com/github.com/imdario/mergo?badge [13]: https://app.fossa.io/api/projects/git%2Bgithub.com%2Fimdario%2Fmergo.svg?type=shield [14]: https://app.fossa.io/projects/git%2Bgithub.com%2Fimdario%2Fmergo?ref=badge_shield -[15]: https://search.gocenter.io/api/ui/badge/github.com%2Fimdario%2Fmergo -[16]: https://search.gocenter.io/github.com/imdario/mergo +[15]: https://img.shields.io/github/sponsors/imdario +[16]: https://github.com/sponsors/imdario A helper to merge structs and maps in Golang. Useful for configuration default values, avoiding messy if-statements. @@ -36,11 +35,11 @@ Also a lovely [comune](http://en.wikipedia.org/wiki/Mergo) (municipality) in the ## Status -It is ready for production use. [It is used in several projects by Docker, Google, The Linux Foundation, VMWare, Shopify, etc](https://github.com/imdario/mergo#mergo-in-the-wild). +It is ready for production use. [It is used in several projects by Docker, Google, The Linux Foundation, VMWare, Shopify, Microsoft, etc](https://github.com/imdario/mergo#mergo-in-the-wild). ### Important note -Please keep in mind that a problematic PR broke [0.3.9](//github.com/imdario/mergo/releases/tag/0.3.9). I reverted it in [0.3.10](//github.com/imdario/mergo/releases/tag/0.3.10), and I consider it stable but not bug-free. Also, this version adds suppot for go modules. +Please keep in mind that a problematic PR broke [0.3.9](//github.com/imdario/mergo/releases/tag/0.3.9). I reverted it in [0.3.10](//github.com/imdario/mergo/releases/tag/0.3.10), and I consider it stable but not bug-free. Also, this version adds support for go modules. Keep in mind that in [0.3.2](//github.com/imdario/mergo/releases/tag/0.3.2), Mergo changed `Merge()`and `Map()` signatures to support [transformers](#transformers). I added an optional/variadic argument so that it won't break the existing code. @@ -51,12 +50,12 @@ If you were using Mergo before April 6th, 2015, please check your project works If Mergo is useful to you, consider buying me a coffee, a beer, or making a monthly donation to allow me to keep building great free software. :heart_eyes: <a href='https://ko-fi.com/B0B58839' target='_blank'><img height='36' style='border:0px;height:36px;' src='https://az743702.vo.msecnd.net/cdn/kofi1.png?v=0' border='0' alt='Buy Me a Coffee at ko-fi.com' /></a> -[](https://beerpay.io/imdario/mergo) -[](https://beerpay.io/imdario/mergo) <a href="https://liberapay.com/dario/donate"><img alt="Donate using Liberapay" src="https://liberapay.com/assets/widgets/donate.svg"></a> +<a href='https://github.com/sponsors/imdario' target='_blank'><img alt="Become my sponsor" src="https://img.shields.io/github/sponsors/imdario?style=for-the-badge" /></a> ### Mergo in the wild +- [cli/cli](https://github.com/cli/cli) - [moby/moby](https://github.com/moby/moby) - [kubernetes/kubernetes](https://github.com/kubernetes/kubernetes) - [vmware/dispatch](https://github.com/vmware/dispatch) @@ -98,6 +97,8 @@ If Mergo is useful to you, consider buying me a coffee, a beer, or making a mont - [jnuthong/item_search](https://github.com/jnuthong/item_search) - [bukalapak/snowboard](https://github.com/bukalapak/snowboard) - [containerssh/containerssh](https://github.com/containerssh/containerssh) +- [goreleaser/goreleaser](https://github.com/goreleaser/goreleaser) +- [tjpnz/structbot](https://github.com/tjpnz/structbot) ## Install @@ -168,7 +169,7 @@ func main() { Note: if test are failing due missing package, please execute: - go get gopkg.in/yaml.v2 + go get gopkg.in/yaml.v3 ### Transformers @@ -218,7 +219,6 @@ func main() { } ``` - ## Contact me If I can help you, you have an idea or you are using Mergo in your projects, don't hesitate to drop me a line (or a pull request): [@im_dario](https://twitter.com/im_dario) @@ -227,18 +227,6 @@ If I can help you, you have an idea or you are using Mergo in your projects, don Written by [Dario Castañé](http://dario.im). -## Top Contributors - -[](https://sourcerer.io/fame/imdario/imdario/mergo/links/0) -[](https://sourcerer.io/fame/imdario/imdario/mergo/links/1) -[](https://sourcerer.io/fame/imdario/imdario/mergo/links/2) -[](https://sourcerer.io/fame/imdario/imdario/mergo/links/3) -[](https://sourcerer.io/fame/imdario/imdario/mergo/links/4) -[](https://sourcerer.io/fame/imdario/imdario/mergo/links/5) -[](https://sourcerer.io/fame/imdario/imdario/mergo/links/6) -[](https://sourcerer.io/fame/imdario/imdario/mergo/links/7) - - ## License [BSD 3-Clause](http://opensource.org/licenses/BSD-3-Clause) license, as [Go language](http://golang.org/LICENSE). diff --git a/vendor/github.com/imdario/mergo/go.mod b/vendor/github.com/imdario/mergo/go.mod index 3d689d93e..6f476d6ff 100644 --- a/vendor/github.com/imdario/mergo/go.mod +++ b/vendor/github.com/imdario/mergo/go.mod @@ -2,4 +2,4 @@ module github.com/imdario/mergo go 1.13 -require gopkg.in/yaml.v2 v2.3.0 +require gopkg.in/yaml.v3 v3.0.0 diff --git a/vendor/github.com/imdario/mergo/go.sum b/vendor/github.com/imdario/mergo/go.sum index 168980da5..223abcb42 100644 --- a/vendor/github.com/imdario/mergo/go.sum +++ b/vendor/github.com/imdario/mergo/go.sum @@ -1,4 +1,4 @@ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= -gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA= +gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/vendor/github.com/imdario/mergo/merge.go b/vendor/github.com/imdario/mergo/merge.go index 8c2a8fcd9..8b4e2f47a 100644 --- a/vendor/github.com/imdario/mergo/merge.go +++ b/vendor/github.com/imdario/mergo/merge.go @@ -79,7 +79,7 @@ func deepMerge(dst, src reflect.Value, visited map[uintptr]*visit, depth int, co visited[h] = &visit{addr, typ, seen} } - if config.Transformers != nil && !isEmptyValue(dst) { + if config.Transformers != nil && !isReflectNil(dst) && dst.IsValid() { if fn := config.Transformers.Transformer(dst.Type()); fn != nil { err = fn(dst, src) return diff --git a/vendor/github.com/imdario/mergo/mergo.go b/vendor/github.com/imdario/mergo/mergo.go index 3cc926c7f..9fe362d47 100644 --- a/vendor/github.com/imdario/mergo/mergo.go +++ b/vendor/github.com/imdario/mergo/mergo.go @@ -17,7 +17,7 @@ import ( var ( ErrNilArguments = errors.New("src and dst must not be nil") ErrDifferentArgumentsTypes = errors.New("src and dst must be of same type") - ErrNotSupported = errors.New("only structs and maps are supported") + ErrNotSupported = errors.New("only structs, maps, and slices are supported") ErrExpectedMapAsDestination = errors.New("dst was expected to be a map") ErrExpectedStructAsDestination = errors.New("dst was expected to be a struct") ErrNonPointerAgument = errors.New("dst must be a pointer") @@ -65,7 +65,7 @@ func resolveValues(dst, src interface{}) (vDst, vSrc reflect.Value, err error) { return } vDst = reflect.ValueOf(dst).Elem() - if vDst.Kind() != reflect.Struct && vDst.Kind() != reflect.Map { + if vDst.Kind() != reflect.Struct && vDst.Kind() != reflect.Map && vDst.Kind() != reflect.Slice { err = ErrNotSupported return } diff --git a/vendor/github.com/ishidawataru/sctp/.gitignore b/vendor/github.com/ishidawataru/sctp/.gitignore deleted file mode 100644 index cf2d826c1..000000000 --- a/vendor/github.com/ishidawataru/sctp/.gitignore +++ /dev/null @@ -1,16 +0,0 @@ -# Binaries for programs and plugins -*.exe -*.dll -*.so -*.dylib - -# Test binary, build with `go test -c` -*.test - -# Output of the go coverage tool, specifically when used with LiteIDE -*.out - -# Project-local glide cache, RE: https://github.com/Masterminds/glide/issues/736 -.glide/ - -example/example diff --git a/vendor/github.com/ishidawataru/sctp/.travis.yml b/vendor/github.com/ishidawataru/sctp/.travis.yml deleted file mode 100644 index a1c693c01..000000000 --- a/vendor/github.com/ishidawataru/sctp/.travis.yml +++ /dev/null @@ -1,29 +0,0 @@ -language: go -arch: - - amd64 - - ppc64le -go: - - 1.9.x - - 1.10.x - - 1.11.x - - 1.12.x - - 1.13.x -# allowing test cases to fail for the versions were not suppotred by ppc64le -matrix: - allow_failures: - - go: 1.9.x - - go: 1.10.x - - go: 1.13.x - - -script: - - go test -v -race ./... - - GOOS=linux GOARCH=amd64 go build . - - GOOS=linux GOARCH=arm go build . - - GOOS=linux GOARCH=arm64 go build . - - GOOS=linux GOARCH=ppc64le go build . - - GOOS=linux GOARCH=mips64le go build . - - (go version | grep go1.6 > /dev/null) || GOOS=linux GOARCH=s390x go build . -# can be compiled but not functional: - - GOOS=linux GOARCH=386 go build . - - GOOS=windows GOARCH=amd64 go build . diff --git a/vendor/github.com/ishidawataru/sctp/GO_LICENSE b/vendor/github.com/ishidawataru/sctp/GO_LICENSE deleted file mode 100644 index 6a66aea5e..000000000 --- a/vendor/github.com/ishidawataru/sctp/GO_LICENSE +++ /dev/null @@ -1,27 +0,0 @@ -Copyright (c) 2009 The Go Authors. All rights reserved. - -Redistribution and use in source and binary forms, with or without -modification, are permitted provided that the following conditions are -met: - - * Redistributions of source code must retain the above copyright -notice, this list of conditions and the following disclaimer. - * Redistributions in binary form must reproduce the above -copyright notice, this list of conditions and the following disclaimer -in the documentation and/or other materials provided with the -distribution. - * Neither the name of Google Inc. nor the names of its -contributors may be used to endorse or promote products derived from -this software without specific prior written permission. - -THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS -"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT -LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR -A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT -OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, -SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT -LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, -DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY -THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT -(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE -OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/github.com/ishidawataru/sctp/LICENSE b/vendor/github.com/ishidawataru/sctp/LICENSE deleted file mode 100644 index 8dada3eda..000000000 --- a/vendor/github.com/ishidawataru/sctp/LICENSE +++ /dev/null @@ -1,201 +0,0 @@ - Apache License - Version 2.0, January 2004 - http://www.apache.org/licenses/ - - TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION - - 1. Definitions. - - "License" shall mean the terms and conditions for use, reproduction, - and distribution as defined by Sections 1 through 9 of this document. - - "Licensor" shall mean the copyright owner or entity authorized by - the copyright owner that is granting the License. - - "Legal Entity" shall mean the union of the acting entity and all - other entities that control, are controlled by, or are under common - control with that entity. For the purposes of this definition, - "control" means (i) the power, direct or indirect, to cause the - direction or management of such entity, whether by contract or - otherwise, or (ii) ownership of fifty percent (50%) or more of the - outstanding shares, or (iii) beneficial ownership of such entity. - - "You" (or "Your") shall mean an individual or Legal Entity - exercising permissions granted by this License. - - "Source" form shall mean the preferred form for making modifications, - including but not limited to software source code, documentation - source, and configuration files. - - "Object" form shall mean any form resulting from mechanical - transformation or translation of a Source form, including but - not limited to compiled object code, generated documentation, - and conversions to other media types. - - "Work" shall mean the work of authorship, whether in Source or - Object form, made available under the License, as indicated by a - copyright notice that is included in or attached to the work - (an example is provided in the Appendix below). - - "Derivative Works" shall mean any work, whether in Source or Object - form, that is based on (or derived from) the Work and for which the - editorial revisions, annotations, elaborations, or other modifications - represent, as a whole, an original work of authorship. For the purposes - of this License, Derivative Works shall not include works that remain - separable from, or merely link (or bind by name) to the interfaces of, - the Work and Derivative Works thereof. - - "Contribution" shall mean any work of authorship, including - the original version of the Work and any modifications or additions - to that Work or Derivative Works thereof, that is intentionally - submitted to Licensor for inclusion in the Work by the copyright owner - or by an individual or Legal Entity authorized to submit on behalf of - the copyright owner. For the purposes of this definition, "submitted" - means any form of electronic, verbal, or written communication sent - to the Licensor or its representatives, including but not limited to - communication on electronic mailing lists, source code control systems, - and issue tracking systems that are managed by, or on behalf of, the - Licensor for the purpose of discussing and improving the Work, but - excluding communication that is conspicuously marked or otherwise - designated in writing by the copyright owner as "Not a Contribution." - - "Contributor" shall mean Licensor and any individual or Legal Entity - on behalf of whom a Contribution has been received by Licensor and - subsequently incorporated within the Work. - - 2. Grant of Copyright License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - copyright license to reproduce, prepare Derivative Works of, - publicly display, publicly perform, sublicense, and distribute the - Work and such Derivative Works in Source or Object form. - - 3. Grant of Patent License. Subject to the terms and conditions of - this License, each Contributor hereby grants to You a perpetual, - worldwide, non-exclusive, no-charge, royalty-free, irrevocable - (except as stated in this section) patent license to make, have made, - use, offer to sell, sell, import, and otherwise transfer the Work, - where such license applies only to those patent claims licensable - by such Contributor that are necessarily infringed by their - Contribution(s) alone or by combination of their Contribution(s) - with the Work to which such Contribution(s) was submitted. If You - institute patent litigation against any entity (including a - cross-claim or counterclaim in a lawsuit) alleging that the Work - or a Contribution incorporated within the Work constitutes direct - or contributory patent infringement, then any patent licenses - granted to You under this License for that Work shall terminate - as of the date such litigation is filed. - - 4. Redistribution. You may reproduce and distribute copies of the - Work or Derivative Works thereof in any medium, with or without - modifications, and in Source or Object form, provided that You - meet the following conditions: - - (a) You must give any other recipients of the Work or - Derivative Works a copy of this License; and - - (b) You must cause any modified files to carry prominent notices - stating that You changed the files; and - - (c) You must retain, in the Source form of any Derivative Works - that You distribute, all copyright, patent, trademark, and - attribution notices from the Source form of the Work, - excluding those notices that do not pertain to any part of - the Derivative Works; and - - (d) If the Work includes a "NOTICE" text file as part of its - distribution, then any Derivative Works that You distribute must - include a readable copy of the attribution notices contained - within such NOTICE file, excluding those notices that do not - pertain to any part of the Derivative Works, in at least one - of the following places: within a NOTICE text file distributed - as part of the Derivative Works; within the Source form or - documentation, if provided along with the Derivative Works; or, - within a display generated by the Derivative Works, if and - wherever such third-party notices normally appear. The contents - of the NOTICE file are for informational purposes only and - do not modify the License. You may add Your own attribution - notices within Derivative Works that You distribute, alongside - or as an addendum to the NOTICE text from the Work, provided - that such additional attribution notices cannot be construed - as modifying the License. - - You may add Your own copyright statement to Your modifications and - may provide additional or different license terms and conditions - for use, reproduction, or distribution of Your modifications, or - for any such Derivative Works as a whole, provided Your use, - reproduction, and distribution of the Work otherwise complies with - the conditions stated in this License. - - 5. Submission of Contributions. Unless You explicitly state otherwise, - any Contribution intentionally submitted for inclusion in the Work - by You to the Licensor shall be under the terms and conditions of - this License, without any additional terms or conditions. - Notwithstanding the above, nothing herein shall supersede or modify - the terms of any separate license agreement you may have executed - with Licensor regarding such Contributions. - - 6. Trademarks. This License does not grant permission to use the trade - names, trademarks, service marks, or product names of the Licensor, - except as required for reasonable and customary use in describing the - origin of the Work and reproducing the content of the NOTICE file. - - 7. Disclaimer of Warranty. Unless required by applicable law or - agreed to in writing, Licensor provides the Work (and each - Contributor provides its Contributions) on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or - implied, including, without limitation, any warranties or conditions - of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A - PARTICULAR PURPOSE. You are solely responsible for determining the - appropriateness of using or redistributing the Work and assume any - risks associated with Your exercise of permissions under this License. - - 8. Limitation of Liability. In no event and under no legal theory, - whether in tort (including negligence), contract, or otherwise, - unless required by applicable law (such as deliberate and grossly - negligent acts) or agreed to in writing, shall any Contributor be - liable to You for damages, including any direct, indirect, special, - incidental, or consequential damages of any character arising as a - result of this License or out of the use or inability to use the - Work (including but not limited to damages for loss of goodwill, - work stoppage, computer failure or malfunction, or any and all - other commercial damages or losses), even if such Contributor - has been advised of the possibility of such damages. - - 9. Accepting Warranty or Additional Liability. While redistributing - the Work or Derivative Works thereof, You may choose to offer, - and charge a fee for, acceptance of support, warranty, indemnity, - or other liability obligations and/or rights consistent with this - License. However, in accepting such obligations, You may act only - on Your own behalf and on Your sole responsibility, not on behalf - of any other Contributor, and only if You agree to indemnify, - defend, and hold each Contributor harmless for any liability - incurred by, or claims asserted against, such Contributor by reason - of your accepting any such warranty or additional liability. - - END OF TERMS AND CONDITIONS - - APPENDIX: How to apply the Apache License to your work. - - To apply the Apache License to your work, attach the following - boilerplate notice, with the fields enclosed by brackets "{}" - replaced with your own identifying information. (Don't include - the brackets!) The text should be enclosed in the appropriate - comment syntax for the file format. We also recommend that a - file or class name and description of purpose be included on the - same "printed page" as the copyright notice for easier - identification within third-party archives. - - Copyright {yyyy} {name of copyright owner} - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. diff --git a/vendor/github.com/ishidawataru/sctp/NOTICE b/vendor/github.com/ishidawataru/sctp/NOTICE deleted file mode 100644 index cfb675fd4..000000000 --- a/vendor/github.com/ishidawataru/sctp/NOTICE +++ /dev/null @@ -1,3 +0,0 @@ -This source code includes following third party code - -- ipsock_linux.go : licensed by the Go authors, see GO_LICENSE file for the license which applies to the code diff --git a/vendor/github.com/ishidawataru/sctp/README.md b/vendor/github.com/ishidawataru/sctp/README.md deleted file mode 100644 index 574ececa8..000000000 --- a/vendor/github.com/ishidawataru/sctp/README.md +++ /dev/null @@ -1,18 +0,0 @@ -Stream Control Transmission Protocol (SCTP) ----- - -[](https://travis-ci.org/ishidawataru/sctp/builds) - -Examples ----- - -See `example/sctp.go` - -```go -$ cd example -$ go build -$ # run example SCTP server -$ ./example -server -port 1000 -ip 10.10.0.1,10.20.0.1 -$ # run example SCTP client -$ ./example -port 1000 -ip 10.10.0.1,10.20.0.1 -``` diff --git a/vendor/github.com/ishidawataru/sctp/go.mod b/vendor/github.com/ishidawataru/sctp/go.mod deleted file mode 100644 index 5adf982b0..000000000 --- a/vendor/github.com/ishidawataru/sctp/go.mod +++ /dev/null @@ -1,3 +0,0 @@ -module github.com/ishidawataru/sctp - -go 1.12 diff --git a/vendor/github.com/ishidawataru/sctp/ipsock_linux.go b/vendor/github.com/ishidawataru/sctp/ipsock_linux.go deleted file mode 100644 index 3df30fa46..000000000 --- a/vendor/github.com/ishidawataru/sctp/ipsock_linux.go +++ /dev/null @@ -1,222 +0,0 @@ -// Copyright 2009 The Go Authors. All rights reserved. -// Use of this source code is governed by a BSD-style -// license that can be found in the GO_LICENSE file. - -package sctp - -import ( - "net" - "os" - "sync" - "syscall" -) - -//from https://github.com/golang/go -// Boolean to int. -func boolint(b bool) int { - if b { - return 1 - } - return 0 -} - -//from https://github.com/golang/go -func ipToSockaddr(family int, ip net.IP, port int, zone string) (syscall.Sockaddr, error) { - switch family { - case syscall.AF_INET: - if len(ip) == 0 { - ip = net.IPv4zero - } - ip4 := ip.To4() - if ip4 == nil { - return nil, &net.AddrError{Err: "non-IPv4 address", Addr: ip.String()} - } - sa := &syscall.SockaddrInet4{Port: port} - copy(sa.Addr[:], ip4) - return sa, nil - case syscall.AF_INET6: - // In general, an IP wildcard address, which is either - // "0.0.0.0" or "::", means the entire IP addressing - // space. For some historical reason, it is used to - // specify "any available address" on some operations - // of IP node. - // - // When the IP node supports IPv4-mapped IPv6 address, - // we allow an listener to listen to the wildcard - // address of both IP addressing spaces by specifying - // IPv6 wildcard address. - if len(ip) == 0 || ip.Equal(net.IPv4zero) { - ip = net.IPv6zero - } - // We accept any IPv6 address including IPv4-mapped - // IPv6 address. - ip6 := ip.To16() - if ip6 == nil { - return nil, &net.AddrError{Err: "non-IPv6 address", Addr: ip.String()} - } - //we set ZoneId to 0, as currently we use this functon only to probe the IP capabilities of the host - //if real Zone handling is required, the zone cache implementation in golang/net should be pulled here - sa := &syscall.SockaddrInet6{Port: port, ZoneId: 0} - copy(sa.Addr[:], ip6) - return sa, nil - } - return nil, &net.AddrError{Err: "invalid address family", Addr: ip.String()} -} - -//from https://github.com/golang/go -func sockaddr(a *net.TCPAddr, family int) (syscall.Sockaddr, error) { - if a == nil { - return nil, nil - } - return ipToSockaddr(family, a.IP, a.Port, a.Zone) -} - -//from https://github.com/golang/go -type ipStackCapabilities struct { - sync.Once // guards following - ipv4Enabled bool - ipv6Enabled bool - ipv4MappedIPv6Enabled bool -} - -//from https://github.com/golang/go -var ipStackCaps ipStackCapabilities - -//from https://github.com/golang/go -// supportsIPv4 reports whether the platform supports IPv4 networking -// functionality. -func supportsIPv4() bool { - ipStackCaps.Once.Do(ipStackCaps.probe) - return ipStackCaps.ipv4Enabled -} - -//from https://github.com/golang/go -// supportsIPv6 reports whether the platform supports IPv6 networking -// functionality. -func supportsIPv6() bool { - ipStackCaps.Once.Do(ipStackCaps.probe) - return ipStackCaps.ipv6Enabled -} - -//from https://github.com/golang/go -// supportsIPv4map reports whether the platform supports mapping an -// IPv4 address inside an IPv6 address at transport layer -// protocols. See RFC 4291, RFC 4038 and RFC 3493. -func supportsIPv4map() bool { - ipStackCaps.Once.Do(ipStackCaps.probe) - return ipStackCaps.ipv4MappedIPv6Enabled -} - -//from https://github.com/golang/go -// Probe probes IPv4, IPv6 and IPv4-mapped IPv6 communication -// capabilities which are controlled by the IPV6_V6ONLY socket option -// and kernel configuration. -// -// Should we try to use the IPv4 socket interface if we're only -// dealing with IPv4 sockets? As long as the host system understands -// IPv4-mapped IPv6, it's okay to pass IPv4-mapeed IPv6 addresses to -// the IPv6 interface. That simplifies our code and is most -// general. Unfortunately, we need to run on kernels built without -// IPv6 support too. So probe the kernel to figure it out. -func (p *ipStackCapabilities) probe() { - s, err := syscall.Socket(syscall.AF_INET, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) - switch err { - case syscall.EAFNOSUPPORT, syscall.EPROTONOSUPPORT: - case nil: - syscall.Close(s) - p.ipv4Enabled = true - } - var probes = []struct { - laddr net.TCPAddr - value int - }{ - // IPv6 communication capability - {laddr: net.TCPAddr{IP: net.IPv6loopback}, value: 1}, - // IPv4-mapped IPv6 address communication capability - {laddr: net.TCPAddr{IP: net.IPv4(127, 0, 0, 1)}, value: 0}, - } - - for i := range probes { - s, err := syscall.Socket(syscall.AF_INET6, syscall.SOCK_STREAM, syscall.IPPROTO_TCP) - if err != nil { - continue - } - defer syscall.Close(s) - syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, probes[i].value) - sa, err := sockaddr(&(probes[i].laddr), syscall.AF_INET6) - if err != nil { - continue - } - if err := syscall.Bind(s, sa); err != nil { - continue - } - if i == 0 { - p.ipv6Enabled = true - } else { - p.ipv4MappedIPv6Enabled = true - } - } -} - -//from https://github.com/golang/go -//Change: we check the first IP address in the list of candidate SCTP IP addresses -func (a *SCTPAddr) isWildcard() bool { - if a == nil { - return true - } - if 0 == len(a.IPAddrs) { - return true - } - - return a.IPAddrs[0].IP.IsUnspecified() -} - -func (a *SCTPAddr) family() int { - if a != nil { - for _, ip := range a.IPAddrs { - if ip.IP.To4() == nil { - return syscall.AF_INET6 - } - } - } - return syscall.AF_INET -} - -//from https://github.com/golang/go -func favoriteAddrFamily(network string, laddr *SCTPAddr, raddr *SCTPAddr, mode string) (family int, ipv6only bool) { - switch network[len(network)-1] { - case '4': - return syscall.AF_INET, false - case '6': - return syscall.AF_INET6, true - } - - if mode == "listen" && (laddr == nil || laddr.isWildcard()) { - if supportsIPv4map() || !supportsIPv4() { - return syscall.AF_INET6, false - } - if laddr == nil { - return syscall.AF_INET, false - } - return laddr.family(), false - } - - if (laddr == nil || laddr.family() == syscall.AF_INET) && - (raddr == nil || raddr.family() == syscall.AF_INET) { - return syscall.AF_INET, false - } - return syscall.AF_INET6, false -} - -//from https://github.com/golang/go -//Changes: it is for SCTP only -func setDefaultSockopts(s int, family int, ipv6only bool) error { - if family == syscall.AF_INET6 { - // Allow both IP versions even if the OS default - // is otherwise. Note that some operating systems - // never admit this option. - syscall.SetsockoptInt(s, syscall.IPPROTO_IPV6, syscall.IPV6_V6ONLY, boolint(ipv6only)) - } - // Allow broadcast. - return os.NewSyscallError("setsockopt", syscall.SetsockoptInt(s, syscall.SOL_SOCKET, syscall.SO_BROADCAST, 1)) -} diff --git a/vendor/github.com/ishidawataru/sctp/sctp.go b/vendor/github.com/ishidawataru/sctp/sctp.go deleted file mode 100644 index 94842f427..000000000 --- a/vendor/github.com/ishidawataru/sctp/sctp.go +++ /dev/null @@ -1,729 +0,0 @@ -// Copyright 2019 Wataru Ishida. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sctp - -import ( - "bytes" - "encoding/binary" - "fmt" - "net" - "strconv" - "strings" - "sync" - "sync/atomic" - "syscall" - "time" - "unsafe" -) - -const ( - SOL_SCTP = 132 - - SCTP_BINDX_ADD_ADDR = 0x01 - SCTP_BINDX_REM_ADDR = 0x02 - - MSG_NOTIFICATION = 0x8000 -) - -const ( - SCTP_RTOINFO = iota - SCTP_ASSOCINFO - SCTP_INITMSG - SCTP_NODELAY - SCTP_AUTOCLOSE - SCTP_SET_PEER_PRIMARY_ADDR - SCTP_PRIMARY_ADDR - SCTP_ADAPTATION_LAYER - SCTP_DISABLE_FRAGMENTS - SCTP_PEER_ADDR_PARAMS - SCTP_DEFAULT_SENT_PARAM - SCTP_EVENTS - SCTP_I_WANT_MAPPED_V4_ADDR - SCTP_MAXSEG - SCTP_STATUS - SCTP_GET_PEER_ADDR_INFO - SCTP_DELAYED_ACK_TIME - SCTP_DELAYED_ACK = SCTP_DELAYED_ACK_TIME - SCTP_DELAYED_SACK = SCTP_DELAYED_ACK_TIME - - SCTP_SOCKOPT_BINDX_ADD = 100 - SCTP_SOCKOPT_BINDX_REM = 101 - SCTP_SOCKOPT_PEELOFF = 102 - SCTP_GET_PEER_ADDRS = 108 - SCTP_GET_LOCAL_ADDRS = 109 - SCTP_SOCKOPT_CONNECTX = 110 - SCTP_SOCKOPT_CONNECTX3 = 111 -) - -const ( - SCTP_EVENT_DATA_IO = 1 << iota - SCTP_EVENT_ASSOCIATION - SCTP_EVENT_ADDRESS - SCTP_EVENT_SEND_FAILURE - SCTP_EVENT_PEER_ERROR - SCTP_EVENT_SHUTDOWN - SCTP_EVENT_PARTIAL_DELIVERY - SCTP_EVENT_ADAPTATION_LAYER - SCTP_EVENT_AUTHENTICATION - SCTP_EVENT_SENDER_DRY - - SCTP_EVENT_ALL = SCTP_EVENT_DATA_IO | SCTP_EVENT_ASSOCIATION | SCTP_EVENT_ADDRESS | SCTP_EVENT_SEND_FAILURE | SCTP_EVENT_PEER_ERROR | SCTP_EVENT_SHUTDOWN | SCTP_EVENT_PARTIAL_DELIVERY | SCTP_EVENT_ADAPTATION_LAYER | SCTP_EVENT_AUTHENTICATION | SCTP_EVENT_SENDER_DRY -) - -type SCTPNotificationType int - -const ( - SCTP_SN_TYPE_BASE = SCTPNotificationType(iota + (1 << 15)) - SCTP_ASSOC_CHANGE - SCTP_PEER_ADDR_CHANGE - SCTP_SEND_FAILED - SCTP_REMOTE_ERROR - SCTP_SHUTDOWN_EVENT - SCTP_PARTIAL_DELIVERY_EVENT - SCTP_ADAPTATION_INDICATION - SCTP_AUTHENTICATION_INDICATION - SCTP_SENDER_DRY_EVENT -) - -type NotificationHandler func([]byte) error - -type EventSubscribe struct { - DataIO uint8 - Association uint8 - Address uint8 - SendFailure uint8 - PeerError uint8 - Shutdown uint8 - PartialDelivery uint8 - AdaptationLayer uint8 - Authentication uint8 - SenderDry uint8 -} - -const ( - SCTP_CMSG_INIT = iota - SCTP_CMSG_SNDRCV - SCTP_CMSG_SNDINFO - SCTP_CMSG_RCVINFO - SCTP_CMSG_NXTINFO -) - -const ( - SCTP_UNORDERED = 1 << iota - SCTP_ADDR_OVER - SCTP_ABORT - SCTP_SACK_IMMEDIATELY - SCTP_EOF -) - -const ( - SCTP_MAX_STREAM = 0xffff -) - -type InitMsg struct { - NumOstreams uint16 - MaxInstreams uint16 - MaxAttempts uint16 - MaxInitTimeout uint16 -} - -type SndRcvInfo struct { - Stream uint16 - SSN uint16 - Flags uint16 - _ uint16 - PPID uint32 - Context uint32 - TTL uint32 - TSN uint32 - CumTSN uint32 - AssocID int32 -} - -type SndInfo struct { - SID uint16 - Flags uint16 - PPID uint32 - Context uint32 - AssocID int32 -} - -type GetAddrsOld struct { - AssocID int32 - AddrNum int32 - Addrs uintptr -} - -type NotificationHeader struct { - Type uint16 - Flags uint16 - Length uint32 -} - -type SCTPState uint16 - -const ( - SCTP_COMM_UP = SCTPState(iota) - SCTP_COMM_LOST - SCTP_RESTART - SCTP_SHUTDOWN_COMP - SCTP_CANT_STR_ASSOC -) - -var nativeEndian binary.ByteOrder -var sndRcvInfoSize uintptr - -func init() { - i := uint16(1) - if *(*byte)(unsafe.Pointer(&i)) == 0 { - nativeEndian = binary.BigEndian - } else { - nativeEndian = binary.LittleEndian - } - info := SndRcvInfo{} - sndRcvInfoSize = unsafe.Sizeof(info) -} - -func toBuf(v interface{}) []byte { - var buf bytes.Buffer - binary.Write(&buf, nativeEndian, v) - return buf.Bytes() -} - -func htons(h uint16) uint16 { - if nativeEndian == binary.LittleEndian { - return (h << 8 & 0xff00) | (h >> 8 & 0xff) - } - return h -} - -var ntohs = htons - -// setInitOpts sets options for an SCTP association initialization -// see https://tools.ietf.org/html/rfc4960#page-25 -func setInitOpts(fd int, options InitMsg) error { - optlen := unsafe.Sizeof(options) - _, _, err := setsockopt(fd, SCTP_INITMSG, uintptr(unsafe.Pointer(&options)), uintptr(optlen)) - return err -} - -func setNumOstreams(fd, num int) error { - return setInitOpts(fd, InitMsg{NumOstreams: uint16(num)}) -} - -type SCTPAddr struct { - IPAddrs []net.IPAddr - Port int -} - -func (a *SCTPAddr) ToRawSockAddrBuf() []byte { - p := htons(uint16(a.Port)) - if len(a.IPAddrs) == 0 { // if a.IPAddrs list is empty - fall back to IPv4 zero addr - s := syscall.RawSockaddrInet4{ - Family: syscall.AF_INET, - Port: p, - } - copy(s.Addr[:], net.IPv4zero) - return toBuf(s) - } - buf := []byte{} - for _, ip := range a.IPAddrs { - ipBytes := ip.IP - if len(ipBytes) == 0 { - ipBytes = net.IPv4zero - } - if ip4 := ipBytes.To4(); ip4 != nil { - s := syscall.RawSockaddrInet4{ - Family: syscall.AF_INET, - Port: p, - } - copy(s.Addr[:], ip4) - buf = append(buf, toBuf(s)...) - } else { - var scopeid uint32 - ifi, err := net.InterfaceByName(ip.Zone) - if err == nil { - scopeid = uint32(ifi.Index) - } - s := syscall.RawSockaddrInet6{ - Family: syscall.AF_INET6, - Port: p, - Scope_id: scopeid, - } - copy(s.Addr[:], ipBytes) - buf = append(buf, toBuf(s)...) - } - } - return buf -} - -func (a *SCTPAddr) String() string { - var b bytes.Buffer - - for n, i := range a.IPAddrs { - if i.IP.To4() != nil { - b.WriteString(i.String()) - } else if i.IP.To16() != nil { - b.WriteRune('[') - b.WriteString(i.String()) - b.WriteRune(']') - } - if n < len(a.IPAddrs)-1 { - b.WriteRune('/') - } - } - b.WriteRune(':') - b.WriteString(strconv.Itoa(a.Port)) - return b.String() -} - -func (a *SCTPAddr) Network() string { return "sctp" } - -func ResolveSCTPAddr(network, addrs string) (*SCTPAddr, error) { - tcpnet := "" - switch network { - case "", "sctp": - tcpnet = "tcp" - case "sctp4": - tcpnet = "tcp4" - case "sctp6": - tcpnet = "tcp6" - default: - return nil, fmt.Errorf("invalid net: %s", network) - } - elems := strings.Split(addrs, "/") - if len(elems) == 0 { - return nil, fmt.Errorf("invalid input: %s", addrs) - } - ipaddrs := make([]net.IPAddr, 0, len(elems)) - for _, e := range elems[:len(elems)-1] { - tcpa, err := net.ResolveTCPAddr(tcpnet, e+":") - if err != nil { - return nil, err - } - ipaddrs = append(ipaddrs, net.IPAddr{IP: tcpa.IP, Zone: tcpa.Zone}) - } - tcpa, err := net.ResolveTCPAddr(tcpnet, elems[len(elems)-1]) - if err != nil { - return nil, err - } - if tcpa.IP != nil { - ipaddrs = append(ipaddrs, net.IPAddr{IP: tcpa.IP, Zone: tcpa.Zone}) - } else { - ipaddrs = nil - } - return &SCTPAddr{ - IPAddrs: ipaddrs, - Port: tcpa.Port, - }, nil -} - -func SCTPConnect(fd int, addr *SCTPAddr) (int, error) { - buf := addr.ToRawSockAddrBuf() - param := GetAddrsOld{ - AddrNum: int32(len(buf)), - Addrs: uintptr(uintptr(unsafe.Pointer(&buf[0]))), - } - optlen := unsafe.Sizeof(param) - _, _, err := getsockopt(fd, SCTP_SOCKOPT_CONNECTX3, uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) - if err == nil { - return int(param.AssocID), nil - } else if err != syscall.ENOPROTOOPT { - return 0, err - } - r0, _, err := setsockopt(fd, SCTP_SOCKOPT_CONNECTX, uintptr(unsafe.Pointer(&buf[0])), uintptr(len(buf))) - return int(r0), err -} - -func SCTPBind(fd int, addr *SCTPAddr, flags int) error { - var option uintptr - switch flags { - case SCTP_BINDX_ADD_ADDR: - option = SCTP_SOCKOPT_BINDX_ADD - case SCTP_BINDX_REM_ADDR: - option = SCTP_SOCKOPT_BINDX_REM - default: - return syscall.EINVAL - } - - buf := addr.ToRawSockAddrBuf() - _, _, err := setsockopt(fd, option, uintptr(unsafe.Pointer(&buf[0])), uintptr(len(buf))) - return err -} - -type SCTPConn struct { - _fd int32 - notificationHandler NotificationHandler -} - -func (c *SCTPConn) fd() int { - return int(atomic.LoadInt32(&c._fd)) -} - -func NewSCTPConn(fd int, handler NotificationHandler) *SCTPConn { - conn := &SCTPConn{ - _fd: int32(fd), - notificationHandler: handler, - } - return conn -} - -func (c *SCTPConn) Write(b []byte) (int, error) { - return c.SCTPWrite(b, nil) -} - -func (c *SCTPConn) Read(b []byte) (int, error) { - n, _, err := c.SCTPRead(b) - if n < 0 { - n = 0 - } - return n, err -} - -func (c *SCTPConn) SetInitMsg(numOstreams, maxInstreams, maxAttempts, maxInitTimeout int) error { - return setInitOpts(c.fd(), InitMsg{ - NumOstreams: uint16(numOstreams), - MaxInstreams: uint16(maxInstreams), - MaxAttempts: uint16(maxAttempts), - MaxInitTimeout: uint16(maxInitTimeout), - }) -} - -func (c *SCTPConn) SubscribeEvents(flags int) error { - var d, a, ad, sf, p, sh, pa, ada, au, se uint8 - if flags&SCTP_EVENT_DATA_IO > 0 { - d = 1 - } - if flags&SCTP_EVENT_ASSOCIATION > 0 { - a = 1 - } - if flags&SCTP_EVENT_ADDRESS > 0 { - ad = 1 - } - if flags&SCTP_EVENT_SEND_FAILURE > 0 { - sf = 1 - } - if flags&SCTP_EVENT_PEER_ERROR > 0 { - p = 1 - } - if flags&SCTP_EVENT_SHUTDOWN > 0 { - sh = 1 - } - if flags&SCTP_EVENT_PARTIAL_DELIVERY > 0 { - pa = 1 - } - if flags&SCTP_EVENT_ADAPTATION_LAYER > 0 { - ada = 1 - } - if flags&SCTP_EVENT_AUTHENTICATION > 0 { - au = 1 - } - if flags&SCTP_EVENT_SENDER_DRY > 0 { - se = 1 - } - param := EventSubscribe{ - DataIO: d, - Association: a, - Address: ad, - SendFailure: sf, - PeerError: p, - Shutdown: sh, - PartialDelivery: pa, - AdaptationLayer: ada, - Authentication: au, - SenderDry: se, - } - optlen := unsafe.Sizeof(param) - _, _, err := setsockopt(c.fd(), SCTP_EVENTS, uintptr(unsafe.Pointer(¶m)), uintptr(optlen)) - return err -} - -func (c *SCTPConn) SubscribedEvents() (int, error) { - param := EventSubscribe{} - optlen := unsafe.Sizeof(param) - _, _, err := getsockopt(c.fd(), SCTP_EVENTS, uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) - if err != nil { - return 0, err - } - var flags int - if param.DataIO > 0 { - flags |= SCTP_EVENT_DATA_IO - } - if param.Association > 0 { - flags |= SCTP_EVENT_ASSOCIATION - } - if param.Address > 0 { - flags |= SCTP_EVENT_ADDRESS - } - if param.SendFailure > 0 { - flags |= SCTP_EVENT_SEND_FAILURE - } - if param.PeerError > 0 { - flags |= SCTP_EVENT_PEER_ERROR - } - if param.Shutdown > 0 { - flags |= SCTP_EVENT_SHUTDOWN - } - if param.PartialDelivery > 0 { - flags |= SCTP_EVENT_PARTIAL_DELIVERY - } - if param.AdaptationLayer > 0 { - flags |= SCTP_EVENT_ADAPTATION_LAYER - } - if param.Authentication > 0 { - flags |= SCTP_EVENT_AUTHENTICATION - } - if param.SenderDry > 0 { - flags |= SCTP_EVENT_SENDER_DRY - } - return flags, nil -} - -func (c *SCTPConn) SetDefaultSentParam(info *SndRcvInfo) error { - optlen := unsafe.Sizeof(*info) - _, _, err := setsockopt(c.fd(), SCTP_DEFAULT_SENT_PARAM, uintptr(unsafe.Pointer(info)), uintptr(optlen)) - return err -} - -func (c *SCTPConn) GetDefaultSentParam() (*SndRcvInfo, error) { - info := &SndRcvInfo{} - optlen := unsafe.Sizeof(*info) - _, _, err := getsockopt(c.fd(), SCTP_DEFAULT_SENT_PARAM, uintptr(unsafe.Pointer(info)), uintptr(unsafe.Pointer(&optlen))) - return info, err -} - -func resolveFromRawAddr(ptr unsafe.Pointer, n int) (*SCTPAddr, error) { - addr := &SCTPAddr{ - IPAddrs: make([]net.IPAddr, n), - } - - switch family := (*(*syscall.RawSockaddrAny)(ptr)).Addr.Family; family { - case syscall.AF_INET: - addr.Port = int(ntohs(uint16((*(*syscall.RawSockaddrInet4)(ptr)).Port))) - tmp := syscall.RawSockaddrInet4{} - size := unsafe.Sizeof(tmp) - for i := 0; i < n; i++ { - a := *(*syscall.RawSockaddrInet4)(unsafe.Pointer( - uintptr(ptr) + size*uintptr(i))) - addr.IPAddrs[i] = net.IPAddr{IP: a.Addr[:]} - } - case syscall.AF_INET6: - addr.Port = int(ntohs(uint16((*(*syscall.RawSockaddrInet4)(ptr)).Port))) - tmp := syscall.RawSockaddrInet6{} - size := unsafe.Sizeof(tmp) - for i := 0; i < n; i++ { - a := *(*syscall.RawSockaddrInet6)(unsafe.Pointer( - uintptr(ptr) + size*uintptr(i))) - var zone string - ifi, err := net.InterfaceByIndex(int(a.Scope_id)) - if err == nil { - zone = ifi.Name - } - addr.IPAddrs[i] = net.IPAddr{IP: a.Addr[:], Zone: zone} - } - default: - return nil, fmt.Errorf("unknown address family: %d", family) - } - return addr, nil -} - -func sctpGetAddrs(fd, id, optname int) (*SCTPAddr, error) { - - type getaddrs struct { - assocId int32 - addrNum uint32 - addrs [4096]byte - } - param := getaddrs{ - assocId: int32(id), - } - optlen := unsafe.Sizeof(param) - _, _, err := getsockopt(fd, uintptr(optname), uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) - if err != nil { - return nil, err - } - return resolveFromRawAddr(unsafe.Pointer(¶m.addrs), int(param.addrNum)) -} - -func (c *SCTPConn) SCTPGetPrimaryPeerAddr() (*SCTPAddr, error) { - - type sctpGetSetPrim struct { - assocId int32 - addrs [128]byte - } - param := sctpGetSetPrim{ - assocId: int32(0), - } - optlen := unsafe.Sizeof(param) - _, _, err := getsockopt(c.fd(), SCTP_PRIMARY_ADDR, uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) - if err != nil { - return nil, err - } - return resolveFromRawAddr(unsafe.Pointer(¶m.addrs), 1) -} - -func (c *SCTPConn) SCTPLocalAddr(id int) (*SCTPAddr, error) { - return sctpGetAddrs(c.fd(), id, SCTP_GET_LOCAL_ADDRS) -} - -func (c *SCTPConn) SCTPRemoteAddr(id int) (*SCTPAddr, error) { - return sctpGetAddrs(c.fd(), id, SCTP_GET_PEER_ADDRS) -} - -func (c *SCTPConn) LocalAddr() net.Addr { - addr, err := sctpGetAddrs(c.fd(), 0, SCTP_GET_LOCAL_ADDRS) - if err != nil { - return nil - } - return addr -} - -func (c *SCTPConn) RemoteAddr() net.Addr { - addr, err := sctpGetAddrs(c.fd(), 0, SCTP_GET_PEER_ADDRS) - if err != nil { - return nil - } - return addr -} - -func (c *SCTPConn) PeelOff(id int) (*SCTPConn, error) { - type peeloffArg struct { - assocId int32 - sd int - } - param := peeloffArg{ - assocId: int32(id), - } - optlen := unsafe.Sizeof(param) - _, _, err := getsockopt(c.fd(), SCTP_SOCKOPT_PEELOFF, uintptr(unsafe.Pointer(¶m)), uintptr(unsafe.Pointer(&optlen))) - if err != nil { - return nil, err - } - return &SCTPConn{_fd: int32(param.sd)}, nil -} - -func (c *SCTPConn) SetDeadline(t time.Time) error { - return syscall.EOPNOTSUPP -} - -func (c *SCTPConn) SetReadDeadline(t time.Time) error { - return syscall.EOPNOTSUPP -} - -func (c *SCTPConn) SetWriteDeadline(t time.Time) error { - return syscall.EOPNOTSUPP -} - -type SCTPListener struct { - fd int - m sync.Mutex -} - -func (ln *SCTPListener) Addr() net.Addr { - laddr, err := sctpGetAddrs(ln.fd, 0, SCTP_GET_LOCAL_ADDRS) - if err != nil { - return nil - } - return laddr -} - -type SCTPSndRcvInfoWrappedConn struct { - conn *SCTPConn -} - -func NewSCTPSndRcvInfoWrappedConn(conn *SCTPConn) *SCTPSndRcvInfoWrappedConn { - conn.SubscribeEvents(SCTP_EVENT_DATA_IO) - return &SCTPSndRcvInfoWrappedConn{conn} -} - -func (c *SCTPSndRcvInfoWrappedConn) Write(b []byte) (int, error) { - if len(b) < int(sndRcvInfoSize) { - return 0, syscall.EINVAL - } - info := (*SndRcvInfo)(unsafe.Pointer(&b[0])) - n, err := c.conn.SCTPWrite(b[sndRcvInfoSize:], info) - return n + int(sndRcvInfoSize), err -} - -func (c *SCTPSndRcvInfoWrappedConn) Read(b []byte) (int, error) { - if len(b) < int(sndRcvInfoSize) { - return 0, syscall.EINVAL - } - n, info, err := c.conn.SCTPRead(b[sndRcvInfoSize:]) - if err != nil { - return n, err - } - copy(b, toBuf(info)) - return n + int(sndRcvInfoSize), err -} - -func (c *SCTPSndRcvInfoWrappedConn) Close() error { - return c.conn.Close() -} - -func (c *SCTPSndRcvInfoWrappedConn) LocalAddr() net.Addr { - return c.conn.LocalAddr() -} - -func (c *SCTPSndRcvInfoWrappedConn) RemoteAddr() net.Addr { - return c.conn.RemoteAddr() -} - -func (c *SCTPSndRcvInfoWrappedConn) SetDeadline(t time.Time) error { - return c.conn.SetDeadline(t) -} - -func (c *SCTPSndRcvInfoWrappedConn) SetReadDeadline(t time.Time) error { - return c.conn.SetReadDeadline(t) -} - -func (c *SCTPSndRcvInfoWrappedConn) SetWriteDeadline(t time.Time) error { - return c.conn.SetWriteDeadline(t) -} - -func (c *SCTPSndRcvInfoWrappedConn) SetWriteBuffer(bytes int) error { - return c.conn.SetWriteBuffer(bytes) -} - -func (c *SCTPSndRcvInfoWrappedConn) GetWriteBuffer() (int, error) { - return c.conn.GetWriteBuffer() -} - -func (c *SCTPSndRcvInfoWrappedConn) SetReadBuffer(bytes int) error { - return c.conn.SetReadBuffer(bytes) -} - -func (c *SCTPSndRcvInfoWrappedConn) GetReadBuffer() (int, error) { - return c.conn.GetReadBuffer() -} - -// SocketConfig contains options for the SCTP socket. -type SocketConfig struct { - // If Control is not nil it is called after the socket is created but before - // it is bound or connected. - Control func(network, address string, c syscall.RawConn) error - - // InitMsg is the options to send in the initial SCTP message - InitMsg InitMsg -} - -func (cfg *SocketConfig) Listen(net string, laddr *SCTPAddr) (*SCTPListener, error) { - return listenSCTPExtConfig(net, laddr, cfg.InitMsg, cfg.Control) -} - -func (cfg *SocketConfig) Dial(net string, laddr, raddr *SCTPAddr) (*SCTPConn, error) { - return dialSCTPExtConfig(net, laddr, raddr, cfg.InitMsg, cfg.Control) -} diff --git a/vendor/github.com/ishidawataru/sctp/sctp_linux.go b/vendor/github.com/ishidawataru/sctp/sctp_linux.go deleted file mode 100644 index d96d09e5c..000000000 --- a/vendor/github.com/ishidawataru/sctp/sctp_linux.go +++ /dev/null @@ -1,305 +0,0 @@ -// +build linux,!386 -// Copyright 2019 Wataru Ishida. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sctp - -import ( - "io" - "net" - "sync/atomic" - "syscall" - "unsafe" -) - -func setsockopt(fd int, optname, optval, optlen uintptr) (uintptr, uintptr, error) { - // FIXME: syscall.SYS_SETSOCKOPT is undefined on 386 - r0, r1, errno := syscall.Syscall6(syscall.SYS_SETSOCKOPT, - uintptr(fd), - SOL_SCTP, - optname, - optval, - optlen, - 0) - if errno != 0 { - return r0, r1, errno - } - return r0, r1, nil -} - -func getsockopt(fd int, optname, optval, optlen uintptr) (uintptr, uintptr, error) { - // FIXME: syscall.SYS_GETSOCKOPT is undefined on 386 - r0, r1, errno := syscall.Syscall6(syscall.SYS_GETSOCKOPT, - uintptr(fd), - SOL_SCTP, - optname, - optval, - optlen, - 0) - if errno != 0 { - return r0, r1, errno - } - return r0, r1, nil -} - -type rawConn struct { - sockfd int -} - -func (r rawConn) Control(f func(fd uintptr)) error { - f(uintptr(r.sockfd)) - return nil -} - -func (r rawConn) Read(f func(fd uintptr) (done bool)) error { - panic("not implemented") -} - -func (r rawConn) Write(f func(fd uintptr) (done bool)) error { - panic("not implemented") -} - -func (c *SCTPConn) SCTPWrite(b []byte, info *SndRcvInfo) (int, error) { - var cbuf []byte - if info != nil { - cmsgBuf := toBuf(info) - hdr := &syscall.Cmsghdr{ - Level: syscall.IPPROTO_SCTP, - Type: SCTP_CMSG_SNDRCV, - } - - // bitwidth of hdr.Len is platform-specific, - // so we use hdr.SetLen() rather than directly setting hdr.Len - hdr.SetLen(syscall.CmsgSpace(len(cmsgBuf))) - cbuf = append(toBuf(hdr), cmsgBuf...) - } - return syscall.SendmsgN(c.fd(), b, cbuf, nil, 0) -} - -func parseSndRcvInfo(b []byte) (*SndRcvInfo, error) { - msgs, err := syscall.ParseSocketControlMessage(b) - if err != nil { - return nil, err - } - for _, m := range msgs { - if m.Header.Level == syscall.IPPROTO_SCTP { - switch m.Header.Type { - case SCTP_CMSG_SNDRCV: - return (*SndRcvInfo)(unsafe.Pointer(&m.Data[0])), nil - } - } - } - return nil, nil -} - -func (c *SCTPConn) SCTPRead(b []byte) (int, *SndRcvInfo, error) { - oob := make([]byte, 254) - for { - n, oobn, recvflags, _, err := syscall.Recvmsg(c.fd(), b, oob, 0) - if err != nil { - return n, nil, err - } - - if n == 0 && oobn == 0 { - return 0, nil, io.EOF - } - - if recvflags&MSG_NOTIFICATION > 0 && c.notificationHandler != nil { - if err := c.notificationHandler(b[:n]); err != nil { - return 0, nil, err - } - } else { - var info *SndRcvInfo - if oobn > 0 { - info, err = parseSndRcvInfo(oob[:oobn]) - } - return n, info, err - } - } -} - -func (c *SCTPConn) Close() error { - if c != nil { - fd := atomic.SwapInt32(&c._fd, -1) - if fd > 0 { - info := &SndRcvInfo{ - Flags: SCTP_EOF, - } - c.SCTPWrite(nil, info) - syscall.Shutdown(int(fd), syscall.SHUT_RDWR) - return syscall.Close(int(fd)) - } - } - return syscall.EBADF -} - -func (c *SCTPConn) SetWriteBuffer(bytes int) error { - return syscall.SetsockoptInt(c.fd(), syscall.SOL_SOCKET, syscall.SO_SNDBUF, bytes) -} - -func (c *SCTPConn) GetWriteBuffer() (int, error) { - return syscall.GetsockoptInt(c.fd(), syscall.SOL_SOCKET, syscall.SO_SNDBUF) -} - -func (c *SCTPConn) SetReadBuffer(bytes int) error { - return syscall.SetsockoptInt(c.fd(), syscall.SOL_SOCKET, syscall.SO_RCVBUF, bytes) -} - -func (c *SCTPConn) GetReadBuffer() (int, error) { - return syscall.GetsockoptInt(c.fd(), syscall.SOL_SOCKET, syscall.SO_RCVBUF) -} - -// ListenSCTP - start listener on specified address/port -func ListenSCTP(net string, laddr *SCTPAddr) (*SCTPListener, error) { - return ListenSCTPExt(net, laddr, InitMsg{NumOstreams: SCTP_MAX_STREAM}) -} - -// ListenSCTPExt - start listener on specified address/port with given SCTP options -func ListenSCTPExt(network string, laddr *SCTPAddr, options InitMsg) (*SCTPListener, error) { - return listenSCTPExtConfig(network, laddr, options, nil) -} - -// listenSCTPExtConfig - start listener on specified address/port with given SCTP options and socket configuration -func listenSCTPExtConfig(network string, laddr *SCTPAddr, options InitMsg, control func(network, address string, c syscall.RawConn) error) (*SCTPListener, error) { - af, ipv6only := favoriteAddrFamily(network, laddr, nil, "listen") - sock, err := syscall.Socket( - af, - syscall.SOCK_STREAM, - syscall.IPPROTO_SCTP, - ) - if err != nil { - return nil, err - } - - // close socket on error - defer func() { - if err != nil { - syscall.Close(sock) - } - }() - if err = setDefaultSockopts(sock, af, ipv6only); err != nil { - return nil, err - } - if control != nil { - rc := rawConn{sockfd: sock} - if err = control(network, laddr.String(), rc); err != nil { - return nil, err - } - } - err = setInitOpts(sock, options) - if err != nil { - return nil, err - } - - if laddr != nil { - // If IP address and/or port was not provided so far, let's use the unspecified IPv4 or IPv6 address - if len(laddr.IPAddrs) == 0 { - if af == syscall.AF_INET { - laddr.IPAddrs = append(laddr.IPAddrs, net.IPAddr{IP: net.IPv4zero}) - } else if af == syscall.AF_INET6 { - laddr.IPAddrs = append(laddr.IPAddrs, net.IPAddr{IP: net.IPv6zero}) - } - } - err = SCTPBind(sock, laddr, SCTP_BINDX_ADD_ADDR) - if err != nil { - return nil, err - } - } - err = syscall.Listen(sock, syscall.SOMAXCONN) - if err != nil { - return nil, err - } - return &SCTPListener{ - fd: sock, - }, nil -} - -// AcceptSCTP waits for and returns the next SCTP connection to the listener. -func (ln *SCTPListener) AcceptSCTP() (*SCTPConn, error) { - fd, _, err := syscall.Accept4(ln.fd, 0) - return NewSCTPConn(fd, nil), err -} - -// Accept waits for and returns the next connection connection to the listener. -func (ln *SCTPListener) Accept() (net.Conn, error) { - return ln.AcceptSCTP() -} - -func (ln *SCTPListener) Close() error { - syscall.Shutdown(ln.fd, syscall.SHUT_RDWR) - return syscall.Close(ln.fd) -} - -// DialSCTP - bind socket to laddr (if given) and connect to raddr -func DialSCTP(net string, laddr, raddr *SCTPAddr) (*SCTPConn, error) { - return DialSCTPExt(net, laddr, raddr, InitMsg{NumOstreams: SCTP_MAX_STREAM}) -} - -// DialSCTPExt - same as DialSCTP but with given SCTP options -func DialSCTPExt(network string, laddr, raddr *SCTPAddr, options InitMsg) (*SCTPConn, error) { - return dialSCTPExtConfig(network, laddr, raddr, options, nil) -} - -// dialSCTPExtConfig - same as DialSCTP but with given SCTP options and socket configuration -func dialSCTPExtConfig(network string, laddr, raddr *SCTPAddr, options InitMsg, control func(network, address string, c syscall.RawConn) error) (*SCTPConn, error) { - af, ipv6only := favoriteAddrFamily(network, laddr, raddr, "dial") - sock, err := syscall.Socket( - af, - syscall.SOCK_STREAM, - syscall.IPPROTO_SCTP, - ) - if err != nil { - return nil, err - } - - // close socket on error - defer func() { - if err != nil { - syscall.Close(sock) - } - }() - if err = setDefaultSockopts(sock, af, ipv6only); err != nil { - return nil, err - } - if control != nil { - rc := rawConn{sockfd: sock} - if err = control(network, laddr.String(), rc); err != nil { - return nil, err - } - } - err = setInitOpts(sock, options) - if err != nil { - return nil, err - } - if laddr != nil { - // If IP address and/or port was not provided so far, let's use the unspecified IPv4 or IPv6 address - if len(laddr.IPAddrs) == 0 { - if af == syscall.AF_INET { - laddr.IPAddrs = append(laddr.IPAddrs, net.IPAddr{IP: net.IPv4zero}) - } else if af == syscall.AF_INET6 { - laddr.IPAddrs = append(laddr.IPAddrs, net.IPAddr{IP: net.IPv6zero}) - } - } - err := SCTPBind(sock, laddr, SCTP_BINDX_ADD_ADDR) - if err != nil { - return nil, err - } - } - _, err = SCTPConnect(sock, raddr) - if err != nil { - return nil, err - } - return NewSCTPConn(sock, nil), nil -} diff --git a/vendor/github.com/ishidawataru/sctp/sctp_unsupported.go b/vendor/github.com/ishidawataru/sctp/sctp_unsupported.go deleted file mode 100644 index 118fe159e..000000000 --- a/vendor/github.com/ishidawataru/sctp/sctp_unsupported.go +++ /dev/null @@ -1,98 +0,0 @@ -// +build !linux linux,386 -// Copyright 2019 Wataru Ishida. All rights reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or -// implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package sctp - -import ( - "errors" - "net" - "runtime" - "syscall" -) - -var ErrUnsupported = errors.New("SCTP is unsupported on " + runtime.GOOS + "/" + runtime.GOARCH) - -func setsockopt(fd int, optname, optval, optlen uintptr) (uintptr, uintptr, error) { - return 0, 0, ErrUnsupported -} - -func getsockopt(fd int, optname, optval, optlen uintptr) (uintptr, uintptr, error) { - return 0, 0, ErrUnsupported -} - -func (c *SCTPConn) SCTPWrite(b []byte, info *SndRcvInfo) (int, error) { - return 0, ErrUnsupported -} - -func (c *SCTPConn) SCTPRead(b []byte) (int, *SndRcvInfo, error) { - return 0, nil, ErrUnsupported -} - -func (c *SCTPConn) Close() error { - return ErrUnsupported -} - -func (c *SCTPConn) SetWriteBuffer(bytes int) error { - return ErrUnsupported -} - -func (c *SCTPConn) GetWriteBuffer() (int, error) { - return 0, ErrUnsupported -} - -func (c *SCTPConn) SetReadBuffer(bytes int) error { - return ErrUnsupported -} - -func (c *SCTPConn) GetReadBuffer() (int, error) { - return 0, ErrUnsupported -} - -func ListenSCTP(net string, laddr *SCTPAddr) (*SCTPListener, error) { - return nil, ErrUnsupported -} - -func ListenSCTPExt(net string, laddr *SCTPAddr, options InitMsg) (*SCTPListener, error) { - return nil, ErrUnsupported -} - -func listenSCTPExtConfig(network string, laddr *SCTPAddr, options InitMsg, control func(network, address string, c syscall.RawConn) error) (*SCTPListener, error) { - return nil, ErrUnsupported -} - -func (ln *SCTPListener) Accept() (net.Conn, error) { - return nil, ErrUnsupported -} - -func (ln *SCTPListener) AcceptSCTP() (*SCTPConn, error) { - return nil, ErrUnsupported -} - -func (ln *SCTPListener) Close() error { - return ErrUnsupported -} - -func DialSCTP(net string, laddr, raddr *SCTPAddr) (*SCTPConn, error) { - return nil, ErrUnsupported -} - -func DialSCTPExt(network string, laddr, raddr *SCTPAddr, options InitMsg) (*SCTPConn, error) { - return nil, ErrUnsupported -} - -func dialSCTPExtConfig(network string, laddr, raddr *SCTPAddr, options InitMsg, control func(network, address string, c syscall.RawConn) error) (*SCTPConn, error) { - return nil, ErrUnsupported -} diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index c3ec9d8a7..5c3c2a258 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -17,6 +17,24 @@ This package provides various compression algorithms. # changelog
+* May 25, 2022 (v1.15.5)
+ * s2: Add concurrent stream decompression https://github.com/klauspost/compress/pull/602
+ * s2: Fix final emit oob read crash on amd64 https://github.com/klauspost/compress/pull/601
+ * huff0: asm implementation of Decompress1X by @WojciechMula https://github.com/klauspost/compress/pull/596
+ * zstd: Use 1 less goroutine for stream decoding https://github.com/klauspost/compress/pull/588
+ * zstd: Copy literal in 16 byte blocks when possible https://github.com/klauspost/compress/pull/592
+ * zstd: Speed up when WithDecoderLowmem(false) https://github.com/klauspost/compress/pull/599
+ * zstd: faster next state update in BMI2 version of decode by @WojciechMula in https://github.com/klauspost/compress/pull/593
+ * huff0: Do not check max size when reading table. https://github.com/klauspost/compress/pull/586
+ * flate: Inplace hashing for level 7-9 by @klauspost in https://github.com/klauspost/compress/pull/590
+
+
+* May 11, 2022 (v1.15.4)
+ * huff0: decompress directly into output by @WojciechMula in [#577](https://github.com/klauspost/compress/pull/577)
+ * inflate: Keep dict on stack [#581](https://github.com/klauspost/compress/pull/581)
+ * zstd: Faster decoding memcopy in asm [#583](https://github.com/klauspost/compress/pull/583)
+ * zstd: Fix ignored crc [#580](https://github.com/klauspost/compress/pull/580)
+
* May 5, 2022 (v1.15.3)
* zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572)
* s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575)
@@ -77,6 +95,9 @@ While the release has been extensively tested, it is recommended to testing when * zstd: add arm64 xxhash assembly in [#464](https://github.com/klauspost/compress/pull/464)
* Add garbled for binaries for s2 in [#445](https://github.com/klauspost/compress/pull/445)
+<details>
+ <summary>See changes to v1.13.x</summary>
+
* Aug 30, 2021 (v1.13.5)
* gz/zlib/flate: Alias stdlib errors [#425](https://github.com/klauspost/compress/pull/425)
* s2: Add block support to commandline tools [#413](https://github.com/klauspost/compress/pull/413)
@@ -105,6 +126,8 @@ While the release has been extensively tested, it is recommended to testing when * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors.
* zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382)
* zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380)
+</details>
+
<details>
<summary>See changes to v1.12.x</summary>
diff --git a/vendor/github.com/klauspost/compress/flate/deflate.go b/vendor/github.com/klauspost/compress/flate/deflate.go index bffa2f332..f8435998e 100644 --- a/vendor/github.com/klauspost/compress/flate/deflate.go +++ b/vendor/github.com/klauspost/compress/flate/deflate.go @@ -84,24 +84,23 @@ type advancedState struct { length int offset int maxInsertIndex int + chainHead int + hashOffset int - // Input hash chains - // hashHead[hashValue] contains the largest inputIndex with the specified hash value - // If hashHead[hashValue] is within the current window, then - // hashPrev[hashHead[hashValue] & windowMask] contains the previous index - // with the same hash value. - chainHead int - hashHead [hashSize]uint32 - hashPrev [windowSize]uint32 - hashOffset int + ii uint16 // position of last match, intended to overflow to reset. // input window: unprocessed data is window[index:windowEnd] index int estBitsPerByte int hashMatch [maxMatchLength + minMatchLength]uint32 - hash uint32 - ii uint16 // position of last match, intended to overflow to reset. + // Input hash chains + // hashHead[hashValue] contains the largest inputIndex with the specified hash value + // If hashHead[hashValue] is within the current window, then + // hashPrev[hashHead[hashValue] & windowMask] contains the previous index + // with the same hash value. + hashHead [hashSize]uint32 + hashPrev [windowSize]uint32 } type compressor struct { @@ -259,7 +258,6 @@ func (d *compressor) fillWindow(b []byte) { // Set the head of the hash chain to us. s.hashHead[newH] = uint32(di + s.hashOffset) } - s.hash = newH } // Update window information. d.windowEnd += n @@ -403,7 +401,6 @@ func (d *compressor) initDeflate() { s.hashOffset = 1 s.length = minMatchLength - 1 s.offset = 0 - s.hash = 0 s.chainHead = -1 } @@ -432,9 +429,6 @@ func (d *compressor) deflateLazy() { } s.maxInsertIndex = d.windowEnd - (minMatchLength - 1) - if s.index < s.maxInsertIndex { - s.hash = hash4(d.window[s.index:]) - } for { if sanity && s.index > d.windowEnd { @@ -466,11 +460,11 @@ func (d *compressor) deflateLazy() { } if s.index < s.maxInsertIndex { // Update the hash - s.hash = hash4(d.window[s.index:]) - ch := s.hashHead[s.hash&hashMask] + hash := hash4(d.window[s.index:]) + ch := s.hashHead[hash] s.chainHead = int(ch) s.hashPrev[s.index&windowMask] = ch - s.hashHead[s.hash&hashMask] = uint32(s.index + s.hashOffset) + s.hashHead[hash] = uint32(s.index + s.hashOffset) } prevLength := s.length prevOffset := s.offset @@ -503,7 +497,7 @@ func (d *compressor) deflateLazy() { end += prevIndex idx := prevIndex + prevLength - (4 - checkOff) h := hash4(d.window[idx:]) - ch2 := int(s.hashHead[h&hashMask]) - s.hashOffset - prevLength + (4 - checkOff) + ch2 := int(s.hashHead[h]) - s.hashOffset - prevLength + (4 - checkOff) if ch2 > minIndex { length := matchLen(d.window[prevIndex:end], d.window[ch2:]) // It seems like a pure length metric is best. @@ -547,7 +541,6 @@ func (d *compressor) deflateLazy() { // Set the head of the hash chain to us. s.hashHead[newH] = uint32(di + s.hashOffset) } - s.hash = newH } s.index = newIndex @@ -793,7 +786,6 @@ func (d *compressor) reset(w io.Writer) { d.tokens.Reset() s.length = minMatchLength - 1 s.offset = 0 - s.hash = 0 s.ii = 0 s.maxInsertIndex = 0 } diff --git a/vendor/github.com/klauspost/compress/flate/fast_encoder.go b/vendor/github.com/klauspost/compress/flate/fast_encoder.go index d55ea2a77..f781aaa62 100644 --- a/vendor/github.com/klauspost/compress/flate/fast_encoder.go +++ b/vendor/github.com/klauspost/compress/flate/fast_encoder.go @@ -117,7 +117,7 @@ func (e *fastGen) addBlock(src []byte) int32 { // hash4 returns the hash of u to fit in a hash table with h bits. // Preferably h should be a constant and should always be <32. func hash4u(u uint32, h uint8) uint32 { - return (u * prime4bytes) >> ((32 - h) & reg8SizeMask32) + return (u * prime4bytes) >> (32 - h) } type tableEntryPrev struct { diff --git a/vendor/github.com/klauspost/compress/huff0/bitreader.go b/vendor/github.com/klauspost/compress/huff0/bitreader.go index 451160edd..504a7be9d 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitreader.go +++ b/vendor/github.com/klauspost/compress/huff0/bitreader.go @@ -165,11 +165,6 @@ func (b *bitReaderShifted) peekBitsFast(n uint8) uint16 { return uint16(b.value >> ((64 - n) & 63)) } -// peekTopBits(n) is equvialent to peekBitFast(64 - n) -func (b *bitReaderShifted) peekTopBits(n uint8) uint16 { - return uint16(b.value >> n) -} - func (b *bitReaderShifted) advance(n uint8) { b.bitsRead += n b.value <<= n & 63 @@ -220,11 +215,6 @@ func (b *bitReaderShifted) fill() { } } -// finished returns true if all bits have been read from the bit stream. -func (b *bitReaderShifted) finished() bool { - return b.off == 0 && b.bitsRead >= 64 -} - func (b *bitReaderShifted) remaining() uint { return b.off*8 + uint(64-b.bitsRead) } diff --git a/vendor/github.com/klauspost/compress/huff0/bitwriter.go b/vendor/github.com/klauspost/compress/huff0/bitwriter.go index 6bce4e87d..ec71f7a34 100644 --- a/vendor/github.com/klauspost/compress/huff0/bitwriter.go +++ b/vendor/github.com/klauspost/compress/huff0/bitwriter.go @@ -5,8 +5,6 @@ package huff0 -import "fmt" - // bitWriter will write bits. // First bit will be LSB of the first byte of output. type bitWriter struct { @@ -23,14 +21,6 @@ var bitMask16 = [32]uint16{ 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF, 0xFFFF} /* up to 16 bits */ -// addBits16NC will add up to 16 bits. -// It will not check if there is space for them, -// so the caller must ensure that it has flushed recently. -func (b *bitWriter) addBits16NC(value uint16, bits uint8) { - b.bitContainer |= uint64(value&bitMask16[bits&31]) << (b.nBits & 63) - b.nBits += bits -} - // addBits16Clean will add up to 16 bits. value may not contain more set bits than indicated. // It will not check if there is space for them, so the caller must ensure that it has flushed recently. func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { @@ -70,104 +60,6 @@ func (b *bitWriter) encTwoSymbols(ct cTable, av, bv byte) { b.nBits += encA.nBits + encB.nBits } -// addBits16ZeroNC will add up to 16 bits. -// It will not check if there is space for them, -// so the caller must ensure that it has flushed recently. -// This is fastest if bits can be zero. -func (b *bitWriter) addBits16ZeroNC(value uint16, bits uint8) { - if bits == 0 { - return - } - value <<= (16 - bits) & 15 - value >>= (16 - bits) & 15 - b.bitContainer |= uint64(value) << (b.nBits & 63) - b.nBits += bits -} - -// flush will flush all pending full bytes. -// There will be at least 56 bits available for writing when this has been called. -// Using flush32 is faster, but leaves less space for writing. -func (b *bitWriter) flush() { - v := b.nBits >> 3 - switch v { - case 0: - return - case 1: - b.out = append(b.out, - byte(b.bitContainer), - ) - b.bitContainer >>= 1 << 3 - case 2: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - ) - b.bitContainer >>= 2 << 3 - case 3: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - ) - b.bitContainer >>= 3 << 3 - case 4: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - ) - b.bitContainer >>= 4 << 3 - case 5: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - ) - b.bitContainer >>= 5 << 3 - case 6: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - ) - b.bitContainer >>= 6 << 3 - case 7: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - byte(b.bitContainer>>48), - ) - b.bitContainer >>= 7 << 3 - case 8: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - byte(b.bitContainer>>48), - byte(b.bitContainer>>56), - ) - b.bitContainer = 0 - b.nBits = 0 - return - default: - panic(fmt.Errorf("bits (%d) > 64", b.nBits)) - } - b.nBits &= 7 -} - // flush32 will flush out, so there are at least 32 bits available for writing. func (b *bitWriter) flush32() { if b.nBits < 32 { @@ -201,10 +93,3 @@ func (b *bitWriter) close() error { b.flushAlign() return nil } - -// reset and continue writing by appending to out. -func (b *bitWriter) reset(out []byte) { - b.bitContainer = 0 - b.nBits = 0 - b.out = out -} diff --git a/vendor/github.com/klauspost/compress/huff0/bytereader.go b/vendor/github.com/klauspost/compress/huff0/bytereader.go index 50bcdf6ea..4dcab8d23 100644 --- a/vendor/github.com/klauspost/compress/huff0/bytereader.go +++ b/vendor/github.com/klauspost/compress/huff0/bytereader.go @@ -20,11 +20,6 @@ func (b *byteReader) init(in []byte) { b.off = 0 } -// advance the stream b n bytes. -func (b *byteReader) advance(n uint) { - b.off += int(n) -} - // Int32 returns a little endian int32 starting at current offset. func (b byteReader) Int32() int32 { v3 := int32(b.b[b.off+3]) @@ -43,11 +38,6 @@ func (b byteReader) Uint32() uint32 { return (v3 << 24) | (v2 << 16) | (v1 << 8) | v0 } -// unread returns the unread portion of the input. -func (b byteReader) unread() []byte { - return b.b[b.off:] -} - // remain will return the number of bytes remaining. func (b byteReader) remain() int { return len(b.b) - b.off diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go index bc95ac623..4d14542fa 100644 --- a/vendor/github.com/klauspost/compress/huff0/compress.go +++ b/vendor/github.com/klauspost/compress/huff0/compress.go @@ -404,6 +404,7 @@ func (s *Scratch) canUseTable(c cTable) bool { return true } +//lint:ignore U1000 used for debugging func (s *Scratch) validateTable(c cTable) bool { if len(c) < int(s.symbolLen) { return false diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index 04f652995..c0c48bd70 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -11,7 +11,6 @@ import ( type dTable struct { single []dEntrySingle - double []dEntryDouble } // single-symbols decoding @@ -19,13 +18,6 @@ type dEntrySingle struct { entry uint16 } -// double-symbols decoding -type dEntryDouble struct { - seq [4]byte - nBits uint8 - len uint8 -} - // Uses special code for all tables that are < 8 bits. const use8BitTables = true @@ -35,7 +27,7 @@ const use8BitTables = true // If no Scratch is provided a new one is allocated. // The returned Scratch can be used for encoding or decoding input using this table. func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { - s, err = s.prepare(in) + s, err = s.prepare(nil) if err != nil { return s, nil, err } @@ -236,108 +228,6 @@ func (d *Decoder) buffer() *[4][256]byte { return &[4][256]byte{} } -// Decompress1X will decompress a 1X encoded stream. -// The cap of the output buffer will be the maximum decompressed size. -// The length of the supplied input must match the end of a block exactly. -func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { - if len(d.dt.single) == 0 { - return nil, errors.New("no table loaded") - } - if use8BitTables && d.actualTableLog <= 8 { - return d.decompress1X8Bit(dst, src) - } - var br bitReaderShifted - err := br.init(src) - if err != nil { - return dst, err - } - maxDecodedSize := cap(dst) - dst = dst[:0] - - // Avoid bounds check by always having full sized table. - const tlSize = 1 << tableLogMax - const tlMask = tlSize - 1 - dt := d.dt.single[:tlSize] - - // Use temp table to avoid bound checks/append penalty. - bufs := d.buffer() - buf := &bufs[0] - var off uint8 - - for br.off >= 8 { - br.fillFast() - v := dt[br.peekBitsFast(d.actualTableLog)&tlMask] - br.advance(uint8(v.entry)) - buf[off+0] = uint8(v.entry >> 8) - - v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] - br.advance(uint8(v.entry)) - buf[off+1] = uint8(v.entry >> 8) - - // Refill - br.fillFast() - - v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] - br.advance(uint8(v.entry)) - buf[off+2] = uint8(v.entry >> 8) - - v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] - br.advance(uint8(v.entry)) - buf[off+3] = uint8(v.entry >> 8) - - off += 4 - if off == 0 { - if len(dst)+256 > maxDecodedSize { - br.close() - d.bufs.Put(bufs) - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:]...) - } - } - - if len(dst)+int(off) > maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - dst = append(dst, buf[:off]...) - - // br < 8, so uint8 is fine - bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead - for bitsLeft > 0 { - br.fill() - if false && br.bitsRead >= 32 { - if br.off >= 4 { - v := br.in[br.off-4:] - v = v[:4] - low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - br.value = (br.value << 32) | uint64(low) - br.bitsRead -= 32 - br.off -= 4 - } else { - for br.off > 0 { - br.value = (br.value << 8) | uint64(br.in[br.off-1]) - br.bitsRead -= 8 - br.off-- - } - } - } - if len(dst) >= maxDecodedSize { - d.bufs.Put(bufs) - br.close() - return nil, ErrMaxDecodedSizeExceeded - } - v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] - nBits := uint8(v.entry) - br.advance(nBits) - bitsLeft -= nBits - dst = append(dst, uint8(v.entry>>8)) - } - d.bufs.Put(bufs) - return dst, br.close() -} - // decompress1X8Bit will decompress a 1X encoded stream with tablelog <= 8. // The cap of the output buffer will be the maximum decompressed size. // The length of the supplied input must match the end of a block exactly. @@ -995,7 +885,6 @@ func (d *Decoder) decompress4X8bitExactly(dst, src []byte) ([]byte, error) { const shift = 56 const tlSize = 1 << 8 - const tlMask = tlSize - 1 single := d.dt.single[:tlSize] // Use temp table to avoid bound checks/append penalty. diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go index 3415e5da2..671e630a8 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -2,12 +2,14 @@ // +build amd64,!appengine,!noasm,gc // This file contains the specialisation of Decoder.Decompress4X -// that uses an asm implementation of its main loop. +// and Decoder.Decompress1X that use an asm implementation of thir main loops. package huff0 import ( "errors" "fmt" + + "github.com/klauspost/compress/internal/cpuinfo" ) // decompress4x_main_loop_x86 is an x86 assembler implementation @@ -146,3 +148,81 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { } return dst, nil } + +// decompress4x_main_loop_x86 is an x86 assembler implementation +// of Decompress1X when tablelog > 8. +//go:noescape +func decompress1x_main_loop_amd64(ctx *decompress1xContext) + +// decompress4x_main_loop_x86 is an x86 with BMI2 assembler implementation +// of Decompress1X when tablelog > 8. +//go:noescape +func decompress1x_main_loop_bmi2(ctx *decompress1xContext) + +type decompress1xContext struct { + pbr *bitReaderShifted + peekBits uint8 + out *byte + outCap int + tbl *dEntrySingle + decoded int +} + +// Error reported by asm implementations +const error_max_decoded_size_exeeded = -1 + +// Decompress1X will decompress a 1X encoded stream. +// The cap of the output buffer will be the maximum decompressed size. +// The length of the supplied input must match the end of a block exactly. +func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + var br bitReaderShifted + err := br.init(src) + if err != nil { + return dst, err + } + maxDecodedSize := cap(dst) + dst = dst[:maxDecodedSize] + + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + + if maxDecodedSize >= 4 { + ctx := decompress1xContext{ + pbr: &br, + out: &dst[0], + outCap: maxDecodedSize, + peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() + tbl: &d.dt.single[0], + } + + if cpuinfo.HasBMI2() { + decompress1x_main_loop_bmi2(&ctx) + } else { + decompress1x_main_loop_amd64(&ctx) + } + if ctx.decoded == error_max_decoded_size_exeeded { + return nil, ErrMaxDecodedSizeExceeded + } + + dst = dst[:ctx.decoded] + } + + // br < 8, so uint8 is fine + bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead + for bitsLeft > 0 { + br.fill() + if len(dst) >= maxDecodedSize { + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] + nBits := uint8(v.entry) + br.advance(nBits) + bitsLeft -= nBits + dst = append(dst, uint8(v.entry>>8)) + } + return dst, br.close() +} diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s index 06287f568..6c65c6e2b 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -660,3 +660,206 @@ skip_fill1003: SHLQ $0x02, DX MOVQ DX, 64(AX) RET + +// func decompress1x_main_loop_amd64(ctx *decompress1xContext) +TEXT ·decompress1x_main_loop_amd64(SB), $0-8 + MOVQ ctx+0(FP), CX + MOVQ 16(CX), DX + MOVQ 24(CX), BX + CMPQ BX, $0x04 + JB error_max_decoded_size_exeeded + LEAQ (DX)(BX*1), BX + MOVQ (CX), SI + MOVQ (SI), R8 + MOVQ 24(SI), R9 + MOVQ 32(SI), R10 + MOVBQZX 40(SI), R11 + MOVQ 32(CX), SI + MOVBQZX 8(CX), DI + JMP loop_condition + +main_loop: + // Check if we have room for 4 bytes in the output buffer + LEAQ 4(DX), CX + CMPQ CX, BX + JGE error_max_decoded_size_exeeded + + // Decode 4 values + CMPQ R11, $0x20 + JL bitReader_fillFast_1_end + SUBQ $0x20, R11 + SUBQ $0x04, R9 + MOVL (R8)(R9*1), R12 + MOVQ R11, CX + SHLQ CL, R12 + ORQ R12, R10 + +bitReader_fillFast_1_end: + MOVQ DI, CX + MOVQ R10, R12 + SHRQ CL, R12 + MOVW (SI)(R12*2), CX + MOVB CH, AL + MOVBQZX CL, CX + ADDQ CX, R11 + SHLQ CL, R10 + MOVQ DI, CX + MOVQ R10, R12 + SHRQ CL, R12 + MOVW (SI)(R12*2), CX + MOVB CH, AH + MOVBQZX CL, CX + ADDQ CX, R11 + SHLQ CL, R10 + BSWAPL AX + CMPQ R11, $0x20 + JL bitReader_fillFast_2_end + SUBQ $0x20, R11 + SUBQ $0x04, R9 + MOVL (R8)(R9*1), R12 + MOVQ R11, CX + SHLQ CL, R12 + ORQ R12, R10 + +bitReader_fillFast_2_end: + MOVQ DI, CX + MOVQ R10, R12 + SHRQ CL, R12 + MOVW (SI)(R12*2), CX + MOVB CH, AH + MOVBQZX CL, CX + ADDQ CX, R11 + SHLQ CL, R10 + MOVQ DI, CX + MOVQ R10, R12 + SHRQ CL, R12 + MOVW (SI)(R12*2), CX + MOVB CH, AL + MOVBQZX CL, CX + ADDQ CX, R11 + SHLQ CL, R10 + BSWAPL AX + + // Store the decoded values + MOVL AX, (DX) + ADDQ $0x04, DX + +loop_condition: + CMPQ R9, $0x08 + JGE main_loop + + // Update ctx structure + MOVQ ctx+0(FP), AX + MOVQ DX, CX + MOVQ 16(AX), DX + SUBQ DX, CX + MOVQ CX, 40(AX) + MOVQ (AX), AX + MOVQ R9, 24(AX) + MOVQ R10, 32(AX) + MOVB R11, 40(AX) + RET + + // Report error +error_max_decoded_size_exeeded: + MOVQ ctx+0(FP), AX + MOVQ $-1, CX + MOVQ CX, 40(AX) + RET + +// func decompress1x_main_loop_bmi2(ctx *decompress1xContext) +// Requires: BMI2 +TEXT ·decompress1x_main_loop_bmi2(SB), $0-8 + MOVQ ctx+0(FP), CX + MOVQ 16(CX), DX + MOVQ 24(CX), BX + CMPQ BX, $0x04 + JB error_max_decoded_size_exeeded + LEAQ (DX)(BX*1), BX + MOVQ (CX), SI + MOVQ (SI), R8 + MOVQ 24(SI), R9 + MOVQ 32(SI), R10 + MOVBQZX 40(SI), R11 + MOVQ 32(CX), SI + MOVBQZX 8(CX), DI + JMP loop_condition + +main_loop: + // Check if we have room for 4 bytes in the output buffer + LEAQ 4(DX), CX + CMPQ CX, BX + JGE error_max_decoded_size_exeeded + + // Decode 4 values + CMPQ R11, $0x20 + JL bitReader_fillFast_1_end + SUBQ $0x20, R11 + SUBQ $0x04, R9 + MOVL (R8)(R9*1), CX + SHLXQ R11, CX, CX + ORQ CX, R10 + +bitReader_fillFast_1_end: + SHRXQ DI, R10, CX + MOVW (SI)(CX*2), CX + MOVB CH, AL + MOVBQZX CL, CX + ADDQ CX, R11 + SHLXQ CX, R10, R10 + SHRXQ DI, R10, CX + MOVW (SI)(CX*2), CX + MOVB CH, AH + MOVBQZX CL, CX + ADDQ CX, R11 + SHLXQ CX, R10, R10 + BSWAPL AX + CMPQ R11, $0x20 + JL bitReader_fillFast_2_end + SUBQ $0x20, R11 + SUBQ $0x04, R9 + MOVL (R8)(R9*1), CX + SHLXQ R11, CX, CX + ORQ CX, R10 + +bitReader_fillFast_2_end: + SHRXQ DI, R10, CX + MOVW (SI)(CX*2), CX + MOVB CH, AH + MOVBQZX CL, CX + ADDQ CX, R11 + SHLXQ CX, R10, R10 + SHRXQ DI, R10, CX + MOVW (SI)(CX*2), CX + MOVB CH, AL + MOVBQZX CL, CX + ADDQ CX, R11 + SHLXQ CX, R10, R10 + BSWAPL AX + + // Store the decoded values + MOVL AX, (DX) + ADDQ $0x04, DX + +loop_condition: + CMPQ R9, $0x08 + JGE main_loop + + // Update ctx structure + MOVQ ctx+0(FP), AX + MOVQ DX, CX + MOVQ 16(AX), DX + SUBQ DX, CX + MOVQ CX, 40(AX) + MOVQ (AX), AX + MOVQ R9, 24(AX) + MOVQ R10, 32(AX) + MOVB R11, 40(AX) + RET + + // Report error +error_max_decoded_size_exeeded: + MOVQ ctx+0(FP), AX + MOVQ $-1, CX + MOVQ CX, 40(AX) + RET diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go index 126b4d68a..4f6f37cb2 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_generic.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_generic.go @@ -191,3 +191,105 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { } return dst, nil } + +// Decompress1X will decompress a 1X encoded stream. +// The cap of the output buffer will be the maximum decompressed size. +// The length of the supplied input must match the end of a block exactly. +func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { + return nil, errors.New("no table loaded") + } + if use8BitTables && d.actualTableLog <= 8 { + return d.decompress1X8Bit(dst, src) + } + var br bitReaderShifted + err := br.init(src) + if err != nil { + return dst, err + } + maxDecodedSize := cap(dst) + dst = dst[:0] + + // Avoid bounds check by always having full sized table. + const tlSize = 1 << tableLogMax + const tlMask = tlSize - 1 + dt := d.dt.single[:tlSize] + + // Use temp table to avoid bound checks/append penalty. + bufs := d.buffer() + buf := &bufs[0] + var off uint8 + + for br.off >= 8 { + br.fillFast() + v := dt[br.peekBitsFast(d.actualTableLog)&tlMask] + br.advance(uint8(v.entry)) + buf[off+0] = uint8(v.entry >> 8) + + v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] + br.advance(uint8(v.entry)) + buf[off+1] = uint8(v.entry >> 8) + + // Refill + br.fillFast() + + v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] + br.advance(uint8(v.entry)) + buf[off+2] = uint8(v.entry >> 8) + + v = dt[br.peekBitsFast(d.actualTableLog)&tlMask] + br.advance(uint8(v.entry)) + buf[off+3] = uint8(v.entry >> 8) + + off += 4 + if off == 0 { + if len(dst)+256 > maxDecodedSize { + br.close() + d.bufs.Put(bufs) + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:]...) + } + } + + if len(dst)+int(off) > maxDecodedSize { + d.bufs.Put(bufs) + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + dst = append(dst, buf[:off]...) + + // br < 8, so uint8 is fine + bitsLeft := uint8(br.off)*8 + 64 - br.bitsRead + for bitsLeft > 0 { + br.fill() + if false && br.bitsRead >= 32 { + if br.off >= 4 { + v := br.in[br.off-4:] + v = v[:4] + low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) + br.value = (br.value << 32) | uint64(low) + br.bitsRead -= 32 + br.off -= 4 + } else { + for br.off > 0 { + br.value = (br.value << 8) | uint64(br.in[br.off-1]) + br.bitsRead -= 8 + br.off-- + } + } + } + if len(dst) >= maxDecodedSize { + d.bufs.Put(bufs) + br.close() + return nil, ErrMaxDecodedSizeExceeded + } + v := d.dt.single[br.peekBitsFast(d.actualTableLog)&tlMask] + nBits := uint8(v.entry) + br.advance(nBits) + bitsLeft -= nBits + dst = append(dst, uint8(v.entry>>8)) + } + d.bufs.Put(bufs) + return dst, br.close() +} diff --git a/vendor/github.com/klauspost/compress/zstd/bitreader.go b/vendor/github.com/klauspost/compress/zstd/bitreader.go index d7cd15ba2..97299d499 100644 --- a/vendor/github.com/klauspost/compress/zstd/bitreader.go +++ b/vendor/github.com/klauspost/compress/zstd/bitreader.go @@ -63,13 +63,6 @@ func (b *bitReader) get32BitsFast(n uint8) uint32 { return v } -func (b *bitReader) get16BitsFast(n uint8) uint16 { - const regMask = 64 - 1 - v := uint16((b.value << (b.bitsRead & regMask)) >> ((regMask + 1 - n) & regMask)) - b.bitsRead += n - return v -} - // fillFast() will make sure at least 32 bits are available. // There must be at least 4 bytes available. func (b *bitReader) fillFast() { diff --git a/vendor/github.com/klauspost/compress/zstd/bitwriter.go b/vendor/github.com/klauspost/compress/zstd/bitwriter.go index b36618285..78b3c61be 100644 --- a/vendor/github.com/klauspost/compress/zstd/bitwriter.go +++ b/vendor/github.com/klauspost/compress/zstd/bitwriter.go @@ -5,8 +5,6 @@ package zstd -import "fmt" - // bitWriter will write bits. // First bit will be LSB of the first byte of output. type bitWriter struct { @@ -73,80 +71,6 @@ func (b *bitWriter) addBits16Clean(value uint16, bits uint8) { b.nBits += bits } -// flush will flush all pending full bytes. -// There will be at least 56 bits available for writing when this has been called. -// Using flush32 is faster, but leaves less space for writing. -func (b *bitWriter) flush() { - v := b.nBits >> 3 - switch v { - case 0: - case 1: - b.out = append(b.out, - byte(b.bitContainer), - ) - case 2: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - ) - case 3: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - ) - case 4: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - ) - case 5: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - ) - case 6: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - ) - case 7: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - byte(b.bitContainer>>48), - ) - case 8: - b.out = append(b.out, - byte(b.bitContainer), - byte(b.bitContainer>>8), - byte(b.bitContainer>>16), - byte(b.bitContainer>>24), - byte(b.bitContainer>>32), - byte(b.bitContainer>>40), - byte(b.bitContainer>>48), - byte(b.bitContainer>>56), - ) - default: - panic(fmt.Errorf("bits (%d) > 64", b.nBits)) - } - b.bitContainer >>= v << 3 - b.nBits &= 7 -} - // flush32 will flush out, so there are at least 32 bits available for writing. func (b *bitWriter) flush32() { if b.nBits < 32 { diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index b2bca3301..7eed729be 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -49,11 +49,8 @@ const ( // Maximum possible block size (all Raw+Uncompressed). maxBlockSize = (1 << 21) - 1 - // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#literals_section_header - maxCompressedLiteralSize = 1 << 18 - maxRLELiteralSize = 1 << 20 - maxMatchLen = 131074 - maxSequences = 0x7f00 + 0xffff + maxMatchLen = 131074 + maxSequences = 0x7f00 + 0xffff // We support slightly less than the reference decoder to be able to // use ints on 32 bit archs. @@ -105,7 +102,6 @@ type blockDec struct { // Block is RLE, this is the size. RLESize uint32 - tmp [4]byte Type blockType @@ -368,14 +364,9 @@ func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err } if cap(b.literalBuf) < litRegenSize { if b.lowMem { - b.literalBuf = make([]byte, litRegenSize) + b.literalBuf = make([]byte, litRegenSize, litRegenSize+compressedBlockOverAlloc) } else { - if litRegenSize > maxCompressedLiteralSize { - // Exceptional - b.literalBuf = make([]byte, litRegenSize) - } else { - b.literalBuf = make([]byte, litRegenSize, maxCompressedLiteralSize) - } + b.literalBuf = make([]byte, litRegenSize, maxCompressedBlockSize+compressedBlockOverAlloc) } } literals = b.literalBuf[:litRegenSize] @@ -405,14 +396,14 @@ func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err // Ensure we have space to store it. if cap(b.literalBuf) < litRegenSize { if b.lowMem { - b.literalBuf = make([]byte, 0, litRegenSize) + b.literalBuf = make([]byte, 0, litRegenSize+compressedBlockOverAlloc) } else { - b.literalBuf = make([]byte, 0, maxCompressedLiteralSize) + b.literalBuf = make([]byte, 0, maxCompressedBlockSize+compressedBlockOverAlloc) } } var err error // Use our out buffer. - huff.MaxDecodedSize = maxCompressedBlockSize + huff.MaxDecodedSize = litRegenSize if fourStreams { literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) } else { @@ -437,9 +428,9 @@ func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err // Ensure we have space to store it. if cap(b.literalBuf) < litRegenSize { if b.lowMem { - b.literalBuf = make([]byte, 0, litRegenSize) + b.literalBuf = make([]byte, 0, litRegenSize+compressedBlockOverAlloc) } else { - b.literalBuf = make([]byte, 0, maxCompressedBlockSize) + b.literalBuf = make([]byte, 0, maxCompressedBlockSize+compressedBlockOverAlloc) } } huff := hist.huffTree @@ -456,7 +447,7 @@ func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err return in, err } hist.huffTree = huff - huff.MaxDecodedSize = maxCompressedBlockSize + huff.MaxDecodedSize = litRegenSize // Use our out buffer. if fourStreams { literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) @@ -471,6 +462,8 @@ func (b *blockDec) decodeLiterals(in []byte, hist *history) (remain []byte, err if len(literals) != litRegenSize { return in, fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) } + // Re-cap to get extra size. + literals = b.literalBuf[:len(literals)] if debugDecoder { printf("Decompressed %d literals into %d bytes\n", litCompSize, litRegenSize) } diff --git a/vendor/github.com/klauspost/compress/zstd/bytebuf.go b/vendor/github.com/klauspost/compress/zstd/bytebuf.go index b80191e4b..4493baa75 100644 --- a/vendor/github.com/klauspost/compress/zstd/bytebuf.go +++ b/vendor/github.com/klauspost/compress/zstd/bytebuf.go @@ -52,10 +52,6 @@ func (b *byteBuf) readBig(n int, dst []byte) ([]byte, error) { return r, nil } -func (b *byteBuf) remain() []byte { - return *b -} - func (b *byteBuf) readByte() (byte, error) { bb := *b if len(bb) < 1 { diff --git a/vendor/github.com/klauspost/compress/zstd/bytereader.go b/vendor/github.com/klauspost/compress/zstd/bytereader.go index 2c4fca17f..0e59a242d 100644 --- a/vendor/github.com/klauspost/compress/zstd/bytereader.go +++ b/vendor/github.com/klauspost/compress/zstd/bytereader.go @@ -13,12 +13,6 @@ type byteReader struct { off int } -// init will initialize the reader and set the input. -func (b *byteReader) init(in []byte) { - b.b = in - b.off = 0 -} - // advance the stream b n bytes. func (b *byteReader) advance(n uint) { b.off += int(n) diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 36119f385..286c8f9d7 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -637,60 +637,18 @@ func (d *Decoder) startSyncDecoder(r io.Reader) error { // Create Decoder: // ASYNC: -// Spawn 4 go routines. -// 0: Read frames and decode blocks. -// 1: Decode block and literals. Receives hufftree and seqdecs, returns seqdecs and huff tree. -// 2: Wait for recentOffsets if needed. Decode sequences, send recentOffsets. -// 3: Wait for stream history, execute sequences, send stream history. +// Spawn 3 go routines. +// 0: Read frames and decode block literals. +// 1: Decode sequences. +// 2: Execute sequences, send to output. func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output chan decodeOutput) { defer d.streamWg.Done() br := readerWrapper{r: r} - var seqPrepare = make(chan *blockDec, d.o.concurrent) var seqDecode = make(chan *blockDec, d.o.concurrent) var seqExecute = make(chan *blockDec, d.o.concurrent) - // Async 1: Prepare blocks... - go func() { - var hist history - var hasErr bool - for block := range seqPrepare { - if hasErr { - if block != nil { - seqDecode <- block - } - continue - } - if block.async.newHist != nil { - if debugDecoder { - println("Async 1: new history") - } - hist.reset() - if block.async.newHist.dict != nil { - hist.setDict(block.async.newHist.dict) - } - } - if block.err != nil || block.Type != blockTypeCompressed { - hasErr = block.err != nil - seqDecode <- block - continue - } - - remain, err := block.decodeLiterals(block.data, &hist) - block.err = err - hasErr = block.err != nil - if err == nil { - block.async.literals = hist.decoders.literals - block.async.seqData = remain - } else if debugDecoder { - println("decodeLiterals error:", err) - } - seqDecode <- block - } - close(seqDecode) - }() - - // Async 2: Decode sequences... + // Async 1: Decode sequences... go func() { var hist history var hasErr bool @@ -704,7 +662,7 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch } if block.async.newHist != nil { if debugDecoder { - println("Async 2: new history, recent:", block.async.newHist.recentOffsets) + println("Async 1: new history, recent:", block.async.newHist.recentOffsets) } hist.decoders = block.async.newHist.decoders hist.recentOffsets = block.async.newHist.recentOffsets @@ -758,7 +716,7 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch } if block.async.newHist != nil { if debugDecoder { - println("Async 3: new history") + println("Async 2: new history") } hist.windowSize = block.async.newHist.windowSize hist.allocFrameBuffer = block.async.newHist.allocFrameBuffer @@ -845,6 +803,33 @@ func (d *Decoder) startStreamDecoder(ctx context.Context, r io.Reader, output ch decodeStream: for { + var hist history + var hasErr bool + + decodeBlock := func(block *blockDec) { + if hasErr { + if block != nil { + seqDecode <- block + } + return + } + if block.err != nil || block.Type != blockTypeCompressed { + hasErr = block.err != nil + seqDecode <- block + return + } + + remain, err := block.decodeLiterals(block.data, &hist) + block.err = err + hasErr = block.err != nil + if err == nil { + block.async.literals = hist.decoders.literals + block.async.seqData = remain + } else if debugDecoder { + println("decodeLiterals error:", err) + } + seqDecode <- block + } frame := d.frame if debugDecoder { println("New frame...") @@ -871,7 +856,7 @@ decodeStream: case <-ctx.Done(): case dec := <-d.decoders: dec.sendErr(err) - seqPrepare <- dec + decodeBlock(dec) } break decodeStream } @@ -891,6 +876,10 @@ decodeStream: if debugDecoder { println("Alloc History:", h.allocFrameBuffer) } + hist.reset() + if h.dict != nil { + hist.setDict(h.dict) + } dec.async.newHist = &h dec.async.fcs = frame.FrameContentSize historySent = true @@ -917,7 +906,7 @@ decodeStream: } err = dec.err last := dec.Last - seqPrepare <- dec + decodeBlock(dec) if err != nil { break decodeStream } @@ -926,7 +915,7 @@ decodeStream: } } } - close(seqPrepare) + close(seqDecode) wg.Wait() d.frame.history.b = frameHistCache } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index 602c05ee0..c769f6941 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -156,8 +156,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -518,8 +518,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -674,8 +674,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -1047,8 +1047,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index d6b310424..7ff0c64fa 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -127,8 +127,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -439,8 +439,8 @@ encodeLoop: var t int32 for { - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -785,8 +785,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -969,7 +969,7 @@ encodeLoop: te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} longHash1 := hashLen(cv0, dFastLongTableBits, dFastLongLen) - longHash2 := hashLen(cv0, dFastLongTableBits, dFastLongLen) + longHash2 := hashLen(cv1, dFastLongTableBits, dFastLongLen) e.longTable[longHash1] = te0 e.longTable[longHash2] = te1 e.markLongShardDirty(longHash1) @@ -1002,8 +1002,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index dcc987a7c..e6b1d01cf 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -551,7 +551,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } // If we can do everything in one block, prefer that. - if len(src) <= maxCompressedBlockSize { + if len(src) <= e.o.blockSize { enc.Reset(e.o.dict, true) // Slightly faster with no history and everything in one block. if e.o.crc { diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index 3ff109cce..fa0a633f3 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -253,10 +253,11 @@ func (d *frameDec) reset(br byteBuffer) error { return ErrWindowSizeTooSmall } d.history.windowSize = int(d.WindowSize) - if d.o.lowMem && d.history.windowSize < maxBlockSize { + if !d.o.lowMem || d.history.windowSize < maxBlockSize { + // Alloc 2x window size if not low-mem, or very small window size. d.history.allocFrameBuffer = d.history.windowSize * 2 - // TODO: Maybe use FrameContent size } else { + // Alloc with one additional block d.history.allocFrameBuffer = d.history.windowSize + maxBlockSize } diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go index fde4e6b60..23333b969 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go @@ -229,18 +229,10 @@ func (d decSymbol) newState() uint16 { return uint16(d >> 16) } -func (d decSymbol) baseline() uint32 { - return uint32(d >> 32) -} - func (d decSymbol) baselineInt() int { return int(d >> 32) } -func (d *decSymbol) set(nbits, addBits uint8, newState uint16, baseline uint32) { - *d = decSymbol(nbits) | (decSymbol(addBits) << 8) | (decSymbol(newState) << 16) | (decSymbol(baseline) << 32) -} - func (d *decSymbol) setNBits(nBits uint8) { const mask = 0xffffffffffffff00 *d = (*d & mask) | decSymbol(nBits) @@ -256,11 +248,6 @@ func (d *decSymbol) setNewState(state uint16) { *d = (*d & mask) | decSymbol(state)<<16 } -func (d *decSymbol) setBaseline(baseline uint32) { - const mask = 0xffffffff - *d = (*d & mask) | decSymbol(baseline)<<32 -} - func (d *decSymbol) setExt(addBits uint8, baseline uint32) { const mask = 0xffff00ff *d = (*d & mask) | (decSymbol(addBits) << 8) | (decSymbol(baseline) << 32) @@ -377,34 +364,7 @@ func (s *fseState) init(br *bitReader, tableLog uint8, dt []decSymbol) { s.state = dt[br.getBits(tableLog)] } -// next returns the current symbol and sets the next state. -// At least tablelog bits must be available in the bit reader. -func (s *fseState) next(br *bitReader) { - lowBits := uint16(br.getBits(s.state.nbBits())) - s.state = s.dt[s.state.newState()+lowBits] -} - -// finished returns true if all bits have been read from the bitstream -// and the next state would require reading bits from the input. -func (s *fseState) finished(br *bitReader) bool { - return br.finished() && s.state.nbBits() > 0 -} - -// final returns the current state symbol without decoding the next. -func (s *fseState) final() (int, uint8) { - return s.state.baselineInt(), s.state.addBits() -} - // final returns the current state symbol without decoding the next. func (s decSymbol) final() (int, uint8) { return s.baselineInt(), s.addBits() } - -// nextFast returns the next symbol and sets the next state. -// This can only be used if no symbols are 0 bits. -// At least tablelog bits must be available in the bit reader. -func (s *fseState) nextFast(br *bitReader) (uint32, uint8) { - lowBits := br.get16BitsFast(s.state.nbBits()) - s.state = s.dt[s.state.newState()+lowBits] - return s.state.baseline(), s.state.addBits() -} diff --git a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go index 5442061b1..ab26326a8 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go @@ -76,21 +76,6 @@ func (s *fseEncoder) HistogramFinished(maxSymbol uint8, maxCount int) { s.clearCount = maxCount != 0 } -// prepare will prepare and allocate scratch tables used for both compression and decompression. -func (s *fseEncoder) prepare() (*fseEncoder, error) { - if s == nil { - s = &fseEncoder{} - } - s.useRLE = false - if s.clearCount && s.maxCount == 0 { - for i := range s.count { - s.count[i] = 0 - } - s.clearCount = false - } - return s, nil -} - // allocCtable will allocate tables needed for compression. // If existing tables a re big enough, they are simply re-used. func (s *fseEncoder) allocCtable() { @@ -709,14 +694,6 @@ func (c *cState) init(bw *bitWriter, ct *cTable, first symbolTransform) { c.state = c.stateTable[lu] } -// encode the output symbol provided and write it to the bitstream. -func (c *cState) encode(symbolTT symbolTransform) { - nbBitsOut := (uint32(c.state) + symbolTT.deltaNbBits) >> 16 - dstState := int32(c.state>>(nbBitsOut&15)) + int32(symbolTT.deltaFindState) - c.bw.addBits16NC(c.state, uint8(nbBitsOut)) - c.state = c.stateTable[dstState] -} - // flush will write the tablelog to the output and flush the remaining full bytes. func (c *cState) flush(tableLog uint8) { c.bw.flush32() diff --git a/vendor/github.com/klauspost/compress/zstd/hash.go b/vendor/github.com/klauspost/compress/zstd/hash.go index cf33f29a1..5d73c21eb 100644 --- a/vendor/github.com/klauspost/compress/zstd/hash.go +++ b/vendor/github.com/klauspost/compress/zstd/hash.go @@ -33,9 +33,3 @@ func hashLen(u uint64, length, mls uint8) uint32 { return (uint32(u) * prime4bytes) >> (32 - length) } } - -// hash3 returns the hash of the lower 3 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <32. -func hash3(u uint32, h uint8) uint32 { - return ((u << (32 - 24)) * prime3bytes) >> ((32 - h) & 31) -} diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go index e80139dd9..df0447203 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -188,6 +188,7 @@ func (s *sequenceDecs) execute(seqs []seqVals, hist []byte) error { } } } + // Add final literals copy(out[t:], s.literals) if debugDecoder { @@ -203,12 +204,11 @@ func (s *sequenceDecs) execute(seqs []seqVals, hist []byte) error { // decode sequences from the stream with the provided history. func (s *sequenceDecs) decodeSync(hist []byte) error { - if true { - supported, err := s.decodeSyncSimple(hist) - if supported { - return err - } + supported, err := s.decodeSyncSimple(hist) + if supported { + return err } + br := s.br seqs := s.nSeqs startSize := len(s.out) @@ -396,6 +396,7 @@ func (s *sequenceDecs) decodeSync(hist []byte) error { ofState = ofTable[ofState.newState()&maxTableMask] } else { bits := br.get32BitsFast(nBits) + lowBits := uint16(bits >> ((ofState.nbBits() + mlState.nbBits()) & 31)) llState = llTable[(llState.newState()+lowBits)&maxTableMask] @@ -418,16 +419,6 @@ func (s *sequenceDecs) decodeSync(hist []byte) error { return br.close() } -// update states, at least 27 bits must be available. -func (s *sequenceDecs) update(br *bitReader) { - // Max 8 bits - s.litLengths.state.next(br) - // Max 9 bits - s.matchLengths.state.next(br) - // Max 8 bits - s.offsets.state.next(br) -} - var bitMask [16]uint16 func init() { @@ -436,87 +427,6 @@ func init() { } } -// update states, at least 27 bits must be available. -func (s *sequenceDecs) updateAlt(br *bitReader) { - // Update all 3 states at once. Approx 20% faster. - a, b, c := s.litLengths.state.state, s.matchLengths.state.state, s.offsets.state.state - - nBits := a.nbBits() + b.nbBits() + c.nbBits() - if nBits == 0 { - s.litLengths.state.state = s.litLengths.state.dt[a.newState()] - s.matchLengths.state.state = s.matchLengths.state.dt[b.newState()] - s.offsets.state.state = s.offsets.state.dt[c.newState()] - return - } - bits := br.get32BitsFast(nBits) - lowBits := uint16(bits >> ((c.nbBits() + b.nbBits()) & 31)) - s.litLengths.state.state = s.litLengths.state.dt[a.newState()+lowBits] - - lowBits = uint16(bits >> (c.nbBits() & 31)) - lowBits &= bitMask[b.nbBits()&15] - s.matchLengths.state.state = s.matchLengths.state.dt[b.newState()+lowBits] - - lowBits = uint16(bits) & bitMask[c.nbBits()&15] - s.offsets.state.state = s.offsets.state.dt[c.newState()+lowBits] -} - -// nextFast will return new states when there are at least 4 unused bytes left on the stream when done. -func (s *sequenceDecs) nextFast(br *bitReader, llState, mlState, ofState decSymbol) (ll, mo, ml int) { - // Final will not read from stream. - ll, llB := llState.final() - ml, mlB := mlState.final() - mo, moB := ofState.final() - - // extra bits are stored in reverse order. - br.fillFast() - mo += br.getBits(moB) - if s.maxBits > 32 { - br.fillFast() - } - ml += br.getBits(mlB) - ll += br.getBits(llB) - - if moB > 1 { - s.prevOffset[2] = s.prevOffset[1] - s.prevOffset[1] = s.prevOffset[0] - s.prevOffset[0] = mo - return - } - // mo = s.adjustOffset(mo, ll, moB) - // Inlined for rather big speedup - if ll == 0 { - // There is an exception though, when current sequence's literals_length = 0. - // In this case, repeated offsets are shifted by one, so an offset_value of 1 means Repeated_Offset2, - // an offset_value of 2 means Repeated_Offset3, and an offset_value of 3 means Repeated_Offset1 - 1_byte. - mo++ - } - - if mo == 0 { - mo = s.prevOffset[0] - return - } - var temp int - if mo == 3 { - temp = s.prevOffset[0] - 1 - } else { - temp = s.prevOffset[mo] - } - - if temp == 0 { - // 0 is not valid; input is corrupted; force offset to 1 - println("temp was 0") - temp = 1 - } - - if mo != 1 { - s.prevOffset[2] = s.prevOffset[1] - } - s.prevOffset[1] = s.prevOffset[0] - s.prevOffset[0] = temp - mo = temp - return -} - func (s *sequenceDecs) next(br *bitReader, llState, mlState, ofState decSymbol) (ll, mo, ml int) { // Final will not read from stream. ll, llB := llState.final() diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go index 4676b09cc..847b322ae 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go @@ -62,6 +62,10 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { if s.maxSyncLen > 0 && cap(s.out)-len(s.out)-compressedBlockOverAlloc < int(s.maxSyncLen) { useSafe = true } + if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { + useSafe = true + } + br := s.br maxBlockSize := maxCompressedBlockSize @@ -301,6 +305,10 @@ type executeAsmContext struct { //go:noescape func sequenceDecs_executeSimple_amd64(ctx *executeAsmContext) bool +// Same as above, but with safe memcopies +//go:noescape +func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool + // executeSimple handles cases when dictionary is not used. func (s *sequenceDecs) executeSimple(seqs []seqVals, hist []byte) error { // Ensure we have enough output size... @@ -327,8 +335,12 @@ func (s *sequenceDecs) executeSimple(seqs []seqVals, hist []byte) error { literals: s.literals, windowSize: s.windowSize, } - - ok := sequenceDecs_executeSimple_amd64(&ctx) + var ok bool + if cap(s.literals) < len(s.literals)+compressedBlockOverAlloc { + ok = sequenceDecs_executeSimple_safe_amd64(&ctx) + } else { + ok = sequenceDecs_executeSimple_amd64(&ctx) + } if !ok { return fmt.Errorf("match offset (%d) bigger than current history (%d)", seqs[ctx.seqIndex].mo, ctx.outPosition+len(hist)) diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 2585b2e98..212c6cac3 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -705,60 +705,55 @@ sequenceDecs_decode_bmi2_fill_2_end: MOVQ CX, (R9) // Fill bitreader for state updates - MOVQ R13, (SP) - MOVQ $0x00000808, CX - BEXTRQ CX, R8, R13 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 - JZ sequenceDecs_decode_bmi2_skip_update - - // Update Literal Length State - MOVBQZX SI, R14 - MOVQ $0x00001010, CX - BEXTRQ CX, SI, SI + MOVQ R13, (SP) + MOVQ $0x00000808, CX + BEXTRQ CX, R8, R13 + MOVQ ctx+16(FP), CX + CMPQ 96(CX), $0x00 + JZ sequenceDecs_decode_bmi2_skip_update + LEAQ (SI)(DI*1), R14 + ADDQ R8, R14 + MOVBQZX R14, R14 LEAQ (DX)(R14*1), CX MOVQ AX, R15 MOVQ CX, DX ROLQ CL, R15 BZHIQ R14, R15, R15 - ADDQ R15, SI - // Load ctx.llTable + // Update Offset State + BZHIQ R8, R15, CX + SHRXQ R8, R15, R15 + MOVQ $0x00001010, R14 + BEXTRQ R14, R8, R8 + ADDQ CX, R8 + + // Load ctx.ofTable MOVQ ctx+16(FP), CX - MOVQ (CX), CX - MOVQ (CX)(SI*8), SI + MOVQ 48(CX), CX + MOVQ (CX)(R8*8), R8 // Update Match Length State - MOVBQZX DI, R14 - MOVQ $0x00001010, CX - BEXTRQ CX, DI, DI - LEAQ (DX)(R14*1), CX - MOVQ AX, R15 - MOVQ CX, DX - ROLQ CL, R15 - BZHIQ R14, R15, R15 - ADDQ R15, DI + BZHIQ DI, R15, CX + SHRXQ DI, R15, R15 + MOVQ $0x00001010, R14 + BEXTRQ R14, DI, DI + ADDQ CX, DI // Load ctx.mlTable MOVQ ctx+16(FP), CX MOVQ 24(CX), CX MOVQ (CX)(DI*8), DI - // Update Offset State - MOVBQZX R8, R14 - MOVQ $0x00001010, CX - BEXTRQ CX, R8, R8 - LEAQ (DX)(R14*1), CX - MOVQ AX, R15 - MOVQ CX, DX - ROLQ CL, R15 - BZHIQ R14, R15, R15 - ADDQ R15, R8 + // Update Literal Length State + BZHIQ SI, R15, CX + MOVQ $0x00001010, R14 + BEXTRQ R14, SI, SI + ADDQ CX, SI - // Load ctx.ofTable + // Load ctx.llTable MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX - MOVQ (CX)(R8*8), R8 + MOVQ (CX), CX + MOVQ (CX)(SI*8), SI sequenceDecs_decode_bmi2_skip_update: // Adjust offset @@ -965,60 +960,55 @@ sequenceDecs_decode_56_bmi2_fill_end: MOVQ CX, (R9) // Fill bitreader for state updates - MOVQ R13, (SP) - MOVQ $0x00000808, CX - BEXTRQ CX, R8, R13 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 - JZ sequenceDecs_decode_56_bmi2_skip_update - - // Update Literal Length State - MOVBQZX SI, R14 - MOVQ $0x00001010, CX - BEXTRQ CX, SI, SI + MOVQ R13, (SP) + MOVQ $0x00000808, CX + BEXTRQ CX, R8, R13 + MOVQ ctx+16(FP), CX + CMPQ 96(CX), $0x00 + JZ sequenceDecs_decode_56_bmi2_skip_update + LEAQ (SI)(DI*1), R14 + ADDQ R8, R14 + MOVBQZX R14, R14 LEAQ (DX)(R14*1), CX MOVQ AX, R15 MOVQ CX, DX ROLQ CL, R15 BZHIQ R14, R15, R15 - ADDQ R15, SI - // Load ctx.llTable + // Update Offset State + BZHIQ R8, R15, CX + SHRXQ R8, R15, R15 + MOVQ $0x00001010, R14 + BEXTRQ R14, R8, R8 + ADDQ CX, R8 + + // Load ctx.ofTable MOVQ ctx+16(FP), CX - MOVQ (CX), CX - MOVQ (CX)(SI*8), SI + MOVQ 48(CX), CX + MOVQ (CX)(R8*8), R8 // Update Match Length State - MOVBQZX DI, R14 - MOVQ $0x00001010, CX - BEXTRQ CX, DI, DI - LEAQ (DX)(R14*1), CX - MOVQ AX, R15 - MOVQ CX, DX - ROLQ CL, R15 - BZHIQ R14, R15, R15 - ADDQ R15, DI + BZHIQ DI, R15, CX + SHRXQ DI, R15, R15 + MOVQ $0x00001010, R14 + BEXTRQ R14, DI, DI + ADDQ CX, DI // Load ctx.mlTable MOVQ ctx+16(FP), CX MOVQ 24(CX), CX MOVQ (CX)(DI*8), DI - // Update Offset State - MOVBQZX R8, R14 - MOVQ $0x00001010, CX - BEXTRQ CX, R8, R8 - LEAQ (DX)(R14*1), CX - MOVQ AX, R15 - MOVQ CX, DX - ROLQ CL, R15 - BZHIQ R14, R15, R15 - ADDQ R15, R8 + // Update Literal Length State + BZHIQ SI, R15, CX + MOVQ $0x00001010, R14 + BEXTRQ R14, SI, SI + ADDQ CX, SI - // Load ctx.ofTable + // Load ctx.llTable MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX - MOVQ (CX)(R8*8), R8 + MOVQ (CX), CX + MOVQ (CX)(SI*8), SI sequenceDecs_decode_56_bmi2_skip_update: // Adjust offset @@ -1171,6 +1161,228 @@ main_loop: TESTQ R11, R11 JZ check_offset XORQ R14, R14 + +copy_1: + MOVUPS (SI)(R14*1), X0 + MOVUPS X0, (BX)(R14*1) + ADDQ $0x10, R14 + CMPQ R14, R11 + JB copy_1 + ADDQ R11, SI + ADDQ R11, BX + ADDQ R11, DI + + // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) +check_offset: + LEAQ (DI)(R10*1), R11 + CMPQ R12, R11 + JG error_match_off_too_big + CMPQ R12, R8 + JG error_match_off_too_big + + // Copy match from history + MOVQ R12, R11 + SUBQ DI, R11 + JLS copy_match + MOVQ R9, R14 + SUBQ R11, R14 + CMPQ R13, R11 + JGE copy_all_from_history + XORQ R11, R11 + TESTQ $0x00000001, R13 + JZ copy_4_word + MOVB (R14)(R11*1), R12 + MOVB R12, (BX)(R11*1) + ADDQ $0x01, R11 + +copy_4_word: + TESTQ $0x00000002, R13 + JZ copy_4_dword + MOVW (R14)(R11*1), R12 + MOVW R12, (BX)(R11*1) + ADDQ $0x02, R11 + +copy_4_dword: + TESTQ $0x00000004, R13 + JZ copy_4_qword + MOVL (R14)(R11*1), R12 + MOVL R12, (BX)(R11*1) + ADDQ $0x04, R11 + +copy_4_qword: + TESTQ $0x00000008, R13 + JZ copy_4_test + MOVQ (R14)(R11*1), R12 + MOVQ R12, (BX)(R11*1) + ADDQ $0x08, R11 + JMP copy_4_test + +copy_4: + MOVUPS (R14)(R11*1), X0 + MOVUPS X0, (BX)(R11*1) + ADDQ $0x10, R11 + +copy_4_test: + CMPQ R11, R13 + JB copy_4 + ADDQ R13, DI + ADDQ R13, BX + ADDQ $0x18, AX + INCQ DX + CMPQ DX, CX + JB main_loop + JMP loop_finished + +copy_all_from_history: + XORQ R15, R15 + TESTQ $0x00000001, R11 + JZ copy_5_word + MOVB (R14)(R15*1), BP + MOVB BP, (BX)(R15*1) + ADDQ $0x01, R15 + +copy_5_word: + TESTQ $0x00000002, R11 + JZ copy_5_dword + MOVW (R14)(R15*1), BP + MOVW BP, (BX)(R15*1) + ADDQ $0x02, R15 + +copy_5_dword: + TESTQ $0x00000004, R11 + JZ copy_5_qword + MOVL (R14)(R15*1), BP + MOVL BP, (BX)(R15*1) + ADDQ $0x04, R15 + +copy_5_qword: + TESTQ $0x00000008, R11 + JZ copy_5_test + MOVQ (R14)(R15*1), BP + MOVQ BP, (BX)(R15*1) + ADDQ $0x08, R15 + JMP copy_5_test + +copy_5: + MOVUPS (R14)(R15*1), X0 + MOVUPS X0, (BX)(R15*1) + ADDQ $0x10, R15 + +copy_5_test: + CMPQ R15, R11 + JB copy_5 + ADDQ R11, BX + ADDQ R11, DI + SUBQ R11, R13 + + // Copy match from the current buffer +copy_match: + TESTQ R13, R13 + JZ handle_loop + MOVQ BX, R11 + SUBQ R12, R11 + + // ml <= mo + CMPQ R13, R12 + JA copy_overlapping_match + + // Copy non-overlapping match + ADDQ R13, DI + MOVQ BX, R12 + ADDQ R13, BX + +copy_2: + MOVUPS (R11), X0 + MOVUPS X0, (R12) + ADDQ $0x10, R11 + ADDQ $0x10, R12 + SUBQ $0x10, R13 + JHI copy_2 + JMP handle_loop + + // Copy overlapping match +copy_overlapping_match: + ADDQ R13, DI + +copy_slow_3: + MOVB (R11), R12 + MOVB R12, (BX) + INCQ R11 + INCQ BX + DECQ R13 + JNZ copy_slow_3 + +handle_loop: + ADDQ $0x18, AX + INCQ DX + CMPQ DX, CX + JB main_loop + +loop_finished: + // Return value + MOVB $0x01, ret+8(FP) + + // Update the context + MOVQ ctx+0(FP), AX + MOVQ DX, 24(AX) + MOVQ DI, 104(AX) + MOVQ 80(AX), CX + SUBQ CX, SI + MOVQ SI, 112(AX) + RET + +error_match_off_too_big: + // Return value + MOVB $0x00, ret+8(FP) + + // Update the context + MOVQ ctx+0(FP), AX + MOVQ DX, 24(AX) + MOVQ DI, 104(AX) + MOVQ 80(AX), CX + SUBQ CX, SI + MOVQ SI, 112(AX) + RET + +empty_seqs: + // Return value + MOVB $0x01, ret+8(FP) + RET + +// func sequenceDecs_executeSimple_safe_amd64(ctx *executeAsmContext) bool +// Requires: SSE +TEXT ·sequenceDecs_executeSimple_safe_amd64(SB), $8-9 + MOVQ ctx+0(FP), R10 + MOVQ 8(R10), CX + TESTQ CX, CX + JZ empty_seqs + MOVQ (R10), AX + MOVQ 24(R10), DX + MOVQ 32(R10), BX + MOVQ 80(R10), SI + MOVQ 104(R10), DI + MOVQ 120(R10), R8 + MOVQ 56(R10), R9 + MOVQ 64(R10), R10 + ADDQ R10, R9 + + // seqsBase += 24 * seqIndex + LEAQ (DX)(DX*2), R11 + SHLQ $0x03, R11 + ADDQ R11, AX + + // outBase += outPosition + ADDQ DI, BX + +main_loop: + MOVQ (AX), R11 + MOVQ 16(AX), R12 + MOVQ 8(AX), R13 + + // Copy literals + TESTQ R11, R11 + JZ check_offset + XORQ R14, R14 TESTQ $0x00000001, R11 JZ copy_1_word MOVB (SI)(R14*1), R15 @@ -1326,18 +1538,46 @@ copy_match: JA copy_overlapping_match // Copy non-overlapping match - ADDQ R13, DI - MOVQ BX, R12 - ADDQ R13, BX + ADDQ R13, DI + XORQ R12, R12 + TESTQ $0x00000001, R13 + JZ copy_2_word + MOVB (R11)(R12*1), R14 + MOVB R14, (BX)(R12*1) + ADDQ $0x01, R12 + +copy_2_word: + TESTQ $0x00000002, R13 + JZ copy_2_dword + MOVW (R11)(R12*1), R14 + MOVW R14, (BX)(R12*1) + ADDQ $0x02, R12 + +copy_2_dword: + TESTQ $0x00000004, R13 + JZ copy_2_qword + MOVL (R11)(R12*1), R14 + MOVL R14, (BX)(R12*1) + ADDQ $0x04, R12 + +copy_2_qword: + TESTQ $0x00000008, R13 + JZ copy_2_test + MOVQ (R11)(R12*1), R14 + MOVQ R14, (BX)(R12*1) + ADDQ $0x08, R12 + JMP copy_2_test copy_2: - MOVUPS (R11), X0 - MOVUPS X0, (R12) - ADDQ $0x10, R11 + MOVUPS (R11)(R12*1), X0 + MOVUPS X0, (BX)(R12*1) ADDQ $0x10, R12 - SUBQ $0x10, R13 - JHI copy_2 - JMP handle_loop + +copy_2_test: + CMPQ R12, R13 + JB copy_2 + ADDQ R13, BX + JMP handle_loop // Copy overlapping match copy_overlapping_match: @@ -1673,45 +1913,16 @@ sequenceDecs_decodeSync_amd64_match_len_ofs_ok: TESTQ AX, AX JZ check_offset XORQ R14, R14 - TESTQ $0x00000001, AX - JZ copy_1_word - MOVB (R11)(R14*1), R15 - MOVB R15, (R10)(R14*1) - ADDQ $0x01, R14 - -copy_1_word: - TESTQ $0x00000002, AX - JZ copy_1_dword - MOVW (R11)(R14*1), R15 - MOVW R15, (R10)(R14*1) - ADDQ $0x02, R14 - -copy_1_dword: - TESTQ $0x00000004, AX - JZ copy_1_qword - MOVL (R11)(R14*1), R15 - MOVL R15, (R10)(R14*1) - ADDQ $0x04, R14 - -copy_1_qword: - TESTQ $0x00000008, AX - JZ copy_1_test - MOVQ (R11)(R14*1), R15 - MOVQ R15, (R10)(R14*1) - ADDQ $0x08, R14 - JMP copy_1_test copy_1: MOVUPS (R11)(R14*1), X0 MOVUPS X0, (R10)(R14*1) ADDQ $0x10, R14 - -copy_1_test: - CMPQ R14, AX - JB copy_1 - ADDQ AX, R11 - ADDQ AX, R10 - ADDQ AX, R12 + CMPQ R14, AX + JB copy_1 + ADDQ AX, R11 + ADDQ AX, R10 + ADDQ AX, R12 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: @@ -2044,60 +2255,55 @@ sequenceDecs_decodeSync_bmi2_fill_2_end: MOVQ CX, 24(SP) // Fill bitreader for state updates - MOVQ R12, (SP) - MOVQ $0x00000808, CX - BEXTRQ CX, R8, R12 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 - JZ sequenceDecs_decodeSync_bmi2_skip_update - - // Update Literal Length State - MOVBQZX SI, R13 - MOVQ $0x00001010, CX - BEXTRQ CX, SI, SI + MOVQ R12, (SP) + MOVQ $0x00000808, CX + BEXTRQ CX, R8, R12 + MOVQ ctx+16(FP), CX + CMPQ 96(CX), $0x00 + JZ sequenceDecs_decodeSync_bmi2_skip_update + LEAQ (SI)(DI*1), R13 + ADDQ R8, R13 + MOVBQZX R13, R13 LEAQ (DX)(R13*1), CX MOVQ AX, R14 MOVQ CX, DX ROLQ CL, R14 BZHIQ R13, R14, R14 - ADDQ R14, SI - // Load ctx.llTable + // Update Offset State + BZHIQ R8, R14, CX + SHRXQ R8, R14, R14 + MOVQ $0x00001010, R13 + BEXTRQ R13, R8, R8 + ADDQ CX, R8 + + // Load ctx.ofTable MOVQ ctx+16(FP), CX - MOVQ (CX), CX - MOVQ (CX)(SI*8), SI + MOVQ 48(CX), CX + MOVQ (CX)(R8*8), R8 // Update Match Length State - MOVBQZX DI, R13 - MOVQ $0x00001010, CX - BEXTRQ CX, DI, DI - LEAQ (DX)(R13*1), CX - MOVQ AX, R14 - MOVQ CX, DX - ROLQ CL, R14 - BZHIQ R13, R14, R14 - ADDQ R14, DI + BZHIQ DI, R14, CX + SHRXQ DI, R14, R14 + MOVQ $0x00001010, R13 + BEXTRQ R13, DI, DI + ADDQ CX, DI // Load ctx.mlTable MOVQ ctx+16(FP), CX MOVQ 24(CX), CX MOVQ (CX)(DI*8), DI - // Update Offset State - MOVBQZX R8, R13 - MOVQ $0x00001010, CX - BEXTRQ CX, R8, R8 - LEAQ (DX)(R13*1), CX - MOVQ AX, R14 - MOVQ CX, DX - ROLQ CL, R14 - BZHIQ R13, R14, R14 - ADDQ R14, R8 + // Update Literal Length State + BZHIQ SI, R14, CX + MOVQ $0x00001010, R13 + BEXTRQ R13, SI, SI + ADDQ CX, SI - // Load ctx.ofTable + // Load ctx.llTable MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX - MOVQ (CX)(R8*8), R8 + MOVQ (CX), CX + MOVQ (CX)(SI*8), SI sequenceDecs_decodeSync_bmi2_skip_update: // Adjust offset @@ -2180,45 +2386,16 @@ sequenceDecs_decodeSync_bmi2_match_len_ofs_ok: TESTQ CX, CX JZ check_offset XORQ R14, R14 - TESTQ $0x00000001, CX - JZ copy_1_word - MOVB (R10)(R14*1), R15 - MOVB R15, (R9)(R14*1) - ADDQ $0x01, R14 - -copy_1_word: - TESTQ $0x00000002, CX - JZ copy_1_dword - MOVW (R10)(R14*1), R15 - MOVW R15, (R9)(R14*1) - ADDQ $0x02, R14 - -copy_1_dword: - TESTQ $0x00000004, CX - JZ copy_1_qword - MOVL (R10)(R14*1), R15 - MOVL R15, (R9)(R14*1) - ADDQ $0x04, R14 - -copy_1_qword: - TESTQ $0x00000008, CX - JZ copy_1_test - MOVQ (R10)(R14*1), R15 - MOVQ R15, (R9)(R14*1) - ADDQ $0x08, R14 - JMP copy_1_test copy_1: MOVUPS (R10)(R14*1), X0 MOVUPS X0, (R9)(R14*1) ADDQ $0x10, R14 - -copy_1_test: - CMPQ R14, CX - JB copy_1 - ADDQ CX, R10 - ADDQ CX, R9 - ADDQ CX, R11 + CMPQ R14, CX + JB copy_1 + ADDQ CX, R10 + ADDQ CX, R9 + ADDQ CX, R11 // Malformed input if seq.mo > t+len(hist) || seq.mo > s.windowSize) check_offset: @@ -3108,60 +3285,55 @@ sequenceDecs_decodeSync_safe_bmi2_fill_2_end: MOVQ CX, 24(SP) // Fill bitreader for state updates - MOVQ R12, (SP) - MOVQ $0x00000808, CX - BEXTRQ CX, R8, R12 - MOVQ ctx+16(FP), CX - CMPQ 96(CX), $0x00 - JZ sequenceDecs_decodeSync_safe_bmi2_skip_update - - // Update Literal Length State - MOVBQZX SI, R13 - MOVQ $0x00001010, CX - BEXTRQ CX, SI, SI + MOVQ R12, (SP) + MOVQ $0x00000808, CX + BEXTRQ CX, R8, R12 + MOVQ ctx+16(FP), CX + CMPQ 96(CX), $0x00 + JZ sequenceDecs_decodeSync_safe_bmi2_skip_update + LEAQ (SI)(DI*1), R13 + ADDQ R8, R13 + MOVBQZX R13, R13 LEAQ (DX)(R13*1), CX MOVQ AX, R14 MOVQ CX, DX ROLQ CL, R14 BZHIQ R13, R14, R14 - ADDQ R14, SI - // Load ctx.llTable + // Update Offset State + BZHIQ R8, R14, CX + SHRXQ R8, R14, R14 + MOVQ $0x00001010, R13 + BEXTRQ R13, R8, R8 + ADDQ CX, R8 + + // Load ctx.ofTable MOVQ ctx+16(FP), CX - MOVQ (CX), CX - MOVQ (CX)(SI*8), SI + MOVQ 48(CX), CX + MOVQ (CX)(R8*8), R8 // Update Match Length State - MOVBQZX DI, R13 - MOVQ $0x00001010, CX - BEXTRQ CX, DI, DI - LEAQ (DX)(R13*1), CX - MOVQ AX, R14 - MOVQ CX, DX - ROLQ CL, R14 - BZHIQ R13, R14, R14 - ADDQ R14, DI + BZHIQ DI, R14, CX + SHRXQ DI, R14, R14 + MOVQ $0x00001010, R13 + BEXTRQ R13, DI, DI + ADDQ CX, DI // Load ctx.mlTable MOVQ ctx+16(FP), CX MOVQ 24(CX), CX MOVQ (CX)(DI*8), DI - // Update Offset State - MOVBQZX R8, R13 - MOVQ $0x00001010, CX - BEXTRQ CX, R8, R8 - LEAQ (DX)(R13*1), CX - MOVQ AX, R14 - MOVQ CX, DX - ROLQ CL, R14 - BZHIQ R13, R14, R14 - ADDQ R14, R8 + // Update Literal Length State + BZHIQ SI, R14, CX + MOVQ $0x00001010, R13 + BEXTRQ R13, SI, SI + ADDQ CX, SI - // Load ctx.ofTable + // Load ctx.llTable MOVQ ctx+16(FP), CX - MOVQ 48(CX), CX - MOVQ (CX)(R8*8), R8 + MOVQ (CX), CX + MOVQ (CX)(SI*8), SI sequenceDecs_decodeSync_safe_bmi2_skip_update: // Adjust offset diff --git a/vendor/github.com/klauspost/compress/zstd/zip.go b/vendor/github.com/klauspost/compress/zstd/zip.go index b53f606a1..29c15c8c4 100644 --- a/vendor/github.com/klauspost/compress/zstd/zip.go +++ b/vendor/github.com/klauspost/compress/zstd/zip.go @@ -18,7 +18,14 @@ const ZipMethodWinZip = 93 // See https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.9.TXT const ZipMethodPKWare = 20 -var zipReaderPool sync.Pool +// zipReaderPool is the default reader pool. +var zipReaderPool = sync.Pool{New: func() interface{} { + z, err := NewReader(nil, WithDecoderLowmem(true), WithDecoderMaxWindow(128<<20), WithDecoderConcurrency(1)) + if err != nil { + panic(err) + } + return z +}} // newZipReader creates a pooled zip decompressor. func newZipReader(opts ...DOption) func(r io.Reader) io.ReadCloser { diff --git a/vendor/github.com/klauspost/compress/zstd/zstd.go b/vendor/github.com/klauspost/compress/zstd/zstd.go index c1c90b4a0..3eb3f1c82 100644 --- a/vendor/github.com/klauspost/compress/zstd/zstd.go +++ b/vendor/github.com/klauspost/compress/zstd/zstd.go @@ -110,17 +110,6 @@ func printf(format string, a ...interface{}) { } } -// matchLenFast does matching, but will not match the last up to 7 bytes. -func matchLenFast(a, b []byte) int { - endI := len(a) & (math.MaxInt32 - 7) - for i := 0; i < endI; i += 8 { - if diff := load64(a, i) ^ load64(b, i); diff != 0 { - return i + bits.TrailingZeros64(diff)>>3 - } - } - return endI -} - // matchLen returns the maximum length. // a must be the shortest of the two. // The function also returns whether all bytes matched. |