diff options
96 files changed, 2907 insertions, 435 deletions
@@ -22,7 +22,7 @@ ETCDIR ?= /etc TMPFILESDIR ?= ${PREFIX}/lib/tmpfiles.d SYSTEMDDIR ?= ${PREFIX}/lib/systemd/system USERSYSTEMDDIR ?= ${PREFIX}/lib/systemd/user -REMOTETAGS ?= !ABISupport remote exclude_graphdriver_btrfs btrfs_noversion exclude_graphdriver_devicemapper containers_image_openpgp +REMOTETAGS ?= remote exclude_graphdriver_btrfs btrfs_noversion exclude_graphdriver_devicemapper containers_image_openpgp BUILDTAGS ?= \ $(shell hack/apparmor_tag.sh) \ $(shell hack/btrfs_installed_tag.sh) \ @@ -44,7 +44,7 @@ ifeq ($(shell go help mod >/dev/null 2>&1 && echo true), true) GO_BUILD=GO111MODULE=on $(GO) build -mod=vendor endif -BUILDTAGS_CROSS ?= ABISupport containers_image_openpgp exclude_graphdriver_btrfs exclude_graphdriver_devicemapper exclude_graphdriver_overlay +BUILDTAGS_CROSS ?= containers_image_openpgp exclude_graphdriver_btrfs exclude_graphdriver_devicemapper exclude_graphdriver_overlay ifneq (,$(findstring varlink,$(BUILDTAGS))) PODMAN_VARLINK_DEPENDENCIES = pkg/varlink/iopodman.go endif @@ -185,7 +185,7 @@ ifeq (,$(findstring systemd,$(BUILDTAGS))) @echo "Podman is being compiled without the systemd build tag. Install libsystemd on \ Ubuntu or systemd-devel on rpm based distro for journald support." endif - $(GO_BUILD) $(BUILDFLAGS) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "ABISupport $(BUILDTAGS)" -o $@ $(PROJECT)/cmd/podman + $(GO_BUILD) $(BUILDFLAGS) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "$(BUILDTAGS)" -o $@ $(PROJECT)/cmd/podman .PHONY: podman podman: bin/podman diff --git a/changelog.txt b/changelog.txt index 47a99ba64..360b83e6d 100644 --- a/changelog.txt +++ b/changelog.txt @@ -555,7 +555,7 @@ * podmanv2: implement pod top * v2 api: implement pods top endpoint * podmanv2 commit - * Bump to buildah v1.14.5 + * Bump to buildah v1.14.5 (Edit 2020-06-03: Addresses CVE-2020-10696) * Add support for containers.conf * API v2 tests: usability improvements * Sanitize port parsing for pods in play kube @@ -878,7 +878,7 @@ * rootlessport: drop Pdeathsig in favor of Kill * rootlessport: fix potential hang * add pkg/seccomp - * Do not copy up when volume is not empty + * Do not copy up when volume is not empty (Edit 2020-06-03: Addresses CVE-2020-1726) * api: pull: fix reference parsing * cmd/podman/pull: refactor code * stats: add SystemUsage @@ -1573,7 +1573,7 @@ * get runtime for podman-remote push earlier * rootless: report the correct error * Report errors when trying to pause rootless containers - * Do not support wildcards on cp + * Do not support wildcards on cp (Edit 2020-06-03: Addresses CVE-2019-18466) * Podman-remote run should wait for exit code * Use exit code constants * exec: Register resize func a bit later diff --git a/cmd/podman/containers/create.go b/cmd/podman/containers/create.go index c8007bc2f..ed09585ba 100644 --- a/cmd/podman/containers/create.go +++ b/cmd/podman/containers/create.go @@ -161,24 +161,25 @@ func createInit(c *cobra.Command) error { if c.Flag("no-hosts").Changed && c.Flag("add-host").Changed { return errors.Errorf("--no-hosts and --add-host cannot be set together") } - if c.Flag("userns").Changed { - cliVals.UserNS = c.Flag("userns").Value.String() - } - if c.Flag("ipc").Changed { - cliVals.IPC = c.Flag("ipc").Value.String() - } - if c.Flag("uts").Changed { - cliVals.UTS = c.Flag("uts").Value.String() - } - if c.Flag("pid").Changed { - cliVals.PID = c.Flag("pid").Value.String() + cliVals.UserNS = c.Flag("userns").Value.String() + // if user did not modify --userns flag and did turn on + // uid/gid mappsings, set userns flag to "private" + if !c.Flag("userns").Changed && cliVals.UserNS == "host" { + if len(cliVals.UIDMap) > 0 || + len(cliVals.GIDMap) > 0 || + cliVals.SubUIDName != "" || + cliVals.SubGIDName != "" { + cliVals.UserNS = "private" + } } + + cliVals.IPC = c.Flag("ipc").Value.String() + cliVals.UTS = c.Flag("uts").Value.String() + cliVals.PID = c.Flag("pid").Value.String() + cliVals.CGroupsNS = c.Flag("cgroupns").Value.String() if !c.Flag("pids-limit").Changed { cliVals.PIDsLimit = -1 } - if c.Flag("cgroupns").Changed { - cliVals.CGroupsNS = c.Flag("cgroupns").Value.String() - } if c.Flag("entrypoint").Changed { val := c.Flag("entrypoint").Value.String() cliVals.Entrypoint = &val diff --git a/cmd/podman/containers/inspect.go b/cmd/podman/containers/inspect.go index 4549a4ef6..8556ebe83 100644 --- a/cmd/podman/containers/inspect.go +++ b/cmd/podman/containers/inspect.go @@ -26,9 +26,15 @@ func init() { Command: inspectCmd, Parent: containerCmd, }) - inspectOpts = inspect.AddInspectFlagSet(inspectCmd) + inspectOpts = new(entities.InspectOptions) + flags := inspectCmd.Flags() + flags.BoolVarP(&inspectOpts.Size, "size", "s", false, "Display total file size") + flags.StringVarP(&inspectOpts.Format, "format", "f", "json", "Format the output to a Go template or json") + flags.BoolVarP(&inspectOpts.Latest, "latest", "l", false, "Act on the latest container Podman is aware of") } func inspectExec(cmd *cobra.Command, args []string) error { + // Force container type + inspectOpts.Type = inspect.ContainerType return inspect.Inspect(args, *inspectOpts) } diff --git a/cmd/podman/containers/run.go b/cmd/podman/containers/run.go index 890c6e827..8a02c63c0 100644 --- a/cmd/podman/containers/run.go +++ b/cmd/podman/containers/run.go @@ -3,6 +3,7 @@ package containers import ( "fmt" "os" + "strconv" "strings" "github.com/containers/libpod/cmd/podman/common" @@ -10,7 +11,9 @@ import ( "github.com/containers/libpod/libpod/define" "github.com/containers/libpod/pkg/domain/entities" "github.com/containers/libpod/pkg/errorhandling" + "github.com/containers/libpod/pkg/rootless" "github.com/containers/libpod/pkg/specgen" + "github.com/containers/libpod/pkg/util" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -92,6 +95,15 @@ func run(cmd *cobra.Command, args []string) error { return err } + if rootless.IsRootless() && !registry.IsRemote() { + userspec := strings.SplitN(cliVals.User, ":", 2)[0] + if uid, err := strconv.ParseInt(userspec, 10, 32); err == nil { + if err := util.CheckRootlessUIDRange(int(uid)); err != nil { + return err + } + } + } + if af := cliVals.Authfile; len(af) > 0 { if _, err := os.Stat(af); err != nil { return errors.Wrapf(err, "error checking authfile path %s", af) diff --git a/cmd/podman/containers/wait.go b/cmd/podman/containers/wait.go index 1f4d4159b..ca3883091 100644 --- a/cmd/podman/containers/wait.go +++ b/cmd/podman/containers/wait.go @@ -24,8 +24,7 @@ var ( Long: waitDescription, RunE: wait, Args: validate.IdOrLatestArgs, - Example: `podman wait --latest - podman wait --interval 5000 ctrID + Example: `podman wait --interval 5000 ctrID podman wait ctrID1 ctrID2`, } @@ -35,8 +34,7 @@ var ( Long: waitCommand.Long, RunE: waitCommand.RunE, Args: validate.IdOrLatestArgs, - Example: `podman container wait --latest - podman container wait --interval 5000 ctrID + Example: `podman container wait --interval 5000 ctrID podman container wait ctrID1 ctrID2`, } ) @@ -48,11 +46,9 @@ var ( func waitFlags(flags *pflag.FlagSet) { flags.DurationVarP(&waitOptions.Interval, "interval", "i", time.Duration(250), "Milliseconds to wait before polling for completion") - flags.BoolVarP(&waitOptions.Latest, "latest", "l", false, "Act on the latest container podman is aware of") flags.StringVar(&waitCondition, "condition", "stopped", "Condition to wait on") - if registry.IsRemote() { - // TODO: This is the same as V1. We could skip creating the flag altogether in V2... - _ = flags.MarkHidden("latest") + if !registry.IsRemote() { + flags.BoolVarP(&waitOptions.Latest, "latest", "l", false, "Act on the latest container podman is aware of") } } diff --git a/cmd/podman/images/inspect.go b/cmd/podman/images/inspect.go index 8c727eb07..f6a10ba44 100644 --- a/cmd/podman/images/inspect.go +++ b/cmd/podman/images/inspect.go @@ -27,11 +27,12 @@ func init() { Command: inspectCmd, Parent: imageCmd, }) - inspectOpts = inspect.AddInspectFlagSet(inspectCmd) + inspectOpts = new(entities.InspectOptions) flags := inspectCmd.Flags() - _ = flags.MarkHidden("latest") // Shared with container-inspect but not wanted here. + flags.StringVarP(&inspectOpts.Format, "format", "f", "json", "Format the output to a Go template or json") } func inspectExec(cmd *cobra.Command, args []string) error { + inspectOpts.Type = inspect.ImageType return inspect.Inspect(args, *inspectOpts) } diff --git a/cmd/podman/images/list.go b/cmd/podman/images/list.go index 4f8948b8b..23757104b 100644 --- a/cmd/podman/images/list.go +++ b/cmd/podman/images/list.go @@ -234,11 +234,7 @@ func imageListFormat(flags listFlagType) (string, string) { } hdr += "\tIMAGE ID" - if flags.noTrunc { - row += "\tsha256:{{.ID}}" - } else { - row += "\t{{.ID}}" - } + row += "\t{{.ID}}" hdr += "\tCREATED\tSIZE" row += "\t{{.Created}}\t{{.Size}}" diff --git a/cmd/podman/registry/config_abi.go b/cmd/podman/registry/config_abi.go index 55430e1bf..4a909c17e 100644 --- a/cmd/podman/registry/config_abi.go +++ b/cmd/podman/registry/config_abi.go @@ -1,4 +1,4 @@ -// +build ABISupport +// +build !remote package registry diff --git a/cmd/podman/registry/config_tunnel.go b/cmd/podman/registry/config_tunnel.go index 29e744dac..bb3da947e 100644 --- a/cmd/podman/registry/config_tunnel.go +++ b/cmd/podman/registry/config_tunnel.go @@ -1,4 +1,4 @@ -// +build !ABISupport +// +build remote package registry diff --git a/cmd/podman/root.go b/cmd/podman/root.go index 3796b8e27..59d536d0b 100644 --- a/cmd/podman/root.go +++ b/cmd/podman/root.go @@ -103,6 +103,11 @@ func persistentPreRunE(cmd *cobra.Command, args []string) error { cfg := registry.PodmanConfig() + // Help is a special case, no need for more setup + if cmd.Name() == "help" { + return nil + } + // Prep the engines if _, err := registry.NewImageEngine(cmd, args); err != nil { return err @@ -150,6 +155,11 @@ func persistentPostRunE(cmd *cobra.Command, args []string) error { // TODO: Remove trace statement in podman V2.1 logrus.Debugf("Called %s.PersistentPostRunE(%s)", cmd.Name(), strings.Join(os.Args, " ")) + // Help is a special case, no need for more cleanup + if cmd.Name() == "help" { + return nil + } + cfg := registry.PodmanConfig() if cmd.Flag("cpu-profile").Changed { pprof.StopCPUProfile() @@ -191,8 +201,11 @@ func loggingHook() { func rootFlags(opts *entities.PodmanConfig, flags *pflag.FlagSet) { // V2 flags - flags.StringVarP(&opts.Uri, "remote", "r", registry.DefaultAPIAddress(), "URL to access Podman service") - flags.StringSliceVar(&opts.Identities, "identity", []string{}, "path to SSH identity file") + flags.BoolVarP(&opts.Remote, "remote", "r", false, "Access remote Podman service (default false)") + // TODO Read uri from containers.config when available + flags.StringVar(&opts.Uri, "url", registry.DefaultAPIAddress(), "URL to access Podman service (CONTAINER_HOST)") + flags.StringSliceVar(&opts.Identities, "identity", []string{}, "path to SSH identity file, (CONTAINER_SSHKEY)") + flags.StringVar(&opts.PassPhrase, "passphrase", "", "passphrase for identity file (not secure, CONTAINER_PASSPHRASE), ssh-agent always supported") cfg := opts.Config flags.StringVar(&cfg.Engine.CgroupManager, "cgroup-manager", cfg.Engine.CgroupManager, "Cgroup manager to use (\"cgroupfs\"|\"systemd\")") diff --git a/cmd/podman/system/service_abi.go b/cmd/podman/system/service_abi.go index 501650839..f5386c4f1 100644 --- a/cmd/podman/system/service_abi.go +++ b/cmd/podman/system/service_abi.go @@ -1,4 +1,4 @@ -// +build ABISupport,!remote +// +build linux,!remote package system diff --git a/cmd/podman/system/service_unsupported.go b/cmd/podman/system/service_unsupported.go deleted file mode 100644 index 82272c882..000000000 --- a/cmd/podman/system/service_unsupported.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build !ABISupport,!remote - -package system - -import ( - "errors" - - "github.com/containers/libpod/pkg/domain/entities" - "github.com/spf13/pflag" -) - -func restService(opts entities.ServiceOptions, flags *pflag.FlagSet, cfg *entities.PodmanConfig) error { - return errors.New("not supported") -} diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 945b33909..25b7ff941 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -46,6 +46,9 @@ case "${OS_RELEASE_ID}" in workaround_bfq_bug + # HACK: Need Conmon 2.0.17, currently in updates-testing on F31. + dnf update -y --enablerepo=updates-testing conmon + if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then bash "$SCRIPT_BASE/add_second_partition.sh" fi diff --git a/contrib/gate/Dockerfile b/contrib/gate/Dockerfile index f86709b00..657b5accf 100644 --- a/contrib/gate/Dockerfile +++ b/contrib/gate/Dockerfile @@ -13,7 +13,7 @@ COPY . $GOSRC # Note: adding conmon and crun so podman command checks will work RUN dnf -y install \ $(grep "^[^#]" $GOSRC/contrib/dependencies.txt) diffutils containers-common fuse-overlayfs conmon crun runc --exclude container-selinux; \ - sed -i -e 's|^#mount_program|mount_program|g' /etc/containers/storage.conf \ + sed -i -e 's|^#mount_program|mount_program|g' -e 's/# size.*/skip_mount_home = "true"/g' /etc/containers/storage.conf \ && dnf clean all # Install dependencies diff --git a/contrib/spec/podman.spec.in b/contrib/spec/podman.spec.in index 701b5f0b1..8d3cba612 100644 --- a/contrib/spec/podman.spec.in +++ b/contrib/spec/podman.spec.in @@ -33,7 +33,7 @@ # People want conmon packaged with the copr rpm %global import_path_conmon github.com/containers/conmon %global git_conmon https://%{import_path_conmon} -%global commit_conmon d532caebc788fafdd2a305b68cd1983b4039bea4 +%global commit_conmon 41877362fc4685d55e0473d2e4a1cbe5e1debee0 %global shortcommit_conmon %(c=%{commit_conmon}; echo ${c:0:7}) Name: podman @@ -80,8 +80,12 @@ Requires: iptables %if 0%{?rhel} <= 7 Requires: container-selinux %else +%if 0%{?rhel} || 0%{?centos} +Requires: runc +%else Requires: oci-runtime Recommends: crun +%endif Recommends: container-selinux Recommends: slirp4netns Recommends: fuse-overlayfs diff --git a/docs/source/markdown/podman.1.md b/docs/source/markdown/podman.1.md index 02f23e6cc..8dc486b65 100644 --- a/docs/source/markdown/podman.1.md +++ b/docs/source/markdown/podman.1.md @@ -61,6 +61,9 @@ Podman and libpod currently support an additional `precreate` state which is cal **--identity**=*path* Path to SSH identity file +**--passphrase**=*secret* +pass phrase for SSH identity file + **--log-level**=*level* Log messages above specified level: debug, info, warn, error (default), fatal or panic (default: "error") @@ -73,18 +76,21 @@ When namespace is set, created containers and pods will join the given namespace **--network-cmd-path**=*path* Path to the command binary to use for setting up a network. It is currently only used for setting up a slirp4netns network. If "" is used then the binary is looked up using the $PATH environment variable. -**--remote**, **-r**=*url* -URL to access Podman service (default "unix:/run/user/3267/podman/podman.sock") +**--remote**, **-r** +Access Podman service will be remote + +**--url**=*value* +URL to access Podman service (default from `containers.conf`, rootless "unix://run/user/$UID/podman/podman.sock" or as root "unix://run/podman/podman.sock). **--root**=*value* Storage root dir in which data, including images, is stored (default: "/var/lib/containers/storage" for UID 0, "$HOME/.local/share/containers/storage" for other users). -Default root dir is configured in `/etc/containers/storage.conf`. +Default root dir configured in `/etc/containers/storage.conf`. **--runroot**=*value* Storage state directory where all state information is stored (default: "/var/run/containers/storage" for UID 0, "/var/run/user/$UID/run" for other users). -Default state dir is configured in `/etc/containers/storage.conf`. +Default state dir configured in `/etc/containers/storage.conf`. **--runtime**=*value* @@ -14,9 +14,9 @@ require ( github.com/containers/common v0.12.0 github.com/containers/conmon v2.0.17+incompatible github.com/containers/image/v5 v5.4.5-0.20200529084758-46b2ee6aebb0 - github.com/containers/psgo v1.5.0 - github.com/containers/storage v1.20.1 - github.com/coreos/go-systemd/v22 v22.0.0 + github.com/containers/psgo v1.5.1 + github.com/containers/storage v1.20.2 + github.com/coreos/go-systemd/v22 v22.1.0 github.com/cri-o/ocicni v0.2.0 github.com/cyphar/filepath-securejoin v0.2.2 github.com/davecgh/go-spew v1.1.1 @@ -35,11 +35,11 @@ require ( github.com/hpcloud/tail v1.0.0 github.com/json-iterator/go v1.1.9 github.com/mrunalp/fileutils v0.0.0-20171103030105-7d4729fb3618 - github.com/onsi/ginkgo v1.12.2 + github.com/onsi/ginkgo v1.12.3 github.com/onsi/gomega v1.10.1 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 - github.com/opencontainers/runc v1.0.0-rc9 + github.com/opencontainers/runc v1.0.0-rc90 github.com/opencontainers/runtime-spec v1.0.3-0.20200520003142-237cc4f519e2 github.com/opencontainers/runtime-tools v0.9.0 github.com/opencontainers/selinux v1.5.2 @@ -47,6 +47,7 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd h1:qMd81Ts1T github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1 h1:uict5mhHFTzKLUCufdSLym7z/J0CbBJT59lYbP9wtbg= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.2.10/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -81,13 +82,15 @@ github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDpl github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.0.2 h1:Q0/IPs8ohfbXNxEfyJ2pFVmvJu5BhqJUAmc6ES9NKbo= github.com/containers/ocicrypt v1.0.2/go.mod h1:nsOhbP19flrX6rE7ieGFvBlr7modwmNjsqWarIUce4M= -github.com/containers/psgo v1.5.0 h1:uofUREsrm0Ls5K4tkEIFPqWSHKyg3Bvoqo/Q2eDmj8g= -github.com/containers/psgo v1.5.0/go.mod h1:2ubh0SsreMZjSXW1Hif58JrEcFudQyIy9EzPUWfawVU= +github.com/containers/psgo v1.5.1 h1:MQNb7FLbXqBdqz6u4lI2QWizVz4RSTzs1+Nk9XT1iVA= +github.com/containers/psgo v1.5.1/go.mod h1:2ubh0SsreMZjSXW1Hif58JrEcFudQyIy9EzPUWfawVU= github.com/containers/storage v1.18.2/go.mod h1:WTBMf+a9ZZ/LbmEVeLHH2TX4CikWbO1Bt+/m58ZHVPg= github.com/containers/storage v1.19.1/go.mod h1:KbXjSwKnx17ejOsjFcCXSf78mCgZkQSLPBNTMRc3XrQ= github.com/containers/storage v1.19.2/go.mod h1:gYCp3jzgXkvubO0rI14QAjz5Mxm/qKJgLmHFyqayDnw= github.com/containers/storage v1.20.1 h1:2XE4eRIqSa6YjhAZjNwIkIKE6+Miy+5WV8l1KzY2ZKk= github.com/containers/storage v1.20.1/go.mod h1:RoKzO8KSDogCT6c06rEbanZTcKYxshorB33JikEGc3A= +github.com/containers/storage v1.20.2 h1:tw/uKRPDnmVrluIzer3dawTFG/bTJLP8IEUyHFhltYk= +github.com/containers/storage v1.20.2/go.mod h1:oOB9Ie8OVPojvoaKWEGSEtHbXUAs+tSyr7RO7ZGteMc= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5 h1:DpHb9vJrZQEFMcVLFKAAGMUVX0XoRC0ptCthinRYm38= @@ -96,8 +99,8 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= -github.com/coreos/go-systemd/v22 v22.0.0 h1:XJIw/+VlJ+87J+doOxznsAWIdmWuViOVhkQamW5YV28= -github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.1.0 h1:kq/SbG2BCKLkDKkjQf5OWwKWUKj1lgs3lFI4PxnR5lg= +github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -256,6 +259,8 @@ github.com/klauspost/compress v1.10.5 h1:7q6vHIqubShURwQz8cQK6yIe/xC3IF0Vm7TGfqj github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.6 h1:SP6zavvTG3YjOosWePXFDlExpKIWMTO4SE/Y8MZB2vI= github.com/klauspost/compress v1.10.6/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= +github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/pgzip v1.2.3/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -315,6 +320,8 @@ github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0 github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.12.2 h1:Ke9m3h2Hu0wsZ45yewCqhYr3Z+emcNTuLY2nMWCkrSI= github.com/onsi/ginkgo v1.12.2/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= +github.com/onsi/ginkgo v1.12.3 h1:+RYp9QczoWz9zfUyLP/5SLXQVhfr6gZOoKGfQqHuLZQ= +github.com/onsi/ginkgo v1.12.3/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= github.com/onsi/gomega v0.0.0-20151007035656-2152b45fa28a/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v0.0.0-20190113212917-5533ce8a0da3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -336,6 +343,8 @@ github.com/opencontainers/runc v0.0.0-20190425234816-dae70e8efea4/go.mod h1:qT5X github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runc v1.0.0-rc9 h1:/k06BMULKF5hidyoZymkoDCzdJzltZpz/UU4LguQVtc= github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc90 h1:4+xo8mtWixbHoEm451+WJNUrq12o2/tDsyK9Vgc/NcA= +github.com/opencontainers/runc v1.0.0-rc90/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v0.1.2-0.20190618234442-a950415649c7/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.3-0.20200520003142-237cc4f519e2 h1:9mv9SC7GWmRWE0J/+oD8w3GsN2KYGKtg6uwLN7hfP5E= @@ -448,6 +457,7 @@ github.com/uber/jaeger-lib v2.2.0+incompatible/go.mod h1:ComeNDZlWwrWnDv8aPp0Ba6 github.com/ugorji/go v1.1.4/go.mod h1:uQMGLiO92mf5W77hV/PUCpI3pbzQx3CRekS0kk+RGrc= github.com/ulikunitz/xz v0.5.7 h1:YvTNdFzX6+W5m9msiYg/zpkSURPPtOlzbqYjrFn7Yt4= github.com/ulikunitz/xz v0.5.7/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5 h1:MCfT24H3f//U5+UCrZp1/riVO3B50BovxtDiNn0XKkk= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli/v2 v2.2.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/varlink/go v0.0.0-20190502142041-0f1d566d194b h1:hdDRrn9OP/roL8a/e/5Zu85ldrcdndu9IeBj2OEvQm0= @@ -554,6 +564,7 @@ golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191127021746-63cb32ae39b2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/hack/golangci-lint.sh b/hack/golangci-lint.sh index f4e60d8f5..8c81a3743 100755 --- a/hack/golangci-lint.sh +++ b/hack/golangci-lint.sh @@ -5,12 +5,12 @@ declare -A BUILD_TAGS # TODO: add systemd tag BUILD_TAGS[default]="apparmor,seccomp,selinux" -BUILD_TAGS[abi]="${BUILD_TAGS[default]},ABISupport,varlink,!remoteclient" -BUILD_TAGS[tunnel]="${BUILD_TAGS[default]},!ABISupport,varlink,remoteclient" +BUILD_TAGS[abi]="${BUILD_TAGS[default]},varlink,!remoteclient" +BUILD_TAGS[tunnel]="${BUILD_TAGS[default]},remote,varlink,remoteclient" declare -A SKIP_DIRS SKIP_DIRS[abi]="" -# TODO: add "ABISupport" build tag to pkg/api +# TODO: add "remote" build tag to pkg/api SKIP_DIRS[tunnel]="pkg/api" [[ $1 == run ]] && shift diff --git a/libpod/container_exec.go b/libpod/container_exec.go index f2943b73c..a0e8904dc 100644 --- a/libpod/container_exec.go +++ b/libpod/container_exec.go @@ -69,6 +69,10 @@ type ExecConfig struct { // The ID of the exec session, and the ID of the container the exec // session is a part of (in that order). ExitCommand []string `json:"exitCommand,omitempty"` + // ExitCommandDelay is a delay (in seconds) between the container + // exiting, and the exit command being executed. If set to 0, there is + // no delay. If set, ExitCommand must also be set. + ExitCommandDelay uint `json:"exitCommandDelay,omitempty"` } // ExecSession contains information on a single exec session attached to a given @@ -165,6 +169,9 @@ func (c *Container) ExecCreate(config *ExecConfig) (string, error) { if len(config.Command) == 0 { return "", errors.Wrapf(define.ErrInvalidArg, "must provide a non-empty command to start an exec session") } + if config.ExitCommandDelay > 0 && len(config.ExitCommand) == 0 { + return "", errors.Wrapf(define.ErrInvalidArg, "must provide a non-empty exit command if giving an exit command delay") + } // Verify that we are in a good state to continue if !c.ensureState(define.ContainerStateRunning) { @@ -984,6 +991,7 @@ func prepareForExec(c *Container, session *ExecSession) (*ExecOptions, error) { opts.PreserveFDs = session.Config.PreserveFDs opts.DetachKeys = session.Config.DetachKeys opts.ExitCommand = session.Config.ExitCommand + opts.ExitCommandDelay = session.Config.ExitCommandDelay return opts, nil } diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index 2bd6099f0..d08e012a6 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -325,6 +325,11 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { } if c.config.User != "" { + if rootless.IsRootless() { + if err := util.CheckRootlessUIDRange(execUser.Uid); err != nil { + return nil, err + } + } // User and Group must go together g.SetProcessUID(uint32(execUser.Uid)) g.SetProcessGID(uint32(execUser.Gid)) diff --git a/libpod/oci.go b/libpod/oci.go index 7c5218319..684a7ba42 100644 --- a/libpod/oci.go +++ b/libpod/oci.go @@ -172,6 +172,9 @@ type ExecOptions struct { // ExitCommand is a command that will be run after the exec session // exits. ExitCommand []string + // ExitCommandDelay is a delay (in seconds) between the exec session + // exiting, and the exit command being invoked. + ExitCommandDelay uint } // HTTPAttachStreams informs the HTTPAttach endpoint which of the container's diff --git a/libpod/oci_conmon_exec_linux.go b/libpod/oci_conmon_exec_linux.go index 51819f90a..bc39100f8 100644 --- a/libpod/oci_conmon_exec_linux.go +++ b/libpod/oci_conmon_exec_linux.go @@ -421,12 +421,14 @@ func (r *ConmonOCIRuntime) startExec(c *Container, sessionID string, options *Ex for _, arg := range options.ExitCommand[1:] { args = append(args, []string{"--exit-command-arg", arg}...) } + if options.ExitCommandDelay > 0 { + args = append(args, []string{"--exit-delay", fmt.Sprintf("%d", options.ExitCommandDelay)}...) + } } logrus.WithFields(logrus.Fields{ "args": args, }).Debugf("running conmon: %s", r.conmonPath) - // TODO: Need to pass this back so we can wait on it. execCmd := exec.Command(r.conmonPath, args...) // TODO: This is commented because it doesn't make much sense in HTTP diff --git a/pkg/api/handlers/compat/exec.go b/pkg/api/handlers/compat/exec.go index 6865a3319..8f7016903 100644 --- a/pkg/api/handlers/compat/exec.go +++ b/pkg/api/handlers/compat/exec.go @@ -10,6 +10,7 @@ import ( "github.com/containers/libpod/libpod/define" "github.com/containers/libpod/pkg/api/handlers" "github.com/containers/libpod/pkg/api/handlers/utils" + "github.com/containers/libpod/pkg/specgen/generate" "github.com/gorilla/mux" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -54,6 +55,24 @@ func ExecCreateHandler(w http.ResponseWriter, r *http.Request) { libpodConfig.Privileged = input.Privileged libpodConfig.User = input.User + // Make our exit command + storageConfig := runtime.StorageConfig() + runtimeConfig, err := runtime.GetConfig() + if err != nil { + utils.InternalServerError(w, err) + return + } + exitCommandArgs, err := generate.CreateExitCommandArgs(storageConfig, runtimeConfig, false, true, true) + if err != nil { + utils.InternalServerError(w, err) + return + } + libpodConfig.ExitCommand = exitCommandArgs + + // Run the exit command after 5 minutes, to mimic Docker's exec cleanup + // behavior. + libpodConfig.ExitCommandDelay = 5 * 60 + sessID, err := ctr.ExecCreate(libpodConfig) if err != nil { if errors.Cause(err) == define.ErrCtrStateInvalid { @@ -104,15 +123,6 @@ func ExecInspectHandler(w http.ResponseWriter, r *http.Request) { } utils.WriteResponse(w, http.StatusOK, inspectOut) - - // Only for the Compat API: we want to remove sessions that were - // stopped. This is very hacky, but should suffice for now. - if !utils.IsLibpodRequest(r) && inspectOut.CanRemove { - logrus.Infof("Pruning stale exec session %s from container %s", sessionID, sessionCtr.ID()) - if err := sessionCtr.ExecRemove(sessionID, false); err != nil && errors.Cause(err) != define.ErrNoSuchExecSession { - logrus.Errorf("Error removing stale exec session %s from container %s: %v", sessionID, sessionCtr.ID(), err) - } - } } // ExecStartHandler runs a given exec session. @@ -121,7 +131,7 @@ func ExecStartHandler(w http.ResponseWriter, r *http.Request) { sessionID := mux.Vars(r)["id"] - // TODO: We should read/support Tty and Detach from here. + // TODO: We should read/support Tty from here. bodyParams := new(handlers.ExecStartConfig) if err := json.NewDecoder(r.Body).Decode(&bodyParams); err != nil { @@ -129,11 +139,6 @@ func ExecStartHandler(w http.ResponseWriter, r *http.Request) { errors.Wrapf(err, "failed to decode parameters for %s", r.URL.String())) return } - if bodyParams.Detach { - utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, - errors.Errorf("Detached exec is not yet supported")) - return - } // TODO: Verify TTY setting against what inspect session was made with sessionCtr, err := runtime.GetExecSessionContainer(sessionID) @@ -154,6 +159,19 @@ func ExecStartHandler(w http.ResponseWriter, r *http.Request) { return } + if bodyParams.Detach { + // If we are detaching, we do NOT want to hijack. + // Instead, we perform a detached start, and return 200 if + // successful. + if err := sessionCtr.ExecStart(sessionID); err != nil { + utils.InternalServerError(w, err) + return + } + // This is a 200 despite having no content + utils.WriteResponse(w, http.StatusOK, "") + return + } + // Hijack the connection hijacker, ok := w.(http.Hijacker) if !ok { diff --git a/pkg/api/server/register_exec.go b/pkg/api/server/register_exec.go index 17181d286..af9a83496 100644 --- a/pkg/api/server/register_exec.go +++ b/pkg/api/server/register_exec.go @@ -13,7 +13,7 @@ func (s *APIServer) registerExecHandlers(r *mux.Router) error { // tags: // - exec (compat) // summary: Create an exec instance - // description: Run a command inside a running container. + // description: Create an exec session to run a command inside a running container. Exec sessions will be automatically removed 5 minutes after they exit. // parameters: // - in: path // name: name @@ -153,7 +153,7 @@ func (s *APIServer) registerExecHandlers(r *mux.Router) error { // tags: // - exec (compat) // summary: Inspect an exec instance - // description: Return low-level information about an exec instance. Stale (stopped) exec sessions will be auto-removed after inspect runs. + // description: Return low-level information about an exec instance. // parameters: // - in: path // name: id @@ -182,7 +182,7 @@ func (s *APIServer) registerExecHandlers(r *mux.Router) error { // tags: // - exec // summary: Create an exec instance - // description: Run a command inside a running container. + // description: Create an exec session to run a command inside a running container. Exec sessions will be automatically removed 5 minutes after they exit. // parameters: // - in: path // name: name diff --git a/pkg/bindings/bindings.go b/pkg/bindings/bindings.go index 7e2a444bd..da47ea713 100644 --- a/pkg/bindings/bindings.go +++ b/pkg/bindings/bindings.go @@ -9,7 +9,13 @@ package bindings import ( + "errors" + "fmt" + "io" + "os" + "github.com/blang/semver" + "golang.org/x/crypto/ssh/terminal" ) var ( @@ -25,3 +31,40 @@ var ( // _*YES*- podman will fail to run if this value is wrong APIVersion = semver.MustParse("1.0.0") ) + +// readPassword prompts for a secret and returns value input by user from stdin +// Unlike terminal.ReadPassword(), $(echo $SECRET | podman...) is supported. +// Additionally, all input after `<secret>/n` is queued to podman command. +func readPassword(prompt string) (pw []byte, err error) { + fd := int(os.Stdin.Fd()) + if terminal.IsTerminal(fd) { + fmt.Fprint(os.Stderr, prompt) + pw, err = terminal.ReadPassword(fd) + fmt.Fprintln(os.Stderr) + return + } + + var b [1]byte + for { + n, err := os.Stdin.Read(b[:]) + // terminal.ReadPassword discards any '\r', so we do the same + if n > 0 && b[0] != '\r' { + if b[0] == '\n' { + return pw, nil + } + pw = append(pw, b[0]) + // limit size, so that a wrong input won't fill up the memory + if len(pw) > 1024 { + err = errors.New("password too long, 1024 byte limit") + } + } + if err != nil { + // terminal.ReadPassword accepts EOF-terminated passwords + // if non-empty, so we do the same + if err == io.EOF && len(pw) > 0 { + err = nil + } + return pw, err + } + } +} diff --git a/pkg/bindings/connection.go b/pkg/bindings/connection.go index c26093a7f..b130b9598 100644 --- a/pkg/bindings/connection.go +++ b/pkg/bindings/connection.go @@ -13,6 +13,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "time" "github.com/blang/semver" @@ -20,6 +21,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" "k8s.io/client-go/util/homedir" ) @@ -29,6 +31,8 @@ var ( Host: "d", Path: "/v" + APIVersion.String() + "/libpod", } + passPhrase []byte + phraseSync sync.Once ) type APIResponse struct { @@ -61,6 +65,10 @@ func JoinURL(elements ...string) string { return "/" + strings.Join(elements, "/") } +func NewConnection(ctx context.Context, uri string) (context.Context, error) { + return NewConnectionWithIdentity(ctx, uri, "") +} + // NewConnection takes a URI as a string and returns a context with the // Connection embedded as a value. This context needs to be passed to each // endpoint to work correctly. @@ -69,23 +77,28 @@ func JoinURL(elements ...string) string { // For example tcp://localhost:<port> // or unix:///run/podman/podman.sock // or ssh://<user>@<host>[:port]/run/podman/podman.sock?secure=True -func NewConnection(ctx context.Context, uri string, identity ...string) (context.Context, error) { +func NewConnectionWithIdentity(ctx context.Context, uri string, passPhrase string, identities ...string) (context.Context, error) { var ( err error secure bool ) - if v, found := os.LookupEnv("PODMAN_HOST"); found { + if v, found := os.LookupEnv("CONTAINER_HOST"); found && uri == "" { uri = v } - if v, found := os.LookupEnv("PODMAN_SSHKEY"); found { - identity = []string{v} + if v, found := os.LookupEnv("CONTAINER_SSHKEY"); found && len(identities) == 0 { + identities = append(identities, v) + } + + if v, found := os.LookupEnv("CONTAINER_PASSPHRASE"); found && passPhrase == "" { + passPhrase = v } _url, err := url.Parse(uri) if err != nil { - return nil, errors.Wrapf(err, "Value of PODMAN_HOST is not a valid url: %s", uri) + return nil, errors.Wrapf(err, "Value of CONTAINER_HOST is not a valid url: %s", uri) } + // TODO Fill in missing defaults for _url... // Now we setup the http Client to use the connection above var connection Connection @@ -95,7 +108,7 @@ func NewConnection(ctx context.Context, uri string, identity ...string) (context if err != nil { secure = false } - connection, err = sshClient(_url, identity[0], secure) + connection, err = sshClient(_url, secure, passPhrase, identities...) case "unix": if !strings.HasPrefix(uri, "unix:///") { // autofix unix://path_element vs unix:///path_element @@ -172,10 +185,31 @@ func pingNewConnection(ctx context.Context) error { return errors.Errorf("ping response was %q", response.StatusCode) } -func sshClient(_url *url.URL, identity string, secure bool) (Connection, error) { - auth, err := publicKey(identity) - if err != nil { - return Connection{}, errors.Wrapf(err, "Failed to parse identity %s: %v\n", _url.String(), identity) +func sshClient(_url *url.URL, secure bool, passPhrase string, identities ...string) (Connection, error) { + var authMethods []ssh.AuthMethod + + for _, i := range identities { + auth, err := publicKey(i, []byte(passPhrase)) + if err != nil { + fmt.Fprint(os.Stderr, errors.Wrapf(err, "failed to parse identity %q", i).Error()+"\n") + continue + } + authMethods = append(authMethods, auth) + } + + if sock, found := os.LookupEnv("SSH_AUTH_SOCK"); found { + logrus.Debugf("Found SSH_AUTH_SOCK %q, ssh-agent signer enabled", sock) + + c, err := net.Dial("unix", sock) + if err != nil { + return Connection{}, err + } + a := agent.NewClient(c) + authMethods = append(authMethods, ssh.PublicKeysCallback(a.Signers)) + } + + if pw, found := _url.User.Password(); found { + authMethods = append(authMethods, ssh.Password(pw)) } callback := ssh.InsecureIgnoreHostKey() @@ -195,7 +229,7 @@ func sshClient(_url *url.URL, identity string, secure bool) (Connection, error) net.JoinHostPort(_url.Hostname(), port), &ssh.ClientConfig{ User: _url.User.Username(), - Auth: []ssh.AuthMethod{auth}, + Auth: authMethods, HostKeyCallback: callback, HostKeyAlgorithms: []string{ ssh.KeyAlgoRSA, @@ -307,7 +341,7 @@ func (h *APIResponse) IsServerError() bool { return h.Response.StatusCode/100 == 5 } -func publicKey(path string) (ssh.AuthMethod, error) { +func publicKey(path string, passphrase []byte) (ssh.AuthMethod, error) { key, err := ioutil.ReadFile(path) if err != nil { return nil, err @@ -315,12 +349,30 @@ func publicKey(path string) (ssh.AuthMethod, error) { signer, err := ssh.ParsePrivateKey(key) if err != nil { - return nil, err + if _, ok := err.(*ssh.PassphraseMissingError); !ok { + return nil, err + } + if len(passphrase) == 0 { + phraseSync.Do(promptPassphrase) + passphrase = passPhrase + } + signer, err = ssh.ParsePrivateKeyWithPassphrase(key, passphrase) + if err != nil { + return nil, err + } } - return ssh.PublicKeys(signer), nil } +func promptPassphrase() { + phrase, err := readPassword("Key Passphrase: ") + if err != nil { + passPhrase = []byte{} + return + } + passPhrase = phrase +} + func hostKey(host string) ssh.PublicKey { // parse OpenSSH known_hosts file // ssh or use ssh-keyscan to get initial key diff --git a/pkg/bindings/containers/exec.go b/pkg/bindings/containers/exec.go index 2aeeae1f8..73cfb5079 100644 --- a/pkg/bindings/containers/exec.go +++ b/pkg/bindings/containers/exec.go @@ -1,6 +1,7 @@ package containers import ( + "bytes" "context" "net/http" "strings" @@ -69,3 +70,31 @@ func ExecInspect(ctx context.Context, sessionID string) (*define.InspectExecSess return respStruct, nil } + +// ExecStart starts (but does not attach to) a given exec session. +func ExecStart(ctx context.Context, sessionID string) error { + conn, err := bindings.GetClient(ctx) + if err != nil { + return err + } + + logrus.Debugf("Starting exec session ID %q", sessionID) + + // We force Detach to true + body := struct { + Detach bool `json:"Detach"` + }{ + Detach: true, + } + bodyJSON, err := json.Marshal(body) + if err != nil { + return err + } + + resp, err := conn.DoRequest(bytes.NewReader(bodyJSON), http.MethodPost, "/exec/%s/start", nil, nil, sessionID) + if err != nil { + return err + } + + return resp.Process(nil) +} diff --git a/pkg/domain/entities/engine.go b/pkg/domain/entities/engine.go index db58befa5..b2bef0eea 100644 --- a/pkg/domain/entities/engine.go +++ b/pkg/domain/entities/engine.go @@ -43,14 +43,16 @@ type PodmanConfig struct { EngineMode EngineMode // ABI or Tunneling mode Identities []string // ssh identities for connecting to server MaxWorks int // maximum number of parallel threads + PassPhrase string // ssh passphrase for identity for connecting to server RegistriesConf string // allows for specifying a custom registries.conf + Remote bool // Connection to Podman API Service will use RESTful API RuntimePath string // --runtime flag will set Engine.RuntimePath + Span opentracing.Span // tracing object SpanCloser io.Closer // Close() for tracing object SpanCtx context.Context // context to use when tracing - Span opentracing.Span // tracing object Syslog bool // write to StdOut and Syslog, not supported when tunneling Trace bool // Hidden: Trace execution - Uri string // URI to API Service + Uri string // URI to RESTful API Service Runroot string StorageDriver string diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index e982c7c11..19232eff1 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -44,8 +44,10 @@ func getContainersAndInputByContext(all, latest bool, names []string, runtime *l ctrs, err = runtime.GetAllContainers() case latest: ctr, err = runtime.GetLatestContainer() - rawInput = append(rawInput, ctr.ID()) - ctrs = append(ctrs, ctr) + if err == nil { + rawInput = append(rawInput, ctr.ID()) + ctrs = append(ctrs, ctr) + } default: for _, n := range names { ctr, e := runtime.LookupContainer(n) @@ -177,6 +179,12 @@ func (ic *ContainerEngine) ContainerStop(ctx context.Context, namesOrIds []strin report.Err = err reports = append(reports, &report) continue + } else if err := con.Cleanup(ctx); err != nil { + // Only if no error, proceed to cleanup to ensure all + // mounts are removed before we exit. + report.Err = err + reports = append(reports, &report) + continue } reports = append(reports, &report) } @@ -613,12 +621,11 @@ func (ic *ContainerEngine) ContainerExecDetached(ctx context.Context, nameOrId s if err != nil { return "", errors.Wrapf(err, "error retrieving Libpod configuration to build exec exit command") } - podmanPath, err := os.Executable() + // TODO: Add some ability to toggle syslog + exitCommandArgs, err := generate.CreateExitCommandArgs(storageConfig, runtimeConfig, false, true, true) if err != nil { - return "", errors.Wrapf(err, "error retrieving executable to build exec exit command") + return "", errors.Wrapf(err, "error constructing exit command for exec session") } - // TODO: Add some ability to toggle syslog - exitCommandArgs := generate.CreateExitCommandArgs(storageConfig, runtimeConfig, podmanPath, false, true, true) execConfig.ExitCommand = exitCommandArgs // Create and start the exec session diff --git a/pkg/domain/infra/abi/manifest.go b/pkg/domain/infra/abi/manifest.go index 6e311dec7..a2b5fc0fc 100644 --- a/pkg/domain/infra/abi/manifest.go +++ b/pkg/domain/infra/abi/manifest.go @@ -1,4 +1,4 @@ -// +build ABISupport +// +build !remote package abi diff --git a/pkg/domain/infra/runtime_abi.go b/pkg/domain/infra/runtime_abi.go index 67c1cd534..0a82b9a6b 100644 --- a/pkg/domain/infra/runtime_abi.go +++ b/pkg/domain/infra/runtime_abi.go @@ -1,4 +1,4 @@ -// +build ABISupport +// +build !remote package infra @@ -20,7 +20,7 @@ func NewContainerEngine(facts *entities.PodmanConfig) (entities.ContainerEngine, r, err := NewLibpodRuntime(facts.FlagSet, facts) return r, err case entities.TunnelMode: - ctx, err := bindings.NewConnection(context.Background(), facts.Uri, facts.Identities...) + ctx, err := bindings.NewConnectionWithIdentity(context.Background(), facts.Uri, facts.PassPhrase, facts.Identities...) return &tunnel.ContainerEngine{ClientCxt: ctx}, err } return nil, fmt.Errorf("runtime mode '%v' is not supported", facts.EngineMode) @@ -33,7 +33,7 @@ func NewImageEngine(facts *entities.PodmanConfig) (entities.ImageEngine, error) r, err := NewLibpodImageRuntime(facts.FlagSet, facts) return r, err case entities.TunnelMode: - ctx, err := bindings.NewConnection(context.Background(), facts.Uri, facts.Identities...) + ctx, err := bindings.NewConnectionWithIdentity(context.Background(), facts.Uri, facts.PassPhrase, facts.Identities...) return &tunnel.ImageEngine{ClientCxt: ctx}, err } return nil, fmt.Errorf("runtime mode '%v' is not supported", facts.EngineMode) diff --git a/pkg/domain/infra/runtime_abi_unsupported.go b/pkg/domain/infra/runtime_abi_unsupported.go index c4e25e990..3d7d457fc 100644 --- a/pkg/domain/infra/runtime_abi_unsupported.go +++ b/pkg/domain/infra/runtime_abi_unsupported.go @@ -1,4 +1,4 @@ -// +build !ABISupport +// +build remote package infra diff --git a/pkg/domain/infra/runtime_libpod.go b/pkg/domain/infra/runtime_libpod.go index a57eadc63..2f2b0f90f 100644 --- a/pkg/domain/infra/runtime_libpod.go +++ b/pkg/domain/infra/runtime_libpod.go @@ -1,4 +1,4 @@ -// +build ABISupport +// +build !remote package infra diff --git a/pkg/domain/infra/runtime_proxy.go b/pkg/domain/infra/runtime_proxy.go index e7002e20f..fed9b1008 100644 --- a/pkg/domain/infra/runtime_proxy.go +++ b/pkg/domain/infra/runtime_proxy.go @@ -1,4 +1,4 @@ -// +build ABISupport +// +build !remote package infra diff --git a/pkg/domain/infra/runtime_tunnel.go b/pkg/domain/infra/runtime_tunnel.go index 752218aaf..bba7d2c0c 100644 --- a/pkg/domain/infra/runtime_tunnel.go +++ b/pkg/domain/infra/runtime_tunnel.go @@ -1,4 +1,4 @@ -// +build !ABISupport +// +build remote package infra @@ -16,7 +16,7 @@ func NewContainerEngine(facts *entities.PodmanConfig) (entities.ContainerEngine, case entities.ABIMode: return nil, fmt.Errorf("direct runtime not supported") case entities.TunnelMode: - ctx, err := bindings.NewConnection(context.Background(), facts.Uri, facts.Identities...) + ctx, err := bindings.NewConnectionWithIdentity(context.Background(), facts.Uri, facts.PassPhrase, facts.Identities...) return &tunnel.ContainerEngine{ClientCxt: ctx}, err } return nil, fmt.Errorf("runtime mode '%v' is not supported", facts.EngineMode) @@ -28,7 +28,7 @@ func NewImageEngine(facts *entities.PodmanConfig) (entities.ImageEngine, error) case entities.ABIMode: return nil, fmt.Errorf("direct image runtime not supported") case entities.TunnelMode: - ctx, err := bindings.NewConnection(context.Background(), facts.Uri, facts.Identities...) + ctx, err := bindings.NewConnectionWithIdentity(context.Background(), facts.Uri, facts.PassPhrase, facts.Identities...) return &tunnel.ImageEngine{ClientCxt: ctx}, err } return nil, fmt.Errorf("runtime mode '%v' is not supported", facts.EngineMode) diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go index e1c859e7c..97b98eec2 100644 --- a/pkg/domain/infra/tunnel/containers.go +++ b/pkg/domain/infra/tunnel/containers.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "io/ioutil" "os" "strconv" "strings" @@ -162,6 +163,14 @@ func (ic *ContainerEngine) ContainerRm(ctx context.Context, namesOrIds []string, var ( reports []*entities.RmReport ) + for _, cidFile := range options.CIDFiles { + content, err := ioutil.ReadFile(cidFile) + if err != nil { + return nil, errors.Wrapf(err, "error reading CIDFile %s", cidFile) + } + id := strings.Split(string(content), "\n")[0] + namesOrIds = append(namesOrIds, id) + } ctrs, err := getContainersByContext(ic.ClientCxt, options.All, namesOrIds) if err != nil { return nil, err @@ -376,7 +385,7 @@ func (ic *ContainerEngine) ContainerAttach(ctx context.Context, nameOrId string, return containers.Attach(ic.ClientCxt, nameOrId, &options.DetachKeys, nil, bindings.PTrue, options.Stdin, options.Stdout, options.Stderr, nil) } -func (ic *ContainerEngine) ContainerExec(ctx context.Context, nameOrId string, options entities.ExecOptions, streams define.AttachStreams) (int, error) { +func makeExecConfig(options entities.ExecOptions) *handlers.ExecCreateConfig { env := []string{} for k, v := range options.Envs { env = append(env, fmt.Sprintf("%s=%s", k, v)) @@ -395,6 +404,12 @@ func (ic *ContainerEngine) ContainerExec(ctx context.Context, nameOrId string, o createConfig.WorkingDir = options.WorkDir createConfig.Cmd = options.Cmd + return createConfig +} + +func (ic *ContainerEngine) ContainerExec(ctx context.Context, nameOrId string, options entities.ExecOptions, streams define.AttachStreams) (int, error) { + createConfig := makeExecConfig(options) + sessionID, err := containers.ExecCreate(ic.ClientCxt, nameOrId, createConfig) if err != nil { return 125, err @@ -412,8 +427,19 @@ func (ic *ContainerEngine) ContainerExec(ctx context.Context, nameOrId string, o return inspectOut.ExitCode, nil } -func (ic *ContainerEngine) ContainerExecDetached(ctx context.Context, nameOrID string, options entities.ExecOptions) (string, error) { - return "", errors.New("not implemented") +func (ic *ContainerEngine) ContainerExecDetached(ctx context.Context, nameOrId string, options entities.ExecOptions) (string, error) { + createConfig := makeExecConfig(options) + + sessionID, err := containers.ExecCreate(ic.ClientCxt, nameOrId, createConfig) + if err != nil { + return "", err + } + + if err := containers.ExecStart(ic.ClientCxt, sessionID); err != nil { + return "", err + } + + return sessionID, nil } func startAndAttach(ic *ContainerEngine, name string, detachKeys *string, input, output, errput *os.File) error { //nolint diff --git a/pkg/env/env.go b/pkg/env/env.go index c6a1a0d28..a16007a50 100644 --- a/pkg/env/env.go +++ b/pkg/env/env.go @@ -20,18 +20,6 @@ var DefaultEnvVariables = map[string]string{ const whiteSpaces = " \t" -// ParseSlice parses the specified slice and transforms it into an environment -// map. -func ParseSlice(s []string) (map[string]string, error) { - env := make(map[string]string, len(s)) - for _, e := range s { - if err := parseEnv(env, e); err != nil { - return nil, err - } - } - return env, nil -} - // Slice transforms the specified map of environment variables into a // slice. If a value is non-empty, the key and value are joined with '='. func Slice(m map[string]string) []string { @@ -96,7 +84,6 @@ func parseEnv(env map[string]string, line string) error { if data[0] == "" { return errors.Errorf("invalid environment variable: %q", line) } - // trim the front of a variable, but nothing else name := strings.TrimLeft(data[0], whiteSpaces) if strings.ContainsAny(name, whiteSpaces) { diff --git a/pkg/env/env_supported.go b/pkg/env/env_supported.go new file mode 100644 index 000000000..8be9f9592 --- /dev/null +++ b/pkg/env/env_supported.go @@ -0,0 +1,15 @@ +// +build linux darwin + +package env + +// ParseSlice parses the specified slice and transforms it into an environment +// map. +func ParseSlice(s []string) (map[string]string, error) { + env := make(map[string]string, len(s)) + for _, e := range s { + if err := parseEnv(env, e); err != nil { + return nil, err + } + } + return env, nil +} diff --git a/pkg/env/env_unsupported.go b/pkg/env/env_unsupported.go new file mode 100644 index 000000000..a71c2956d --- /dev/null +++ b/pkg/env/env_unsupported.go @@ -0,0 +1,8 @@ +// +build !linux,!darwin + +package env + +func ParseSlice(s []string) (map[string]string, error) { + m := make(map[string]string) + return m, nil +} diff --git a/pkg/specgen/generate/container_create.go b/pkg/specgen/generate/container_create.go index ffd7fd4dd..7ddfed339 100644 --- a/pkg/specgen/generate/container_create.go +++ b/pkg/specgen/generate/container_create.go @@ -107,12 +107,12 @@ func MakeContainer(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGener } options = append(options, opts...) - podmanPath, err := os.Executable() + // TODO: Enable syslog support - we'll need to put this in SpecGen. + exitCommandArgs, err := CreateExitCommandArgs(rt.StorageConfig(), rtc, false, s.Remove, false) if err != nil { return nil, err } - // TODO: Enable syslog support - we'll need to put this in SpecGen. - options = append(options, libpod.WithExitCommand(CreateExitCommandArgs(rt.StorageConfig(), rtc, podmanPath, false, s.Remove, false))) + options = append(options, libpod.WithExitCommand(exitCommandArgs)) runtimeSpec, err := SpecGenToOCI(ctx, s, rt, rtc, newImage, finalMounts) if err != nil { @@ -229,13 +229,18 @@ func createContainerOptions(ctx context.Context, rt *libpod.Runtime, s *specgen. return options, nil } -func CreateExitCommandArgs(storageConfig storage.StoreOptions, config *config.Config, podmanPath string, syslog, rm bool, exec bool) []string { +func CreateExitCommandArgs(storageConfig storage.StoreOptions, config *config.Config, syslog, rm, exec bool) ([]string, error) { // We need a cleanup process for containers in the current model. // But we can't assume that the caller is Podman - it could be another // user of the API. // As such, provide a way to specify a path to Podman, so we can // still invoke a cleanup process. + podmanPath, err := os.Executable() + if err != nil { + return nil, err + } + command := []string{podmanPath, "--root", storageConfig.GraphRoot, "--runroot", storageConfig.RunRoot, @@ -265,9 +270,11 @@ func CreateExitCommandArgs(storageConfig storage.StoreOptions, config *config.Co command = append(command, "--rm") } + // This has to be absolutely last, to ensure that the exec session ID + // will be added after it by Libpod. if exec { command = append(command, "--exec") } - return command + return command, nil } diff --git a/pkg/util/utils_linux.go b/pkg/util/utils_linux.go index 288137ca5..5e4dc4a51 100644 --- a/pkg/util/utils_linux.go +++ b/pkg/util/utils_linux.go @@ -6,6 +6,7 @@ import ( "path/filepath" "syscall" + "github.com/containers/libpod/pkg/rootless" "github.com/containers/psgo" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -52,3 +53,18 @@ func FindDeviceNodes() (map[string]string, error) { return nodes, nil } + +// CheckRootlessUIDRange checks the uid within the rootless container is in the range from /etc/subuid +func CheckRootlessUIDRange(uid int) error { + uids, _, err := rootless.GetConfiguredMappings() + if err != nil { + return err + } + for _, u := range uids { + // add 1 since we also map in the user's own UID + if uid > u.Size+1 { + return errors.Errorf("requested user's UID %d is too large for the rootless user namespace", uid) + } + } + return nil +} diff --git a/pkg/util/utils_unsupported.go b/pkg/util/utils_unsupported.go index 62805d7c8..f8d5a37c1 100644 --- a/pkg/util/utils_unsupported.go +++ b/pkg/util/utils_unsupported.go @@ -10,3 +10,8 @@ import ( func FindDeviceNodes() (map[string]string, error) { return nil, errors.Errorf("not supported on non-Linux OSes") } + +// CheckRootlessUIDRange is not implemented anywhere except Linux. +func CheckRootlessUIDRange(uid int) error { + return nil +} diff --git a/test/apiv2/20-containers.at b/test/apiv2/20-containers.at index 7fb39b221..8b535928a 100644 --- a/test/apiv2/20-containers.at +++ b/test/apiv2/20-containers.at @@ -30,4 +30,56 @@ cid=$(jq -r '.[0].Id' <<<"$output") t DELETE libpod/containers/$cid 204 +CNAME=myfoo +podman run --name $CNAME $IMAGE -td top +t GET libpod/containers/json?all=true 200 \ + .[0].Id~[0-9a-f]\\{12\\} +cid=$(jq -r '.[0].Id' <<<"$output") + +# No such container +t POST "libpod/commit?container=nonesuch" '' 404 + +# Comment can only be used with docker format, not OCI +cparam="repo=newrepo&comment=foo&author=bob" +t POST "libpod/commit?container=$CNAME&$cparam" '' 500 + +# Commit a new image from the container +t POST "libpod/commit?container=$CNAME" '' 200 \ + .Id~[0-9a-f]\\{12\\} +iid=$(jq -r '.Id' <<<"$output") +t GET libpod/images/$iid/json 200 \ + .RepoTags[0]=null \ + .Author="" \ + .Comment="" + +# Commit a new image w/o tag +cparam="repo=newrepo&comment=foo&author=bob&format=docker" +t POST "libpod/commit?container=$CNAME&$cparam" '' 200 +t GET libpod/images/newrepo:latest/json 200 \ + .RepoTags[0]=localhost/newrepo:latest \ + .Author=bob \ + .Comment=foo + +# Commit a new image w/ specified tag and author +cparam="repo=newrepo&tag=v1&author=alice" +t POST "libpod/commit?container=$cid&$cparam&pause=false" '' 200 +t GET libpod/images/newrepo:v1/json 200 \ + .RepoTags[0]=localhost/newrepo:v1 \ + .Author=alice + +# Commit a new image w/ full parameters +cparam="repo=newrepo&tag=v2&comment=bar&author=eric" +cparam="$cparam&format=docker&changes=CMD=/bin/foo" +t POST "libpod/commit?container=${cid:0:12}&$cparam&pause=true" '' 200 +t GET libpod/images/newrepo:v2/json 200 \ + .RepoTags[0]=localhost/newrepo:v2 \ + .Author=eric \ + .Comment=bar \ + .Config.Cmd[-1]="/bin/foo" + +t DELETE images/localhost/newrepo:latest?force=true 200 +t DELETE images/localhost/newrepo:v1?force=true 200 +t DELETE images/localhost/newrepo:v2?force=true 200 +t DELETE libpod/containers/$cid 204 + # vim: filetype=sh diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go index 0a6373bfa..f40472a7c 100644 --- a/test/e2e/create_test.go +++ b/test/e2e/create_test.go @@ -301,7 +301,6 @@ var _ = Describe("Podman create", func() { }) It("podman create --authfile with nonexist authfile", func() { - SkipIfRemote() session := podmanTest.PodmanNoCache([]string{"create", "--authfile", "/tmp/nonexist", "--name=foo", ALPINE}) session.WaitWithDefaultTimeout() Expect(session).To(Not(Equal(0))) diff --git a/test/e2e/diff_test.go b/test/e2e/diff_test.go index d273f9463..fbbe49eac 100644 --- a/test/e2e/diff_test.go +++ b/test/e2e/diff_test.go @@ -61,7 +61,6 @@ var _ = Describe("Podman diff", func() { }) It("podman diff container and committed image", func() { - SkipIfRemote() session := podmanTest.Podman([]string{"run", "--name=diff-test", ALPINE, "touch", "/tmp/diff-test"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -82,11 +81,10 @@ var _ = Describe("Podman diff", func() { }) It("podman diff latest container", func() { - SkipIfRemote() - session := podmanTest.Podman([]string{"run", "--name=diff-test", ALPINE, "touch", "/tmp/diff-test"}) + session := podmanTest.Podman([]string{"run", "--name", "diff-test", ALPINE, "touch", "/tmp/diff-test"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"diff", "-l"}) + session = podmanTest.Podman([]string{"diff", "diff-test"}) session.WaitWithDefaultTimeout() containerDiff := session.OutputToStringArray() sort.Strings(containerDiff) diff --git a/test/e2e/exec_test.go b/test/e2e/exec_test.go index 8ec666c2b..b152da9e6 100644 --- a/test/e2e/exec_test.go +++ b/test/e2e/exec_test.go @@ -25,6 +25,10 @@ var _ = Describe("Podman exec", func() { podmanTest = PodmanTestCreate(tempdir) podmanTest.Setup() podmanTest.SeedImages() + // HACK: Remove this once we get Conmon 2.0.17 on Ubuntu + if podmanTest.Host.Distribution == "ubuntu" { + Skip("Unable to perform test on Ubuntu distributions due to too-old Conmon (need 2.0.17)") + } }) AfterEach(func() { @@ -284,7 +288,6 @@ var _ = Describe("Podman exec", func() { }) It("podman exec --detach", func() { - Skip(v2remotefail) ctrName := "testctr" ctr := podmanTest.Podman([]string{"run", "-t", "-i", "-d", "--name", ctrName, ALPINE, "top"}) ctr.WaitWithDefaultTimeout() diff --git a/test/e2e/exists_test.go b/test/e2e/exists_test.go index e26fad51d..8f3b371d8 100644 --- a/test/e2e/exists_test.go +++ b/test/e2e/exists_test.go @@ -112,7 +112,6 @@ var _ = Describe("Podman image|container exists", func() { }) It("podman pod does not exist in local storage", func() { // The exit code for non-existing pod is incorrect (125 vs 1) - SkipIfRemote() session := podmanTest.Podman([]string{"pod", "exists", "foobar"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(1)) diff --git a/test/e2e/healthcheck_run_test.go b/test/e2e/healthcheck_run_test.go index 8e63d9f4c..d8e3f045f 100644 --- a/test/e2e/healthcheck_run_test.go +++ b/test/e2e/healthcheck_run_test.go @@ -42,7 +42,6 @@ var _ = Describe("Podman healthcheck run", func() { }) It("podman disable healthcheck with --no-healthcheck on valid container", func() { - SkipIfRemote() session := podmanTest.Podman([]string{"run", "-dt", "--no-healthcheck", "--name", "hc", healthcheck}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -52,7 +51,6 @@ var _ = Describe("Podman healthcheck run", func() { }) It("podman disable healthcheck with --health-cmd=none on valid container", func() { - SkipIfRemote() session := podmanTest.Podman([]string{"run", "-dt", "--health-cmd", "none", "--name", "hc", healthcheck}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) diff --git a/test/e2e/inspect_test.go b/test/e2e/inspect_test.go index 77cfe4fd3..62f69f1c1 100644 --- a/test/e2e/inspect_test.go +++ b/test/e2e/inspect_test.go @@ -132,28 +132,27 @@ var _ = Describe("Podman inspect", func() { }) It("podman inspect with mount filters", func() { - SkipIfRemote() - ctrSession := podmanTest.Podman([]string{"create", "-v", "/tmp:/test1", ALPINE, "top"}) + ctrSession := podmanTest.Podman([]string{"create", "--name", "test", "-v", "/tmp:/test1", ALPINE, "top"}) ctrSession.WaitWithDefaultTimeout() Expect(ctrSession.ExitCode()).To(Equal(0)) - inspectSource := podmanTest.Podman([]string{"inspect", "-l", "--format", "{{(index .Mounts 0).Source}}"}) + inspectSource := podmanTest.Podman([]string{"inspect", "test", "--format", "{{(index .Mounts 0).Source}}"}) inspectSource.WaitWithDefaultTimeout() Expect(inspectSource.ExitCode()).To(Equal(0)) Expect(inspectSource.OutputToString()).To(Equal("/tmp")) - inspectSrc := podmanTest.Podman([]string{"inspect", "-l", "--format", "{{(index .Mounts 0).Src}}"}) + inspectSrc := podmanTest.Podman([]string{"inspect", "test", "--format", "{{(index .Mounts 0).Src}}"}) inspectSrc.WaitWithDefaultTimeout() Expect(inspectSrc.ExitCode()).To(Equal(0)) Expect(inspectSrc.OutputToString()).To(Equal("/tmp")) - inspectDestination := podmanTest.Podman([]string{"inspect", "-l", "--format", "{{(index .Mounts 0).Destination}}"}) + inspectDestination := podmanTest.Podman([]string{"inspect", "test", "--format", "{{(index .Mounts 0).Destination}}"}) inspectDestination.WaitWithDefaultTimeout() Expect(inspectDestination.ExitCode()).To(Equal(0)) Expect(inspectDestination.OutputToString()).To(Equal("/test1")) - inspectDst := podmanTest.Podman([]string{"inspect", "-l", "--format", "{{(index .Mounts 0).Dst}}"}) + inspectDst := podmanTest.Podman([]string{"inspect", "test", "--format", "{{(index .Mounts 0).Dst}}"}) inspectDst.WaitWithDefaultTimeout() Expect(inspectDst.ExitCode()).To(Equal(0)) Expect(inspectDst.OutputToString()).To(Equal("/test1")) @@ -171,4 +170,57 @@ var _ = Describe("Podman inspect", func() { Expect(imageData[0].HealthCheck.Interval).To(BeNumerically("==", 60000000000)) Expect(imageData[0].HealthCheck.Test).To(Equal([]string{"CMD-SHELL", "curl -f http://localhost/ || exit 1"})) }) + + It("podman inspect --latest with no container fails", func() { + SkipIfRemote() + + session := podmanTest.Podman([]string{"inspect", "--latest"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Not(Equal(0))) + }) + + It("podman [image,container] inspect on image", func() { + baseInspect := podmanTest.Podman([]string{"inspect", ALPINE}) + baseInspect.WaitWithDefaultTimeout() + Expect(baseInspect.ExitCode()).To(Equal(0)) + baseJSON := baseInspect.InspectImageJSON() + Expect(len(baseJSON)).To(Equal(1)) + + ctrInspect := podmanTest.Podman([]string{"container", "inspect", ALPINE}) + ctrInspect.WaitWithDefaultTimeout() + Expect(ctrInspect.ExitCode()).To(Not(Equal(0))) + + imageInspect := podmanTest.Podman([]string{"image", "inspect", ALPINE}) + imageInspect.WaitWithDefaultTimeout() + Expect(imageInspect.ExitCode()).To(Equal(0)) + imageJSON := imageInspect.InspectImageJSON() + Expect(len(imageJSON)).To(Equal(1)) + + Expect(baseJSON[0].ID).To(Equal(imageJSON[0].ID)) + }) + + It("podman [image, container] inspect on container", func() { + ctrName := "testCtr" + create := podmanTest.Podman([]string{"create", "--name", ctrName, ALPINE, "sh"}) + create.WaitWithDefaultTimeout() + Expect(create.ExitCode()).To(Equal(0)) + + baseInspect := podmanTest.Podman([]string{"inspect", ctrName}) + baseInspect.WaitWithDefaultTimeout() + Expect(baseInspect.ExitCode()).To(Equal(0)) + baseJSON := baseInspect.InspectContainerToJSON() + Expect(len(baseJSON)).To(Equal(1)) + + ctrInspect := podmanTest.Podman([]string{"container", "inspect", ctrName}) + ctrInspect.WaitWithDefaultTimeout() + Expect(ctrInspect.ExitCode()).To(Equal(0)) + ctrJSON := ctrInspect.InspectContainerToJSON() + Expect(len(ctrJSON)).To(Equal(1)) + + imageInspect := podmanTest.Podman([]string{"image", "inspect", ctrName}) + imageInspect.WaitWithDefaultTimeout() + Expect(imageInspect.ExitCode()).To(Not(Equal(0))) + + Expect(baseJSON[0].ID).To(Equal(ctrJSON[0].ID)) + }) }) diff --git a/test/e2e/libpod_suite_remote_test.go b/test/e2e/libpod_suite_remote_test.go index dde853413..b94b6e267 100644 --- a/test/e2e/libpod_suite_remote_test.go +++ b/test/e2e/libpod_suite_remote_test.go @@ -36,7 +36,7 @@ func SkipIfRootlessV2() { // Podman is the exec call to podman on the filesystem func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration { - var remoteArgs = []string{"--remote", p.RemoteSocket} + var remoteArgs = []string{"--remote", "--url", p.RemoteSocket} remoteArgs = append(remoteArgs, args...) podmanSession := p.PodmanBase(remoteArgs, false, false) return &PodmanSessionIntegration{podmanSession} @@ -44,7 +44,7 @@ func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration // PodmanExtraFiles is the exec call to podman on the filesystem and passes down extra files func (p *PodmanTestIntegration) PodmanExtraFiles(args []string, extraFiles []*os.File) *PodmanSessionIntegration { - var remoteArgs = []string{"--remote", p.RemoteSocket} + var remoteArgs = []string{"--remote", "--url", p.RemoteSocket} remoteArgs = append(remoteArgs, args...) podmanSession := p.PodmanAsUserBase(remoteArgs, 0, 0, "", nil, false, false, extraFiles) return &PodmanSessionIntegration{podmanSession} @@ -52,7 +52,7 @@ func (p *PodmanTestIntegration) PodmanExtraFiles(args []string, extraFiles []*os // PodmanNoCache calls podman with out adding the imagecache func (p *PodmanTestIntegration) PodmanNoCache(args []string) *PodmanSessionIntegration { - var remoteArgs = []string{"--remote", p.RemoteSocket} + var remoteArgs = []string{"--remote", "--url", p.RemoteSocket} remoteArgs = append(remoteArgs, args...) podmanSession := p.PodmanBase(remoteArgs, false, true) return &PodmanSessionIntegration{podmanSession} diff --git a/test/e2e/pod_create_test.go b/test/e2e/pod_create_test.go index e56db54a2..a7d5783cb 100644 --- a/test/e2e/pod_create_test.go +++ b/test/e2e/pod_create_test.go @@ -238,7 +238,6 @@ var _ = Describe("Podman pod create", func() { }) It("podman create pod with IP address", func() { - SkipIfRemote() SkipIfRootless() name := "test" ip := GetRandomIPAddress() diff --git a/test/e2e/ps_test.go b/test/e2e/ps_test.go index 12ce4661f..0dc8e01af 100644 --- a/test/e2e/ps_test.go +++ b/test/e2e/ps_test.go @@ -362,8 +362,6 @@ var _ = Describe("Podman ps", func() { }) It("podman --pod with a non-empty pod name", func() { - SkipIfRemote() - podName := "testPodName" _, ec, podid := podmanTest.CreatePod(podName) Expect(ec).To(Equal(0)) diff --git a/test/e2e/pull_test.go b/test/e2e/pull_test.go index 96340ef30..ac882927f 100644 --- a/test/e2e/pull_test.go +++ b/test/e2e/pull_test.go @@ -351,7 +351,6 @@ var _ = Describe("Podman pull", func() { }) It("podman pull from docker with nonexist --authfile", func() { - SkipIfRemote() session := podmanTest.PodmanNoCache([]string{"pull", "--authfile", "/tmp/nonexist", ALPINE}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Not(Equal(0))) diff --git a/test/e2e/rm_test.go b/test/e2e/rm_test.go index 87e3de922..6b554cd84 100644 --- a/test/e2e/rm_test.go +++ b/test/e2e/rm_test.go @@ -141,7 +141,6 @@ var _ = Describe("Podman rm", func() { }) It("podman rm --cidfile", func() { - SkipIfRemote() tmpDir, err := ioutil.TempDir("", "") Expect(err).To(BeNil()) @@ -164,7 +163,6 @@ var _ = Describe("Podman rm", func() { }) It("podman rm multiple --cidfile", func() { - SkipIfRemote() tmpDir, err := ioutil.TempDir("", "") Expect(err).To(BeNil()) diff --git a/test/e2e/volume_rm_test.go b/test/e2e/volume_rm_test.go index 6f2020828..742d4e0dc 100644 --- a/test/e2e/volume_rm_test.go +++ b/test/e2e/volume_rm_test.go @@ -48,7 +48,6 @@ var _ = Describe("Podman volume rm", func() { }) It("podman volume rm with --force flag", func() { - SkipIfRemote() session := podmanTest.Podman([]string{"create", "-v", "myvol:/myvol", ALPINE, "ls"}) cid := session.OutputToString() session.WaitWithDefaultTimeout() diff --git a/test/utils/utils.go b/test/utils/utils.go index 1d59e5468..0597cd292 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -73,7 +73,7 @@ func (p *PodmanTest) PodmanAsUserBase(args []string, uid, gid uint32, cwd string podmanBinary = p.RemotePodmanBinary } if p.RemoteTest { - podmanOptions = append([]string{"--remote", p.RemoteSocket}, podmanOptions...) + podmanOptions = append([]string{"--remote", "--url", p.RemoteSocket}, podmanOptions...) } if env == nil { fmt.Printf("Running: %s %s\n", podmanBinary, strings.Join(podmanOptions, " ")) diff --git a/troubleshooting.md b/troubleshooting.md index bad9d8102..8175073cd 100644 --- a/troubleshooting.md +++ b/troubleshooting.md @@ -247,7 +247,7 @@ cannot find newuidmap: exec: "newuidmap": executable file not found in $PATH #### Solution -Install a version of shadow-utils that includes these executables. Note RHEL 7 and CentOS 7 will not have support for this until RHEL7.7 is released. +Install a version of shadow-utils that includes these executables. Note that for RHEL and CentOS 7, at least the 7.7 release must be installed for support to be available. ### 11) rootless setup user: invalid argument diff --git a/vendor/github.com/containers/psgo/SECURITY.md b/vendor/github.com/containers/psgo/SECURITY.md new file mode 100644 index 000000000..5d5ba254a --- /dev/null +++ b/vendor/github.com/containers/psgo/SECURITY.md @@ -0,0 +1,3 @@ +## Security and Disclosure Information Policy for the psgo Project + +The psgo Project follows the [Security and Disclosure Information Policy](https://github.com/containers/common/blob/master/SECURITY.md) for the Containers Projects. diff --git a/vendor/github.com/containers/psgo/psgo.go b/vendor/github.com/containers/psgo/psgo.go index 57132c94e..c75fc3815 100644 --- a/vendor/github.com/containers/psgo/psgo.go +++ b/vendor/github.com/containers/psgo/psgo.go @@ -847,7 +847,7 @@ func processHPID(p *process.Process, ctx *psContext) (string, error) { func processHUSER(p *process.Process, ctx *psContext) (string, error) { if hp := findHostProcess(p, ctx); hp != nil { if ctx.opts != nil && len(ctx.opts.UIDMap) > 0 { - return findID(p.Status.Uids[1], ctx.opts.UIDMap, process.LookupUID, "/proc/sys/fs/overflowuid") + return findID(hp.Status.Uids[1], ctx.opts.UIDMap, process.LookupUID, "/proc/sys/fs/overflowuid") } return hp.Huser, nil } @@ -860,7 +860,7 @@ func processHUSER(p *process.Process, ctx *psContext) (string, error) { func processHGROUP(p *process.Process, ctx *psContext) (string, error) { if hp := findHostProcess(p, ctx); hp != nil { if ctx.opts != nil && len(ctx.opts.GIDMap) > 0 { - return findID(p.Status.Gids[1], ctx.opts.GIDMap, process.LookupGID, "/proc/sys/fs/overflowgid") + return findID(hp.Status.Gids[1], ctx.opts.GIDMap, process.LookupGID, "/proc/sys/fs/overflowgid") } return hp.Hgroup, nil } diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index 0044d6cb9..769e37e15 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.20.1 +1.20.2 diff --git a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go index f1c941f11..ff6e297f4 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go @@ -1544,8 +1544,8 @@ func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) { } dev := stat.Rdev - majorNum := major(dev) - minorNum := minor(dev) + majorNum := major(uint64(dev)) + minorNum := minor(uint64(dev)) logrus.Debugf("devmapper: Major:Minor for device: %s is:%v:%v", file.Name(), majorNum, minorNum) return majorNum, minorNum, nil diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 2906e3e08..930a57a97 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -311,6 +311,9 @@ func parseOptions(options []string) (*overlayOptions, error) { return nil, fmt.Errorf("overlay: can't stat program %s: %v", val, err) } o.mountProgram = val + case "overlay2.skip_mount_home", "overlay.skip_mount_home", ".skip_mount_home": + logrus.Debugf("overlay: skip_mount_home=%s", val) + o.skipMountHome, err = strconv.ParseBool(val) case ".ignore_chown_errors", "overlay2.ignore_chown_errors", "overlay.ignore_chown_errors": logrus.Debugf("overlay: ignore_chown_errors=%s", val) o.ignoreChownErrors, err = strconv.ParseBool(val) diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index a7d9ade60..01ac1827b 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -6,23 +6,23 @@ require ( github.com/Microsoft/hcsshim v0.8.9 github.com/docker/go-units v0.4.0 github.com/hashicorp/go-multierror v1.0.0 - github.com/klauspost/compress v1.10.5 + github.com/klauspost/compress v1.10.7 github.com/klauspost/pgzip v1.2.4 github.com/mattn/go-shellwords v1.0.10 github.com/mistifyio/go-zfs v2.1.1+incompatible github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/runc v1.0.0-rc9 + github.com/opencontainers/runc v1.0.0-rc90 github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700 - github.com/opencontainers/selinux v1.5.1 + github.com/opencontainers/selinux v1.5.2 github.com/pkg/errors v0.9.1 github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 github.com/sirupsen/logrus v1.6.0 - github.com/stretchr/testify v1.5.1 + github.com/stretchr/testify v1.6.0 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 github.com/tchap/go-patricia v2.3.0+incompatible github.com/vbatts/tar-split v0.11.1 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 - golang.org/x/sys v0.0.0-20191127021746-63cb32ae39b2 + golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 gotest.tools v2.2.0+incompatible ) diff --git a/vendor/github.com/containers/storage/go.sum b/vendor/github.com/containers/storage/go.sum index eab0fd61e..2b5a279c6 100644 --- a/vendor/github.com/containers/storage/go.sum +++ b/vendor/github.com/containers/storage/go.sum @@ -8,6 +8,7 @@ github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg3 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1 h1:uict5mhHFTzKLUCufdSLym7z/J0CbBJT59lYbP9wtbg= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -15,12 +16,14 @@ github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= @@ -29,6 +32,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= @@ -42,8 +46,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.10.5 h1:7q6vHIqubShURwQz8cQK6yIe/xC3IF0Vm7TGfqjewrc= -github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= +github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -62,12 +66,12 @@ github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc9 h1:/k06BMULKF5hidyoZymkoDCzdJzltZpz/UU4LguQVtc= -github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc90 h1:4+xo8mtWixbHoEm451+WJNUrq12o2/tDsyK9Vgc/NcA= +github.com/opencontainers/runc v1.0.0-rc90/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700 h1:eNUVfm/RFLIi1G7flU5/ZRTHvd4kcVuzfRnL6OFlzCI= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.5.1 h1:jskKwSMFYqyTrHEuJgQoUlTcId0av64S6EWObrIfn5Y= -github.com/opencontainers/selinux v1.5.1/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g= +github.com/opencontainers/selinux v1.5.2 h1:F6DgIsjgBIcDksLW4D5RG9bXok6oqZ3nvMwj4ZoFu/Q= +github.com/opencontainers/selinux v1.5.2/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -81,15 +85,17 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgho= +github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tchap/go-patricia v2.3.0+incompatible h1:GkY4dP3cEfEASBPPkWd+AmjYxhmDkqO9/zg7R0lSQRs= github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5 h1:MCfT24H3f//U5+UCrZp1/riVO3B50BovxtDiNn0XKkk= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vbatts/tar-split v0.11.1 h1:0Odu65rhcZ3JZaPHxl7tCI3V/C/Q9Zf82UFravl02dE= github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g= @@ -120,8 +126,8 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191127021746-63cb32ae39b2 h1:/J2nHFg1MTqaRLFO7M+J78ASNsJoz3r0cvHBPQ77fsE= -golang.org/x/sys v0.0.0-20191127021746-63cb32ae39b2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 h1:1/DFK4b7JH8DmkqhUk48onnSfrPzImPoVxuomtbT2nk= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -145,6 +151,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/containers/storage/pkg/config/config.go b/vendor/github.com/containers/storage/pkg/config/config.go index 1ac6c04e3..4a35997ea 100644 --- a/vendor/github.com/containers/storage/pkg/config/config.go +++ b/vendor/github.com/containers/storage/pkg/config/config.go @@ -2,8 +2,6 @@ package config import ( "fmt" - - "github.com/sirupsen/logrus" ) // ThinpoolOptionsConfig represents the "storage.options.thinpool" @@ -269,11 +267,11 @@ func GetGraphDriverOptions(driverName string, options OptionsConfig) []string { } else if options.Size != "" { doptions = append(doptions, fmt.Sprintf("%s.size=%s", driverName, options.Size)) } - - if options.Overlay.SkipMountHome != "" || options.SkipMountHome != "" { - logrus.Warn("skip_mount_home option is no longer supported, ignoring option") + if options.Overlay.SkipMountHome != "" { + doptions = append(doptions, fmt.Sprintf("%s.skip_mount_home=%s", driverName, options.Overlay.SkipMountHome)) + } else if options.SkipMountHome != "" { + doptions = append(doptions, fmt.Sprintf("%s.skip_mount_home=%s", driverName, options.SkipMountHome)) } - case "vfs": if options.Vfs.IgnoreChownErrors != "" { doptions = append(doptions, fmt.Sprintf("%s.ignore_chown_errors=%s", driverName, options.Vfs.IgnoreChownErrors)) diff --git a/vendor/github.com/containers/storage/storage.conf b/vendor/github.com/containers/storage/storage.conf index c7f9b2cf8..19909e9c6 100644 --- a/vendor/github.com/containers/storage/storage.conf +++ b/vendor/github.com/containers/storage/storage.conf @@ -76,6 +76,9 @@ additionalimagestores = [ # mountopt specifies comma separated list of extra mount options mountopt = "nodev" +# Set to skip a PRIVATE bind mount on the storage home directory. +# skip_mount_home = "false" + # Size is used to set a maximum size of the container image. # size = "" diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index eaf622f43..263f4edb2 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -3481,6 +3481,9 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) { if config.Storage.Options.MountProgram != "" { storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.mount_program=%s", config.Storage.Driver, config.Storage.Options.MountProgram)) } + if config.Storage.Options.SkipMountHome != "" { + storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.skip_mount_home=%s", config.Storage.Driver, config.Storage.Options.SkipMountHome)) + } if config.Storage.Options.IgnoreChownErrors != "" { storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.ignore_chown_errors=%s", config.Storage.Driver, config.Storage.Options.IgnoreChownErrors)) } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index 97ae66a4a..fb42a398b 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -155,20 +155,70 @@ func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { // The length of the supplied input must match the end of a block exactly. // Before this is called, the table must be initialized with ReadTable unless // the encoder re-used the table. +// deprecated: Use the stateless Decoder() to get a concurrent version. func (s *Scratch) Decompress1X(in []byte) (out []byte, err error) { - if len(s.dt.single) == 0 { + if cap(s.Out) < s.MaxDecodedSize { + s.Out = make([]byte, s.MaxDecodedSize) + } + s.Out = s.Out[:0:s.MaxDecodedSize] + s.Out, err = s.Decoder().Decompress1X(s.Out, in) + return s.Out, err +} + +// Decompress4X will decompress a 4X encoded stream. +// Before this is called, the table must be initialized with ReadTable unless +// the encoder re-used the table. +// The length of the supplied input must match the end of a block exactly. +// The destination size of the uncompressed data must be known and provided. +// deprecated: Use the stateless Decoder() to get a concurrent version. +func (s *Scratch) Decompress4X(in []byte, dstSize int) (out []byte, err error) { + if dstSize > s.MaxDecodedSize { + return nil, ErrMaxDecodedSizeExceeded + } + if cap(s.Out) < dstSize { + s.Out = make([]byte, s.MaxDecodedSize) + } + s.Out = s.Out[:0:dstSize] + s.Out, err = s.Decoder().Decompress4X(s.Out, in) + return s.Out, err +} + +// Decoder will return a stateless decoder that can be used by multiple +// decompressors concurrently. +// Before this is called, the table must be initialized with ReadTable. +// The Decoder is still linked to the scratch buffer so that cannot be reused. +// However, it is safe to discard the scratch. +func (s *Scratch) Decoder() *Decoder { + return &Decoder{ + dt: s.dt, + actualTableLog: s.actualTableLog, + } +} + +// Decoder provides stateless decoding. +type Decoder struct { + dt dTable + actualTableLog uint8 +} + +// Decompress1X will decompress a 1X encoded stream. +// The cap of the output buffer will be the maximum decompressed size. +// The length of the supplied input must match the end of a block exactly. +func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { return nil, errors.New("no table loaded") } var br bitReader - err = br.init(in) + err := br.init(src) if err != nil { - return nil, err + return dst, err } - s.Out = s.Out[:0] + maxDecodedSize := cap(dst) + dst = dst[:0] decode := func() byte { - val := br.peekBitsFast(s.actualTableLog) /* note : actualTableLog >= 1 */ - v := s.dt.single[val] + val := br.peekBitsFast(d.actualTableLog) /* note : actualTableLog >= 1 */ + v := d.dt.single[val] br.bitsRead += uint8(v.entry) return uint8(v.entry >> 8) } @@ -180,88 +230,80 @@ func (s *Scratch) Decompress1X(in []byte) (out []byte, err error) { // Avoid bounds check by always having full sized table. const tlSize = 1 << tableLogMax const tlMask = tlSize - 1 - dt := s.dt.single[:tlSize] + dt := d.dt.single[:tlSize] // Use temp table to avoid bound checks/append penalty. - var tmp = s.huffWeight[:256] + var buf [256]byte var off uint8 for br.off >= 8 { br.fillFast() - tmp[off+0] = hasDec(dt[br.peekBitsFast(s.actualTableLog)&tlMask]) - tmp[off+1] = hasDec(dt[br.peekBitsFast(s.actualTableLog)&tlMask]) + buf[off+0] = hasDec(dt[br.peekBitsFast(d.actualTableLog)&tlMask]) + buf[off+1] = hasDec(dt[br.peekBitsFast(d.actualTableLog)&tlMask]) br.fillFast() - tmp[off+2] = hasDec(dt[br.peekBitsFast(s.actualTableLog)&tlMask]) - tmp[off+3] = hasDec(dt[br.peekBitsFast(s.actualTableLog)&tlMask]) + buf[off+2] = hasDec(dt[br.peekBitsFast(d.actualTableLog)&tlMask]) + buf[off+3] = hasDec(dt[br.peekBitsFast(d.actualTableLog)&tlMask]) off += 4 if off == 0 { - if len(s.Out)+256 > s.MaxDecodedSize { + if len(dst)+256 > maxDecodedSize { br.close() return nil, ErrMaxDecodedSizeExceeded } - s.Out = append(s.Out, tmp...) + dst = append(dst, buf[:]...) } } - if len(s.Out)+int(off) > s.MaxDecodedSize { + if len(dst)+int(off) > maxDecodedSize { br.close() return nil, ErrMaxDecodedSizeExceeded } - s.Out = append(s.Out, tmp[:off]...) + dst = append(dst, buf[:off]...) for !br.finished() { br.fill() - if len(s.Out) >= s.MaxDecodedSize { + if len(dst) >= maxDecodedSize { br.close() return nil, ErrMaxDecodedSizeExceeded } - s.Out = append(s.Out, decode()) + dst = append(dst, decode()) } - return s.Out, br.close() + return dst, br.close() } // Decompress4X will decompress a 4X encoded stream. -// Before this is called, the table must be initialized with ReadTable unless -// the encoder re-used the table. // The length of the supplied input must match the end of a block exactly. -// The destination size of the uncompressed data must be known and provided. -func (s *Scratch) Decompress4X(in []byte, dstSize int) (out []byte, err error) { +// The *capacity* of the dst slice must match the destination size of +// the uncompressed data exactly. +func (s *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { if len(s.dt.single) == 0 { return nil, errors.New("no table loaded") } - if len(in) < 6+(4*1) { + if len(src) < 6+(4*1) { return nil, errors.New("input too small") } - if dstSize > s.MaxDecodedSize { - return nil, ErrMaxDecodedSizeExceeded - } - // TODO: We do not detect when we overrun a buffer, except if the last one does. var br [4]bitReader start := 6 for i := 0; i < 3; i++ { - length := int(in[i*2]) | (int(in[i*2+1]) << 8) - if start+length >= len(in) { + length := int(src[i*2]) | (int(src[i*2+1]) << 8) + if start+length >= len(src) { return nil, errors.New("truncated input (or invalid offset)") } - err = br[i].init(in[start : start+length]) + err := br[i].init(src[start : start+length]) if err != nil { return nil, err } start += length } - err = br[3].init(in[start:]) + err := br[3].init(src[start:]) if err != nil { return nil, err } - // Prepare output - if cap(s.Out) < dstSize { - s.Out = make([]byte, 0, dstSize) - } - s.Out = s.Out[:dstSize] // destination, offset to match first output - dstOut := s.Out + dstSize := cap(dst) + dst = dst[:dstSize] + out := dst dstEvery := (dstSize + 3) / 4 const tlSize = 1 << tableLogMax @@ -276,7 +318,7 @@ func (s *Scratch) Decompress4X(in []byte, dstSize int) (out []byte, err error) { } // Use temp table to avoid bound checks/append penalty. - var tmp = s.huffWeight[:256] + var buf [256]byte var off uint8 var decoded int @@ -300,8 +342,8 @@ bigloop: val2 := br[stream].peekBitsFast(s.actualTableLog) v2 := single[val2&tlMask] - tmp[off+bufoff*stream+1] = uint8(v2.entry >> 8) - tmp[off+bufoff*stream] = uint8(v.entry >> 8) + buf[off+bufoff*stream+1] = uint8(v2.entry >> 8) + buf[off+bufoff*stream] = uint8(v.entry >> 8) br[stream].bitsRead += uint8(v2.entry) } @@ -313,8 +355,8 @@ bigloop: val2 := br[stream].peekBitsFast(s.actualTableLog) v2 := single[val2&tlMask] - tmp[off+bufoff*stream+1] = uint8(v2.entry >> 8) - tmp[off+bufoff*stream] = uint8(v.entry >> 8) + buf[off+bufoff*stream+1] = uint8(v2.entry >> 8) + buf[off+bufoff*stream] = uint8(v.entry >> 8) br[stream].bitsRead += uint8(v2.entry) } @@ -326,8 +368,8 @@ bigloop: val2 := br[stream].peekBitsFast(s.actualTableLog) v2 := single[val2&tlMask] - tmp[off+bufoff*stream+1] = uint8(v2.entry >> 8) - tmp[off+bufoff*stream] = uint8(v.entry >> 8) + buf[off+bufoff*stream+1] = uint8(v2.entry >> 8) + buf[off+bufoff*stream] = uint8(v.entry >> 8) br[stream].bitsRead += uint8(v2.entry) } @@ -339,8 +381,8 @@ bigloop: val2 := br[stream].peekBitsFast(s.actualTableLog) v2 := single[val2&tlMask] - tmp[off+bufoff*stream+1] = uint8(v2.entry >> 8) - tmp[off+bufoff*stream] = uint8(v.entry >> 8) + buf[off+bufoff*stream+1] = uint8(v2.entry >> 8) + buf[off+bufoff*stream] = uint8(v.entry >> 8) br[stream].bitsRead += uint8(v2.entry) } @@ -350,30 +392,30 @@ bigloop: if bufoff > dstEvery { return nil, errors.New("corruption detected: stream overrun 1") } - copy(dstOut, tmp[:bufoff]) - copy(dstOut[dstEvery:], tmp[bufoff:bufoff*2]) - copy(dstOut[dstEvery*2:], tmp[bufoff*2:bufoff*3]) - copy(dstOut[dstEvery*3:], tmp[bufoff*3:bufoff*4]) + copy(out, buf[:bufoff]) + copy(out[dstEvery:], buf[bufoff:bufoff*2]) + copy(out[dstEvery*2:], buf[bufoff*2:bufoff*3]) + copy(out[dstEvery*3:], buf[bufoff*3:bufoff*4]) off = 0 - dstOut = dstOut[bufoff:] + out = out[bufoff:] decoded += 256 // There must at least be 3 buffers left. - if len(dstOut) < dstEvery*3 { + if len(out) < dstEvery*3 { return nil, errors.New("corruption detected: stream overrun 2") } } } if off > 0 { ioff := int(off) - if len(dstOut) < dstEvery*3+ioff { + if len(out) < dstEvery*3+ioff { return nil, errors.New("corruption detected: stream overrun 3") } - copy(dstOut, tmp[:off]) - copy(dstOut[dstEvery:dstEvery+ioff], tmp[bufoff:bufoff*2]) - copy(dstOut[dstEvery*2:dstEvery*2+ioff], tmp[bufoff*2:bufoff*3]) - copy(dstOut[dstEvery*3:dstEvery*3+ioff], tmp[bufoff*3:bufoff*4]) + copy(out, buf[:off]) + copy(out[dstEvery:dstEvery+ioff], buf[bufoff:bufoff*2]) + copy(out[dstEvery*2:dstEvery*2+ioff], buf[bufoff*2:bufoff*3]) + copy(out[dstEvery*3:dstEvery*3+ioff], buf[bufoff*3:bufoff*4]) decoded += int(off) * 4 - dstOut = dstOut[off:] + out = out[off:] } // Decode remaining. @@ -382,10 +424,10 @@ bigloop: br := &br[i] for !br.finished() { br.fill() - if offset >= len(dstOut) { + if offset >= len(out) { return nil, errors.New("corruption detected: stream overrun 4") } - dstOut[offset] = decode(br) + out[offset] = decode(br) offset++ } decoded += offset - dstEvery*i @@ -397,7 +439,7 @@ bigloop: if dstSize != decoded { return nil, errors.New("corruption detected: short output block") } - return s.Out, nil + return dst, nil } // matches will compare a decoding table to a coding table. diff --git a/vendor/github.com/klauspost/compress/zstd/README.md b/vendor/github.com/klauspost/compress/zstd/README.md index bc977a302..f2a80b5d0 100644 --- a/vendor/github.com/klauspost/compress/zstd/README.md +++ b/vendor/github.com/klauspost/compress/zstd/README.md @@ -309,6 +309,20 @@ The decoder can be used for *concurrent* decompression of multiple buffers. It will only allow a certain number of concurrent operations to run. To tweak that yourself use the `WithDecoderConcurrency(n)` option when creating the decoder. +### Dictionaries + +Data compressed with [dictionaries](https://github.com/facebook/zstd#the-case-for-small-data-compression) can be decompressed. + +Dictionaries are added individually to Decoders. +Dictionaries are generated by the `zstd --train` command and contains an initial state for the decoder. +To add a dictionary use the `RegisterDict(data)` with the dictionary data before starting any decompression. + +The dictionary will be used automatically for the data that specifies them. + +A re-used Decoder will still contain the dictionaries registered. + +When registering a dictionary with the same ID it will override the existing. + ### Allocation-less operation The decoder has been designed to operate without allocations after a warmup. diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 19181caea..4a14242c7 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -461,26 +461,22 @@ func (b *blockDec) decodeCompressed(hist *history) error { if huff == nil { huff = &huff0.Scratch{} } - huff.Out = b.literalBuf[:0] huff, literals, err = huff0.ReadTable(literals, huff) if err != nil { println("reading huffman table:", err) return err } // Use our out buffer. - huff.Out = b.literalBuf[:0] - huff.MaxDecodedSize = litRegenSize if fourStreams { - literals, err = huff.Decompress4X(literals, litRegenSize) + literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) } else { - literals, err = huff.Decompress1X(literals) + literals, err = huff.Decoder().Decompress1X(b.literalBuf[:0:litRegenSize], literals) } if err != nil { println("decoding compressed literals:", err) return err } // Make sure we don't leak our literals buffer - huff.Out = nil if len(literals) != litRegenSize { return fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) } @@ -631,15 +627,12 @@ func (b *blockDec) decodeCompressed(hist *history) error { var err error // Use our out buffer. huff = hist.huffTree - huff.Out = b.literalBuf[:0] - huff.MaxDecodedSize = litRegenSize if fourStreams { - literals, err = huff.Decompress4X(literals, litRegenSize) + literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) } else { - literals, err = huff.Decompress1X(literals) + literals, err = huff.Decoder().Decompress1X(b.literalBuf[:0:litRegenSize], literals) } // Make sure we don't leak our literals buffer - huff.Out = nil if err != nil { println("decompressing literals:", err) return err @@ -649,12 +642,13 @@ func (b *blockDec) decodeCompressed(hist *history) error { } } else { if hist.huffTree != nil && huff != nil { - huffDecoderPool.Put(hist.huffTree) + if hist.dict == nil || hist.dict.litDec != hist.huffTree { + huffDecoderPool.Put(hist.huffTree) + } hist.huffTree = nil } } if huff != nil { - huff.Out = nil hist.huffTree = huff } if debug { @@ -687,14 +681,20 @@ func (b *blockDec) decodeCompressed(hist *history) error { // If only recent offsets were not transferred, this would be an obvious win. // Also, if first 3 sequences don't reference recent offsets, all sequences can be decoded. - if err := seqs.initialize(br, hist, literals, b.dst); err != nil { - println("initializing sequences:", err) - return err - } hbytes := hist.b if len(hbytes) > hist.windowSize { hbytes = hbytes[len(hbytes)-hist.windowSize:] + // We do not need history any more. + if hist.dict != nil { + hist.dict.content = nil + } } + + if err := seqs.initialize(br, hist, literals, b.dst); err != nil { + println("initializing sequences:", err) + return err + } + err = seqs.decode(nSeqs, br, hbytes) if err != nil { return err diff --git a/vendor/github.com/klauspost/compress/zstd/bytereader.go b/vendor/github.com/klauspost/compress/zstd/bytereader.go index dc4378b64..f708df1c4 100644 --- a/vendor/github.com/klauspost/compress/zstd/bytereader.go +++ b/vendor/github.com/klauspost/compress/zstd/bytereader.go @@ -4,6 +4,8 @@ package zstd +import "encoding/binary" + // byteReader provides a byte reader that reads // little endian values from a byte stream. // The input stream is manually advanced. @@ -55,12 +57,7 @@ func (b byteReader) Uint32() uint32 { } return v } - b2 := b.b[b.off : b.off+4 : b.off+4] - v3 := uint32(b2[3]) - v2 := uint32(b2[2]) - v1 := uint32(b2[1]) - v0 := uint32(b2[0]) - return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) + return binary.LittleEndian.Uint32(b.b[b.off : b.off+4]) } // unread returns the unread portion of the input. diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 324347623..8e34479ff 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -32,8 +32,9 @@ type Decoder struct { // Current read position used for Reader functionality. current decoderState - // Custom dictionaries - dicts map[uint32]struct{} + // Custom dictionaries. + // Always uses copies. + dicts map[uint32]dict // streamWg is the waitgroup for all streams streamWg sync.WaitGroup @@ -295,10 +296,18 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { frame.bBuf = input for { + frame.history.reset() err := frame.reset(&frame.bBuf) if err == io.EOF { return dst, nil } + if frame.DictionaryID != nil { + dict, ok := d.dicts[*frame.DictionaryID] + if !ok { + return nil, ErrUnknownDictionary + } + frame.history.setDict(&dict) + } if err != nil { return dst, err } @@ -393,6 +402,19 @@ func (d *Decoder) Close() { d.current.err = ErrDecoderClosed } +// RegisterDict will load a dictionary +func (d *Decoder) RegisterDict(b []byte) error { + dc, err := loadDict(b) + if err != nil { + return err + } + if d.dicts == nil { + d.dicts = make(map[uint32]dict, 1) + } + d.dicts[dc.id] = *dc + return nil +} + // IOReadCloser returns the decoder as an io.ReadCloser for convenience. // Any changes to the decoder will be reflected, so the returned ReadCloser // can be reused along with the decoder. @@ -466,6 +488,14 @@ func (d *Decoder) startStreamDecoder(inStream chan decodeStream) { if debug && err != nil { println("Frame decoder returned", err) } + if err == nil && frame.DictionaryID != nil { + dict, ok := d.dicts[*frame.DictionaryID] + if !ok { + err = ErrUnknownDictionary + } else { + frame.history.setDict(&dict) + } + } if err != nil { stream.output <- decodeOutput{ err: err, diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go new file mode 100644 index 000000000..8eb6f6ba3 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -0,0 +1,104 @@ +package zstd + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + + "github.com/klauspost/compress/huff0" +) + +type dict struct { + id uint32 + + litDec *huff0.Scratch + llDec, ofDec, mlDec sequenceDec + offsets [3]int + content []byte +} + +var dictMagic = [4]byte{0x37, 0xa4, 0x30, 0xec} + +// Load a dictionary as described in +// https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format +func loadDict(b []byte) (*dict, error) { + // Check static field size. + if len(b) <= 8+(3*4) { + return nil, io.ErrUnexpectedEOF + } + d := dict{ + llDec: sequenceDec{fse: &fseDecoder{}}, + ofDec: sequenceDec{fse: &fseDecoder{}}, + mlDec: sequenceDec{fse: &fseDecoder{}}, + } + if !bytes.Equal(b[:4], dictMagic[:]) { + return nil, ErrMagicMismatch + } + d.id = binary.LittleEndian.Uint32(b[4:8]) + if d.id == 0 { + return nil, errors.New("dictionaries cannot have ID 0") + } + + // Read literal table + var err error + d.litDec, b, err = huff0.ReadTable(b[8:], nil) + if err != nil { + return nil, err + } + + br := byteReader{ + b: b, + off: 0, + } + readDec := func(i tableIndex, dec *fseDecoder) error { + if err := dec.readNCount(&br, uint16(maxTableSymbol[i])); err != nil { + return err + } + if br.overread() { + return io.ErrUnexpectedEOF + } + err = dec.transform(symbolTableX[i]) + if err != nil { + println("Transform table error:", err) + return err + } + if debug { + println("Read table ok", "symbolLen:", dec.symbolLen) + } + // Set decoders as predefined so they aren't reused. + dec.preDefined = true + return nil + } + + if err := readDec(tableOffsets, d.ofDec.fse); err != nil { + return nil, err + } + if err := readDec(tableMatchLengths, d.mlDec.fse); err != nil { + return nil, err + } + if err := readDec(tableLiteralLengths, d.llDec.fse); err != nil { + return nil, err + } + if br.remain() < 12 { + return nil, io.ErrUnexpectedEOF + } + + d.offsets[0] = int(br.Uint32()) + br.advance(4) + d.offsets[1] = int(br.Uint32()) + br.advance(4) + d.offsets[2] = int(br.Uint32()) + br.advance(4) + if d.offsets[0] <= 0 || d.offsets[1] <= 0 || d.offsets[2] <= 0 { + return nil, errors.New("invalid offset in dictionary") + } + d.content = make([]byte, br.remain()) + copy(d.content, br.unread()) + if d.offsets[0] > len(d.content) || d.offsets[1] > len(d.content) || d.offsets[2] > len(d.content) { + return nil, fmt.Errorf("initial offset bigger than dictionary content size %d, offsets: %v", len(d.content), d.offsets) + } + + return &d, nil +} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index 5ebead9dc..50276bcde 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -671,4 +671,8 @@ encodeLoop: println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } + // We do not store history, so we must offset e.cur to avoid false matches for next user. + if e.cur < bufferReset { + e.cur += int32(len(src)) + } } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index d1d3658e6..4104b456c 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -383,6 +383,7 @@ func (e *fastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { panic("src too big") } } + // Protect against e.cur wraparound. if e.cur >= bufferReset { for i := range e.table[:] { @@ -516,6 +517,9 @@ encodeLoop: if debugAsserts && s-t > e.maxMatchOff { panic("s - t >e.maxMatchOff") } + if debugAsserts && t < 0 { + panic(fmt.Sprintf("t (%d) < 0, candidate.offset: %d, e.cur: %d, coffset0: %d, e.maxMatchOff: %d", t, candidate.offset, e.cur, coffset0, e.maxMatchOff)) + } break } @@ -548,6 +552,9 @@ encodeLoop: panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) } + if debugAsserts && t < 0 { + panic(fmt.Sprintf("t (%d) < 0 ", t)) + } // Extend the 4-byte match as long as possible. //l := e.matchlenNoHist(s+4, t+4, src) + 4 // l := int32(matchLen(src[s+4:], src[t+4:])) + 4 @@ -647,6 +654,10 @@ encodeLoop: if debug { println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } + // We do not store history, so we must offset e.cur to avoid false matches for next user. + if e.cur < bufferReset { + e.cur += int32(len(src)) + } } func (e *fastBase) addBlock(src []byte) int32 { @@ -714,7 +725,7 @@ func (e *fastBase) matchlen(s, t int32, src []byte) int32 { } // Reset the encoding table. -func (e *fastBase) Reset() { +func (e *fastBase) Reset(singleBlock bool) { if e.blk == nil { e.blk = &blockEnc{} e.blk.init() @@ -727,7 +738,7 @@ func (e *fastBase) Reset() { } else { e.crc.Reset() } - if cap(e.hist) < int(e.maxMatchOff*2) { + if !singleBlock && cap(e.hist) < int(e.maxMatchOff*2) { l := e.maxMatchOff * 2 // Make it at least 1MB. if l < 1<<20 { diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index af4f00b73..bf42bb1cf 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -35,7 +35,7 @@ type encoder interface { AppendCRC([]byte) []byte WindowSize(size int) int32 UseBlock(*blockEnc) - Reset() + Reset(singleBlock bool) } type encoderState struct { @@ -82,7 +82,10 @@ func (e *Encoder) initialize() { } e.encoders = make(chan encoder, e.o.concurrent) for i := 0; i < e.o.concurrent; i++ { - e.encoders <- e.o.encoder() + enc := e.o.encoder() + // If not single block, history will be allocated on first use. + enc.Reset(true) + e.encoders <- enc } } @@ -112,7 +115,7 @@ func (e *Encoder) Reset(w io.Writer) { s.filling = s.filling[:0] s.current = s.current[:0] s.previous = s.previous[:0] - s.encoder.Reset() + s.encoder.Reset(false) s.headerWritten = false s.eofWritten = false s.fullFrameWritten = false @@ -445,11 +448,10 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { enc := <-e.encoders defer func() { // Release encoder reference to last block. - enc.Reset() + // If a non-single block is needed the encoder will reset again. + enc.Reset(true) e.encoders <- enc }() - enc.Reset() - blk := enc.Block() // Use single segments when above minimum window and below 1MB. single := len(src) < 1<<20 && len(src) > MinWindowSize if e.o.single != nil { @@ -472,12 +474,13 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { panic(err) } - if len(src) <= e.o.blockSize && len(src) <= maxBlockSize { + // If we can do everything in one block, prefer that. + if len(src) <= maxCompressedBlockSize { // Slightly faster with no history and everything in one block. if e.o.crc { _, _ = enc.CRC().Write(src) } - blk.reset(nil) + blk := enc.Block() blk.last = true enc.EncodeNoHist(blk, src) @@ -504,6 +507,8 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } blk.output = oldout } else { + enc.Reset(false) + blk := enc.Block() for len(src) > 0 { todo := src if len(todo) > e.o.blockSize { diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index 780880ebe..fc4a566d3 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -40,7 +40,7 @@ type frameDec struct { FrameContentSize uint64 frameDone sync.WaitGroup - DictionaryID uint32 + DictionaryID *uint32 HasCheckSum bool SingleSegment bool @@ -142,7 +142,7 @@ func (d *frameDec) reset(br byteBuffer) error { // Read Dictionary_ID // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary_id - d.DictionaryID = 0 + d.DictionaryID = nil if size := fhd & 3; size != 0 { if size == 3 { size = 4 @@ -154,19 +154,22 @@ func (d *frameDec) reset(br byteBuffer) error { } return io.ErrUnexpectedEOF } + var id uint32 switch size { case 1: - d.DictionaryID = uint32(b[0]) + id = uint32(b[0]) case 2: - d.DictionaryID = uint32(b[0]) | (uint32(b[1]) << 8) + id = uint32(b[0]) | (uint32(b[1]) << 8) case 4: - d.DictionaryID = uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) + id = uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) } if debug { - println("Dict size", size, "ID:", d.DictionaryID) + println("Dict size", size, "ID:", id) } - if d.DictionaryID != 0 { - return ErrUnknownDictionary + if id > 0 { + // ID 0 means "sorry, no dictionary anyway". + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format + d.DictionaryID = &id } } @@ -351,8 +354,6 @@ func (d *frameDec) initAsync() { // When the frame has finished decoding the *bufio.Reader // containing the remaining input will be sent on frameDec.frameDone. func (d *frameDec) startDecoder(output chan decodeOutput) { - // TODO: Init to dictionary - d.history.reset() written := int64(0) defer func() { @@ -445,8 +446,6 @@ func (d *frameDec) startDecoder(output chan decodeOutput) { // runDecoder will create a sync decoder that will decode a block of data. func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { - // TODO: Init to dictionary - d.history.reset() saved := d.history.b // We use the history for output to avoid copying it. diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go index e002be98b..957cfeb79 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go @@ -19,7 +19,7 @@ const ( * Increasing memory usage improves compression ratio * Reduced memory usage can improve speed, due to cache effect * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ - maxMemoryUsage = 11 + maxMemoryUsage = tablelogAbsoluteMax + 2 maxTableLog = maxMemoryUsage - 2 maxTablesize = 1 << maxTableLog diff --git a/vendor/github.com/klauspost/compress/zstd/history.go b/vendor/github.com/klauspost/compress/zstd/history.go index e8c419bd5..f418f50fc 100644 --- a/vendor/github.com/klauspost/compress/zstd/history.go +++ b/vendor/github.com/klauspost/compress/zstd/history.go @@ -17,6 +17,7 @@ type history struct { windowSize int maxSize int error bool + dict *dict } // reset will reset the history to initial state of a frame. @@ -36,12 +37,27 @@ func (h *history) reset() { } h.decoders = sequenceDecs{} if h.huffTree != nil { - huffDecoderPool.Put(h.huffTree) + if h.dict == nil || h.dict.litDec != h.huffTree { + huffDecoderPool.Put(h.huffTree) + } } h.huffTree = nil + h.dict = nil //printf("history created: %+v (l: %d, c: %d)", *h, len(h.b), cap(h.b)) } +func (h *history) setDict(dict *dict) { + if dict == nil { + return + } + h.dict = dict + h.decoders.litLengths = dict.llDec + h.decoders.offsets = dict.ofDec + h.decoders.matchLengths = dict.mlDec + h.recentOffsets = dict.offsets + h.huffTree = dict.litDec +} + // append bytes to history. // This function will make sure there is space for it, // if the buffer has been allocated with enough extra space. diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go index 39238e16a..7ff870400 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -62,6 +62,7 @@ type sequenceDecs struct { matchLengths sequenceDec prevOffset [3]int hist []byte + dict []byte literals []byte out []byte windowSize int @@ -85,6 +86,10 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, literals, out [] s.maxBits = s.litLengths.fse.maxBits + s.offsets.fse.maxBits + s.matchLengths.fse.maxBits s.windowSize = hist.windowSize s.out = out + s.dict = nil + if hist.dict != nil { + s.dict = hist.dict.content + } return nil } @@ -100,23 +105,78 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { printf("reading sequence %d, exceeded available data\n", seqs-i) return io.ErrUnexpectedEOF } - var litLen, matchOff, matchLen int + var ll, mo, ml int if br.off > 4+((maxOffsetBits+16+16)>>3) { - litLen, matchOff, matchLen = s.nextFast(br, llState, mlState, ofState) + // inlined function: + // ll, mo, ml = s.nextFast(br, llState, mlState, ofState) + + // Final will not read from stream. + var llB, mlB, moB uint8 + ll, llB = llState.final() + ml, mlB = mlState.final() + mo, moB = ofState.final() + + // extra bits are stored in reverse order. + br.fillFast() + mo += br.getBits(moB) + if s.maxBits > 32 { + br.fillFast() + } + ml += br.getBits(mlB) + ll += br.getBits(llB) + + if moB > 1 { + s.prevOffset[2] = s.prevOffset[1] + s.prevOffset[1] = s.prevOffset[0] + s.prevOffset[0] = mo + } else { + // mo = s.adjustOffset(mo, ll, moB) + // Inlined for rather big speedup + if ll == 0 { + // There is an exception though, when current sequence's literals_length = 0. + // In this case, repeated offsets are shifted by one, so an offset_value of 1 means Repeated_Offset2, + // an offset_value of 2 means Repeated_Offset3, and an offset_value of 3 means Repeated_Offset1 - 1_byte. + mo++ + } + + if mo == 0 { + mo = s.prevOffset[0] + } else { + var temp int + if mo == 3 { + temp = s.prevOffset[0] - 1 + } else { + temp = s.prevOffset[mo] + } + + if temp == 0 { + // 0 is not valid; input is corrupted; force offset to 1 + println("temp was 0") + temp = 1 + } + + if mo != 1 { + s.prevOffset[2] = s.prevOffset[1] + } + s.prevOffset[1] = s.prevOffset[0] + s.prevOffset[0] = temp + mo = temp + } + } br.fillFast() } else { - litLen, matchOff, matchLen = s.next(br, llState, mlState, ofState) + ll, mo, ml = s.next(br, llState, mlState, ofState) br.fill() } if debugSequences { - println("Seq", seqs-i-1, "Litlen:", litLen, "matchOff:", matchOff, "(abs) matchLen:", matchLen) + println("Seq", seqs-i-1, "Litlen:", ll, "mo:", mo, "(abs) ml:", ml) } - if litLen > len(s.literals) { - return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", litLen, len(s.literals)) + if ll > len(s.literals) { + return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, len(s.literals)) } - size := litLen + matchLen + len(s.out) + size := ll + ml + len(s.out) if size-startSize > maxBlockSize { return fmt.Errorf("output (%d) bigger than max block size", size) } @@ -127,52 +187,70 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { s.out = append(s.out, make([]byte, maxBlockSize)...) s.out = s.out[:len(s.out)-maxBlockSize] } - if matchLen > maxMatchLen { - return fmt.Errorf("match len (%d) bigger than max allowed length", matchLen) - } - if matchOff > len(s.out)+len(hist)+litLen { - return fmt.Errorf("match offset (%d) bigger than current history (%d)", matchOff, len(s.out)+len(hist)+litLen) - } - if matchOff > s.windowSize { - return fmt.Errorf("match offset (%d) bigger than window size (%d)", matchOff, s.windowSize) - } - if matchOff == 0 && matchLen > 0 { - return fmt.Errorf("zero matchoff and matchlen > 0") + if ml > maxMatchLen { + return fmt.Errorf("match len (%d) bigger than max allowed length", ml) } - s.out = append(s.out, s.literals[:litLen]...) - s.literals = s.literals[litLen:] + // Add literals + s.out = append(s.out, s.literals[:ll]...) + s.literals = s.literals[ll:] out := s.out + if mo > len(s.out)+len(hist) || mo > s.windowSize { + if len(s.dict) == 0 { + return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(s.out)+len(hist)) + } + + // we may be in dictionary. + dictO := len(s.dict) - (mo - (len(s.out) + len(hist))) + if dictO < 0 || dictO >= len(s.dict) { + return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(s.out)+len(hist)) + } + end := dictO + ml + if end > len(s.dict) { + out = append(out, s.dict[dictO:]...) + mo -= len(s.dict) - dictO + ml -= len(s.dict) - dictO + } else { + out = append(out, s.dict[dictO:end]...) + mo = 0 + ml = 0 + } + } + + if mo == 0 && ml > 0 { + return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) + } + // Copy from history. // TODO: Blocks without history could be made to ignore this completely. - if v := matchOff - len(s.out); v > 0 { + if v := mo - len(s.out); v > 0 { // v is the start position in history from end. start := len(s.hist) - v - if matchLen > v { + if ml > v { // Some goes into current block. // Copy remainder of history out = append(out, s.hist[start:]...) - matchOff -= v - matchLen -= v + mo -= v + ml -= v } else { - out = append(out, s.hist[start:start+matchLen]...) - matchLen = 0 + out = append(out, s.hist[start:start+ml]...) + ml = 0 } } // We must be in current buffer now - if matchLen > 0 { - start := len(s.out) - matchOff - if matchLen <= len(s.out)-start { + if ml > 0 { + start := len(s.out) - mo + if ml <= len(s.out)-start { // No overlap - out = append(out, s.out[start:start+matchLen]...) + out = append(out, s.out[start:start+ml]...) } else { // Overlapping copy // Extend destination slice and copy one byte at the time. - out = out[:len(out)+matchLen] - src := out[start : start+matchLen] + out = out[:len(out)+ml] + src := out[start : start+ml] // Destination is the space we just added. - dst := out[len(out)-matchLen:] + dst := out[len(out)-ml:] dst = dst[:len(src)] for i := range src { dst[i] = src[i] diff --git a/vendor/github.com/onsi/ginkgo/CHANGELOG.md b/vendor/github.com/onsi/ginkgo/CHANGELOG.md index 52d8ca36d..395fa90f4 100644 --- a/vendor/github.com/onsi/ginkgo/CHANGELOG.md +++ b/vendor/github.com/onsi/ginkgo/CHANGELOG.md @@ -1,3 +1,8 @@ +## 1.12.3 + +### Fixes +- Print correct code location of failing table test (#666) [c6d7afb] + ## 1.12.2 ### Fixes diff --git a/vendor/github.com/onsi/ginkgo/README.md b/vendor/github.com/onsi/ginkgo/README.md index 2fda7b9eb..ddb0cd882 100644 --- a/vendor/github.com/onsi/ginkgo/README.md +++ b/vendor/github.com/onsi/ginkgo/README.md @@ -4,7 +4,23 @@ Jump to the [docs](https://onsi.github.io/ginkgo/) to learn more. To start rolling your Ginkgo tests *now* [keep reading](#set-me-up)! -If you have a question, comment, bug report, feature request, etc. please open a GitHub issue. +If you have a question, comment, bug report, feature request, etc. please open a GitHub issue, or visit the [Ginkgo Slack channel](https://app.slack.com/client/T029RQSE6/CQQ50BBNW). + +## TLDR +Ginkgo builds on Go's `testing` package, allowing expressive [Behavior-Driven Development](https://en.wikipedia.org/wiki/Behavior-driven_development) ("BDD") style tests. +It is typically (and optionally) paired with the [Gomega](https://github.com/onsi/gomega) matcher library. + +```go +Describe("the strings package", func() { + Context("strings.Contains()", func() { + When("the string contains the substring in the middle", func() { + It("returns `true`", func() { + Expect(strings.Contains("Ginkgo is awesome", "is")).To(BeTrue()) + }) + }) + }) +}) +``` ## Feature List diff --git a/vendor/github.com/onsi/ginkgo/config/config.go b/vendor/github.com/onsi/ginkgo/config/config.go index 342645022..5d4b35732 100644 --- a/vendor/github.com/onsi/ginkgo/config/config.go +++ b/vendor/github.com/onsi/ginkgo/config/config.go @@ -20,7 +20,7 @@ import ( "fmt" ) -const VERSION = "1.12.2" +const VERSION = "1.12.3" type GinkgoConfigType struct { RandomSeed int64 diff --git a/vendor/github.com/onsi/ginkgo/extensions/table/table.go b/vendor/github.com/onsi/ginkgo/extensions/table/table.go index ae8ab7d24..2ed5314f2 100644 --- a/vendor/github.com/onsi/ginkgo/extensions/table/table.go +++ b/vendor/github.com/onsi/ginkgo/extensions/table/table.go @@ -12,7 +12,9 @@ import ( "fmt" "reflect" - "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/internal/codelocation" + "github.com/onsi/ginkgo/internal/global" + "github.com/onsi/ginkgo/types" ) /* @@ -42,7 +44,7 @@ It's important to understand that the `Describe`s and `It`s are generated at eva Individual Entries can be focused (with FEntry) or marked pending (with PEntry or XEntry). In addition, the entire table can be focused or marked pending with FDescribeTable and PDescribeTable/XDescribeTable. */ func DescribeTable(description string, itBody interface{}, entries ...TableEntry) bool { - describeTable(description, itBody, entries, false, false) + describeTable(description, itBody, entries, types.FlagTypeNone) return true } @@ -50,7 +52,7 @@ func DescribeTable(description string, itBody interface{}, entries ...TableEntry You can focus a table with `FDescribeTable`. This is equivalent to `FDescribe`. */ func FDescribeTable(description string, itBody interface{}, entries ...TableEntry) bool { - describeTable(description, itBody, entries, false, true) + describeTable(description, itBody, entries, types.FlagTypeFocused) return true } @@ -58,7 +60,7 @@ func FDescribeTable(description string, itBody interface{}, entries ...TableEntr You can mark a table as pending with `PDescribeTable`. This is equivalent to `PDescribe`. */ func PDescribeTable(description string, itBody interface{}, entries ...TableEntry) bool { - describeTable(description, itBody, entries, true, false) + describeTable(description, itBody, entries, types.FlagTypePending) return true } @@ -66,33 +68,24 @@ func PDescribeTable(description string, itBody interface{}, entries ...TableEntr You can mark a table as pending with `XDescribeTable`. This is equivalent to `XDescribe`. */ func XDescribeTable(description string, itBody interface{}, entries ...TableEntry) bool { - describeTable(description, itBody, entries, true, false) + describeTable(description, itBody, entries, types.FlagTypePending) return true } -func describeTable(description string, itBody interface{}, entries []TableEntry, pending bool, focused bool) { +func describeTable(description string, itBody interface{}, entries []TableEntry, flag types.FlagType) { itBodyValue := reflect.ValueOf(itBody) if itBodyValue.Kind() != reflect.Func { panic(fmt.Sprintf("DescribeTable expects a function, got %#v", itBody)) } - if pending { - ginkgo.PDescribe(description, func() { + global.Suite.PushContainerNode( + description, + func() { for _, entry := range entries { entry.generateIt(itBodyValue) } - }) - } else if focused { - ginkgo.FDescribe(description, func() { - for _, entry := range entries { - entry.generateIt(itBodyValue) - } - }) - } else { - ginkgo.Describe(description, func() { - for _, entry := range entries { - entry.generateIt(itBodyValue) - } - }) - } + }, + flag, + codelocation.New(2), + ) } diff --git a/vendor/github.com/onsi/ginkgo/extensions/table/table_entry.go b/vendor/github.com/onsi/ginkgo/extensions/table/table_entry.go index 93f3bc3b0..1a919a1fe 100644 --- a/vendor/github.com/onsi/ginkgo/extensions/table/table_entry.go +++ b/vendor/github.com/onsi/ginkgo/extensions/table/table_entry.go @@ -3,22 +3,31 @@ package table import ( "reflect" - "github.com/onsi/ginkgo" + "github.com/onsi/ginkgo/internal/codelocation" + "github.com/onsi/ginkgo/internal/global" + "github.com/onsi/ginkgo/types" ) /* TableEntry represents an entry in a table test. You generally use the `Entry` constructor. */ type TableEntry struct { - Description string - Parameters []interface{} - Pending bool - Focused bool + Description string + Parameters []interface{} + Pending bool + Focused bool + codeLocation types.CodeLocation } func (t TableEntry) generateIt(itBody reflect.Value) { + if t.codeLocation == (types.CodeLocation{}) { + // The user created the TableEntry struct directly instead of having used the (F/P/X)Entry constructors. + // Therefore default to the code location of the surrounding DescribeTable. + t.codeLocation = codelocation.New(5) + } + if t.Pending { - ginkgo.PIt(t.Description) + global.Suite.PushItNode(t.Description, func() {}, types.FlagTypePending, t.codeLocation, 0) return } @@ -38,9 +47,9 @@ func (t TableEntry) generateIt(itBody reflect.Value) { } if t.Focused { - ginkgo.FIt(t.Description, body) + global.Suite.PushItNode(t.Description, body, types.FlagTypeFocused, t.codeLocation, global.DefaultTimeout) } else { - ginkgo.It(t.Description, body) + global.Suite.PushItNode(t.Description, body, types.FlagTypeNone, t.codeLocation, global.DefaultTimeout) } } @@ -53,26 +62,26 @@ Subsequent parameters are saved off and sent to the callback passed in to `Descr Each Entry ends up generating an individual Ginkgo It. */ func Entry(description string, parameters ...interface{}) TableEntry { - return TableEntry{description, parameters, false, false} + return TableEntry{description, parameters, false, false, codelocation.New(1)} } /* You can focus a particular entry with FEntry. This is equivalent to FIt. */ func FEntry(description string, parameters ...interface{}) TableEntry { - return TableEntry{description, parameters, false, true} + return TableEntry{description, parameters, false, true, codelocation.New(1)} } /* You can mark a particular entry as pending with PEntry. This is equivalent to PIt. */ func PEntry(description string, parameters ...interface{}) TableEntry { - return TableEntry{description, parameters, true, false} + return TableEntry{description, parameters, true, false, codelocation.New(1)} } /* You can mark a particular entry as pending with XEntry. This is equivalent to XIt. */ func XEntry(description string, parameters ...interface{}) TableEntry { - return TableEntry{description, parameters, true, false} + return TableEntry{description, parameters, true, false, codelocation.New(1)} } diff --git a/vendor/github.com/onsi/ginkgo/ginkgo_dsl.go b/vendor/github.com/onsi/ginkgo/ginkgo_dsl.go index 3cbf89a35..30ff86f59 100644 --- a/vendor/github.com/onsi/ginkgo/ginkgo_dsl.go +++ b/vendor/github.com/onsi/ginkgo/ginkgo_dsl.go @@ -22,9 +22,8 @@ import ( "github.com/onsi/ginkgo/config" "github.com/onsi/ginkgo/internal/codelocation" - "github.com/onsi/ginkgo/internal/failer" + "github.com/onsi/ginkgo/internal/global" "github.com/onsi/ginkgo/internal/remote" - "github.com/onsi/ginkgo/internal/suite" "github.com/onsi/ginkgo/internal/testingtproxy" "github.com/onsi/ginkgo/internal/writer" "github.com/onsi/ginkgo/reporters" @@ -46,16 +45,10 @@ To circumvent this, you should call at the top of the goroutine that caused this panic. ` -const defaultTimeout = 1 - -var globalSuite *suite.Suite -var globalFailer *failer.Failer func init() { config.Flags(flag.CommandLine, "ginkgo", true) GinkgoWriter = writer.New(os.Stdout) - globalFailer = failer.New() - globalSuite = suite.New(globalFailer) } //GinkgoWriter implements an io.Writer @@ -156,7 +149,7 @@ type GinkgoTestDescription struct { //CurrentGinkgoTestDescripton returns information about the current running test. func CurrentGinkgoTestDescription() GinkgoTestDescription { - summary, ok := globalSuite.CurrentRunningSpecSummary() + summary, ok := global.Suite.CurrentRunningSpecSummary() if !ok { return GinkgoTestDescription{} } @@ -223,7 +216,7 @@ func RunSpecsWithCustomReporters(t GinkgoTestingT, description string, specRepor for i, reporter := range specReporters { reporters[i] = reporter } - passed, hasFocusedTests := globalSuite.Run(t, description, reporters, writer, config.GinkgoConfig) + passed, hasFocusedTests := global.Suite.Run(t, description, reporters, writer, config.GinkgoConfig) if passed && hasFocusedTests && strings.TrimSpace(os.Getenv("GINKGO_EDITOR_INTEGRATION")) == "" { fmt.Println("PASS | FOCUSED") os.Exit(types.GINKGO_FOCUS_EXIT_CODE) @@ -252,7 +245,7 @@ func Skip(message string, callerSkip ...int) { skip = callerSkip[0] } - globalFailer.Skip(message, codelocation.New(skip+1)) + global.Failer.Skip(message, codelocation.New(skip+1)) panic(GINKGO_PANIC) } @@ -263,7 +256,7 @@ func Fail(message string, callerSkip ...int) { skip = callerSkip[0] } - globalFailer.Fail(message, codelocation.New(skip+1)) + global.Failer.Fail(message, codelocation.New(skip+1)) panic(GINKGO_PANIC) } @@ -280,7 +273,7 @@ func Fail(message string, callerSkip ...int) { func GinkgoRecover() { e := recover() if e != nil { - globalFailer.Panic(codelocation.New(1), e) + global.Failer.Panic(codelocation.New(1), e) } } @@ -291,25 +284,25 @@ func GinkgoRecover() { //equivalent. The difference is purely semantic -- you typically Describe the behavior of an object //or method and, within that Describe, outline a number of Contexts and Whens. func Describe(text string, body func()) bool { - globalSuite.PushContainerNode(text, body, types.FlagTypeNone, codelocation.New(1)) + global.Suite.PushContainerNode(text, body, types.FlagTypeNone, codelocation.New(1)) return true } //You can focus the tests within a describe block using FDescribe func FDescribe(text string, body func()) bool { - globalSuite.PushContainerNode(text, body, types.FlagTypeFocused, codelocation.New(1)) + global.Suite.PushContainerNode(text, body, types.FlagTypeFocused, codelocation.New(1)) return true } //You can mark the tests within a describe block as pending using PDescribe func PDescribe(text string, body func()) bool { - globalSuite.PushContainerNode(text, body, types.FlagTypePending, codelocation.New(1)) + global.Suite.PushContainerNode(text, body, types.FlagTypePending, codelocation.New(1)) return true } //You can mark the tests within a describe block as pending using XDescribe func XDescribe(text string, body func()) bool { - globalSuite.PushContainerNode(text, body, types.FlagTypePending, codelocation.New(1)) + global.Suite.PushContainerNode(text, body, types.FlagTypePending, codelocation.New(1)) return true } @@ -320,25 +313,25 @@ func XDescribe(text string, body func()) bool { //equivalent. The difference is purely semantic -- you typical Describe the behavior of an object //or method and, within that Describe, outline a number of Contexts and Whens. func Context(text string, body func()) bool { - globalSuite.PushContainerNode(text, body, types.FlagTypeNone, codelocation.New(1)) + global.Suite.PushContainerNode(text, body, types.FlagTypeNone, codelocation.New(1)) return true } //You can focus the tests within a describe block using FContext func FContext(text string, body func()) bool { - globalSuite.PushContainerNode(text, body, types.FlagTypeFocused, codelocation.New(1)) + global.Suite.PushContainerNode(text, body, types.FlagTypeFocused, codelocation.New(1)) return true } //You can mark the tests within a describe block as pending using PContext func PContext(text string, body func()) bool { - globalSuite.PushContainerNode(text, body, types.FlagTypePending, codelocation.New(1)) + global.Suite.PushContainerNode(text, body, types.FlagTypePending, codelocation.New(1)) return true } //You can mark the tests within a describe block as pending using XContext func XContext(text string, body func()) bool { - globalSuite.PushContainerNode(text, body, types.FlagTypePending, codelocation.New(1)) + global.Suite.PushContainerNode(text, body, types.FlagTypePending, codelocation.New(1)) return true } @@ -349,25 +342,25 @@ func XContext(text string, body func()) bool { //equivalent. The difference is purely semantic -- you typical Describe the behavior of an object //or method and, within that Describe, outline a number of Contexts and Whens. func When(text string, body func()) bool { - globalSuite.PushContainerNode("when "+text, body, types.FlagTypeNone, codelocation.New(1)) + global.Suite.PushContainerNode("when "+text, body, types.FlagTypeNone, codelocation.New(1)) return true } //You can focus the tests within a describe block using FWhen func FWhen(text string, body func()) bool { - globalSuite.PushContainerNode("when "+text, body, types.FlagTypeFocused, codelocation.New(1)) + global.Suite.PushContainerNode("when "+text, body, types.FlagTypeFocused, codelocation.New(1)) return true } //You can mark the tests within a describe block as pending using PWhen func PWhen(text string, body func()) bool { - globalSuite.PushContainerNode("when "+text, body, types.FlagTypePending, codelocation.New(1)) + global.Suite.PushContainerNode("when "+text, body, types.FlagTypePending, codelocation.New(1)) return true } //You can mark the tests within a describe block as pending using XWhen func XWhen(text string, body func()) bool { - globalSuite.PushContainerNode("when "+text, body, types.FlagTypePending, codelocation.New(1)) + global.Suite.PushContainerNode("when "+text, body, types.FlagTypePending, codelocation.New(1)) return true } @@ -377,25 +370,25 @@ func XWhen(text string, body func()) bool { //Ginkgo will normally run It blocks synchronously. To perform asynchronous tests, pass a //function that accepts a Done channel. When you do this, you can also provide an optional timeout. func It(text string, body interface{}, timeout ...float64) bool { - globalSuite.PushItNode(text, body, types.FlagTypeNone, codelocation.New(1), parseTimeout(timeout...)) + global.Suite.PushItNode(text, body, types.FlagTypeNone, codelocation.New(1), parseTimeout(timeout...)) return true } //You can focus individual Its using FIt func FIt(text string, body interface{}, timeout ...float64) bool { - globalSuite.PushItNode(text, body, types.FlagTypeFocused, codelocation.New(1), parseTimeout(timeout...)) + global.Suite.PushItNode(text, body, types.FlagTypeFocused, codelocation.New(1), parseTimeout(timeout...)) return true } //You can mark Its as pending using PIt func PIt(text string, _ ...interface{}) bool { - globalSuite.PushItNode(text, func() {}, types.FlagTypePending, codelocation.New(1), 0) + global.Suite.PushItNode(text, func() {}, types.FlagTypePending, codelocation.New(1), 0) return true } //You can mark Its as pending using XIt func XIt(text string, _ ...interface{}) bool { - globalSuite.PushItNode(text, func() {}, types.FlagTypePending, codelocation.New(1), 0) + global.Suite.PushItNode(text, func() {}, types.FlagTypePending, codelocation.New(1), 0) return true } @@ -403,25 +396,25 @@ func XIt(text string, _ ...interface{}) bool { //which "It" does not fit into a natural sentence flow. All the same protocols apply for Specify blocks //which apply to It blocks. func Specify(text string, body interface{}, timeout ...float64) bool { - globalSuite.PushItNode(text, body, types.FlagTypeNone, codelocation.New(1), parseTimeout(timeout...)) + global.Suite.PushItNode(text, body, types.FlagTypeNone, codelocation.New(1), parseTimeout(timeout...)) return true } //You can focus individual Specifys using FSpecify func FSpecify(text string, body interface{}, timeout ...float64) bool { - globalSuite.PushItNode(text, body, types.FlagTypeFocused, codelocation.New(1), parseTimeout(timeout...)) + global.Suite.PushItNode(text, body, types.FlagTypeFocused, codelocation.New(1), parseTimeout(timeout...)) return true } //You can mark Specifys as pending using PSpecify func PSpecify(text string, is ...interface{}) bool { - globalSuite.PushItNode(text, func() {}, types.FlagTypePending, codelocation.New(1), 0) + global.Suite.PushItNode(text, func() {}, types.FlagTypePending, codelocation.New(1), 0) return true } //You can mark Specifys as pending using XSpecify func XSpecify(text string, is ...interface{}) bool { - globalSuite.PushItNode(text, func() {}, types.FlagTypePending, codelocation.New(1), 0) + global.Suite.PushItNode(text, func() {}, types.FlagTypePending, codelocation.New(1), 0) return true } @@ -452,25 +445,25 @@ func By(text string, callbacks ...func()) { //The body function must have the signature: // func(b Benchmarker) func Measure(text string, body interface{}, samples int) bool { - globalSuite.PushMeasureNode(text, body, types.FlagTypeNone, codelocation.New(1), samples) + global.Suite.PushMeasureNode(text, body, types.FlagTypeNone, codelocation.New(1), samples) return true } //You can focus individual Measures using FMeasure func FMeasure(text string, body interface{}, samples int) bool { - globalSuite.PushMeasureNode(text, body, types.FlagTypeFocused, codelocation.New(1), samples) + global.Suite.PushMeasureNode(text, body, types.FlagTypeFocused, codelocation.New(1), samples) return true } //You can mark Measurements as pending using PMeasure func PMeasure(text string, _ ...interface{}) bool { - globalSuite.PushMeasureNode(text, func(b Benchmarker) {}, types.FlagTypePending, codelocation.New(1), 0) + global.Suite.PushMeasureNode(text, func(b Benchmarker) {}, types.FlagTypePending, codelocation.New(1), 0) return true } //You can mark Measurements as pending using XMeasure func XMeasure(text string, _ ...interface{}) bool { - globalSuite.PushMeasureNode(text, func(b Benchmarker) {}, types.FlagTypePending, codelocation.New(1), 0) + global.Suite.PushMeasureNode(text, func(b Benchmarker) {}, types.FlagTypePending, codelocation.New(1), 0) return true } @@ -481,7 +474,7 @@ func XMeasure(text string, _ ...interface{}) bool { // //You may only register *one* BeforeSuite handler per test suite. You typically do so in your bootstrap file at the top level. func BeforeSuite(body interface{}, timeout ...float64) bool { - globalSuite.SetBeforeSuiteNode(body, codelocation.New(1), parseTimeout(timeout...)) + global.Suite.SetBeforeSuiteNode(body, codelocation.New(1), parseTimeout(timeout...)) return true } @@ -494,7 +487,7 @@ func BeforeSuite(body interface{}, timeout ...float64) bool { // //You may only register *one* AfterSuite handler per test suite. You typically do so in your bootstrap file at the top level. func AfterSuite(body interface{}, timeout ...float64) bool { - globalSuite.SetAfterSuiteNode(body, codelocation.New(1), parseTimeout(timeout...)) + global.Suite.SetAfterSuiteNode(body, codelocation.New(1), parseTimeout(timeout...)) return true } @@ -539,7 +532,7 @@ func AfterSuite(body interface{}, timeout ...float64) bool { // Ω(err).ShouldNot(HaveOccurred()) // }) func SynchronizedBeforeSuite(node1Body interface{}, allNodesBody interface{}, timeout ...float64) bool { - globalSuite.SetSynchronizedBeforeSuiteNode( + global.Suite.SetSynchronizedBeforeSuiteNode( node1Body, allNodesBody, codelocation.New(1), @@ -566,7 +559,7 @@ func SynchronizedBeforeSuite(node1Body interface{}, allNodesBody interface{}, ti // dbRunner.Stop() // }) func SynchronizedAfterSuite(allNodesBody interface{}, node1Body interface{}, timeout ...float64) bool { - globalSuite.SetSynchronizedAfterSuiteNode( + global.Suite.SetSynchronizedAfterSuiteNode( allNodesBody, node1Body, codelocation.New(1), @@ -581,7 +574,7 @@ func SynchronizedAfterSuite(allNodesBody interface{}, node1Body interface{}, tim //Like It blocks, BeforeEach blocks can be made asynchronous by providing a body function that accepts //a Done channel func BeforeEach(body interface{}, timeout ...float64) bool { - globalSuite.PushBeforeEachNode(body, codelocation.New(1), parseTimeout(timeout...)) + global.Suite.PushBeforeEachNode(body, codelocation.New(1), parseTimeout(timeout...)) return true } @@ -591,7 +584,7 @@ func BeforeEach(body interface{}, timeout ...float64) bool { //Like It blocks, BeforeEach blocks can be made asynchronous by providing a body function that accepts //a Done channel func JustBeforeEach(body interface{}, timeout ...float64) bool { - globalSuite.PushJustBeforeEachNode(body, codelocation.New(1), parseTimeout(timeout...)) + global.Suite.PushJustBeforeEachNode(body, codelocation.New(1), parseTimeout(timeout...)) return true } @@ -601,7 +594,7 @@ func JustBeforeEach(body interface{}, timeout ...float64) bool { //Like It blocks, JustAfterEach blocks can be made asynchronous by providing a body function that accepts //a Done channel func JustAfterEach(body interface{}, timeout ...float64) bool { - globalSuite.PushJustAfterEachNode(body, codelocation.New(1), parseTimeout(timeout...)) + global.Suite.PushJustAfterEachNode(body, codelocation.New(1), parseTimeout(timeout...)) return true } @@ -611,13 +604,13 @@ func JustAfterEach(body interface{}, timeout ...float64) bool { //Like It blocks, AfterEach blocks can be made asynchronous by providing a body function that accepts //a Done channel func AfterEach(body interface{}, timeout ...float64) bool { - globalSuite.PushAfterEachNode(body, codelocation.New(1), parseTimeout(timeout...)) + global.Suite.PushAfterEachNode(body, codelocation.New(1), parseTimeout(timeout...)) return true } func parseTimeout(timeout ...float64) time.Duration { if len(timeout) == 0 { - return time.Duration(defaultTimeout * int64(time.Second)) + return global.DefaultTimeout } else { return time.Duration(timeout[0] * float64(time.Second)) } diff --git a/vendor/github.com/onsi/ginkgo/internal/global/init.go b/vendor/github.com/onsi/ginkgo/internal/global/init.go new file mode 100644 index 000000000..711443200 --- /dev/null +++ b/vendor/github.com/onsi/ginkgo/internal/global/init.go @@ -0,0 +1,18 @@ +package global + +import ( + "time" + + "github.com/onsi/ginkgo/internal/failer" + "github.com/onsi/ginkgo/internal/suite" +) + +const DefaultTimeout = time.Duration(1 * time.Second) + +var Suite *suite.Suite +var Failer *failer.Failer + +func init() { + Failer = failer.New() + Suite = suite.New(Failer) +} diff --git a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/cgroups.go b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/cgroups.go index 25ff51589..c0a965923 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/cgroups.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/cgroups.go @@ -37,8 +37,18 @@ type Manager interface { // restore the object later. GetPaths() map[string]string + // GetUnifiedPath returns the unified path when running in unified mode. + // The value corresponds to the all values of GetPaths() map. + // + // GetUnifiedPath returns error when running in hybrid mode as well as + // in legacy mode. + GetUnifiedPath() (string, error) + // Sets the cgroup as configured. Set(container *configs.Config) error + + // Gets the cgroup as configured. + GetCgroups() (*configs.Cgroup, error) } type NotFoundError struct { diff --git a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils.go b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils.go index 60790f83b..dbcc58f5b 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/cgroups/utils.go @@ -20,8 +20,9 @@ import ( ) const ( - CgroupNamePrefix = "name=" - CgroupProcesses = "cgroup.procs" + CgroupNamePrefix = "name=" + CgroupProcesses = "cgroup.procs" + unifiedMountpoint = "/sys/fs/cgroup" ) var ( @@ -40,7 +41,7 @@ var HugePageSizeUnitList = []string{"B", "KB", "MB", "GB", "TB", "PB"} func IsCgroup2UnifiedMode() bool { isUnifiedOnce.Do(func() { var st syscall.Statfs_t - if err := syscall.Statfs("/sys/fs/cgroup", &st); err != nil { + if err := syscall.Statfs(unifiedMountpoint, &st); err != nil { panic("cannot statfs cgroup root") } isUnified = st.Type == unix.CGROUP2_SUPER_MAGIC @@ -50,6 +51,9 @@ func IsCgroup2UnifiedMode() bool { // https://www.kernel.org/doc/Documentation/cgroup-v1/cgroups.txt func FindCgroupMountpoint(cgroupPath, subsystem string) (string, error) { + if IsCgroup2UnifiedMode() { + return unifiedMountpoint, nil + } mnt, _, err := FindCgroupMountpointAndRoot(cgroupPath, subsystem) return mnt, err } @@ -235,8 +239,8 @@ func GetCgroupMounts(all bool) ([]Mount, error) { return nil, err } m := Mount{ - Mountpoint: "/sys/fs/cgroup", - Root: "/sys/fs/cgroup", + Mountpoint: unifiedMountpoint, + Root: unifiedMountpoint, Subsystems: availableControllers, } return []Mount{m}, nil @@ -262,6 +266,21 @@ func GetCgroupMounts(all bool) ([]Mount, error) { // GetAllSubsystems returns all the cgroup subsystems supported by the kernel func GetAllSubsystems() ([]string, error) { + // /proc/cgroups is meaningless for v2 + // https://github.com/torvalds/linux/blob/v5.3/Documentation/admin-guide/cgroup-v2.rst#deprecated-v1-core-features + if IsCgroup2UnifiedMode() { + // "pseudo" controllers do not appear in /sys/fs/cgroup/cgroup.controllers. + // - devices: implemented in kernel 4.15 + // - freezer: implemented in kernel 5.2 + // We assume these are always available, as it is hard to detect availability. + pseudo := []string{"devices", "freezer"} + data, err := ioutil.ReadFile("/sys/fs/cgroup/cgroup.controllers") + if err != nil { + return nil, err + } + subsystems := append(pseudo, strings.Fields(string(data))...) + return subsystems, nil + } f, err := os.Open("/proc/cgroups") if err != nil { return nil, err diff --git a/vendor/golang.org/x/crypto/ssh/agent/client.go b/vendor/golang.org/x/crypto/ssh/agent/client.go new file mode 100644 index 000000000..b909471cc --- /dev/null +++ b/vendor/golang.org/x/crypto/ssh/agent/client.go @@ -0,0 +1,813 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package agent implements the ssh-agent protocol, and provides both +// a client and a server. The client can talk to a standard ssh-agent +// that uses UNIX sockets, and one could implement an alternative +// ssh-agent process using the sample server. +// +// References: +// [PROTOCOL.agent]: https://tools.ietf.org/html/draft-miller-ssh-agent-00 +package agent // import "golang.org/x/crypto/ssh/agent" + +import ( + "bytes" + "crypto/dsa" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rsa" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "io" + "math/big" + "sync" + + "crypto" + "golang.org/x/crypto/ed25519" + "golang.org/x/crypto/ssh" +) + +// SignatureFlags represent additional flags that can be passed to the signature +// requests an defined in [PROTOCOL.agent] section 4.5.1. +type SignatureFlags uint32 + +// SignatureFlag values as defined in [PROTOCOL.agent] section 5.3. +const ( + SignatureFlagReserved SignatureFlags = 1 << iota + SignatureFlagRsaSha256 + SignatureFlagRsaSha512 +) + +// Agent represents the capabilities of an ssh-agent. +type Agent interface { + // List returns the identities known to the agent. + List() ([]*Key, error) + + // Sign has the agent sign the data using a protocol 2 key as defined + // in [PROTOCOL.agent] section 2.6.2. + Sign(key ssh.PublicKey, data []byte) (*ssh.Signature, error) + + // Add adds a private key to the agent. + Add(key AddedKey) error + + // Remove removes all identities with the given public key. + Remove(key ssh.PublicKey) error + + // RemoveAll removes all identities. + RemoveAll() error + + // Lock locks the agent. Sign and Remove will fail, and List will empty an empty list. + Lock(passphrase []byte) error + + // Unlock undoes the effect of Lock + Unlock(passphrase []byte) error + + // Signers returns signers for all the known keys. + Signers() ([]ssh.Signer, error) +} + +type ExtendedAgent interface { + Agent + + // SignWithFlags signs like Sign, but allows for additional flags to be sent/received + SignWithFlags(key ssh.PublicKey, data []byte, flags SignatureFlags) (*ssh.Signature, error) + + // Extension processes a custom extension request. Standard-compliant agents are not + // required to support any extensions, but this method allows agents to implement + // vendor-specific methods or add experimental features. See [PROTOCOL.agent] section 4.7. + // If agent extensions are unsupported entirely this method MUST return an + // ErrExtensionUnsupported error. Similarly, if just the specific extensionType in + // the request is unsupported by the agent then ErrExtensionUnsupported MUST be + // returned. + // + // In the case of success, since [PROTOCOL.agent] section 4.7 specifies that the contents + // of the response are unspecified (including the type of the message), the complete + // response will be returned as a []byte slice, including the "type" byte of the message. + Extension(extensionType string, contents []byte) ([]byte, error) +} + +// ConstraintExtension describes an optional constraint defined by users. +type ConstraintExtension struct { + // ExtensionName consist of a UTF-8 string suffixed by the + // implementation domain following the naming scheme defined + // in Section 4.2 of [RFC4251], e.g. "foo@example.com". + ExtensionName string + // ExtensionDetails contains the actual content of the extended + // constraint. + ExtensionDetails []byte +} + +// AddedKey describes an SSH key to be added to an Agent. +type AddedKey struct { + // PrivateKey must be a *rsa.PrivateKey, *dsa.PrivateKey, + // ed25519.PrivateKey or *ecdsa.PrivateKey, which will be inserted into the + // agent. + PrivateKey interface{} + // Certificate, if not nil, is communicated to the agent and will be + // stored with the key. + Certificate *ssh.Certificate + // Comment is an optional, free-form string. + Comment string + // LifetimeSecs, if not zero, is the number of seconds that the + // agent will store the key for. + LifetimeSecs uint32 + // ConfirmBeforeUse, if true, requests that the agent confirm with the + // user before each use of this key. + ConfirmBeforeUse bool + // ConstraintExtensions are the experimental or private-use constraints + // defined by users. + ConstraintExtensions []ConstraintExtension +} + +// See [PROTOCOL.agent], section 3. +const ( + agentRequestV1Identities = 1 + agentRemoveAllV1Identities = 9 + + // 3.2 Requests from client to agent for protocol 2 key operations + agentAddIdentity = 17 + agentRemoveIdentity = 18 + agentRemoveAllIdentities = 19 + agentAddIDConstrained = 25 + + // 3.3 Key-type independent requests from client to agent + agentAddSmartcardKey = 20 + agentRemoveSmartcardKey = 21 + agentLock = 22 + agentUnlock = 23 + agentAddSmartcardKeyConstrained = 26 + + // 3.7 Key constraint identifiers + agentConstrainLifetime = 1 + agentConstrainConfirm = 2 + agentConstrainExtension = 3 +) + +// maxAgentResponseBytes is the maximum agent reply size that is accepted. This +// is a sanity check, not a limit in the spec. +const maxAgentResponseBytes = 16 << 20 + +// Agent messages: +// These structures mirror the wire format of the corresponding ssh agent +// messages found in [PROTOCOL.agent]. + +// 3.4 Generic replies from agent to client +const agentFailure = 5 + +type failureAgentMsg struct{} + +const agentSuccess = 6 + +type successAgentMsg struct{} + +// See [PROTOCOL.agent], section 2.5.2. +const agentRequestIdentities = 11 + +type requestIdentitiesAgentMsg struct{} + +// See [PROTOCOL.agent], section 2.5.2. +const agentIdentitiesAnswer = 12 + +type identitiesAnswerAgentMsg struct { + NumKeys uint32 `sshtype:"12"` + Keys []byte `ssh:"rest"` +} + +// See [PROTOCOL.agent], section 2.6.2. +const agentSignRequest = 13 + +type signRequestAgentMsg struct { + KeyBlob []byte `sshtype:"13"` + Data []byte + Flags uint32 +} + +// See [PROTOCOL.agent], section 2.6.2. + +// 3.6 Replies from agent to client for protocol 2 key operations +const agentSignResponse = 14 + +type signResponseAgentMsg struct { + SigBlob []byte `sshtype:"14"` +} + +type publicKey struct { + Format string + Rest []byte `ssh:"rest"` +} + +// 3.7 Key constraint identifiers +type constrainLifetimeAgentMsg struct { + LifetimeSecs uint32 `sshtype:"1"` +} + +type constrainExtensionAgentMsg struct { + ExtensionName string `sshtype:"3"` + ExtensionDetails []byte + + // Rest is a field used for parsing, not part of message + Rest []byte `ssh:"rest"` +} + +// See [PROTOCOL.agent], section 4.7 +const agentExtension = 27 +const agentExtensionFailure = 28 + +// ErrExtensionUnsupported indicates that an extension defined in +// [PROTOCOL.agent] section 4.7 is unsupported by the agent. Specifically this +// error indicates that the agent returned a standard SSH_AGENT_FAILURE message +// as the result of a SSH_AGENTC_EXTENSION request. Note that the protocol +// specification (and therefore this error) does not distinguish between a +// specific extension being unsupported and extensions being unsupported entirely. +var ErrExtensionUnsupported = errors.New("agent: extension unsupported") + +type extensionAgentMsg struct { + ExtensionType string `sshtype:"27"` + Contents []byte +} + +// Key represents a protocol 2 public key as defined in +// [PROTOCOL.agent], section 2.5.2. +type Key struct { + Format string + Blob []byte + Comment string +} + +func clientErr(err error) error { + return fmt.Errorf("agent: client error: %v", err) +} + +// String returns the storage form of an agent key with the format, base64 +// encoded serialized key, and the comment if it is not empty. +func (k *Key) String() string { + s := string(k.Format) + " " + base64.StdEncoding.EncodeToString(k.Blob) + + if k.Comment != "" { + s += " " + k.Comment + } + + return s +} + +// Type returns the public key type. +func (k *Key) Type() string { + return k.Format +} + +// Marshal returns key blob to satisfy the ssh.PublicKey interface. +func (k *Key) Marshal() []byte { + return k.Blob +} + +// Verify satisfies the ssh.PublicKey interface. +func (k *Key) Verify(data []byte, sig *ssh.Signature) error { + pubKey, err := ssh.ParsePublicKey(k.Blob) + if err != nil { + return fmt.Errorf("agent: bad public key: %v", err) + } + return pubKey.Verify(data, sig) +} + +type wireKey struct { + Format string + Rest []byte `ssh:"rest"` +} + +func parseKey(in []byte) (out *Key, rest []byte, err error) { + var record struct { + Blob []byte + Comment string + Rest []byte `ssh:"rest"` + } + + if err := ssh.Unmarshal(in, &record); err != nil { + return nil, nil, err + } + + var wk wireKey + if err := ssh.Unmarshal(record.Blob, &wk); err != nil { + return nil, nil, err + } + + return &Key{ + Format: wk.Format, + Blob: record.Blob, + Comment: record.Comment, + }, record.Rest, nil +} + +// client is a client for an ssh-agent process. +type client struct { + // conn is typically a *net.UnixConn + conn io.ReadWriter + // mu is used to prevent concurrent access to the agent + mu sync.Mutex +} + +// NewClient returns an Agent that talks to an ssh-agent process over +// the given connection. +func NewClient(rw io.ReadWriter) ExtendedAgent { + return &client{conn: rw} +} + +// call sends an RPC to the agent. On success, the reply is +// unmarshaled into reply and replyType is set to the first byte of +// the reply, which contains the type of the message. +func (c *client) call(req []byte) (reply interface{}, err error) { + buf, err := c.callRaw(req) + if err != nil { + return nil, err + } + reply, err = unmarshal(buf) + if err != nil { + return nil, clientErr(err) + } + return reply, nil +} + +// callRaw sends an RPC to the agent. On success, the raw +// bytes of the response are returned; no unmarshalling is +// performed on the response. +func (c *client) callRaw(req []byte) (reply []byte, err error) { + c.mu.Lock() + defer c.mu.Unlock() + + msg := make([]byte, 4+len(req)) + binary.BigEndian.PutUint32(msg, uint32(len(req))) + copy(msg[4:], req) + if _, err = c.conn.Write(msg); err != nil { + return nil, clientErr(err) + } + + var respSizeBuf [4]byte + if _, err = io.ReadFull(c.conn, respSizeBuf[:]); err != nil { + return nil, clientErr(err) + } + respSize := binary.BigEndian.Uint32(respSizeBuf[:]) + if respSize > maxAgentResponseBytes { + return nil, clientErr(errors.New("response too large")) + } + + buf := make([]byte, respSize) + if _, err = io.ReadFull(c.conn, buf); err != nil { + return nil, clientErr(err) + } + return buf, nil +} + +func (c *client) simpleCall(req []byte) error { + resp, err := c.call(req) + if err != nil { + return err + } + if _, ok := resp.(*successAgentMsg); ok { + return nil + } + return errors.New("agent: failure") +} + +func (c *client) RemoveAll() error { + return c.simpleCall([]byte{agentRemoveAllIdentities}) +} + +func (c *client) Remove(key ssh.PublicKey) error { + req := ssh.Marshal(&agentRemoveIdentityMsg{ + KeyBlob: key.Marshal(), + }) + return c.simpleCall(req) +} + +func (c *client) Lock(passphrase []byte) error { + req := ssh.Marshal(&agentLockMsg{ + Passphrase: passphrase, + }) + return c.simpleCall(req) +} + +func (c *client) Unlock(passphrase []byte) error { + req := ssh.Marshal(&agentUnlockMsg{ + Passphrase: passphrase, + }) + return c.simpleCall(req) +} + +// List returns the identities known to the agent. +func (c *client) List() ([]*Key, error) { + // see [PROTOCOL.agent] section 2.5.2. + req := []byte{agentRequestIdentities} + + msg, err := c.call(req) + if err != nil { + return nil, err + } + + switch msg := msg.(type) { + case *identitiesAnswerAgentMsg: + if msg.NumKeys > maxAgentResponseBytes/8 { + return nil, errors.New("agent: too many keys in agent reply") + } + keys := make([]*Key, msg.NumKeys) + data := msg.Keys + for i := uint32(0); i < msg.NumKeys; i++ { + var key *Key + var err error + if key, data, err = parseKey(data); err != nil { + return nil, err + } + keys[i] = key + } + return keys, nil + case *failureAgentMsg: + return nil, errors.New("agent: failed to list keys") + } + panic("unreachable") +} + +// Sign has the agent sign the data using a protocol 2 key as defined +// in [PROTOCOL.agent] section 2.6.2. +func (c *client) Sign(key ssh.PublicKey, data []byte) (*ssh.Signature, error) { + return c.SignWithFlags(key, data, 0) +} + +func (c *client) SignWithFlags(key ssh.PublicKey, data []byte, flags SignatureFlags) (*ssh.Signature, error) { + req := ssh.Marshal(signRequestAgentMsg{ + KeyBlob: key.Marshal(), + Data: data, + Flags: uint32(flags), + }) + + msg, err := c.call(req) + if err != nil { + return nil, err + } + + switch msg := msg.(type) { + case *signResponseAgentMsg: + var sig ssh.Signature + if err := ssh.Unmarshal(msg.SigBlob, &sig); err != nil { + return nil, err + } + + return &sig, nil + case *failureAgentMsg: + return nil, errors.New("agent: failed to sign challenge") + } + panic("unreachable") +} + +// unmarshal parses an agent message in packet, returning the parsed +// form and the message type of packet. +func unmarshal(packet []byte) (interface{}, error) { + if len(packet) < 1 { + return nil, errors.New("agent: empty packet") + } + var msg interface{} + switch packet[0] { + case agentFailure: + return new(failureAgentMsg), nil + case agentSuccess: + return new(successAgentMsg), nil + case agentIdentitiesAnswer: + msg = new(identitiesAnswerAgentMsg) + case agentSignResponse: + msg = new(signResponseAgentMsg) + case agentV1IdentitiesAnswer: + msg = new(agentV1IdentityMsg) + default: + return nil, fmt.Errorf("agent: unknown type tag %d", packet[0]) + } + if err := ssh.Unmarshal(packet, msg); err != nil { + return nil, err + } + return msg, nil +} + +type rsaKeyMsg struct { + Type string `sshtype:"17|25"` + N *big.Int + E *big.Int + D *big.Int + Iqmp *big.Int // IQMP = Inverse Q Mod P + P *big.Int + Q *big.Int + Comments string + Constraints []byte `ssh:"rest"` +} + +type dsaKeyMsg struct { + Type string `sshtype:"17|25"` + P *big.Int + Q *big.Int + G *big.Int + Y *big.Int + X *big.Int + Comments string + Constraints []byte `ssh:"rest"` +} + +type ecdsaKeyMsg struct { + Type string `sshtype:"17|25"` + Curve string + KeyBytes []byte + D *big.Int + Comments string + Constraints []byte `ssh:"rest"` +} + +type ed25519KeyMsg struct { + Type string `sshtype:"17|25"` + Pub []byte + Priv []byte + Comments string + Constraints []byte `ssh:"rest"` +} + +// Insert adds a private key to the agent. +func (c *client) insertKey(s interface{}, comment string, constraints []byte) error { + var req []byte + switch k := s.(type) { + case *rsa.PrivateKey: + if len(k.Primes) != 2 { + return fmt.Errorf("agent: unsupported RSA key with %d primes", len(k.Primes)) + } + k.Precompute() + req = ssh.Marshal(rsaKeyMsg{ + Type: ssh.KeyAlgoRSA, + N: k.N, + E: big.NewInt(int64(k.E)), + D: k.D, + Iqmp: k.Precomputed.Qinv, + P: k.Primes[0], + Q: k.Primes[1], + Comments: comment, + Constraints: constraints, + }) + case *dsa.PrivateKey: + req = ssh.Marshal(dsaKeyMsg{ + Type: ssh.KeyAlgoDSA, + P: k.P, + Q: k.Q, + G: k.G, + Y: k.Y, + X: k.X, + Comments: comment, + Constraints: constraints, + }) + case *ecdsa.PrivateKey: + nistID := fmt.Sprintf("nistp%d", k.Params().BitSize) + req = ssh.Marshal(ecdsaKeyMsg{ + Type: "ecdsa-sha2-" + nistID, + Curve: nistID, + KeyBytes: elliptic.Marshal(k.Curve, k.X, k.Y), + D: k.D, + Comments: comment, + Constraints: constraints, + }) + case ed25519.PrivateKey: + req = ssh.Marshal(ed25519KeyMsg{ + Type: ssh.KeyAlgoED25519, + Pub: []byte(k)[32:], + Priv: []byte(k), + Comments: comment, + Constraints: constraints, + }) + // This function originally supported only *ed25519.PrivateKey, however the + // general idiom is to pass ed25519.PrivateKey by value, not by pointer. + // We still support the pointer variant for backwards compatibility. + case *ed25519.PrivateKey: + req = ssh.Marshal(ed25519KeyMsg{ + Type: ssh.KeyAlgoED25519, + Pub: []byte(*k)[32:], + Priv: []byte(*k), + Comments: comment, + Constraints: constraints, + }) + default: + return fmt.Errorf("agent: unsupported key type %T", s) + } + + // if constraints are present then the message type needs to be changed. + if len(constraints) != 0 { + req[0] = agentAddIDConstrained + } + + resp, err := c.call(req) + if err != nil { + return err + } + if _, ok := resp.(*successAgentMsg); ok { + return nil + } + return errors.New("agent: failure") +} + +type rsaCertMsg struct { + Type string `sshtype:"17|25"` + CertBytes []byte + D *big.Int + Iqmp *big.Int // IQMP = Inverse Q Mod P + P *big.Int + Q *big.Int + Comments string + Constraints []byte `ssh:"rest"` +} + +type dsaCertMsg struct { + Type string `sshtype:"17|25"` + CertBytes []byte + X *big.Int + Comments string + Constraints []byte `ssh:"rest"` +} + +type ecdsaCertMsg struct { + Type string `sshtype:"17|25"` + CertBytes []byte + D *big.Int + Comments string + Constraints []byte `ssh:"rest"` +} + +type ed25519CertMsg struct { + Type string `sshtype:"17|25"` + CertBytes []byte + Pub []byte + Priv []byte + Comments string + Constraints []byte `ssh:"rest"` +} + +// Add adds a private key to the agent. If a certificate is given, +// that certificate is added instead as public key. +func (c *client) Add(key AddedKey) error { + var constraints []byte + + if secs := key.LifetimeSecs; secs != 0 { + constraints = append(constraints, ssh.Marshal(constrainLifetimeAgentMsg{secs})...) + } + + if key.ConfirmBeforeUse { + constraints = append(constraints, agentConstrainConfirm) + } + + cert := key.Certificate + if cert == nil { + return c.insertKey(key.PrivateKey, key.Comment, constraints) + } + return c.insertCert(key.PrivateKey, cert, key.Comment, constraints) +} + +func (c *client) insertCert(s interface{}, cert *ssh.Certificate, comment string, constraints []byte) error { + var req []byte + switch k := s.(type) { + case *rsa.PrivateKey: + if len(k.Primes) != 2 { + return fmt.Errorf("agent: unsupported RSA key with %d primes", len(k.Primes)) + } + k.Precompute() + req = ssh.Marshal(rsaCertMsg{ + Type: cert.Type(), + CertBytes: cert.Marshal(), + D: k.D, + Iqmp: k.Precomputed.Qinv, + P: k.Primes[0], + Q: k.Primes[1], + Comments: comment, + Constraints: constraints, + }) + case *dsa.PrivateKey: + req = ssh.Marshal(dsaCertMsg{ + Type: cert.Type(), + CertBytes: cert.Marshal(), + X: k.X, + Comments: comment, + Constraints: constraints, + }) + case *ecdsa.PrivateKey: + req = ssh.Marshal(ecdsaCertMsg{ + Type: cert.Type(), + CertBytes: cert.Marshal(), + D: k.D, + Comments: comment, + Constraints: constraints, + }) + case ed25519.PrivateKey: + req = ssh.Marshal(ed25519CertMsg{ + Type: cert.Type(), + CertBytes: cert.Marshal(), + Pub: []byte(k)[32:], + Priv: []byte(k), + Comments: comment, + Constraints: constraints, + }) + // This function originally supported only *ed25519.PrivateKey, however the + // general idiom is to pass ed25519.PrivateKey by value, not by pointer. + // We still support the pointer variant for backwards compatibility. + case *ed25519.PrivateKey: + req = ssh.Marshal(ed25519CertMsg{ + Type: cert.Type(), + CertBytes: cert.Marshal(), + Pub: []byte(*k)[32:], + Priv: []byte(*k), + Comments: comment, + Constraints: constraints, + }) + default: + return fmt.Errorf("agent: unsupported key type %T", s) + } + + // if constraints are present then the message type needs to be changed. + if len(constraints) != 0 { + req[0] = agentAddIDConstrained + } + + signer, err := ssh.NewSignerFromKey(s) + if err != nil { + return err + } + if bytes.Compare(cert.Key.Marshal(), signer.PublicKey().Marshal()) != 0 { + return errors.New("agent: signer and cert have different public key") + } + + resp, err := c.call(req) + if err != nil { + return err + } + if _, ok := resp.(*successAgentMsg); ok { + return nil + } + return errors.New("agent: failure") +} + +// Signers provides a callback for client authentication. +func (c *client) Signers() ([]ssh.Signer, error) { + keys, err := c.List() + if err != nil { + return nil, err + } + + var result []ssh.Signer + for _, k := range keys { + result = append(result, &agentKeyringSigner{c, k}) + } + return result, nil +} + +type agentKeyringSigner struct { + agent *client + pub ssh.PublicKey +} + +func (s *agentKeyringSigner) PublicKey() ssh.PublicKey { + return s.pub +} + +func (s *agentKeyringSigner) Sign(rand io.Reader, data []byte) (*ssh.Signature, error) { + // The agent has its own entropy source, so the rand argument is ignored. + return s.agent.Sign(s.pub, data) +} + +func (s *agentKeyringSigner) SignWithOpts(rand io.Reader, data []byte, opts crypto.SignerOpts) (*ssh.Signature, error) { + var flags SignatureFlags + if opts != nil { + switch opts.HashFunc() { + case crypto.SHA256: + flags = SignatureFlagRsaSha256 + case crypto.SHA512: + flags = SignatureFlagRsaSha512 + } + } + return s.agent.SignWithFlags(s.pub, data, flags) +} + +// Calls an extension method. It is up to the agent implementation as to whether or not +// any particular extension is supported and may always return an error. Because the +// type of the response is up to the implementation, this returns the bytes of the +// response and does not attempt any type of unmarshalling. +func (c *client) Extension(extensionType string, contents []byte) ([]byte, error) { + req := ssh.Marshal(extensionAgentMsg{ + ExtensionType: extensionType, + Contents: contents, + }) + buf, err := c.callRaw(req) + if err != nil { + return nil, err + } + if len(buf) == 0 { + return nil, errors.New("agent: failure; empty response") + } + // [PROTOCOL.agent] section 4.7 indicates that an SSH_AGENT_FAILURE message + // represents an agent that does not support the extension + if buf[0] == agentFailure { + return nil, ErrExtensionUnsupported + } + if buf[0] == agentExtensionFailure { + return nil, errors.New("agent: generic extension failure") + } + + return buf, nil +} diff --git a/vendor/golang.org/x/crypto/ssh/agent/forward.go b/vendor/golang.org/x/crypto/ssh/agent/forward.go new file mode 100644 index 000000000..fd24ba900 --- /dev/null +++ b/vendor/golang.org/x/crypto/ssh/agent/forward.go @@ -0,0 +1,103 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package agent + +import ( + "errors" + "io" + "net" + "sync" + + "golang.org/x/crypto/ssh" +) + +// RequestAgentForwarding sets up agent forwarding for the session. +// ForwardToAgent or ForwardToRemote should be called to route +// the authentication requests. +func RequestAgentForwarding(session *ssh.Session) error { + ok, err := session.SendRequest("auth-agent-req@openssh.com", true, nil) + if err != nil { + return err + } + if !ok { + return errors.New("forwarding request denied") + } + return nil +} + +// ForwardToAgent routes authentication requests to the given keyring. +func ForwardToAgent(client *ssh.Client, keyring Agent) error { + channels := client.HandleChannelOpen(channelType) + if channels == nil { + return errors.New("agent: already have handler for " + channelType) + } + + go func() { + for ch := range channels { + channel, reqs, err := ch.Accept() + if err != nil { + continue + } + go ssh.DiscardRequests(reqs) + go func() { + ServeAgent(keyring, channel) + channel.Close() + }() + } + }() + return nil +} + +const channelType = "auth-agent@openssh.com" + +// ForwardToRemote routes authentication requests to the ssh-agent +// process serving on the given unix socket. +func ForwardToRemote(client *ssh.Client, addr string) error { + channels := client.HandleChannelOpen(channelType) + if channels == nil { + return errors.New("agent: already have handler for " + channelType) + } + conn, err := net.Dial("unix", addr) + if err != nil { + return err + } + conn.Close() + + go func() { + for ch := range channels { + channel, reqs, err := ch.Accept() + if err != nil { + continue + } + go ssh.DiscardRequests(reqs) + go forwardUnixSocket(channel, addr) + } + }() + return nil +} + +func forwardUnixSocket(channel ssh.Channel, addr string) { + conn, err := net.Dial("unix", addr) + if err != nil { + return + } + + var wg sync.WaitGroup + wg.Add(2) + go func() { + io.Copy(conn, channel) + conn.(*net.UnixConn).CloseWrite() + wg.Done() + }() + go func() { + io.Copy(channel, conn) + channel.CloseWrite() + wg.Done() + }() + + wg.Wait() + conn.Close() + channel.Close() +} diff --git a/vendor/golang.org/x/crypto/ssh/agent/keyring.go b/vendor/golang.org/x/crypto/ssh/agent/keyring.go new file mode 100644 index 000000000..c9d979430 --- /dev/null +++ b/vendor/golang.org/x/crypto/ssh/agent/keyring.go @@ -0,0 +1,241 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package agent + +import ( + "bytes" + "crypto/rand" + "crypto/subtle" + "errors" + "fmt" + "sync" + "time" + + "golang.org/x/crypto/ssh" +) + +type privKey struct { + signer ssh.Signer + comment string + expire *time.Time +} + +type keyring struct { + mu sync.Mutex + keys []privKey + + locked bool + passphrase []byte +} + +var errLocked = errors.New("agent: locked") + +// NewKeyring returns an Agent that holds keys in memory. It is safe +// for concurrent use by multiple goroutines. +func NewKeyring() Agent { + return &keyring{} +} + +// RemoveAll removes all identities. +func (r *keyring) RemoveAll() error { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + return errLocked + } + + r.keys = nil + return nil +} + +// removeLocked does the actual key removal. The caller must already be holding the +// keyring mutex. +func (r *keyring) removeLocked(want []byte) error { + found := false + for i := 0; i < len(r.keys); { + if bytes.Equal(r.keys[i].signer.PublicKey().Marshal(), want) { + found = true + r.keys[i] = r.keys[len(r.keys)-1] + r.keys = r.keys[:len(r.keys)-1] + continue + } else { + i++ + } + } + + if !found { + return errors.New("agent: key not found") + } + return nil +} + +// Remove removes all identities with the given public key. +func (r *keyring) Remove(key ssh.PublicKey) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + return errLocked + } + + return r.removeLocked(key.Marshal()) +} + +// Lock locks the agent. Sign and Remove will fail, and List will return an empty list. +func (r *keyring) Lock(passphrase []byte) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + return errLocked + } + + r.locked = true + r.passphrase = passphrase + return nil +} + +// Unlock undoes the effect of Lock +func (r *keyring) Unlock(passphrase []byte) error { + r.mu.Lock() + defer r.mu.Unlock() + if !r.locked { + return errors.New("agent: not locked") + } + if 1 != subtle.ConstantTimeCompare(passphrase, r.passphrase) { + return fmt.Errorf("agent: incorrect passphrase") + } + + r.locked = false + r.passphrase = nil + return nil +} + +// expireKeysLocked removes expired keys from the keyring. If a key was added +// with a lifetimesecs contraint and seconds >= lifetimesecs seconds have +// ellapsed, it is removed. The caller *must* be holding the keyring mutex. +func (r *keyring) expireKeysLocked() { + for _, k := range r.keys { + if k.expire != nil && time.Now().After(*k.expire) { + r.removeLocked(k.signer.PublicKey().Marshal()) + } + } +} + +// List returns the identities known to the agent. +func (r *keyring) List() ([]*Key, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + // section 2.7: locked agents return empty. + return nil, nil + } + + r.expireKeysLocked() + var ids []*Key + for _, k := range r.keys { + pub := k.signer.PublicKey() + ids = append(ids, &Key{ + Format: pub.Type(), + Blob: pub.Marshal(), + Comment: k.comment}) + } + return ids, nil +} + +// Insert adds a private key to the keyring. If a certificate +// is given, that certificate is added as public key. Note that +// any constraints given are ignored. +func (r *keyring) Add(key AddedKey) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + return errLocked + } + signer, err := ssh.NewSignerFromKey(key.PrivateKey) + + if err != nil { + return err + } + + if cert := key.Certificate; cert != nil { + signer, err = ssh.NewCertSigner(cert, signer) + if err != nil { + return err + } + } + + p := privKey{ + signer: signer, + comment: key.Comment, + } + + if key.LifetimeSecs > 0 { + t := time.Now().Add(time.Duration(key.LifetimeSecs) * time.Second) + p.expire = &t + } + + r.keys = append(r.keys, p) + + return nil +} + +// Sign returns a signature for the data. +func (r *keyring) Sign(key ssh.PublicKey, data []byte) (*ssh.Signature, error) { + return r.SignWithFlags(key, data, 0) +} + +func (r *keyring) SignWithFlags(key ssh.PublicKey, data []byte, flags SignatureFlags) (*ssh.Signature, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + return nil, errLocked + } + + r.expireKeysLocked() + wanted := key.Marshal() + for _, k := range r.keys { + if bytes.Equal(k.signer.PublicKey().Marshal(), wanted) { + if flags == 0 { + return k.signer.Sign(rand.Reader, data) + } else { + if algorithmSigner, ok := k.signer.(ssh.AlgorithmSigner); !ok { + return nil, fmt.Errorf("agent: signature does not support non-default signature algorithm: %T", k.signer) + } else { + var algorithm string + switch flags { + case SignatureFlagRsaSha256: + algorithm = ssh.SigAlgoRSASHA2256 + case SignatureFlagRsaSha512: + algorithm = ssh.SigAlgoRSASHA2512 + default: + return nil, fmt.Errorf("agent: unsupported signature flags: %d", flags) + } + return algorithmSigner.SignWithAlgorithm(rand.Reader, data, algorithm) + } + } + } + } + return nil, errors.New("not found") +} + +// Signers returns signers for all the known keys. +func (r *keyring) Signers() ([]ssh.Signer, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + return nil, errLocked + } + + r.expireKeysLocked() + s := make([]ssh.Signer, 0, len(r.keys)) + for _, k := range r.keys { + s = append(s, k.signer) + } + return s, nil +} + +// The keyring does not support any extensions +func (r *keyring) Extension(extensionType string, contents []byte) ([]byte, error) { + return nil, ErrExtensionUnsupported +} diff --git a/vendor/golang.org/x/crypto/ssh/agent/server.go b/vendor/golang.org/x/crypto/ssh/agent/server.go new file mode 100644 index 000000000..6e7a1e02f --- /dev/null +++ b/vendor/golang.org/x/crypto/ssh/agent/server.go @@ -0,0 +1,570 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package agent + +import ( + "crypto/dsa" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rsa" + "encoding/binary" + "errors" + "fmt" + "io" + "log" + "math/big" + + "golang.org/x/crypto/ed25519" + "golang.org/x/crypto/ssh" +) + +// Server wraps an Agent and uses it to implement the agent side of +// the SSH-agent, wire protocol. +type server struct { + agent Agent +} + +func (s *server) processRequestBytes(reqData []byte) []byte { + rep, err := s.processRequest(reqData) + if err != nil { + if err != errLocked { + // TODO(hanwen): provide better logging interface? + log.Printf("agent %d: %v", reqData[0], err) + } + return []byte{agentFailure} + } + + if err == nil && rep == nil { + return []byte{agentSuccess} + } + + return ssh.Marshal(rep) +} + +func marshalKey(k *Key) []byte { + var record struct { + Blob []byte + Comment string + } + record.Blob = k.Marshal() + record.Comment = k.Comment + + return ssh.Marshal(&record) +} + +// See [PROTOCOL.agent], section 2.5.1. +const agentV1IdentitiesAnswer = 2 + +type agentV1IdentityMsg struct { + Numkeys uint32 `sshtype:"2"` +} + +type agentRemoveIdentityMsg struct { + KeyBlob []byte `sshtype:"18"` +} + +type agentLockMsg struct { + Passphrase []byte `sshtype:"22"` +} + +type agentUnlockMsg struct { + Passphrase []byte `sshtype:"23"` +} + +func (s *server) processRequest(data []byte) (interface{}, error) { + switch data[0] { + case agentRequestV1Identities: + return &agentV1IdentityMsg{0}, nil + + case agentRemoveAllV1Identities: + return nil, nil + + case agentRemoveIdentity: + var req agentRemoveIdentityMsg + if err := ssh.Unmarshal(data, &req); err != nil { + return nil, err + } + + var wk wireKey + if err := ssh.Unmarshal(req.KeyBlob, &wk); err != nil { + return nil, err + } + + return nil, s.agent.Remove(&Key{Format: wk.Format, Blob: req.KeyBlob}) + + case agentRemoveAllIdentities: + return nil, s.agent.RemoveAll() + + case agentLock: + var req agentLockMsg + if err := ssh.Unmarshal(data, &req); err != nil { + return nil, err + } + + return nil, s.agent.Lock(req.Passphrase) + + case agentUnlock: + var req agentUnlockMsg + if err := ssh.Unmarshal(data, &req); err != nil { + return nil, err + } + return nil, s.agent.Unlock(req.Passphrase) + + case agentSignRequest: + var req signRequestAgentMsg + if err := ssh.Unmarshal(data, &req); err != nil { + return nil, err + } + + var wk wireKey + if err := ssh.Unmarshal(req.KeyBlob, &wk); err != nil { + return nil, err + } + + k := &Key{ + Format: wk.Format, + Blob: req.KeyBlob, + } + + var sig *ssh.Signature + var err error + if extendedAgent, ok := s.agent.(ExtendedAgent); ok { + sig, err = extendedAgent.SignWithFlags(k, req.Data, SignatureFlags(req.Flags)) + } else { + sig, err = s.agent.Sign(k, req.Data) + } + + if err != nil { + return nil, err + } + return &signResponseAgentMsg{SigBlob: ssh.Marshal(sig)}, nil + + case agentRequestIdentities: + keys, err := s.agent.List() + if err != nil { + return nil, err + } + + rep := identitiesAnswerAgentMsg{ + NumKeys: uint32(len(keys)), + } + for _, k := range keys { + rep.Keys = append(rep.Keys, marshalKey(k)...) + } + return rep, nil + + case agentAddIDConstrained, agentAddIdentity: + return nil, s.insertIdentity(data) + + case agentExtension: + // Return a stub object where the whole contents of the response gets marshaled. + var responseStub struct { + Rest []byte `ssh:"rest"` + } + + if extendedAgent, ok := s.agent.(ExtendedAgent); !ok { + // If this agent doesn't implement extensions, [PROTOCOL.agent] section 4.7 + // requires that we return a standard SSH_AGENT_FAILURE message. + responseStub.Rest = []byte{agentFailure} + } else { + var req extensionAgentMsg + if err := ssh.Unmarshal(data, &req); err != nil { + return nil, err + } + res, err := extendedAgent.Extension(req.ExtensionType, req.Contents) + if err != nil { + // If agent extensions are unsupported, return a standard SSH_AGENT_FAILURE + // message as required by [PROTOCOL.agent] section 4.7. + if err == ErrExtensionUnsupported { + responseStub.Rest = []byte{agentFailure} + } else { + // As the result of any other error processing an extension request, + // [PROTOCOL.agent] section 4.7 requires that we return a + // SSH_AGENT_EXTENSION_FAILURE code. + responseStub.Rest = []byte{agentExtensionFailure} + } + } else { + if len(res) == 0 { + return nil, nil + } + responseStub.Rest = res + } + } + + return responseStub, nil + } + + return nil, fmt.Errorf("unknown opcode %d", data[0]) +} + +func parseConstraints(constraints []byte) (lifetimeSecs uint32, confirmBeforeUse bool, extensions []ConstraintExtension, err error) { + for len(constraints) != 0 { + switch constraints[0] { + case agentConstrainLifetime: + lifetimeSecs = binary.BigEndian.Uint32(constraints[1:5]) + constraints = constraints[5:] + case agentConstrainConfirm: + confirmBeforeUse = true + constraints = constraints[1:] + case agentConstrainExtension: + var msg constrainExtensionAgentMsg + if err = ssh.Unmarshal(constraints, &msg); err != nil { + return 0, false, nil, err + } + extensions = append(extensions, ConstraintExtension{ + ExtensionName: msg.ExtensionName, + ExtensionDetails: msg.ExtensionDetails, + }) + constraints = msg.Rest + default: + return 0, false, nil, fmt.Errorf("unknown constraint type: %d", constraints[0]) + } + } + return +} + +func setConstraints(key *AddedKey, constraintBytes []byte) error { + lifetimeSecs, confirmBeforeUse, constraintExtensions, err := parseConstraints(constraintBytes) + if err != nil { + return err + } + + key.LifetimeSecs = lifetimeSecs + key.ConfirmBeforeUse = confirmBeforeUse + key.ConstraintExtensions = constraintExtensions + return nil +} + +func parseRSAKey(req []byte) (*AddedKey, error) { + var k rsaKeyMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + if k.E.BitLen() > 30 { + return nil, errors.New("agent: RSA public exponent too large") + } + priv := &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + E: int(k.E.Int64()), + N: k.N, + }, + D: k.D, + Primes: []*big.Int{k.P, k.Q}, + } + priv.Precompute() + + addedKey := &AddedKey{PrivateKey: priv, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func parseEd25519Key(req []byte) (*AddedKey, error) { + var k ed25519KeyMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + priv := ed25519.PrivateKey(k.Priv) + + addedKey := &AddedKey{PrivateKey: &priv, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func parseDSAKey(req []byte) (*AddedKey, error) { + var k dsaKeyMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + priv := &dsa.PrivateKey{ + PublicKey: dsa.PublicKey{ + Parameters: dsa.Parameters{ + P: k.P, + Q: k.Q, + G: k.G, + }, + Y: k.Y, + }, + X: k.X, + } + + addedKey := &AddedKey{PrivateKey: priv, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func unmarshalECDSA(curveName string, keyBytes []byte, privScalar *big.Int) (priv *ecdsa.PrivateKey, err error) { + priv = &ecdsa.PrivateKey{ + D: privScalar, + } + + switch curveName { + case "nistp256": + priv.Curve = elliptic.P256() + case "nistp384": + priv.Curve = elliptic.P384() + case "nistp521": + priv.Curve = elliptic.P521() + default: + return nil, fmt.Errorf("agent: unknown curve %q", curveName) + } + + priv.X, priv.Y = elliptic.Unmarshal(priv.Curve, keyBytes) + if priv.X == nil || priv.Y == nil { + return nil, errors.New("agent: point not on curve") + } + + return priv, nil +} + +func parseEd25519Cert(req []byte) (*AddedKey, error) { + var k ed25519CertMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + pubKey, err := ssh.ParsePublicKey(k.CertBytes) + if err != nil { + return nil, err + } + priv := ed25519.PrivateKey(k.Priv) + cert, ok := pubKey.(*ssh.Certificate) + if !ok { + return nil, errors.New("agent: bad ED25519 certificate") + } + + addedKey := &AddedKey{PrivateKey: &priv, Certificate: cert, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func parseECDSAKey(req []byte) (*AddedKey, error) { + var k ecdsaKeyMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + + priv, err := unmarshalECDSA(k.Curve, k.KeyBytes, k.D) + if err != nil { + return nil, err + } + + addedKey := &AddedKey{PrivateKey: priv, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func parseRSACert(req []byte) (*AddedKey, error) { + var k rsaCertMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + + pubKey, err := ssh.ParsePublicKey(k.CertBytes) + if err != nil { + return nil, err + } + + cert, ok := pubKey.(*ssh.Certificate) + if !ok { + return nil, errors.New("agent: bad RSA certificate") + } + + // An RSA publickey as marshaled by rsaPublicKey.Marshal() in keys.go + var rsaPub struct { + Name string + E *big.Int + N *big.Int + } + if err := ssh.Unmarshal(cert.Key.Marshal(), &rsaPub); err != nil { + return nil, fmt.Errorf("agent: Unmarshal failed to parse public key: %v", err) + } + + if rsaPub.E.BitLen() > 30 { + return nil, errors.New("agent: RSA public exponent too large") + } + + priv := rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + E: int(rsaPub.E.Int64()), + N: rsaPub.N, + }, + D: k.D, + Primes: []*big.Int{k.Q, k.P}, + } + priv.Precompute() + + addedKey := &AddedKey{PrivateKey: &priv, Certificate: cert, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func parseDSACert(req []byte) (*AddedKey, error) { + var k dsaCertMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + pubKey, err := ssh.ParsePublicKey(k.CertBytes) + if err != nil { + return nil, err + } + cert, ok := pubKey.(*ssh.Certificate) + if !ok { + return nil, errors.New("agent: bad DSA certificate") + } + + // A DSA publickey as marshaled by dsaPublicKey.Marshal() in keys.go + var w struct { + Name string + P, Q, G, Y *big.Int + } + if err := ssh.Unmarshal(cert.Key.Marshal(), &w); err != nil { + return nil, fmt.Errorf("agent: Unmarshal failed to parse public key: %v", err) + } + + priv := &dsa.PrivateKey{ + PublicKey: dsa.PublicKey{ + Parameters: dsa.Parameters{ + P: w.P, + Q: w.Q, + G: w.G, + }, + Y: w.Y, + }, + X: k.X, + } + + addedKey := &AddedKey{PrivateKey: priv, Certificate: cert, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func parseECDSACert(req []byte) (*AddedKey, error) { + var k ecdsaCertMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + + pubKey, err := ssh.ParsePublicKey(k.CertBytes) + if err != nil { + return nil, err + } + cert, ok := pubKey.(*ssh.Certificate) + if !ok { + return nil, errors.New("agent: bad ECDSA certificate") + } + + // An ECDSA publickey as marshaled by ecdsaPublicKey.Marshal() in keys.go + var ecdsaPub struct { + Name string + ID string + Key []byte + } + if err := ssh.Unmarshal(cert.Key.Marshal(), &ecdsaPub); err != nil { + return nil, err + } + + priv, err := unmarshalECDSA(ecdsaPub.ID, ecdsaPub.Key, k.D) + if err != nil { + return nil, err + } + + addedKey := &AddedKey{PrivateKey: priv, Certificate: cert, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func (s *server) insertIdentity(req []byte) error { + var record struct { + Type string `sshtype:"17|25"` + Rest []byte `ssh:"rest"` + } + + if err := ssh.Unmarshal(req, &record); err != nil { + return err + } + + var addedKey *AddedKey + var err error + + switch record.Type { + case ssh.KeyAlgoRSA: + addedKey, err = parseRSAKey(req) + case ssh.KeyAlgoDSA: + addedKey, err = parseDSAKey(req) + case ssh.KeyAlgoECDSA256, ssh.KeyAlgoECDSA384, ssh.KeyAlgoECDSA521: + addedKey, err = parseECDSAKey(req) + case ssh.KeyAlgoED25519: + addedKey, err = parseEd25519Key(req) + case ssh.CertAlgoRSAv01: + addedKey, err = parseRSACert(req) + case ssh.CertAlgoDSAv01: + addedKey, err = parseDSACert(req) + case ssh.CertAlgoECDSA256v01, ssh.CertAlgoECDSA384v01, ssh.CertAlgoECDSA521v01: + addedKey, err = parseECDSACert(req) + case ssh.CertAlgoED25519v01: + addedKey, err = parseEd25519Cert(req) + default: + return fmt.Errorf("agent: not implemented: %q", record.Type) + } + + if err != nil { + return err + } + return s.agent.Add(*addedKey) +} + +// ServeAgent serves the agent protocol on the given connection. It +// returns when an I/O error occurs. +func ServeAgent(agent Agent, c io.ReadWriter) error { + s := &server{agent} + + var length [4]byte + for { + if _, err := io.ReadFull(c, length[:]); err != nil { + return err + } + l := binary.BigEndian.Uint32(length[:]) + if l == 0 { + return fmt.Errorf("agent: request size is 0") + } + if l > maxAgentResponseBytes { + // We also cap requests. + return fmt.Errorf("agent: request too large: %d", l) + } + + req := make([]byte, l) + if _, err := io.ReadFull(c, req); err != nil { + return err + } + + repData := s.processRequestBytes(req) + if len(repData) > maxAgentResponseBytes { + return fmt.Errorf("agent: reply too large: %d bytes", len(repData)) + } + + binary.BigEndian.PutUint32(length[:], uint32(len(repData))) + if _, err := c.Write(length[:]); err != nil { + return err + } + if _, err := c.Write(repData); err != nil { + return err + } + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index be041c7c5..b6498b3af 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -146,7 +146,7 @@ github.com/containers/ocicrypt/keywrap/pgp github.com/containers/ocicrypt/keywrap/pkcs7 github.com/containers/ocicrypt/spec github.com/containers/ocicrypt/utils -# github.com/containers/psgo v1.5.0 +# github.com/containers/psgo v1.5.1 github.com/containers/psgo github.com/containers/psgo/internal/capabilities github.com/containers/psgo/internal/cgroups @@ -154,7 +154,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.20.1 +# github.com/containers/storage v1.20.2 github.com/containers/storage github.com/containers/storage/drivers github.com/containers/storage/drivers/aufs @@ -197,7 +197,7 @@ github.com/containers/storage/pkg/truncindex github.com/containers/storage/pkg/unshare # github.com/coreos/go-iptables v0.4.5 github.com/coreos/go-iptables/iptables -# github.com/coreos/go-systemd/v22 v22.0.0 +# github.com/coreos/go-systemd/v22 v22.1.0 github.com/coreos/go-systemd/v22/activation github.com/coreos/go-systemd/v22/dbus github.com/coreos/go-systemd/v22/internal/dlopen @@ -324,7 +324,7 @@ github.com/inconshreveable/mousetrap github.com/ishidawataru/sctp # github.com/json-iterator/go v1.1.9 github.com/json-iterator/go -# github.com/klauspost/compress v1.10.6 +# github.com/klauspost/compress v1.10.7 github.com/klauspost/compress/flate github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 @@ -361,7 +361,7 @@ github.com/nxadm/tail/ratelimiter github.com/nxadm/tail/util github.com/nxadm/tail/watch github.com/nxadm/tail/winfile -# github.com/onsi/ginkgo v1.12.2 +# github.com/onsi/ginkgo v1.12.3 github.com/onsi/ginkgo github.com/onsi/ginkgo/config github.com/onsi/ginkgo/extensions/table @@ -375,6 +375,7 @@ github.com/onsi/ginkgo/ginkgo/watch github.com/onsi/ginkgo/internal/codelocation github.com/onsi/ginkgo/internal/containernode github.com/onsi/ginkgo/internal/failer +github.com/onsi/ginkgo/internal/global github.com/onsi/ginkgo/internal/leafnodes github.com/onsi/ginkgo/internal/remote github.com/onsi/ginkgo/internal/spec @@ -408,7 +409,7 @@ github.com/opencontainers/go-digest # github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 github.com/opencontainers/image-spec/specs-go github.com/opencontainers/image-spec/specs-go/v1 -# github.com/opencontainers/runc v1.0.0-rc9 +# github.com/opencontainers/runc v1.0.0-rc90 github.com/opencontainers/runc/libcontainer/apparmor github.com/opencontainers/runc/libcontainer/cgroups github.com/opencontainers/runc/libcontainer/configs @@ -574,6 +575,7 @@ golang.org/x/crypto/openpgp/s2k golang.org/x/crypto/pbkdf2 golang.org/x/crypto/poly1305 golang.org/x/crypto/ssh +golang.org/x/crypto/ssh/agent golang.org/x/crypto/ssh/internal/bcrypt_pbkdf golang.org/x/crypto/ssh/terminal # golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 |