diff options
65 files changed, 2530 insertions, 265 deletions
@@ -22,7 +22,7 @@ ETCDIR ?= /etc TMPFILESDIR ?= ${PREFIX}/lib/tmpfiles.d SYSTEMDDIR ?= ${PREFIX}/lib/systemd/system USERSYSTEMDDIR ?= ${PREFIX}/lib/systemd/user -REMOTETAGS ?= !ABISupport remote exclude_graphdriver_btrfs btrfs_noversion exclude_graphdriver_devicemapper containers_image_openpgp +REMOTETAGS ?= remote exclude_graphdriver_btrfs btrfs_noversion exclude_graphdriver_devicemapper containers_image_openpgp BUILDTAGS ?= \ $(shell hack/apparmor_tag.sh) \ $(shell hack/btrfs_installed_tag.sh) \ @@ -44,7 +44,7 @@ ifeq ($(shell go help mod >/dev/null 2>&1 && echo true), true) GO_BUILD=GO111MODULE=on $(GO) build -mod=vendor endif -BUILDTAGS_CROSS ?= ABISupport containers_image_openpgp exclude_graphdriver_btrfs exclude_graphdriver_devicemapper exclude_graphdriver_overlay +BUILDTAGS_CROSS ?= containers_image_openpgp exclude_graphdriver_btrfs exclude_graphdriver_devicemapper exclude_graphdriver_overlay ifneq (,$(findstring varlink,$(BUILDTAGS))) PODMAN_VARLINK_DEPENDENCIES = pkg/varlink/iopodman.go endif @@ -185,7 +185,7 @@ ifeq (,$(findstring systemd,$(BUILDTAGS))) @echo "Podman is being compiled without the systemd build tag. Install libsystemd on \ Ubuntu or systemd-devel on rpm based distro for journald support." endif - $(GO_BUILD) $(BUILDFLAGS) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "ABISupport $(BUILDTAGS)" -o $@ $(PROJECT)/cmd/podman + $(GO_BUILD) $(BUILDFLAGS) -gcflags '$(GCFLAGS)' -asmflags '$(ASMFLAGS)' -ldflags '$(LDFLAGS_PODMAN)' -tags "$(BUILDTAGS)" -o $@ $(PROJECT)/cmd/podman .PHONY: podman podman: bin/podman diff --git a/cmd/podman/containers/inspect.go b/cmd/podman/containers/inspect.go index 4549a4ef6..8556ebe83 100644 --- a/cmd/podman/containers/inspect.go +++ b/cmd/podman/containers/inspect.go @@ -26,9 +26,15 @@ func init() { Command: inspectCmd, Parent: containerCmd, }) - inspectOpts = inspect.AddInspectFlagSet(inspectCmd) + inspectOpts = new(entities.InspectOptions) + flags := inspectCmd.Flags() + flags.BoolVarP(&inspectOpts.Size, "size", "s", false, "Display total file size") + flags.StringVarP(&inspectOpts.Format, "format", "f", "json", "Format the output to a Go template or json") + flags.BoolVarP(&inspectOpts.Latest, "latest", "l", false, "Act on the latest container Podman is aware of") } func inspectExec(cmd *cobra.Command, args []string) error { + // Force container type + inspectOpts.Type = inspect.ContainerType return inspect.Inspect(args, *inspectOpts) } diff --git a/cmd/podman/containers/wait.go b/cmd/podman/containers/wait.go index 1f4d4159b..ca3883091 100644 --- a/cmd/podman/containers/wait.go +++ b/cmd/podman/containers/wait.go @@ -24,8 +24,7 @@ var ( Long: waitDescription, RunE: wait, Args: validate.IdOrLatestArgs, - Example: `podman wait --latest - podman wait --interval 5000 ctrID + Example: `podman wait --interval 5000 ctrID podman wait ctrID1 ctrID2`, } @@ -35,8 +34,7 @@ var ( Long: waitCommand.Long, RunE: waitCommand.RunE, Args: validate.IdOrLatestArgs, - Example: `podman container wait --latest - podman container wait --interval 5000 ctrID + Example: `podman container wait --interval 5000 ctrID podman container wait ctrID1 ctrID2`, } ) @@ -48,11 +46,9 @@ var ( func waitFlags(flags *pflag.FlagSet) { flags.DurationVarP(&waitOptions.Interval, "interval", "i", time.Duration(250), "Milliseconds to wait before polling for completion") - flags.BoolVarP(&waitOptions.Latest, "latest", "l", false, "Act on the latest container podman is aware of") flags.StringVar(&waitCondition, "condition", "stopped", "Condition to wait on") - if registry.IsRemote() { - // TODO: This is the same as V1. We could skip creating the flag altogether in V2... - _ = flags.MarkHidden("latest") + if !registry.IsRemote() { + flags.BoolVarP(&waitOptions.Latest, "latest", "l", false, "Act on the latest container podman is aware of") } } diff --git a/cmd/podman/images/inspect.go b/cmd/podman/images/inspect.go index 8c727eb07..f6a10ba44 100644 --- a/cmd/podman/images/inspect.go +++ b/cmd/podman/images/inspect.go @@ -27,11 +27,12 @@ func init() { Command: inspectCmd, Parent: imageCmd, }) - inspectOpts = inspect.AddInspectFlagSet(inspectCmd) + inspectOpts = new(entities.InspectOptions) flags := inspectCmd.Flags() - _ = flags.MarkHidden("latest") // Shared with container-inspect but not wanted here. + flags.StringVarP(&inspectOpts.Format, "format", "f", "json", "Format the output to a Go template or json") } func inspectExec(cmd *cobra.Command, args []string) error { + inspectOpts.Type = inspect.ImageType return inspect.Inspect(args, *inspectOpts) } diff --git a/cmd/podman/registry/config_abi.go b/cmd/podman/registry/config_abi.go index 55430e1bf..4a909c17e 100644 --- a/cmd/podman/registry/config_abi.go +++ b/cmd/podman/registry/config_abi.go @@ -1,4 +1,4 @@ -// +build ABISupport +// +build !remote package registry diff --git a/cmd/podman/registry/config_tunnel.go b/cmd/podman/registry/config_tunnel.go index 29e744dac..bb3da947e 100644 --- a/cmd/podman/registry/config_tunnel.go +++ b/cmd/podman/registry/config_tunnel.go @@ -1,4 +1,4 @@ -// +build !ABISupport +// +build remote package registry diff --git a/cmd/podman/root.go b/cmd/podman/root.go index 3796b8e27..59d536d0b 100644 --- a/cmd/podman/root.go +++ b/cmd/podman/root.go @@ -103,6 +103,11 @@ func persistentPreRunE(cmd *cobra.Command, args []string) error { cfg := registry.PodmanConfig() + // Help is a special case, no need for more setup + if cmd.Name() == "help" { + return nil + } + // Prep the engines if _, err := registry.NewImageEngine(cmd, args); err != nil { return err @@ -150,6 +155,11 @@ func persistentPostRunE(cmd *cobra.Command, args []string) error { // TODO: Remove trace statement in podman V2.1 logrus.Debugf("Called %s.PersistentPostRunE(%s)", cmd.Name(), strings.Join(os.Args, " ")) + // Help is a special case, no need for more cleanup + if cmd.Name() == "help" { + return nil + } + cfg := registry.PodmanConfig() if cmd.Flag("cpu-profile").Changed { pprof.StopCPUProfile() @@ -191,8 +201,11 @@ func loggingHook() { func rootFlags(opts *entities.PodmanConfig, flags *pflag.FlagSet) { // V2 flags - flags.StringVarP(&opts.Uri, "remote", "r", registry.DefaultAPIAddress(), "URL to access Podman service") - flags.StringSliceVar(&opts.Identities, "identity", []string{}, "path to SSH identity file") + flags.BoolVarP(&opts.Remote, "remote", "r", false, "Access remote Podman service (default false)") + // TODO Read uri from containers.config when available + flags.StringVar(&opts.Uri, "url", registry.DefaultAPIAddress(), "URL to access Podman service (CONTAINER_HOST)") + flags.StringSliceVar(&opts.Identities, "identity", []string{}, "path to SSH identity file, (CONTAINER_SSHKEY)") + flags.StringVar(&opts.PassPhrase, "passphrase", "", "passphrase for identity file (not secure, CONTAINER_PASSPHRASE), ssh-agent always supported") cfg := opts.Config flags.StringVar(&cfg.Engine.CgroupManager, "cgroup-manager", cfg.Engine.CgroupManager, "Cgroup manager to use (\"cgroupfs\"|\"systemd\")") diff --git a/cmd/podman/system/service_abi.go b/cmd/podman/system/service_abi.go index 501650839..f5386c4f1 100644 --- a/cmd/podman/system/service_abi.go +++ b/cmd/podman/system/service_abi.go @@ -1,4 +1,4 @@ -// +build ABISupport,!remote +// +build linux,!remote package system diff --git a/cmd/podman/system/service_unsupported.go b/cmd/podman/system/service_unsupported.go deleted file mode 100644 index 82272c882..000000000 --- a/cmd/podman/system/service_unsupported.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build !ABISupport,!remote - -package system - -import ( - "errors" - - "github.com/containers/libpod/pkg/domain/entities" - "github.com/spf13/pflag" -) - -func restService(opts entities.ServiceOptions, flags *pflag.FlagSet, cfg *entities.PodmanConfig) error { - return errors.New("not supported") -} diff --git a/contrib/gate/Dockerfile b/contrib/gate/Dockerfile index f86709b00..657b5accf 100644 --- a/contrib/gate/Dockerfile +++ b/contrib/gate/Dockerfile @@ -13,7 +13,7 @@ COPY . $GOSRC # Note: adding conmon and crun so podman command checks will work RUN dnf -y install \ $(grep "^[^#]" $GOSRC/contrib/dependencies.txt) diffutils containers-common fuse-overlayfs conmon crun runc --exclude container-selinux; \ - sed -i -e 's|^#mount_program|mount_program|g' /etc/containers/storage.conf \ + sed -i -e 's|^#mount_program|mount_program|g' -e 's/# size.*/skip_mount_home = "true"/g' /etc/containers/storage.conf \ && dnf clean all # Install dependencies diff --git a/contrib/spec/podman.spec.in b/contrib/spec/podman.spec.in index 9ab0db3ae..8d3cba612 100644 --- a/contrib/spec/podman.spec.in +++ b/contrib/spec/podman.spec.in @@ -80,8 +80,12 @@ Requires: iptables %if 0%{?rhel} <= 7 Requires: container-selinux %else +%if 0%{?rhel} || 0%{?centos} +Requires: runc +%else Requires: oci-runtime Recommends: crun +%endif Recommends: container-selinux Recommends: slirp4netns Recommends: fuse-overlayfs diff --git a/docs/source/markdown/podman.1.md b/docs/source/markdown/podman.1.md index 02f23e6cc..8dc486b65 100644 --- a/docs/source/markdown/podman.1.md +++ b/docs/source/markdown/podman.1.md @@ -61,6 +61,9 @@ Podman and libpod currently support an additional `precreate` state which is cal **--identity**=*path* Path to SSH identity file +**--passphrase**=*secret* +pass phrase for SSH identity file + **--log-level**=*level* Log messages above specified level: debug, info, warn, error (default), fatal or panic (default: "error") @@ -73,18 +76,21 @@ When namespace is set, created containers and pods will join the given namespace **--network-cmd-path**=*path* Path to the command binary to use for setting up a network. It is currently only used for setting up a slirp4netns network. If "" is used then the binary is looked up using the $PATH environment variable. -**--remote**, **-r**=*url* -URL to access Podman service (default "unix:/run/user/3267/podman/podman.sock") +**--remote**, **-r** +Access Podman service will be remote + +**--url**=*value* +URL to access Podman service (default from `containers.conf`, rootless "unix://run/user/$UID/podman/podman.sock" or as root "unix://run/podman/podman.sock). **--root**=*value* Storage root dir in which data, including images, is stored (default: "/var/lib/containers/storage" for UID 0, "$HOME/.local/share/containers/storage" for other users). -Default root dir is configured in `/etc/containers/storage.conf`. +Default root dir configured in `/etc/containers/storage.conf`. **--runroot**=*value* Storage state directory where all state information is stored (default: "/var/run/containers/storage" for UID 0, "/var/run/user/$UID/run" for other users). -Default state dir is configured in `/etc/containers/storage.conf`. +Default state dir configured in `/etc/containers/storage.conf`. **--runtime**=*value* @@ -15,7 +15,7 @@ require ( github.com/containers/conmon v2.0.16+incompatible github.com/containers/image/v5 v5.4.5-0.20200529084758-46b2ee6aebb0 github.com/containers/psgo v1.5.1 - github.com/containers/storage v1.20.1 + github.com/containers/storage v1.20.2 github.com/coreos/go-systemd/v22 v22.1.0 github.com/cri-o/ocicni v0.2.0 github.com/cyphar/filepath-securejoin v0.2.2 @@ -89,6 +89,8 @@ github.com/containers/storage v1.19.1/go.mod h1:KbXjSwKnx17ejOsjFcCXSf78mCgZkQSL github.com/containers/storage v1.19.2/go.mod h1:gYCp3jzgXkvubO0rI14QAjz5Mxm/qKJgLmHFyqayDnw= github.com/containers/storage v1.20.1 h1:2XE4eRIqSa6YjhAZjNwIkIKE6+Miy+5WV8l1KzY2ZKk= github.com/containers/storage v1.20.1/go.mod h1:RoKzO8KSDogCT6c06rEbanZTcKYxshorB33JikEGc3A= +github.com/containers/storage v1.20.2 h1:tw/uKRPDnmVrluIzer3dawTFG/bTJLP8IEUyHFhltYk= +github.com/containers/storage v1.20.2/go.mod h1:oOB9Ie8OVPojvoaKWEGSEtHbXUAs+tSyr7RO7ZGteMc= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5 h1:DpHb9vJrZQEFMcVLFKAAGMUVX0XoRC0ptCthinRYm38= @@ -257,6 +259,8 @@ github.com/klauspost/compress v1.10.5 h1:7q6vHIqubShURwQz8cQK6yIe/xC3IF0Vm7TGfqj github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.10.6 h1:SP6zavvTG3YjOosWePXFDlExpKIWMTO4SE/Y8MZB2vI= github.com/klauspost/compress v1.10.6/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= +github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/pgzip v1.2.3/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= @@ -560,6 +564,7 @@ golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191127021746-63cb32ae39b2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200116001909-b77594299b42/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/hack/golangci-lint.sh b/hack/golangci-lint.sh index f4e60d8f5..8c81a3743 100755 --- a/hack/golangci-lint.sh +++ b/hack/golangci-lint.sh @@ -5,12 +5,12 @@ declare -A BUILD_TAGS # TODO: add systemd tag BUILD_TAGS[default]="apparmor,seccomp,selinux" -BUILD_TAGS[abi]="${BUILD_TAGS[default]},ABISupport,varlink,!remoteclient" -BUILD_TAGS[tunnel]="${BUILD_TAGS[default]},!ABISupport,varlink,remoteclient" +BUILD_TAGS[abi]="${BUILD_TAGS[default]},varlink,!remoteclient" +BUILD_TAGS[tunnel]="${BUILD_TAGS[default]},remote,varlink,remoteclient" declare -A SKIP_DIRS SKIP_DIRS[abi]="" -# TODO: add "ABISupport" build tag to pkg/api +# TODO: add "remote" build tag to pkg/api SKIP_DIRS[tunnel]="pkg/api" [[ $1 == run ]] && shift diff --git a/pkg/bindings/bindings.go b/pkg/bindings/bindings.go index 7e2a444bd..da47ea713 100644 --- a/pkg/bindings/bindings.go +++ b/pkg/bindings/bindings.go @@ -9,7 +9,13 @@ package bindings import ( + "errors" + "fmt" + "io" + "os" + "github.com/blang/semver" + "golang.org/x/crypto/ssh/terminal" ) var ( @@ -25,3 +31,40 @@ var ( // _*YES*- podman will fail to run if this value is wrong APIVersion = semver.MustParse("1.0.0") ) + +// readPassword prompts for a secret and returns value input by user from stdin +// Unlike terminal.ReadPassword(), $(echo $SECRET | podman...) is supported. +// Additionally, all input after `<secret>/n` is queued to podman command. +func readPassword(prompt string) (pw []byte, err error) { + fd := int(os.Stdin.Fd()) + if terminal.IsTerminal(fd) { + fmt.Fprint(os.Stderr, prompt) + pw, err = terminal.ReadPassword(fd) + fmt.Fprintln(os.Stderr) + return + } + + var b [1]byte + for { + n, err := os.Stdin.Read(b[:]) + // terminal.ReadPassword discards any '\r', so we do the same + if n > 0 && b[0] != '\r' { + if b[0] == '\n' { + return pw, nil + } + pw = append(pw, b[0]) + // limit size, so that a wrong input won't fill up the memory + if len(pw) > 1024 { + err = errors.New("password too long, 1024 byte limit") + } + } + if err != nil { + // terminal.ReadPassword accepts EOF-terminated passwords + // if non-empty, so we do the same + if err == io.EOF && len(pw) > 0 { + err = nil + } + return pw, err + } + } +} diff --git a/pkg/bindings/connection.go b/pkg/bindings/connection.go index c26093a7f..b130b9598 100644 --- a/pkg/bindings/connection.go +++ b/pkg/bindings/connection.go @@ -13,6 +13,7 @@ import ( "path/filepath" "strconv" "strings" + "sync" "time" "github.com/blang/semver" @@ -20,6 +21,7 @@ import ( "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" + "golang.org/x/crypto/ssh/agent" "k8s.io/client-go/util/homedir" ) @@ -29,6 +31,8 @@ var ( Host: "d", Path: "/v" + APIVersion.String() + "/libpod", } + passPhrase []byte + phraseSync sync.Once ) type APIResponse struct { @@ -61,6 +65,10 @@ func JoinURL(elements ...string) string { return "/" + strings.Join(elements, "/") } +func NewConnection(ctx context.Context, uri string) (context.Context, error) { + return NewConnectionWithIdentity(ctx, uri, "") +} + // NewConnection takes a URI as a string and returns a context with the // Connection embedded as a value. This context needs to be passed to each // endpoint to work correctly. @@ -69,23 +77,28 @@ func JoinURL(elements ...string) string { // For example tcp://localhost:<port> // or unix:///run/podman/podman.sock // or ssh://<user>@<host>[:port]/run/podman/podman.sock?secure=True -func NewConnection(ctx context.Context, uri string, identity ...string) (context.Context, error) { +func NewConnectionWithIdentity(ctx context.Context, uri string, passPhrase string, identities ...string) (context.Context, error) { var ( err error secure bool ) - if v, found := os.LookupEnv("PODMAN_HOST"); found { + if v, found := os.LookupEnv("CONTAINER_HOST"); found && uri == "" { uri = v } - if v, found := os.LookupEnv("PODMAN_SSHKEY"); found { - identity = []string{v} + if v, found := os.LookupEnv("CONTAINER_SSHKEY"); found && len(identities) == 0 { + identities = append(identities, v) + } + + if v, found := os.LookupEnv("CONTAINER_PASSPHRASE"); found && passPhrase == "" { + passPhrase = v } _url, err := url.Parse(uri) if err != nil { - return nil, errors.Wrapf(err, "Value of PODMAN_HOST is not a valid url: %s", uri) + return nil, errors.Wrapf(err, "Value of CONTAINER_HOST is not a valid url: %s", uri) } + // TODO Fill in missing defaults for _url... // Now we setup the http Client to use the connection above var connection Connection @@ -95,7 +108,7 @@ func NewConnection(ctx context.Context, uri string, identity ...string) (context if err != nil { secure = false } - connection, err = sshClient(_url, identity[0], secure) + connection, err = sshClient(_url, secure, passPhrase, identities...) case "unix": if !strings.HasPrefix(uri, "unix:///") { // autofix unix://path_element vs unix:///path_element @@ -172,10 +185,31 @@ func pingNewConnection(ctx context.Context) error { return errors.Errorf("ping response was %q", response.StatusCode) } -func sshClient(_url *url.URL, identity string, secure bool) (Connection, error) { - auth, err := publicKey(identity) - if err != nil { - return Connection{}, errors.Wrapf(err, "Failed to parse identity %s: %v\n", _url.String(), identity) +func sshClient(_url *url.URL, secure bool, passPhrase string, identities ...string) (Connection, error) { + var authMethods []ssh.AuthMethod + + for _, i := range identities { + auth, err := publicKey(i, []byte(passPhrase)) + if err != nil { + fmt.Fprint(os.Stderr, errors.Wrapf(err, "failed to parse identity %q", i).Error()+"\n") + continue + } + authMethods = append(authMethods, auth) + } + + if sock, found := os.LookupEnv("SSH_AUTH_SOCK"); found { + logrus.Debugf("Found SSH_AUTH_SOCK %q, ssh-agent signer enabled", sock) + + c, err := net.Dial("unix", sock) + if err != nil { + return Connection{}, err + } + a := agent.NewClient(c) + authMethods = append(authMethods, ssh.PublicKeysCallback(a.Signers)) + } + + if pw, found := _url.User.Password(); found { + authMethods = append(authMethods, ssh.Password(pw)) } callback := ssh.InsecureIgnoreHostKey() @@ -195,7 +229,7 @@ func sshClient(_url *url.URL, identity string, secure bool) (Connection, error) net.JoinHostPort(_url.Hostname(), port), &ssh.ClientConfig{ User: _url.User.Username(), - Auth: []ssh.AuthMethod{auth}, + Auth: authMethods, HostKeyCallback: callback, HostKeyAlgorithms: []string{ ssh.KeyAlgoRSA, @@ -307,7 +341,7 @@ func (h *APIResponse) IsServerError() bool { return h.Response.StatusCode/100 == 5 } -func publicKey(path string) (ssh.AuthMethod, error) { +func publicKey(path string, passphrase []byte) (ssh.AuthMethod, error) { key, err := ioutil.ReadFile(path) if err != nil { return nil, err @@ -315,12 +349,30 @@ func publicKey(path string) (ssh.AuthMethod, error) { signer, err := ssh.ParsePrivateKey(key) if err != nil { - return nil, err + if _, ok := err.(*ssh.PassphraseMissingError); !ok { + return nil, err + } + if len(passphrase) == 0 { + phraseSync.Do(promptPassphrase) + passphrase = passPhrase + } + signer, err = ssh.ParsePrivateKeyWithPassphrase(key, passphrase) + if err != nil { + return nil, err + } } - return ssh.PublicKeys(signer), nil } +func promptPassphrase() { + phrase, err := readPassword("Key Passphrase: ") + if err != nil { + passPhrase = []byte{} + return + } + passPhrase = phrase +} + func hostKey(host string) ssh.PublicKey { // parse OpenSSH known_hosts file // ssh or use ssh-keyscan to get initial key diff --git a/pkg/domain/entities/engine.go b/pkg/domain/entities/engine.go index db58befa5..b2bef0eea 100644 --- a/pkg/domain/entities/engine.go +++ b/pkg/domain/entities/engine.go @@ -43,14 +43,16 @@ type PodmanConfig struct { EngineMode EngineMode // ABI or Tunneling mode Identities []string // ssh identities for connecting to server MaxWorks int // maximum number of parallel threads + PassPhrase string // ssh passphrase for identity for connecting to server RegistriesConf string // allows for specifying a custom registries.conf + Remote bool // Connection to Podman API Service will use RESTful API RuntimePath string // --runtime flag will set Engine.RuntimePath + Span opentracing.Span // tracing object SpanCloser io.Closer // Close() for tracing object SpanCtx context.Context // context to use when tracing - Span opentracing.Span // tracing object Syslog bool // write to StdOut and Syslog, not supported when tunneling Trace bool // Hidden: Trace execution - Uri string // URI to API Service + Uri string // URI to RESTful API Service Runroot string StorageDriver string diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index 706fcec47..19232eff1 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -44,8 +44,10 @@ func getContainersAndInputByContext(all, latest bool, names []string, runtime *l ctrs, err = runtime.GetAllContainers() case latest: ctr, err = runtime.GetLatestContainer() - rawInput = append(rawInput, ctr.ID()) - ctrs = append(ctrs, ctr) + if err == nil { + rawInput = append(rawInput, ctr.ID()) + ctrs = append(ctrs, ctr) + } default: for _, n := range names { ctr, e := runtime.LookupContainer(n) @@ -177,6 +179,12 @@ func (ic *ContainerEngine) ContainerStop(ctx context.Context, namesOrIds []strin report.Err = err reports = append(reports, &report) continue + } else if err := con.Cleanup(ctx); err != nil { + // Only if no error, proceed to cleanup to ensure all + // mounts are removed before we exit. + report.Err = err + reports = append(reports, &report) + continue } reports = append(reports, &report) } diff --git a/pkg/domain/infra/abi/manifest.go b/pkg/domain/infra/abi/manifest.go index 6e311dec7..a2b5fc0fc 100644 --- a/pkg/domain/infra/abi/manifest.go +++ b/pkg/domain/infra/abi/manifest.go @@ -1,4 +1,4 @@ -// +build ABISupport +// +build !remote package abi diff --git a/pkg/domain/infra/runtime_abi.go b/pkg/domain/infra/runtime_abi.go index 67c1cd534..0a82b9a6b 100644 --- a/pkg/domain/infra/runtime_abi.go +++ b/pkg/domain/infra/runtime_abi.go @@ -1,4 +1,4 @@ -// +build ABISupport +// +build !remote package infra @@ -20,7 +20,7 @@ func NewContainerEngine(facts *entities.PodmanConfig) (entities.ContainerEngine, r, err := NewLibpodRuntime(facts.FlagSet, facts) return r, err case entities.TunnelMode: - ctx, err := bindings.NewConnection(context.Background(), facts.Uri, facts.Identities...) + ctx, err := bindings.NewConnectionWithIdentity(context.Background(), facts.Uri, facts.PassPhrase, facts.Identities...) return &tunnel.ContainerEngine{ClientCxt: ctx}, err } return nil, fmt.Errorf("runtime mode '%v' is not supported", facts.EngineMode) @@ -33,7 +33,7 @@ func NewImageEngine(facts *entities.PodmanConfig) (entities.ImageEngine, error) r, err := NewLibpodImageRuntime(facts.FlagSet, facts) return r, err case entities.TunnelMode: - ctx, err := bindings.NewConnection(context.Background(), facts.Uri, facts.Identities...) + ctx, err := bindings.NewConnectionWithIdentity(context.Background(), facts.Uri, facts.PassPhrase, facts.Identities...) return &tunnel.ImageEngine{ClientCxt: ctx}, err } return nil, fmt.Errorf("runtime mode '%v' is not supported", facts.EngineMode) diff --git a/pkg/domain/infra/runtime_abi_unsupported.go b/pkg/domain/infra/runtime_abi_unsupported.go index c4e25e990..3d7d457fc 100644 --- a/pkg/domain/infra/runtime_abi_unsupported.go +++ b/pkg/domain/infra/runtime_abi_unsupported.go @@ -1,4 +1,4 @@ -// +build !ABISupport +// +build remote package infra diff --git a/pkg/domain/infra/runtime_libpod.go b/pkg/domain/infra/runtime_libpod.go index a57eadc63..2f2b0f90f 100644 --- a/pkg/domain/infra/runtime_libpod.go +++ b/pkg/domain/infra/runtime_libpod.go @@ -1,4 +1,4 @@ -// +build ABISupport +// +build !remote package infra diff --git a/pkg/domain/infra/runtime_proxy.go b/pkg/domain/infra/runtime_proxy.go index e7002e20f..fed9b1008 100644 --- a/pkg/domain/infra/runtime_proxy.go +++ b/pkg/domain/infra/runtime_proxy.go @@ -1,4 +1,4 @@ -// +build ABISupport +// +build !remote package infra diff --git a/pkg/domain/infra/runtime_tunnel.go b/pkg/domain/infra/runtime_tunnel.go index 752218aaf..bba7d2c0c 100644 --- a/pkg/domain/infra/runtime_tunnel.go +++ b/pkg/domain/infra/runtime_tunnel.go @@ -1,4 +1,4 @@ -// +build !ABISupport +// +build remote package infra @@ -16,7 +16,7 @@ func NewContainerEngine(facts *entities.PodmanConfig) (entities.ContainerEngine, case entities.ABIMode: return nil, fmt.Errorf("direct runtime not supported") case entities.TunnelMode: - ctx, err := bindings.NewConnection(context.Background(), facts.Uri, facts.Identities...) + ctx, err := bindings.NewConnectionWithIdentity(context.Background(), facts.Uri, facts.PassPhrase, facts.Identities...) return &tunnel.ContainerEngine{ClientCxt: ctx}, err } return nil, fmt.Errorf("runtime mode '%v' is not supported", facts.EngineMode) @@ -28,7 +28,7 @@ func NewImageEngine(facts *entities.PodmanConfig) (entities.ImageEngine, error) case entities.ABIMode: return nil, fmt.Errorf("direct image runtime not supported") case entities.TunnelMode: - ctx, err := bindings.NewConnection(context.Background(), facts.Uri, facts.Identities...) + ctx, err := bindings.NewConnectionWithIdentity(context.Background(), facts.Uri, facts.PassPhrase, facts.Identities...) return &tunnel.ImageEngine{ClientCxt: ctx}, err } return nil, fmt.Errorf("runtime mode '%v' is not supported", facts.EngineMode) diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go index af9d94293..97b98eec2 100644 --- a/pkg/domain/infra/tunnel/containers.go +++ b/pkg/domain/infra/tunnel/containers.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io" + "io/ioutil" "os" "strconv" "strings" @@ -162,6 +163,14 @@ func (ic *ContainerEngine) ContainerRm(ctx context.Context, namesOrIds []string, var ( reports []*entities.RmReport ) + for _, cidFile := range options.CIDFiles { + content, err := ioutil.ReadFile(cidFile) + if err != nil { + return nil, errors.Wrapf(err, "error reading CIDFile %s", cidFile) + } + id := strings.Split(string(content), "\n")[0] + namesOrIds = append(namesOrIds, id) + } ctrs, err := getContainersByContext(ic.ClientCxt, options.All, namesOrIds) if err != nil { return nil, err diff --git a/test/apiv2/20-containers.at b/test/apiv2/20-containers.at index 7fb39b221..8b535928a 100644 --- a/test/apiv2/20-containers.at +++ b/test/apiv2/20-containers.at @@ -30,4 +30,56 @@ cid=$(jq -r '.[0].Id' <<<"$output") t DELETE libpod/containers/$cid 204 +CNAME=myfoo +podman run --name $CNAME $IMAGE -td top +t GET libpod/containers/json?all=true 200 \ + .[0].Id~[0-9a-f]\\{12\\} +cid=$(jq -r '.[0].Id' <<<"$output") + +# No such container +t POST "libpod/commit?container=nonesuch" '' 404 + +# Comment can only be used with docker format, not OCI +cparam="repo=newrepo&comment=foo&author=bob" +t POST "libpod/commit?container=$CNAME&$cparam" '' 500 + +# Commit a new image from the container +t POST "libpod/commit?container=$CNAME" '' 200 \ + .Id~[0-9a-f]\\{12\\} +iid=$(jq -r '.Id' <<<"$output") +t GET libpod/images/$iid/json 200 \ + .RepoTags[0]=null \ + .Author="" \ + .Comment="" + +# Commit a new image w/o tag +cparam="repo=newrepo&comment=foo&author=bob&format=docker" +t POST "libpod/commit?container=$CNAME&$cparam" '' 200 +t GET libpod/images/newrepo:latest/json 200 \ + .RepoTags[0]=localhost/newrepo:latest \ + .Author=bob \ + .Comment=foo + +# Commit a new image w/ specified tag and author +cparam="repo=newrepo&tag=v1&author=alice" +t POST "libpod/commit?container=$cid&$cparam&pause=false" '' 200 +t GET libpod/images/newrepo:v1/json 200 \ + .RepoTags[0]=localhost/newrepo:v1 \ + .Author=alice + +# Commit a new image w/ full parameters +cparam="repo=newrepo&tag=v2&comment=bar&author=eric" +cparam="$cparam&format=docker&changes=CMD=/bin/foo" +t POST "libpod/commit?container=${cid:0:12}&$cparam&pause=true" '' 200 +t GET libpod/images/newrepo:v2/json 200 \ + .RepoTags[0]=localhost/newrepo:v2 \ + .Author=eric \ + .Comment=bar \ + .Config.Cmd[-1]="/bin/foo" + +t DELETE images/localhost/newrepo:latest?force=true 200 +t DELETE images/localhost/newrepo:v1?force=true 200 +t DELETE images/localhost/newrepo:v2?force=true 200 +t DELETE libpod/containers/$cid 204 + # vim: filetype=sh diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go index 0a6373bfa..f40472a7c 100644 --- a/test/e2e/create_test.go +++ b/test/e2e/create_test.go @@ -301,7 +301,6 @@ var _ = Describe("Podman create", func() { }) It("podman create --authfile with nonexist authfile", func() { - SkipIfRemote() session := podmanTest.PodmanNoCache([]string{"create", "--authfile", "/tmp/nonexist", "--name=foo", ALPINE}) session.WaitWithDefaultTimeout() Expect(session).To(Not(Equal(0))) diff --git a/test/e2e/diff_test.go b/test/e2e/diff_test.go index d273f9463..fbbe49eac 100644 --- a/test/e2e/diff_test.go +++ b/test/e2e/diff_test.go @@ -61,7 +61,6 @@ var _ = Describe("Podman diff", func() { }) It("podman diff container and committed image", func() { - SkipIfRemote() session := podmanTest.Podman([]string{"run", "--name=diff-test", ALPINE, "touch", "/tmp/diff-test"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -82,11 +81,10 @@ var _ = Describe("Podman diff", func() { }) It("podman diff latest container", func() { - SkipIfRemote() - session := podmanTest.Podman([]string{"run", "--name=diff-test", ALPINE, "touch", "/tmp/diff-test"}) + session := podmanTest.Podman([]string{"run", "--name", "diff-test", ALPINE, "touch", "/tmp/diff-test"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"diff", "-l"}) + session = podmanTest.Podman([]string{"diff", "diff-test"}) session.WaitWithDefaultTimeout() containerDiff := session.OutputToStringArray() sort.Strings(containerDiff) diff --git a/test/e2e/exists_test.go b/test/e2e/exists_test.go index e26fad51d..8f3b371d8 100644 --- a/test/e2e/exists_test.go +++ b/test/e2e/exists_test.go @@ -112,7 +112,6 @@ var _ = Describe("Podman image|container exists", func() { }) It("podman pod does not exist in local storage", func() { // The exit code for non-existing pod is incorrect (125 vs 1) - SkipIfRemote() session := podmanTest.Podman([]string{"pod", "exists", "foobar"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(1)) diff --git a/test/e2e/healthcheck_run_test.go b/test/e2e/healthcheck_run_test.go index 8e63d9f4c..d8e3f045f 100644 --- a/test/e2e/healthcheck_run_test.go +++ b/test/e2e/healthcheck_run_test.go @@ -42,7 +42,6 @@ var _ = Describe("Podman healthcheck run", func() { }) It("podman disable healthcheck with --no-healthcheck on valid container", func() { - SkipIfRemote() session := podmanTest.Podman([]string{"run", "-dt", "--no-healthcheck", "--name", "hc", healthcheck}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -52,7 +51,6 @@ var _ = Describe("Podman healthcheck run", func() { }) It("podman disable healthcheck with --health-cmd=none on valid container", func() { - SkipIfRemote() session := podmanTest.Podman([]string{"run", "-dt", "--health-cmd", "none", "--name", "hc", healthcheck}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) diff --git a/test/e2e/inspect_test.go b/test/e2e/inspect_test.go index 77cfe4fd3..62f69f1c1 100644 --- a/test/e2e/inspect_test.go +++ b/test/e2e/inspect_test.go @@ -132,28 +132,27 @@ var _ = Describe("Podman inspect", func() { }) It("podman inspect with mount filters", func() { - SkipIfRemote() - ctrSession := podmanTest.Podman([]string{"create", "-v", "/tmp:/test1", ALPINE, "top"}) + ctrSession := podmanTest.Podman([]string{"create", "--name", "test", "-v", "/tmp:/test1", ALPINE, "top"}) ctrSession.WaitWithDefaultTimeout() Expect(ctrSession.ExitCode()).To(Equal(0)) - inspectSource := podmanTest.Podman([]string{"inspect", "-l", "--format", "{{(index .Mounts 0).Source}}"}) + inspectSource := podmanTest.Podman([]string{"inspect", "test", "--format", "{{(index .Mounts 0).Source}}"}) inspectSource.WaitWithDefaultTimeout() Expect(inspectSource.ExitCode()).To(Equal(0)) Expect(inspectSource.OutputToString()).To(Equal("/tmp")) - inspectSrc := podmanTest.Podman([]string{"inspect", "-l", "--format", "{{(index .Mounts 0).Src}}"}) + inspectSrc := podmanTest.Podman([]string{"inspect", "test", "--format", "{{(index .Mounts 0).Src}}"}) inspectSrc.WaitWithDefaultTimeout() Expect(inspectSrc.ExitCode()).To(Equal(0)) Expect(inspectSrc.OutputToString()).To(Equal("/tmp")) - inspectDestination := podmanTest.Podman([]string{"inspect", "-l", "--format", "{{(index .Mounts 0).Destination}}"}) + inspectDestination := podmanTest.Podman([]string{"inspect", "test", "--format", "{{(index .Mounts 0).Destination}}"}) inspectDestination.WaitWithDefaultTimeout() Expect(inspectDestination.ExitCode()).To(Equal(0)) Expect(inspectDestination.OutputToString()).To(Equal("/test1")) - inspectDst := podmanTest.Podman([]string{"inspect", "-l", "--format", "{{(index .Mounts 0).Dst}}"}) + inspectDst := podmanTest.Podman([]string{"inspect", "test", "--format", "{{(index .Mounts 0).Dst}}"}) inspectDst.WaitWithDefaultTimeout() Expect(inspectDst.ExitCode()).To(Equal(0)) Expect(inspectDst.OutputToString()).To(Equal("/test1")) @@ -171,4 +170,57 @@ var _ = Describe("Podman inspect", func() { Expect(imageData[0].HealthCheck.Interval).To(BeNumerically("==", 60000000000)) Expect(imageData[0].HealthCheck.Test).To(Equal([]string{"CMD-SHELL", "curl -f http://localhost/ || exit 1"})) }) + + It("podman inspect --latest with no container fails", func() { + SkipIfRemote() + + session := podmanTest.Podman([]string{"inspect", "--latest"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Not(Equal(0))) + }) + + It("podman [image,container] inspect on image", func() { + baseInspect := podmanTest.Podman([]string{"inspect", ALPINE}) + baseInspect.WaitWithDefaultTimeout() + Expect(baseInspect.ExitCode()).To(Equal(0)) + baseJSON := baseInspect.InspectImageJSON() + Expect(len(baseJSON)).To(Equal(1)) + + ctrInspect := podmanTest.Podman([]string{"container", "inspect", ALPINE}) + ctrInspect.WaitWithDefaultTimeout() + Expect(ctrInspect.ExitCode()).To(Not(Equal(0))) + + imageInspect := podmanTest.Podman([]string{"image", "inspect", ALPINE}) + imageInspect.WaitWithDefaultTimeout() + Expect(imageInspect.ExitCode()).To(Equal(0)) + imageJSON := imageInspect.InspectImageJSON() + Expect(len(imageJSON)).To(Equal(1)) + + Expect(baseJSON[0].ID).To(Equal(imageJSON[0].ID)) + }) + + It("podman [image, container] inspect on container", func() { + ctrName := "testCtr" + create := podmanTest.Podman([]string{"create", "--name", ctrName, ALPINE, "sh"}) + create.WaitWithDefaultTimeout() + Expect(create.ExitCode()).To(Equal(0)) + + baseInspect := podmanTest.Podman([]string{"inspect", ctrName}) + baseInspect.WaitWithDefaultTimeout() + Expect(baseInspect.ExitCode()).To(Equal(0)) + baseJSON := baseInspect.InspectContainerToJSON() + Expect(len(baseJSON)).To(Equal(1)) + + ctrInspect := podmanTest.Podman([]string{"container", "inspect", ctrName}) + ctrInspect.WaitWithDefaultTimeout() + Expect(ctrInspect.ExitCode()).To(Equal(0)) + ctrJSON := ctrInspect.InspectContainerToJSON() + Expect(len(ctrJSON)).To(Equal(1)) + + imageInspect := podmanTest.Podman([]string{"image", "inspect", ctrName}) + imageInspect.WaitWithDefaultTimeout() + Expect(imageInspect.ExitCode()).To(Not(Equal(0))) + + Expect(baseJSON[0].ID).To(Equal(ctrJSON[0].ID)) + }) }) diff --git a/test/e2e/libpod_suite_remote_test.go b/test/e2e/libpod_suite_remote_test.go index dde853413..b94b6e267 100644 --- a/test/e2e/libpod_suite_remote_test.go +++ b/test/e2e/libpod_suite_remote_test.go @@ -36,7 +36,7 @@ func SkipIfRootlessV2() { // Podman is the exec call to podman on the filesystem func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration { - var remoteArgs = []string{"--remote", p.RemoteSocket} + var remoteArgs = []string{"--remote", "--url", p.RemoteSocket} remoteArgs = append(remoteArgs, args...) podmanSession := p.PodmanBase(remoteArgs, false, false) return &PodmanSessionIntegration{podmanSession} @@ -44,7 +44,7 @@ func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration // PodmanExtraFiles is the exec call to podman on the filesystem and passes down extra files func (p *PodmanTestIntegration) PodmanExtraFiles(args []string, extraFiles []*os.File) *PodmanSessionIntegration { - var remoteArgs = []string{"--remote", p.RemoteSocket} + var remoteArgs = []string{"--remote", "--url", p.RemoteSocket} remoteArgs = append(remoteArgs, args...) podmanSession := p.PodmanAsUserBase(remoteArgs, 0, 0, "", nil, false, false, extraFiles) return &PodmanSessionIntegration{podmanSession} @@ -52,7 +52,7 @@ func (p *PodmanTestIntegration) PodmanExtraFiles(args []string, extraFiles []*os // PodmanNoCache calls podman with out adding the imagecache func (p *PodmanTestIntegration) PodmanNoCache(args []string) *PodmanSessionIntegration { - var remoteArgs = []string{"--remote", p.RemoteSocket} + var remoteArgs = []string{"--remote", "--url", p.RemoteSocket} remoteArgs = append(remoteArgs, args...) podmanSession := p.PodmanBase(remoteArgs, false, true) return &PodmanSessionIntegration{podmanSession} diff --git a/test/e2e/pod_create_test.go b/test/e2e/pod_create_test.go index e56db54a2..a7d5783cb 100644 --- a/test/e2e/pod_create_test.go +++ b/test/e2e/pod_create_test.go @@ -238,7 +238,6 @@ var _ = Describe("Podman pod create", func() { }) It("podman create pod with IP address", func() { - SkipIfRemote() SkipIfRootless() name := "test" ip := GetRandomIPAddress() diff --git a/test/e2e/ps_test.go b/test/e2e/ps_test.go index 12ce4661f..0dc8e01af 100644 --- a/test/e2e/ps_test.go +++ b/test/e2e/ps_test.go @@ -362,8 +362,6 @@ var _ = Describe("Podman ps", func() { }) It("podman --pod with a non-empty pod name", func() { - SkipIfRemote() - podName := "testPodName" _, ec, podid := podmanTest.CreatePod(podName) Expect(ec).To(Equal(0)) diff --git a/test/e2e/pull_test.go b/test/e2e/pull_test.go index 96340ef30..ac882927f 100644 --- a/test/e2e/pull_test.go +++ b/test/e2e/pull_test.go @@ -351,7 +351,6 @@ var _ = Describe("Podman pull", func() { }) It("podman pull from docker with nonexist --authfile", func() { - SkipIfRemote() session := podmanTest.PodmanNoCache([]string{"pull", "--authfile", "/tmp/nonexist", ALPINE}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Not(Equal(0))) diff --git a/test/e2e/rm_test.go b/test/e2e/rm_test.go index 87e3de922..6b554cd84 100644 --- a/test/e2e/rm_test.go +++ b/test/e2e/rm_test.go @@ -141,7 +141,6 @@ var _ = Describe("Podman rm", func() { }) It("podman rm --cidfile", func() { - SkipIfRemote() tmpDir, err := ioutil.TempDir("", "") Expect(err).To(BeNil()) @@ -164,7 +163,6 @@ var _ = Describe("Podman rm", func() { }) It("podman rm multiple --cidfile", func() { - SkipIfRemote() tmpDir, err := ioutil.TempDir("", "") Expect(err).To(BeNil()) diff --git a/test/e2e/volume_rm_test.go b/test/e2e/volume_rm_test.go index 6f2020828..742d4e0dc 100644 --- a/test/e2e/volume_rm_test.go +++ b/test/e2e/volume_rm_test.go @@ -48,7 +48,6 @@ var _ = Describe("Podman volume rm", func() { }) It("podman volume rm with --force flag", func() { - SkipIfRemote() session := podmanTest.Podman([]string{"create", "-v", "myvol:/myvol", ALPINE, "ls"}) cid := session.OutputToString() session.WaitWithDefaultTimeout() diff --git a/test/utils/utils.go b/test/utils/utils.go index 1d59e5468..0597cd292 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -73,7 +73,7 @@ func (p *PodmanTest) PodmanAsUserBase(args []string, uid, gid uint32, cwd string podmanBinary = p.RemotePodmanBinary } if p.RemoteTest { - podmanOptions = append([]string{"--remote", p.RemoteSocket}, podmanOptions...) + podmanOptions = append([]string{"--remote", "--url", p.RemoteSocket}, podmanOptions...) } if env == nil { fmt.Printf("Running: %s %s\n", podmanBinary, strings.Join(podmanOptions, " ")) diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index 0044d6cb9..769e37e15 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.20.1 +1.20.2 diff --git a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go index f1c941f11..ff6e297f4 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go @@ -1544,8 +1544,8 @@ func getDeviceMajorMinor(file *os.File) (uint64, uint64, error) { } dev := stat.Rdev - majorNum := major(dev) - minorNum := minor(dev) + majorNum := major(uint64(dev)) + minorNum := minor(uint64(dev)) logrus.Debugf("devmapper: Major:Minor for device: %s is:%v:%v", file.Name(), majorNum, minorNum) return majorNum, minorNum, nil diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 2906e3e08..930a57a97 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -311,6 +311,9 @@ func parseOptions(options []string) (*overlayOptions, error) { return nil, fmt.Errorf("overlay: can't stat program %s: %v", val, err) } o.mountProgram = val + case "overlay2.skip_mount_home", "overlay.skip_mount_home", ".skip_mount_home": + logrus.Debugf("overlay: skip_mount_home=%s", val) + o.skipMountHome, err = strconv.ParseBool(val) case ".ignore_chown_errors", "overlay2.ignore_chown_errors", "overlay.ignore_chown_errors": logrus.Debugf("overlay: ignore_chown_errors=%s", val) o.ignoreChownErrors, err = strconv.ParseBool(val) diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index a7d9ade60..01ac1827b 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -6,23 +6,23 @@ require ( github.com/Microsoft/hcsshim v0.8.9 github.com/docker/go-units v0.4.0 github.com/hashicorp/go-multierror v1.0.0 - github.com/klauspost/compress v1.10.5 + github.com/klauspost/compress v1.10.7 github.com/klauspost/pgzip v1.2.4 github.com/mattn/go-shellwords v1.0.10 github.com/mistifyio/go-zfs v2.1.1+incompatible github.com/opencontainers/go-digest v1.0.0 - github.com/opencontainers/runc v1.0.0-rc9 + github.com/opencontainers/runc v1.0.0-rc90 github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700 - github.com/opencontainers/selinux v1.5.1 + github.com/opencontainers/selinux v1.5.2 github.com/pkg/errors v0.9.1 github.com/pquerna/ffjson v0.0.0-20181028064349-e517b90714f7 github.com/sirupsen/logrus v1.6.0 - github.com/stretchr/testify v1.5.1 + github.com/stretchr/testify v1.6.0 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 github.com/tchap/go-patricia v2.3.0+incompatible github.com/vbatts/tar-split v0.11.1 golang.org/x/net v0.0.0-20191004110552-13f9640d40b9 - golang.org/x/sys v0.0.0-20191127021746-63cb32ae39b2 + golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 gotest.tools v2.2.0+incompatible ) diff --git a/vendor/github.com/containers/storage/go.sum b/vendor/github.com/containers/storage/go.sum index eab0fd61e..2b5a279c6 100644 --- a/vendor/github.com/containers/storage/go.sum +++ b/vendor/github.com/containers/storage/go.sum @@ -8,6 +8,7 @@ github.com/Microsoft/hcsshim v0.8.9/go.mod h1:5692vkUqntj1idxauYlpoINNKeqCiG6Sg3 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f h1:tSNMc+rJDfmYntojat8lljbt1mgKNpTxUZJsSzJ9Y1s= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= +github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1 h1:uict5mhHFTzKLUCufdSLym7z/J0CbBJT59lYbP9wtbg= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -15,12 +16,14 @@ github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e h1:BWhy2j3IXJhjCbC68FptL43tDKIq8FladmaTs3Xs7Z8= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= github.com/gogo/protobuf v1.3.1 h1:DqDEcV5aeaTmdFBePNpYsp3FlcVH/2ISVVM9Qf8PSls= @@ -29,6 +32,7 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= @@ -42,8 +46,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQLJ+jE2L00= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= -github.com/klauspost/compress v1.10.5 h1:7q6vHIqubShURwQz8cQK6yIe/xC3IF0Vm7TGfqjewrc= -github.com/klauspost/compress v1.10.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.10.7 h1:7rix8v8GpI3ZBb0nSozFRgbtXKv+hOe+qfEpZqybrAg= +github.com/klauspost/compress v1.10.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/pgzip v1.2.4 h1:TQ7CNpYKovDOmqzRHKxJh0BeaBI7UdQZYc6p7pMQh1A= github.com/klauspost/pgzip v1.2.4/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -62,12 +66,12 @@ github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v1.0.0-rc9 h1:/k06BMULKF5hidyoZymkoDCzdJzltZpz/UU4LguQVtc= -github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc90 h1:4+xo8mtWixbHoEm451+WJNUrq12o2/tDsyK9Vgc/NcA= +github.com/opencontainers/runc v1.0.0-rc90/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700 h1:eNUVfm/RFLIi1G7flU5/ZRTHvd4kcVuzfRnL6OFlzCI= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/selinux v1.5.1 h1:jskKwSMFYqyTrHEuJgQoUlTcId0av64S6EWObrIfn5Y= -github.com/opencontainers/selinux v1.5.1/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g= +github.com/opencontainers/selinux v1.5.2 h1:F6DgIsjgBIcDksLW4D5RG9bXok6oqZ3nvMwj4ZoFu/Q= +github.com/opencontainers/selinux v1.5.2/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwyzAJufJyiTt7s0g= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -81,15 +85,17 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0 h1:UBcNElsrwanuuMsnGSlYmtmgbb23qDR5dG+6X6Oo89I= github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1 h1:2vfRuCMp5sSVIDSqO8oNnWJq7mPa6KVP3iPIwFBuy8A= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= -github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.6.0 h1:jlIyCplCJFULU/01vCkhKuTyc3OorI3bJFuw6obfgho= +github.com/stretchr/testify v1.6.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tchap/go-patricia v2.3.0+incompatible h1:GkY4dP3cEfEASBPPkWd+AmjYxhmDkqO9/zg7R0lSQRs= github.com/tchap/go-patricia v2.3.0+incompatible/go.mod h1:bmLyhP68RS6kStMGxByiQ23RP/odRBOTVjwp2cDyi6I= +github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5 h1:MCfT24H3f//U5+UCrZp1/riVO3B50BovxtDiNn0XKkk= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vbatts/tar-split v0.11.1 h1:0Odu65rhcZ3JZaPHxl7tCI3V/C/Q9Zf82UFravl02dE= github.com/vbatts/tar-split v0.11.1/go.mod h1:LEuURwDEiWjRjwu46yU3KVGuUdVv/dcnpcEPSzR8z6g= @@ -120,8 +126,8 @@ golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190514135907-3a4b5fb9f71f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191127021746-63cb32ae39b2 h1:/J2nHFg1MTqaRLFO7M+J78ASNsJoz3r0cvHBPQ77fsE= -golang.org/x/sys v0.0.0-20191127021746-63cb32ae39b2/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9 h1:1/DFK4b7JH8DmkqhUk48onnSfrPzImPoVxuomtbT2nk= +golang.org/x/sys v0.0.0-20200124204421-9fbb57f87de9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -145,6 +151,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/vendor/github.com/containers/storage/pkg/config/config.go b/vendor/github.com/containers/storage/pkg/config/config.go index 1ac6c04e3..4a35997ea 100644 --- a/vendor/github.com/containers/storage/pkg/config/config.go +++ b/vendor/github.com/containers/storage/pkg/config/config.go @@ -2,8 +2,6 @@ package config import ( "fmt" - - "github.com/sirupsen/logrus" ) // ThinpoolOptionsConfig represents the "storage.options.thinpool" @@ -269,11 +267,11 @@ func GetGraphDriverOptions(driverName string, options OptionsConfig) []string { } else if options.Size != "" { doptions = append(doptions, fmt.Sprintf("%s.size=%s", driverName, options.Size)) } - - if options.Overlay.SkipMountHome != "" || options.SkipMountHome != "" { - logrus.Warn("skip_mount_home option is no longer supported, ignoring option") + if options.Overlay.SkipMountHome != "" { + doptions = append(doptions, fmt.Sprintf("%s.skip_mount_home=%s", driverName, options.Overlay.SkipMountHome)) + } else if options.SkipMountHome != "" { + doptions = append(doptions, fmt.Sprintf("%s.skip_mount_home=%s", driverName, options.SkipMountHome)) } - case "vfs": if options.Vfs.IgnoreChownErrors != "" { doptions = append(doptions, fmt.Sprintf("%s.ignore_chown_errors=%s", driverName, options.Vfs.IgnoreChownErrors)) diff --git a/vendor/github.com/containers/storage/storage.conf b/vendor/github.com/containers/storage/storage.conf index c7f9b2cf8..19909e9c6 100644 --- a/vendor/github.com/containers/storage/storage.conf +++ b/vendor/github.com/containers/storage/storage.conf @@ -76,6 +76,9 @@ additionalimagestores = [ # mountopt specifies comma separated list of extra mount options mountopt = "nodev" +# Set to skip a PRIVATE bind mount on the storage home directory. +# skip_mount_home = "false" + # Size is used to set a maximum size of the container image. # size = "" diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index eaf622f43..263f4edb2 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -3481,6 +3481,9 @@ func ReloadConfigurationFile(configFile string, storeOptions *StoreOptions) { if config.Storage.Options.MountProgram != "" { storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.mount_program=%s", config.Storage.Driver, config.Storage.Options.MountProgram)) } + if config.Storage.Options.SkipMountHome != "" { + storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.skip_mount_home=%s", config.Storage.Driver, config.Storage.Options.SkipMountHome)) + } if config.Storage.Options.IgnoreChownErrors != "" { storeOptions.GraphDriverOptions = append(storeOptions.GraphDriverOptions, fmt.Sprintf("%s.ignore_chown_errors=%s", config.Storage.Driver, config.Storage.Options.IgnoreChownErrors)) } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index 97ae66a4a..fb42a398b 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -155,20 +155,70 @@ func ReadTable(in []byte, s *Scratch) (s2 *Scratch, remain []byte, err error) { // The length of the supplied input must match the end of a block exactly. // Before this is called, the table must be initialized with ReadTable unless // the encoder re-used the table. +// deprecated: Use the stateless Decoder() to get a concurrent version. func (s *Scratch) Decompress1X(in []byte) (out []byte, err error) { - if len(s.dt.single) == 0 { + if cap(s.Out) < s.MaxDecodedSize { + s.Out = make([]byte, s.MaxDecodedSize) + } + s.Out = s.Out[:0:s.MaxDecodedSize] + s.Out, err = s.Decoder().Decompress1X(s.Out, in) + return s.Out, err +} + +// Decompress4X will decompress a 4X encoded stream. +// Before this is called, the table must be initialized with ReadTable unless +// the encoder re-used the table. +// The length of the supplied input must match the end of a block exactly. +// The destination size of the uncompressed data must be known and provided. +// deprecated: Use the stateless Decoder() to get a concurrent version. +func (s *Scratch) Decompress4X(in []byte, dstSize int) (out []byte, err error) { + if dstSize > s.MaxDecodedSize { + return nil, ErrMaxDecodedSizeExceeded + } + if cap(s.Out) < dstSize { + s.Out = make([]byte, s.MaxDecodedSize) + } + s.Out = s.Out[:0:dstSize] + s.Out, err = s.Decoder().Decompress4X(s.Out, in) + return s.Out, err +} + +// Decoder will return a stateless decoder that can be used by multiple +// decompressors concurrently. +// Before this is called, the table must be initialized with ReadTable. +// The Decoder is still linked to the scratch buffer so that cannot be reused. +// However, it is safe to discard the scratch. +func (s *Scratch) Decoder() *Decoder { + return &Decoder{ + dt: s.dt, + actualTableLog: s.actualTableLog, + } +} + +// Decoder provides stateless decoding. +type Decoder struct { + dt dTable + actualTableLog uint8 +} + +// Decompress1X will decompress a 1X encoded stream. +// The cap of the output buffer will be the maximum decompressed size. +// The length of the supplied input must match the end of a block exactly. +func (d *Decoder) Decompress1X(dst, src []byte) ([]byte, error) { + if len(d.dt.single) == 0 { return nil, errors.New("no table loaded") } var br bitReader - err = br.init(in) + err := br.init(src) if err != nil { - return nil, err + return dst, err } - s.Out = s.Out[:0] + maxDecodedSize := cap(dst) + dst = dst[:0] decode := func() byte { - val := br.peekBitsFast(s.actualTableLog) /* note : actualTableLog >= 1 */ - v := s.dt.single[val] + val := br.peekBitsFast(d.actualTableLog) /* note : actualTableLog >= 1 */ + v := d.dt.single[val] br.bitsRead += uint8(v.entry) return uint8(v.entry >> 8) } @@ -180,88 +230,80 @@ func (s *Scratch) Decompress1X(in []byte) (out []byte, err error) { // Avoid bounds check by always having full sized table. const tlSize = 1 << tableLogMax const tlMask = tlSize - 1 - dt := s.dt.single[:tlSize] + dt := d.dt.single[:tlSize] // Use temp table to avoid bound checks/append penalty. - var tmp = s.huffWeight[:256] + var buf [256]byte var off uint8 for br.off >= 8 { br.fillFast() - tmp[off+0] = hasDec(dt[br.peekBitsFast(s.actualTableLog)&tlMask]) - tmp[off+1] = hasDec(dt[br.peekBitsFast(s.actualTableLog)&tlMask]) + buf[off+0] = hasDec(dt[br.peekBitsFast(d.actualTableLog)&tlMask]) + buf[off+1] = hasDec(dt[br.peekBitsFast(d.actualTableLog)&tlMask]) br.fillFast() - tmp[off+2] = hasDec(dt[br.peekBitsFast(s.actualTableLog)&tlMask]) - tmp[off+3] = hasDec(dt[br.peekBitsFast(s.actualTableLog)&tlMask]) + buf[off+2] = hasDec(dt[br.peekBitsFast(d.actualTableLog)&tlMask]) + buf[off+3] = hasDec(dt[br.peekBitsFast(d.actualTableLog)&tlMask]) off += 4 if off == 0 { - if len(s.Out)+256 > s.MaxDecodedSize { + if len(dst)+256 > maxDecodedSize { br.close() return nil, ErrMaxDecodedSizeExceeded } - s.Out = append(s.Out, tmp...) + dst = append(dst, buf[:]...) } } - if len(s.Out)+int(off) > s.MaxDecodedSize { + if len(dst)+int(off) > maxDecodedSize { br.close() return nil, ErrMaxDecodedSizeExceeded } - s.Out = append(s.Out, tmp[:off]...) + dst = append(dst, buf[:off]...) for !br.finished() { br.fill() - if len(s.Out) >= s.MaxDecodedSize { + if len(dst) >= maxDecodedSize { br.close() return nil, ErrMaxDecodedSizeExceeded } - s.Out = append(s.Out, decode()) + dst = append(dst, decode()) } - return s.Out, br.close() + return dst, br.close() } // Decompress4X will decompress a 4X encoded stream. -// Before this is called, the table must be initialized with ReadTable unless -// the encoder re-used the table. // The length of the supplied input must match the end of a block exactly. -// The destination size of the uncompressed data must be known and provided. -func (s *Scratch) Decompress4X(in []byte, dstSize int) (out []byte, err error) { +// The *capacity* of the dst slice must match the destination size of +// the uncompressed data exactly. +func (s *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { if len(s.dt.single) == 0 { return nil, errors.New("no table loaded") } - if len(in) < 6+(4*1) { + if len(src) < 6+(4*1) { return nil, errors.New("input too small") } - if dstSize > s.MaxDecodedSize { - return nil, ErrMaxDecodedSizeExceeded - } - // TODO: We do not detect when we overrun a buffer, except if the last one does. var br [4]bitReader start := 6 for i := 0; i < 3; i++ { - length := int(in[i*2]) | (int(in[i*2+1]) << 8) - if start+length >= len(in) { + length := int(src[i*2]) | (int(src[i*2+1]) << 8) + if start+length >= len(src) { return nil, errors.New("truncated input (or invalid offset)") } - err = br[i].init(in[start : start+length]) + err := br[i].init(src[start : start+length]) if err != nil { return nil, err } start += length } - err = br[3].init(in[start:]) + err := br[3].init(src[start:]) if err != nil { return nil, err } - // Prepare output - if cap(s.Out) < dstSize { - s.Out = make([]byte, 0, dstSize) - } - s.Out = s.Out[:dstSize] // destination, offset to match first output - dstOut := s.Out + dstSize := cap(dst) + dst = dst[:dstSize] + out := dst dstEvery := (dstSize + 3) / 4 const tlSize = 1 << tableLogMax @@ -276,7 +318,7 @@ func (s *Scratch) Decompress4X(in []byte, dstSize int) (out []byte, err error) { } // Use temp table to avoid bound checks/append penalty. - var tmp = s.huffWeight[:256] + var buf [256]byte var off uint8 var decoded int @@ -300,8 +342,8 @@ bigloop: val2 := br[stream].peekBitsFast(s.actualTableLog) v2 := single[val2&tlMask] - tmp[off+bufoff*stream+1] = uint8(v2.entry >> 8) - tmp[off+bufoff*stream] = uint8(v.entry >> 8) + buf[off+bufoff*stream+1] = uint8(v2.entry >> 8) + buf[off+bufoff*stream] = uint8(v.entry >> 8) br[stream].bitsRead += uint8(v2.entry) } @@ -313,8 +355,8 @@ bigloop: val2 := br[stream].peekBitsFast(s.actualTableLog) v2 := single[val2&tlMask] - tmp[off+bufoff*stream+1] = uint8(v2.entry >> 8) - tmp[off+bufoff*stream] = uint8(v.entry >> 8) + buf[off+bufoff*stream+1] = uint8(v2.entry >> 8) + buf[off+bufoff*stream] = uint8(v.entry >> 8) br[stream].bitsRead += uint8(v2.entry) } @@ -326,8 +368,8 @@ bigloop: val2 := br[stream].peekBitsFast(s.actualTableLog) v2 := single[val2&tlMask] - tmp[off+bufoff*stream+1] = uint8(v2.entry >> 8) - tmp[off+bufoff*stream] = uint8(v.entry >> 8) + buf[off+bufoff*stream+1] = uint8(v2.entry >> 8) + buf[off+bufoff*stream] = uint8(v.entry >> 8) br[stream].bitsRead += uint8(v2.entry) } @@ -339,8 +381,8 @@ bigloop: val2 := br[stream].peekBitsFast(s.actualTableLog) v2 := single[val2&tlMask] - tmp[off+bufoff*stream+1] = uint8(v2.entry >> 8) - tmp[off+bufoff*stream] = uint8(v.entry >> 8) + buf[off+bufoff*stream+1] = uint8(v2.entry >> 8) + buf[off+bufoff*stream] = uint8(v.entry >> 8) br[stream].bitsRead += uint8(v2.entry) } @@ -350,30 +392,30 @@ bigloop: if bufoff > dstEvery { return nil, errors.New("corruption detected: stream overrun 1") } - copy(dstOut, tmp[:bufoff]) - copy(dstOut[dstEvery:], tmp[bufoff:bufoff*2]) - copy(dstOut[dstEvery*2:], tmp[bufoff*2:bufoff*3]) - copy(dstOut[dstEvery*3:], tmp[bufoff*3:bufoff*4]) + copy(out, buf[:bufoff]) + copy(out[dstEvery:], buf[bufoff:bufoff*2]) + copy(out[dstEvery*2:], buf[bufoff*2:bufoff*3]) + copy(out[dstEvery*3:], buf[bufoff*3:bufoff*4]) off = 0 - dstOut = dstOut[bufoff:] + out = out[bufoff:] decoded += 256 // There must at least be 3 buffers left. - if len(dstOut) < dstEvery*3 { + if len(out) < dstEvery*3 { return nil, errors.New("corruption detected: stream overrun 2") } } } if off > 0 { ioff := int(off) - if len(dstOut) < dstEvery*3+ioff { + if len(out) < dstEvery*3+ioff { return nil, errors.New("corruption detected: stream overrun 3") } - copy(dstOut, tmp[:off]) - copy(dstOut[dstEvery:dstEvery+ioff], tmp[bufoff:bufoff*2]) - copy(dstOut[dstEvery*2:dstEvery*2+ioff], tmp[bufoff*2:bufoff*3]) - copy(dstOut[dstEvery*3:dstEvery*3+ioff], tmp[bufoff*3:bufoff*4]) + copy(out, buf[:off]) + copy(out[dstEvery:dstEvery+ioff], buf[bufoff:bufoff*2]) + copy(out[dstEvery*2:dstEvery*2+ioff], buf[bufoff*2:bufoff*3]) + copy(out[dstEvery*3:dstEvery*3+ioff], buf[bufoff*3:bufoff*4]) decoded += int(off) * 4 - dstOut = dstOut[off:] + out = out[off:] } // Decode remaining. @@ -382,10 +424,10 @@ bigloop: br := &br[i] for !br.finished() { br.fill() - if offset >= len(dstOut) { + if offset >= len(out) { return nil, errors.New("corruption detected: stream overrun 4") } - dstOut[offset] = decode(br) + out[offset] = decode(br) offset++ } decoded += offset - dstEvery*i @@ -397,7 +439,7 @@ bigloop: if dstSize != decoded { return nil, errors.New("corruption detected: short output block") } - return s.Out, nil + return dst, nil } // matches will compare a decoding table to a coding table. diff --git a/vendor/github.com/klauspost/compress/zstd/README.md b/vendor/github.com/klauspost/compress/zstd/README.md index bc977a302..f2a80b5d0 100644 --- a/vendor/github.com/klauspost/compress/zstd/README.md +++ b/vendor/github.com/klauspost/compress/zstd/README.md @@ -309,6 +309,20 @@ The decoder can be used for *concurrent* decompression of multiple buffers. It will only allow a certain number of concurrent operations to run. To tweak that yourself use the `WithDecoderConcurrency(n)` option when creating the decoder. +### Dictionaries + +Data compressed with [dictionaries](https://github.com/facebook/zstd#the-case-for-small-data-compression) can be decompressed. + +Dictionaries are added individually to Decoders. +Dictionaries are generated by the `zstd --train` command and contains an initial state for the decoder. +To add a dictionary use the `RegisterDict(data)` with the dictionary data before starting any decompression. + +The dictionary will be used automatically for the data that specifies them. + +A re-used Decoder will still contain the dictionaries registered. + +When registering a dictionary with the same ID it will override the existing. + ### Allocation-less operation The decoder has been designed to operate without allocations after a warmup. diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 19181caea..4a14242c7 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -461,26 +461,22 @@ func (b *blockDec) decodeCompressed(hist *history) error { if huff == nil { huff = &huff0.Scratch{} } - huff.Out = b.literalBuf[:0] huff, literals, err = huff0.ReadTable(literals, huff) if err != nil { println("reading huffman table:", err) return err } // Use our out buffer. - huff.Out = b.literalBuf[:0] - huff.MaxDecodedSize = litRegenSize if fourStreams { - literals, err = huff.Decompress4X(literals, litRegenSize) + literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) } else { - literals, err = huff.Decompress1X(literals) + literals, err = huff.Decoder().Decompress1X(b.literalBuf[:0:litRegenSize], literals) } if err != nil { println("decoding compressed literals:", err) return err } // Make sure we don't leak our literals buffer - huff.Out = nil if len(literals) != litRegenSize { return fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) } @@ -631,15 +627,12 @@ func (b *blockDec) decodeCompressed(hist *history) error { var err error // Use our out buffer. huff = hist.huffTree - huff.Out = b.literalBuf[:0] - huff.MaxDecodedSize = litRegenSize if fourStreams { - literals, err = huff.Decompress4X(literals, litRegenSize) + literals, err = huff.Decoder().Decompress4X(b.literalBuf[:0:litRegenSize], literals) } else { - literals, err = huff.Decompress1X(literals) + literals, err = huff.Decoder().Decompress1X(b.literalBuf[:0:litRegenSize], literals) } // Make sure we don't leak our literals buffer - huff.Out = nil if err != nil { println("decompressing literals:", err) return err @@ -649,12 +642,13 @@ func (b *blockDec) decodeCompressed(hist *history) error { } } else { if hist.huffTree != nil && huff != nil { - huffDecoderPool.Put(hist.huffTree) + if hist.dict == nil || hist.dict.litDec != hist.huffTree { + huffDecoderPool.Put(hist.huffTree) + } hist.huffTree = nil } } if huff != nil { - huff.Out = nil hist.huffTree = huff } if debug { @@ -687,14 +681,20 @@ func (b *blockDec) decodeCompressed(hist *history) error { // If only recent offsets were not transferred, this would be an obvious win. // Also, if first 3 sequences don't reference recent offsets, all sequences can be decoded. - if err := seqs.initialize(br, hist, literals, b.dst); err != nil { - println("initializing sequences:", err) - return err - } hbytes := hist.b if len(hbytes) > hist.windowSize { hbytes = hbytes[len(hbytes)-hist.windowSize:] + // We do not need history any more. + if hist.dict != nil { + hist.dict.content = nil + } } + + if err := seqs.initialize(br, hist, literals, b.dst); err != nil { + println("initializing sequences:", err) + return err + } + err = seqs.decode(nSeqs, br, hbytes) if err != nil { return err diff --git a/vendor/github.com/klauspost/compress/zstd/bytereader.go b/vendor/github.com/klauspost/compress/zstd/bytereader.go index dc4378b64..f708df1c4 100644 --- a/vendor/github.com/klauspost/compress/zstd/bytereader.go +++ b/vendor/github.com/klauspost/compress/zstd/bytereader.go @@ -4,6 +4,8 @@ package zstd +import "encoding/binary" + // byteReader provides a byte reader that reads // little endian values from a byte stream. // The input stream is manually advanced. @@ -55,12 +57,7 @@ func (b byteReader) Uint32() uint32 { } return v } - b2 := b.b[b.off : b.off+4 : b.off+4] - v3 := uint32(b2[3]) - v2 := uint32(b2[2]) - v1 := uint32(b2[1]) - v0 := uint32(b2[0]) - return v0 | (v1 << 8) | (v2 << 16) | (v3 << 24) + return binary.LittleEndian.Uint32(b.b[b.off : b.off+4]) } // unread returns the unread portion of the input. diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 324347623..8e34479ff 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -32,8 +32,9 @@ type Decoder struct { // Current read position used for Reader functionality. current decoderState - // Custom dictionaries - dicts map[uint32]struct{} + // Custom dictionaries. + // Always uses copies. + dicts map[uint32]dict // streamWg is the waitgroup for all streams streamWg sync.WaitGroup @@ -295,10 +296,18 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { frame.bBuf = input for { + frame.history.reset() err := frame.reset(&frame.bBuf) if err == io.EOF { return dst, nil } + if frame.DictionaryID != nil { + dict, ok := d.dicts[*frame.DictionaryID] + if !ok { + return nil, ErrUnknownDictionary + } + frame.history.setDict(&dict) + } if err != nil { return dst, err } @@ -393,6 +402,19 @@ func (d *Decoder) Close() { d.current.err = ErrDecoderClosed } +// RegisterDict will load a dictionary +func (d *Decoder) RegisterDict(b []byte) error { + dc, err := loadDict(b) + if err != nil { + return err + } + if d.dicts == nil { + d.dicts = make(map[uint32]dict, 1) + } + d.dicts[dc.id] = *dc + return nil +} + // IOReadCloser returns the decoder as an io.ReadCloser for convenience. // Any changes to the decoder will be reflected, so the returned ReadCloser // can be reused along with the decoder. @@ -466,6 +488,14 @@ func (d *Decoder) startStreamDecoder(inStream chan decodeStream) { if debug && err != nil { println("Frame decoder returned", err) } + if err == nil && frame.DictionaryID != nil { + dict, ok := d.dicts[*frame.DictionaryID] + if !ok { + err = ErrUnknownDictionary + } else { + frame.history.setDict(&dict) + } + } if err != nil { stream.output <- decodeOutput{ err: err, diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go new file mode 100644 index 000000000..8eb6f6ba3 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -0,0 +1,104 @@ +package zstd + +import ( + "bytes" + "encoding/binary" + "errors" + "fmt" + "io" + + "github.com/klauspost/compress/huff0" +) + +type dict struct { + id uint32 + + litDec *huff0.Scratch + llDec, ofDec, mlDec sequenceDec + offsets [3]int + content []byte +} + +var dictMagic = [4]byte{0x37, 0xa4, 0x30, 0xec} + +// Load a dictionary as described in +// https://github.com/facebook/zstd/blob/master/doc/zstd_compression_format.md#dictionary-format +func loadDict(b []byte) (*dict, error) { + // Check static field size. + if len(b) <= 8+(3*4) { + return nil, io.ErrUnexpectedEOF + } + d := dict{ + llDec: sequenceDec{fse: &fseDecoder{}}, + ofDec: sequenceDec{fse: &fseDecoder{}}, + mlDec: sequenceDec{fse: &fseDecoder{}}, + } + if !bytes.Equal(b[:4], dictMagic[:]) { + return nil, ErrMagicMismatch + } + d.id = binary.LittleEndian.Uint32(b[4:8]) + if d.id == 0 { + return nil, errors.New("dictionaries cannot have ID 0") + } + + // Read literal table + var err error + d.litDec, b, err = huff0.ReadTable(b[8:], nil) + if err != nil { + return nil, err + } + + br := byteReader{ + b: b, + off: 0, + } + readDec := func(i tableIndex, dec *fseDecoder) error { + if err := dec.readNCount(&br, uint16(maxTableSymbol[i])); err != nil { + return err + } + if br.overread() { + return io.ErrUnexpectedEOF + } + err = dec.transform(symbolTableX[i]) + if err != nil { + println("Transform table error:", err) + return err + } + if debug { + println("Read table ok", "symbolLen:", dec.symbolLen) + } + // Set decoders as predefined so they aren't reused. + dec.preDefined = true + return nil + } + + if err := readDec(tableOffsets, d.ofDec.fse); err != nil { + return nil, err + } + if err := readDec(tableMatchLengths, d.mlDec.fse); err != nil { + return nil, err + } + if err := readDec(tableLiteralLengths, d.llDec.fse); err != nil { + return nil, err + } + if br.remain() < 12 { + return nil, io.ErrUnexpectedEOF + } + + d.offsets[0] = int(br.Uint32()) + br.advance(4) + d.offsets[1] = int(br.Uint32()) + br.advance(4) + d.offsets[2] = int(br.Uint32()) + br.advance(4) + if d.offsets[0] <= 0 || d.offsets[1] <= 0 || d.offsets[2] <= 0 { + return nil, errors.New("invalid offset in dictionary") + } + d.content = make([]byte, br.remain()) + copy(d.content, br.unread()) + if d.offsets[0] > len(d.content) || d.offsets[1] > len(d.content) || d.offsets[2] > len(d.content) { + return nil, fmt.Errorf("initial offset bigger than dictionary content size %d, offsets: %v", len(d.content), d.offsets) + } + + return &d, nil +} diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index 5ebead9dc..50276bcde 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -671,4 +671,8 @@ encodeLoop: println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } + // We do not store history, so we must offset e.cur to avoid false matches for next user. + if e.cur < bufferReset { + e.cur += int32(len(src)) + } } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index d1d3658e6..4104b456c 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -383,6 +383,7 @@ func (e *fastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { panic("src too big") } } + // Protect against e.cur wraparound. if e.cur >= bufferReset { for i := range e.table[:] { @@ -516,6 +517,9 @@ encodeLoop: if debugAsserts && s-t > e.maxMatchOff { panic("s - t >e.maxMatchOff") } + if debugAsserts && t < 0 { + panic(fmt.Sprintf("t (%d) < 0, candidate.offset: %d, e.cur: %d, coffset0: %d, e.maxMatchOff: %d", t, candidate.offset, e.cur, coffset0, e.maxMatchOff)) + } break } @@ -548,6 +552,9 @@ encodeLoop: panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) } + if debugAsserts && t < 0 { + panic(fmt.Sprintf("t (%d) < 0 ", t)) + } // Extend the 4-byte match as long as possible. //l := e.matchlenNoHist(s+4, t+4, src) + 4 // l := int32(matchLen(src[s+4:], src[t+4:])) + 4 @@ -647,6 +654,10 @@ encodeLoop: if debug { println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } + // We do not store history, so we must offset e.cur to avoid false matches for next user. + if e.cur < bufferReset { + e.cur += int32(len(src)) + } } func (e *fastBase) addBlock(src []byte) int32 { @@ -714,7 +725,7 @@ func (e *fastBase) matchlen(s, t int32, src []byte) int32 { } // Reset the encoding table. -func (e *fastBase) Reset() { +func (e *fastBase) Reset(singleBlock bool) { if e.blk == nil { e.blk = &blockEnc{} e.blk.init() @@ -727,7 +738,7 @@ func (e *fastBase) Reset() { } else { e.crc.Reset() } - if cap(e.hist) < int(e.maxMatchOff*2) { + if !singleBlock && cap(e.hist) < int(e.maxMatchOff*2) { l := e.maxMatchOff * 2 // Make it at least 1MB. if l < 1<<20 { diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index af4f00b73..bf42bb1cf 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -35,7 +35,7 @@ type encoder interface { AppendCRC([]byte) []byte WindowSize(size int) int32 UseBlock(*blockEnc) - Reset() + Reset(singleBlock bool) } type encoderState struct { @@ -82,7 +82,10 @@ func (e *Encoder) initialize() { } e.encoders = make(chan encoder, e.o.concurrent) for i := 0; i < e.o.concurrent; i++ { - e.encoders <- e.o.encoder() + enc := e.o.encoder() + // If not single block, history will be allocated on first use. + enc.Reset(true) + e.encoders <- enc } } @@ -112,7 +115,7 @@ func (e *Encoder) Reset(w io.Writer) { s.filling = s.filling[:0] s.current = s.current[:0] s.previous = s.previous[:0] - s.encoder.Reset() + s.encoder.Reset(false) s.headerWritten = false s.eofWritten = false s.fullFrameWritten = false @@ -445,11 +448,10 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { enc := <-e.encoders defer func() { // Release encoder reference to last block. - enc.Reset() + // If a non-single block is needed the encoder will reset again. + enc.Reset(true) e.encoders <- enc }() - enc.Reset() - blk := enc.Block() // Use single segments when above minimum window and below 1MB. single := len(src) < 1<<20 && len(src) > MinWindowSize if e.o.single != nil { @@ -472,12 +474,13 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { panic(err) } - if len(src) <= e.o.blockSize && len(src) <= maxBlockSize { + // If we can do everything in one block, prefer that. + if len(src) <= maxCompressedBlockSize { // Slightly faster with no history and everything in one block. if e.o.crc { _, _ = enc.CRC().Write(src) } - blk.reset(nil) + blk := enc.Block() blk.last = true enc.EncodeNoHist(blk, src) @@ -504,6 +507,8 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } blk.output = oldout } else { + enc.Reset(false) + blk := enc.Block() for len(src) > 0 { todo := src if len(todo) > e.o.blockSize { diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index 780880ebe..fc4a566d3 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -40,7 +40,7 @@ type frameDec struct { FrameContentSize uint64 frameDone sync.WaitGroup - DictionaryID uint32 + DictionaryID *uint32 HasCheckSum bool SingleSegment bool @@ -142,7 +142,7 @@ func (d *frameDec) reset(br byteBuffer) error { // Read Dictionary_ID // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary_id - d.DictionaryID = 0 + d.DictionaryID = nil if size := fhd & 3; size != 0 { if size == 3 { size = 4 @@ -154,19 +154,22 @@ func (d *frameDec) reset(br byteBuffer) error { } return io.ErrUnexpectedEOF } + var id uint32 switch size { case 1: - d.DictionaryID = uint32(b[0]) + id = uint32(b[0]) case 2: - d.DictionaryID = uint32(b[0]) | (uint32(b[1]) << 8) + id = uint32(b[0]) | (uint32(b[1]) << 8) case 4: - d.DictionaryID = uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) + id = uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) } if debug { - println("Dict size", size, "ID:", d.DictionaryID) + println("Dict size", size, "ID:", id) } - if d.DictionaryID != 0 { - return ErrUnknownDictionary + if id > 0 { + // ID 0 means "sorry, no dictionary anyway". + // https://github.com/facebook/zstd/blob/dev/doc/zstd_compression_format.md#dictionary-format + d.DictionaryID = &id } } @@ -351,8 +354,6 @@ func (d *frameDec) initAsync() { // When the frame has finished decoding the *bufio.Reader // containing the remaining input will be sent on frameDec.frameDone. func (d *frameDec) startDecoder(output chan decodeOutput) { - // TODO: Init to dictionary - d.history.reset() written := int64(0) defer func() { @@ -445,8 +446,6 @@ func (d *frameDec) startDecoder(output chan decodeOutput) { // runDecoder will create a sync decoder that will decode a block of data. func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { - // TODO: Init to dictionary - d.history.reset() saved := d.history.b // We use the history for output to avoid copying it. diff --git a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go index e002be98b..957cfeb79 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_decoder.go @@ -19,7 +19,7 @@ const ( * Increasing memory usage improves compression ratio * Reduced memory usage can improve speed, due to cache effect * Recommended max value is 14, for 16KB, which nicely fits into Intel x86 L1 cache */ - maxMemoryUsage = 11 + maxMemoryUsage = tablelogAbsoluteMax + 2 maxTableLog = maxMemoryUsage - 2 maxTablesize = 1 << maxTableLog diff --git a/vendor/github.com/klauspost/compress/zstd/history.go b/vendor/github.com/klauspost/compress/zstd/history.go index e8c419bd5..f418f50fc 100644 --- a/vendor/github.com/klauspost/compress/zstd/history.go +++ b/vendor/github.com/klauspost/compress/zstd/history.go @@ -17,6 +17,7 @@ type history struct { windowSize int maxSize int error bool + dict *dict } // reset will reset the history to initial state of a frame. @@ -36,12 +37,27 @@ func (h *history) reset() { } h.decoders = sequenceDecs{} if h.huffTree != nil { - huffDecoderPool.Put(h.huffTree) + if h.dict == nil || h.dict.litDec != h.huffTree { + huffDecoderPool.Put(h.huffTree) + } } h.huffTree = nil + h.dict = nil //printf("history created: %+v (l: %d, c: %d)", *h, len(h.b), cap(h.b)) } +func (h *history) setDict(dict *dict) { + if dict == nil { + return + } + h.dict = dict + h.decoders.litLengths = dict.llDec + h.decoders.offsets = dict.ofDec + h.decoders.matchLengths = dict.mlDec + h.recentOffsets = dict.offsets + h.huffTree = dict.litDec +} + // append bytes to history. // This function will make sure there is space for it, // if the buffer has been allocated with enough extra space. diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec.go b/vendor/github.com/klauspost/compress/zstd/seqdec.go index 39238e16a..7ff870400 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec.go @@ -62,6 +62,7 @@ type sequenceDecs struct { matchLengths sequenceDec prevOffset [3]int hist []byte + dict []byte literals []byte out []byte windowSize int @@ -85,6 +86,10 @@ func (s *sequenceDecs) initialize(br *bitReader, hist *history, literals, out [] s.maxBits = s.litLengths.fse.maxBits + s.offsets.fse.maxBits + s.matchLengths.fse.maxBits s.windowSize = hist.windowSize s.out = out + s.dict = nil + if hist.dict != nil { + s.dict = hist.dict.content + } return nil } @@ -100,23 +105,78 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { printf("reading sequence %d, exceeded available data\n", seqs-i) return io.ErrUnexpectedEOF } - var litLen, matchOff, matchLen int + var ll, mo, ml int if br.off > 4+((maxOffsetBits+16+16)>>3) { - litLen, matchOff, matchLen = s.nextFast(br, llState, mlState, ofState) + // inlined function: + // ll, mo, ml = s.nextFast(br, llState, mlState, ofState) + + // Final will not read from stream. + var llB, mlB, moB uint8 + ll, llB = llState.final() + ml, mlB = mlState.final() + mo, moB = ofState.final() + + // extra bits are stored in reverse order. + br.fillFast() + mo += br.getBits(moB) + if s.maxBits > 32 { + br.fillFast() + } + ml += br.getBits(mlB) + ll += br.getBits(llB) + + if moB > 1 { + s.prevOffset[2] = s.prevOffset[1] + s.prevOffset[1] = s.prevOffset[0] + s.prevOffset[0] = mo + } else { + // mo = s.adjustOffset(mo, ll, moB) + // Inlined for rather big speedup + if ll == 0 { + // There is an exception though, when current sequence's literals_length = 0. + // In this case, repeated offsets are shifted by one, so an offset_value of 1 means Repeated_Offset2, + // an offset_value of 2 means Repeated_Offset3, and an offset_value of 3 means Repeated_Offset1 - 1_byte. + mo++ + } + + if mo == 0 { + mo = s.prevOffset[0] + } else { + var temp int + if mo == 3 { + temp = s.prevOffset[0] - 1 + } else { + temp = s.prevOffset[mo] + } + + if temp == 0 { + // 0 is not valid; input is corrupted; force offset to 1 + println("temp was 0") + temp = 1 + } + + if mo != 1 { + s.prevOffset[2] = s.prevOffset[1] + } + s.prevOffset[1] = s.prevOffset[0] + s.prevOffset[0] = temp + mo = temp + } + } br.fillFast() } else { - litLen, matchOff, matchLen = s.next(br, llState, mlState, ofState) + ll, mo, ml = s.next(br, llState, mlState, ofState) br.fill() } if debugSequences { - println("Seq", seqs-i-1, "Litlen:", litLen, "matchOff:", matchOff, "(abs) matchLen:", matchLen) + println("Seq", seqs-i-1, "Litlen:", ll, "mo:", mo, "(abs) ml:", ml) } - if litLen > len(s.literals) { - return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", litLen, len(s.literals)) + if ll > len(s.literals) { + return fmt.Errorf("unexpected literal count, want %d bytes, but only %d is available", ll, len(s.literals)) } - size := litLen + matchLen + len(s.out) + size := ll + ml + len(s.out) if size-startSize > maxBlockSize { return fmt.Errorf("output (%d) bigger than max block size", size) } @@ -127,52 +187,70 @@ func (s *sequenceDecs) decode(seqs int, br *bitReader, hist []byte) error { s.out = append(s.out, make([]byte, maxBlockSize)...) s.out = s.out[:len(s.out)-maxBlockSize] } - if matchLen > maxMatchLen { - return fmt.Errorf("match len (%d) bigger than max allowed length", matchLen) - } - if matchOff > len(s.out)+len(hist)+litLen { - return fmt.Errorf("match offset (%d) bigger than current history (%d)", matchOff, len(s.out)+len(hist)+litLen) - } - if matchOff > s.windowSize { - return fmt.Errorf("match offset (%d) bigger than window size (%d)", matchOff, s.windowSize) - } - if matchOff == 0 && matchLen > 0 { - return fmt.Errorf("zero matchoff and matchlen > 0") + if ml > maxMatchLen { + return fmt.Errorf("match len (%d) bigger than max allowed length", ml) } - s.out = append(s.out, s.literals[:litLen]...) - s.literals = s.literals[litLen:] + // Add literals + s.out = append(s.out, s.literals[:ll]...) + s.literals = s.literals[ll:] out := s.out + if mo > len(s.out)+len(hist) || mo > s.windowSize { + if len(s.dict) == 0 { + return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(s.out)+len(hist)) + } + + // we may be in dictionary. + dictO := len(s.dict) - (mo - (len(s.out) + len(hist))) + if dictO < 0 || dictO >= len(s.dict) { + return fmt.Errorf("match offset (%d) bigger than current history (%d)", mo, len(s.out)+len(hist)) + } + end := dictO + ml + if end > len(s.dict) { + out = append(out, s.dict[dictO:]...) + mo -= len(s.dict) - dictO + ml -= len(s.dict) - dictO + } else { + out = append(out, s.dict[dictO:end]...) + mo = 0 + ml = 0 + } + } + + if mo == 0 && ml > 0 { + return fmt.Errorf("zero matchoff and matchlen (%d) > 0", ml) + } + // Copy from history. // TODO: Blocks without history could be made to ignore this completely. - if v := matchOff - len(s.out); v > 0 { + if v := mo - len(s.out); v > 0 { // v is the start position in history from end. start := len(s.hist) - v - if matchLen > v { + if ml > v { // Some goes into current block. // Copy remainder of history out = append(out, s.hist[start:]...) - matchOff -= v - matchLen -= v + mo -= v + ml -= v } else { - out = append(out, s.hist[start:start+matchLen]...) - matchLen = 0 + out = append(out, s.hist[start:start+ml]...) + ml = 0 } } // We must be in current buffer now - if matchLen > 0 { - start := len(s.out) - matchOff - if matchLen <= len(s.out)-start { + if ml > 0 { + start := len(s.out) - mo + if ml <= len(s.out)-start { // No overlap - out = append(out, s.out[start:start+matchLen]...) + out = append(out, s.out[start:start+ml]...) } else { // Overlapping copy // Extend destination slice and copy one byte at the time. - out = out[:len(out)+matchLen] - src := out[start : start+matchLen] + out = out[:len(out)+ml] + src := out[start : start+ml] // Destination is the space we just added. - dst := out[len(out)-matchLen:] + dst := out[len(out)-ml:] dst = dst[:len(src)] for i := range src { dst[i] = src[i] diff --git a/vendor/golang.org/x/crypto/ssh/agent/client.go b/vendor/golang.org/x/crypto/ssh/agent/client.go new file mode 100644 index 000000000..b909471cc --- /dev/null +++ b/vendor/golang.org/x/crypto/ssh/agent/client.go @@ -0,0 +1,813 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package agent implements the ssh-agent protocol, and provides both +// a client and a server. The client can talk to a standard ssh-agent +// that uses UNIX sockets, and one could implement an alternative +// ssh-agent process using the sample server. +// +// References: +// [PROTOCOL.agent]: https://tools.ietf.org/html/draft-miller-ssh-agent-00 +package agent // import "golang.org/x/crypto/ssh/agent" + +import ( + "bytes" + "crypto/dsa" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rsa" + "encoding/base64" + "encoding/binary" + "errors" + "fmt" + "io" + "math/big" + "sync" + + "crypto" + "golang.org/x/crypto/ed25519" + "golang.org/x/crypto/ssh" +) + +// SignatureFlags represent additional flags that can be passed to the signature +// requests an defined in [PROTOCOL.agent] section 4.5.1. +type SignatureFlags uint32 + +// SignatureFlag values as defined in [PROTOCOL.agent] section 5.3. +const ( + SignatureFlagReserved SignatureFlags = 1 << iota + SignatureFlagRsaSha256 + SignatureFlagRsaSha512 +) + +// Agent represents the capabilities of an ssh-agent. +type Agent interface { + // List returns the identities known to the agent. + List() ([]*Key, error) + + // Sign has the agent sign the data using a protocol 2 key as defined + // in [PROTOCOL.agent] section 2.6.2. + Sign(key ssh.PublicKey, data []byte) (*ssh.Signature, error) + + // Add adds a private key to the agent. + Add(key AddedKey) error + + // Remove removes all identities with the given public key. + Remove(key ssh.PublicKey) error + + // RemoveAll removes all identities. + RemoveAll() error + + // Lock locks the agent. Sign and Remove will fail, and List will empty an empty list. + Lock(passphrase []byte) error + + // Unlock undoes the effect of Lock + Unlock(passphrase []byte) error + + // Signers returns signers for all the known keys. + Signers() ([]ssh.Signer, error) +} + +type ExtendedAgent interface { + Agent + + // SignWithFlags signs like Sign, but allows for additional flags to be sent/received + SignWithFlags(key ssh.PublicKey, data []byte, flags SignatureFlags) (*ssh.Signature, error) + + // Extension processes a custom extension request. Standard-compliant agents are not + // required to support any extensions, but this method allows agents to implement + // vendor-specific methods or add experimental features. See [PROTOCOL.agent] section 4.7. + // If agent extensions are unsupported entirely this method MUST return an + // ErrExtensionUnsupported error. Similarly, if just the specific extensionType in + // the request is unsupported by the agent then ErrExtensionUnsupported MUST be + // returned. + // + // In the case of success, since [PROTOCOL.agent] section 4.7 specifies that the contents + // of the response are unspecified (including the type of the message), the complete + // response will be returned as a []byte slice, including the "type" byte of the message. + Extension(extensionType string, contents []byte) ([]byte, error) +} + +// ConstraintExtension describes an optional constraint defined by users. +type ConstraintExtension struct { + // ExtensionName consist of a UTF-8 string suffixed by the + // implementation domain following the naming scheme defined + // in Section 4.2 of [RFC4251], e.g. "foo@example.com". + ExtensionName string + // ExtensionDetails contains the actual content of the extended + // constraint. + ExtensionDetails []byte +} + +// AddedKey describes an SSH key to be added to an Agent. +type AddedKey struct { + // PrivateKey must be a *rsa.PrivateKey, *dsa.PrivateKey, + // ed25519.PrivateKey or *ecdsa.PrivateKey, which will be inserted into the + // agent. + PrivateKey interface{} + // Certificate, if not nil, is communicated to the agent and will be + // stored with the key. + Certificate *ssh.Certificate + // Comment is an optional, free-form string. + Comment string + // LifetimeSecs, if not zero, is the number of seconds that the + // agent will store the key for. + LifetimeSecs uint32 + // ConfirmBeforeUse, if true, requests that the agent confirm with the + // user before each use of this key. + ConfirmBeforeUse bool + // ConstraintExtensions are the experimental or private-use constraints + // defined by users. + ConstraintExtensions []ConstraintExtension +} + +// See [PROTOCOL.agent], section 3. +const ( + agentRequestV1Identities = 1 + agentRemoveAllV1Identities = 9 + + // 3.2 Requests from client to agent for protocol 2 key operations + agentAddIdentity = 17 + agentRemoveIdentity = 18 + agentRemoveAllIdentities = 19 + agentAddIDConstrained = 25 + + // 3.3 Key-type independent requests from client to agent + agentAddSmartcardKey = 20 + agentRemoveSmartcardKey = 21 + agentLock = 22 + agentUnlock = 23 + agentAddSmartcardKeyConstrained = 26 + + // 3.7 Key constraint identifiers + agentConstrainLifetime = 1 + agentConstrainConfirm = 2 + agentConstrainExtension = 3 +) + +// maxAgentResponseBytes is the maximum agent reply size that is accepted. This +// is a sanity check, not a limit in the spec. +const maxAgentResponseBytes = 16 << 20 + +// Agent messages: +// These structures mirror the wire format of the corresponding ssh agent +// messages found in [PROTOCOL.agent]. + +// 3.4 Generic replies from agent to client +const agentFailure = 5 + +type failureAgentMsg struct{} + +const agentSuccess = 6 + +type successAgentMsg struct{} + +// See [PROTOCOL.agent], section 2.5.2. +const agentRequestIdentities = 11 + +type requestIdentitiesAgentMsg struct{} + +// See [PROTOCOL.agent], section 2.5.2. +const agentIdentitiesAnswer = 12 + +type identitiesAnswerAgentMsg struct { + NumKeys uint32 `sshtype:"12"` + Keys []byte `ssh:"rest"` +} + +// See [PROTOCOL.agent], section 2.6.2. +const agentSignRequest = 13 + +type signRequestAgentMsg struct { + KeyBlob []byte `sshtype:"13"` + Data []byte + Flags uint32 +} + +// See [PROTOCOL.agent], section 2.6.2. + +// 3.6 Replies from agent to client for protocol 2 key operations +const agentSignResponse = 14 + +type signResponseAgentMsg struct { + SigBlob []byte `sshtype:"14"` +} + +type publicKey struct { + Format string + Rest []byte `ssh:"rest"` +} + +// 3.7 Key constraint identifiers +type constrainLifetimeAgentMsg struct { + LifetimeSecs uint32 `sshtype:"1"` +} + +type constrainExtensionAgentMsg struct { + ExtensionName string `sshtype:"3"` + ExtensionDetails []byte + + // Rest is a field used for parsing, not part of message + Rest []byte `ssh:"rest"` +} + +// See [PROTOCOL.agent], section 4.7 +const agentExtension = 27 +const agentExtensionFailure = 28 + +// ErrExtensionUnsupported indicates that an extension defined in +// [PROTOCOL.agent] section 4.7 is unsupported by the agent. Specifically this +// error indicates that the agent returned a standard SSH_AGENT_FAILURE message +// as the result of a SSH_AGENTC_EXTENSION request. Note that the protocol +// specification (and therefore this error) does not distinguish between a +// specific extension being unsupported and extensions being unsupported entirely. +var ErrExtensionUnsupported = errors.New("agent: extension unsupported") + +type extensionAgentMsg struct { + ExtensionType string `sshtype:"27"` + Contents []byte +} + +// Key represents a protocol 2 public key as defined in +// [PROTOCOL.agent], section 2.5.2. +type Key struct { + Format string + Blob []byte + Comment string +} + +func clientErr(err error) error { + return fmt.Errorf("agent: client error: %v", err) +} + +// String returns the storage form of an agent key with the format, base64 +// encoded serialized key, and the comment if it is not empty. +func (k *Key) String() string { + s := string(k.Format) + " " + base64.StdEncoding.EncodeToString(k.Blob) + + if k.Comment != "" { + s += " " + k.Comment + } + + return s +} + +// Type returns the public key type. +func (k *Key) Type() string { + return k.Format +} + +// Marshal returns key blob to satisfy the ssh.PublicKey interface. +func (k *Key) Marshal() []byte { + return k.Blob +} + +// Verify satisfies the ssh.PublicKey interface. +func (k *Key) Verify(data []byte, sig *ssh.Signature) error { + pubKey, err := ssh.ParsePublicKey(k.Blob) + if err != nil { + return fmt.Errorf("agent: bad public key: %v", err) + } + return pubKey.Verify(data, sig) +} + +type wireKey struct { + Format string + Rest []byte `ssh:"rest"` +} + +func parseKey(in []byte) (out *Key, rest []byte, err error) { + var record struct { + Blob []byte + Comment string + Rest []byte `ssh:"rest"` + } + + if err := ssh.Unmarshal(in, &record); err != nil { + return nil, nil, err + } + + var wk wireKey + if err := ssh.Unmarshal(record.Blob, &wk); err != nil { + return nil, nil, err + } + + return &Key{ + Format: wk.Format, + Blob: record.Blob, + Comment: record.Comment, + }, record.Rest, nil +} + +// client is a client for an ssh-agent process. +type client struct { + // conn is typically a *net.UnixConn + conn io.ReadWriter + // mu is used to prevent concurrent access to the agent + mu sync.Mutex +} + +// NewClient returns an Agent that talks to an ssh-agent process over +// the given connection. +func NewClient(rw io.ReadWriter) ExtendedAgent { + return &client{conn: rw} +} + +// call sends an RPC to the agent. On success, the reply is +// unmarshaled into reply and replyType is set to the first byte of +// the reply, which contains the type of the message. +func (c *client) call(req []byte) (reply interface{}, err error) { + buf, err := c.callRaw(req) + if err != nil { + return nil, err + } + reply, err = unmarshal(buf) + if err != nil { + return nil, clientErr(err) + } + return reply, nil +} + +// callRaw sends an RPC to the agent. On success, the raw +// bytes of the response are returned; no unmarshalling is +// performed on the response. +func (c *client) callRaw(req []byte) (reply []byte, err error) { + c.mu.Lock() + defer c.mu.Unlock() + + msg := make([]byte, 4+len(req)) + binary.BigEndian.PutUint32(msg, uint32(len(req))) + copy(msg[4:], req) + if _, err = c.conn.Write(msg); err != nil { + return nil, clientErr(err) + } + + var respSizeBuf [4]byte + if _, err = io.ReadFull(c.conn, respSizeBuf[:]); err != nil { + return nil, clientErr(err) + } + respSize := binary.BigEndian.Uint32(respSizeBuf[:]) + if respSize > maxAgentResponseBytes { + return nil, clientErr(errors.New("response too large")) + } + + buf := make([]byte, respSize) + if _, err = io.ReadFull(c.conn, buf); err != nil { + return nil, clientErr(err) + } + return buf, nil +} + +func (c *client) simpleCall(req []byte) error { + resp, err := c.call(req) + if err != nil { + return err + } + if _, ok := resp.(*successAgentMsg); ok { + return nil + } + return errors.New("agent: failure") +} + +func (c *client) RemoveAll() error { + return c.simpleCall([]byte{agentRemoveAllIdentities}) +} + +func (c *client) Remove(key ssh.PublicKey) error { + req := ssh.Marshal(&agentRemoveIdentityMsg{ + KeyBlob: key.Marshal(), + }) + return c.simpleCall(req) +} + +func (c *client) Lock(passphrase []byte) error { + req := ssh.Marshal(&agentLockMsg{ + Passphrase: passphrase, + }) + return c.simpleCall(req) +} + +func (c *client) Unlock(passphrase []byte) error { + req := ssh.Marshal(&agentUnlockMsg{ + Passphrase: passphrase, + }) + return c.simpleCall(req) +} + +// List returns the identities known to the agent. +func (c *client) List() ([]*Key, error) { + // see [PROTOCOL.agent] section 2.5.2. + req := []byte{agentRequestIdentities} + + msg, err := c.call(req) + if err != nil { + return nil, err + } + + switch msg := msg.(type) { + case *identitiesAnswerAgentMsg: + if msg.NumKeys > maxAgentResponseBytes/8 { + return nil, errors.New("agent: too many keys in agent reply") + } + keys := make([]*Key, msg.NumKeys) + data := msg.Keys + for i := uint32(0); i < msg.NumKeys; i++ { + var key *Key + var err error + if key, data, err = parseKey(data); err != nil { + return nil, err + } + keys[i] = key + } + return keys, nil + case *failureAgentMsg: + return nil, errors.New("agent: failed to list keys") + } + panic("unreachable") +} + +// Sign has the agent sign the data using a protocol 2 key as defined +// in [PROTOCOL.agent] section 2.6.2. +func (c *client) Sign(key ssh.PublicKey, data []byte) (*ssh.Signature, error) { + return c.SignWithFlags(key, data, 0) +} + +func (c *client) SignWithFlags(key ssh.PublicKey, data []byte, flags SignatureFlags) (*ssh.Signature, error) { + req := ssh.Marshal(signRequestAgentMsg{ + KeyBlob: key.Marshal(), + Data: data, + Flags: uint32(flags), + }) + + msg, err := c.call(req) + if err != nil { + return nil, err + } + + switch msg := msg.(type) { + case *signResponseAgentMsg: + var sig ssh.Signature + if err := ssh.Unmarshal(msg.SigBlob, &sig); err != nil { + return nil, err + } + + return &sig, nil + case *failureAgentMsg: + return nil, errors.New("agent: failed to sign challenge") + } + panic("unreachable") +} + +// unmarshal parses an agent message in packet, returning the parsed +// form and the message type of packet. +func unmarshal(packet []byte) (interface{}, error) { + if len(packet) < 1 { + return nil, errors.New("agent: empty packet") + } + var msg interface{} + switch packet[0] { + case agentFailure: + return new(failureAgentMsg), nil + case agentSuccess: + return new(successAgentMsg), nil + case agentIdentitiesAnswer: + msg = new(identitiesAnswerAgentMsg) + case agentSignResponse: + msg = new(signResponseAgentMsg) + case agentV1IdentitiesAnswer: + msg = new(agentV1IdentityMsg) + default: + return nil, fmt.Errorf("agent: unknown type tag %d", packet[0]) + } + if err := ssh.Unmarshal(packet, msg); err != nil { + return nil, err + } + return msg, nil +} + +type rsaKeyMsg struct { + Type string `sshtype:"17|25"` + N *big.Int + E *big.Int + D *big.Int + Iqmp *big.Int // IQMP = Inverse Q Mod P + P *big.Int + Q *big.Int + Comments string + Constraints []byte `ssh:"rest"` +} + +type dsaKeyMsg struct { + Type string `sshtype:"17|25"` + P *big.Int + Q *big.Int + G *big.Int + Y *big.Int + X *big.Int + Comments string + Constraints []byte `ssh:"rest"` +} + +type ecdsaKeyMsg struct { + Type string `sshtype:"17|25"` + Curve string + KeyBytes []byte + D *big.Int + Comments string + Constraints []byte `ssh:"rest"` +} + +type ed25519KeyMsg struct { + Type string `sshtype:"17|25"` + Pub []byte + Priv []byte + Comments string + Constraints []byte `ssh:"rest"` +} + +// Insert adds a private key to the agent. +func (c *client) insertKey(s interface{}, comment string, constraints []byte) error { + var req []byte + switch k := s.(type) { + case *rsa.PrivateKey: + if len(k.Primes) != 2 { + return fmt.Errorf("agent: unsupported RSA key with %d primes", len(k.Primes)) + } + k.Precompute() + req = ssh.Marshal(rsaKeyMsg{ + Type: ssh.KeyAlgoRSA, + N: k.N, + E: big.NewInt(int64(k.E)), + D: k.D, + Iqmp: k.Precomputed.Qinv, + P: k.Primes[0], + Q: k.Primes[1], + Comments: comment, + Constraints: constraints, + }) + case *dsa.PrivateKey: + req = ssh.Marshal(dsaKeyMsg{ + Type: ssh.KeyAlgoDSA, + P: k.P, + Q: k.Q, + G: k.G, + Y: k.Y, + X: k.X, + Comments: comment, + Constraints: constraints, + }) + case *ecdsa.PrivateKey: + nistID := fmt.Sprintf("nistp%d", k.Params().BitSize) + req = ssh.Marshal(ecdsaKeyMsg{ + Type: "ecdsa-sha2-" + nistID, + Curve: nistID, + KeyBytes: elliptic.Marshal(k.Curve, k.X, k.Y), + D: k.D, + Comments: comment, + Constraints: constraints, + }) + case ed25519.PrivateKey: + req = ssh.Marshal(ed25519KeyMsg{ + Type: ssh.KeyAlgoED25519, + Pub: []byte(k)[32:], + Priv: []byte(k), + Comments: comment, + Constraints: constraints, + }) + // This function originally supported only *ed25519.PrivateKey, however the + // general idiom is to pass ed25519.PrivateKey by value, not by pointer. + // We still support the pointer variant for backwards compatibility. + case *ed25519.PrivateKey: + req = ssh.Marshal(ed25519KeyMsg{ + Type: ssh.KeyAlgoED25519, + Pub: []byte(*k)[32:], + Priv: []byte(*k), + Comments: comment, + Constraints: constraints, + }) + default: + return fmt.Errorf("agent: unsupported key type %T", s) + } + + // if constraints are present then the message type needs to be changed. + if len(constraints) != 0 { + req[0] = agentAddIDConstrained + } + + resp, err := c.call(req) + if err != nil { + return err + } + if _, ok := resp.(*successAgentMsg); ok { + return nil + } + return errors.New("agent: failure") +} + +type rsaCertMsg struct { + Type string `sshtype:"17|25"` + CertBytes []byte + D *big.Int + Iqmp *big.Int // IQMP = Inverse Q Mod P + P *big.Int + Q *big.Int + Comments string + Constraints []byte `ssh:"rest"` +} + +type dsaCertMsg struct { + Type string `sshtype:"17|25"` + CertBytes []byte + X *big.Int + Comments string + Constraints []byte `ssh:"rest"` +} + +type ecdsaCertMsg struct { + Type string `sshtype:"17|25"` + CertBytes []byte + D *big.Int + Comments string + Constraints []byte `ssh:"rest"` +} + +type ed25519CertMsg struct { + Type string `sshtype:"17|25"` + CertBytes []byte + Pub []byte + Priv []byte + Comments string + Constraints []byte `ssh:"rest"` +} + +// Add adds a private key to the agent. If a certificate is given, +// that certificate is added instead as public key. +func (c *client) Add(key AddedKey) error { + var constraints []byte + + if secs := key.LifetimeSecs; secs != 0 { + constraints = append(constraints, ssh.Marshal(constrainLifetimeAgentMsg{secs})...) + } + + if key.ConfirmBeforeUse { + constraints = append(constraints, agentConstrainConfirm) + } + + cert := key.Certificate + if cert == nil { + return c.insertKey(key.PrivateKey, key.Comment, constraints) + } + return c.insertCert(key.PrivateKey, cert, key.Comment, constraints) +} + +func (c *client) insertCert(s interface{}, cert *ssh.Certificate, comment string, constraints []byte) error { + var req []byte + switch k := s.(type) { + case *rsa.PrivateKey: + if len(k.Primes) != 2 { + return fmt.Errorf("agent: unsupported RSA key with %d primes", len(k.Primes)) + } + k.Precompute() + req = ssh.Marshal(rsaCertMsg{ + Type: cert.Type(), + CertBytes: cert.Marshal(), + D: k.D, + Iqmp: k.Precomputed.Qinv, + P: k.Primes[0], + Q: k.Primes[1], + Comments: comment, + Constraints: constraints, + }) + case *dsa.PrivateKey: + req = ssh.Marshal(dsaCertMsg{ + Type: cert.Type(), + CertBytes: cert.Marshal(), + X: k.X, + Comments: comment, + Constraints: constraints, + }) + case *ecdsa.PrivateKey: + req = ssh.Marshal(ecdsaCertMsg{ + Type: cert.Type(), + CertBytes: cert.Marshal(), + D: k.D, + Comments: comment, + Constraints: constraints, + }) + case ed25519.PrivateKey: + req = ssh.Marshal(ed25519CertMsg{ + Type: cert.Type(), + CertBytes: cert.Marshal(), + Pub: []byte(k)[32:], + Priv: []byte(k), + Comments: comment, + Constraints: constraints, + }) + // This function originally supported only *ed25519.PrivateKey, however the + // general idiom is to pass ed25519.PrivateKey by value, not by pointer. + // We still support the pointer variant for backwards compatibility. + case *ed25519.PrivateKey: + req = ssh.Marshal(ed25519CertMsg{ + Type: cert.Type(), + CertBytes: cert.Marshal(), + Pub: []byte(*k)[32:], + Priv: []byte(*k), + Comments: comment, + Constraints: constraints, + }) + default: + return fmt.Errorf("agent: unsupported key type %T", s) + } + + // if constraints are present then the message type needs to be changed. + if len(constraints) != 0 { + req[0] = agentAddIDConstrained + } + + signer, err := ssh.NewSignerFromKey(s) + if err != nil { + return err + } + if bytes.Compare(cert.Key.Marshal(), signer.PublicKey().Marshal()) != 0 { + return errors.New("agent: signer and cert have different public key") + } + + resp, err := c.call(req) + if err != nil { + return err + } + if _, ok := resp.(*successAgentMsg); ok { + return nil + } + return errors.New("agent: failure") +} + +// Signers provides a callback for client authentication. +func (c *client) Signers() ([]ssh.Signer, error) { + keys, err := c.List() + if err != nil { + return nil, err + } + + var result []ssh.Signer + for _, k := range keys { + result = append(result, &agentKeyringSigner{c, k}) + } + return result, nil +} + +type agentKeyringSigner struct { + agent *client + pub ssh.PublicKey +} + +func (s *agentKeyringSigner) PublicKey() ssh.PublicKey { + return s.pub +} + +func (s *agentKeyringSigner) Sign(rand io.Reader, data []byte) (*ssh.Signature, error) { + // The agent has its own entropy source, so the rand argument is ignored. + return s.agent.Sign(s.pub, data) +} + +func (s *agentKeyringSigner) SignWithOpts(rand io.Reader, data []byte, opts crypto.SignerOpts) (*ssh.Signature, error) { + var flags SignatureFlags + if opts != nil { + switch opts.HashFunc() { + case crypto.SHA256: + flags = SignatureFlagRsaSha256 + case crypto.SHA512: + flags = SignatureFlagRsaSha512 + } + } + return s.agent.SignWithFlags(s.pub, data, flags) +} + +// Calls an extension method. It is up to the agent implementation as to whether or not +// any particular extension is supported and may always return an error. Because the +// type of the response is up to the implementation, this returns the bytes of the +// response and does not attempt any type of unmarshalling. +func (c *client) Extension(extensionType string, contents []byte) ([]byte, error) { + req := ssh.Marshal(extensionAgentMsg{ + ExtensionType: extensionType, + Contents: contents, + }) + buf, err := c.callRaw(req) + if err != nil { + return nil, err + } + if len(buf) == 0 { + return nil, errors.New("agent: failure; empty response") + } + // [PROTOCOL.agent] section 4.7 indicates that an SSH_AGENT_FAILURE message + // represents an agent that does not support the extension + if buf[0] == agentFailure { + return nil, ErrExtensionUnsupported + } + if buf[0] == agentExtensionFailure { + return nil, errors.New("agent: generic extension failure") + } + + return buf, nil +} diff --git a/vendor/golang.org/x/crypto/ssh/agent/forward.go b/vendor/golang.org/x/crypto/ssh/agent/forward.go new file mode 100644 index 000000000..fd24ba900 --- /dev/null +++ b/vendor/golang.org/x/crypto/ssh/agent/forward.go @@ -0,0 +1,103 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package agent + +import ( + "errors" + "io" + "net" + "sync" + + "golang.org/x/crypto/ssh" +) + +// RequestAgentForwarding sets up agent forwarding for the session. +// ForwardToAgent or ForwardToRemote should be called to route +// the authentication requests. +func RequestAgentForwarding(session *ssh.Session) error { + ok, err := session.SendRequest("auth-agent-req@openssh.com", true, nil) + if err != nil { + return err + } + if !ok { + return errors.New("forwarding request denied") + } + return nil +} + +// ForwardToAgent routes authentication requests to the given keyring. +func ForwardToAgent(client *ssh.Client, keyring Agent) error { + channels := client.HandleChannelOpen(channelType) + if channels == nil { + return errors.New("agent: already have handler for " + channelType) + } + + go func() { + for ch := range channels { + channel, reqs, err := ch.Accept() + if err != nil { + continue + } + go ssh.DiscardRequests(reqs) + go func() { + ServeAgent(keyring, channel) + channel.Close() + }() + } + }() + return nil +} + +const channelType = "auth-agent@openssh.com" + +// ForwardToRemote routes authentication requests to the ssh-agent +// process serving on the given unix socket. +func ForwardToRemote(client *ssh.Client, addr string) error { + channels := client.HandleChannelOpen(channelType) + if channels == nil { + return errors.New("agent: already have handler for " + channelType) + } + conn, err := net.Dial("unix", addr) + if err != nil { + return err + } + conn.Close() + + go func() { + for ch := range channels { + channel, reqs, err := ch.Accept() + if err != nil { + continue + } + go ssh.DiscardRequests(reqs) + go forwardUnixSocket(channel, addr) + } + }() + return nil +} + +func forwardUnixSocket(channel ssh.Channel, addr string) { + conn, err := net.Dial("unix", addr) + if err != nil { + return + } + + var wg sync.WaitGroup + wg.Add(2) + go func() { + io.Copy(conn, channel) + conn.(*net.UnixConn).CloseWrite() + wg.Done() + }() + go func() { + io.Copy(channel, conn) + channel.CloseWrite() + wg.Done() + }() + + wg.Wait() + conn.Close() + channel.Close() +} diff --git a/vendor/golang.org/x/crypto/ssh/agent/keyring.go b/vendor/golang.org/x/crypto/ssh/agent/keyring.go new file mode 100644 index 000000000..c9d979430 --- /dev/null +++ b/vendor/golang.org/x/crypto/ssh/agent/keyring.go @@ -0,0 +1,241 @@ +// Copyright 2014 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package agent + +import ( + "bytes" + "crypto/rand" + "crypto/subtle" + "errors" + "fmt" + "sync" + "time" + + "golang.org/x/crypto/ssh" +) + +type privKey struct { + signer ssh.Signer + comment string + expire *time.Time +} + +type keyring struct { + mu sync.Mutex + keys []privKey + + locked bool + passphrase []byte +} + +var errLocked = errors.New("agent: locked") + +// NewKeyring returns an Agent that holds keys in memory. It is safe +// for concurrent use by multiple goroutines. +func NewKeyring() Agent { + return &keyring{} +} + +// RemoveAll removes all identities. +func (r *keyring) RemoveAll() error { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + return errLocked + } + + r.keys = nil + return nil +} + +// removeLocked does the actual key removal. The caller must already be holding the +// keyring mutex. +func (r *keyring) removeLocked(want []byte) error { + found := false + for i := 0; i < len(r.keys); { + if bytes.Equal(r.keys[i].signer.PublicKey().Marshal(), want) { + found = true + r.keys[i] = r.keys[len(r.keys)-1] + r.keys = r.keys[:len(r.keys)-1] + continue + } else { + i++ + } + } + + if !found { + return errors.New("agent: key not found") + } + return nil +} + +// Remove removes all identities with the given public key. +func (r *keyring) Remove(key ssh.PublicKey) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + return errLocked + } + + return r.removeLocked(key.Marshal()) +} + +// Lock locks the agent. Sign and Remove will fail, and List will return an empty list. +func (r *keyring) Lock(passphrase []byte) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + return errLocked + } + + r.locked = true + r.passphrase = passphrase + return nil +} + +// Unlock undoes the effect of Lock +func (r *keyring) Unlock(passphrase []byte) error { + r.mu.Lock() + defer r.mu.Unlock() + if !r.locked { + return errors.New("agent: not locked") + } + if 1 != subtle.ConstantTimeCompare(passphrase, r.passphrase) { + return fmt.Errorf("agent: incorrect passphrase") + } + + r.locked = false + r.passphrase = nil + return nil +} + +// expireKeysLocked removes expired keys from the keyring. If a key was added +// with a lifetimesecs contraint and seconds >= lifetimesecs seconds have +// ellapsed, it is removed. The caller *must* be holding the keyring mutex. +func (r *keyring) expireKeysLocked() { + for _, k := range r.keys { + if k.expire != nil && time.Now().After(*k.expire) { + r.removeLocked(k.signer.PublicKey().Marshal()) + } + } +} + +// List returns the identities known to the agent. +func (r *keyring) List() ([]*Key, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + // section 2.7: locked agents return empty. + return nil, nil + } + + r.expireKeysLocked() + var ids []*Key + for _, k := range r.keys { + pub := k.signer.PublicKey() + ids = append(ids, &Key{ + Format: pub.Type(), + Blob: pub.Marshal(), + Comment: k.comment}) + } + return ids, nil +} + +// Insert adds a private key to the keyring. If a certificate +// is given, that certificate is added as public key. Note that +// any constraints given are ignored. +func (r *keyring) Add(key AddedKey) error { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + return errLocked + } + signer, err := ssh.NewSignerFromKey(key.PrivateKey) + + if err != nil { + return err + } + + if cert := key.Certificate; cert != nil { + signer, err = ssh.NewCertSigner(cert, signer) + if err != nil { + return err + } + } + + p := privKey{ + signer: signer, + comment: key.Comment, + } + + if key.LifetimeSecs > 0 { + t := time.Now().Add(time.Duration(key.LifetimeSecs) * time.Second) + p.expire = &t + } + + r.keys = append(r.keys, p) + + return nil +} + +// Sign returns a signature for the data. +func (r *keyring) Sign(key ssh.PublicKey, data []byte) (*ssh.Signature, error) { + return r.SignWithFlags(key, data, 0) +} + +func (r *keyring) SignWithFlags(key ssh.PublicKey, data []byte, flags SignatureFlags) (*ssh.Signature, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + return nil, errLocked + } + + r.expireKeysLocked() + wanted := key.Marshal() + for _, k := range r.keys { + if bytes.Equal(k.signer.PublicKey().Marshal(), wanted) { + if flags == 0 { + return k.signer.Sign(rand.Reader, data) + } else { + if algorithmSigner, ok := k.signer.(ssh.AlgorithmSigner); !ok { + return nil, fmt.Errorf("agent: signature does not support non-default signature algorithm: %T", k.signer) + } else { + var algorithm string + switch flags { + case SignatureFlagRsaSha256: + algorithm = ssh.SigAlgoRSASHA2256 + case SignatureFlagRsaSha512: + algorithm = ssh.SigAlgoRSASHA2512 + default: + return nil, fmt.Errorf("agent: unsupported signature flags: %d", flags) + } + return algorithmSigner.SignWithAlgorithm(rand.Reader, data, algorithm) + } + } + } + } + return nil, errors.New("not found") +} + +// Signers returns signers for all the known keys. +func (r *keyring) Signers() ([]ssh.Signer, error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.locked { + return nil, errLocked + } + + r.expireKeysLocked() + s := make([]ssh.Signer, 0, len(r.keys)) + for _, k := range r.keys { + s = append(s, k.signer) + } + return s, nil +} + +// The keyring does not support any extensions +func (r *keyring) Extension(extensionType string, contents []byte) ([]byte, error) { + return nil, ErrExtensionUnsupported +} diff --git a/vendor/golang.org/x/crypto/ssh/agent/server.go b/vendor/golang.org/x/crypto/ssh/agent/server.go new file mode 100644 index 000000000..6e7a1e02f --- /dev/null +++ b/vendor/golang.org/x/crypto/ssh/agent/server.go @@ -0,0 +1,570 @@ +// Copyright 2012 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package agent + +import ( + "crypto/dsa" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rsa" + "encoding/binary" + "errors" + "fmt" + "io" + "log" + "math/big" + + "golang.org/x/crypto/ed25519" + "golang.org/x/crypto/ssh" +) + +// Server wraps an Agent and uses it to implement the agent side of +// the SSH-agent, wire protocol. +type server struct { + agent Agent +} + +func (s *server) processRequestBytes(reqData []byte) []byte { + rep, err := s.processRequest(reqData) + if err != nil { + if err != errLocked { + // TODO(hanwen): provide better logging interface? + log.Printf("agent %d: %v", reqData[0], err) + } + return []byte{agentFailure} + } + + if err == nil && rep == nil { + return []byte{agentSuccess} + } + + return ssh.Marshal(rep) +} + +func marshalKey(k *Key) []byte { + var record struct { + Blob []byte + Comment string + } + record.Blob = k.Marshal() + record.Comment = k.Comment + + return ssh.Marshal(&record) +} + +// See [PROTOCOL.agent], section 2.5.1. +const agentV1IdentitiesAnswer = 2 + +type agentV1IdentityMsg struct { + Numkeys uint32 `sshtype:"2"` +} + +type agentRemoveIdentityMsg struct { + KeyBlob []byte `sshtype:"18"` +} + +type agentLockMsg struct { + Passphrase []byte `sshtype:"22"` +} + +type agentUnlockMsg struct { + Passphrase []byte `sshtype:"23"` +} + +func (s *server) processRequest(data []byte) (interface{}, error) { + switch data[0] { + case agentRequestV1Identities: + return &agentV1IdentityMsg{0}, nil + + case agentRemoveAllV1Identities: + return nil, nil + + case agentRemoveIdentity: + var req agentRemoveIdentityMsg + if err := ssh.Unmarshal(data, &req); err != nil { + return nil, err + } + + var wk wireKey + if err := ssh.Unmarshal(req.KeyBlob, &wk); err != nil { + return nil, err + } + + return nil, s.agent.Remove(&Key{Format: wk.Format, Blob: req.KeyBlob}) + + case agentRemoveAllIdentities: + return nil, s.agent.RemoveAll() + + case agentLock: + var req agentLockMsg + if err := ssh.Unmarshal(data, &req); err != nil { + return nil, err + } + + return nil, s.agent.Lock(req.Passphrase) + + case agentUnlock: + var req agentUnlockMsg + if err := ssh.Unmarshal(data, &req); err != nil { + return nil, err + } + return nil, s.agent.Unlock(req.Passphrase) + + case agentSignRequest: + var req signRequestAgentMsg + if err := ssh.Unmarshal(data, &req); err != nil { + return nil, err + } + + var wk wireKey + if err := ssh.Unmarshal(req.KeyBlob, &wk); err != nil { + return nil, err + } + + k := &Key{ + Format: wk.Format, + Blob: req.KeyBlob, + } + + var sig *ssh.Signature + var err error + if extendedAgent, ok := s.agent.(ExtendedAgent); ok { + sig, err = extendedAgent.SignWithFlags(k, req.Data, SignatureFlags(req.Flags)) + } else { + sig, err = s.agent.Sign(k, req.Data) + } + + if err != nil { + return nil, err + } + return &signResponseAgentMsg{SigBlob: ssh.Marshal(sig)}, nil + + case agentRequestIdentities: + keys, err := s.agent.List() + if err != nil { + return nil, err + } + + rep := identitiesAnswerAgentMsg{ + NumKeys: uint32(len(keys)), + } + for _, k := range keys { + rep.Keys = append(rep.Keys, marshalKey(k)...) + } + return rep, nil + + case agentAddIDConstrained, agentAddIdentity: + return nil, s.insertIdentity(data) + + case agentExtension: + // Return a stub object where the whole contents of the response gets marshaled. + var responseStub struct { + Rest []byte `ssh:"rest"` + } + + if extendedAgent, ok := s.agent.(ExtendedAgent); !ok { + // If this agent doesn't implement extensions, [PROTOCOL.agent] section 4.7 + // requires that we return a standard SSH_AGENT_FAILURE message. + responseStub.Rest = []byte{agentFailure} + } else { + var req extensionAgentMsg + if err := ssh.Unmarshal(data, &req); err != nil { + return nil, err + } + res, err := extendedAgent.Extension(req.ExtensionType, req.Contents) + if err != nil { + // If agent extensions are unsupported, return a standard SSH_AGENT_FAILURE + // message as required by [PROTOCOL.agent] section 4.7. + if err == ErrExtensionUnsupported { + responseStub.Rest = []byte{agentFailure} + } else { + // As the result of any other error processing an extension request, + // [PROTOCOL.agent] section 4.7 requires that we return a + // SSH_AGENT_EXTENSION_FAILURE code. + responseStub.Rest = []byte{agentExtensionFailure} + } + } else { + if len(res) == 0 { + return nil, nil + } + responseStub.Rest = res + } + } + + return responseStub, nil + } + + return nil, fmt.Errorf("unknown opcode %d", data[0]) +} + +func parseConstraints(constraints []byte) (lifetimeSecs uint32, confirmBeforeUse bool, extensions []ConstraintExtension, err error) { + for len(constraints) != 0 { + switch constraints[0] { + case agentConstrainLifetime: + lifetimeSecs = binary.BigEndian.Uint32(constraints[1:5]) + constraints = constraints[5:] + case agentConstrainConfirm: + confirmBeforeUse = true + constraints = constraints[1:] + case agentConstrainExtension: + var msg constrainExtensionAgentMsg + if err = ssh.Unmarshal(constraints, &msg); err != nil { + return 0, false, nil, err + } + extensions = append(extensions, ConstraintExtension{ + ExtensionName: msg.ExtensionName, + ExtensionDetails: msg.ExtensionDetails, + }) + constraints = msg.Rest + default: + return 0, false, nil, fmt.Errorf("unknown constraint type: %d", constraints[0]) + } + } + return +} + +func setConstraints(key *AddedKey, constraintBytes []byte) error { + lifetimeSecs, confirmBeforeUse, constraintExtensions, err := parseConstraints(constraintBytes) + if err != nil { + return err + } + + key.LifetimeSecs = lifetimeSecs + key.ConfirmBeforeUse = confirmBeforeUse + key.ConstraintExtensions = constraintExtensions + return nil +} + +func parseRSAKey(req []byte) (*AddedKey, error) { + var k rsaKeyMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + if k.E.BitLen() > 30 { + return nil, errors.New("agent: RSA public exponent too large") + } + priv := &rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + E: int(k.E.Int64()), + N: k.N, + }, + D: k.D, + Primes: []*big.Int{k.P, k.Q}, + } + priv.Precompute() + + addedKey := &AddedKey{PrivateKey: priv, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func parseEd25519Key(req []byte) (*AddedKey, error) { + var k ed25519KeyMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + priv := ed25519.PrivateKey(k.Priv) + + addedKey := &AddedKey{PrivateKey: &priv, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func parseDSAKey(req []byte) (*AddedKey, error) { + var k dsaKeyMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + priv := &dsa.PrivateKey{ + PublicKey: dsa.PublicKey{ + Parameters: dsa.Parameters{ + P: k.P, + Q: k.Q, + G: k.G, + }, + Y: k.Y, + }, + X: k.X, + } + + addedKey := &AddedKey{PrivateKey: priv, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func unmarshalECDSA(curveName string, keyBytes []byte, privScalar *big.Int) (priv *ecdsa.PrivateKey, err error) { + priv = &ecdsa.PrivateKey{ + D: privScalar, + } + + switch curveName { + case "nistp256": + priv.Curve = elliptic.P256() + case "nistp384": + priv.Curve = elliptic.P384() + case "nistp521": + priv.Curve = elliptic.P521() + default: + return nil, fmt.Errorf("agent: unknown curve %q", curveName) + } + + priv.X, priv.Y = elliptic.Unmarshal(priv.Curve, keyBytes) + if priv.X == nil || priv.Y == nil { + return nil, errors.New("agent: point not on curve") + } + + return priv, nil +} + +func parseEd25519Cert(req []byte) (*AddedKey, error) { + var k ed25519CertMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + pubKey, err := ssh.ParsePublicKey(k.CertBytes) + if err != nil { + return nil, err + } + priv := ed25519.PrivateKey(k.Priv) + cert, ok := pubKey.(*ssh.Certificate) + if !ok { + return nil, errors.New("agent: bad ED25519 certificate") + } + + addedKey := &AddedKey{PrivateKey: &priv, Certificate: cert, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func parseECDSAKey(req []byte) (*AddedKey, error) { + var k ecdsaKeyMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + + priv, err := unmarshalECDSA(k.Curve, k.KeyBytes, k.D) + if err != nil { + return nil, err + } + + addedKey := &AddedKey{PrivateKey: priv, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func parseRSACert(req []byte) (*AddedKey, error) { + var k rsaCertMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + + pubKey, err := ssh.ParsePublicKey(k.CertBytes) + if err != nil { + return nil, err + } + + cert, ok := pubKey.(*ssh.Certificate) + if !ok { + return nil, errors.New("agent: bad RSA certificate") + } + + // An RSA publickey as marshaled by rsaPublicKey.Marshal() in keys.go + var rsaPub struct { + Name string + E *big.Int + N *big.Int + } + if err := ssh.Unmarshal(cert.Key.Marshal(), &rsaPub); err != nil { + return nil, fmt.Errorf("agent: Unmarshal failed to parse public key: %v", err) + } + + if rsaPub.E.BitLen() > 30 { + return nil, errors.New("agent: RSA public exponent too large") + } + + priv := rsa.PrivateKey{ + PublicKey: rsa.PublicKey{ + E: int(rsaPub.E.Int64()), + N: rsaPub.N, + }, + D: k.D, + Primes: []*big.Int{k.Q, k.P}, + } + priv.Precompute() + + addedKey := &AddedKey{PrivateKey: &priv, Certificate: cert, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func parseDSACert(req []byte) (*AddedKey, error) { + var k dsaCertMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + pubKey, err := ssh.ParsePublicKey(k.CertBytes) + if err != nil { + return nil, err + } + cert, ok := pubKey.(*ssh.Certificate) + if !ok { + return nil, errors.New("agent: bad DSA certificate") + } + + // A DSA publickey as marshaled by dsaPublicKey.Marshal() in keys.go + var w struct { + Name string + P, Q, G, Y *big.Int + } + if err := ssh.Unmarshal(cert.Key.Marshal(), &w); err != nil { + return nil, fmt.Errorf("agent: Unmarshal failed to parse public key: %v", err) + } + + priv := &dsa.PrivateKey{ + PublicKey: dsa.PublicKey{ + Parameters: dsa.Parameters{ + P: w.P, + Q: w.Q, + G: w.G, + }, + Y: w.Y, + }, + X: k.X, + } + + addedKey := &AddedKey{PrivateKey: priv, Certificate: cert, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func parseECDSACert(req []byte) (*AddedKey, error) { + var k ecdsaCertMsg + if err := ssh.Unmarshal(req, &k); err != nil { + return nil, err + } + + pubKey, err := ssh.ParsePublicKey(k.CertBytes) + if err != nil { + return nil, err + } + cert, ok := pubKey.(*ssh.Certificate) + if !ok { + return nil, errors.New("agent: bad ECDSA certificate") + } + + // An ECDSA publickey as marshaled by ecdsaPublicKey.Marshal() in keys.go + var ecdsaPub struct { + Name string + ID string + Key []byte + } + if err := ssh.Unmarshal(cert.Key.Marshal(), &ecdsaPub); err != nil { + return nil, err + } + + priv, err := unmarshalECDSA(ecdsaPub.ID, ecdsaPub.Key, k.D) + if err != nil { + return nil, err + } + + addedKey := &AddedKey{PrivateKey: priv, Certificate: cert, Comment: k.Comments} + if err := setConstraints(addedKey, k.Constraints); err != nil { + return nil, err + } + return addedKey, nil +} + +func (s *server) insertIdentity(req []byte) error { + var record struct { + Type string `sshtype:"17|25"` + Rest []byte `ssh:"rest"` + } + + if err := ssh.Unmarshal(req, &record); err != nil { + return err + } + + var addedKey *AddedKey + var err error + + switch record.Type { + case ssh.KeyAlgoRSA: + addedKey, err = parseRSAKey(req) + case ssh.KeyAlgoDSA: + addedKey, err = parseDSAKey(req) + case ssh.KeyAlgoECDSA256, ssh.KeyAlgoECDSA384, ssh.KeyAlgoECDSA521: + addedKey, err = parseECDSAKey(req) + case ssh.KeyAlgoED25519: + addedKey, err = parseEd25519Key(req) + case ssh.CertAlgoRSAv01: + addedKey, err = parseRSACert(req) + case ssh.CertAlgoDSAv01: + addedKey, err = parseDSACert(req) + case ssh.CertAlgoECDSA256v01, ssh.CertAlgoECDSA384v01, ssh.CertAlgoECDSA521v01: + addedKey, err = parseECDSACert(req) + case ssh.CertAlgoED25519v01: + addedKey, err = parseEd25519Cert(req) + default: + return fmt.Errorf("agent: not implemented: %q", record.Type) + } + + if err != nil { + return err + } + return s.agent.Add(*addedKey) +} + +// ServeAgent serves the agent protocol on the given connection. It +// returns when an I/O error occurs. +func ServeAgent(agent Agent, c io.ReadWriter) error { + s := &server{agent} + + var length [4]byte + for { + if _, err := io.ReadFull(c, length[:]); err != nil { + return err + } + l := binary.BigEndian.Uint32(length[:]) + if l == 0 { + return fmt.Errorf("agent: request size is 0") + } + if l > maxAgentResponseBytes { + // We also cap requests. + return fmt.Errorf("agent: request too large: %d", l) + } + + req := make([]byte, l) + if _, err := io.ReadFull(c, req); err != nil { + return err + } + + repData := s.processRequestBytes(req) + if len(repData) > maxAgentResponseBytes { + return fmt.Errorf("agent: reply too large: %d bytes", len(repData)) + } + + binary.BigEndian.PutUint32(length[:], uint32(len(repData))) + if _, err := c.Write(length[:]); err != nil { + return err + } + if _, err := c.Write(repData); err != nil { + return err + } + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index b84d9e017..e7232d5a2 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -154,7 +154,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.20.1 +# github.com/containers/storage v1.20.2 github.com/containers/storage github.com/containers/storage/drivers github.com/containers/storage/drivers/aufs @@ -324,7 +324,7 @@ github.com/inconshreveable/mousetrap github.com/ishidawataru/sctp # github.com/json-iterator/go v1.1.9 github.com/json-iterator/go -# github.com/klauspost/compress v1.10.6 +# github.com/klauspost/compress v1.10.7 github.com/klauspost/compress/flate github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 @@ -575,6 +575,7 @@ golang.org/x/crypto/openpgp/s2k golang.org/x/crypto/pbkdf2 golang.org/x/crypto/poly1305 golang.org/x/crypto/ssh +golang.org/x/crypto/ssh/agent golang.org/x/crypto/ssh/internal/bcrypt_pbkdf golang.org/x/crypto/ssh/terminal # golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7 |