diff options
96 files changed, 1105 insertions, 454 deletions
diff --git a/CODE-OF-CONDUCT.md b/CODE-OF-CONDUCT.md index b23672b1a..e418693f7 100644 --- a/CODE-OF-CONDUCT.md +++ b/CODE-OF-CONDUCT.md @@ -1,3 +1,3 @@ ## The Podman Project Community Code of Conduct -The Podman project which includes Libpod, follows the [Containers Community Code of Conduct](https://github.com/containers/common/blob/master/CODE-OF-CONDUCT.md). +The Podman project which includes Libpod, follows the [Containers Community Code of Conduct](https://github.com/containers/common/blob/main/CODE-OF-CONDUCT.md). @@ -389,10 +389,10 @@ cross: local-cross .PHONY: nixpkgs nixpkgs: @nix run \ - -f channel:nixos-20.09 nix-prefetch-git \ + -f channel:nixos-21.05 nix-prefetch-git \ -c nix-prefetch-git \ --no-deepClone \ - https://github.com/nixos/nixpkgs refs/heads/nixos-20.09 > nix/nixpkgs.json + https://github.com/nixos/nixpkgs refs/heads/nixos-21.05 > nix/nixpkgs.json # Build statically linked binary .PHONY: static diff --git a/SECURITY.md b/SECURITY.md index 03a192044..1f6d5088d 100644 --- a/SECURITY.md +++ b/SECURITY.md @@ -1,3 +1,3 @@ ## Security and Disclosure Information Policy for the Libpod Project -The Libpod Project follows the [Security and Disclosure Information Policy](https://github.com/containers/common/blob/master/SECURITY.md) for the Containers Projects. +The Libpod Project follows the [Security and Disclosure Information Policy](https://github.com/containers/common/blob/main/SECURITY.md) for the Containers Projects. diff --git a/cmd/podman/common/create_opts.go b/cmd/podman/common/create_opts.go index 76d7345fc..66778f519 100644 --- a/cmd/podman/common/create_opts.go +++ b/cmd/podman/common/create_opts.go @@ -249,7 +249,7 @@ func ContainerCreateToContainerCLIOpts(cc handlers.CreateContainerConfig, rtc *c } // netMode - nsmode, _, err := specgen.ParseNetworkNamespace(string(cc.HostConfig.NetworkMode), true) + nsmode, networks, err := specgen.ParseNetworkNamespace(string(cc.HostConfig.NetworkMode), true) if err != nil { return nil, nil, err } @@ -322,7 +322,7 @@ func ContainerCreateToContainerCLIOpts(cc handlers.CreateContainerConfig, rtc *c netInfo.Aliases = aliases netInfo.CNINetworks = cniNetworks case len(cc.HostConfig.NetworkMode) > 0: - netInfo.CNINetworks = []string{string(cc.HostConfig.NetworkMode)} + netInfo.CNINetworks = networks } parsedTmp := make([]string, 0, len(cc.HostConfig.Tmpfs)) diff --git a/cmd/podman/images/prune.go b/cmd/podman/images/prune.go index a082255f6..6ecf4f2aa 100644 --- a/cmd/podman/images/prune.go +++ b/cmd/podman/images/prune.go @@ -59,7 +59,15 @@ func prune(cmd *cobra.Command, args []string) error { return nil } } - + filterMap, err := common.ParseFilters(filter) + if err != nil { + return err + } + for k, v := range filterMap { + for _, val := range v { + pruneOpts.Filter = append(pruneOpts.Filter, fmt.Sprintf("%s=%s", k, val)) + } + } results, err := registry.ImageEngine().Prune(registry.GetContext(), pruneOpts) if err != nil { return err diff --git a/docs/source/markdown/podman-manifest-rm.1.md b/docs/source/markdown/podman-manifest-rm.1.md index 396dd49c7..a65f32936 100644 --- a/docs/source/markdown/podman-manifest-rm.1.md +++ b/docs/source/markdown/podman-manifest-rm.1.md @@ -11,7 +11,7 @@ Removes one or more locally stored manifest lists. ## EXAMPLE -podman manifest rm <list> +podman manifest rm `<list>` podman manifest rm listid1 listid2 diff --git a/docs/source/markdown/podman-network-create.1.md b/docs/source/markdown/podman-network-create.1.md index 3d5d98055..d110c4ceb 100644 --- a/docs/source/markdown/podman-network-create.1.md +++ b/docs/source/markdown/podman-network-create.1.md @@ -9,7 +9,7 @@ podman\-network-create - Create a Podman CNI network ## DESCRIPTION Create a CNI-network configuration for use with Podman. By default, Podman creates a bridge connection. A *Macvlan* connection can be created with the *-d macvlan* option. A parent device for macvlan can -be designated with the *-o parent=\<device>* option. In the case of *Macvlan* connections, the +be designated with the *-o parent=`<device>`* option. In the case of *Macvlan* connections, the CNI *dhcp* plugin needs to be activated or the container image must have a DHCP client to interact with the host network's DHCP server. diff --git a/docs/source/markdown/podman-pod-create.1.md b/docs/source/markdown/podman-pod-create.1.md index 37eb098d1..4b890a7af 100644 --- a/docs/source/markdown/podman-pod-create.1.md +++ b/docs/source/markdown/podman-pod-create.1.md @@ -10,8 +10,8 @@ podman\-pod\-create - Create a new pod Creates an empty pod, or unit of multiple containers, and prepares it to have containers added to it. The pod id is printed to STDOUT. You can then use -**podman create --pod \<pod_id|pod_name\> ...** to add containers to the pod, and -**podman pod start \<pod_id|pod_name\>** to start the pod. +**podman create --pod `<pod_id|pod_name>` ...** to add containers to the pod, and +**podman pod start `<pod_id|pod_name>`** to start the pod. ## OPTIONS diff --git a/docs/tutorials/mac_experimental.md b/docs/tutorials/mac_experimental.md new file mode 100644 index 000000000..8df64dc99 --- /dev/null +++ b/docs/tutorials/mac_experimental.md @@ -0,0 +1,99 @@ +# Using podman-machine on MacOS (x86_64 and Apple silicon) + +## Setup + +You must obtain a compressed tarball that contains the following: +* a qcow image +* a podman binary +* a gvproxy binary + +You must also have installed brew prior to following this process. See https://brew.sh/ for +installation instructions. + +Note: If your user has admin rights, you can ignore the use of `sudo` in these instructions. + + +1. Install qemu from brew to obtain the required runtime dependencies. + + ``` + brew install qemu + ``` + +2. If you are running MacOS on the Intel architecture, you can skip to step 8. +3. Uninstall the brew package + + ``` + brew uninstall qemu + ``` + +4. Get upstream qemu source code. + + ``` + git clone https://github.com/qemu/qemu + ``` + +5. Apply patches that have not been merged into upstream qemu. + + ``` + cd qemu + git config user.name "YOUR_NAME" + git config user.email johndoe@example.com + git checkout v5.2.0 + curl https://patchwork.kernel.org/series/418581/mbox/ | git am --exclude=MAINTAINERS + curl -L https://gist.github.com/citruz/9896cd6fb63288ac95f81716756cb9aa/raw/2d613e9a003b28dfe688f33055706d3873025a40/xcode-12-4.patch | git apply - + ``` + +6. Install qemu build dependencies + + ``` + brew install libffi gettext pkg-config autoconf automake pixman ninja make + ``` + +7. Configure, compile, and install qemu + ``` + mkdir build + cd build + ../configure --target-list=aarch64-softmmu --disable-gnutls + gmake -j8 + sudo gmake install + ``` + + +8. Uncompress and place provided binaries into filesystem + + **Note**: In the following instructions, you need to know the name of the compressed file +that you were given. It will be used in two of the steps below. + + ``` + cd ~ + tar xvf `compressed_file_ending_in_xz` + sudo cp -v `unpacked_directory`/{gvproxy,podman} /usr/local/bin + ``` + +9. Sign all binaries + + If you have a Mac with Apple Silicon, issue the following command: + ``` + sudo codesign --entitlements ~/qemu/accel/hvf/entitlements.plist --force -s - /usr/local/bin/qemu-* /usr/local/bin/gvproxy /usr/local/bin/podman + ``` + + If you have a Mac with an Intel processor, issue the following command: + + ``` + echo '<?xml version="1.0" encoding="utf-8"?> + <!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd"> + <plist version="1.0"> <dict> <key>com.apple.security.hypervisor</key> <true/> </dict> </plist> + ' > ~/entitlements.plist + sudo codesign --entitlements ~/entitlements.plist --force -s - /usr/local/bin/qemu-* /usr/local/bin/gvproxy /usr/local/bin/podman + ``` + + +## Test podman + +1. podman machine init --image-path /path/to/image +2. podman machine start +3. podman images +4. git clone http://github.com/baude/alpine_nginx && cd alpine_nginx +5. podman build -t alpine_nginx . +4. podman run -dt -p 9999:80 alpine_nginx +5. curl http://localhost:9999 @@ -11,13 +11,13 @@ require ( github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9 github.com/containernetworking/cni v0.8.1 github.com/containernetworking/plugins v0.9.1 - github.com/containers/buildah v1.21.0 + github.com/containers/buildah v1.21.1 github.com/containers/common v0.39.1-0.20210527140106-e5800a20386a github.com/containers/conmon v2.0.20+incompatible github.com/containers/image/v5 v5.12.0 github.com/containers/ocicrypt v1.1.1 github.com/containers/psgo v1.5.2 - github.com/containers/storage v1.32.1 + github.com/containers/storage v1.32.2 github.com/coreos/go-systemd/v22 v22.3.2 github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3 github.com/cri-o/ocicni v0.2.1-0.20210301205850-541cf7c703cf @@ -218,8 +218,8 @@ github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHV github.com/containernetworking/plugins v0.8.7/go.mod h1:R7lXeZaBzpfqapcAbHRW8/CYwm0dHzbz0XEjofx0uB0= github.com/containernetworking/plugins v0.9.1 h1:FD1tADPls2EEi3flPc2OegIY1M9pUa9r2Quag7HMLV8= github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= -github.com/containers/buildah v1.21.0 h1:LuwuqRPjan3X3AIdGwfkEkqMgmrDMNpQznFqNdHgCz8= -github.com/containers/buildah v1.21.0/go.mod h1:yPdlpVd93T+i91yGxrJbW1YOWrqN64j5ZhHOZmHUejs= +github.com/containers/buildah v1.21.1 h1:e9LmTCUKUBLg72v5DnIOT/wc8ffkfB7LbpQBywLZo20= +github.com/containers/buildah v1.21.1/go.mod h1:yPdlpVd93T+i91yGxrJbW1YOWrqN64j5ZhHOZmHUejs= github.com/containers/common v0.38.4/go.mod h1:egfpX/Y3+19Dz4Wa1eRZDdgzoEOeneieF9CQppKzLBg= github.com/containers/common v0.39.1-0.20210527140106-e5800a20386a h1:XzYOUf7qjgVJ59YGqAzehlbT63EgjUJhMnfhsPSSJV0= github.com/containers/common v0.39.1-0.20210527140106-e5800a20386a/go.mod h1:CxHAf4iQOZZ8nASIjMdYHHRyA8dMR4tINSS7WQWlv90= @@ -239,8 +239,8 @@ github.com/containers/storage v1.23.5/go.mod h1:ha26Q6ngehFNhf3AWoXldvAvwI4jFe3E github.com/containers/storage v1.30.1/go.mod h1:NDJkiwxnSHD1Is+4DGcyR3SIEYSDOa0xnAW+uGQFx9E= github.com/containers/storage v1.31.1/go.mod h1:IFEf+yRTS0pvCGQt2tBv1Kzz2XUSPvED6uFBmWG7V/E= github.com/containers/storage v1.32.0/go.mod h1:J3q772EVbN9vgqoN/dkvInKnp4xK9ZXm7wHNfuiIDgE= -github.com/containers/storage v1.32.1 h1:JgvHY5dokiff+Ee4TdvPYO++Oq2BAave5DmyPetH2iU= -github.com/containers/storage v1.32.1/go.mod h1:do6oIF71kfkVS3CPUZr+6He94fIaj6pzF8ywevPuuOw= +github.com/containers/storage v1.32.2 h1:V1oKAKmH5e6OTgP7Uf8+T+ntVVYk2MNFnDDkPBI9kxU= +github.com/containers/storage v1.32.2/go.mod h1:YIBxxjfXZTi04Ah49sh1uSGfmT1V89+I5i3deRobzQo= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -536,8 +536,8 @@ github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= -github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.0 h1:2T7tUoQrQT+fQWdaY5rjWztFGAFwbGD04iPJg90ZiOs= +github.com/klauspost/compress v1.13.0/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -576,8 +576,9 @@ github.com/mattn/go-runewidth v0.0.12 h1:Y41i/hVW3Pgwr8gV+J23B9YEY0zxjptBuCWEaxm github.com/mattn/go-runewidth v0.0.12/go.mod h1:RAqKPSqVFrSLVXbA8x7dzmKdmGzieGRCM46jaSJTDAk= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= github.com/mattn/go-shellwords v1.0.10/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= -github.com/mattn/go-shellwords v1.0.11 h1:vCoR9VPpsk/TZFW2JwK5I9S0xdrtUq2bph6/YjEPnaw= github.com/mattn/go-shellwords v1.0.11/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= +github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 h1:I0XW9+e1XWDxdcEniV4rQAIOPUGDq67JSCiRCgGCZLI= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= diff --git a/libpod/container.go b/libpod/container.go index c6f0cd618..4b9bea5fc 100644 --- a/libpod/container.go +++ b/libpod/container.go @@ -957,6 +957,12 @@ func (c *Container) cGroupPath() (string, error) { // is the libpod-specific one we're looking for. // // See #8397 on the need for the longest-path look up. + // + // And another workaround for containers running systemd as the payload. + // containers running systemd moves themselves into a child subgroup of + // the named systemd cgroup hierarchy. Ignore any named cgroups during + // the lookup. + // See #10602 for more details. procPath := fmt.Sprintf("/proc/%d/cgroup", c.state.PID) lines, err := ioutil.ReadFile(procPath) if err != nil { @@ -972,6 +978,10 @@ func (c *Container) cGroupPath() (string, error) { logrus.Debugf("Error parsing cgroup: expected 3 fields but got %d: %s", len(fields), procPath) continue } + // Ignore named cgroups like name=systemd. + if bytes.Contains(fields[1], []byte("=")) { + continue + } path := string(fields[2]) if len(path) > len(cgroupPath) { cgroupPath = path diff --git a/libpod/container_exec.go b/libpod/container_exec.go index c359f1e5d..737bf74ad 100644 --- a/libpod/container_exec.go +++ b/libpod/container_exec.go @@ -1,6 +1,7 @@ package libpod import ( + "context" "io/ioutil" "net/http" "os" @@ -539,18 +540,7 @@ func (c *Container) ExecStop(sessionID string, timeout *uint) error { var cleanupErr error // Retrieve exit code and update status - exitCode, err := c.readExecExitCode(session.ID()) - if err != nil { - cleanupErr = err - } - session.ExitCode = exitCode - session.PID = 0 - session.State = define.ExecStateStopped - - if err := c.save(); err != nil { - if cleanupErr != nil { - logrus.Errorf("Error stopping container %s exec session %s: %v", c.ID(), session.ID(), cleanupErr) - } + if err := retrieveAndWriteExecExitCode(c, session.ID()); err != nil { cleanupErr = err } @@ -592,15 +582,7 @@ func (c *Container) ExecCleanup(sessionID string) error { return errors.Wrapf(define.ErrExecSessionStateInvalid, "cannot clean up container %s exec session %s as it is running", c.ID(), session.ID()) } - exitCode, err := c.readExecExitCode(session.ID()) - if err != nil { - return err - } - session.ExitCode = exitCode - session.PID = 0 - session.State = define.ExecStateStopped - - if err := c.save(); err != nil { + if err := retrieveAndWriteExecExitCode(c, session.ID()); err != nil { return err } } @@ -637,9 +619,9 @@ func (c *Container) ExecRemove(sessionID string, force bool) error { return err } if !running { - session.State = define.ExecStateStopped - // TODO: should we retrieve exit code here? - // TODO: Might be worth saving state here. + if err := retrieveAndWriteExecExitCode(c, session.ID()); err != nil { + return err + } } } @@ -653,6 +635,10 @@ func (c *Container) ExecRemove(sessionID string, force bool) error { return err } + if err := retrieveAndWriteExecExitCode(c, session.ID()); err != nil { + return err + } + if err := c.cleanupExecBundle(session.ID()); err != nil { return err } @@ -757,10 +743,25 @@ func (c *Container) Exec(config *ExecConfig, streams *define.AttachStreams, resi session, err := c.ExecSession(sessionID) if err != nil { + if errors.Cause(err) == define.ErrNoSuchExecSession { + // TODO: If a proper Context is ever plumbed in here, we + // should use it. + // As things stand, though, it's not worth it - this + // should always terminate quickly since it's not + // streaming. + diedEvent, err := c.runtime.GetExecDiedEvent(context.Background(), c.ID(), sessionID) + if err != nil { + return -1, errors.Wrapf(err, "error retrieving exec session %s exit code", sessionID) + } + return diedEvent.ContainerExitCode, nil + } return -1, err } exitCode := session.ExitCode if err := c.ExecRemove(sessionID, false); err != nil { + if errors.Cause(err) == define.ErrNoSuchExecSession { + return exitCode, nil + } return -1, err } @@ -927,6 +928,8 @@ func (c *Container) getActiveExecSessions() ([]string, error) { session.PID = 0 session.State = define.ExecStateStopped + c.newExecDiedEvent(session.ID(), exitCode) + needSave = true } if err := c.cleanupExecBundle(id); err != nil { @@ -1036,6 +1039,22 @@ func writeExecExitCode(c *Container, sessionID string, exitCode int) error { return errors.Wrapf(err, "error syncing container %s state to remove exec session %s", c.ID(), sessionID) } + return justWriteExecExitCode(c, sessionID, exitCode) +} + +func retrieveAndWriteExecExitCode(c *Container, sessionID string) error { + exitCode, err := c.readExecExitCode(sessionID) + if err != nil { + return err + } + + return justWriteExecExitCode(c, sessionID, exitCode) +} + +func justWriteExecExitCode(c *Container, sessionID string, exitCode int) error { + // Write an event first + c.newExecDiedEvent(sessionID, exitCode) + session, ok := c.state.ExecSessions[sessionID] if !ok { // Exec session already removed. diff --git a/libpod/container_internal.go b/libpod/container_internal.go index f77825efd..3e4eea003 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -42,6 +42,7 @@ const ( // name of the directory holding the artifacts artifactsDir = "artifacts" execDirPermission = 0755 + preCheckpointDir = "pre-checkpoint" ) // rootFsSize gets the size of the container's root filesystem @@ -141,7 +142,7 @@ func (c *Container) CheckpointPath() string { // PreCheckpointPath returns the path to the directory containing the pre-checkpoint-images func (c *Container) PreCheckPointPath() string { - return filepath.Join(c.bundlePath(), "pre-checkpoint") + return filepath.Join(c.bundlePath(), preCheckpointDir) } // AttachSocketPath retrieves the path of the container's attach socket @@ -427,7 +428,7 @@ func (c *Container) setupStorage(ctx context.Context) error { }, LabelOpts: c.config.LabelOpts, } - if c.restoreFromCheckpoint { + if c.restoreFromCheckpoint && !c.config.Privileged { // If restoring from a checkpoint, the root file-system // needs to be mounted with the same SELinux labels as // it was mounted previously. diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index a3acc3198..ddfccb999 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -909,14 +909,15 @@ func (c *Container) exportCheckpoint(options ContainerCheckpointOptions) error { includeFiles := []string{ "artifacts", "ctr.log", - metadata.CheckpointDirectory, metadata.ConfigDumpFile, metadata.SpecDumpFile, metadata.NetworkStatusFile, } if options.PreCheckPoint { - includeFiles[0] = "pre-checkpoint" + includeFiles = append(includeFiles, preCheckpointDir) + } else { + includeFiles = append(includeFiles, metadata.CheckpointDirectory) } // Get root file-system changes included in the checkpoint archive var addToTarFiles []string @@ -1650,22 +1651,20 @@ func (c *Container) generateResolvConf() (string, error) { } } - // Determine the endpoint for resolv.conf in case it is a symlink - resolvPath, err := filepath.EvalSymlinks(resolvConf) + contents, err := ioutil.ReadFile(resolvConf) // resolv.conf doesn't have to exists if err != nil && !os.IsNotExist(err) { return "", err } - // Determine if symlink points to any of the systemd-resolved files - if strings.HasPrefix(resolvPath, "/run/systemd/resolve/") { - resolvPath = "/run/systemd/resolve/resolv.conf" - } - - contents, err := ioutil.ReadFile(resolvPath) - // resolv.conf doesn't have to exists - if err != nil && !os.IsNotExist(err) { - return "", err + ns := resolvconf.GetNameservers(contents) + // check if systemd-resolved is used, assume it is used when 127.0.0.53 is the only nameserver + if len(ns) == 1 && ns[0] == "127.0.0.53" { + // read the actual resolv.conf file for systemd-resolved + contents, err = ioutil.ReadFile("/run/systemd/resolve/resolv.conf") + if err != nil { + return "", errors.Wrapf(err, "detected that systemd-resolved is in use, but could not locate real resolv.conf") + } } ipv6 := false diff --git a/libpod/container_log.go b/libpod/container_log.go index c207df819..a30e4f5cc 100644 --- a/libpod/container_log.go +++ b/libpod/container_log.go @@ -4,11 +4,10 @@ import ( "context" "fmt" "os" - "time" "github.com/containers/podman/v3/libpod/define" + "github.com/containers/podman/v3/libpod/events" "github.com/containers/podman/v3/libpod/logs" - "github.com/hpcloud/tail/watch" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -94,27 +93,40 @@ func (c *Container) readFromLogFile(ctx context.Context, options *logs.LogOption }() // Check if container is still running or paused if options.Follow { + state, err := c.State() + if err != nil || state != define.ContainerStateRunning { + // If the container isn't running or if we encountered + // an error getting its state, instruct the logger to + // read the file until EOF. + tailError := t.StopAtEOF() + if tailError != nil && fmt.Sprintf("%v", tailError) != "tail: stop at eof" { + logrus.Error(tailError) + } + if errors.Cause(err) != define.ErrNoSuchCtr { + logrus.Error(err) + } + return nil + } + + // The container is running, so we need to wait until the container exited go func() { - for { - state, err := c.State() - time.Sleep(watch.POLL_DURATION) - if err != nil { - tailError := t.StopAtEOF() - if tailError != nil && fmt.Sprintf("%v", tailError) != "tail: stop at eof" { - logrus.Error(tailError) - } - if errors.Cause(err) != define.ErrNoSuchCtr { - logrus.Error(err) - } - break - } - if state != define.ContainerStateRunning && state != define.ContainerStatePaused { - tailError := t.StopAtEOF() - if tailError != nil && fmt.Sprintf("%v", tailError) != "tail: stop at eof" { - logrus.Error(tailError) - } - break + eventChannel := make(chan *events.Event) + eventOptions := events.ReadOptions{ + EventChannel: eventChannel, + Filters: []string{"event=died", "container=" + c.ID()}, + Stream: true, + } + go func() { + if err := c.runtime.Events(ctx, eventOptions); err != nil { + logrus.Errorf("Error waiting for container to exit: %v", err) } + }() + // Now wait for the died event and signal to finish + // reading the log until EOF. + <-eventChannel + tailError := t.StopAtEOF() + if tailError != nil && fmt.Sprintf("%v", tailError) != "tail: stop at eof" { + logrus.Error(tailError) } }() } diff --git a/libpod/events.go b/libpod/events.go index 839229674..22c51aeec 100644 --- a/libpod/events.go +++ b/libpod/events.go @@ -46,7 +46,22 @@ func (c *Container) newContainerExitedEvent(exitCode int32) { e.Type = events.Container e.ContainerExitCode = int(exitCode) if err := c.runtime.eventer.Write(e); err != nil { - logrus.Errorf("unable to write pod event: %q", err) + logrus.Errorf("unable to write container exited event: %q", err) + } +} + +// newExecDiedEvent creates a new event for an exec session's death +func (c *Container) newExecDiedEvent(sessionID string, exitCode int) { + e := events.NewEvent(events.ExecDied) + e.ID = c.ID() + e.Name = c.Name() + e.Image = c.config.RootfsImageName + e.Type = events.Container + e.ContainerExitCode = exitCode + e.Attributes = make(map[string]string) + e.Attributes["execID"] = sessionID + if err := c.runtime.eventer.Write(e); err != nil { + logrus.Errorf("unable to write exec died event: %q", err) } } @@ -154,3 +169,25 @@ func (r *Runtime) GetLastContainerEvent(ctx context.Context, nameOrID string, co // return the last element in the slice return containerEvents[len(containerEvents)-1], nil } + +// GetExecDiedEvent takes a container name or ID, exec session ID, and returns +// that exec session's Died event (if it has already occurred). +func (r *Runtime) GetExecDiedEvent(ctx context.Context, nameOrID, execSessionID string) (*events.Event, error) { + filters := []string{ + fmt.Sprintf("container=%s", nameOrID), + "event=exec_died", + "type=container", + fmt.Sprintf("label=execID=%s", execSessionID), + } + + containerEvents, err := r.GetEvents(ctx, filters) + if err != nil { + return nil, err + } + // There *should* only be one event maximum. + // But... just in case... let's not blow up if there's more than one. + if len(containerEvents) < 1 { + return nil, errors.Wrapf(events.ErrEventNotFound, "exec died event for session %s (container %s) not found", execSessionID, nameOrID) + } + return containerEvents[len(containerEvents)-1], nil +} diff --git a/libpod/events/config.go b/libpod/events/config.go index 085fa9d52..d88d7b6e3 100644 --- a/libpod/events/config.go +++ b/libpod/events/config.go @@ -127,6 +127,8 @@ const ( Create Status = "create" // Exec ... Exec Status = "exec" + // ExecDied indicates that an exec session in a container died. + ExecDied Status = "exec_died" // Exited indicates that a container's process died Exited Status = "died" // Export ... diff --git a/libpod/events/events.go b/libpod/events/events.go index 01ea6a386..e03215eff 100644 --- a/libpod/events/events.go +++ b/libpod/events/events.go @@ -149,6 +149,8 @@ func StringToStatus(name string) (Status, error) { return Create, nil case Exec.String(): return Exec, nil + case ExecDied.String(): + return ExecDied, nil case Exited.String(): return Exited, nil case Export.String(): diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go index c928e02a6..5446841f6 100644 --- a/libpod/networking_linux.go +++ b/libpod/networking_linux.go @@ -1090,7 +1090,7 @@ func (c *Container) NetworkDisconnect(nameOrID, netName string, force bool) erro } c.newNetworkEvent(events.NetworkDisconnect, netName) - if c.state.State != define.ContainerStateRunning { + if !c.ensureState(define.ContainerStateRunning, define.ContainerStateCreated) { return nil } @@ -1145,7 +1145,7 @@ func (c *Container) NetworkConnect(nameOrID, netName string, aliases []string) e return err } c.newNetworkEvent(events.NetworkConnect, netName) - if c.state.State != define.ContainerStateRunning { + if !c.ensureState(define.ContainerStateRunning, define.ContainerStateCreated) { return nil } if c.state.NetNS == nil { diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go index 3da49b85f..2914bd1a1 100644 --- a/libpod/oci_conmon_linux.go +++ b/libpod/oci_conmon_linux.go @@ -787,7 +787,11 @@ func (r *ConmonOCIRuntime) CheckpointContainer(ctr *Container, options Container args = append(args, "--pre-dump") } if !options.PreCheckPoint && options.WithPrevious { - args = append(args, "--parent-path", ctr.PreCheckPointPath()) + args = append( + args, + "--parent-path", + filepath.Join("..", preCheckpointDir), + ) } runtimeDir, err := util.GetRuntimeDir() if err != nil { diff --git a/libpod/options.go b/libpod/options.go index f942d264b..d3be46ad8 100644 --- a/libpod/options.go +++ b/libpod/options.go @@ -1641,6 +1641,19 @@ func WithVolumeGID(gid int) VolumeCreateOption { } } +// WithVolumeNoChown prevents the volume from being chowned to the process uid at first use. +func WithVolumeNoChown() VolumeCreateOption { + return func(volume *Volume) error { + if volume.valid { + return define.ErrVolumeFinalized + } + + volume.state.NeedsChown = false + + return nil + } +} + // withSetAnon sets a bool notifying libpod that this volume is anonymous and // should be removed when containers using it are removed and volumes are // specified for removal. diff --git a/libpod/volume_internal.go b/libpod/volume_internal.go index 694cdd149..19008a253 100644 --- a/libpod/volume_internal.go +++ b/libpod/volume_internal.go @@ -39,8 +39,23 @@ func (v *Volume) needsMount() bool { return true } - // Local driver with options needs mount - return len(v.config.Options) > 0 + // Commit 28138dafcc added the UID and GID options to this map + // However we should only mount when options other than uid and gid are set. + // see https://github.com/containers/podman/issues/10620 + index := 0 + if _, ok := v.config.Options["UID"]; ok { + index++ + } + if _, ok := v.config.Options["GID"]; ok { + index++ + } + // when uid or gid is set there is also the "o" option + // set so we have to ignore this one as well + if index > 0 { + index++ + } + // Local driver with options other than uid,gid needs mount + return len(v.config.Options) > index } // update() updates the volume state from the DB. diff --git a/nix/default-arm64.nix b/nix/default-arm64.nix index 3e374c1bd..d29f75520 100644 --- a/nix/default-arm64.nix +++ b/nix/default-arm64.nix @@ -25,6 +25,21 @@ let -i "$dev"/include/glib-2.0/gobject/gobjectnotifyqueue.c ''; }); + pcsclite = (static pkg.pcsclite).overrideAttrs (x: { + configureFlags = [ + "--enable-confdir=/etc" + "--enable-usbdropdir=/var/lib/pcsc/drivers" + "--disable-libsystemd" + ]; + buildInputs = [ pkgs.python3 pkgs.udev pkgs.dbus pkgs.systemd ]; + }); + systemd = (static pkg.systemd).overrideAttrs (x: { + outputs = [ "out" "dev" ]; + mesonFlags = x.mesonFlags ++ [ + "-Dglib=false" + "-Dstatic-libsystemd=true" + ]; + }); }; }; }); @@ -47,8 +62,8 @@ let doCheck = false; enableParallelBuilding = true; outputs = [ "out" ]; - nativeBuildInputs = [ bash gitMinimal go-md2man installShellFiles makeWrapper pkg-config which ]; - buildInputs = [ glibc glibc.static gpgme libassuan libgpgerror libseccomp libapparmor libselinux ]; + nativeBuildInputs = [ bash gitMinimal go-md2man pkg-config which ]; + buildInputs = [ glibc glibc.static glib gpgme libassuan libgpgerror libseccomp libapparmor libselinux ]; prePatch = '' export CFLAGS='-static -pthread' export LDFLAGS='-s -w -static-libgcc -static' diff --git a/nix/default.nix b/nix/default.nix index aae520c9e..f6194e3d9 100644 --- a/nix/default.nix +++ b/nix/default.nix @@ -23,6 +23,21 @@ let -i "$dev"/include/glib-2.0/gobject/gobjectnotifyqueue.c ''; }); + pcsclite = (static pkg.pcsclite).overrideAttrs (x: { + configureFlags = [ + "--enable-confdir=/etc" + "--enable-usbdropdir=/var/lib/pcsc/drivers" + "--with-systemdsystemunitdir=${placeholder "bin"}/lib/systemd/system" + ]; + buildInputs = [ pkgs.python3 pkgs.udev pkgs.dbus pkgs.systemd ]; + }); + systemd = (static pkg.systemd).overrideAttrs (x: { + outputs = [ "out" "dev" ]; + mesonFlags = x.mesonFlags ++ [ + "-Dglib=false" + "-Dstatic-libsystemd=true" + ]; + }); }; }; }); @@ -45,8 +60,8 @@ let doCheck = false; enableParallelBuilding = true; outputs = [ "out" ]; - nativeBuildInputs = [ bash gitMinimal go-md2man installShellFiles makeWrapper pkg-config which ]; - buildInputs = [ glibc glibc.static gpgme libassuan libgpgerror libseccomp libapparmor libselinux ]; + nativeBuildInputs = [ bash gitMinimal go-md2man pkg-config which ]; + buildInputs = [ glibc glibc.static glib gpgme libassuan libgpgerror libseccomp libapparmor libselinux ]; prePatch = '' export CFLAGS='-static -pthread' export LDFLAGS='-s -w -static-libgcc -static' diff --git a/nix/nixpkgs.json b/nix/nixpkgs.json index 9b8b8289e..a9771eade 100644 --- a/nix/nixpkgs.json +++ b/nix/nixpkgs.json @@ -1,9 +1,9 @@ { "url": "https://github.com/nixos/nixpkgs", - "rev": "eb7e1ef185f6c990cda5f71fdc4fb02e76ab06d5", - "date": "2021-05-05T23:16:00+02:00", - "path": "/nix/store/a98lkhjlsqh32ic2kkrv5kkik6jy25wh-nixpkgs", - "sha256": "1ibz204c41g7baqga2iaj11yz9l75cfdylkiqjnk5igm81ivivxg", + "rev": "60cce7e5e1fdf62421ef6d4184ee399b46209366", + "date": "2021-06-09T01:18:50-04:00", + "path": "/nix/store/fixgn194626rb7gf99l9jaqm0hbqn2ix-nixpkgs", + "sha256": "100xrb925cana1kfd0c7gwkjjalq891vfgr0rn1gl9j8gp3l3gx6", "fetchSubmodules": false, "deepClone": false, "leaveDotGit": false diff --git a/nix/nixpkgs.nix b/nix/nixpkgs.nix index 21e7f17a2..11c20380a 100644 --- a/nix/nixpkgs.nix +++ b/nix/nixpkgs.nix @@ -5,4 +5,5 @@ let url = "${json.url}/archive/${json.rev}.tar.gz"; inherit (json) sha256; }); -in nixpkgs +in +nixpkgs diff --git a/pkg/api/handlers/compat/images.go b/pkg/api/handlers/compat/images.go index 7b336c470..ac212474b 100644 --- a/pkg/api/handlers/compat/images.go +++ b/pkg/api/handlers/compat/images.go @@ -168,6 +168,8 @@ func CreateImageFromSrc(w http.ResponseWriter, r *http.Request) { query := struct { FromSrc string `schema:"fromSrc"` Changes []string `schema:"changes"` + Message string `schema:"message"` + Repo string `shchema:"repo"` }{ // This is where you can override the golang default value for one of fields } @@ -184,14 +186,15 @@ func CreateImageFromSrc(w http.ResponseWriter, r *http.Request) { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "failed to create tempfile")) return } + source = f.Name() if err := SaveFromBody(f, r); err != nil { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "failed to write temporary file")) } } - imageEngine := abi.ImageEngine{Libpod: runtime} - report, err := imageEngine.Import(r.Context(), entities.ImageImportOptions{Source: source, Changes: query.Changes}) + // TODO: add support for ImageImportOptions to take a platform parameter. Also import https://github.com/opencontainers/image-spec/tree/master/specs-go/v1 either here or within imageEngine.Import to get default platform + report, err := imageEngine.Import(r.Context(), entities.ImageImportOptions{Source: source, Changes: query.Changes, Message: query.Message, Reference: query.Repo}) if err != nil { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "unable to import tarball")) return @@ -224,10 +227,10 @@ func CreateImageFromImage(w http.ResponseWriter, r *http.Request) { query := struct { FromImage string `schema:"fromImage"` Tag string `schema:"tag"` + Platform string `schema:"platform"` }{ // This is where you can override the golang default value for one of fields } - if err := decoder.Decode(&query, r.URL.Query()); err != nil { utils.Error(w, "Something went wrong.", http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return @@ -250,12 +253,36 @@ func CreateImageFromImage(w http.ResponseWriter, r *http.Request) { } defer auth.RemoveAuthfile(authfile) + platformSpecs := strings.Split(query.Platform, "/") // split query into its parts + + addOS := true // default assume true due to structure of if/else below + addArch := false + addVariant := false + + if len(platformSpecs) > 1 { // if we have two arguments then we have os and arch + addArch = true + if len(platformSpecs) > 2 { // if we have 3 arguments then we have os arch and variant + addVariant = true + } + } else if len(platformSpecs) == 0 { + addOS = false + } + pullOptions := &libimage.PullOptions{} pullOptions.AuthFilePath = authfile if authConf != nil { pullOptions.Username = authConf.Username pullOptions.Password = authConf.Password pullOptions.IdentityToken = authConf.IdentityToken + if addOS { // if the len is not 0 + pullOptions.OS = platformSpecs[0] + if addArch { + pullOptions.Architecture = platformSpecs[1] + } + if addVariant { + pullOptions.Variant = platformSpecs[2] + } + } } pullOptions.Writer = os.Stderr // allows for debugging on the server @@ -294,7 +321,6 @@ loop: // break out of for/select infinite loop Error string `json:"error,omitempty"` Id string `json:"id,omitempty"` // nolint } - select { case e := <-progress: switch e.Event { diff --git a/pkg/api/handlers/compat/images_build.go b/pkg/api/handlers/compat/images_build.go index 6ff557291..9c4dd8638 100644 --- a/pkg/api/handlers/compat/images_build.go +++ b/pkg/api/handlers/compat/images_build.go @@ -139,6 +139,31 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { addCaps = m } + // convert addcaps formats + containerFiles := []string{} + if _, found := r.URL.Query()["dockerfile"]; found { + var m = []string{} + if err := json.Unmarshal([]byte(query.Dockerfile), &m); err != nil { + // it's not json, assume just a string + m = append(m, query.Dockerfile) + } + containerFiles = m + } else { + containerFiles = []string{"Dockerfile"} + if utils.IsLibpodRequest(r) { + containerFiles = []string{"Containerfile"} + if _, err = os.Stat(filepath.Join(contextDirectory, "Containerfile")); err != nil { + if _, err1 := os.Stat(filepath.Join(contextDirectory, "Dockerfile")); err1 == nil { + containerFiles = []string{"Dockerfile"} + } else { + utils.BadRequest(w, "dockerfile", query.Dockerfile, err) + } + } + } else { + containerFiles = []string{"Dockerfile"} + } + } + addhosts := []string{} if _, found := r.URL.Query()["extrahosts"]; found { if err := json.Unmarshal([]byte(query.AddHosts), &addhosts); err != nil { @@ -470,7 +495,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { runCtx, cancel := context.WithCancel(context.Background()) go func() { defer cancel() - imageID, _, err = runtime.Build(r.Context(), buildOptions, query.Dockerfile) + imageID, _, err = runtime.Build(r.Context(), buildOptions, containerFiles...) if err == nil { success = true } else { diff --git a/pkg/api/handlers/compat/networks.go b/pkg/api/handlers/compat/networks.go index 77ed548d8..4e1f31404 100644 --- a/pkg/api/handlers/compat/networks.go +++ b/pkg/api/handlers/compat/networks.go @@ -28,19 +28,24 @@ import ( func InspectNetwork(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value("runtime").(*libpod.Runtime) - // FYI scope and version are currently unused but are described by the API - // Leaving this for if/when we have to enable these - // query := struct { - // scope string - // verbose bool - // }{ - // // override any golang type defaults - // } - // decoder := r.Context().Value("decoder").(*schema.Decoder) - // if err := decoder.Decode(&query, r.URL.Query()); err != nil { - // utils.Error(w, "Something went wrong.", http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) - // return - // } + // scope is only used to see if the user passes any illegal value, verbose is not used but implemented + // for compatibility purposes only. + query := struct { + scope string `schema:"scope"` + verbose bool `schema:"verbose"` + }{ + scope: "local", + } + decoder := r.Context().Value("decoder").(*schema.Decoder) + if err := decoder.Decode(&query, r.URL.Query()); err != nil { + utils.Error(w, "Something went wrong.", http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) + return + } + + if query.scope != "local" { + utils.Error(w, "Invalid scope value. Can only be local.", http.StatusBadRequest, define.ErrInvalidArg) + return + } config, err := runtime.GetConfig() if err != nil { utils.InternalServerError(w, err) @@ -414,7 +419,7 @@ func Prune(w http.ResponseWriter, r *http.Request) { type response struct { NetworksDeleted []string } - var prunedNetworks []string //nolint + prunedNetworks := []string{} for _, pr := range pruneReports { if pr.Error != nil { logrus.Error(pr.Error) diff --git a/pkg/api/handlers/compat/swagger.go b/pkg/api/handlers/compat/swagger.go index a0783e723..b773799ef 100644 --- a/pkg/api/handlers/compat/swagger.go +++ b/pkg/api/handlers/compat/swagger.go @@ -77,10 +77,3 @@ type swagCompatNetworkDisconnectRequest struct { // in:body Body struct{ types.NetworkDisconnect } } - -// Network prune -// swagger:response NetworkPruneResponse -type swagCompatNetworkPruneResponse struct { - // in:body - Body []string -} diff --git a/pkg/api/handlers/libpod/images_pull.go b/pkg/api/handlers/libpod/images_pull.go index fe56aa31d..e88b53a4b 100644 --- a/pkg/api/handlers/libpod/images_pull.go +++ b/pkg/api/handlers/libpod/images_pull.go @@ -85,7 +85,7 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) { var pulledImages []*libimage.Image var pullError error - runCtx, cancel := context.WithCancel(context.Background()) + runCtx, cancel := context.WithCancel(r.Context()) go func() { defer cancel() pulledImages, pullError = runtime.LibimageRuntime().Pull(runCtx, query.Reference, config.PullPolicyAlways, pullOptions) diff --git a/pkg/api/handlers/libpod/networks.go b/pkg/api/handlers/libpod/networks.go index 5417f778e..e4f450e12 100644 --- a/pkg/api/handlers/libpod/networks.go +++ b/pkg/api/handlers/libpod/networks.go @@ -190,5 +190,8 @@ func Prune(w http.ResponseWriter, r *http.Request) { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, err) return } + if pruneReports == nil { + pruneReports = []*entities.NetworkPruneReport{} + } utils.WriteResponse(w, http.StatusOK, pruneReports) } diff --git a/pkg/api/handlers/libpod/swagger.go b/pkg/api/handlers/libpod/swagger.go index 19eced986..6116a7274 100644 --- a/pkg/api/handlers/libpod/swagger.go +++ b/pkg/api/handlers/libpod/swagger.go @@ -4,6 +4,7 @@ import ( "net/http" "os" + "github.com/containernetworking/cni/libcni" "github.com/containers/image/v5/manifest" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/api/handlers/utils" @@ -102,7 +103,7 @@ type swagNetworkRmReport struct { // swagger:response NetworkInspectReport type swagNetworkInspectReport struct { // in:body - Body entities.NetworkInspectReport + Body libcni.NetworkConfigList } // Network list @@ -119,6 +120,13 @@ type swagNetworkCreateReport struct { Body entities.NetworkCreateReport } +// Network prune +// swagger:response NetworkPruneResponse +type swagNetworkPruneResponse struct { + // in:body + Body []entities.NetworkPruneReport +} + func ServeSwagger(w http.ResponseWriter, r *http.Request) { path := DefaultPodmanSwaggerSpec if p, found := os.LookupEnv("PODMAN_SWAGGER_SPEC"); found { diff --git a/pkg/api/server/register_images.go b/pkg/api/server/register_images.go index 9634bd83b..b28818768 100644 --- a/pkg/api/server/register_images.go +++ b/pkg/api/server/register_images.go @@ -28,15 +28,28 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // - in: query // name: fromImage // type: string - // description: needs description + // description: Name of the image to pull. The name may include a tag or digest. This parameter may only be used when pulling an image. The pull is cancelled if the HTTP connection is closed. // - in: query // name: fromSrc // type: string - // description: needs description + // description: Source to import. The value may be a URL from which the image can be retrieved or - to read the image from the request body. This parameter may only be used when importing an image + // - in: query + // name: repo + // type: string + // description: Repository name given to an image when it is imported. The repo may include a tag. This parameter may only be used when importing an image. // - in: query // name: tag // type: string - // description: needs description + // description: Tag or digest. If empty when pulling an image, this causes all tags for the given image to be pulled. + // - in: query + // name: message + // type: string + // description: Set commit message for imported image. + // - in: query + // name: platform + // type: string + // description: Platform in the format os[/arch[/variant]] + // default: "" // - in: header // name: X-Registry-Auth // type: string @@ -45,6 +58,7 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // name: request // schema: // type: string + // format: binary // description: Image content if fromSrc parameter was used // responses: // 200: diff --git a/pkg/api/server/register_networks.go b/pkg/api/server/register_networks.go index 4d9806316..cacf83a7f 100644 --- a/pkg/api/server/register_networks.go +++ b/pkg/api/server/register_networks.go @@ -44,6 +44,16 @@ func (s *APIServer) registerNetworkHandlers(r *mux.Router) error { // type: string // required: true // description: the name of the network + // - in: query + // name: verbose + // type: boolean + // required: false + // description: Detailed inspect output for troubleshooting + // - in: query + // name: scope + // type: string + // required: false + // description: Filter the network by scope (swarm, global, or local) // produces: // - application/json // responses: @@ -180,9 +190,12 @@ func (s *APIServer) registerNetworkHandlers(r *mux.Router) error { // 200: // description: OK // schema: - // type: array - // items: - // type: string + // type: object + // properties: + // NetworksDeleted: + // type: array + // items: + // type: string // 500: // $ref: "#/responses/InternalError" r.HandleFunc(VersionedPath("/networks/prune"), s.APIHandler(compat.Prune)).Methods(http.MethodPost) @@ -268,7 +281,9 @@ func (s *APIServer) registerNetworkHandlers(r *mux.Router) error { // tags: // - networks // summary: Inspect a network - // description: Display low level configuration for a CNI network + // description: | + // Display low level configuration for a CNI network. + // - In a 200 response, all of the fields named Bytes are returned as a Base64 encoded string. // parameters: // - in: path // name: name diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go index 346d55c47..c7d432b16 100644 --- a/pkg/bindings/images/build.go +++ b/pkg/bindings/images/build.go @@ -282,10 +282,6 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO stdout = options.Out } - entries := make([]string, len(containerFiles)) - copy(entries, containerFiles) - entries = append(entries, options.ContextDirectory) - excludes := options.Excludes if len(excludes) == 0 { excludes, err = parseDockerignore(options.ContextDirectory) @@ -294,33 +290,57 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO } } - tarfile, err := nTar(excludes, entries...) + contextDir, err := filepath.Abs(options.ContextDirectory) if err != nil { - logrus.Errorf("cannot tar container entries %v error: %v", entries, err) + logrus.Errorf("cannot find absolute path of %v: %v", options.ContextDirectory, err) return nil, err } - defer func() { - if err := tarfile.Close(); err != nil { - logrus.Errorf("%v\n", err) + + tarContent := []string{options.ContextDirectory} + newContainerFiles := []string{} + for _, c := range containerFiles { + containerfile, err := filepath.Abs(c) + if err != nil { + logrus.Errorf("cannot find absolute path of %v: %v", c, err) + return nil, err } - }() - containerFile, err := filepath.Abs(entries[0]) - if err != nil { - logrus.Errorf("cannot find absolute path of %v: %v", entries[0], err) - return nil, err + // Check if Containerfile is in the context directory, if so truncate the contextdirectory off path + // Do NOT add to tarfile + if strings.HasPrefix(containerfile, contextDir+string(filepath.Separator)) { + containerfile = strings.TrimPrefix(containerfile, contextDir+string(filepath.Separator)) + } else { + // If Containerfile does not exists assume it is in context directory, do Not add to tarfile + if _, err := os.Lstat(containerfile); err != nil { + if !os.IsNotExist(err) { + return nil, err + } + containerfile = c + } else { + // If Containerfile does exists but is not in context directory add it to the tarfile + tarContent = append(tarContent, containerfile) + } + } + newContainerFiles = append(newContainerFiles, containerfile) } - contextDir, err := filepath.Abs(entries[1]) - if err != nil { - logrus.Errorf("cannot find absolute path of %v: %v", entries[1], err) - return nil, err + if len(newContainerFiles) > 0 { + cFileJSON, err := json.Marshal(newContainerFiles) + if err != nil { + return nil, err + } + params.Set("dockerfile", string(cFileJSON)) } - if strings.HasPrefix(containerFile, contextDir+string(filepath.Separator)) { - containerFile = strings.TrimPrefix(containerFile, contextDir+string(filepath.Separator)) + tarfile, err := nTar(excludes, tarContent...) + if err != nil { + logrus.Errorf("cannot tar container entries %v error: %v", tarContent, err) + return nil, err } - - params.Set("dockerfile", containerFile) + defer func() { + if err := tarfile.Close(); err != nil { + logrus.Errorf("%v\n", err) + } + }() conn, err := bindings.GetClient(ctx) if err != nil { diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index 4908e72f6..5a953c047 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -595,7 +595,7 @@ func (ic *ContainerEngine) ContainerAttach(ctx context.Context, nameOrID string, return nil } -func makeExecConfig(options entities.ExecOptions) *libpod.ExecConfig { +func makeExecConfig(options entities.ExecOptions, rt *libpod.Runtime) (*libpod.ExecConfig, error) { execConfig := new(libpod.ExecConfig) execConfig.Command = options.Cmd execConfig.Terminal = options.Tty @@ -607,7 +607,20 @@ func makeExecConfig(options entities.ExecOptions) *libpod.ExecConfig { execConfig.PreserveFDs = options.PreserveFDs execConfig.AttachStdin = options.Interactive - return execConfig + // Make an exit command + storageConfig := rt.StorageConfig() + runtimeConfig, err := rt.GetConfig() + if err != nil { + return nil, errors.Wrapf(err, "error retrieving Libpod configuration to build exec exit command") + } + // TODO: Add some ability to toggle syslog + exitCommandArgs, err := generate.CreateExitCommandArgs(storageConfig, runtimeConfig, false, true, true) + if err != nil { + return nil, errors.Wrapf(err, "error constructing exit command for exec session") + } + execConfig.ExitCommand = exitCommandArgs + + return execConfig, nil } func checkExecPreserveFDs(options entities.ExecOptions) error { @@ -647,7 +660,10 @@ func (ic *ContainerEngine) ContainerExec(ctx context.Context, nameOrID string, o } ctr := ctrs[0] - execConfig := makeExecConfig(options) + execConfig, err := makeExecConfig(options, ic.Libpod) + if err != nil { + return ec, err + } ec, err = terminal.ExecAttachCtr(ctx, ctr, execConfig, &streams) return define.TranslateExecErrorToExitCode(ec, err), err @@ -664,20 +680,10 @@ func (ic *ContainerEngine) ContainerExecDetached(ctx context.Context, nameOrID s } ctr := ctrs[0] - execConfig := makeExecConfig(options) - - // Make an exit command - storageConfig := ic.Libpod.StorageConfig() - runtimeConfig, err := ic.Libpod.GetConfig() - if err != nil { - return "", errors.Wrapf(err, "error retrieving Libpod configuration to build exec exit command") - } - // TODO: Add some ability to toggle syslog - exitCommandArgs, err := generate.CreateExitCommandArgs(storageConfig, runtimeConfig, false, true, true) + execConfig, err := makeExecConfig(options, ic.Libpod) if err != nil { - return "", errors.Wrapf(err, "error constructing exit command for exec session") + return "", err } - execConfig.ExitCommand = exitCommandArgs // Create and start the exec session id, err := ctr.ExecCreate(execConfig) diff --git a/pkg/domain/infra/abi/parse/parse.go b/pkg/domain/infra/abi/parse/parse.go index 1c590d2d6..56c747711 100644 --- a/pkg/domain/infra/abi/parse/parse.go +++ b/pkg/domain/infra/abi/parse/parse.go @@ -37,7 +37,7 @@ func VolumeOptions(opts map[string]string) ([]libpod.VolumeCreateOption, error) return nil, errors.Wrapf(err, "cannot convert UID %s to integer", splitO[1]) } logrus.Debugf("Removing uid= from options and adding WithVolumeUID for UID %d", intUID) - libpodOptions = append(libpodOptions, libpod.WithVolumeUID(intUID)) + libpodOptions = append(libpodOptions, libpod.WithVolumeUID(intUID), libpod.WithVolumeNoChown()) finalVal = append(finalVal, o) // set option "UID": "$uid" volumeOptions["UID"] = splitO[1] @@ -50,7 +50,7 @@ func VolumeOptions(opts map[string]string) ([]libpod.VolumeCreateOption, error) return nil, errors.Wrapf(err, "cannot convert GID %s to integer", splitO[1]) } logrus.Debugf("Removing gid= from options and adding WithVolumeGID for GID %d", intGID) - libpodOptions = append(libpodOptions, libpod.WithVolumeGID(intGID)) + libpodOptions = append(libpodOptions, libpod.WithVolumeGID(intGID), libpod.WithVolumeNoChown()) finalVal = append(finalVal, o) // set option "GID": "$gid" volumeOptions["GID"] = splitO[1] diff --git a/pkg/machine/config.go b/pkg/machine/config.go index 58794ce42..db9bfa7de 100644 --- a/pkg/machine/config.go +++ b/pkg/machine/config.go @@ -1,3 +1,5 @@ +// +build amd64,linux arm64,linux amd64,darwin arm64,darwin + package machine import ( diff --git a/pkg/machine/connection.go b/pkg/machine/connection.go index e3985d8ac..3edcbd10e 100644 --- a/pkg/machine/connection.go +++ b/pkg/machine/connection.go @@ -1,3 +1,5 @@ +// +build amd64,linux arm64,linux amd64,darwin arm64,darwin + package machine import ( diff --git a/pkg/machine/fcos.go b/pkg/machine/fcos.go index 32f943c87..11936aee7 100644 --- a/pkg/machine/fcos.go +++ b/pkg/machine/fcos.go @@ -1,3 +1,5 @@ +// +build amd64,linux arm64,linux amd64,darwin arm64,darwin + package machine import ( diff --git a/pkg/machine/ignition.go b/pkg/machine/ignition.go index a5c7210af..1d77083d0 100644 --- a/pkg/machine/ignition.go +++ b/pkg/machine/ignition.go @@ -1,3 +1,5 @@ +// +build amd64,linux arm64,linux amd64,darwin arm64,darwin + package machine import ( diff --git a/pkg/machine/ignition_schema.go b/pkg/machine/ignition_schema.go index 9dbd90ba4..6ac8af826 100644 --- a/pkg/machine/ignition_schema.go +++ b/pkg/machine/ignition_schema.go @@ -1,3 +1,5 @@ +// +build amd64,linux arm64,linux amd64,darwin arm64,darwin + package machine /* diff --git a/pkg/machine/keys.go b/pkg/machine/keys.go index 907e28f55..81ec44ea8 100644 --- a/pkg/machine/keys.go +++ b/pkg/machine/keys.go @@ -1,3 +1,5 @@ +// +build amd64,linux arm64,linux amd64,darwin arm64,darwin + package machine import ( diff --git a/pkg/machine/libvirt/config.go b/pkg/machine/libvirt/config.go index 903f15fbc..1ce5ab154 100644 --- a/pkg/machine/libvirt/config.go +++ b/pkg/machine/libvirt/config.go @@ -1,3 +1,5 @@ +// +build amd64,linux arm64,linux amd64,darwin arm64,darwin + package libvirt type MachineVM struct { diff --git a/pkg/machine/libvirt/machine.go b/pkg/machine/libvirt/machine.go index c38f63853..e1aa1569b 100644 --- a/pkg/machine/libvirt/machine.go +++ b/pkg/machine/libvirt/machine.go @@ -1,3 +1,5 @@ +// +build amd64,linux arm64,linux amd64,darwin arm64,darwin + package libvirt import "github.com/containers/podman/v3/pkg/machine" diff --git a/pkg/machine/libvirt/machine_unsupported.go b/pkg/machine/libvirt/machine_unsupported.go new file mode 100644 index 000000000..8b54440fe --- /dev/null +++ b/pkg/machine/libvirt/machine_unsupported.go @@ -0,0 +1,3 @@ +// +build !amd64 amd64,windows + +package libvirt diff --git a/pkg/machine/machine_unsupported.go b/pkg/machine/machine_unsupported.go new file mode 100644 index 000000000..9309d16bc --- /dev/null +++ b/pkg/machine/machine_unsupported.go @@ -0,0 +1,3 @@ +// +build !amd64 amd64,windows + +package machine diff --git a/pkg/machine/pull.go b/pkg/machine/pull.go index 68bb551dc..662896de5 100644 --- a/pkg/machine/pull.go +++ b/pkg/machine/pull.go @@ -1,3 +1,5 @@ +// +build amd64,linux arm64,linux amd64,darwin arm64,darwin + package machine import ( diff --git a/pkg/machine/qemu/config.go b/pkg/machine/qemu/config.go index e4687914d..013f28960 100644 --- a/pkg/machine/qemu/config.go +++ b/pkg/machine/qemu/config.go @@ -1,3 +1,5 @@ +// +build amd64,linux arm64,linux amd64,darwin arm64,darwin + package qemu import "time" diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go index 31c355d4a..22fb78a5c 100644 --- a/pkg/machine/qemu/machine.go +++ b/pkg/machine/qemu/machine.go @@ -1,3 +1,5 @@ +// +build amd64,linux arm64,linux amd64,darwin arm64,darwin + package qemu import ( diff --git a/pkg/machine/qemu/machine_unsupported.go b/pkg/machine/qemu/machine_unsupported.go new file mode 100644 index 000000000..da06ac324 --- /dev/null +++ b/pkg/machine/qemu/machine_unsupported.go @@ -0,0 +1,3 @@ +// +build !amd64 amd64,windows + +package qemu diff --git a/pkg/rootless/rootless_linux.c b/pkg/rootless/rootless_linux.c index 0d1d6e93e..e5f9e88d9 100644 --- a/pkg/rootless/rootless_linux.c +++ b/pkg/rootless/rootless_linux.c @@ -333,7 +333,7 @@ static void __attribute__((constructor)) init() uid_t uid; gid_t gid; char path[PATH_MAX]; - const char *const suffix = "/libpod/pause.pid"; + const char *const suffix = "/libpod/tmp/pause.pid"; char *cwd = getcwd (NULL, 0); char uid_fmt[16]; char gid_fmt[16]; diff --git a/test/apiv2/10-images.at b/test/apiv2/10-images.at index 037a4c01f..9e464dbc7 100644 --- a/test/apiv2/10-images.at +++ b/test/apiv2/10-images.at @@ -147,4 +147,39 @@ t GET "images/get?names=alpine&names=busybox" 200 '[POSIX tar archive]' img_cnt=$(tar xf "$WORKDIR/curl.result.out" manifest.json -O | jq "length") is "$img_cnt" 2 "number of images in tar archive" +# check build works when uploading container file as a tar, see issue #10660 +TMPD=$(mktemp -d podman-apiv2-test.build.XXXXXXXX) +function cleanBuildTest() { + podman rmi -a -f + rm -rf "${TMPD}" &> /dev/null +} +CONTAINERFILE_TAR="${TMPD}/containerfile.tar" +cat > $TMPD/containerfile << EOF +FROM quay.io/libpod/alpine_labels:latest +EOF +tar --format=posix -C $TMPD -cvf ${CONTAINERFILE_TAR} containerfile &> /dev/null + +curl -XPOST --data-binary @<(cat $CONTAINERFILE_TAR) \ + -H "content-type: application/x-tar" \ + --dump-header "${TMPD}/headers.txt" \ + -o "${TMPD}/response.txt" \ + "http://$HOST:$PORT/v1.40/libpod/build?dockerfile=containerfile" &> /dev/null + +BUILD_TEST_ERROR="" + +if ! grep -q '200 OK' "${TMPD}/headers.txt"; then + echo -e "${red}NOK: Image build from tar failed response was not 200 OK" + BUILD_TEST_ERROR="1" +fi + +if ! grep -q 'quay.io/libpod/alpine_labels' "${TMPD}/response.txt"; then + echo -e "${red}NOK: Image build from tar failed image name not in response" + BUILD_TEST_ERROR="1" +fi + +cleanBuildTest +if [[ "${BUILD_TEST_ERROR}" ]]; then + exit 1 +fi + # vim: filetype=sh diff --git a/test/apiv2/20-containers.at b/test/apiv2/20-containers.at index a81210855..ef51757c9 100644 --- a/test/apiv2/20-containers.at +++ b/test/apiv2/20-containers.at @@ -341,3 +341,12 @@ t GET containers/$cid/json 200 \ .HostConfig.NanoCpus=500000 t DELETE containers/$cid?v=true 204 + +# Test Compat Create with default network mode (#10569) +t POST containers/create Image=$IMAGE HostConfig='{"NetworkMode":"default"}' 201 \ + .Id~[0-9a-f]\\{64\\} +cid=$(jq -r '.Id' <<<"$output") +t GET containers/$cid/json 200 \ + .HostConfig.NetworkMode="bridge" + +t DELETE containers/$cid?v=true 204 diff --git a/test/apiv2/python/rest_api/test_v2_0_0_image.py b/test/apiv2/python/rest_api/test_v2_0_0_image.py index 243b1d5f5..cea34e2e7 100644 --- a/test/apiv2/python/rest_api/test_v2_0_0_image.py +++ b/test/apiv2/python/rest_api/test_v2_0_0_image.py @@ -87,6 +87,18 @@ class ImageTestCase(APITestCase): self.assertTrue(keys["images"], "Expected to find images stanza") self.assertTrue(keys["stream"], "Expected to find stream progress stanza's") + def test_create(self): + r = requests.post( + self.podman_url + "/v1.40/images/create?fromImage=alpine&platform=linux/amd64/v8", + timeout=15, + ) + self.assertEqual(r.status_code, 200, r.text) + r = requests.post( + self.podman_url + "/v1.40/images/create?fromSrc=-&repo=fedora&message=testing123", + timeout=15, + ) + self.assertEqual(r.status_code, 200, r.text) + def test_search_compat(self): url = self.podman_url + "/v1.40/images/search" diff --git a/test/apiv2/python/rest_api/test_v2_0_0_network.py b/test/apiv2/python/rest_api/test_v2_0_0_network.py index 3888123fb..d606b9351 100644 --- a/test/apiv2/python/rest_api/test_v2_0_0_network.py +++ b/test/apiv2/python/rest_api/test_v2_0_0_network.py @@ -102,6 +102,33 @@ class NetworkTestCase(APITestCase): "TestNetwork", payload["NetworkSettings"]["Networks"]["TestNetwork"]["NetworkID"], ) + def test_inspect(self): + name = f"Network_{random.getrandbits(160):x}" + create = requests.post(self.podman_url + "/v1.40/networks/create", json={"Name": name}) + self.assertEqual(create.status_code, 201, create.text) + self.assertId(create.content) + + net = create.json() + self.assertIsInstance(net, dict) + self.assertNotEqual(net["Id"], name) + ident = net["Id"] + + ls = requests.get(self.podman_url + "/v1.40/networks") + self.assertEqual(ls.status_code, 200, ls.text) + + networks = ls.json() + self.assertIsInstance(networks, list) + + found = False + for net in networks: + if net["Name"] == name: + found = True + break + self.assertTrue(found, f"Network '{name}' not found") + + inspect = requests.get(self.podman_url + f"/v1.40/networks/{ident}?verbose=false&scope=local") + self.assertEqual(inspect.status_code, 200, inspect.text) + def test_crud(self): name = f"Network_{random.getrandbits(160):x}" diff --git a/test/compose/test-compose b/test/compose/test-compose index 981f78a79..70db6dd55 100755 --- a/test/compose/test-compose +++ b/test/compose/test-compose @@ -183,6 +183,8 @@ function test_port() { fi echo "# cat $WORKDIR/server.log:" cat $WORKDIR/server.log + echo "# cat $logfile:" + cat $logfile return fi diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go index 7ffee961c..1aeeca4cb 100644 --- a/test/e2e/common_test.go +++ b/test/e2e/common_test.go @@ -451,19 +451,13 @@ func (p *PodmanTestIntegration) RunLsContainerInPod(name, pod string) (*PodmanSe // BuildImage uses podman build and buildah to build an image // called imageName based on a string dockerfile func (p *PodmanTestIntegration) BuildImage(dockerfile, imageName string, layers string) string { - dockerfilePath := filepath.Join(p.TempDir, "Dockerfile") - err := ioutil.WriteFile(dockerfilePath, []byte(dockerfile), 0755) - Expect(err).To(BeNil()) - cmd := []string{"build", "--pull-never", "--layers=" + layers, "--file", dockerfilePath} - if len(imageName) > 0 { - cmd = append(cmd, []string{"-t", imageName}...) - } - cmd = append(cmd, p.TempDir) - session := p.Podman(cmd) - session.Wait(240) - Expect(session).Should(Exit(0), fmt.Sprintf("BuildImage session output: %q", session.OutputToString())) - output := session.OutputToStringArray() - return output[len(output)-1] + return p.buildImage(dockerfile, imageName, layers, "") +} + +// BuildImageWithLabel uses podman build and buildah to build an image +// called imageName based on a string dockerfile, adds desired label to paramset +func (p *PodmanTestIntegration) BuildImageWithLabel(dockerfile, imageName string, layers string, label string) string { + return p.buildImage(dockerfile, imageName, layers, label) } // PodmanPID execs podman and returns its PID @@ -828,3 +822,22 @@ func (p *PodmanSessionIntegration) jq(jqCommand string) (string, error) { err := cmd.Run() return strings.TrimRight(out.String(), "\n"), err } + +func (p *PodmanTestIntegration) buildImage(dockerfile, imageName string, layers string, label string) string { + dockerfilePath := filepath.Join(p.TempDir, "Dockerfile") + err := ioutil.WriteFile(dockerfilePath, []byte(dockerfile), 0755) + Expect(err).To(BeNil()) + cmd := []string{"build", "--pull-never", "--layers=" + layers, "--file", dockerfilePath} + if label != "" { + cmd = append(cmd, "--label="+label) + } + if len(imageName) > 0 { + cmd = append(cmd, []string{"-t", imageName}...) + } + cmd = append(cmd, p.TempDir) + session := p.Podman(cmd) + session.Wait(240) + Expect(session).Should(Exit(0), fmt.Sprintf("BuildImage session output: %q", session.OutputToString())) + output := session.OutputToStringArray() + return output[len(output)-1] +} diff --git a/test/e2e/images_test.go b/test/e2e/images_test.go index f6321ec1c..b4ec7447e 100644 --- a/test/e2e/images_test.go +++ b/test/e2e/images_test.go @@ -425,4 +425,25 @@ LABEL "com.example.vendor"="Example Vendor" Expect(result.OutputToStringArray()).To(Not(Equal(result1.OutputToStringArray()))) }) + It("podman image prune --filter", func() { + dockerfile := `FROM quay.io/libpod/alpine:latest +RUN > file +` + dockerfile2 := `FROM quay.io/libpod/alpine:latest +RUN > file2 +` + podmanTest.BuildImageWithLabel(dockerfile, "foobar.com/workdir:latest", "false", "abc") + podmanTest.BuildImageWithLabel(dockerfile2, "foobar.com/workdir:latest", "false", "xyz") + // --force used to to avoid y/n question + result := podmanTest.Podman([]string{"image", "prune", "--filter", "label=abc", "--force"}) + result.WaitWithDefaultTimeout() + Expect(result).Should(Exit(0)) + Expect(len(result.OutputToStringArray())).To(Equal(1)) + + //check if really abc is removed + result = podmanTest.Podman([]string{"image", "list", "--filter", "label=abc"}) + Expect(len(result.OutputToStringArray())).To(Equal(0)) + + }) + }) diff --git a/test/e2e/logs_test.go b/test/e2e/logs_test.go index 4d9cbb48b..b576fa072 100644 --- a/test/e2e/logs_test.go +++ b/test/e2e/logs_test.go @@ -173,9 +173,9 @@ var _ = Describe("Podman logs", func() { }) It("streaming output: "+log, func() { - containerName := "logs-f-rm" + containerName := "logs-f" - logc := podmanTest.Podman([]string{"run", "--log-driver", log, "--rm", "--name", containerName, "-dt", ALPINE, "sh", "-c", "echo podman; sleep 1; echo podman"}) + logc := podmanTest.Podman([]string{"run", "--log-driver", log, "--name", containerName, "-dt", ALPINE, "sh", "-c", "echo podman-1; sleep 1; echo podman-2"}) logc.WaitWithDefaultTimeout() Expect(logc).To(Exit(0)) @@ -183,10 +183,8 @@ var _ = Describe("Podman logs", func() { results.WaitWithDefaultTimeout() Expect(results).To(Exit(0)) - // TODO: we should actually check for two podman lines, - // but as of 2020-06-17 there's a race condition in which - // 'logs -f' may not catch all output from a container - Expect(results.OutputToString()).To(ContainSubstring("podman")) + Expect(results.OutputToString()).To(ContainSubstring("podman-1")) + Expect(results.OutputToString()).To(ContainSubstring("podman-2")) // Container should now be terminatING or terminatED, but we // have no guarantee of which: 'logs -f' does not necessarily @@ -199,6 +197,10 @@ var _ = Describe("Podman logs", func() { } else { Expect(inspect.ErrorToString()).To(ContainSubstring("no such container")) } + + results = podmanTest.Podman([]string{"rm", "-f", containerName}) + results.WaitWithDefaultTimeout() + Expect(results).To(Exit(0)) }) It("follow output stopped container: "+log, func() { diff --git a/test/e2e/run_volume_test.go b/test/e2e/run_volume_test.go index 9b77aaef8..4be1b2009 100644 --- a/test/e2e/run_volume_test.go +++ b/test/e2e/run_volume_test.go @@ -668,4 +668,36 @@ USER testuser`, fedoraMinimal) Expect(strings.Contains(test2.OutputToString(), testString)).To(BeTrue()) }) + + It("podman volume with uid and gid works", func() { + volName := "testVol" + volCreate := podmanTest.Podman([]string{"volume", "create", "--opt", "o=uid=1000", volName}) + volCreate.WaitWithDefaultTimeout() + Expect(volCreate.ExitCode()).To(Equal(0)) + + volMount := podmanTest.Podman([]string{"run", "--rm", "-v", fmt.Sprintf("%s:/test", volName), ALPINE, "stat", "-c", "%u", "/test"}) + volMount.WaitWithDefaultTimeout() + Expect(volMount.ExitCode()).To(Equal(0)) + Expect(volMount.OutputToString()).To(Equal("1000")) + + volName = "testVol2" + volCreate = podmanTest.Podman([]string{"volume", "create", "--opt", "o=gid=1000", volName}) + volCreate.WaitWithDefaultTimeout() + Expect(volCreate.ExitCode()).To(Equal(0)) + + volMount = podmanTest.Podman([]string{"run", "--rm", "-v", fmt.Sprintf("%s:/test", volName), ALPINE, "stat", "-c", "%g", "/test"}) + volMount.WaitWithDefaultTimeout() + Expect(volMount.ExitCode()).To(Equal(0)) + Expect(volMount.OutputToString()).To(Equal("1000")) + + volName = "testVol3" + volCreate = podmanTest.Podman([]string{"volume", "create", "--opt", "o=uid=1000,gid=1000", volName}) + volCreate.WaitWithDefaultTimeout() + Expect(volCreate.ExitCode()).To(Equal(0)) + + volMount = podmanTest.Podman([]string{"run", "--rm", "-v", fmt.Sprintf("%s:/test", volName), ALPINE, "stat", "-c", "%u:%g", "/test"}) + volMount.WaitWithDefaultTimeout() + Expect(volMount.ExitCode()).To(Equal(0)) + Expect(volMount.OutputToString()).To(Equal("1000:1000")) + }) }) diff --git a/test/e2e/systemd_test.go b/test/e2e/systemd_test.go index b132750b0..8dc14d5f7 100644 --- a/test/e2e/systemd_test.go +++ b/test/e2e/systemd_test.go @@ -6,6 +6,7 @@ import ( "strings" "time" + "github.com/containers/podman/v3/pkg/rootless" . "github.com/containers/podman/v3/test/utils" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -115,6 +116,12 @@ WantedBy=multi-user.target conData := result.InspectContainerToJSON() Expect(len(conData)).To(Equal(1)) Expect(conData[0].Config.SystemdMode).To(BeTrue()) + + if CGROUPSV2 || !rootless.IsRootless() { + stats := podmanTest.Podman([]string{"stats", "--no-stream", ctrName}) + stats.WaitWithDefaultTimeout() + Expect(stats.ExitCode()).To(Equal(0)) + } }) It("podman create container with systemd entrypoint triggers systemd mode", func() { diff --git a/test/system/070-build.bats b/test/system/070-build.bats index 0f3f3fa7f..40622d6cc 100644 --- a/test/system/070-build.bats +++ b/test/system/070-build.bats @@ -794,6 +794,32 @@ EOF run_podman rmi -f build_test } +@test "podman build -f test " { + tmpdir=$PODMAN_TMPDIR/build-test + subdir=$tmpdir/subdir + mkdir -p $subdir + + containerfile1=$tmpdir/Containerfile1 + cat >$containerfile1 <<EOF +FROM scratch +copy . /tmp +EOF + containerfile2=$PODMAN_TMPDIR/Containerfile2 + cat >$containerfile2 <<EOF +FROM $IMAGE +EOF + run_podman build -t build_test -f Containerfile1 $tmpdir + run_podman 125 build -t build_test -f Containerfile2 $tmpdir + is "$output" ".*Containerfile2: no such file or directory" "Containerfile2 should not exist" + run_podman build -t build_test -f $containerfile1 $tmpdir + run_podman build -t build_test -f $containerfile2 $tmpdir + run_podman build -t build_test -f $containerfile1 + run_podman build -t build_test -f $containerfile2 + run_podman build -t build_test -f $containerfile1 -f $containerfile2 $tmpdir + is "$output" ".*$IMAGE" "Containerfile2 is also passed to server" + run_podman rmi -f build_test +} + function teardown() { # A timeout or other error in 'build' can leave behind stale images # that podman can't even see and which will cascade into subsequent diff --git a/test/system/255-auto-update.bats b/test/system/255-auto-update.bats index 59f53f775..9bfb44791 100644 --- a/test/system/255-auto-update.bats +++ b/test/system/255-auto-update.bats @@ -43,12 +43,12 @@ function teardown() { # 5. Remove the origin container # 6. Start the container from service function generate_service() { - target_img_basename=$1 - autoupdate=$2 + local target_img_basename=$1 + local autoupdate=$2 - # Please keep variable name for cname and ori_image. The - # scripts will use them directly in following tests. - cname=c_$(random_string) + # Container name. Include the autoupdate type, to make debugging easier. + # IMPORTANT: variable 'cname' is passed (out of scope) up to caller! + cname=c_${autoupdate//\'/}_$(random_string) target_img="quay.io/libpod/$target_img_basename:latest" run_podman tag $IMAGE $target_img if [[ -n "$autoupdate" ]]; then @@ -67,6 +67,8 @@ function generate_service() { systemctl start container-$cname systemctl status container-$cname + # Original image ID. + # IMPORTANT: variable 'ori_image' is passed (out of scope) up to caller! run_podman inspect --format "{{.Image}}" $cname ori_image=$output } @@ -76,8 +78,7 @@ function _wait_service_ready() { local timeout=6 while [[ $timeout -gt 1 ]]; do - run systemctl is-active $sname - if [[ $output == "active" ]]; then + if systemctl -q is-active $sname; then return fi sleep 1 @@ -89,65 +90,63 @@ function _wait_service_ready() { die "Timed out waiting for $sname to start" } +# Wait for container to update, as confirmed by its image ID changing function _confirm_update() { - local sname=$1 - - local timeout=6 - last_log="" - while [[ $timeout -gt 1 ]]; do - run journalctl -u $sname -n 10 - if [[ "$output" == "$last_log" ]]; then + local cname=$1 + local old_iid=$2 + + # Image has already been pulled, so this shouldn't take too long + local timeout=5 + while [[ $timeout -gt 0 ]]; do + run_podman '?' inspect --format "{{.Image}}" $cname + if [[ $status != 0 ]]; then + if [[ $output =~ (no such object|does not exist in database): ]]; then + # this is ok, it just means the container is being restarted + : + else + die "podman inspect $cname failed unexpectedly" + fi + elif [[ $output != $old_iid ]]; then return fi - last_log=$output sleep 1 - let timeout=$timeout-1 done - die "Timed out waiting for $sname to update" + die "Timed out waiting for $cname to update; old IID=$old_iid" } # This test can fail in dev. environment because of SELinux. # quick fix: chcon -t container_runtime_exec_t ./bin/podman @test "podman auto-update - label io.containers.autoupdate=image" { - run_podman images generate_service alpine image _wait_service_ready container-$cname.service - run_podman ps -a run_podman auto-update is "$output" "Trying to pull.*" "Image is updated." - run_podman ps -a - _confirm_update container-$cname.service - run_podman inspect --format "{{.Image}}" $cname - [[ "$output" != "$ori_image" ]] + _confirm_update $cname $ori_image } @test "podman auto-update - label io.containers.autoupdate=disabled" { generate_service alpine disabled _wait_service_ready container-$cname.service - run_podman ps -a run_podman auto-update - is "$output" "" "Image is not updated with disabled." - run_podman ps -a - _confirm_update container-$cname.service + is "$output" "" "Image is not updated when autoupdate=disabled." + run_podman inspect --format "{{.Image}}" $cname - is "$output" "$ori_image" "Image hash should not changed." + is "$output" "$ori_image" "Image ID should not change" } @test "podman auto-update - label io.containers.autoupdate=fakevalue" { - fakevalue=$(random_string) + fakevalue=fake_$(random_string) generate_service alpine $fakevalue _wait_service_ready container-$cname.service - run_podman ps -a - run_podman ? auto-update + run_podman 125 auto-update is "$output" ".*invalid auto-update policy.*" "invalid policy setup" - run_podman ps -a - _confirm_update container-$cname.service + run_podman inspect --format "{{.Image}}" $cname - is "$output" "$ori_image" "Image hash should not changed." + is "$output" "$ori_image" "Image ID should not change" } @test "podman auto-update - label io.containers.autoupdate=local" { @@ -155,25 +154,23 @@ function _confirm_update() { podman commit --change CMD=/bin/bash $cname quay.io/libpod/localtest:latest _wait_service_ready container-$cname.service - run_podman ps -a run_podman auto-update - run_podman ps -a - _confirm_update container-$cname.service - run_podman inspect --format "{{.Image}}" $cname - [[ "$output" != "$ori_image" ]] + _confirm_update $cname $ori_image } @test "podman auto-update with multiple services" { - fakevalue=$(random_string) + # Preserve original image ID, to confirm that it changes (or not) run_podman inspect --format "{{.Id}}" $IMAGE - img_id="$output" - cnames=() + local img_id="$output" + + local cnames=() local -A expect_update local -A will_update=([image]=1 [registry]=1 [local]=1) + local fakevalue=fake_$(random_string) for auto_update in image registry "" disabled "''" $fakevalue local do - img_base="alpine" + local img_base="alpine" if [[ $auto_update == "registry" ]]; then img_base="alpine_nginx" elif [[ $auto_update == "local" ]]; then @@ -184,6 +181,7 @@ function _confirm_update() { if [[ $auto_update == "local" ]]; then local_cname=$cname fi + if [[ -n "$auto_update" && -n "${will_update[$auto_update]}" ]]; then expect_update[$cname]=1 fi @@ -192,30 +190,28 @@ function _confirm_update() { # Only check the last service is started. Previous services should already actived. _wait_service_ready container-$cname.service run_podman commit --change CMD=/bin/bash $local_cname quay.io/libpod/localtest:latest - run_podman ? auto-update + # Exit code is expected, due to invalid 'fakevalue' + run_podman 125 auto-update update_log=$output - for cname in "${cnames[@]}"; do - _confirm_update container-$cname.service - done - count=0 - while read line; do - if [[ "$line" =~ "Trying to pull" ]]; then - ((count+=1)) - fi - done <<< "$update_log" is "$update_log" ".*invalid auto-update policy.*" "invalid policy setup" is "$update_log" ".*1 error occurred.*" "invalid policy setup" - is "$count" "2" "There are two images being updated from registry." - for cname in "${!expect_update[@]}"; do + local n_updated=$(grep -c 'Trying to pull' <<<"$update_log") + is "$n_updated" "2" "Number of images updated from registry." + for cname in "${!expect_update[@]}"; do is "$update_log" ".*$cname.*" "container with auto-update policy image updated" + # Just because podman says it fetched, doesn't mean it actually updated + _confirm_update $cname $img_id done + # Final confirmation that all image IDs have/haven't changed for cname in "${cnames[@]}"; do run_podman inspect --format "{{.Image}}" $cname if [[ -n "${expect_update[$cname]}" ]]; then - [[ "$output" != "$img_id" ]] + if [[ "$output" == "$img_id" ]]; then + die "$cname: image ID ($output) did not change" + fi else is "$output" "$img_id" "Image should not be changed." fi @@ -255,25 +251,24 @@ EOF systemctl enable --now podman-auto-update-$cname.timer systemctl list-timers --all - count=0 - failed_start=1 + local expect='Finished Podman auto-update testing service' + local failed_start=failed + local count=0 while [ $count -lt 120 ]; do run journalctl -n 15 -u podman-auto-update-$cname.service - if [[ "$output" =~ "Finished Podman auto-update testing service" ]]; then - failed_start=0 + if [[ "$output" =~ $expect ]]; then + failed_start= break fi ((count+=1)) sleep 1 done - echo $output - _confirm_update container-$cname.service - run_podman inspect --format "{{.Image}}" $cname - if [[ $failed_start == 1 ]]; then - die "Failed to get podman auto-update service finished" + if [[ -n "$failed_start" ]]; then + die "Did not find expected string '$expect' in journalctl output for $cname" fi - [[ "$output" != "$ori_image" ]] + + _confirm_update $cname $ori_image } # vim: filetype=sh diff --git a/test/system/410-selinux.bats b/test/system/410-selinux.bats index f8cee0e59..4ef9c8b30 100644 --- a/test/system/410-selinux.bats +++ b/test/system/410-selinux.bats @@ -183,7 +183,10 @@ function check_label() { # runc and crun emit different diagnostics runtime=$(podman_runtime) case "$runtime" in - crun) expect="\`/proc/thread-self/attr/exec\`: OCI runtime error: unable to assign security attribute" ;; + # crun 0.20.1 changes the error message + # from /proc/thread-self/attr/exec`: .* unable to assign + # to /proc/self/attr/keycreate`: .* unable to process + crun) expect="\`/proc/.*\`: OCI runtime error: unable to \(assign\|process\) security attribute" ;; runc) expect="OCI runtime error: .*: failed to set /proc/self/attr/keycreate on procfs" ;; *) skip "Unknown runtime '$runtime'";; esac diff --git a/test/system/build-testimage b/test/system/build-testimage index 3e5b982ce..eb5849b5e 100755 --- a/test/system/build-testimage +++ b/test/system/build-testimage @@ -61,8 +61,8 @@ chmod 755 pause # - check for updates @ https://hub.docker.com/_/alpine # busybox-extras provides httpd needed in 500-networking.bats cat >Containerfile <<EOF -ARG ARCH=please-override-arch -FROM docker.io/\${ARCH}/alpine:3.12.0 +ARG REPO=please-override-repo +FROM docker.io/\${REPO}/alpine:3.13.5 RUN apk add busybox-extras ADD testimage-id pause /home/podman/ LABEL created_by=$create_script @@ -74,17 +74,46 @@ EOF # --squash-all : needed by 'tree' test in 070-build.bats podman rmi -f testimage &> /dev/null || true +# There should always be a testimage tagged ':0000000<X>' (eight digits, +# zero-padded sequence ID) in the same location; this is used by tests +# which need to pull a non-locally-cached image. This image will rarely +# if ever need to change, nor in fact does it even have to be a copy of +# this testimage since all we use it for is 'true'. +# However, it does need to be multiarch :-( +zerotag_latest=$(skopeo list-tags docker://quay.io/libpod/testimage |\ + jq -r '.Tags[]' |\ + sort --version-sort |\ + grep '^000' |\ + tail -n 1) +zerotag_next=$(printf "%08d" $((zerotag_latest + 1))) + +# We don't always need to push the :00xx image, but build it anyway. +zeroimg=quay.io/libpod/testimage:${zerotag_next} +buildah manifest create $zeroimg + # We need to use buildah because (as of 2021-02-23) only buildah has --manifest # and because Dan says arch emulation is not currently working on podman # (no further details). # Arch emulation on Fedora requires the qemu-user-static package. -for arch in amd64 arm64v8 ppc64le s390x;do +for arch in amd64 arm64 ppc64le s390x;do + # docker.io repo is usually the same name as the desired arch; except + # for arm64, where podman needs to have the arch be 'arm64' but the + # image lives in 'arm64v8'. + repo=$arch + if [[ $repo = "arm64" ]]; then + repo="${repo}v8" + fi + ${BUILDAH} bud \ --arch=$arch \ - --build-arg ARCH=$arch \ + --build-arg REPO=$repo \ --manifest=testimage \ --squash \ . + + # The zero-tag image + ${BUILDAH} pull --arch $arch docker.io/$repo/busybox:1.33.1 + ${BUILDAH} manifest add $zeroimg docker.io/$repo/busybox:1.33.1 done # Clean up @@ -94,23 +123,13 @@ rm -rf $tmpdir # Tag image and push (all arches) to quay. remote_tag=quay.io/libpod/testimage:$YMD podman tag testimage ${remote_tag} -${BUILDAH} manifest push --all ${remote_tag} docker://${remote_tag} +cat <<EOF -# Side note: there should always be a testimage tagged ':0000000<X>' -# (eight digits, zero-padded sequence ID) in the same location; this is -# used by tests which need to pull a non-locally-cached image. This -# image will rarely if ever need to change, nor in fact does it even -# have to be a copy of this testimage since all we use it for is 'true'. -# However, it does need to be multiarch :-( -# -# As of 2021-02-24 it is simply busybox, because it is super small, -# but it's complicated because of multiarch: -# -# img=quay.io/libpod/testimage:0000000<current+1> -# buildah manifest create $img -# for arch in amd64 arm64v8 ppc64le s390x;do -# buildah pull --arch $arch docker.io/$arch/busybox:1.32.0 -# buildah manifest add $img docker.io/$arch/busybox:1.32.0 -# done -# buildah manifest push --all $img docker://$img -# +If you're happy with these images, run: + + ${BUILDAH} manifest push --all ${remote_tag} docker://${remote_tag} + ${BUILDAH} manifest push --all ${zeroimg} docker://${zeroimg} + +(You do not always need to push the :0000 image) + +EOF diff --git a/test/system/helpers.bash b/test/system/helpers.bash index e0c208f57..1859a2168 100644 --- a/test/system/helpers.bash +++ b/test/system/helpers.bash @@ -7,14 +7,15 @@ PODMAN=${PODMAN:-podman} PODMAN_TEST_IMAGE_REGISTRY=${PODMAN_TEST_IMAGE_REGISTRY:-"quay.io"} PODMAN_TEST_IMAGE_USER=${PODMAN_TEST_IMAGE_USER:-"libpod"} PODMAN_TEST_IMAGE_NAME=${PODMAN_TEST_IMAGE_NAME:-"testimage"} -PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"20210427"} +PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"20210610"} PODMAN_TEST_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG" PODMAN_TEST_IMAGE_ID= # Remote image that we *DO NOT* fetch or keep by default; used for testing pull -# This changed from 0 to 1 on 2021-02-24 due to multiarch considerations; it -# should change only very rarely. -PODMAN_NONLOCAL_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:00000002" +# This has changed in 2021, from 0 through 3, various iterations of getting +# multiarch to work. It should change only very rarely. +PODMAN_NONLOCAL_IMAGE_TAG=${PODMAN_NONLOCAL_IMAGE_TAG:-"00000003"} +PODMAN_NONLOCAL_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:$PODMAN_NONLOCAL_IMAGE_TAG" # Because who wants to spell that out each time? IMAGE=$PODMAN_TEST_IMAGE_FQN diff --git a/vendor/github.com/containers/buildah/CHANGELOG.md b/vendor/github.com/containers/buildah/CHANGELOG.md index 7a0af9298..ab704400f 100644 --- a/vendor/github.com/containers/buildah/CHANGELOG.md +++ b/vendor/github.com/containers/buildah/CHANGELOG.md @@ -1,6 +1,9 @@  # Changelog +## v1.21.1 (2021-06-02) + Fix handling of auth.json file while in a user namespace + ## v1.21.0 (2021-05-19) Don't blow up if cpp detects errors Vendor in containers/common v0.38.4 diff --git a/vendor/github.com/containers/buildah/changelog.txt b/vendor/github.com/containers/buildah/changelog.txt index 3bde05a3e..ee0e9b61f 100644 --- a/vendor/github.com/containers/buildah/changelog.txt +++ b/vendor/github.com/containers/buildah/changelog.txt @@ -1,3 +1,6 @@ +- Changelog for v1.21.1 (2021-06-03) + * Fix handling of auth.json file while in a user namespace + - Changelog for v1.21.0 (2021-05-19) * Don't blow up if cpp detects errors * Vendor in containers/common v0.38.4 diff --git a/vendor/github.com/containers/buildah/define/types.go b/vendor/github.com/containers/buildah/define/types.go index f2327e8df..ac2071b37 100644 --- a/vendor/github.com/containers/buildah/define/types.go +++ b/vendor/github.com/containers/buildah/define/types.go @@ -28,7 +28,7 @@ const ( Package = "buildah" // Version for the Package. Bump version in contrib/rpm/buildah.spec // too. - Version = "1.21.0" + Version = "1.21.1" // DefaultRuntime if containers.conf fails. DefaultRuntime = "runc" diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index 96cd6ee1e..c78d39b8e 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.32.1 +1.32.2 diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index d5d161bfd..87d608393 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -1189,7 +1189,11 @@ func (d *Driver) get(id string, disableShifting bool, options graphdriver.MountO if d.usingMetacopy { optsList = append(optsList, "metacopy=on") } else { - logrus.Warnf("ignoring metacopy option from storage.conf, not supported with booted kernel") + logLevel := logrus.WarnLevel + if unshare.IsRootless() { + logLevel = logrus.DebugLevel + } + logrus.StandardLogger().Logf(logLevel, "ignoring metacopy option from storage.conf, not supported with booted kernel") } } } diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index e7ca56e64..2217ea739 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -10,15 +10,15 @@ require ( github.com/google/go-intervals v0.0.2 github.com/hashicorp/go-multierror v1.1.1 github.com/json-iterator/go v1.1.11 - github.com/klauspost/compress v1.12.3 + github.com/klauspost/compress v1.13.0 github.com/klauspost/pgzip v1.2.5 - github.com/mattn/go-shellwords v1.0.11 + github.com/mattn/go-shellwords v1.0.12 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible github.com/moby/sys/mountinfo v0.4.1 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/runc v1.0.0-rc95 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 - github.com/opencontainers/selinux v1.8.1 + github.com/opencontainers/selinux v1.8.2 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 diff --git a/vendor/github.com/containers/storage/go.sum b/vendor/github.com/containers/storage/go.sum index 5373d0597..6cd6df17c 100644 --- a/vendor/github.com/containers/storage/go.sum +++ b/vendor/github.com/containers/storage/go.sum @@ -77,6 +77,8 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= +github.com/bits-and-blooms/bitset v1.2.0 h1:Kn4yilvwNtMACtf1eYDlG8H77R07mZSPbMjLyS07ChA= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= @@ -383,8 +385,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.12.3 h1:G5AfA94pHPysR56qqrkO2pxEexdDzrpFJ6yt/VqWxVU= -github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.0 h1:2T7tUoQrQT+fQWdaY5rjWztFGAFwbGD04iPJg90ZiOs= +github.com/klauspost/compress v1.13.0/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -408,8 +410,8 @@ github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaO github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vqg+NOMyg4B2o= -github.com/mattn/go-shellwords v1.0.11 h1:vCoR9VPpsk/TZFW2JwK5I9S0xdrtUq2bph6/YjEPnaw= -github.com/mattn/go-shellwords v1.0.11/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= +github.com/mattn/go-shellwords v1.0.12 h1:M2zGm7EW6UQJvDeQxo4T51eKPurbeFbe8WtebGE2xrk= +github.com/mattn/go-shellwords v1.0.12/go.mod h1:EZzvwXDESEeg03EKmM+RmDnNOPKG4lLtQsUlTZDWQ8Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= @@ -476,8 +478,8 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.m github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= -github.com/opencontainers/selinux v1.8.1 h1:yvEZh7CsfnJNwKzG9ZeXwbvR05RAZsu5RS/3vA6qFTA= -github.com/opencontainers/selinux v1.8.1/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= +github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc= +github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= @@ -588,7 +590,6 @@ github.com/vishvananda/netns v0.0.0-20180720170159-13995c7128cc/go.mod h1:ZjcWmF github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/vishvananda/netns v0.0.0-20200728191858-db3c7e526aae/go.mod h1:DD4vA1DwXk04H54A1oHXtwZmA0grkVMdPxx/VGLCah0= github.com/willf/bitset v1.1.11-0.20200630133818-d5bec3311243/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= -github.com/willf/bitset v1.1.11 h1:N7Z7E9UvjW+sGsEl7k/SJrvY2reP1A07MrGuCjIOjRE= github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= diff --git a/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go b/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go index 0a92da2c0..f0e46b6b6 100644 --- a/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go +++ b/vendor/github.com/containers/storage/pkg/lockfile/lockfile_unix.go @@ -194,11 +194,7 @@ func (l *lockfile) Touch() error { defer l.stateMutex.Unlock() l.lw = stringid.GenerateRandomID() id := []byte(l.lw) - _, err := unix.Seek(int(l.fd), 0, os.SEEK_SET) - if err != nil { - return err - } - n, err := unix.Write(int(l.fd), id) + n, err := unix.Pwrite(int(l.fd), id, 0) if err != nil { return err } @@ -217,11 +213,7 @@ func (l *lockfile) Modified() (bool, error) { panic("attempted to check last-writer in lockfile without locking it first") } defer l.stateMutex.Unlock() - _, err := unix.Seek(int(l.fd), 0, os.SEEK_SET) - if err != nil { - return true, err - } - n, err := unix.Read(int(l.fd), id) + n, err := unix.Pread(int(l.fd), id, 0) if err != nil { return true, err } diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index d6d547c64..dc6eaafa2 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -937,7 +937,7 @@ func (s *store) ContainerStore() (ContainerStore, error) { } func (s *store) canUseShifting(uidmap, gidmap []idtools.IDMap) bool { - if !s.graphDriver.SupportsShifting() { + if s.graphDriver == nil || !s.graphDriver.SupportsShifting() { return false } if uidmap != nil && !idtools.IsContiguous(uidmap) { @@ -2668,6 +2668,10 @@ func (s *store) mount(id string, options drivers.MountOpts) (string, error) { s.lastLoaded = time.Now() } + if options.UidMaps != nil || options.GidMaps != nil { + options.DisableShifting = !s.canUseShifting(options.UidMaps, options.GidMaps) + } + if rlstore.Exists(id) { return rlstore.Mount(id, options) } @@ -2708,7 +2712,6 @@ func (s *store) Mount(id, mountLabel string) (string, error) { options.Volatile = v.(bool) } } - options.DisableShifting = !s.canUseShifting(container.UIDMap, container.GIDMap) } return s.mount(id, options) } diff --git a/vendor/github.com/containers/storage/types/default_override_test.conf b/vendor/github.com/containers/storage/types/default_override_test.conf new file mode 100644 index 000000000..caa537ba9 --- /dev/null +++ b/vendor/github.com/containers/storage/types/default_override_test.conf @@ -0,0 +1,11 @@ +[storage] + +# Default Storage Driver +driver = "" + +# Primary Read/Write location of container storage +graphroot = "environment_override_graphroot" + +# Storage path for rootless users +# +rootless_storage_path = "environment_override_rootless_storage_path" diff --git a/vendor/github.com/containers/storage/types/utils.go b/vendor/github.com/containers/storage/types/utils.go index d2dca7b68..4d62b151a 100644 --- a/vendor/github.com/containers/storage/types/utils.go +++ b/vendor/github.com/containers/storage/types/utils.go @@ -160,7 +160,14 @@ func expandEnvPath(path string, rootlessUID int) (string, error) { } func DefaultConfigFile(rootless bool) (string, error) { - if defaultConfigFileSet || !rootless { + if defaultConfigFileSet { + return defaultConfigFile, nil + } + + if path, ok := os.LookupEnv("CONTAINERS_STORAGE_CONF"); ok { + return path, nil + } + if !rootless { return defaultConfigFile, nil } diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 6cea054d2..e30af505c 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -144,7 +144,7 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { } cSize = 1 case blockTypeCompressed: - if debug { + if debugDecoder { println("Data size on stream:", cSize) } b.RLESize = 0 @@ -153,7 +153,7 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { maxSize = int(windowSize) } if cSize > maxCompressedBlockSize || uint64(cSize) > b.WindowSize { - if debug { + if debugDecoder { printf("compressed block too big: csize:%d block: %+v\n", uint64(cSize), b) } return ErrCompressedSizeTooBig @@ -179,7 +179,7 @@ func (b *blockDec) reset(br byteBuffer, windowSize uint64) error { } b.data, err = br.readBig(cSize, b.dataStorage) if err != nil { - if debug { + if debugDecoder { println("Reading block:", err, "(", cSize, ")", len(b.data)) printf("%T", br) } @@ -249,7 +249,7 @@ func (b *blockDec) startDecoder() { b: b.dst, err: err, } - if debug { + if debugDecoder { println("Decompressed to", len(b.dst), "bytes, error:", err) } b.result <- o @@ -264,7 +264,7 @@ func (b *blockDec) startDecoder() { default: panic("Invalid block type") } - if debug { + if debugDecoder { println("blockDec: Finished block") } } @@ -297,7 +297,7 @@ func (b *blockDec) decodeBuf(hist *history) error { b.dst = hist.b hist.b = nil err := b.decodeCompressed(hist) - if debug { + if debugDecoder { println("Decompressed to total", len(b.dst), "bytes, hash:", xxhash.Sum64(b.dst), "error:", err) } hist.b = b.dst @@ -390,7 +390,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { in = in[5:] } } - if debug { + if debugDecoder { println("literals type:", litType, "litRegenSize:", litRegenSize, "litCompSize:", litCompSize, "sizeFormat:", sizeFormat, "4X:", fourStreams) } var literals []byte @@ -428,7 +428,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { literals[i] = v } in = in[1:] - if debug { + if debugDecoder { printf("Found %d RLE compressed literals\n", litRegenSize) } case literalsBlockTreeless: @@ -439,7 +439,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { // Store compressed literals, so we defer decoding until we get history. literals = in[:litCompSize] in = in[litCompSize:] - if debug { + if debugDecoder { printf("Found %d compressed literals\n", litCompSize) } case literalsBlockCompressed: @@ -481,7 +481,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { if len(literals) != litRegenSize { return fmt.Errorf("literal output size mismatch want %d, got %d", litRegenSize, len(literals)) } - if debug { + if debugDecoder { printf("Decompressed %d literals into %d bytes\n", litCompSize, litRegenSize) } } @@ -532,12 +532,12 @@ func (b *blockDec) decodeCompressed(hist *history) error { br := byteReader{b: in, off: 0} compMode := br.Uint8() br.advance(1) - if debug { + if debugDecoder { printf("Compression modes: 0b%b", compMode) } for i := uint(0); i < 3; i++ { mode := seqCompMode((compMode >> (6 - i*2)) & 3) - if debug { + if debugDecoder { println("Table", tableIndex(i), "is", mode) } var seq *sequenceDec @@ -568,7 +568,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { } dec.setRLE(symb) seq.fse = dec - if debug { + if debugDecoder { printf("RLE set to %+v, code: %v", symb, v) } case compModeFSE: @@ -584,7 +584,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { println("Transform table error:", err) return err } - if debug { + if debugDecoder { println("Read table ok", "symbolLen:", dec.symbolLen) } seq.fse = dec @@ -652,7 +652,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { if huff != nil { hist.huffTree = huff } - if debug { + if debugDecoder { println("Final literals:", len(literals), "hash:", xxhash.Sum64(literals), "and", nSeqs, "sequences.") } @@ -669,7 +669,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { if err != nil { return err } - if debug { + if debugDecoder { println("History merged ok") } br := &bitReader{} @@ -728,7 +728,7 @@ func (b *blockDec) decodeCompressed(hist *history) error { } hist.append(b.dst) hist.recentOffsets = seqs.prevOffset - if debug { + if debugDecoder { println("Finished block with literals:", len(literals), "and", nSeqs, "sequences.") } diff --git a/vendor/github.com/klauspost/compress/zstd/blockenc.go b/vendor/github.com/klauspost/compress/zstd/blockenc.go index e1be092f3..3df185ee4 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockenc.go +++ b/vendor/github.com/klauspost/compress/zstd/blockenc.go @@ -156,7 +156,7 @@ func (h *literalsHeader) setSize(regenLen int) { switch { case inBits < 5: lh |= (uint64(regenLen) << 3) | (1 << 60) - if debug { + if debugEncoder { got := int(lh>>3) & 0xff if got != regenLen { panic(fmt.Sprint("litRegenSize = ", regenLen, "(want) != ", got, "(got)")) @@ -184,7 +184,7 @@ func (h *literalsHeader) setSizes(compLen, inLen int, single bool) { lh |= 1 << 2 } lh |= (uint64(inLen) << 4) | (uint64(compLen) << (10 + 4)) | (3 << 60) - if debug { + if debugEncoder { const mmask = (1 << 24) - 1 n := (lh >> 4) & mmask if int(n&1023) != inLen { @@ -312,7 +312,7 @@ func (b *blockEnc) encodeRaw(a []byte) { bh.setType(blockTypeRaw) b.output = bh.appendTo(b.output[:0]) b.output = append(b.output, a...) - if debug { + if debugEncoder { println("Adding RAW block, length", len(a), "last:", b.last) } } @@ -325,7 +325,7 @@ func (b *blockEnc) encodeRawTo(dst, src []byte) []byte { bh.setType(blockTypeRaw) dst = bh.appendTo(dst) dst = append(dst, src...) - if debug { + if debugEncoder { println("Adding RAW block, length", len(src), "last:", b.last) } return dst @@ -339,7 +339,7 @@ func (b *blockEnc) encodeLits(lits []byte, raw bool) error { // Don't compress extremely small blocks if len(lits) < 8 || (len(lits) < 32 && b.dictLitEnc == nil) || raw { - if debug { + if debugEncoder { println("Adding RAW block, length", len(lits), "last:", b.last) } bh.setType(blockTypeRaw) @@ -371,7 +371,7 @@ func (b *blockEnc) encodeLits(lits []byte, raw bool) error { switch err { case huff0.ErrIncompressible: - if debug { + if debugEncoder { println("Adding RAW block, length", len(lits), "last:", b.last) } bh.setType(blockTypeRaw) @@ -379,7 +379,7 @@ func (b *blockEnc) encodeLits(lits []byte, raw bool) error { b.output = append(b.output, lits...) return nil case huff0.ErrUseRLE: - if debug { + if debugEncoder { println("Adding RLE block, length", len(lits)) } bh.setType(blockTypeRLE) @@ -396,12 +396,12 @@ func (b *blockEnc) encodeLits(lits []byte, raw bool) error { bh.setType(blockTypeCompressed) var lh literalsHeader if reUsed { - if debug { + if debugEncoder { println("Reused tree, compressed to", len(out)) } lh.setType(literalsBlockTreeless) } else { - if debug { + if debugEncoder { println("New tree, compressed to", len(out), "tree size:", len(b.litEnc.OutTable)) } lh.setType(literalsBlockCompressed) @@ -517,7 +517,7 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { lh.setSize(len(b.literals)) b.output = lh.appendTo(b.output) b.output = append(b.output, b.literals...) - if debug { + if debugEncoder { println("Adding literals RAW, length", len(b.literals)) } case huff0.ErrUseRLE: @@ -525,22 +525,22 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { lh.setSize(len(b.literals)) b.output = lh.appendTo(b.output) b.output = append(b.output, b.literals[0]) - if debug { + if debugEncoder { println("Adding literals RLE") } case nil: // Compressed litLen... if reUsed { - if debug { + if debugEncoder { println("reused tree") } lh.setType(literalsBlockTreeless) } else { - if debug { + if debugEncoder { println("new tree, size:", len(b.litEnc.OutTable)) } lh.setType(literalsBlockCompressed) - if debug { + if debugEncoder { _, _, err := huff0.ReadTable(out, nil) if err != nil { panic(err) @@ -548,18 +548,18 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { } } lh.setSizes(len(out), len(b.literals), single) - if debug { + if debugEncoder { printf("Compressed %d literals to %d bytes", len(b.literals), len(out)) println("Adding literal header:", lh) } b.output = lh.appendTo(b.output) b.output = append(b.output, out...) b.litEnc.Reuse = huff0.ReusePolicyAllow - if debug { + if debugEncoder { println("Adding literals compressed") } default: - if debug { + if debugEncoder { println("Adding literals ERROR:", err) } return err @@ -577,7 +577,7 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { n := len(b.sequences) - 0x7f00 b.output = append(b.output, 255, uint8(n), uint8(n>>8)) } - if debug { + if debugEncoder { println("Encoding", len(b.sequences), "sequences") } b.genCodes() @@ -611,17 +611,17 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { nSize = nSize + (nSize+2*8*16)>>4 switch { case predefSize <= prevSize && predefSize <= nSize || forcePreDef: - if debug { + if debugEncoder { println("Using predefined", predefSize>>3, "<=", nSize>>3) } return preDef, compModePredefined case prevSize <= nSize: - if debug { + if debugEncoder { println("Using previous", prevSize>>3, "<=", nSize>>3) } return prev, compModeRepeat default: - if debug { + if debugEncoder { println("Using new, predef", predefSize>>3, ". previous:", prevSize>>3, ">", nSize>>3, "header max:", cur.maxHeaderSize()>>3, "bytes") println("tl:", cur.actualTableLog, "symbolLen:", cur.symbolLen, "norm:", cur.norm[:cur.symbolLen], "hist", cur.count[:cur.symbolLen]) } @@ -634,7 +634,7 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { if llEnc.useRLE { mode |= uint8(compModeRLE) << 6 llEnc.setRLE(b.sequences[0].llCode) - if debug { + if debugEncoder { println("llEnc.useRLE") } } else { @@ -645,7 +645,7 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { if ofEnc.useRLE { mode |= uint8(compModeRLE) << 4 ofEnc.setRLE(b.sequences[0].ofCode) - if debug { + if debugEncoder { println("ofEnc.useRLE") } } else { @@ -657,7 +657,7 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { if mlEnc.useRLE { mode |= uint8(compModeRLE) << 2 mlEnc.setRLE(b.sequences[0].mlCode) - if debug { + if debugEncoder { println("mlEnc.useRLE, code: ", b.sequences[0].mlCode, "value", b.sequences[0].matchLen) } } else { @@ -666,7 +666,7 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { mode |= uint8(m) << 2 } b.output = append(b.output, mode) - if debug { + if debugEncoder { printf("Compression modes: 0b%b", mode) } b.output, err = llEnc.writeCount(b.output) @@ -786,7 +786,7 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { // Size is output minus block header. bh.setSize(uint32(len(b.output)-bhOffset) - 3) - if debug { + if debugEncoder { println("Rewriting block header", bh) } _ = bh.appendTo(b.output[bhOffset:bhOffset]) diff --git a/vendor/github.com/klauspost/compress/zstd/bytebuf.go b/vendor/github.com/klauspost/compress/zstd/bytebuf.go index 17e820a6a..aab71c6cf 100644 --- a/vendor/github.com/klauspost/compress/zstd/bytebuf.go +++ b/vendor/github.com/klauspost/compress/zstd/bytebuf.go @@ -91,7 +91,7 @@ func (r *readerWrapper) readSmall(n int) ([]byte, error) { if err == io.EOF { return nil, io.ErrUnexpectedEOF } - if debug { + if debugDecoder { println("readSmall: got", n2, "want", n, "err", err) } return nil, err diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index f593e464b..4d984c3b2 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -113,9 +113,6 @@ func NewReader(r io.Reader, opts ...DOption) (*Decoder, error) { // Returns the number of bytes written and any error that occurred. // When the stream is done, io.EOF will be returned. func (d *Decoder) Read(p []byte) (int, error) { - if d.stream == nil { - return 0, ErrDecoderNilInput - } var n int for { if len(d.current.b) > 0 { @@ -138,7 +135,7 @@ func (d *Decoder) Read(p []byte) (int, error) { } } if len(d.current.b) > 0 { - if debug { + if debugDecoder { println("returning", n, "still bytes left:", len(d.current.b)) } // Only return error at end of block @@ -147,7 +144,7 @@ func (d *Decoder) Read(p []byte) (int, error) { if d.current.err != nil { d.drainOutput() } - if debug { + if debugDecoder { println("returning", n, d.current.err, len(d.decoders)) } return n, d.current.err @@ -167,20 +164,17 @@ func (d *Decoder) Reset(r io.Reader) error { if r == nil { d.current.err = ErrDecoderNilInput + if len(d.current.b) > 0 { + d.current.b = d.current.b[:0] + } d.current.flushed = true return nil } - if d.stream == nil { - d.stream = make(chan decodeStream, 1) - d.streamWg.Add(1) - go d.startStreamDecoder(d.stream) - } - - // If bytes buffer and < 1MB, do sync decoding anyway. - if bb, ok := r.(byter); ok && bb.Len() < 1<<20 { + // If bytes buffer and < 5MB, do sync decoding anyway. + if bb, ok := r.(byter); ok && bb.Len() < 5<<20 { bb2 := bb - if debug { + if debugDecoder { println("*bytes.Buffer detected, doing sync decode, len:", bb.Len()) } b := bb2.Bytes() @@ -196,12 +190,18 @@ func (d *Decoder) Reset(r io.Reader) error { d.current.b = dst d.current.err = err d.current.flushed = true - if debug { + if debugDecoder { println("sync decode to", len(dst), "bytes, err:", err) } return nil } + if d.stream == nil { + d.stream = make(chan decodeStream, 1) + d.streamWg.Add(1) + go d.startStreamDecoder(d.stream) + } + // Remove current block. d.current.decodeOutput = decodeOutput{} d.current.err = nil @@ -225,7 +225,7 @@ func (d *Decoder) drainOutput() { d.current.cancel = nil } if d.current.d != nil { - if debug { + if debugDecoder { printf("re-adding current decoder %p, decoders: %d", d.current.d, len(d.decoders)) } d.decoders <- d.current.d @@ -238,7 +238,7 @@ func (d *Decoder) drainOutput() { } for v := range d.current.output { if v.d != nil { - if debug { + if debugDecoder { printf("re-adding decoder %p", v.d) } d.decoders <- v.d @@ -255,9 +255,6 @@ func (d *Decoder) drainOutput() { // The return value n is the number of bytes written. // Any error encountered during the write is also returned. func (d *Decoder) WriteTo(w io.Writer) (int64, error) { - if d.stream == nil { - return 0, ErrDecoderNilInput - } var n int64 for { if len(d.current.b) > 0 { @@ -297,7 +294,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { block := <-d.decoders frame := block.localFrame defer func() { - if debug { + if debugDecoder { printf("re-adding decoder: %p", block) } frame.rawInput = nil @@ -310,7 +307,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { frame.history.reset() err := frame.reset(&frame.bBuf) if err == io.EOF { - if debug { + if debugDecoder { println("frame reset return EOF") } return dst, nil @@ -355,7 +352,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { return dst, err } if len(frame.bBuf) == 0 { - if debug { + if debugDecoder { println("frame dbuf empty") } break @@ -371,7 +368,7 @@ func (d *Decoder) DecodeAll(input, dst []byte) ([]byte, error) { // if no data was available without blocking. func (d *Decoder) nextBlock(blocking bool) (ok bool) { if d.current.d != nil { - if debug { + if debugDecoder { printf("re-adding current decoder %p", d.current.d) } d.decoders <- d.current.d @@ -391,7 +388,7 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) { return false } } - if debug { + if debugDecoder { println("got", len(d.current.b), "bytes, error:", d.current.err) } return true @@ -485,7 +482,7 @@ func (d *Decoder) startStreamDecoder(inStream chan decodeStream) { defer d.streamWg.Done() frame := newFrameDec(d.o) for stream := range inStream { - if debug { + if debugDecoder { println("got new stream") } br := readerWrapper{r: stream.r} @@ -493,7 +490,7 @@ func (d *Decoder) startStreamDecoder(inStream chan decodeStream) { for { frame.history.reset() err := frame.reset(&br) - if debug && err != nil { + if debugDecoder && err != nil { println("Frame decoder returned", err) } if err == nil && frame.DictionaryID != nil { @@ -510,7 +507,7 @@ func (d *Decoder) startStreamDecoder(inStream chan decodeStream) { } break } - if debug { + if debugDecoder { println("starting frame decoder") } diff --git a/vendor/github.com/klauspost/compress/zstd/dict.go b/vendor/github.com/klauspost/compress/zstd/dict.go index fa25a18d8..a36ae83ef 100644 --- a/vendor/github.com/klauspost/compress/zstd/dict.go +++ b/vendor/github.com/klauspost/compress/zstd/dict.go @@ -82,7 +82,7 @@ func loadDict(b []byte) (*dict, error) { println("Transform table error:", err) return err } - if debug { + if debugDecoder || debugEncoder { println("Read table ok", "symbolLen:", dec.symbolLen) } // Set decoders as predefined so they aren't reused. diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go index dc1eed5f0..b7d4b9004 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_best.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go @@ -132,7 +132,7 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) { } _ = addLiterals - if debug { + if debugEncoder { println("recent offsets:", blk.recentOffsets) } @@ -274,7 +274,7 @@ encodeLoop: nextEmit = s if s >= sLimit { - if debug { + if debugEncoder { println("repeat ended", s, best.length) } @@ -412,7 +412,7 @@ encodeLoop: blk.recentOffsets[0] = uint32(offset1) blk.recentOffsets[1] = uint32(offset2) blk.recentOffsets[2] = uint32(offset3) - if debug { + if debugEncoder { println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index 604954290..eab7b5083 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -138,7 +138,7 @@ func (e *betterFastEncoder) Encode(blk *blockEnc, src []byte) { blk.literals = append(blk.literals, src[nextEmit:until]...) s.litLen = uint32(until - nextEmit) } - if debug { + if debugEncoder { println("recent offsets:", blk.recentOffsets) } @@ -204,7 +204,7 @@ encodeLoop: nextEmit = s if s >= sLimit { - if debug { + if debugEncoder { println("repeat ended", s, lenght) } @@ -264,7 +264,7 @@ encodeLoop: s += lenght + repOff2 nextEmit = s if s >= sLimit { - if debug { + if debugEncoder { println("repeat ended", s, lenght) } @@ -553,7 +553,7 @@ encodeLoop: } blk.recentOffsets[0] = uint32(offset1) blk.recentOffsets[1] = uint32(offset2) - if debug { + if debugEncoder { println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } } @@ -656,7 +656,7 @@ func (e *betterFastEncoderDict) Encode(blk *blockEnc, src []byte) { blk.literals = append(blk.literals, src[nextEmit:until]...) s.litLen = uint32(until - nextEmit) } - if debug { + if debugEncoder { println("recent offsets:", blk.recentOffsets) } @@ -724,7 +724,7 @@ encodeLoop: nextEmit = s if s >= sLimit { - if debug { + if debugEncoder { println("repeat ended", s, lenght) } @@ -787,7 +787,7 @@ encodeLoop: s += lenght + repOff2 nextEmit = s if s >= sLimit { - if debug { + if debugEncoder { println("repeat ended", s, lenght) } @@ -1084,7 +1084,7 @@ encodeLoop: } blk.recentOffsets[0] = uint32(offset1) blk.recentOffsets[1] = uint32(offset2) - if debug { + if debugEncoder { println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index 8629d43d8..96b21b90e 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -109,7 +109,7 @@ func (e *doubleFastEncoder) Encode(blk *blockEnc, src []byte) { blk.literals = append(blk.literals, src[nextEmit:until]...) s.litLen = uint32(until - nextEmit) } - if debug { + if debugEncoder { println("recent offsets:", blk.recentOffsets) } @@ -170,7 +170,7 @@ encodeLoop: s += lenght + repOff nextEmit = s if s >= sLimit { - if debug { + if debugEncoder { println("repeat ended", s, lenght) } @@ -368,7 +368,7 @@ encodeLoop: } blk.recentOffsets[0] = uint32(offset1) blk.recentOffsets[1] = uint32(offset2) - if debug { + if debugEncoder { println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } } @@ -427,7 +427,7 @@ func (e *doubleFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { blk.literals = append(blk.literals, src[nextEmit:until]...) s.litLen = uint32(until - nextEmit) } - if debug { + if debugEncoder { println("recent offsets:", blk.recentOffsets) } @@ -483,7 +483,7 @@ encodeLoop: s += length + repOff nextEmit = s if s >= sLimit { - if debug { + if debugEncoder { println("repeat ended", s, length) } @@ -677,7 +677,7 @@ encodeLoop: blk.literals = append(blk.literals, src[nextEmit:]...) blk.extraLits = len(src) - int(nextEmit) } - if debug { + if debugEncoder { println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } @@ -767,7 +767,7 @@ func (e *doubleFastEncoderDict) Encode(blk *blockEnc, src []byte) { blk.literals = append(blk.literals, src[nextEmit:until]...) s.litLen = uint32(until - nextEmit) } - if debug { + if debugEncoder { println("recent offsets:", blk.recentOffsets) } @@ -830,7 +830,7 @@ encodeLoop: s += lenght + repOff nextEmit = s if s >= sLimit { - if debug { + if debugEncoder { println("repeat ended", s, lenght) } @@ -1039,7 +1039,7 @@ encodeLoop: } blk.recentOffsets[0] = uint32(offset1) blk.recentOffsets[1] = uint32(offset2) - if debug { + if debugEncoder { println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } // If we encoded more than 64K mark all dirty. diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index ba4a17e10..2246d286d 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -103,7 +103,7 @@ func (e *fastEncoder) Encode(blk *blockEnc, src []byte) { blk.literals = append(blk.literals, src[nextEmit:until]...) s.litLen = uint32(until - nextEmit) } - if debug { + if debugEncoder { println("recent offsets:", blk.recentOffsets) } @@ -178,7 +178,7 @@ encodeLoop: s += length + 2 nextEmit = s if s >= sLimit { - if debug { + if debugEncoder { println("repeat ended", s, length) } @@ -330,7 +330,7 @@ encodeLoop: } blk.recentOffsets[0] = uint32(offset1) blk.recentOffsets[1] = uint32(offset2) - if debug { + if debugEncoder { println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } } @@ -343,7 +343,7 @@ func (e *fastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { inputMargin = 8 minNonLiteralBlockSize = 1 + 1 + inputMargin ) - if debug { + if debugEncoder { if len(src) > maxBlockSize { panic("src too big") } @@ -391,7 +391,7 @@ func (e *fastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { blk.literals = append(blk.literals, src[nextEmit:until]...) s.litLen = uint32(until - nextEmit) } - if debug { + if debugEncoder { println("recent offsets:", blk.recentOffsets) } @@ -462,7 +462,7 @@ encodeLoop: s += length + 2 nextEmit = s if s >= sLimit { - if debug { + if debugEncoder { println("repeat ended", s, length) } @@ -616,7 +616,7 @@ encodeLoop: blk.literals = append(blk.literals, src[nextEmit:]...) blk.extraLits = len(src) - int(nextEmit) } - if debug { + if debugEncoder { println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } // We do not store history, so we must offset e.cur to avoid false matches for next user. @@ -696,7 +696,7 @@ func (e *fastEncoderDict) Encode(blk *blockEnc, src []byte) { blk.literals = append(blk.literals, src[nextEmit:until]...) s.litLen = uint32(until - nextEmit) } - if debug { + if debugEncoder { println("recent offsets:", blk.recentOffsets) } @@ -773,7 +773,7 @@ encodeLoop: s += length + 2 nextEmit = s if s >= sLimit { - if debug { + if debugEncoder { println("repeat ended", s, length) } @@ -926,7 +926,7 @@ encodeLoop: } blk.recentOffsets[0] = uint32(offset1) blk.recentOffsets[1] = uint32(offset2) - if debug { + if debugEncoder { println("returning, recent offsets:", blk.recentOffsets, "extra literals:", blk.extraLits) } } diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index 4871dd03a..ea85548fc 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -245,7 +245,7 @@ func (e *Encoder) nextBlock(final bool) error { s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current s.wg.Add(1) go func(src []byte) { - if debug { + if debugEncoder { println("Adding block,", len(src), "bytes, final:", final) } defer func() { @@ -290,7 +290,7 @@ func (e *Encoder) nextBlock(final bool) error { } switch err { case errIncompressible: - if debug { + if debugEncoder { println("Storing incompressible block as raw") } blk.encodeRaw(src) @@ -313,7 +313,7 @@ func (e *Encoder) nextBlock(final bool) error { // // The Copy function uses ReaderFrom if available. func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) { - if debug { + if debugEncoder { println("Using ReadFrom") } @@ -336,20 +336,20 @@ func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) { switch err { case io.EOF: e.state.filling = e.state.filling[:len(e.state.filling)-len(src)] - if debug { + if debugEncoder { println("ReadFrom: got EOF final block:", len(e.state.filling)) } return n, nil case nil: default: - if debug { + if debugEncoder { println("ReadFrom: got error:", err) } e.state.err = err return n, err } if len(src) > 0 { - if debug { + if debugEncoder { println("ReadFrom: got space left in source:", len(src)) } continue @@ -512,7 +512,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { switch err { case errIncompressible: - if debug { + if debugEncoder { println("Storing incompressible block as raw") } dst = blk.encodeRawTo(dst, src) @@ -548,7 +548,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { switch err { case errIncompressible: - if debug { + if debugEncoder { println("Storing incompressible block as raw") } dst = blk.encodeRawTo(dst, todo) diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index 4dc151213..e8cc9a2c2 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -78,20 +78,33 @@ func newFrameDec(o decoderOptions) *frameDec { func (d *frameDec) reset(br byteBuffer) error { d.HasCheckSum = false d.WindowSize = 0 - var b []byte + var signature [4]byte for { var err error - b, err = br.readSmall(4) + // Check if we can read more... + b, err := br.readSmall(1) switch err { case io.EOF, io.ErrUnexpectedEOF: return io.EOF default: return err case nil: + signature[0] = b[0] + } + // Read the rest, don't allow io.ErrUnexpectedEOF + b, err = br.readSmall(3) + switch err { + case io.EOF: + return io.EOF + default: + return err + case nil: + copy(signature[1:], b) } - if !bytes.Equal(b[1:4], skippableFrameMagic) || b[0]&0xf0 != 0x50 { - if debug { - println("Not skippable", hex.EncodeToString(b), hex.EncodeToString(skippableFrameMagic)) + + if !bytes.Equal(signature[1:4], skippableFrameMagic) || signature[0]&0xf0 != 0x50 { + if debugDecoder { + println("Not skippable", hex.EncodeToString(signature[:]), hex.EncodeToString(skippableFrameMagic)) } // Break if not skippable frame. break @@ -99,28 +112,34 @@ func (d *frameDec) reset(br byteBuffer) error { // Read size to skip b, err = br.readSmall(4) if err != nil { - println("Reading Frame Size", err) + if debugDecoder { + println("Reading Frame Size", err) + } return err } n := uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) println("Skipping frame with", n, "bytes.") err = br.skipN(int(n)) if err != nil { - if debug { + if debugDecoder { println("Reading discarded frame", err) } return err } } - if !bytes.Equal(b, frameMagic) { - println("Got magic numbers: ", b, "want:", frameMagic) + if !bytes.Equal(signature[:], frameMagic) { + if debugDecoder { + println("Got magic numbers: ", signature, "want:", frameMagic) + } return ErrMagicMismatch } // Read Frame_Header_Descriptor fhd, err := br.readByte() if err != nil { - println("Reading Frame_Header_Descriptor", err) + if debugDecoder { + println("Reading Frame_Header_Descriptor", err) + } return err } d.SingleSegment = fhd&(1<<5) != 0 @@ -135,7 +154,9 @@ func (d *frameDec) reset(br byteBuffer) error { if !d.SingleSegment { wd, err := br.readByte() if err != nil { - println("Reading Window_Descriptor", err) + if debugDecoder { + println("Reading Window_Descriptor", err) + } return err } printf("raw: %x, mantissa: %d, exponent: %d\n", wd, wd&7, wd>>3) @@ -153,7 +174,7 @@ func (d *frameDec) reset(br byteBuffer) error { size = 4 } - b, err = br.readSmall(int(size)) + b, err := br.readSmall(int(size)) if err != nil { println("Reading Dictionary_ID", err) return err @@ -167,7 +188,7 @@ func (d *frameDec) reset(br byteBuffer) error { case 4: id = uint32(b[0]) | (uint32(b[1]) << 8) | (uint32(b[2]) << 16) | (uint32(b[3]) << 24) } - if debug { + if debugDecoder { println("Dict size", size, "ID:", id) } if id > 0 { @@ -191,7 +212,7 @@ func (d *frameDec) reset(br byteBuffer) error { } d.FrameContentSize = 0 if fcsSize > 0 { - b, err = br.readSmall(fcsSize) + b, err := br.readSmall(fcsSize) if err != nil { println("Reading Frame content", err) return err @@ -209,7 +230,7 @@ func (d *frameDec) reset(br byteBuffer) error { d2 := uint32(b[4]) | (uint32(b[5]) << 8) | (uint32(b[6]) << 16) | (uint32(b[7]) << 24) d.FrameContentSize = uint64(d1) | (uint64(d2) << 32) } - if debug { + if debugDecoder { println("field size bits:", v, "fcsSize:", fcsSize, "FrameContentSize:", d.FrameContentSize, hex.EncodeToString(b[:fcsSize]), "singleseg:", d.SingleSegment, "window:", d.WindowSize) } } @@ -252,7 +273,7 @@ func (d *frameDec) reset(br byteBuffer) error { // next will start decoding the next block from stream. func (d *frameDec) next(block *blockDec) error { - if debug { + if debugDecoder { printf("decoding new block %p:%p", block, block.data) } err := block.reset(d.rawInput, d.WindowSize) @@ -263,7 +284,7 @@ func (d *frameDec) next(block *blockDec) error { return err } block.input <- struct{}{} - if debug { + if debugDecoder { println("next block:", block) } d.asyncRunningMu.Lock() @@ -318,12 +339,12 @@ func (d *frameDec) checkCRC() error { } if !bytes.Equal(tmp[:], want) { - if debug { + if debugDecoder { println("CRC Check Failed:", tmp[:], "!=", want) } return ErrCRCMismatch } - if debug { + if debugDecoder { println("CRC ok", tmp[:]) } return nil @@ -344,7 +365,7 @@ func (d *frameDec) initAsync() { if cap(d.decoding) < d.o.concurrent { d.decoding = make(chan *blockDec, d.o.concurrent) } - if debug { + if debugDecoder { h := d.history printf("history init. len: %d, cap: %d", len(h.b), cap(h.b)) } @@ -392,7 +413,7 @@ func (d *frameDec) startDecoder(output chan decodeOutput) { output <- r return } - if debug { + if debugDecoder { println("got result, from ", d.offset, "to", d.offset+int64(len(r.b))) d.offset += int64(len(r.b)) } @@ -400,7 +421,7 @@ func (d *frameDec) startDecoder(output chan decodeOutput) { // Send history to next block select { case next = <-d.decoding: - if debug { + if debugDecoder { println("Sending ", len(d.history.b), "bytes as history") } next.history <- &d.history @@ -438,7 +459,7 @@ func (d *frameDec) startDecoder(output chan decodeOutput) { output <- r if next == nil { // There was no decoder available, we wait for one now that we have sent to the writer. - if debug { + if debugDecoder { println("Sending ", len(d.history.b), " bytes as history") } next = <-d.decoding @@ -462,7 +483,7 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { if err != nil { break } - if debug { + if debugDecoder { println("next block:", dec) } err = dec.decodeBuf(&d.history) diff --git a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go index c74681b99..b4757ee3f 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go @@ -229,7 +229,7 @@ func (s *fseEncoder) setRLE(val byte) { deltaFindState: 0, deltaNbBits: 0, } - if debug { + if debugEncoder { println("setRLE: val", val, "symbolTT", s.ct.symbolTT[val]) } s.rleVal = val diff --git a/vendor/github.com/klauspost/compress/zstd/snappy.go b/vendor/github.com/klauspost/compress/zstd/snappy.go index 9d9d1d567..0372b1714 100644 --- a/vendor/github.com/klauspost/compress/zstd/snappy.go +++ b/vendor/github.com/klauspost/compress/zstd/snappy.go @@ -203,7 +203,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) { written += int64(n) continue case chunkTypeUncompressedData: - if debug { + if debugEncoder { println("Uncompressed, chunklen", chunkLen) } // Section 4.3. Uncompressed data (chunk type 0x01). @@ -246,7 +246,7 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) { continue case chunkTypeStreamIdentifier: - if debug { + if debugEncoder { println("stream id", chunkLen, len(snappyMagicBody)) } // Section 4.1. Stream identifier (chunk type 0xff). diff --git a/vendor/github.com/klauspost/compress/zstd/zstd.go b/vendor/github.com/klauspost/compress/zstd/zstd.go index 1ba308c8b..ef1d49a00 100644 --- a/vendor/github.com/klauspost/compress/zstd/zstd.go +++ b/vendor/github.com/klauspost/compress/zstd/zstd.go @@ -15,6 +15,12 @@ import ( // enable debug printing const debug = false +// enable encoding debug printing +const debugEncoder = debug + +// enable decoding debug printing +const debugDecoder = debug + // Enable extra assertions. const debugAsserts = debug || false @@ -82,13 +88,13 @@ var ( ) func println(a ...interface{}) { - if debug { + if debug || debugDecoder || debugEncoder { log.Println(a...) } } func printf(format string, a ...interface{}) { - if debug { + if debug || debugDecoder || debugEncoder { log.Printf(format, a...) } } diff --git a/vendor/github.com/mattn/go-shellwords/shellwords.go b/vendor/github.com/mattn/go-shellwords/shellwords.go index 01afd94d1..1b42a0017 100644 --- a/vendor/github.com/mattn/go-shellwords/shellwords.go +++ b/vendor/github.com/mattn/go-shellwords/shellwords.go @@ -232,7 +232,7 @@ loop: case '\'': if !doubleQuoted && !dollarQuote { if singleQuoted { - got = argSingle + got = argQuoted } singleQuoted = !singleQuoted continue diff --git a/vendor/modules.txt b/vendor/modules.txt index 786096f45..f2a38caca 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -77,7 +77,7 @@ github.com/containernetworking/plugins/pkg/utils/hwaddr github.com/containernetworking/plugins/pkg/utils/sysctl github.com/containernetworking/plugins/plugins/ipam/host-local/backend github.com/containernetworking/plugins/plugins/ipam/host-local/backend/allocator -# github.com/containers/buildah v1.21.0 +# github.com/containers/buildah v1.21.1 github.com/containers/buildah github.com/containers/buildah/bind github.com/containers/buildah/chroot @@ -195,7 +195,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.32.1 +# github.com/containers/storage v1.32.2 github.com/containers/storage github.com/containers/storage/drivers github.com/containers/storage/drivers/aufs @@ -397,7 +397,7 @@ github.com/json-iterator/go # github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a github.com/juju/ansiterm github.com/juju/ansiterm/tabwriter -# github.com/klauspost/compress v1.12.3 +# github.com/klauspost/compress v1.13.0 github.com/klauspost/compress/flate github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 @@ -417,7 +417,7 @@ github.com/mattn/go-colorable github.com/mattn/go-isatty # github.com/mattn/go-runewidth v0.0.12 github.com/mattn/go-runewidth -# github.com/mattn/go-shellwords v1.0.11 +# github.com/mattn/go-shellwords v1.0.12 github.com/mattn/go-shellwords # github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 github.com/matttproud/golang_protobuf_extensions/pbutil |