diff options
304 files changed, 10826 insertions, 1293 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index b6ef69a96..3fcf335ed 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -30,20 +30,17 @@ env: PRIOR_UBUNTU_NAME: "ubuntu-2010" # Google-cloud VM Images - # TODO: At the time of this comment, an selinux-policy regression is blocking use of updated - # Fedora VM images: https://bugzilla.redhat.com/show_bug.cgi?id=1965743 - IMAGE_SUFFIX_UBUNTU: "c5521575421149184" - IMAGE_SUFFIX: "c5348179051806720" + IMAGE_SUFFIX: "c6737534580424704" FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}" - UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX_UBUNTU}" - PRIOR_UBUNTU_CACHE_IMAGE_NAME: "prior-ubuntu-${IMAGE_SUFFIX_UBUNTU}" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}" + PRIOR_UBUNTU_CACHE_IMAGE_NAME: "prior-ubuntu-${IMAGE_SUFFIX}" # Container FQIN's FEDORA_CONTAINER_FQIN: "quay.io/libpod/fedora_podman:${IMAGE_SUFFIX}" PRIOR_FEDORA_CONTAINER_FQIN: "quay.io/libpod/prior-fedora_podman:${IMAGE_SUFFIX}" - UBUNTU_CONTAINER_FQIN: "quay.io/libpod/ubuntu_podman:${IMAGE_SUFFIX_UBUNTU}" - PRIOR_UBUNTU_CONTAINER_FQIN: "quay.io/libpod/prior-ubuntu_podman:${IMAGE_SUFFIX_UBUNTU}" + UBUNTU_CONTAINER_FQIN: "quay.io/libpod/ubuntu_podman:${IMAGE_SUFFIX}" + PRIOR_UBUNTU_CONTAINER_FQIN: "quay.io/libpod/prior-ubuntu_podman:${IMAGE_SUFFIX}" #### #### Control variables that determine what to run and how to run it. @@ -378,8 +375,7 @@ osx_alt_build_task: always: *binary_artifacts -# This task is a stub: In the future it will be used to verify -# podman is compatible with the docker python-module. +# Verify podman is compatible with the docker python-module. docker-py_test_task: name: Docker-py Compat. alias: docker-py_test @@ -633,17 +629,11 @@ rootless_system_test_task: main_script: *main always: *logs_artifacts -# FIXME: we may want to consider running this from nightly cron instead of CI. -# The tests are actually pretty quick (less than a minute) but they do rely -# on pulling images from quay.io, which means we're subject to network flakes. -# -# FIXME: how does this env matrix work, anyway? Does it spin up multiple VMs? -# We might just want to encode the version matrix in runner.sh instead upgrade_test_task: name: "Upgrade test: from $PODMAN_UPGRADE_FROM" alias: upgrade_test skip: *tags - only_if: *not_docs + only_if: $CIRRUS_CHANGE_TITLE !=~ '.*CI:DOCS.*' || $CIRRUS_CRON != '' depends_on: - local_system_test matrix: @@ -678,18 +668,11 @@ meta_task: image: quay.io/libpod/imgts:$IMAGE_SUFFIX env: # Space-separated list of images used by this repository state - # TODO: Protect commonly tagged ubuntu images from puning in case - # workaround for BZ1965743 remains in use beyond the 30-days. - # Ref sha 404d5edb155 IMGNAMES: >- ${FEDORA_CACHE_IMAGE_NAME} ${PRIOR_FEDORA_CACHE_IMAGE_NAME} ${UBUNTU_CACHE_IMAGE_NAME} ${PRIOR_UBUNTU_CACHE_IMAGE_NAME} - fedora-${IMAGE_SUFFIX_UBUNTU} - prior-fedora-${IMAGE_SUFFIX_UBUNTU} - ubuntu-${IMAGE_SUFFIX} - prior-ubuntu-${IMAGE_SUFFIX} BUILDID: "${CIRRUS_BUILD_ID}" REPOREF: "${CIRRUS_REPO_NAME}" GCPJSON: ENCRYPTED[3a198350077849c8df14b723c0f4c9fece9ebe6408d35982e7adf2105a33f8e0e166ed3ed614875a0887e1af2b8775f4] diff --git a/.gitignore b/.gitignore index 0a3caf8ca..08e5309ee 100644 --- a/.gitignore +++ b/.gitignore @@ -33,6 +33,7 @@ release.txt /test/goecho/goecho /test/testvol/testvol .vscode* +tags result # Necessary to prevent hack/tree-status.sh false-positive /*runner_stats.log @@ -90,8 +90,7 @@ else ISODATE ?= $(shell date --iso-8601) endif LIBPOD := ${PROJECT}/v3/libpod -GCFLAGS ?= all=-trimpath=$(CURDIR) -ASMFLAGS ?= all=-trimpath=$(CURDIR) +GOFLAGS ?= -trimpath LDFLAGS_PODMAN ?= \ -X $(LIBPOD)/define.gitCommit=$(GIT_COMMIT) \ -X $(LIBPOD)/define.buildInfo=$(BUILD_INFO) \ @@ -258,7 +257,7 @@ test/goecho/goecho: .gopathok $(wildcard test/goecho/*.go) .PHONY: codespell codespell: - codespell -S bin,vendor,.git,go.sum,changelog.txt,.cirrus.yml,"RELEASE_NOTES.md,*.xz,*.gz,*.tar,*.tgz,bin2img,*ico,*.png,*.1,*.5,copyimg,*.orig,apidoc.go" -L uint,iff,od,seeked,splitted,marge,ERRO,hist,ether -w + codespell -S bin,vendor,.git,go.sum,changelog.txt,.cirrus.yml,"RELEASE_NOTES.md,*.xz,*.gz,*.ps1,*.tar,*.tgz,bin2img,*ico,*.png,*.1,*.5,copyimg,*.orig,apidoc.go" -L uint,iff,od,seeked,splitted,marge,ERRO,hist,ether -w .PHONY: validate validate: gofmt lint .gitvalidation validate.completions man-page-check swagger-check tests-included tests-expect-exit @@ -295,8 +294,6 @@ endif CGO_ENABLED=$(CGO_ENABLED) \ $(GO) build \ $(BUILDFLAGS) \ - -gcflags '$(GCFLAGS)' \ - -asmflags '$(ASMFLAGS)' \ -ldflags '$(LDFLAGS_PODMAN)' \ -tags "$(BUILDTAGS)" \ -o $@ ./cmd/podman @@ -310,8 +307,6 @@ $(SRCBINDIR)/podman$(BINSFX): $(SRCBINDIR) .gopathok $(SOURCES) go.mod go.sum GOOS=$(GOOS) \ $(GO) build \ $(BUILDFLAGS) \ - -gcflags '$(GCFLAGS)' \ - -asmflags '$(ASMFLAGS)' \ -ldflags '$(LDFLAGS_PODMAN)' \ -tags "${REMOTETAGS}" \ -o $@ ./cmd/podman @@ -321,8 +316,6 @@ $(SRCBINDIR)/podman-remote-static: $(SRCBINDIR) .gopathok $(SOURCES) go.mod go.s GOOS=$(GOOS) \ $(GO) build \ $(BUILDFLAGS) \ - -gcflags '$(GCFLAGS)' \ - -asmflags '$(ASMFLAGS)' \ -ldflags '$(LDFLAGS_PODMAN_STATIC)' \ -tags "${REMOTETAGS}" \ -o $@ ./cmd/podman @@ -376,8 +369,6 @@ bin/podman.cross.%: .gopathok CGO_ENABLED=0 \ $(GO) build \ $(BUILDFLAGS) \ - -gcflags '$(GCFLAGS)' \ - -asmflags '$(ASMFLAGS)' \ -ldflags '$(LDFLAGS_PODMAN)' \ -tags '$(BUILDTAGS_CROSS)' \ -o "$@" ./cmd/podman @@ -502,10 +493,12 @@ validate.completions: if [ -x /bin/zsh ]; then /bin/zsh completions/zsh/_podman; fi if [ -x /bin/fish ]; then /bin/fish completions/fish/podman.fish; fi +# Note: Assumes test/python/requirements.txt is installed & available .PHONY: run-docker-py-tests run-docker-py-tests: - $(eval testLogs=$(shell mktemp podman_tmp_XXXX)) - ./bin/podman run --rm --security-opt label=disable --privileged -v $(testLogs):/testLogs --net=host -e DOCKER_HOST=tcp://localhost:8080 $(DOCKERPY_IMAGE) sh -c "pytest $(DOCKERPY_TEST) " + touch test/__init__.py + pytest test/python/docker/ + -rm test/__init__.py .PHONY: localunit localunit: test/goecho/goecho @@ -850,11 +843,13 @@ clean: ## Clean all make artifacts build \ test/checkseccomp/checkseccomp \ test/goecho/goecho \ + test/__init__.py \ test/testdata/redis-image \ libpod/container_ffjson.go \ libpod/pod_ffjson.go \ libpod/container_easyjson.go \ libpod/pod_easyjson.go \ .install.goimports \ - docs/build + docs/build \ + venv make -C docs clean @@ -5,7 +5,7 @@ Podman (the POD MANager) is a tool for managing containers and images, volumes mounted into those containers, and pods made from groups of containers. Podman is based on libpod, a library for container lifecycle management that is also contained in this repository. The libpod library provides APIs for managing containers, pods, container images, and volumes. -* [Latest Version: 3.2.3](https://github.com/containers/podman/releases/latest) +* [Latest Version: 3.3.0](https://github.com/containers/podman/releases/latest) * Latest Remote client for Windows * Latest Remote client for macOS * Latest Static Remote client for Linux diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index ba323e6a0..9649e7abb 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,112 @@ # Release Notes +## 3.3.0 +### Features +- Containers inside VMs created by `podman machine` will now automatically handle port forwarding - containers in `podman machine` VMs that publish ports via `--publish` or `--publish-all` will have these ports not just forwarded on the VM, but also on the host system. +- The `podman play kube` command's `--network` option now accepts advanced network options (e.g. `--network slirp4netns:port_handler=slirp4netns`) ([#10807](https://github.com/containers/podman/issues/10807)). +- The `podman play kube` commmand now supports Kubernetes liveness probes, which will be created as Podman healthchecks. +- Podman now provides a systemd unit, `podman-restart.service`, which, when enabled, will restart all containers that were started with `--restart=always` after the system reboots. +- Rootless Podman can now be configured to use CNI networking by default by using the `rootless_networking` option in `containers.conf`. +- Images can now be pulled using `image:tag@digest` syntax (e.g. `podman pull fedora:34@sha256:1b0d4ddd99b1a8c8a80e885aafe6034c95f266da44ead992aab388e6aa91611a`) ([#6721](https://github.com/containers/podman/issues/6721)). +- The `podman container checkpoint` and `podman container restore` commands can now be used to checkpoint containers that are in pods, and restore those containers into pods. +- The `podman container restore` command now features a new option, `--publish`, to change the ports that are forwarded to a container that is being restored from an exported checkpoint. +- The `podman container checkpoint` command now features a new option, `--compress`, to specify the compression algorithm that will be used on the generated checkpoint. +- The `podman pull` command can now pull multiple images at once (e.g. `podman pull fedora:34 ubi8:latest` will pull both specified images). +- THe `podman cp` command can now copy files from one container into another directly (e.g. `podman cp containera:/etc/hosts containerb:/etc/`) ([#7370](https://github.com/containers/podman/issues/7370)). +- The `podman cp` command now supports a new option, `--archive`, which controls whether copied files will be chown'd to the UID and GID of the user of the destination container. +- The `podman stats` command now provides two additional metrics: Average CPU, and CPU time. +- The `podman pod create` command supports a new flag, `--pid`, to specify the PID namespace of the pod. If specified, containers that join the pod will automatically share its PID namespace. +- The `podman pod create` command supports a new flag, `--infra-name`, which allows the name of the pod's infra container to be set ([#10794](https://github.com/containers/podman/issues/10794)). +- The `podman auto-update` command has had its output reformatted - it is now much clearer what images were pulled and what containers were updated. +- The `podman auto-update` command now supports a new option, `--dry-run`, which reports what would be updated but does not actually perform the update ([#9949](https://github.com/containers/podman/issues/9949)). +- The `podman build` command now supports a new option, `--secret`, to mount secrets into build containers. +- The `podman manifest remove` command now has a new alias, `podman manifest rm`. +- The `podman login` command now supports a new option, `--verbose`, to print detailed information about where the credentials entered were stored. +- The `podman events` command now supports a new event, `exec_died`, which is produced when an exec session exits, and includes the exit code of the exec session. +- The `podman system connection add` command now supports adding connections that connect using the `tcp://` and `unix://` URL schemes. +- The `podman system connection list` command now supports a new flag, `--format`, to determine how the output is printed. +- The `podman volume prune` and `podman volume ls` commands' `--filter` option now support a new filter, `until`, that matches volumes created before a certain time ([#10579](https://github.com/containers/podman/issues/10579)). +- The `podman ps --filter` option's `network` filter now accepts a new value: `container:`, which matches containers that share a network namespace with a specific container ([#10361](https://github.com/containers/podman/issues/10361)). +- The `podman diff` command can now accept two arguments, allowing two images or two containers to be specified; the diff between the two will be printed ([#10649](https://github.com/containers/podman/issues/10649)). +- Podman can now optionally copy-up content from containers into volumes mounted into those containers earlier (at creation time, instead of at runtime) via the `prepare_on_create` option in `containers.conf` ([#10262](https://github.com/containers/podman/issues/10262)). +- A new option, `--gpus`, has been added to `podman create` and `podman run` as a no-op for better compatibility with Docker. If the nvidia-container-runtime package is installed, GPUs should be automatically added to containers without using the flag. +- If an invalid subcommand is provided, similar commands to try will now be suggested in the error message. + +### Changes +- The `podman system reset` command now removes non-Podman (e.g. Buildah and CRI-O) containers as well. +- The new port forwarding offered by `podman machine` requires [gvproxy](https://github.com/containers/gvisor-tap-vsock) in order to function. +- Podman will now automatically create the default CNI network if it does not exist, for both root and rootless users. This will only be done once per user - if the network is subsequently removed, it will not be recreated. +- The `install.cni` makefile option has been removed. It is no longer required to distribute the default `87-podman.conflist` CNI configuration file, as Podman will now automatically create it. +- The `--root` option to Podman will not automatically clear all default storage options when set. Storage options can be set manually using `--storage-opt` ([#10393](https://github.com/containers/podman/issues/10393)). +- The output of `podman system connection list` is now deterministic, with connections being sorted alpabetically by their name. +- The auto-update service (`podman-auto-update.service`) has had its default timer adjusted so it now starts at a random time up to 15 minutes after midnight, to help prevent system congestion from numerous daily services run at once. +- Systemd unit files generated by `podman generate systemd` now depend on `network-online.target` by default ([#10655](https://github.com/containers/podman/issues/10655)). +- Systemd unit files generated by `podman generate systemd` now use `Type=notify` by default, instead of using PID files. +- The `podman info` command's logic for detecting package versions on Gentoo has been improved, and should be significantly faster. + +### Bugfixes +- Fixed a bug where the `podman play kube` command did not perform SELinux relabelling of volumes specified with a `mountPath` that included the `:z` or `:Z` options ([#9371](https://github.com/containers/podman/issues/9371)). +- Fixed a bug where the `podman play kube` command would ignore the `USER` and `EXPOSE` directives in images ([#9609](https://github.com/containers/podman/issues/9609)). +- Fixed a bug where the `podman play kube` command would only accept lowercase pull policies. +- Fixed a bug where named volumes mounted into containers with the `:z` or `:Z` options were not appropriately relabelled for access from the container ([#10273](https://github.com/containers/podman/issues/10273)). +- Fixed a bug where the `podman logs -f` command, with the `journald` log driver, could sometimes fail to pick up the last line of output from a container ([#10323](https://github.com/containers/podman/issues/10323)). +- Fixed a bug where running `podman rm` on a container created with the `--rm` option would occasionally emit an error message saying the container failed to be removed, when it was successfully removed. +- Fixed a bug where starting a Podman container would segfault if the `LISTEN_PID` and `LISTEN_FDS` environment variables were set, but `LISTEN_FDNAMES` was not ([#10435](https://github.com/containers/podman/issues/10435)). +- Fixed a bug where exec sessions in containers were sometimes not cleaned up when run without `-d` and when the associated `podman exec` process was killed before completion. +- Fixed a bug where `podman system service` could, when run in a systemd unit file with sdnotify in use, drop some connections when it was starting up. +- Fixed a bug where containers run using the REST API using the `slirp4netns` network mode would leave zombie processes that were not cleaned up until `podman system service` exited ([#9777](https://github.com/containers/podman/issues/9777)). +- Fixed a bug where the `podman system service` command would leave zombie processes after its initial launch that were not cleaned up until it exited ([#10575](https://github.com/containers/podman/issues/10575)). +- Fixed a bug where VMs created by `podman machine` could not be started after the host system restarted ([#10824](https://github.com/containers/podman/issues/10824)). +- Fixed a bug where the `podman pod ps` command would not show headers for optional information (e.g. container names when the `--ctr-names` option was given). +- Fixed a bug where the remote Podman client's `podman create` and `podman run` commands would ignore timezone configuration from the server's `containers.conf` file ([#11124](https://github.com/containers/podman/issues/11124)). +- Fixed a bug where the remote Podman client's `podman build` command would only respect `.containerignore` and not `.dockerignore` files (when both are present, `.containerignore` will be preferred) ([#10907](https://github.com/containers/podman/issues/10907)). +- Fixed a bug where the remote Podman client's `podman build` command would fail to send the Dockerfile being built to the server when it was excluded by the `.dockerignore` file, resulting in an error ([#9867](https://github.com/containers/podman/issues/9867)). +- Fixed a bug where the remote Podman client's `podman build` command could unexpectedly stop streaming the output of the build ([#10154](https://github.com/containers/podman/issues/10154)). +- Fixed a bug where the remote Podman client's `podman build` command would fail to build when run on Windows ([#11259](https://github.com/containers/podman/issues/11259)). +- Fixed a bug where the `podman manifest create` command accepted at most two arguments (an arbitrary number of images are allowed as arguments, which will be added to the manifest). +- Fixed a bug where named volumes would not be properly chowned to the UID and GID of the directory they were mounted over when first mounted into a container ([#10776](https://github.com/containers/podman/issues/10776)). +- Fixed a bug where named volumes created using a volume plugin would be removed from Podman, even if the plugin reported a failure to remove the volume ([#11214](https://github.com/containers/podman/issues/11214)). +- Fixed a bug where the remote Podman client's `podman exec -i` command would hang when input was provided via shell redirection (e.g. `podman --remote exec -i foo cat <<<"hello"`) ([#7360](https://github.com/containers/podman/issues/7360)). +- Fixed a bug where containers created with `--rm` were not immediately removed after being started by `podman start` if they failed to start ([#10935](https://github.com/containers/podman/issues/10935)). +- Fixed a bug where the `--storage-opt` flag to `podman create` and `podman run` was nonfunctional ([#10264](https://github.com/containers/podman/issues/10264)). +- Fixed a bug where the `--device-cgroup-rule` option to `podman create` and `podman run` was nonfunctional ([#10302](https://github.com/containers/podman/issues/10302)). +- Fixed a bug where the `--tls-verify` option to `podman manifest push` was nonfunctional. +- Fixed a bug where the `podman import` command could, in some circumstances, produce empty images ([#10994](https://github.com/containers/podman/issues/10994)). +- Fixed a bug where images pulled using the `docker-daemon:` transport had the wrong registry (`localhost` instead of `docker.io/library`) ([#10998](https://github.com/containers/podman/issues/10998)). +- Fixed a bug where operations that pruned images (`podman image prune` and `podman system prune`) would prune untagged images with children ([#10832](https://github.com/containers/podman/issues/10832)). +- Fixed a bug where dual-stack networks created by `podman network create` did not properly auto-assign an IPv4 subnet when one was not explicitly specified ([#11032](https://github.com/containers/podman/issues/11032)). +- Fixed a bug where port forwarding using the `rootlessport` port forwarder would break when a network was disconnected and then reconnected ([#10052](https://github.com/containers/podman/issues/10052)). +- Fixed a bug where Podman would ignore user-specified SELinux policies for containers using the Kata OCI runtime, or containers using systemd as PID 1 ([#11100](https://github.com/containers/podman/issues/11100)). +- Fixed a bug where Podman containers created using `--net=host` would add an entry to `/etc/hosts` for the container's hostname pointing to `127.0.1.1` ([#10319](https://github.com/containers/podman/issues/10319)). +- Fixed a bug where the `podman unpause --all` command would throw an error for every container that was not paused ([#11098](https://github.com/containers/podman/issues/11098)). +- Fixed a bug where timestamps for the `since` and `until` filters using Unix timestamps with a nanoseconds portion could not be parsed ([#11131](https://github.com/containers/podman/issues/11131)). +- Fixed a bug where the `podman info` command would sometimes print the wrong path for the `slirp4netns` binary. +- Fixed a bug where rootless Podman containers joined to a CNI network would not have functional DNS when the host used systemd-resolved without the resolved stub resolver being enabled ([#11222](https://github.com/containers/podman/issues/11222)). +- Fixed a bug where `podman network connect` and `podman network disconnect` of rootless containers could sometimes break port forwarding to the container ([#11248](https://github.com/containers/podman/issues/11248)). +- Fixed a bug where joining a container to a CNI network by ID and adding network aliases to this network would cause the container to fail to start ([#11285](https://github.com/containers/podman/issues/11285)). + +### API +- Fixed a bug where the Compat List endpoint for Containers included healthcheck information for all containers, even those that did not have a configured healthcheck. +- Fixed a bug where the Compat Create endpoint for Containers would fail to create containers with the `NetworkMode` parameter set to `default` ([#10569](https://github.com/containers/podman/issues/10569)). +- Fixed a bug where the Compat Create endpoint for Containers did not properly handle healthcheck commands ([#10617](https://github.com/containers/podman/issues/10617)). +- Fixed a bug where the Compat Wait endpoint for Containers would always send an empty string error message when no error occurred. +- Fixed a bug where the Libpod Stats endpoint for Containers would not error when run on rootless containers on cgroups v1 systems (nonsensical results would be returned, as this configuration cannot be supportable). +- Fixed a bug where the Compat List endpoint for Images omitted the `ContainerConfig` field ([#10795](https://github.com/containers/podman/issues/10795)). +- Fixed a bug where the Compat Build endpoint for Images was too strict when validating the `Content-Type` header, rejecting content that Docker would have accepted ([#11022](https://github.com/containers/podman/issues/11012)). +- Fixed a bug where the Compat Pull endpoint for Images could fail, but return a 200 status code, if an image name that could not be parsed was provided. +- Fixed a bug where the Compat Pull endpoint for Images would continue to pull images after the client disconnected. +- Fixed a bug where the Compat List endpoint for Networks would fail for non-bridge (e.g. macvlan) networks ([#10266](https://github.com/containers/podman/issues/10266)). +- Fixed a bug where the Libpod List endpoint for Networks would return nil, instead of an empty list, when no networks were present ([#10495](https://github.com/containers/podman/issues/10495)). +- The Compat and Libpod Logs endpoints for Containers now support the `until` query parameter ([#10859](https://github.com/containers/podman/issues/10859)). +- The Compat Import endpoint for Images now supports the `platform`, `message`, and `repo` query parameters. +- The Compat Pull endpoint for Images now supports the `platform` query parameter. + +### Misc +- Updated Buildah to v1.22.3 +- Updated the containers/storage library to v1.34.1 +- Updated the containers/image library to v5.15.2 +- Updated the containers/common library to v0.42.1 + ## 3.2.3 ### Security - This release addresses CVE-2021-3602, an issue with the `podman build` command with the `--isolation chroot` flag that results in environment variables from the host leaking into build containers. diff --git a/cmd/podman/common/completion.go b/cmd/podman/common/completion.go index 08b2f6235..9a4524b46 100644 --- a/cmd/podman/common/completion.go +++ b/cmd/podman/common/completion.go @@ -323,6 +323,18 @@ func prefixSlice(pre string, slice []string) []string { return slice } +func suffixCompSlice(suf string, slice []string) []string { + for i := range slice { + split := strings.SplitN(slice[i], "\t", 2) + if len(split) > 1 { + slice[i] = split[0] + suf + "\t" + split[1] + } else { + slice[i] = slice[i] + suf + } + } + return slice +} + func completeKeyValues(toComplete string, k keyValueCompletion) ([]string, cobra.ShellCompDirective) { suggestions := make([]string, 0, len(k)) directive := cobra.ShellCompDirectiveNoFileComp @@ -664,6 +676,42 @@ func AutocompleteSystemConnections(cmd *cobra.Command, args []string, toComplete return suggestions, cobra.ShellCompDirectiveNoFileComp } +// AutocompleteScp returns a list of connections, images, or both, depending on the amount of arguments +func AutocompleteScp(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if !validCurrentCmdLine(cmd, args, toComplete) { + return nil, cobra.ShellCompDirectiveNoFileComp + } + switch len(args) { + case 0: + split := strings.SplitN(toComplete, "::", 2) + if len(split) > 1 { + imageSuggestions, _ := getImages(cmd, split[1]) + return prefixSlice(split[0]+"::", imageSuggestions), cobra.ShellCompDirectiveNoFileComp + } + connectionSuggestions, _ := AutocompleteSystemConnections(cmd, args, toComplete) + imageSuggestions, _ := getImages(cmd, toComplete) + totalSuggestions := append(suffixCompSlice("::", connectionSuggestions), imageSuggestions...) + directive := cobra.ShellCompDirectiveNoFileComp + // if we have connections do not add a space after the completion + if len(connectionSuggestions) > 0 { + directive = cobra.ShellCompDirectiveNoFileComp | cobra.ShellCompDirectiveNoSpace + } + return totalSuggestions, directive + case 1: + split := strings.SplitN(args[0], "::", 2) + if len(split) > 1 { + if len(split[1]) > 0 { + return nil, cobra.ShellCompDirectiveNoFileComp + } + imageSuggestions, _ := getImages(cmd, toComplete) + return imageSuggestions, cobra.ShellCompDirectiveNoFileComp + } + connectionSuggestions, _ := AutocompleteSystemConnections(cmd, args, toComplete) + return suffixCompSlice("::", connectionSuggestions), cobra.ShellCompDirectiveNoFileComp + } + return nil, cobra.ShellCompDirectiveNoFileComp +} + /* -------------- Flags ----------------- */ // AutocompleteDetachKeys - Autocomplete detach-keys options. diff --git a/cmd/podman/common/create.go b/cmd/podman/common/create.go index 602ad5d94..401cf2e09 100644 --- a/cmd/podman/common/create.go +++ b/cmd/podman/common/create.go @@ -544,6 +544,15 @@ func DefineCreateFlags(cmd *cobra.Command, cf *ContainerCLIOpts) { ) _ = cmd.RegisterFlagCompletionFunc(podIDFileFlagName, completion.AutocompleteDefault) + // Flag for TLS verification, so that `run` and `create` commands can make use of it. + // Make sure to use `=` while using this flag i.e `--tls-verify=false/true` + tlsVerifyFlagName := "tls-verify" + createFlags.BoolVar( + &cf.TLSVerify, + tlsVerifyFlagName, true, + "Require HTTPS and verify certificates when contacting registries for pulling images", + ) + createFlags.BoolVar( &cf.Privileged, "privileged", false, diff --git a/cmd/podman/common/create_opts.go b/cmd/podman/common/create_opts.go index 0a969bfd2..c94f46cf2 100644 --- a/cmd/podman/common/create_opts.go +++ b/cmd/podman/common/create_opts.go @@ -10,6 +10,7 @@ import ( "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/cmd/podman/registry" + "github.com/containers/podman/v3/libpod/network/types" "github.com/containers/podman/v3/pkg/api/handlers" "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/domain/entities" @@ -112,6 +113,7 @@ type ContainerCLIOpts struct { Sysctl []string Systemd string Timeout uint + TLSVerify bool TmpFS []string TTY bool Timezone string @@ -149,7 +151,7 @@ func ContainerCreateToContainerCLIOpts(cc handlers.CreateContainerConfig, rtc *c cappDrop []string entrypoint *string init bool - specPorts []specgen.PortMapping + specPorts []types.PortMapping ) if cc.HostConfig.Init != nil { @@ -239,7 +241,7 @@ func ContainerCreateToContainerCLIOpts(cc handlers.CreateContainerConfig, rtc *c if err != nil { return nil, nil, err } - tmpPort := specgen.PortMapping{ + tmpPort := types.PortMapping{ HostIP: pb.HostIP, ContainerPort: uint16(port.Int()), HostPort: uint16(hostport), @@ -356,51 +358,55 @@ func ContainerCreateToContainerCLIOpts(cc handlers.CreateContainerConfig, rtc *c CPUSetMems: cc.HostConfig.CpusetMems, // Detach: false, // don't need // DetachKeys: "", // don't need - Devices: devices, - DeviceCGroupRule: nil, - DeviceReadBPs: readBps, - DeviceReadIOPs: readIops, - DeviceWriteBPs: writeBps, - DeviceWriteIOPs: writeIops, - Entrypoint: entrypoint, - Env: cc.Config.Env, - Expose: expose, - GroupAdd: cc.HostConfig.GroupAdd, - Hostname: cc.Config.Hostname, - ImageVolume: "bind", - Init: init, - Interactive: cc.Config.OpenStdin, - IPC: string(cc.HostConfig.IpcMode), - Label: stringMaptoArray(cc.Config.Labels), - LogDriver: cc.HostConfig.LogConfig.Type, - LogOptions: stringMaptoArray(cc.HostConfig.LogConfig.Config), - Name: cc.Name, - OOMScoreAdj: cc.HostConfig.OomScoreAdj, - Arch: "", - OS: "", - Variant: "", - PID: string(cc.HostConfig.PidMode), - PIDsLimit: cc.HostConfig.PidsLimit, - Privileged: cc.HostConfig.Privileged, - PublishAll: cc.HostConfig.PublishAllPorts, - Quiet: false, - ReadOnly: cc.HostConfig.ReadonlyRootfs, - ReadOnlyTmpFS: true, // podman default - Rm: cc.HostConfig.AutoRemove, - SecurityOpt: cc.HostConfig.SecurityOpt, - StopSignal: cc.Config.StopSignal, - StorageOpt: stringMaptoArray(cc.HostConfig.StorageOpt), - Sysctl: stringMaptoArray(cc.HostConfig.Sysctls), - Systemd: "true", // podman default - TmpFS: parsedTmp, - TTY: cc.Config.Tty, - User: cc.Config.User, - UserNS: string(cc.HostConfig.UsernsMode), - UTS: string(cc.HostConfig.UTSMode), - Mount: mounts, - VolumesFrom: cc.HostConfig.VolumesFrom, - Workdir: cc.Config.WorkingDir, - Net: &netInfo, + Devices: devices, + DeviceCGroupRule: nil, + DeviceReadBPs: readBps, + DeviceReadIOPs: readIops, + DeviceWriteBPs: writeBps, + DeviceWriteIOPs: writeIops, + Entrypoint: entrypoint, + Env: cc.Config.Env, + Expose: expose, + GroupAdd: cc.HostConfig.GroupAdd, + Hostname: cc.Config.Hostname, + ImageVolume: "bind", + Init: init, + Interactive: cc.Config.OpenStdin, + IPC: string(cc.HostConfig.IpcMode), + Label: stringMaptoArray(cc.Config.Labels), + LogDriver: cc.HostConfig.LogConfig.Type, + LogOptions: stringMaptoArray(cc.HostConfig.LogConfig.Config), + Name: cc.Name, + OOMScoreAdj: cc.HostConfig.OomScoreAdj, + Arch: "", + OS: "", + Variant: "", + PID: string(cc.HostConfig.PidMode), + PIDsLimit: cc.HostConfig.PidsLimit, + Privileged: cc.HostConfig.Privileged, + PublishAll: cc.HostConfig.PublishAllPorts, + Quiet: false, + ReadOnly: cc.HostConfig.ReadonlyRootfs, + ReadOnlyTmpFS: true, // podman default + Rm: cc.HostConfig.AutoRemove, + SecurityOpt: cc.HostConfig.SecurityOpt, + StopSignal: cc.Config.StopSignal, + StorageOpt: stringMaptoArray(cc.HostConfig.StorageOpt), + Sysctl: stringMaptoArray(cc.HostConfig.Sysctls), + Systemd: "true", // podman default + TmpFS: parsedTmp, + TTY: cc.Config.Tty, + User: cc.Config.User, + UserNS: string(cc.HostConfig.UsernsMode), + UTS: string(cc.HostConfig.UTSMode), + Mount: mounts, + VolumesFrom: cc.HostConfig.VolumesFrom, + Workdir: cc.Config.WorkingDir, + Net: &netInfo, + HealthInterval: DefaultHealthCheckInterval, + HealthRetries: DefaultHealthCheckRetries, + HealthTimeout: DefaultHealthCheckTimeout, + HealthStartPeriod: DefaultHealthCheckStartPeriod, } if !rootless.IsRootless() { var ulimits []string @@ -527,10 +533,18 @@ func ContainerCreateToContainerCLIOpts(cc handlers.CreateContainerConfig, rtc *c finCmd = finCmd[:len(finCmd)-1] } cliOpts.HealthCmd = finCmd - cliOpts.HealthInterval = cc.Config.Healthcheck.Interval.String() - cliOpts.HealthRetries = uint(cc.Config.Healthcheck.Retries) - cliOpts.HealthStartPeriod = cc.Config.Healthcheck.StartPeriod.String() - cliOpts.HealthTimeout = cc.Config.Healthcheck.Timeout.String() + if cc.Config.Healthcheck.Interval > 0 { + cliOpts.HealthInterval = cc.Config.Healthcheck.Interval.String() + } + if cc.Config.Healthcheck.Retries > 0 { + cliOpts.HealthRetries = uint(cc.Config.Healthcheck.Retries) + } + if cc.Config.Healthcheck.StartPeriod > 0 { + cliOpts.HealthStartPeriod = cc.Config.Healthcheck.StartPeriod.String() + } + if cc.Config.Healthcheck.Timeout > 0 { + cliOpts.HealthTimeout = cc.Config.Healthcheck.Timeout.String() + } } // specgen assumes the image name is arg[0] diff --git a/cmd/podman/common/specgen.go b/cmd/podman/common/specgen.go index 8d6a21cb7..59d32f568 100644 --- a/cmd/podman/common/specgen.go +++ b/cmd/podman/common/specgen.go @@ -685,7 +685,7 @@ func makeHealthCheckFromCli(inCmd, interval string, retries uint, timeout, start concat := "" if cmdArr[0] == "CMD" || cmdArr[0] == "none" { // this is for compat, we are already split properly for most compat cases cmdArr = strings.Fields(inCmd) - } else if cmdArr[0] != "CMD-SHELL" { // this is for podman side of things, wont contain the keywords + } else if cmdArr[0] != "CMD-SHELL" { // this is for podman side of things, won't contain the keywords if isArr && len(cmdArr) > 1 { // an array of consecutive commands cmdArr = append([]string{"CMD"}, cmdArr...) } else { // one singular command diff --git a/cmd/podman/common/util.go b/cmd/podman/common/util.go index 6a0af4dff..cdfff9d6f 100644 --- a/cmd/podman/common/util.go +++ b/cmd/podman/common/util.go @@ -6,7 +6,7 @@ import ( "strconv" "strings" - "github.com/containers/podman/v3/pkg/specgen" + "github.com/containers/podman/v3/libpod/network/types" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -90,10 +90,10 @@ func createExpose(expose []string) (map[uint16]string, error) { } // CreatePortBindings iterates ports mappings into SpecGen format. -func CreatePortBindings(ports []string) ([]specgen.PortMapping, error) { +func CreatePortBindings(ports []string) ([]types.PortMapping, error) { // --publish is formatted as follows: // [[hostip:]hostport[-endPort]:]containerport[-endPort][/protocol] - toReturn := make([]specgen.PortMapping, 0, len(ports)) + toReturn := make([]types.PortMapping, 0, len(ports)) for _, p := range ports { var ( @@ -169,8 +169,8 @@ func CreatePortBindings(ports []string) ([]specgen.PortMapping, error) { // parseSplitPort parses individual components of the --publish flag to produce // a single port mapping in SpecGen format. -func parseSplitPort(hostIP, hostPort *string, ctrPort string, protocol *string) (specgen.PortMapping, error) { - newPort := specgen.PortMapping{} +func parseSplitPort(hostIP, hostPort *string, ctrPort string, protocol *string) (types.PortMapping, error) { + newPort := types.PortMapping{} if ctrPort == "" { return newPort, errors.Errorf("must provide a non-empty container port to publish") } diff --git a/cmd/podman/containers/create.go b/cmd/podman/containers/create.go index 895736144..7583a024e 100644 --- a/cmd/podman/containers/create.go +++ b/cmd/podman/containers/create.go @@ -10,9 +10,11 @@ import ( "github.com/containers/common/pkg/completion" "github.com/containers/common/pkg/config" "github.com/containers/image/v5/transports/alltransports" + "github.com/containers/image/v5/types" "github.com/containers/podman/v3/cmd/podman/common" "github.com/containers/podman/v3/cmd/podman/registry" "github.com/containers/podman/v3/cmd/podman/utils" + "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/specgen" "github.com/containers/podman/v3/pkg/util" @@ -95,7 +97,7 @@ func create(cmd *cobra.Command, args []string) error { var ( err error ) - cliVals.Net, err = common.NetFlagsToNetOptions(cmd, cliVals.Pod == "") + cliVals.Net, err = common.NetFlagsToNetOptions(cmd, cliVals.Pod == "" && cliVals.PodIDFile == "") if err != nil { return err } @@ -105,8 +107,8 @@ func create(cmd *cobra.Command, args []string) error { if !cmd.Flags().Changed("pod") { return errors.New("must specify pod value with init-ctr") } - if !util.StringInSlice(initctr, []string{"always", "oneshot"}) { - return errors.New("init-ctr value must be 'always' or 'oneshot'") + if !util.StringInSlice(initctr, []string{define.AlwaysInitContainer, define.OneShotInitContainer}) { + return errors.Errorf("init-ctr value must be '%s' or '%s'", define.AlwaysInitContainer, define.OneShotInitContainer) } cliVals.InitContainerType = initctr } @@ -184,6 +186,9 @@ func createInit(c *cobra.Command) error { if c.Flag("cpu-quota").Changed && c.Flag("cpus").Changed { return errors.Errorf("--cpu-quota and --cpus cannot be set together") } + if c.Flag("pod").Changed && !strings.HasPrefix(c.Flag("pod").Value.String(), "new:") && c.Flag("userns").Changed { + return errors.Errorf("--userns and --pod cannot be set together") + } noHosts, err := c.Flags().GetBool("no-hosts") if err != nil { @@ -257,7 +262,7 @@ func createInit(c *cobra.Command) error { } func pullImage(imageName string) (string, error) { - pullPolicy, err := config.ValidatePullPolicy(cliVals.Pull) + pullPolicy, err := config.ParsePullPolicy(cliVals.Pull) if err != nil { return "", err } @@ -283,6 +288,7 @@ func pullImage(imageName string) (string, error) { Variant: cliVals.Variant, SignaturePolicy: cliVals.SignaturePolicy, PullPolicy: pullPolicy, + SkipTLSVerify: types.NewOptionalBool(!cliVals.TLSVerify), // If Flag changed for TLS Verification }) if pullErr != nil { return "", pullErr @@ -309,6 +315,12 @@ func createPodIfNecessary(s *specgen.SpecGenerator, netOpts *entities.NetOptions if len(podName) < 1 { return nil, errors.Errorf("new pod name must be at least one character") } + + userns, err := specgen.ParseUserNamespace(cliVals.UserNS) + if err != nil { + return nil, err + } + createOptions := entities.PodCreateOptions{ Name: podName, Infra: true, @@ -318,6 +330,7 @@ func createPodIfNecessary(s *specgen.SpecGenerator, netOpts *entities.NetOptions Cpus: cliVals.CPUS, CpusetCpus: cliVals.CPUSetCPUs, Pid: cliVals.PID, + Userns: userns, } // Unset config values we passed to the pod to prevent them being used twice for the container and pod. s.ContainerBasicConfig.Hostname = "" diff --git a/cmd/podman/containers/run.go b/cmd/podman/containers/run.go index 579af4eb1..830d1de7f 100644 --- a/cmd/podman/containers/run.go +++ b/cmd/podman/containers/run.go @@ -106,7 +106,7 @@ func init() { func run(cmd *cobra.Command, args []string) error { var err error - cliVals.Net, err = common.NetFlagsToNetOptions(cmd, cliVals.Pod == "") + cliVals.Net, err = common.NetFlagsToNetOptions(cmd, cliVals.Pod == "" && cliVals.PodIDFile == "") if err != nil { return err } diff --git a/cmd/podman/generate/kube.go b/cmd/podman/generate/kube.go index b4c9f9146..60b8f0af0 100644 --- a/cmd/podman/generate/kube.go +++ b/cmd/podman/generate/kube.go @@ -2,6 +2,7 @@ package pods import ( "fmt" + "io" "io/ioutil" "os" @@ -61,6 +62,10 @@ func kube(cmd *cobra.Command, args []string) error { if err != nil { return err } + if r, ok := report.Reader.(io.ReadCloser); ok { + defer r.Close() + } + if cmd.Flags().Changed("filename") { if _, err := os.Stat(kubeFile); err == nil { return errors.Errorf("cannot write to %q; file exists", kubeFile) diff --git a/cmd/podman/images/build.go b/cmd/podman/images/build.go index 3aeba6fb0..a1a28b809 100644 --- a/cmd/podman/images/build.go +++ b/cmd/podman/images/build.go @@ -67,6 +67,18 @@ var ( podman image build --layers --force-rm --tag imageName .`, } + buildxBuildCmd = &cobra.Command{ + Args: buildCmd.Args, + Use: buildCmd.Use, + Short: buildCmd.Short, + Long: buildCmd.Long, + RunE: buildCmd.RunE, + ValidArgsFunction: buildCmd.ValidArgsFunction, + Example: `podman buildx build . + podman buildx build --creds=username:password -t imageName -f Containerfile.simple . + podman buildx build --layers --force-rm --tag imageName .`, + } + buildOpts = buildFlagsWrapper{} ) @@ -91,11 +103,24 @@ func init() { Parent: imageCmd, }) buildFlags(imageBuildCmd) + registry.Commands = append(registry.Commands, registry.CliCommand{ + Command: buildxBuildCmd, + Parent: buildxCmd, + }) + buildFlags(buildxBuildCmd) } func buildFlags(cmd *cobra.Command) { flags := cmd.Flags() + // buildx build --load ignored, but added for compliance + flags.Bool("load", false, "buildx --load") + _ = flags.MarkHidden("load") + + // buildx build --progress ignored, but added for compliance + flags.String("progress", "auto", "buildx --progress") + _ = flags.MarkHidden("progress") + // Podman flags flags.BoolVarP(&buildOpts.SquashAll, "squash-all", "", false, "Squash all layers into a single layer") diff --git a/cmd/podman/images/buildx.go b/cmd/podman/images/buildx.go new file mode 100644 index 000000000..5c8e5aaa0 --- /dev/null +++ b/cmd/podman/images/buildx.go @@ -0,0 +1,29 @@ +package images + +import ( + "github.com/containers/podman/v3/cmd/podman/registry" + "github.com/containers/podman/v3/cmd/podman/validate" + "github.com/spf13/cobra" +) + +var ( + // Command: podman _buildx_ + // This is a hidden command, which was added to make converting + // from Docker to Podman easier. + // For now podman buildx build just calls into podman build + // If we are adding new buildx features, we will add them by default + // to podman build. + buildxCmd = &cobra.Command{ + Use: "buildx", + Short: "Build images", + Long: "Build images", + RunE: validate.SubCommandExists, + Hidden: true, + } +) + +func init() { + registry.Commands = append(registry.Commands, registry.CliCommand{ + Command: buildxCmd, + }) +} diff --git a/cmd/podman/images/scp.go b/cmd/podman/images/scp.go index a47d01995..176563440 100644 --- a/cmd/podman/images/scp.go +++ b/cmd/podman/images/scp.go @@ -33,7 +33,7 @@ var ( Short: "securely copy images", RunE: scp, Args: cobra.RangeArgs(1, 2), - ValidArgsFunction: common.AutocompleteImages, + ValidArgsFunction: common.AutocompleteScp, Example: `podman image scp myimage:latest otherhost::`, } ) diff --git a/cmd/podman/play/kube.go b/cmd/podman/play/kube.go index ece7d1f98..2eebd9f86 100644 --- a/cmd/podman/play/kube.go +++ b/cmd/podman/play/kube.go @@ -100,6 +100,9 @@ func init() { configmapFlagName := "configmap" flags.StringSliceVar(&kubeOptions.ConfigMaps, configmapFlagName, []string{}, "`Pathname` of a YAML file containing a kubernetes configmap") _ = kubeCmd.RegisterFlagCompletionFunc(configmapFlagName, completion.AutocompleteDefault) + + buildFlagName := "build" + flags.BoolVar(&kubeOptions.Build, buildFlagName, false, "Build all images in a YAML (given Containerfiles exist)") } _ = flags.MarkHidden("signature-policy") } diff --git a/cmd/podman/pods/create.go b/cmd/podman/pods/create.go index abc47164b..bf5b9e350 100644 --- a/cmd/podman/pods/create.go +++ b/cmd/podman/pods/create.go @@ -48,6 +48,7 @@ var ( podIDFile string replace bool share string + userns string ) func init() { @@ -72,6 +73,10 @@ func init() { flags.StringVar(&createOptions.CGroupParent, cgroupParentflagName, "", "Set parent cgroup for the pod") _ = createCommand.RegisterFlagCompletionFunc(cgroupParentflagName, completion.AutocompleteDefault) + usernsFlagName := "userns" + flags.StringVar(&userns, usernsFlagName, os.Getenv("PODMAN_USERNS"), "User namespace to use") + _ = createCommand.RegisterFlagCompletionFunc(usernsFlagName, common.AutocompleteUserNamespace) + flags.BoolVar(&createOptions.Infra, "infra", true, "Create an infra container associated with the pod to share namespaces with") infraConmonPidfileFlagName := "infra-conmon-pidfile" @@ -178,6 +183,11 @@ func create(cmd *cobra.Command, args []string) error { } } + createOptions.Userns, err = specgen.ParseUserNamespace(userns) + if err != nil { + return err + } + if cmd.Flag("pod-id-file").Changed { podIDFD, err = util.OpenExclusiveFile(podIDFile) if err != nil && os.IsExist(err) { diff --git a/cmd/podman/registry/config.go b/cmd/podman/registry/config.go index b512ba341..50e488b02 100644 --- a/cmd/podman/registry/config.go +++ b/cmd/podman/registry/config.go @@ -89,7 +89,12 @@ func newPodmanConfig() { // use for the containers.conf configuration file. func setXdgDirs() error { if !rootless.IsRootless() { - return nil + // unset XDG_RUNTIME_DIR for root + // Sometimes XDG_RUNTIME_DIR is set to /run/user/0 sometimes it is unset, + // the inconsistency is causing issues for the dnsname plugin. + // It is already set to an empty string for conmon so lets do the same + // for podman. see #10806 and #10745 + return os.Unsetenv("XDG_RUNTIME_DIR") } // Setup XDG_RUNTIME_DIR diff --git a/cmd/podman/root.go b/cmd/podman/root.go index dc4ebb952..371ded9a8 100644 --- a/cmd/podman/root.go +++ b/cmd/podman/root.go @@ -6,6 +6,7 @@ import ( "path/filepath" "runtime" "runtime/pprof" + "strconv" "strings" "github.com/containers/common/pkg/completion" @@ -194,6 +195,17 @@ func persistentPreRunE(cmd *cobra.Command, args []string) error { return err } } + if cmd.Flag("memory-profile").Changed { + // Same value as the default in github.com/pkg/profile. + runtime.MemProfileRate = 4096 + if rate := os.Getenv("MemProfileRate"); rate != "" { + r, err := strconv.Atoi(rate) + if err != nil { + return err + } + runtime.MemProfileRate = r + } + } if cfg.MaxWorks <= 0 { return errors.Errorf("maximum workers must be set to a positive number (got %d)", cfg.MaxWorks) @@ -224,14 +236,29 @@ func persistentPostRunE(cmd *cobra.Command, args []string) error { return nil } - if !registry.IsRemote() { - if cmd.Flag("cpu-profile").Changed { - pprof.StopCPUProfile() + registry.ImageEngine().Shutdown(registry.Context()) + registry.ContainerEngine().Shutdown(registry.Context()) + + if registry.IsRemote() { + return nil + } + + // CPU and memory profiling. + if cmd.Flag("cpu-profile").Changed { + pprof.StopCPUProfile() + } + if cmd.Flag("memory-profile").Changed { + f, err := os.Create(registry.PodmanConfig().MemoryProfile) + if err != nil { + return errors.Wrap(err, "creating memory profile") + } + defer f.Close() + runtime.GC() // get up-to-date GC statistics + if err := pprof.WriteHeapProfile(f); err != nil { + return errors.Wrap(err, "writing memory profile") } } - registry.ImageEngine().Shutdown(registry.Context()) - registry.ContainerEngine().Shutdown(registry.Context()) return nil } @@ -294,7 +321,8 @@ func rootFlags(cmd *cobra.Command, opts *entities.PodmanConfig) { pFlags.StringVar(&cfg.Engine.CgroupManager, cgroupManagerFlagName, cfg.Engine.CgroupManager, "Cgroup manager to use (\"cgroupfs\"|\"systemd\")") _ = cmd.RegisterFlagCompletionFunc(cgroupManagerFlagName, common.AutocompleteCgroupManager) - pFlags.StringVar(&opts.CPUProfile, "cpu-profile", "", "Path for the cpu profiling results") + pFlags.StringVar(&opts.CPUProfile, "cpu-profile", "", "Path for the cpu-profiling results") + pFlags.StringVar(&opts.MemoryProfile, "memory-profile", "", "Path for the memory-profiling results") conmonFlagName := "conmon" pFlags.StringVar(&opts.ConmonPath, conmonFlagName, "", "Path of the conmon binary") @@ -354,6 +382,7 @@ func rootFlags(cmd *cobra.Command, opts *entities.PodmanConfig) { "cpu-profile", "default-mounts-file", "max-workers", + "memory-profile", "registries-conf", "trace", } { diff --git a/cmd/podman/shell_completion_test.go b/cmd/podman/shell_completion_test.go index 9bd821d8d..792beeb19 100644 --- a/cmd/podman/shell_completion_test.go +++ b/cmd/podman/shell_completion_test.go @@ -33,7 +33,9 @@ func TestShellCompletionFunctions(t *testing.T) { func checkCommand(t *testing.T, cmd *cobra.Command) { if cmd.HasSubCommands() { for _, childCmd := range cmd.Commands() { - checkCommand(t, childCmd) + if !childCmd.Hidden { + checkCommand(t, childCmd) + } } // if not check if completion for that command is provided diff --git a/cmd/podman/volumes/export.go b/cmd/podman/volumes/export.go new file mode 100644 index 000000000..9e4fecdfa --- /dev/null +++ b/cmd/podman/volumes/export.go @@ -0,0 +1,96 @@ +package volumes + +import ( + "context" + "fmt" + + "github.com/containers/common/pkg/completion" + "github.com/containers/podman/v3/cmd/podman/common" + "github.com/containers/podman/v3/cmd/podman/inspect" + "github.com/containers/podman/v3/cmd/podman/registry" + "github.com/containers/podman/v3/pkg/domain/entities" + "github.com/containers/podman/v3/utils" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/spf13/cobra" +) + +var ( + volumeExportDescription = ` +podman volume export + +Allow content of volume to be exported into external tar.` + exportCommand = &cobra.Command{ + Annotations: map[string]string{registry.EngineMode: registry.ABIMode}, + Use: "export [options] VOLUME", + Short: "Export volumes", + Args: cobra.ExactArgs(1), + Long: volumeExportDescription, + RunE: export, + ValidArgsFunction: common.AutocompleteVolumes, + } +) + +var ( + // Temporary struct to hold cli values. + cliExportOpts = struct { + Output string + }{} +) + +func init() { + registry.Commands = append(registry.Commands, registry.CliCommand{ + Command: exportCommand, + Parent: volumeCmd, + }) + flags := exportCommand.Flags() + + outputFlagName := "output" + flags.StringVarP(&cliExportOpts.Output, outputFlagName, "o", "/dev/stdout", "Write to a specified file (default: stdout, which must be redirected)") + _ = exportCommand.RegisterFlagCompletionFunc(outputFlagName, completion.AutocompleteDefault) +} + +func export(cmd *cobra.Command, args []string) error { + var inspectOpts entities.InspectOptions + containerEngine := registry.ContainerEngine() + ctx := context.Background() + + if cliExportOpts.Output == "" { + return errors.New("expects output path, use --output=[path]") + } + inspectOpts.Type = inspect.VolumeType + volumeData, _, err := containerEngine.VolumeInspect(ctx, args, inspectOpts) + if err != nil { + return err + } + if len(volumeData) < 1 { + return errors.New("no volume data found") + } + mountPoint := volumeData[0].VolumeConfigResponse.Mountpoint + driver := volumeData[0].VolumeConfigResponse.Driver + volumeOptions := volumeData[0].VolumeConfigResponse.Options + volumeMountStatus, err := containerEngine.VolumeMounted(ctx, args[0]) + if err != nil { + return err + } + if mountPoint == "" { + return errors.New("volume is not mounted anywhere on host") + } + // Check if volume is using external plugin and export only if volume is mounted + if driver != "" && driver != "local" { + if !volumeMountStatus.Value { + return fmt.Errorf("volume is using a driver %s and volume is not mounted on %s", driver, mountPoint) + } + } + // Check if volume is using `local` driver and has mount options type other than tmpfs + if driver == "local" { + if mountOptionType, ok := volumeOptions["type"]; ok { + if mountOptionType != "tmpfs" && !volumeMountStatus.Value { + return fmt.Errorf("volume is using a driver %s and volume is not mounted on %s", driver, mountPoint) + } + } + } + logrus.Debugf("Exporting volume data from %s to %s", mountPoint, cliExportOpts.Output) + err = utils.CreateTarFromSrc(mountPoint, cliExportOpts.Output) + return err +} diff --git a/cmd/podman/volumes/import.go b/cmd/podman/volumes/import.go new file mode 100644 index 000000000..441bd0fe4 --- /dev/null +++ b/cmd/podman/volumes/import.go @@ -0,0 +1,97 @@ +package volumes + +import ( + "fmt" + "os" + + "github.com/containers/podman/v3/cmd/podman/common" + "github.com/containers/podman/v3/cmd/podman/inspect" + "github.com/containers/podman/v3/cmd/podman/parse" + "github.com/containers/podman/v3/cmd/podman/registry" + "github.com/containers/podman/v3/pkg/domain/entities" + "github.com/containers/podman/v3/utils" + "github.com/pkg/errors" + "github.com/spf13/cobra" +) + +var ( + importDescription = `Imports contents into a podman volume from specified tarball (.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz).` + importCommand = &cobra.Command{ + Annotations: map[string]string{registry.EngineMode: registry.ABIMode}, + Use: "import VOLUME [SOURCE]", + Short: "Import a tarball contents into a podman volume", + Long: importDescription, + RunE: importVol, + Args: cobra.ExactArgs(2), + ValidArgsFunction: common.AutocompleteVolumes, + Example: `podman volume import my_vol /home/user/import.tar + cat ctr.tar | podman import volume my_vol -`, + } +) + +func init() { + registry.Commands = append(registry.Commands, registry.CliCommand{ + Command: importCommand, + Parent: volumeCmd, + }) +} + +func importVol(cmd *cobra.Command, args []string) error { + var inspectOpts entities.InspectOptions + var tarFile *os.File + containerEngine := registry.ContainerEngine() + ctx := registry.Context() + // create a slice of volumes since inspect expects slice as arg + volumes := []string{args[0]} + tarPath := args[1] + + if tarPath != "-" { + err := parse.ValidateFileName(tarPath) + if err != nil { + return err + } + + // open tar file + tarFile, err = os.Open(tarPath) + if err != nil { + return err + } + } else { + tarFile = os.Stdin + } + + inspectOpts.Type = inspect.VolumeType + volumeData, _, err := containerEngine.VolumeInspect(ctx, volumes, inspectOpts) + if err != nil { + return err + } + if len(volumeData) < 1 { + return errors.New("no volume data found") + } + mountPoint := volumeData[0].VolumeConfigResponse.Mountpoint + driver := volumeData[0].VolumeConfigResponse.Driver + volumeOptions := volumeData[0].VolumeConfigResponse.Options + volumeMountStatus, err := containerEngine.VolumeMounted(ctx, args[0]) + if err != nil { + return err + } + if mountPoint == "" { + return errors.New("volume is not mounted anywhere on host") + } + // Check if volume is using external plugin and export only if volume is mounted + if driver != "" && driver != "local" { + if !volumeMountStatus.Value { + return fmt.Errorf("volume is using a driver %s and volume is not mounted on %s", driver, mountPoint) + } + } + // Check if volume is using `local` driver and has mount options type other than tmpfs + if driver == "local" { + if mountOptionType, ok := volumeOptions["type"]; ok { + if mountOptionType != "tmpfs" && !volumeMountStatus.Value { + return fmt.Errorf("volume is using a driver %s and volume is not mounted on %s", driver, mountPoint) + } + } + } + // dont care if volume is mounted or not we are gonna import everything to mountPoint + return utils.UntarToFileSystem(mountPoint, tarFile, nil) +} diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh index da43ffb0a..c1972b90f 100755 --- a/contrib/cirrus/runner.sh +++ b/contrib/cirrus/runner.sh @@ -88,7 +88,8 @@ function _run_bindings() { } function _run_docker-py() { - msg "This is docker-py stub, it is only a stub" + source venv/bin/activate + make run-docker-py-tests } function _run_endpoint() { diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 86b8d78ef..d0c348d58 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -77,6 +77,13 @@ case "$CG_FS_TYPE" in else echo "OCI_RUNTIME=runc" >> /etc/ci_environment fi + + # As a general policy CGv1 + runc should coincide with the "older" + # VM Images in CI. Verify this is the case. + if [[ -n "$VM_IMAGE_NAME" ]] && [[ ! "$VM_IMAGE_NAME" =~ prior ]] + then + die "Most recent distro. version should never run with CGv1" + fi fi ;; cgroup2fs) @@ -85,6 +92,13 @@ case "$CG_FS_TYPE" in # which uses runc as the default. warn "Forcing testing with crun instead of runc" echo "OCI_RUNTIME=crun" >> /etc/ci_environment + + # As a general policy CGv2 + crun should coincide with the "newer" + # VM Images in CI. Verify this is the case. + if [[ -n "$VM_IMAGE_NAME" ]] && [[ "$VM_IMAGE_NAME" =~ prior ]] + then + die "Least recent distro. version should never run with CGv2" + fi fi ;; *) die_unknown CG_FS_TYPE @@ -191,12 +205,24 @@ case "$TEST_FLAVOR" in bigto dnf install -y glibc-minimal-langpack rpm-build fi ;& - docker-py) ;& + docker-py) + remove_packaged_podman_files + make install PREFIX=/usr ETCDIR=/etc + + # TODO: Don't install stuff at test runtime! Do this from + # cache_images/fedora_packaging.sh in containers/automation_images + # and STRONGLY prefer installing RPMs vs pip packages in venv + dnf install -y python3-virtualenv python3-pytest4 + virtualenv venv + source venv/bin/activate + pip install --upgrade pip + pip install --requirement $GOSRC/test/python/requirements.txt + ;; build) make clean ;; unit) ;; apiv2) ;& # use next item compose) - dnf install -y $PACKAGE_DOWNLOAD_DIR/podman-docker* + rpm -ivh $PACKAGE_DOWNLOAD_DIR/podman-docker* ;& # continue with next item int) ;& sys) ;& diff --git a/contrib/podmanimage/README.md b/contrib/podmanimage/README.md index 6effec38b..b7be328c7 100644 --- a/contrib/podmanimage/README.md +++ b/contrib/podmanimage/README.md @@ -17,10 +17,10 @@ default to `/`. The container images are: * `quay.io/containers/podman:<version>` and `quay.io/podman/stable:<version>` - - These images are built when a new Podman version becomes available in - Fedora. These images are intended to be unchanging and stable, they will - never be updated by automation once they've been pushed. For build details, - please [see the configuration file](stable/Dockerfile). + These images are built daily. They are intended to contain an unchanging + and stable version of podman. Though for the most recent `<version>` tag, + image contents will be updated to incorporate (especially) security upgrades. + For build details, please [see the configuration file](stable/Dockerfile). * `quay.io/containers/podman:latest` and `quay.io/podman/stable:latest` - Built daily using the same Dockerfile as above. The Podman version will remain the "latest" available in Fedora, however the other image diff --git a/docs/source/markdown/podman-build.1.md b/docs/source/markdown/podman-build.1.md index c65fbccb9..a2ed35f89 100644 --- a/docs/source/markdown/podman-build.1.md +++ b/docs/source/markdown/podman-build.1.md @@ -43,6 +43,8 @@ containers can be left in container storage. Use the `podman ps --all --storage` command to see these containers. External containers can be removed with the `podman rm --storage` command. +`podman buildx build` command is an alias of `podman build`. Not all `buildx build` features are available in Podman. The `buildx build` option is provided for scripting compatibility. + ## OPTIONS #### **--add-host**=*host* diff --git a/docs/source/markdown/podman-create.1.md b/docs/source/markdown/podman-create.1.md index 51f51c10a..b5c324459 100644 --- a/docs/source/markdown/podman-create.1.md +++ b/docs/source/markdown/podman-create.1.md @@ -453,10 +453,9 @@ When using pods, create an init style container, which is run after the infra co but before regular pod containers are started. Init containers are useful for running setup operations for the pod's applications. -Valid values for `init-ctr` type are *always* or *oneshot*. The *always* value -means the container will run with each and every `pod start`, whereas the *oneshot* -value means is will ony run once when the pod is started and then the container is -removed. +Valid values for `init-ctr` type are *always* or *once*. The *always* value +means the container will run with each and every `pod start`, whereas the *once* +value means the container will only run once when the pod is started and then the container is removed. Init containers are only run on pod `start`. Restarting a pod will not execute any init containers should they be present. Furthermore, init containers can only be created in a @@ -661,13 +660,13 @@ Set the network mode for the container. Invalid if using **--dns**, **--dns-opt* Valid _mode_ values are: -- **bridge**: create a network stack on the default bridge; -- **none**: no networking; -- **container:**_id_: reuse another container's network stack; -- **host**: use the Podman host network stack. Note: the host mode gives the container full access to local system services such as D-bus and is therefore considered insecure; -- _network-id_: connect to a user-defined network, multiple networks should be comma-separated; -- **ns:**_path_: path to a network namespace to join; -- **private**: create a new namespace for the container (default) +- **bridge**: Create a network stack on the default bridge. This is the default for rootfull containers. +- **none**: Create a network namespace for the container but do not configure network interfaces for it, thus the container has no network connectivity. +- **container:**_id_: Reuse another container's network stack. +- **host**: Do not create a network namespace, the container will use the host's network. Note: The host mode gives the container full access to local system services such as D-bus and is therefore considered insecure. +- **network**: Connect to a user-defined network, multiple networks should be comma-separated. +- **ns:**_path_: Path to a network namespace to join. +- **private**: Create a new namespace for the container. This will use the **bridge** mode for rootfull containers and **slirp4netns** for rootless ones. - **slirp4netns[:OPTIONS,...]**: use **slirp4netns**(1) to create a user network stack. This is the default for rootless containers. It is possible to specify these additional options: - **allow_host_loopback=true|false**: Allow the slirp4netns to reach the host loopback IP (`10.0.2.2`, which is added to `/etc/hosts` as `host.containers.internal` for your convenience). Default is false. - **mtu=MTU**: Specify the MTU to use for this network. (Default is `65520`). @@ -678,7 +677,8 @@ Valid _mode_ values are: - **outbound_addr6=INTERFACE**: Specify the outbound interface slirp should bind to (ipv6 traffic only). - **outbound_addr6=IPv6**: Specify the outbound ipv6 address slirp should bind to. - **port_handler=rootlesskit**: Use rootlesskit for port forwarding. Default. - - **port_handler=slirp4netns**: Use the slirp4netns port forwarding. + Note: Rootlesskit changes the source IP address of incoming packets to a IP address in the container network namespace, usually `10.0.2.100`. If your application requires the real source IP address, e.g. web server logs, use the slirp4netns port handler. The rootlesskit port handler is also used for rootless containers when connected to user-defined networks. + - **port_handler=slirp4netns**: Use the slirp4netns port forwarding, it is slower than rootlesskit but preserves the correct source IP address. This port handler cannot be used for user-defined networks. #### **--network-alias**=*alias* @@ -991,6 +991,10 @@ Maximum time a container is allowed to run before conmon sends it the kill signal. By default containers will run until they exit or are stopped by `podman stop`. +#### **--tls-verify**=**true**|**false** + +Require HTTPS and verify certificates when contacting registries (default: true). If explicitly set to true, then TLS verification will be used. If set to false, then TLS verification will not be used. If not specified, TLS verification will be used unless the target registry is listed as an insecure registry in registries.conf. + #### **--tmpfs**=*fs* Create a tmpfs mount @@ -1123,9 +1127,9 @@ Podman allocates unique ranges of UIDs and GIDs from the `containers` subpordina Valid `auto`options: - - *gidmapping*=_HOST_GID:CONTAINER_GID:SIZE_: to force a GID mapping to be present in the user namespace. + - *gidmapping*=_CONTAINER_GID:HOST_GID:SIZE_: to force a GID mapping to be present in the user namespace. - *size*=_SIZE_: to specify an explicit size for the automatic user namespace. e.g. `--userns=auto:size=8192`. If `size` is not specified, `auto` will estimate a size for the user namespace. - - *uidmapping*=_HOST_UID:CONTAINER_UID:SIZE_: to force a UID mapping to be present in the user namespace. + - *uidmapping*=_CONTAINER_UID:HOST_UID:SIZE_: to force a UID mapping to be present in the user namespace. **container:**_id_: join the user namespace of the specified container. diff --git a/docs/source/markdown/podman-play-kube.1.md b/docs/source/markdown/podman-play-kube.1.md index ad5ae7e4c..268e4bbcb 100644 --- a/docs/source/markdown/podman-play-kube.1.md +++ b/docs/source/markdown/podman-play-kube.1.md @@ -35,6 +35,36 @@ A Kubernetes PersistentVolumeClaim represents a Podman named volume. Only the Pe - volume.podman.io/gid - volume.podman.io/mount-options +Play kube is capable of building images on the fly given the correct directory layout and Containerfiles. This +option is not available for remote clients yet. Consider the following excerpt from a YAML file: +``` +apiVersion: v1 +kind: Pod +metadata: +... +spec: + containers: + - command: + - top + - name: container + value: podman + image: foobar +... +``` + +If there is a directory named `foobar` in the current working directory with a file named `Containerfile` or `Dockerfile`, +Podman play kube will build that image and name it `foobar`. An example directory structure for this example would look +like: +``` +|- mykubefiles + |- myplayfile.yaml + |- foobar + |- Containerfile +``` + +The build will consider `foobar` to be the context directory for the build. If there is an image in local storage +called `foobar`, the image will not be built unless the `--build` flag is used. + ## OPTIONS #### **--authfile**=*path* @@ -45,6 +75,10 @@ If the authorization state is not found there, $HOME/.docker/config.json is chec Note: You can also override the default path of the authentication file by setting the REGISTRY\_AUTH\_FILE environment variable. `export REGISTRY_AUTH_FILE=path` +#### **--build** + +Build images even if they are found in the local storage. + #### **--cert-dir**=*path* Use certificates at *path* (\*.crt, \*.cert, \*.key) to connect to the registry. diff --git a/docs/source/markdown/podman-pod-create.1.md b/docs/source/markdown/podman-pod-create.1.md index 441995cb6..4e822dca5 100644 --- a/docs/source/markdown/podman-pod-create.1.md +++ b/docs/source/markdown/podman-pod-create.1.md @@ -101,12 +101,15 @@ Assign a name to the pod. #### **--network**=*mode*, **--net** -Set network mode for the pod. Supported values are +Set network mode for the pod. Supported values are: - **bridge**: Create a network stack on the default bridge. This is the default for rootfull containers. +- **none**: Create a network namespace for the container but do not configure network interfaces for it, thus the container has no network connectivity. - **host**: Do not create a network namespace, all containers in the pod will use the host's network. Note: the host mode gives the container full access to local system services such as D-bus and is therefore considered insecure. -- Comma-separated list of the names of CNI networks the pod should join. -- **slirp4netns[:OPTIONS,...]**: use slirp4netns to create a user network stack. This is the default for rootless containers. It is possible to specify these additional options: - - **allow_host_loopback=true|false**: Allow the slirp4netns to reach the host loopback IP (`10.0.2.2`). Default is false. +- **network**: Connect to a user-defined network, multiple networks should be comma-separated. +- **private**: Create a new namespace for the container. This will use the **bridge** mode for rootfull containers and **slirp4netns** for rootless ones. +- **slirp4netns[:OPTIONS,...]**: use **slirp4netns**(1) to create a user network stack. This is the default for rootless containers. It is possible to specify these additional options: + - **allow_host_loopback=true|false**: Allow the slirp4netns to reach the host loopback IP (`10.0.2.2`, which is added to `/etc/hosts` as `host.containers.internal` for your convenience). Default is false. + - **mtu=MTU**: Specify the MTU to use for this network. (Default is `65520`). - **cidr=CIDR**: Specify ip range to use for this network. (Default is `10.0.2.0/24`). - **enable_ipv6=true|false**: Enable IPv6. Default is false. (Required for `outbound_addr6`). - **outbound_addr=INTERFACE**: Specify the outbound interface slirp should bind to (ipv4 traffic only). @@ -114,7 +117,8 @@ Set network mode for the pod. Supported values are - **outbound_addr6=INTERFACE**: Specify the outbound interface slirp should bind to (ipv6 traffic only). - **outbound_addr6=IPv6**: Specify the outbound ipv6 address slirp should bind to. - **port_handler=rootlesskit**: Use rootlesskit for port forwarding. Default. - - **port_handler=slirp4netns**: Use the slirp4netns port forwarding. + Note: Rootlesskit changes the source IP address of incoming packets to a IP address in the container network namespace, usually `10.0.2.100`. If your application requires the real source IP address, e.g. web server logs, use the slirp4netns port handler. The rootlesskit port handler is also used for rootless containers when connected to user-defined networks. + - **port_handler=slirp4netns**: Use the slirp4netns port forwarding, it is slower than rootlesskit but preserves the correct source IP address. This port handler cannot be used for user-defined networks. #### **--network-alias**=strings @@ -164,6 +168,19 @@ podman generates a UUID for each pod, and if a name is not assigned to the container with **--name** then a random string name will be generated for it. The name is useful any place you need to identify a pod. +#### **--userns**=*mode* + +Set the user namespace mode for all the containers in a pod. It defaults to the **PODMAN_USERNS** environment variable. An empty value ("") means user namespaces are disabled. + +Valid _mode_ values are: + +- *auto[:*_OPTIONS,..._*]*: automatically create a namespace. It is possible to specify these options to `auto`: + - *gidmapping=*_CONTAINER_GID:HOST_GID:SIZE_ to force a GID mapping to be present in the user namespace. + - *size=*_SIZE_: to specify an explicit size for the automatic user namespace. e.g. `--userns=auto:size=8192`. If `size` is not specified, `auto` will estimate a size for the user namespace. + - *uidmapping=*_CONTAINER_UID:HOST_UID:SIZE_ to force a UID mapping to be present in the user namespace. +- *host*: run in the user namespace of the caller. The processes running in the container will have the same privileges on the host as any other process launched by the calling user (default). +- *keep-id*: creates a user namespace where the current rootless user's UID:GID are mapped to the same values in the container. This option is ignored for containers created by the root user. + ## EXAMPLES ``` diff --git a/docs/source/markdown/podman-pod-ps.1.md b/docs/source/markdown/podman-pod-ps.1.md index 156adccaa..ed0789e93 100644 --- a/docs/source/markdown/podman-pod-ps.1.md +++ b/docs/source/markdown/podman-pod-ps.1.md @@ -98,6 +98,7 @@ Valid filters are listed below: | id | [ID] Pod's ID (accepts regex) | | name | [Name] Pod's name (accepts regex) | | label | [Key] or [Key=Value] Label assigned to a container | +| until | Only list pods created before given timestamp | | status | Pod's status: `stopped`, `running`, `paused`, `exited`, `dead`, `created`, `degraded` | | network | [Network] name or full ID of network | | ctr-names | Container name within the pod (accepts regex) | diff --git a/docs/source/markdown/podman-run.1.md b/docs/source/markdown/podman-run.1.md index 38ed44582..caff714d6 100644 --- a/docs/source/markdown/podman-run.1.md +++ b/docs/source/markdown/podman-run.1.md @@ -680,13 +680,13 @@ Set the network mode for the container. Invalid if using **--dns**, **--dns-opt* Valid _mode_ values are: -- **bridge**: create a network stack on the default bridge; -- **none**: no networking; -- **container:**_id_: reuse another container's network stack; -- **host**: use the Podman host network stack. Note: the host mode gives the container full access to local system services such as D-bus and is therefore considered insecure; -- _network-id_: connect to a user-defined network, multiple networks should be comma-separated; -- **ns:**_path_: path to a network namespace to join; -- **private**: create a new namespace for the container (default) +- **bridge**: Create a network stack on the default bridge. This is the default for rootfull containers. +- **none**: Create a network namespace for the container but do not configure network interfaces for it, thus the container has no network connectivity. +- **container:**_id_: Reuse another container's network stack. +- **host**: Do not create a network namespace, the container will use the host's network. Note: The host mode gives the container full access to local system services such as D-bus and is therefore considered insecure. +- **network**: Connect to a user-defined network, multiple networks should be comma-separated. +- **ns:**_path_: Path to a network namespace to join. +- **private**: Create a new namespace for the container. This will use the **bridge** mode for rootfull containers and **slirp4netns** for rootless ones. - **slirp4netns[:OPTIONS,...]**: use **slirp4netns**(1) to create a user network stack. This is the default for rootless containers. It is possible to specify these additional options: - **allow_host_loopback=true|false**: Allow the slirp4netns to reach the host loopback IP (`10.0.2.2`, which is added to `/etc/hosts` as `host.containers.internal` for your convenience). Default is false. - **mtu=MTU**: Specify the MTU to use for this network. (Default is `65520`). @@ -697,7 +697,8 @@ Valid _mode_ values are: - **outbound_addr6=INTERFACE**: Specify the outbound interface slirp should bind to (ipv6 traffic only). - **outbound_addr6=IPv6**: Specify the outbound ipv6 address slirp should bind to. - **port_handler=rootlesskit**: Use rootlesskit for port forwarding. Default. - - **port_handler=slirp4netns**: Use the slirp4netns port forwarding. + Note: Rootlesskit changes the source IP address of incoming packets to a IP address in the container network namespace, usually `10.0.2.100`. If your application requires the real source IP address, e.g. web server logs, use the slirp4netns port handler. The rootlesskit port handler is also used for rootless containers when connected to user-defined networks. + - **port_handler=slirp4netns**: Use the slirp4netns port forwarding, it is slower than rootlesskit but preserves the correct source IP address. This port handler cannot be used for user-defined networks. #### **--network-alias**=*alias* @@ -1047,6 +1048,10 @@ Maximum time a container is allowed to run before conmon sends it the kill signal. By default containers will run until they exit or are stopped by `podman stop`. +#### **--tls-verify**=**true**|**false** + +Require HTTPS and verify certificates when contacting registries (default: true). If explicitly set to true, then TLS verification will be used. If set to false, then TLS verification will not be used. If not specified, TLS verification will be used unless the target registry is listed as an insecure registry in registries.conf. + #### **--tmpfs**=*fs* Create a tmpfs mount. @@ -1181,9 +1186,9 @@ Podman allocates unique ranges of UIDs and GIDs from the `containers` subpordina Valid `auto`options: - - *gidmapping*=_HOST_GID:CONTAINER_GID:SIZE_: to force a GID mapping to be present in the user namespace. + - *gidmapping*=_CONTAINER_GID:HOST_GID:SIZE_: to force a GID mapping to be present in the user namespace. - *size*=_SIZE_: to specify an explicit size for the automatic user namespace. e.g. `--userns=auto:size=8192`. If `size` is not specified, `auto` will estimate a size for the user namespace. - - *uidmapping*=_HOST_UID:CONTAINER_UID:SIZE_: to force a UID mapping to be present in the user namespace. + - *uidmapping*=_CONTAINER_UID:HOST_UID:SIZE_: to force a UID mapping to be present in the user namespace. **container:**_id_: join the user namespace of the specified container. diff --git a/docs/source/markdown/podman-volume-export.1.md b/docs/source/markdown/podman-volume-export.1.md new file mode 100644 index 000000000..7db1e421d --- /dev/null +++ b/docs/source/markdown/podman-volume-export.1.md @@ -0,0 +1,38 @@ +% podman-volume-export(1) + +## NAME +podman\-volume\-export - Exports volume to external tar + +## SYNOPSIS +**podman volume export** [*options*] *volume* + +## DESCRIPTION + +**podman volume export** exports the contents of a podman volume and saves it as a tarball +on the local machine. **podman volume export** writes to STDOUT by default and can be +redirected to a file using the `--output` flag. + +Note: Following command is not supported by podman-remote. + +**podman volume export [OPTIONS] VOLUME** + +## OPTIONS + +#### **--output**, **-o**=*file* + +Write to a file, default is STDOUT + +#### **--help** + +Print usage statement + + +## EXAMPLES + +``` +$ podman volume export myvol --output myvol.tar + +``` + +## SEE ALSO +podman-volume(1), podman-volume-import(1) diff --git a/docs/source/markdown/podman-volume-import.1.md b/docs/source/markdown/podman-volume-import.1.md new file mode 100644 index 000000000..6bb868774 --- /dev/null +++ b/docs/source/markdown/podman-volume-import.1.md @@ -0,0 +1,35 @@ +% podman-volume-import(1) + +## NAME +podman\-volume\-import - Import tarball contents into a podman volume + +## SYNOPSIS +**podman volume import** *volume* [*source*] + +## DESCRIPTION + +**podman volume import** imports the contents of a tarball into the podman volume's mount point. +**podman volume import** can consume piped input when using `-` as source path. + +Note: Following command is not supported by podman-remote. + +**podman volume import VOLUME [SOURCE]** + +#### **--help** + +Print usage statement + +## EXAMPLES + +``` +$ gunzip -c hellow.tar.gz | podman volume import myvol - +``` +``` +$ podman volume import myvol test.tar +``` +``` +$ podman volume export myvol | podman volume import oldmyvol - +``` + +## SEE ALSO +podman-volume(1), podman-volume-export(1) diff --git a/docs/source/markdown/podman-volume.1.md b/docs/source/markdown/podman-volume.1.md index 5af5eb50e..64b37c28c 100644 --- a/docs/source/markdown/podman-volume.1.md +++ b/docs/source/markdown/podman-volume.1.md @@ -15,6 +15,8 @@ podman volume is a set of subcommands that manage volumes. | ------- | ------------------------------------------------------ | ------------------------------------------------------------------------------ | | create | [podman-volume-create(1)](podman-volume-create.1.md) | Create a new volume. | | exists | [podman-volume-exists(1)](podman-volume-exists.1.md) | Check if the given volume exists. | +| export | [podman-volume-export(1)](podman-volume-export.1.md) | Exports volume to external tar. | +| import | [podman-volume-import(1)](podman-volume-import.1.md) | Import tarball contents into a podman volume. | | inspect | [podman-volume-inspect(1)](podman-volume-inspect.1.md) | Get detailed information on one or more volumes. | | ls | [podman-volume-ls(1)](podman-volume-ls.1.md) | List all the available volumes. | | prune | [podman-volume-prune(1)](podman-volume-prune.1.md) | Remove all unused volumes. | diff --git a/docs/source/volume.rst b/docs/source/volume.rst index ce9ea2cbd..af81f39bc 100644 --- a/docs/source/volume.rst +++ b/docs/source/volume.rst @@ -4,6 +4,10 @@ Volume :doc:`exists <markdown/podman-volume-exists.1>` Check if the given volume exists +:doc:`export <markdown/podman-volume-export.1>` Exports volume to external tar + +:doc:`import <markdown/podman-volume-import.1>` Import tarball contents into a podman volume + :doc:`inspect <markdown/podman-volume-inspect.1>` Display detailed information on one or more volumes :doc:`ls <markdown/podman-volume-ls.1>` List volumes @@ -11,13 +11,13 @@ require ( github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9 github.com/containernetworking/cni v0.8.1 github.com/containernetworking/plugins v0.9.1 - github.com/containers/buildah v1.22.0 - github.com/containers/common v0.42.1 + github.com/containers/buildah v1.22.3 + github.com/containers/common v0.43.2 github.com/containers/conmon v2.0.20+incompatible - github.com/containers/image/v5 v5.15.0 + github.com/containers/image/v5 v5.15.2 github.com/containers/ocicrypt v1.1.2 github.com/containers/psgo v1.5.2 - github.com/containers/storage v1.33.1 + github.com/containers/storage v1.34.1 github.com/coreos/go-systemd/v22 v22.3.2 github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3 github.com/cri-o/ocicni v0.2.1-0.20210621164014-d0acc7862283 @@ -30,7 +30,7 @@ require ( github.com/docker/go-plugins-helpers v0.0.0-20200102110956-c9a8a2d92ccc github.com/docker/go-units v0.4.0 github.com/dtylman/scp v0.0.0-20181017070807-f3000a34aef4 - github.com/fsnotify/fsnotify v1.4.9 + github.com/fsnotify/fsnotify v1.5.1 github.com/ghodss/yaml v1.0.0 github.com/godbus/dbus/v5 v5.0.4 github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf @@ -44,16 +44,16 @@ require ( github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 github.com/mrunalp/fileutils v0.5.0 github.com/onsi/ginkgo v1.16.4 - github.com/onsi/gomega v1.15.0 + github.com/onsi/gomega v1.16.0 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 - github.com/opencontainers/runc v1.0.1 + github.com/opencontainers/runc v1.0.2 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 github.com/opencontainers/runtime-tools v0.9.0 - github.com/opencontainers/selinux v1.8.3 + github.com/opencontainers/selinux v1.8.4 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 - github.com/rootless-containers/rootlesskit v0.14.4 + github.com/rootless-containers/rootlesskit v0.14.5 github.com/sirupsen/logrus v1.8.1 github.com/spf13/cobra v1.2.1 github.com/spf13/pflag v1.0.5 @@ -65,8 +65,8 @@ require ( go.etcd.io/bbolt v1.3.6 golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 + golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b - k8s.io/api v0.22.0 - k8s.io/apimachinery v0.22.0 + k8s.io/api v0.22.1 + k8s.io/apimachinery v0.22.1 ) @@ -238,15 +238,17 @@ github.com/containernetworking/plugins v0.8.6/go.mod h1:qnw5mN19D8fIwkqW7oHHYDHV github.com/containernetworking/plugins v0.8.7/go.mod h1:R7lXeZaBzpfqapcAbHRW8/CYwm0dHzbz0XEjofx0uB0= github.com/containernetworking/plugins v0.9.1 h1:FD1tADPls2EEi3flPc2OegIY1M9pUa9r2Quag7HMLV8= github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRDjeJr6FLK6vuiUwoH7P8= -github.com/containers/buildah v1.22.0 h1:VwDrweEEUkfIB0t+hVhwE6FdoV0PZjCTz9sVkaZyv2g= -github.com/containers/buildah v1.22.0/go.mod h1:a6JsF0iNlJJ5GsiVy16e2fgiUV4S3gWZymrpyqzhar0= -github.com/containers/common v0.42.1 h1:ADOZrVAS8ZY5hBAvr/GoRoPv5Z7TBkxWgxQEXQjlqac= +github.com/containers/buildah v1.22.3 h1:RomxwUa24jMcqzXQetpw4wGMfNlNZLhc9qwyoWHblwc= +github.com/containers/buildah v1.22.3/go.mod h1:JVXRyx5Rkp5w5jwvaXe45kuHtyoxpERMjXrR45+3Wfg= github.com/containers/common v0.42.1/go.mod h1:AaF3ipZfgezsctDuhzLkq4Vl+LkEy7J74ikh2HSXDsg= +github.com/containers/common v0.43.2 h1:oSP5d5sDrq7OkoqLPVrLpi1LZOAwpTwOZXgPDHfmD0E= +github.com/containers/common v0.43.2/go.mod h1:BAoVyRYlxKZKAYpHcFMdrXlIZyzbJp9NwKTgadTd/Dg= github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg= github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= github.com/containers/image/v5 v5.14.0/go.mod h1:SxiBKOcKuT+4yTjD0AskjO+UwFvNcVOJ9qlAw1HNSPU= -github.com/containers/image/v5 v5.15.0 h1:NduhN20ptHNlf0uRny5iTJa2OodB9SLMEB4hKKbzBBs= github.com/containers/image/v5 v5.15.0/go.mod h1:gzdBcooi6AFdiqfzirUqv90hUyHyI0MMdaqKzACKr2s= +github.com/containers/image/v5 v5.15.2 h1:DKicmVr0h1HGkzs9muoErX+fVbV9sV9W5TyMy5perLE= +github.com/containers/image/v5 v5.15.2/go.mod h1:8jejVSzTDfyPwr/HXp9rri34n/vbdavYk6IzTiB3TBw= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= @@ -259,8 +261,10 @@ github.com/containers/psgo v1.5.2/go.mod h1:2ubh0SsreMZjSXW1Hif58JrEcFudQyIy9EzP github.com/containers/storage v1.23.5/go.mod h1:ha26Q6ngehFNhf3AWoXldvAvwI4jFe3ETQAf/CeZPyM= github.com/containers/storage v1.32.6/go.mod h1:mdB+b89p+jU8zpzLTVXA0gWMmIo0WrkfGMh1R8O2IQw= github.com/containers/storage v1.33.0/go.mod h1:FUZPF4nJijX8ixdhByZJXf02cvbyLi6dyDwXdIe8QVY= -github.com/containers/storage v1.33.1 h1:RHUPZ7vQxwoeOoMoKUDsVun4f9Wi8BTXmr/wQiruBYU= github.com/containers/storage v1.33.1/go.mod h1:FUZPF4nJijX8ixdhByZJXf02cvbyLi6dyDwXdIe8QVY= +github.com/containers/storage v1.34.0/go.mod h1:t6I+hTgPU0/tVxQ75vw406wDi/TXwYBqZp4QZV9N7b8= +github.com/containers/storage v1.34.1 h1:PsBGMH7hwuQ3MOr7qTgPznFrE8ebfIbwQbg2gKvg0lE= +github.com/containers/storage v1.34.1/go.mod h1:FY2TcbfgCLMU4lYoKnlZeZXeH353TOTbpDEA+sAcqAY= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -362,8 +366,9 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= -github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= +github.com/fsnotify/fsnotify v1.5.1 h1:mZcQUHVQUQWoPXXtuf9yuEXKudkV2sx1E06UadKWpgI= +github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= github.com/fsouza/go-dockerclient v1.7.3 h1:i6iMcktl688vsKUEExA6gU1UjPgIvmGtJeQ0mbuFqZo= github.com/fsouza/go-dockerclient v1.7.3/go.mod h1:8xfZB8o9SptLNJ13VoV5pMiRbZGWkU/Omu5VOu/KC9Y= github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa/go.mod h1:KnogPXtdwXqoenmZCw6S+25EAm2MkxbG0deNDu4cbSA= @@ -581,8 +586,10 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= +github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -712,8 +719,9 @@ github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1y github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDsH8xc= github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/onsi/gomega v1.14.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= -github.com/onsi/gomega v1.15.0 h1:WjP/FQ/sk43MRmnEcT+MlDw2TFvkrXlprrPST/IudjU= github.com/onsi/gomega v1.15.0/go.mod h1:cIuvLEne0aoVhAgh/O6ac0Op8WWw9H6eYCriF+tEHG0= +github.com/onsi/gomega v1.16.0 h1:6gjqkI8iiRHMvdccRJM8rVKjCWk6ZIm6FTm3ddIe4/c= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/opencontainers/go-digest v0.0.0-20170106003457-a6d0ee40d420/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v0.0.0-20180430190053-c9281466c8b2/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= @@ -732,8 +740,9 @@ github.com/opencontainers/runc v1.0.0-rc9/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rm github.com/opencontainers/runc v1.0.0-rc91/go.mod h1:3Sm6Dt7OT8z88EbdQqqcRN2oCT54jbi72tT/HqgflT8= github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84SM2ImC1fxBuqJ/H0= github.com/opencontainers/runc v1.0.0/go.mod h1:MU2S3KEB2ZExnhnAQYbwjdYV6HwKtDlNbA2Z2OeNDeA= -github.com/opencontainers/runc v1.0.1 h1:G18PGckGdAm3yVQRWDVQ1rLSLntiniKJ0cNRT2Tm5gs= github.com/opencontainers/runc v1.0.1/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= +github.com/opencontainers/runc v1.0.2 h1:opHZMaswlyxz1OuGpBE53Dwe4/xF7EZTY0A2L/FpCOg= +github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= @@ -749,8 +758,9 @@ github.com/opencontainers/selinux v1.5.1/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwy github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= -github.com/opencontainers/selinux v1.8.3 h1:tzZR7AuKB5gU1+53uBkoG4XdIFGZzvJTOVoNbRQI8/4= github.com/opencontainers/selinux v1.8.3/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= +github.com/opencontainers/selinux v1.8.4 h1:krlgQ6/j9CkCXT5oW0yVXdQFOME3NjKuuAZXuR6O7P4= +github.com/opencontainers/selinux v1.8.4/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= github.com/openshift/imagebuilder v1.2.2-0.20210415181909-87f3e48c2656 h1:WaxyNFpmIDu4i6so9r6LVFIbSaXqsj8oitMitt86ae4= github.com/openshift/imagebuilder v1.2.2-0.20210415181909-87f3e48c2656/go.mod h1:9aJRczxCH0mvT6XQ+5STAQaPWz7OsWcU5/mRkt8IWeo= github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913 h1:TnbXhKzrTOyuvWrjI8W6pcoI9XPbLHFXCdN2dtUw7Rw= @@ -811,8 +821,8 @@ github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rootless-containers/rootlesskit v0.14.4 h1:pqx9a+OC/6jjV7sIUKy3D1p6NLEC6WIMiJWAGsGMCUM= -github.com/rootless-containers/rootlesskit v0.14.4/go.mod h1:Ai3detLzryb/4EkzXmNfh8aByUcBXp/qqkQusJs1SO8= +github.com/rootless-containers/rootlesskit v0.14.5 h1:X4eNt2e1h/uSjlssKqpeTY5fatrjDz9F9FX05RJB7Tw= +github.com/rootless-containers/rootlesskit v0.14.5/go.mod h1:Ai3detLzryb/4EkzXmNfh8aByUcBXp/qqkQusJs1SO8= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -1186,8 +1196,9 @@ golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201113234701-d7a72108b828/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -1438,13 +1449,13 @@ honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 k8s.io/api v0.20.1/go.mod h1:KqwcCVogGxQY3nBlRpwt+wpAMF/KjaCc7RpywacvqUo= k8s.io/api v0.20.4/go.mod h1:++lNL1AJMkDymriNniQsWRkMDzRaX2Y/POTUi8yvqYQ= k8s.io/api v0.20.6/go.mod h1:X9e8Qag6JV/bL5G6bU8sdVRltWKmdHsFUGS3eVndqE8= -k8s.io/api v0.22.0 h1:elCpMZ9UE8dLdYxr55E06TmSeji9I3KH494qH70/y+c= -k8s.io/api v0.22.0/go.mod h1:0AoXXqst47OI/L0oGKq9DG61dvGRPXs7X4/B7KyjBCU= +k8s.io/api v0.22.1 h1:ISu3tD/jRhYfSW8jI/Q1e+lRxkR7w9UwQEZ7FgslrwY= +k8s.io/api v0.22.1/go.mod h1:bh13rkTp3F1XEaLGykbyRD2QaTTzPm0e/BMd8ptFONY= k8s.io/apimachinery v0.20.1/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.4/go.mod h1:WlLqWAHZGg07AeltaI0MV5uk1Omp8xaN0JGLY6gkRpU= k8s.io/apimachinery v0.20.6/go.mod h1:ejZXtW1Ra6V1O5H8xPBGz+T3+4gfkTCeExAHKU57MAc= -k8s.io/apimachinery v0.22.0 h1:CqH/BdNAzZl+sr3tc0D3VsK3u6ARVSo3GWyLmfIjbP0= -k8s.io/apimachinery v0.22.0/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= +k8s.io/apimachinery v0.22.1 h1:DTARnyzmdHMz7bFWFDDm22AM4pLWTQECMpRTFu2d2OM= +k8s.io/apimachinery v0.22.1/go.mod h1:O3oNtNadZdeOMxHFVxOreoznohCpy0z6mocxbZr7oJ0= k8s.io/apiserver v0.20.1/go.mod h1:ro5QHeQkgMS7ZGpvf4tSMx6bBOgPfE+f52KwvXfScaU= k8s.io/apiserver v0.20.4/go.mod h1:Mc80thBKOyy7tbvFtB4kJv1kbdD0eIH8k8vianJcbFM= k8s.io/apiserver v0.20.6/go.mod h1:QIJXNt6i6JB+0YQRNcS0hdRHJlMhflFmsBDeSgT1r8Q= diff --git a/libpod/container.go b/libpod/container.go index f3f4b27b7..c57250d72 100644 --- a/libpod/container.go +++ b/libpod/container.go @@ -126,6 +126,10 @@ type Container struct { // This is true if a container is restored from a checkpoint. restoreFromCheckpoint bool + // Used to query the NOTIFY_SOCKET once along with setting up + // mounts etc. + notifySocket string + slirp4netnsSubnet *net.IPNet } @@ -240,7 +244,7 @@ type ContainerImageVolume struct { type ContainerSecret struct { // Secret is the secret *secrets.Secret - // UID is tbe UID of the secret file + // UID is the UID of the secret file UID uint32 // GID is the GID of the secret file GID uint32 @@ -1020,8 +1024,8 @@ func (c *Container) RWSize() (int64, error) { } // IDMappings returns the UID/GID mapping used for the container -func (c *Container) IDMappings() (storage.IDMappingOptions, error) { - return c.config.IDMappings, nil +func (c *Container) IDMappings() storage.IDMappingOptions { + return c.config.IDMappings } // RootUID returns the root user mapping from container diff --git a/libpod/container_commit.go b/libpod/container_commit.go index c1dd42942..87e5d511c 100644 --- a/libpod/container_commit.go +++ b/libpod/container_commit.go @@ -99,6 +99,11 @@ func (c *Container) Commit(ctx context.Context, destImage string, options Contai for _, p := range c.config.PortMappings { importBuilder.SetPort(fmt.Sprintf("%d/%s", p.ContainerPort, p.Protocol)) } + for port, protocols := range c.config.ExposedPorts { + for _, protocol := range protocols { + importBuilder.SetPort(fmt.Sprintf("%d/%s", port, protocol)) + } + } // Labels for k, v := range c.Labels() { importBuilder.SetLabel(k, v) diff --git a/libpod/container_config.go b/libpod/container_config.go index 72a969fe6..b80b23c25 100644 --- a/libpod/container_config.go +++ b/libpod/container_config.go @@ -229,6 +229,12 @@ type ContainerNetworkConfig struct { // namespace // These are not used unless CreateNetNS is true PortMappings []ocicni.PortMapping `json:"portMappings,omitempty"` + // ExposedPorts are the ports which are exposed but not forwarded + // into the container. + // The map key is the port and the string slice contains the protocols, + // e.g. tcp and udp + // These are only set when exposed ports are given but not published. + ExposedPorts map[uint16][]string `json:"exposedPorts,omitempty"` // UseImageResolvConf indicates that resolv.conf should not be // bind-mounted inside the container. // Conflicts with DNSServer, DNSSearch, DNSOption. @@ -376,6 +382,6 @@ type ContainerMiscConfig struct { // EnvSecrets are secrets that are set as environment variables EnvSecrets map[string]*secrets.Secret `json:"secret_env,omitempty"` // InitContainerType specifies if the container is an initcontainer - // and if so, what type: always or oneshot are possible non-nil entries + // and if so, what type: always or once are possible non-nil entries InitContainerType string `json:"init_container_type,omitempty"` } diff --git a/libpod/container_inspect.go b/libpod/container_inspect.go index 8c662c488..97318a2e8 100644 --- a/libpod/container_inspect.go +++ b/libpod/container_inspect.go @@ -624,7 +624,7 @@ func (c *Container) generateInspectContainerHostConfig(ctrSpec *spec.Spec, named // Port bindings. // Only populate if we're using CNI to configure the network. if c.config.CreateNetNS { - hostConfig.PortBindings = makeInspectPortBindings(c.config.PortMappings) + hostConfig.PortBindings = makeInspectPortBindings(c.config.PortMappings, c.config.ExposedPorts) } else { hostConfig.PortBindings = make(map[string][]define.InspectHostPort) } diff --git a/libpod/container_internal.go b/libpod/container_internal.go index 8ffcccf4c..3f7a4807d 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -367,6 +367,12 @@ func (c *Container) setupStorageMapping(dest, from *storage.IDMappingOptions) { return } *dest = *from + // If we are creating a container inside a pod, we always want to inherit the + // userns settings from the infra container. So clear the auto userns settings + // so that we don't request storage for a new uid/gid map. + if c.PodID() != "" && !c.IsInfra() { + dest.AutoUserNs = false + } if dest.AutoUserNs { overrides := c.getUserOverrides() dest.AutoUserNsOpts.PasswdFile = overrides.ContainerEtcPasswdPath diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index f30f622ac..8b73c82de 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -352,6 +352,10 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { return nil, err } + if err := c.mountNotifySocket(g); err != nil { + return nil, err + } + // Get host UID and GID based on the container process UID and GID. hostUID, hostGID, err := butil.GetHostIDs(util.IDtoolsToRuntimeSpec(c.config.IDMappings.UIDMap), util.IDtoolsToRuntimeSpec(c.config.IDMappings.GIDMap), uint32(execUser.Uid), uint32(execUser.Gid)) if err != nil { @@ -659,7 +663,7 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { } } - if c.config.IDMappings.AutoUserNs { + if c.config.UserNsCtr == "" && c.config.IDMappings.AutoUserNs { if err := g.AddOrReplaceLinuxNamespace(string(spec.UserNamespace), ""); err != nil { return nil, err } @@ -777,6 +781,41 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { return g.Config, nil } +// mountNotifySocket mounts the NOTIFY_SOCKET into the container if it's set +// and if the sdnotify mode is set to container. It also sets c.notifySocket +// to avoid redundantly looking up the env variable. +func (c *Container) mountNotifySocket(g generate.Generator) error { + notify, ok := os.LookupEnv("NOTIFY_SOCKET") + if !ok { + return nil + } + c.notifySocket = notify + + if c.config.SdNotifyMode != define.SdNotifyModeContainer { + return nil + } + + notifyDir := filepath.Join(c.bundlePath(), "notify") + logrus.Debugf("checking notify %q dir", notifyDir) + if err := os.MkdirAll(notifyDir, 0755); err != nil { + if !os.IsExist(err) { + return errors.Wrapf(err, "unable to create notify %q dir", notifyDir) + } + } + if err := label.Relabel(notifyDir, c.MountLabel(), true); err != nil { + return errors.Wrapf(err, "relabel failed %q", notifyDir) + } + logrus.Debugf("add bindmount notify %q dir", notifyDir) + if _, ok := c.state.BindMounts["/run/notify"]; !ok { + c.state.BindMounts["/run/notify"] = notifyDir + } + + // Set the container's notify socket to the proxy socket created by conmon + g.AddProcessEnv("NOTIFY_SOCKET", "/run/notify/notify.sock") + + return nil +} + // systemd expects to have /run, /run/lock and /tmp on tmpfs // It also expects to be able to write to /sys/fs/cgroup/systemd and /var/log/journal func (c *Container) setupSystemd(mounts []spec.Mount, g generate.Generator) error { @@ -1730,6 +1769,7 @@ rootless=%d c.state.BindMounts[dest] = src } } + return nil } @@ -1782,7 +1822,7 @@ func (c *Container) generateResolvConf() (string, error) { cniResponse := c.state.NetworkStatus for _, i := range cniResponse { for _, ip := range i.IPs { - // Note: only using To16() does not work since it also returns a vaild ip for ipv4 + // Note: only using To16() does not work since it also returns a valid ip for ipv4 if ip.Address.IP.To4() == nil && ip.Address.IP.To16() != nil { ipv6 = true } @@ -1884,7 +1924,7 @@ func (c *Container) generateResolvConf() (string, error) { return "", err } - return filepath.Join(c.state.RunDir, "resolv.conf"), nil + return destPath, nil } // generateHosts creates a containers hosts file diff --git a/libpod/container_log.go b/libpod/container_log.go index 743c9c61b..3988bb654 100644 --- a/libpod/container_log.go +++ b/libpod/container_log.go @@ -14,6 +14,13 @@ import ( "github.com/sirupsen/logrus" ) +// logDrivers stores the currently available log drivers, do not modify +var logDrivers []string + +func init() { + logDrivers = append(logDrivers, define.KubernetesLogging, define.NoLogging) +} + // Log is a runtime function that can read one or more container logs. func (r *Runtime) Log(ctx context.Context, containers []*Container, options *logs.LogOptions, logChannel chan *logs.LogLine) error { for _, ctr := range containers { diff --git a/libpod/container_log_linux.go b/libpod/container_log_linux.go index d4afaa52a..ca1e11ef5 100644 --- a/libpod/container_log_linux.go +++ b/libpod/container_log_linux.go @@ -9,8 +9,10 @@ import ( "strings" "time" + "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/libpod/events" "github.com/containers/podman/v3/libpod/logs" + "github.com/coreos/go-systemd/v22/journal" "github.com/coreos/go-systemd/v22/sdjournal" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -24,6 +26,23 @@ const ( journaldLogErr = "3" ) +func init() { + logDrivers = append(logDrivers, define.JournaldLogging) +} + +// initializeJournal will write an empty string to the journal +// when a journal is created. This solves a problem when people +// attempt to read logs from a container that has never had stdout/stderr +func (c *Container) initializeJournal(ctx context.Context) error { + m := make(map[string]string) + m["SYSLOG_IDENTIFIER"] = "podman" + m["PODMAN_ID"] = c.ID() + m["CONTAINER_ID_FULL"] = c.ID() + history := events.History + m["PODMAN_EVENT"] = history.String() + return journal.Send("", journal.PriInfo, m) +} + func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOptions, logChannel chan *logs.LogLine) error { journal, err := sdjournal.NewJournal() if err != nil { @@ -58,12 +77,12 @@ func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOption } // API requires Next() immediately after SeekHead(). if _, err := journal.Next(); err != nil { - return errors.Wrap(err, "initial journal cursor") + return errors.Wrap(err, "next journal") } // API requires a next|prev before getting a cursor. if _, err := journal.Previous(); err != nil { - return errors.Wrap(err, "initial journal cursor") + return errors.Wrap(err, "previous journal") } // Note that the initial cursor may not yet be ready, so we'll do an @@ -72,14 +91,14 @@ func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOption var cursorError error for i := 1; i <= 3; i++ { cursor, cursorError = journal.GetCursor() - if err != nil { + if cursorError != nil { + time.Sleep(time.Duration(i*100) * time.Millisecond) continue } - time.Sleep(time.Duration(i*100) * time.Millisecond) break } if cursorError != nil { - return errors.Wrap(cursorError, "inital journal cursor") + return errors.Wrap(cursorError, "initial journal cursor") } // We need the container's events in the same journal to guarantee @@ -99,6 +118,7 @@ func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOption tailQueue := []*logs.LogLine{} // needed for options.Tail doTail := options.Tail > 0 + lastReadCursor := "" for { select { case <-ctx.Done(): @@ -108,18 +128,25 @@ func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOption // Fallthrough } - if _, err := journal.Next(); err != nil { - logrus.Errorf("Failed to move journal cursor to next entry: %v", err) - return + if lastReadCursor != "" { + // Advance to next entry if we read this one. + if _, err := journal.Next(); err != nil { + logrus.Errorf("Failed to move journal cursor to next entry: %v", err) + return + } } - latestCursor, err := journal.GetCursor() + + // Fetch the location of this entry, presumably either + // the one that follows the last one we read, or that + // same last one, if there is no next entry (yet). + cursor, err = journal.GetCursor() if err != nil { logrus.Errorf("Failed to get journal cursor: %v", err) return } - // Hit the end of the journal. - if cursor == latestCursor { + // Hit the end of the journal (so far?). + if cursor == lastReadCursor { if doTail { // Flush *once* we hit the end of the journal. startIndex := int64(len(tailQueue)-1) - options.Tail @@ -140,8 +167,9 @@ func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOption journal.Wait(sdjournal.IndefiniteWait) continue } - cursor = latestCursor + lastReadCursor = cursor + // Read the journal entry. entry, err := journal.GetEntry() if err != nil { logrus.Errorf("Failed to get journal entry: %v", err) diff --git a/libpod/container_log_unsupported.go b/libpod/container_log_unsupported.go index d10082141..a551df942 100644 --- a/libpod/container_log_unsupported.go +++ b/libpod/container_log_unsupported.go @@ -13,3 +13,7 @@ import ( func (c *Container) readFromJournal(_ context.Context, _ *logs.LogOptions, _ chan *logs.LogLine) error { return errors.Wrapf(define.ErrOSNotSupported, "Journald logging only enabled with systemd on linux") } + +func (c *Container) initializeJournal(ctx context.Context) error { + return errors.Wrapf(define.ErrOSNotSupported, "Journald logging only enabled with systemd on linux") +} diff --git a/libpod/define/container.go b/libpod/define/container.go index f0aca92aa..bb44a6a4a 100644 --- a/libpod/define/container.go +++ b/libpod/define/container.go @@ -34,5 +34,5 @@ const ( AlwaysInitContainer = "always" // OneShotInitContainer is a container that only runs as init once // and is then deleted. - OneShotInitContainer = "oneshot" + OneShotInitContainer = "once" ) diff --git a/libpod/define/info.go b/libpod/define/info.go index de709be74..95c1196dd 100644 --- a/libpod/define/info.go +++ b/libpod/define/info.go @@ -8,6 +8,7 @@ type Info struct { Host *HostInfo `json:"host"` Store *StoreInfo `json:"store"` Registries map[string]interface{} `json:"registries"` + Plugins Plugins `json:"plugins"` Version Version `json:"version"` } @@ -123,3 +124,11 @@ type ContainerStore struct { Running int `json:"running"` Stopped int `json:"stopped"` } + +type Plugins struct { + Volume []string `json:"volume"` + Network []string `json:"network"` + Log []string `json:"log"` + // FIXME what should we do with Authorization, docker seems to return nothing by default + // Authorization []string `json:"authorization"` +} diff --git a/libpod/define/pod_inspect.go b/libpod/define/pod_inspect.go index a17304875..f91fd198d 100644 --- a/libpod/define/pod_inspect.go +++ b/libpod/define/pod_inspect.go @@ -105,6 +105,8 @@ type InspectPodInfraConfig struct { CPUSetCPUs string `json:"cpuset_cpus,omitempty"` // Pid is the PID namespace mode of the pod's infra container PidNS string `json:"pid_ns,omitempty"` + // UserNS is the usernamespace that all the containers in the pod will join. + UserNS string `json:"userns,omitempty"` } // InspectPodContainerInfo contains information on a container in a pod. diff --git a/libpod/diff.go b/libpod/diff.go index cdd5e79cb..6a50bef32 100644 --- a/libpod/diff.go +++ b/libpod/diff.go @@ -14,6 +14,7 @@ var initInodes = map[string]bool{ "/etc/resolv.conf": true, "/proc": true, "/run": true, + "/run/notify": true, "/run/.containerenv": true, "/run/secrets": true, "/sys": true, diff --git a/libpod/info.go b/libpod/info.go index cdc73780f..8f4c7f015 100644 --- a/libpod/info.go +++ b/libpod/info.go @@ -18,6 +18,7 @@ import ( "github.com/containers/image/v5/pkg/sysregistriesv2" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/libpod/linkmode" + "github.com/containers/podman/v3/libpod/network" "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/rootless" "github.com/containers/storage" @@ -65,6 +66,16 @@ func (r *Runtime) info() (*define.Info, error) { if len(regs) > 0 { registries["search"] = regs } + volumePlugins := make([]string, 0, len(r.config.Engine.VolumePlugins)+1) + // the local driver always exists + volumePlugins = append(volumePlugins, "local") + for plugin := range r.config.Engine.VolumePlugins { + volumePlugins = append(volumePlugins, plugin) + } + info.Plugins.Volume = volumePlugins + // TODO move this into the new network interface + info.Plugins.Network = []string{network.BridgeNetworkDriver, network.MacVLANNetworkDriver} + info.Plugins.Log = logDrivers info.Registries = registries return &info, nil @@ -141,19 +152,24 @@ func (r *Runtime) hostInfo() (*define.HostInfo, error) { } info.CGroupsVersion = cgroupVersion - if rootless.IsRootless() { - if path, err := exec.LookPath("slirp4netns"); err == nil { - version, err := programVersion(path) - if err != nil { - logrus.Warnf("Failed to retrieve program version for %s: %v", path, err) - } - program := define.SlirpInfo{ - Executable: path, - Package: packageVersion(path), - Version: version, - } - info.Slirp4NetNS = program + slirp4netnsPath := r.config.Engine.NetworkCmdPath + if slirp4netnsPath == "" { + slirp4netnsPath, _ = exec.LookPath("slirp4netns") + } + if slirp4netnsPath != "" { + version, err := programVersion(slirp4netnsPath) + if err != nil { + logrus.Warnf("Failed to retrieve program version for %s: %v", slirp4netnsPath, err) + } + program := define.SlirpInfo{ + Executable: slirp4netnsPath, + Package: packageVersion(slirp4netnsPath), + Version: version, } + info.Slirp4NetNS = program + } + + if rootless.IsRootless() { uidmappings, err := rootless.ReadMappingsProc("/proc/self/uid_map") if err != nil { return nil, errors.Wrapf(err, "error reading uid mappings") diff --git a/libpod/logs/log.go b/libpod/logs/log.go index 1a0223edc..a584de0ee 100644 --- a/libpod/logs/log.go +++ b/libpod/logs/log.go @@ -251,11 +251,19 @@ func (l *LogLine) Write(stdout io.Writer, stderr io.Writer, logOpts *LogOptions) switch l.Device { case "stdout": if stdout != nil { - fmt.Fprintln(stdout, l.String(logOpts)) + if l.Partial() { + fmt.Fprint(stdout, l.String(logOpts)) + } else { + fmt.Fprintln(stdout, l.String(logOpts)) + } } case "stderr": if stderr != nil { - fmt.Fprintln(stderr, l.String(logOpts)) + if l.Partial() { + fmt.Fprint(stderr, l.String(logOpts)) + } else { + fmt.Fprintln(stderr, l.String(logOpts)) + } } default: // Warn the user if the device type does not match. Most likely the file is corrupted. diff --git a/libpod/network/cni/README.md b/libpod/network/cni/README.md new file mode 100644 index 000000000..6f57feff5 --- /dev/null +++ b/libpod/network/cni/README.md @@ -0,0 +1,10 @@ +This package abstracts CNI from libpod. +It implements the `ContainerNetwork` interface defined in [libpod/network/types/network.go](../types/network.go) for the CNI backend. + + +## Testing +Run the tests with: +``` +go test -v -mod=vendor -cover ./libpod/network/cni/ +``` +Run the tests as root to also test setup/teardown. This will execute CNI and therefore the cni plugins have to be installed. diff --git a/libpod/network/cni/cni_conversion.go b/libpod/network/cni/cni_conversion.go new file mode 100644 index 000000000..09943948b --- /dev/null +++ b/libpod/network/cni/cni_conversion.go @@ -0,0 +1,375 @@ +// +build linux + +package cni + +import ( + "encoding/json" + "io/ioutil" + "net" + "os" + "path/filepath" + "strconv" + "strings" + "syscall" + "time" + + "github.com/containernetworking/cni/libcni" + "github.com/containernetworking/cni/pkg/version" + "github.com/containers/podman/v3/libpod/network/types" + "github.com/containers/podman/v3/libpod/network/util" + pkgutil "github.com/containers/podman/v3/pkg/util" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +func createNetworkFromCNIConfigList(conf *libcni.NetworkConfigList, confPath string) (*types.Network, error) { + network := types.Network{ + Name: conf.Name, + ID: getNetworkIDFromName(conf.Name), + Labels: map[string]string{}, + Options: map[string]string{}, + IPAMOptions: map[string]string{}, + } + + cniJSON := make(map[string]interface{}) + err := json.Unmarshal(conf.Bytes, &cniJSON) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal network config %s", conf.Name) + } + if args, ok := cniJSON["args"]; ok { + if key, ok := args.(map[string]interface{}); ok { + // read network labels and options from the conf file + network.Labels = getNetworkArgsFromConfList(key, podmanLabelKey) + network.Options = getNetworkArgsFromConfList(key, podmanOptionsKey) + } + } + + f, err := os.Stat(confPath) + if err != nil { + return nil, err + } + stat := f.Sys().(*syscall.Stat_t) + network.Created = time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec)) + + firstPlugin := conf.Plugins[0] + network.Driver = firstPlugin.Network.Type + + switch firstPlugin.Network.Type { + case types.BridgeNetworkDriver: + var bridge hostLocalBridge + err := json.Unmarshal(firstPlugin.Bytes, &bridge) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal the bridge plugin config in %s", confPath) + } + network.NetworkInterface = bridge.BrName + + // if isGateway is false we have an internal network + if !bridge.IsGW { + network.Internal = true + } + + // set network options + if bridge.MTU != 0 { + network.Options["mtu"] = strconv.Itoa(bridge.MTU) + } + if bridge.Vlan != 0 { + network.Options["vlan"] = strconv.Itoa(bridge.Vlan) + } + + err = convertIPAMConfToNetwork(&network, bridge.IPAM, confPath) + if err != nil { + return nil, err + } + + case types.MacVLANNetworkDriver: + var macvlan macVLANConfig + err := json.Unmarshal(firstPlugin.Bytes, &macvlan) + if err != nil { + return nil, errors.Wrapf(err, "failed to unmarshal the macvlan plugin config in %s", confPath) + } + network.NetworkInterface = macvlan.Master + + // set network options + if macvlan.MTU != 0 { + network.Options["mtu"] = strconv.Itoa(macvlan.MTU) + } + + err = convertIPAMConfToNetwork(&network, macvlan.IPAM, confPath) + if err != nil { + return nil, err + } + + default: + // A warning would be good but users would get this warning everytime so keep this at info level. + logrus.Infof("unsupported CNI config type %s in %s, this network can still be used but inspect or list cannot show all information", + firstPlugin.Network.Type, confPath) + } + + // check if the dnsname plugin is configured + network.DNSEnabled = findPluginByName(conf.Plugins, "dnsname") + + return &network, nil +} + +func findPluginByName(plugins []*libcni.NetworkConfig, name string) bool { + for _, plugin := range plugins { + if plugin.Network.Type == name { + return true + } + } + return false +} + +// convertIPAMConfToNetwork converts A cni IPAMConfig to libpod network subnets. +// It returns an array of subnets and an extra bool if dhcp is configured. +func convertIPAMConfToNetwork(network *types.Network, ipam ipamConfig, confPath string) error { + if ipam.PluginType == types.DHCPIPAMDriver { + network.IPAMOptions["driver"] = types.DHCPIPAMDriver + return nil + } + + if ipam.PluginType != types.HostLocalIPAMDriver { + return errors.Errorf("unsupported ipam plugin %s in %s", ipam.PluginType, confPath) + } + + network.IPAMOptions["driver"] = types.HostLocalIPAMDriver + for _, r := range ipam.Ranges { + for _, ipam := range r { + s := types.Subnet{} + + // Do not use types.ParseCIDR() because we want the ip to be + // the network address and not a random ip in the sub. + _, sub, err := net.ParseCIDR(ipam.Subnet) + if err != nil { + return err + } + s.Subnet = types.IPNet{IPNet: *sub} + + // gateway + var gateway net.IP + if ipam.Gateway != "" { + gateway = net.ParseIP(ipam.Gateway) + if gateway == nil { + return errors.Errorf("failed to parse gateway ip %s", ipam.Gateway) + } + // convert to 4 byte if ipv4 + ipv4 := gateway.To4() + if ipv4 != nil { + gateway = ipv4 + } + } else if !network.Internal { + // only add a gateway address if the network is not internal + gateway, err = util.FirstIPInSubnet(sub) + if err != nil { + return errors.Errorf("failed to get first ip in subnet %s", sub.String()) + } + } + s.Gateway = gateway + + var rangeStart net.IP + var rangeEnd net.IP + if ipam.RangeStart != "" { + rangeStart = net.ParseIP(ipam.RangeStart) + if rangeStart == nil { + return errors.Errorf("failed to parse range start ip %s", ipam.RangeStart) + } + } + if ipam.RangeEnd != "" { + rangeEnd = net.ParseIP(ipam.RangeEnd) + if rangeEnd == nil { + return errors.Errorf("failed to parse range end ip %s", ipam.RangeEnd) + } + } + if rangeStart != nil || rangeEnd != nil { + s.LeaseRange = &types.LeaseRange{} + s.LeaseRange.StartIP = rangeStart + s.LeaseRange.EndIP = rangeEnd + } + network.Subnets = append(network.Subnets, s) + } + } + return nil +} + +// getNetworkArgsFromConfList returns the map of args in a conflist, argType should be labels or options +func getNetworkArgsFromConfList(args map[string]interface{}, argType string) map[string]string { + if args, ok := args[argType]; ok { + if labels, ok := args.(map[string]interface{}); ok { + result := make(map[string]string, len(labels)) + for k, v := range labels { + if v, ok := v.(string); ok { + result[k] = v + } + } + return result + } + } + return nil +} + +// createCNIConfigListFromNetwork will create a cni config file from the given network. +// It returns the cni config and the path to the file where the config was written. +// Set writeToDisk to false to only add this network into memory. +func (n *cniNetwork) createCNIConfigListFromNetwork(network *types.Network, writeToDisk bool) (*libcni.NetworkConfigList, string, error) { + var ( + routes []ipamRoute + ipamRanges [][]ipamLocalHostRangeConf + ipamConf ipamConfig + err error + ) + if len(network.Subnets) > 0 { + for _, subnet := range network.Subnets { + route, err := newIPAMDefaultRoute(util.IsIPv6(subnet.Subnet.IP)) + if err != nil { + return nil, "", err + } + routes = append(routes, route) + ipam := newIPAMLocalHostRange(subnet.Subnet, subnet.LeaseRange, subnet.Gateway) + ipamRanges = append(ipamRanges, []ipamLocalHostRangeConf{*ipam}) + } + ipamConf = newIPAMHostLocalConf(routes, ipamRanges) + } else { + ipamConf = ipamConfig{PluginType: "dhcp"} + } + + vlan := 0 + mtu := 0 + for k, v := range network.Options { + switch k { + case "mtu": + mtu, err = parseMTU(v) + if err != nil { + return nil, "", err + } + + case "vlan": + vlan, err = parseVlan(v) + if err != nil { + return nil, "", err + } + + default: + return nil, "", errors.Errorf("unsupported network option %s", k) + } + } + + isGateway := true + ipMasq := true + if network.Internal { + isGateway = false + ipMasq = false + } + // create CNI plugin configuration + ncList := newNcList(network.Name, version.Current(), network.Labels, network.Options) + var plugins []interface{} + + switch network.Driver { + case types.BridgeNetworkDriver: + bridge := newHostLocalBridge(network.NetworkInterface, isGateway, ipMasq, mtu, vlan, ipamConf) + plugins = append(plugins, bridge, newPortMapPlugin(), newFirewallPlugin(), newTuningPlugin()) + // if we find the dnsname plugin we add configuration for it + if hasDNSNamePlugin(n.cniPluginDirs) && network.DNSEnabled { + // Note: in the future we might like to allow for dynamic domain names + plugins = append(plugins, newDNSNamePlugin(defaultPodmanDomainName)) + } + // Add the podman-machine CNI plugin if we are in a machine + if n.isMachine { + plugins = append(plugins, newPodmanMachinePlugin()) + } + + case types.MacVLANNetworkDriver: + plugins = append(plugins, newMacVLANPlugin(network.NetworkInterface, mtu, ipamConf)) + + default: + return nil, "", errors.Errorf("driver %q is not supported by cni", network.Driver) + } + ncList["plugins"] = plugins + b, err := json.MarshalIndent(ncList, "", " ") + if err != nil { + return nil, "", err + } + cniPathName := "" + if writeToDisk { + cniPathName = filepath.Join(n.cniConfigDir, network.Name+".conflist") + err = ioutil.WriteFile(cniPathName, b, 0644) + if err != nil { + return nil, "", err + } + f, err := os.Stat(cniPathName) + if err != nil { + return nil, "", err + } + stat := f.Sys().(*syscall.Stat_t) + network.Created = time.Unix(int64(stat.Ctim.Sec), int64(stat.Ctim.Nsec)) + } else { + network.Created = time.Now() + } + config, err := libcni.ConfListFromBytes(b) + if err != nil { + return nil, "", err + } + return config, cniPathName, nil +} + +// parseMTU parses the mtu option +func parseMTU(mtu string) (int, error) { + if mtu == "" { + return 0, nil // default + } + m, err := strconv.Atoi(mtu) + if err != nil { + return 0, err + } + if m < 0 { + return 0, errors.Errorf("mtu %d is less than zero", m) + } + return m, nil +} + +// parseVlan parses the vlan option +func parseVlan(vlan string) (int, error) { + if vlan == "" { + return 0, nil // default + } + v, err := strconv.Atoi(vlan) + if err != nil { + return 0, err + } + if v < 0 || v > 4094 { + return 0, errors.Errorf("vlan ID %d must be between 0 and 4094", v) + } + return v, nil +} + +func convertSpecgenPortsToCNIPorts(ports []types.PortMapping) ([]cniPortMapEntry, error) { + cniPorts := make([]cniPortMapEntry, 0, len(ports)) + for _, port := range ports { + if port.Protocol == "" { + return nil, errors.New("port protocol should not be empty") + } + protocols := strings.Split(port.Protocol, ",") + + for _, protocol := range protocols { + if !pkgutil.StringInSlice(protocol, []string{"tcp", "udp", "sctp"}) { + return nil, errors.Errorf("unknown port protocol %s", protocol) + } + cniPort := cniPortMapEntry{ + HostPort: int(port.HostPort), + ContainerPort: int(port.ContainerPort), + HostIP: port.HostIP, + Protocol: protocol, + } + cniPorts = append(cniPorts, cniPort) + for i := 1; i < int(port.Range); i++ { + cniPort := cniPortMapEntry{ + HostPort: int(port.HostPort) + i, + ContainerPort: int(port.ContainerPort) + i, + HostIP: port.HostIP, + Protocol: protocol, + } + cniPorts = append(cniPorts, cniPort) + } + } + } + return cniPorts, nil +} diff --git a/libpod/network/cni/cni_exec.go b/libpod/network/cni/cni_exec.go new file mode 100644 index 000000000..0aec3d4f1 --- /dev/null +++ b/libpod/network/cni/cni_exec.go @@ -0,0 +1,98 @@ +// Copyright 2016 CNI authors +// Copyright 2021 Podman authors +// +// This code has been originally copied from github.com/containernetworking/cni +// but has been changed to better fit the Podman use case. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// +build linux + +package cni + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "os/exec" + "path/filepath" + + "github.com/containernetworking/cni/pkg/invoke" + "github.com/containernetworking/cni/pkg/version" +) + +type cniExec struct { + version.PluginDecoder +} + +type cniPluginError struct { + plugin string + Code uint `json:"code"` + Msg string `json:"msg"` + Details string `json:"details,omitempty"` +} + +// Error returns a nicely formatted error message for the cni plugin errors. +func (e *cniPluginError) Error() string { + err := fmt.Sprintf("cni plugin %s failed", e.plugin) + if e.Msg != "" { + err = fmt.Sprintf("%s: %s", err, e.Msg) + } else if e.Code > 0 { + err = fmt.Sprintf("%s with error code %d", err, e.Code) + } + if e.Details != "" { + err = fmt.Sprintf("%s: %s", err, e.Details) + } + return err +} + +// ExecPlugin execute the cni plugin. Returns the stdout of the plugin or an error. +func (e *cniExec) ExecPlugin(ctx context.Context, pluginPath string, stdinData []byte, environ []string) ([]byte, error) { + stdout := &bytes.Buffer{} + stderr := &bytes.Buffer{} + c := exec.CommandContext(ctx, pluginPath) + c.Env = environ + c.Stdin = bytes.NewBuffer(stdinData) + c.Stdout = stdout + c.Stderr = stderr + + err := c.Run() + if err != nil { + return nil, annotatePluginError(err, pluginPath, stdout.Bytes(), stderr.Bytes()) + } + return stdout.Bytes(), nil +} + +// annotatePluginError parses the common cni plugin error json. +func annotatePluginError(err error, plugin string, stdout []byte, stderr []byte) error { + pluginName := filepath.Base(plugin) + emsg := cniPluginError{ + plugin: pluginName, + } + if len(stdout) == 0 { + if len(stderr) == 0 { + emsg.Msg = err.Error() + } else { + emsg.Msg = string(stderr) + } + } else if perr := json.Unmarshal(stdout, &emsg); perr != nil { + emsg.Msg = fmt.Sprintf("failed to unmarshal error message %q: %v", string(stdout), perr) + } + return &emsg +} + +// FindInPath finds the plugin in the given paths. +func (e *cniExec) FindInPath(plugin string, paths []string) (string, error) { + return invoke.FindInPath(plugin, paths) +} diff --git a/libpod/network/cni/cni_suite_test.go b/libpod/network/cni/cni_suite_test.go new file mode 100644 index 000000000..f98869c96 --- /dev/null +++ b/libpod/network/cni/cni_suite_test.go @@ -0,0 +1,53 @@ +// +build linux + +package cni_test + +import ( + "os" + "path/filepath" + "testing" + + "github.com/containers/podman/v3/libpod/network/cni" + "github.com/containers/podman/v3/libpod/network/types" + "github.com/containers/podman/v3/test/utils" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var cniPluginDirs = []string{ + "/usr/libexec/cni", + "/usr/lib/cni", + "/usr/local/lib/cni", + "/opt/cni/bin", +} + +func TestCni(t *testing.T) { + RegisterFailHandler(Fail) + RunSpecs(t, "CNI Suite") +} + +func getNetworkInterface(cniConfDir string, machine bool) (types.ContainerNetwork, error) { + return cni.NewCNINetworkInterface(cni.InitConfig{ + CNIConfigDir: cniConfDir, + CNIPluginDirs: cniPluginDirs, + IsMachine: machine, + LockFile: filepath.Join(cniConfDir, "cni.lock"), + }) +} + +func SkipIfNoDnsname() { + for _, path := range cniPluginDirs { + f, err := os.Stat(filepath.Join(path, "dnsname")) + if err == nil && f.Mode().IsRegular() { + return + } + } + Skip("dnsname cni plugin needs to be installed for this test") +} + +func SkipIfNotFedora(msg string) { + info := utils.GetHostDistributionInfo() + if info.Distribution != "fedora" { + Skip("Test can only run on Fedora: " + msg) + } +} diff --git a/libpod/network/cni/cni_types.go b/libpod/network/cni/cni_types.go new file mode 100644 index 000000000..91fd1c27b --- /dev/null +++ b/libpod/network/cni/cni_types.go @@ -0,0 +1,292 @@ +// +build linux + +package cni + +import ( + "net" + "os" + "path/filepath" + + "github.com/containers/podman/v3/libpod/network/types" +) + +const ( + defaultIPv4Route = "0.0.0.0/0" + defaultIPv6Route = "::/0" + // defaultPodmanDomainName is used for the dnsname plugin to define + // a localized domain name for a created network + defaultPodmanDomainName = "dns.podman" + + // cniDeviceName is the default name for a new bridge, it should be suffixed with an integer + cniDeviceName = "cni-podman" + + // podmanLabelKey key used to store the podman network label in a cni config + podmanLabelKey = "podman_labels" + + // podmanOptionsKey key used to store the podman network options in a cni config + podmanOptionsKey = "podman_options" +) + +// cniPortMapEntry struct is used by the portmap plugin +// https://github.com/containernetworking/plugins/blob/649e0181fe7b3a61e708f3e4249a798f57f25cc5/plugins/meta/portmap/main.go#L43-L50 +type cniPortMapEntry struct { + HostPort int `json:"hostPort"` + ContainerPort int `json:"containerPort"` + Protocol string `json:"protocol"` + HostIP string `json:"hostIP,omitempty"` +} + +// hostLocalBridge describes a configuration for a bridge plugin +// https://github.com/containernetworking/plugins/tree/master/plugins/main/bridge#network-configuration-reference +type hostLocalBridge struct { + PluginType string `json:"type"` + BrName string `json:"bridge,omitempty"` + IsGW bool `json:"isGateway"` + IsDefaultGW bool `json:"isDefaultGateway,omitempty"` + ForceAddress bool `json:"forceAddress,omitempty"` + IPMasq bool `json:"ipMasq,omitempty"` + MTU int `json:"mtu,omitempty"` + HairpinMode bool `json:"hairpinMode,omitempty"` + PromiscMode bool `json:"promiscMode,omitempty"` + Vlan int `json:"vlan,omitempty"` + IPAM ipamConfig `json:"ipam"` + Capabilities map[string]bool `json:"capabilities"` +} + +// ipamConfig describes an IPAM configuration +// https://github.com/containernetworking/plugins/tree/master/plugins/ipam/host-local#network-configuration-reference +type ipamConfig struct { + PluginType string `json:"type"` + Routes []ipamRoute `json:"routes,omitempty"` + ResolveConf string `json:"resolveConf,omitempty"` + DataDir string `json:"dataDir,omitempty"` + Ranges [][]ipamLocalHostRangeConf `json:"ranges,omitempty"` +} + +// ipamLocalHostRangeConf describes the new style IPAM ranges +type ipamLocalHostRangeConf struct { + Subnet string `json:"subnet"` + RangeStart string `json:"rangeStart,omitempty"` + RangeEnd string `json:"rangeEnd,omitempty"` + Gateway string `json:"gateway,omitempty"` +} + +// ipamRoute describes a route in an ipam config +type ipamRoute struct { + Dest string `json:"dst"` +} + +// portMapConfig describes the default portmapping config +type portMapConfig struct { + PluginType string `json:"type"` + Capabilities map[string]bool `json:"capabilities"` +} + +// macVLANConfig describes the macvlan config +type macVLANConfig struct { + PluginType string `json:"type"` + Master string `json:"master"` + IPAM ipamConfig `json:"ipam"` + MTU int `json:"mtu,omitempty"` + Capabilities map[string]bool `json:"capabilities"` +} + +// firewallConfig describes the firewall plugin +type firewallConfig struct { + PluginType string `json:"type"` + Backend string `json:"backend"` +} + +// tuningConfig describes the tuning plugin +type tuningConfig struct { + PluginType string `json:"type"` +} + +// dnsNameConfig describes the dns container name resolution plugin config +type dnsNameConfig struct { + PluginType string `json:"type"` + DomainName string `json:"domainName"` + Capabilities map[string]bool `json:"capabilities"` +} + +// podmanMachineConfig enables port handling on the host OS +type podmanMachineConfig struct { + PluginType string `json:"type"` + Capabilities map[string]bool `json:"capabilities"` +} + +// ncList describes a generic map +type ncList map[string]interface{} + +// newNcList creates a generic map of values with string +// keys and adds in version and network name +func newNcList(name, version string, labels, options map[string]string) ncList { + n := ncList{} + n["cniVersion"] = version + n["name"] = name + args := map[string]map[string]string{} + if len(labels) > 0 { + args[podmanLabelKey] = labels + } + if len(options) > 0 { + args[podmanOptionsKey] = options + } + if len(args) > 0 { + n["args"] = args + } + return n +} + +// newHostLocalBridge creates a new LocalBridge for host-local +func newHostLocalBridge(name string, isGateWay, ipMasq bool, mtu int, vlan int, ipamConf ipamConfig) *hostLocalBridge { + caps := make(map[string]bool) + caps["ips"] = true + bridge := hostLocalBridge{ + PluginType: "bridge", + BrName: name, + IsGW: isGateWay, + IPMasq: ipMasq, + MTU: mtu, + HairpinMode: true, + Vlan: vlan, + IPAM: ipamConf, + } + // if we use host-local set the ips cap to ensure we can set static ips via runtime config + if ipamConf.PluginType == types.HostLocalIPAMDriver { + bridge.Capabilities = caps + } + return &bridge +} + +// newIPAMHostLocalConf creates a new IPAMHostLocal configuration +func newIPAMHostLocalConf(routes []ipamRoute, ipamRanges [][]ipamLocalHostRangeConf) ipamConfig { + ipamConf := ipamConfig{ + PluginType: "host-local", + Routes: routes, + } + + ipamConf.Ranges = ipamRanges + return ipamConf +} + +// newIPAMLocalHostRange create a new IPAM range +func newIPAMLocalHostRange(subnet types.IPNet, leaseRange *types.LeaseRange, gw net.IP) *ipamLocalHostRangeConf { + hostRange := &ipamLocalHostRangeConf{ + Subnet: subnet.String(), + } + + // an user provided a range, we add it here + if leaseRange != nil { + if leaseRange.StartIP != nil { + hostRange.RangeStart = leaseRange.StartIP.String() + } + if leaseRange.EndIP != nil { + hostRange.RangeStart = leaseRange.EndIP.String() + } + } + + if gw != nil { + hostRange.Gateway = gw.String() + } + return hostRange +} + +// newIPAMRoute creates a new IPAM route configuration +// nolint:interfacer +func newIPAMRoute(r *net.IPNet) ipamRoute { + return ipamRoute{Dest: r.String()} +} + +// newIPAMDefaultRoute creates a new IPAMDefault route of +// 0.0.0.0/0 for IPv4 or ::/0 for IPv6 +func newIPAMDefaultRoute(isIPv6 bool) (ipamRoute, error) { + route := defaultIPv4Route + if isIPv6 { + route = defaultIPv6Route + } + _, n, err := net.ParseCIDR(route) + if err != nil { + return ipamRoute{}, err + } + return newIPAMRoute(n), nil +} + +// newPortMapPlugin creates a predefined, default portmapping +// configuration +func newPortMapPlugin() portMapConfig { + caps := make(map[string]bool) + caps["portMappings"] = true + p := portMapConfig{ + PluginType: "portmap", + Capabilities: caps, + } + return p +} + +// newFirewallPlugin creates a generic firewall plugin +func newFirewallPlugin() firewallConfig { + return firewallConfig{ + PluginType: "firewall", + } +} + +// newTuningPlugin creates a generic tuning section +func newTuningPlugin() tuningConfig { + return tuningConfig{ + PluginType: "tuning", + } +} + +// newDNSNamePlugin creates the dnsname config with a given +// domainname +func newDNSNamePlugin(domainName string) dnsNameConfig { + caps := make(map[string]bool, 1) + caps["aliases"] = true + return dnsNameConfig{ + PluginType: "dnsname", + DomainName: domainName, + Capabilities: caps, + } +} + +// hasDNSNamePlugin looks to see if the dnsname cni plugin is present +func hasDNSNamePlugin(paths []string) bool { + for _, p := range paths { + if _, err := os.Stat(filepath.Join(p, "dnsname")); err == nil { + return true + } + } + return false +} + +// newMacVLANPlugin creates a macvlanconfig with a given device name +func newMacVLANPlugin(device string, mtu int, ipam ipamConfig) macVLANConfig { + m := macVLANConfig{ + PluginType: "macvlan", + IPAM: ipam, + } + if mtu > 0 { + m.MTU = mtu + } + // CNI is supposed to use the default route if a + // parent device is not provided + if len(device) > 0 { + m.Master = device + } + caps := make(map[string]bool) + caps["ips"] = true + // if we use host-local set the ips cap to ensure we can set static ips via runtime config + if ipam.PluginType == types.HostLocalIPAMDriver { + m.Capabilities = caps + } + return m +} + +func newPodmanMachinePlugin() podmanMachineConfig { + caps := make(map[string]bool, 1) + caps["portMappings"] = true + return podmanMachineConfig{ + PluginType: "podman-machine", + Capabilities: caps, + } +} diff --git a/libpod/network/cni/config.go b/libpod/network/cni/config.go new file mode 100644 index 000000000..ee203f80d --- /dev/null +++ b/libpod/network/cni/config.go @@ -0,0 +1,313 @@ +// +build linux + +package cni + +import ( + "net" + "os" + + "github.com/containers/podman/v3/libpod/define" + "github.com/containers/podman/v3/libpod/network/types" + "github.com/containers/podman/v3/libpod/network/util" + pkgutil "github.com/containers/podman/v3/pkg/util" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" +) + +// NetworkCreate will take a partial filled Network and fill the +// missing fields. It creates the Network and returns the full Network. +func (n *cniNetwork) NetworkCreate(net types.Network) (types.Network, error) { + n.lock.Lock() + defer n.lock.Unlock() + err := n.loadNetworks() + if err != nil { + return types.Network{}, err + } + network, err := n.networkCreate(net, true) + if err != nil { + return types.Network{}, err + } + // add the new network to the map + n.networks[network.libpodNet.Name] = network + return *network.libpodNet, nil +} + +func (n *cniNetwork) networkCreate(net types.Network, writeToDisk bool) (*network, error) { + // if no driver is set use the default one + if net.Driver == "" { + net.Driver = types.DefaultNetworkDriver + } + + // FIXME: Should we use a different type for network create without the ID field? + // the caller is not allowed to set a specific ID + if net.ID != "" { + return nil, errors.Wrap(define.ErrInvalidArg, "ID can not be set for network create") + } + + if net.Labels == nil { + net.Labels = map[string]string{} + } + if net.Options == nil { + net.Options = map[string]string{} + } + if net.IPAMOptions == nil { + net.IPAMOptions = map[string]string{} + } + + var name string + var err error + // validate the name when given + if net.Name != "" { + if !define.NameRegex.MatchString(net.Name) { + return nil, errors.Wrapf(define.RegexError, "network name %s invalid", net.Name) + } + if _, ok := n.networks[net.Name]; ok { + return nil, errors.Wrapf(define.ErrNetworkExists, "network name %s already used", net.Name) + } + } else { + name, err = n.getFreeDeviceName() + if err != nil { + return nil, err + } + net.Name = name + } + + switch net.Driver { + case types.BridgeNetworkDriver: + // if the name was created with getFreeDeviceName set the interface to it as well + if name != "" && net.NetworkInterface == "" { + net.NetworkInterface = name + } + err = n.createBridge(&net) + if err != nil { + return nil, err + } + case types.MacVLANNetworkDriver: + err = createMacVLAN(&net) + if err != nil { + return nil, err + } + default: + return nil, errors.Wrapf(define.ErrInvalidArg, "unsupported driver %s", net.Driver) + } + + for i := range net.Subnets { + err := validateSubnet(&net.Subnets[i], !net.Internal) + if err != nil { + return nil, err + } + if util.IsIPv6(net.Subnets[i].Subnet.IP) { + net.IPv6Enabled = true + } + } + + // generate the network ID + net.ID = getNetworkIDFromName(net.Name) + + // FIXME: Should this be a hard error? + if net.DNSEnabled && net.Internal && hasDNSNamePlugin(n.cniPluginDirs) { + logrus.Warnf("dnsname and internal networks are incompatible. dnsname plugin not configured for network %s", net.Name) + net.DNSEnabled = false + } + + cniConf, path, err := n.createCNIConfigListFromNetwork(&net, writeToDisk) + if err != nil { + return nil, err + } + return &network{cniNet: cniConf, libpodNet: &net, filename: path}, nil +} + +// NetworkRemove will remove the Network with the given name or ID. +// It does not ensure that the network is unused. +func (n *cniNetwork) NetworkRemove(nameOrID string) error { + n.lock.Lock() + defer n.lock.Unlock() + err := n.loadNetworks() + if err != nil { + return err + } + + network, err := n.getNetwork(nameOrID) + if err != nil { + return err + } + + // Removing the default network is not allowed. + if network.libpodNet.Name == n.defaultNetwork { + return errors.Errorf("default network %s cannot be removed", n.defaultNetwork) + } + + // Remove the bridge network interface on the host. + if network.libpodNet.Driver == types.BridgeNetworkDriver { + link, err := netlink.LinkByName(network.libpodNet.NetworkInterface) + if err == nil { + err = netlink.LinkDel(link) + // only log the error, it is not fatal + if err != nil { + logrus.Infof("failed to remove network interface %s: %v", network.libpodNet.NetworkInterface, err) + } + } + } + + file := network.filename + delete(n.networks, network.libpodNet.Name) + + return os.Remove(file) +} + +// NetworkList will return all known Networks. Optionally you can +// supply a list of filter functions. Only if a network matches all +// functions it is returned. +func (n *cniNetwork) NetworkList(filters ...types.FilterFunc) ([]types.Network, error) { + n.lock.Lock() + defer n.lock.Unlock() + err := n.loadNetworks() + if err != nil { + return nil, err + } + + networks := make([]types.Network, 0, len(n.networks)) +outer: + for _, net := range n.networks { + for _, filter := range filters { + // All filters have to match, if one does not match we can skip to the next network. + if !filter(*net.libpodNet) { + continue outer + } + } + networks = append(networks, *net.libpodNet) + } + return networks, nil +} + +// NetworkInspect will return the Network with the given name or ID. +func (n *cniNetwork) NetworkInspect(nameOrID string) (types.Network, error) { + n.lock.Lock() + defer n.lock.Unlock() + err := n.loadNetworks() + if err != nil { + return types.Network{}, err + } + + network, err := n.getNetwork(nameOrID) + if err != nil { + return types.Network{}, err + } + return *network.libpodNet, nil +} + +func createMacVLAN(network *types.Network) error { + if network.Internal { + return errors.New("internal is not supported with macvlan") + } + if network.NetworkInterface != "" { + interfaceNames, err := util.GetLiveNetworkNames() + if err != nil { + return err + } + if !pkgutil.StringInSlice(network.NetworkInterface, interfaceNames) { + return errors.Errorf("parent interface %s does not exists", network.NetworkInterface) + } + } + if len(network.Subnets) == 0 { + network.IPAMOptions["driver"] = types.DHCPIPAMDriver + } else { + network.IPAMOptions["driver"] = types.HostLocalIPAMDriver + } + return nil +} + +func (n *cniNetwork) createBridge(network *types.Network) error { + if network.NetworkInterface != "" { + bridges := n.getBridgeInterfaceNames() + if pkgutil.StringInSlice(network.NetworkInterface, bridges) { + return errors.Errorf("bridge name %s already in use", network.NetworkInterface) + } + if !define.NameRegex.MatchString(network.NetworkInterface) { + return errors.Wrapf(define.RegexError, "bridge name %s invalid", network.NetworkInterface) + } + } else { + var err error + network.NetworkInterface, err = n.getFreeDeviceName() + if err != nil { + return err + } + } + + if len(network.Subnets) == 0 { + freeSubnet, err := n.getFreeIPv4NetworkSubnet() + if err != nil { + return err + } + network.Subnets = append(network.Subnets, *freeSubnet) + } + // ipv6 enabled means dual stack, check if we already have + // a ipv4 or ipv6 subnet and add one if not. + if network.IPv6Enabled { + ipv4 := false + ipv6 := false + for _, subnet := range network.Subnets { + if util.IsIPv6(subnet.Subnet.IP) { + ipv6 = true + } + if util.IsIPv4(subnet.Subnet.IP) { + ipv4 = true + } + } + if !ipv4 { + freeSubnet, err := n.getFreeIPv4NetworkSubnet() + if err != nil { + return err + } + network.Subnets = append(network.Subnets, *freeSubnet) + } + if !ipv6 { + freeSubnet, err := n.getFreeIPv6NetworkSubnet() + if err != nil { + return err + } + network.Subnets = append(network.Subnets, *freeSubnet) + } + } + network.IPAMOptions["driver"] = types.HostLocalIPAMDriver + return nil +} + +// validateSubnet will validate a given Subnet. It checks if the +// given gateway and lease range are part of this subnet. If the +// gateway is empty and addGateway is true it will get the first +// available ip in the subnet assigned. +func validateSubnet(s *types.Subnet, addGateway bool) error { + if s == nil { + return errors.New("subnet is nil") + } + // Reparse to ensure subnet is valid. + // Do not use types.ParseCIDR() because we want the ip to be + // the network address and not a random ip in the subnet. + _, net, err := net.ParseCIDR(s.Subnet.String()) + if err != nil { + return errors.Wrap(err, "subnet invalid") + } + s.Subnet = types.IPNet{IPNet: *net} + if s.Gateway != nil { + if !s.Subnet.Contains(s.Gateway) { + return errors.Errorf("gateway %s not in subnet %s", s.Gateway, &s.Subnet) + } + } else if addGateway { + ip, err := util.FirstIPInSubnet(net) + if err != nil { + return err + } + s.Gateway = ip + } + if s.LeaseRange != nil { + if s.LeaseRange.StartIP != nil && !s.Subnet.Contains(s.LeaseRange.StartIP) { + return errors.Errorf("lease range start ip %s not in subnet %s", s.LeaseRange.StartIP, &s.Subnet) + } + if s.LeaseRange.EndIP != nil && !s.Subnet.Contains(s.LeaseRange.EndIP) { + return errors.Errorf("lease range end ip %s not in subnet %s", s.LeaseRange.EndIP, &s.Subnet) + } + } + return nil +} diff --git a/libpod/network/cni/config_test.go b/libpod/network/cni/config_test.go new file mode 100644 index 000000000..f67402657 --- /dev/null +++ b/libpod/network/cni/config_test.go @@ -0,0 +1,1241 @@ +// +build linux + +package cni_test + +import ( + "bytes" + "io/ioutil" + "net" + "os" + "path/filepath" + "time" + + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + gomegaTypes "github.com/onsi/gomega/types" + "github.com/sirupsen/logrus" + + "github.com/containers/podman/v3/libpod/network/types" + "github.com/containers/podman/v3/libpod/network/util" +) + +var _ = Describe("Config", func() { + var ( + libpodNet types.ContainerNetwork + cniConfDir string + logBuffer bytes.Buffer + ) + + BeforeEach(func() { + var err error + cniConfDir, err = ioutil.TempDir("", "podman_cni_test") + if err != nil { + Fail("Failed to create tmpdir") + + } + logBuffer = bytes.Buffer{} + logrus.SetOutput(&logBuffer) + }) + + JustBeforeEach(func() { + var err error + libpodNet, err = getNetworkInterface(cniConfDir, false) + if err != nil { + Fail("Failed to create NewCNINetworkInterface") + } + }) + + AfterEach(func() { + os.RemoveAll(cniConfDir) + }) + + Context("basic network config tests", func() { + + It("check default network config exists", func() { + networks, err := libpodNet.NetworkList() + Expect(err).To(BeNil()) + Expect(networks).To(HaveLen(1)) + Expect(networks[0].Name).To(Equal("podman")) + Expect(networks[0].Driver).To(Equal("bridge")) + Expect(networks[0].NetworkInterface).To(Equal("cni-podman0")) + Expect(networks[0].Created.Before(time.Now())).To(BeTrue()) + Expect(networks[0].Subnets).To(HaveLen(1)) + Expect(networks[0].Subnets[0].Subnet.String()).To(Equal("10.88.0.0/16")) + Expect(networks[0].Subnets[0].Gateway.String()).To(Equal("10.88.0.1")) + Expect(networks[0].Subnets[0].LeaseRange).To(BeNil()) + Expect(networks[0].IPAMOptions).To(HaveKeyWithValue("driver", "host-local")) + Expect(networks[0].Options).To(BeEmpty()) + Expect(networks[0].Labels).To(BeEmpty()) + Expect(networks[0].DNSEnabled).To(BeFalse()) + Expect(networks[0].Internal).To(BeFalse()) + }) + + It("basic network create, inspect and remove", func() { + // Because we get the time from the file create timestamp there is small precision + // loss so lets remove 500 milliseconds to make sure this test does not flake. + now := time.Now().Add(-500 * time.Millisecond) + network := types.Network{} + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + path := filepath.Join(cniConfDir, network1.Name+".conflist") + Expect(path).To(BeARegularFile()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Labels).To(BeEmpty()) + Expect(network1.Options).To(BeEmpty()) + Expect(network1.IPAMOptions).ToNot(BeEmpty()) + Expect(network1.IPAMOptions).To(HaveKeyWithValue("driver", "host-local")) + Expect(network1.Created.After(now)).To(BeTrue()) + Expect(network1.Subnets).To(HaveLen(1)) + Expect(network1.Subnets[0].Subnet.String()).To(Equal("10.89.0.0/24")) + Expect(network1.Subnets[0].Gateway.String()).To(Equal("10.89.0.1")) + Expect(network1.Subnets[0].LeaseRange).To(BeNil()) + Expect(network1.DNSEnabled).To(BeFalse()) + Expect(network1.Internal).To(BeFalse()) + + // inspect by name + network2, err := libpodNet.NetworkInspect(network1.Name) + Expect(err).To(BeNil()) + Expect(network2).To(Equal(network1)) + + // inspect by ID + network2, err = libpodNet.NetworkInspect(network1.ID) + Expect(err).To(BeNil()) + Expect(network2).To(Equal(network1)) + + // inspect by partial ID + network2, err = libpodNet.NetworkInspect(network1.ID[:10]) + Expect(err).To(BeNil()) + Expect(network2).To(Equal(network1)) + + // create a new interface to force a config load from disk + libpodNet, err = getNetworkInterface(cniConfDir, false) + Expect(err).To(BeNil()) + + network2, err = libpodNet.NetworkInspect(network1.Name) + Expect(err).To(BeNil()) + Expect(network2).To(Equal(network1)) + + err = libpodNet.NetworkRemove(network1.Name) + Expect(err).To(BeNil()) + Expect(path).ToNot(BeARegularFile()) + + _, err = libpodNet.NetworkInspect(network1.Name) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network not found")) + }) + + It("create two networks", func() { + network := types.Network{} + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(network1.Subnets).To(HaveLen(1)) + + network = types.Network{} + network2, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network2.Name).ToNot(Equal(network1.Name)) + Expect(network2.ID).ToNot(Equal(network1.ID)) + Expect(network2.NetworkInterface).ToNot(Equal(network1.NetworkInterface)) + Expect(network2.Subnets).To(HaveLen(1)) + Expect(network2.Subnets[0].Subnet.Contains(network1.Subnets[0].Subnet.IP)).To(BeFalse()) + }) + + It("create bridge config", func() { + network := types.Network{Driver: "bridge"} + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(filepath.Join(cniConfDir, network1.Name+".conflist")).To(BeARegularFile()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Labels).To(BeEmpty()) + Expect(network1.Options).To(BeEmpty()) + Expect(network1.IPAMOptions).ToNot(BeEmpty()) + Expect(network1.IPAMOptions).To(HaveKeyWithValue("driver", "host-local")) + Expect(network1.Subnets).To(HaveLen(1)) + Expect(network1.Subnets[0].Subnet.String()).To(Equal("10.89.0.0/24")) + Expect(network1.Subnets[0].Gateway.String()).To(Equal("10.89.0.1")) + Expect(network1.Subnets[0].LeaseRange).To(BeNil()) + Expect(network1.DNSEnabled).To(BeFalse()) + Expect(network1.Internal).To(BeFalse()) + }) + + It("create bridge with same name should fail", func() { + network := types.Network{ + Driver: "bridge", + NetworkInterface: "cni-podman2", + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).To(Equal("cni-podman2")) + Expect(network1.Driver).To(Equal("bridge")) + + _, err = libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("bridge name cni-podman2 already in use")) + }) + + It("create macvlan config", func() { + network := types.Network{Driver: "macvlan"} + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(filepath.Join(cniConfDir, network1.Name+".conflist")).To(BeARegularFile()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("macvlan")) + Expect(network1.Labels).To(BeEmpty()) + Expect(network1.Options).To(BeEmpty()) + Expect(network1.IPAMOptions).ToNot(BeEmpty()) + Expect(network1.IPAMOptions).To(HaveKeyWithValue("driver", "dhcp")) + Expect(network1.Subnets).To(HaveLen(0)) + Expect(network1.DNSEnabled).To(BeFalse()) + Expect(network1.Internal).To(BeFalse()) + }) + + It("create macvlan config with device", func() { + network := types.Network{ + Driver: "macvlan", + NetworkInterface: "lo", + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + path := filepath.Join(cniConfDir, network1.Name+".conflist") + Expect(path).To(BeARegularFile()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("macvlan")) + Expect(network1.Labels).To(BeEmpty()) + Expect(network1.Options).To(BeEmpty()) + Expect(network1.Subnets).To(HaveLen(0)) + Expect(network1.DNSEnabled).To(BeFalse()) + Expect(network1.Internal).To(BeFalse()) + Expect(network1.IPAMOptions).To(HaveKeyWithValue("driver", "dhcp")) + grepInFile(path, `"type": "macvlan"`) + grepInFile(path, `"master": "lo"`) + grepInFile(path, `"type": "dhcp"`) + }) + + It("create macvlan config with subnet", func() { + subnet := "10.1.0.0/24" + n, _ := types.ParseCIDR(subnet) + network := types.Network{ + Driver: "macvlan", + Subnets: []types.Subnet{ + {Subnet: n}, + }, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + path := filepath.Join(cniConfDir, network1.Name+".conflist") + Expect(path).To(BeARegularFile()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("macvlan")) + Expect(network1.Labels).To(BeEmpty()) + Expect(network1.Options).To(BeEmpty()) + Expect(network1.Subnets).To(HaveLen(1)) + Expect(network1.Subnets[0].Subnet.String()).To(Equal(subnet)) + Expect(network1.Subnets[0].Gateway.String()).To(Equal("10.1.0.1")) + Expect(network1.Subnets[0].LeaseRange).To(BeNil()) + Expect(network1.DNSEnabled).To(BeFalse()) + Expect(network1.Internal).To(BeFalse()) + Expect(network1.IPAMOptions).To(HaveKeyWithValue("driver", "host-local")) + grepInFile(path, `"type": "host-local"`) + }) + + It("create macvlan config with invalid device", func() { + network := types.Network{ + Driver: "macvlan", + NetworkInterface: "idonotexists", + } + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("parent interface idonotexists does not exists")) + }) + + It("create macvlan config with internal should fail", func() { + network := types.Network{ + Driver: "macvlan", + Internal: true, + } + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("internal is not supported with macvlan")) + }) + + It("create bridge with subnet", func() { + subnet := "10.0.0.0/24" + n, _ := types.ParseCIDR(subnet) + + network := types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: n}, + }, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Subnets).To(HaveLen(1)) + Expect(network1.Subnets[0].Subnet.String()).To(Equal(subnet)) + Expect(network1.Subnets[0].Gateway.String()).To(Equal("10.0.0.1")) + Expect(network1.Subnets[0].LeaseRange).To(BeNil()) + }) + + It("create bridge with ipv6 subnet", func() { + subnet := "fdcc::/64" + n, _ := types.ParseCIDR(subnet) + + network := types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: n}, + }, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.IPv6Enabled).To(BeTrue()) + Expect(network1.Subnets).To(HaveLen(1)) + Expect(network1.Subnets[0].Subnet.String()).To(Equal(subnet)) + Expect(network1.Subnets[0].Gateway.String()).To(Equal("fdcc::1")) + Expect(network1.Subnets[0].LeaseRange).To(BeNil()) + }) + + It("create bridge with ipv6 enabled", func() { + network := types.Network{ + Driver: "bridge", + IPv6Enabled: true, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Subnets).To(HaveLen(2)) + Expect(network1.Subnets[0].Subnet.String()).To(ContainSubstring(".0/24")) + Expect(network1.Subnets[0].Gateway).ToNot(BeNil()) + Expect(network1.Subnets[0].LeaseRange).To(BeNil()) + Expect(network1.Subnets[1].Subnet.String()).To(ContainSubstring("::/64")) + Expect(network1.Subnets[1].Gateway).ToNot(BeNil()) + Expect(network1.Subnets[1].LeaseRange).To(BeNil()) + }) + + It("create bridge with ipv6 enabled and ipv4 subnet", func() { + subnet := "10.100.0.0/24" + n, _ := types.ParseCIDR(subnet) + + network := types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: n}, + }, + IPv6Enabled: true, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Subnets).To(HaveLen(2)) + Expect(network1.Subnets[0].Subnet.String()).To(Equal(subnet)) + Expect(network1.Subnets[0].Gateway).ToNot(BeNil()) + Expect(network1.Subnets[0].LeaseRange).To(BeNil()) + Expect(network1.Subnets[1].Subnet.String()).To(ContainSubstring("::/64")) + Expect(network1.Subnets[1].Gateway).ToNot(BeNil()) + Expect(network1.Subnets[1].LeaseRange).To(BeNil()) + }) + + It("create bridge with ipv6 enabled and ipv6 subnet", func() { + subnet := "fd66::/64" + n, _ := types.ParseCIDR(subnet) + + network := types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: n}, + }, + IPv6Enabled: true, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Subnets).To(HaveLen(2)) + Expect(network1.Subnets[0].Subnet.String()).To(Equal(subnet)) + Expect(network1.Subnets[0].Gateway).ToNot(BeNil()) + Expect(network1.Subnets[0].LeaseRange).To(BeNil()) + Expect(network1.Subnets[1].Subnet.String()).To(ContainSubstring(".0/24")) + Expect(network1.Subnets[1].Gateway).ToNot(BeNil()) + Expect(network1.Subnets[1].LeaseRange).To(BeNil()) + }) + + It("create bridge with ipv6 enabled and ipv4+ipv6 subnet", func() { + subnet1 := "10.100.0.0/24" + n1, _ := types.ParseCIDR(subnet1) + subnet2 := "fd66::/64" + n2, _ := types.ParseCIDR(subnet2) + + network := types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: n1}, {Subnet: n2}, + }, + IPv6Enabled: true, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Subnets).To(HaveLen(2)) + Expect(network1.Subnets[0].Subnet.String()).To(Equal(subnet1)) + Expect(network1.Subnets[0].Gateway).ToNot(BeNil()) + Expect(network1.Subnets[0].LeaseRange).To(BeNil()) + Expect(network1.Subnets[1].Subnet.String()).To(Equal(subnet2)) + Expect(network1.Subnets[1].Gateway).ToNot(BeNil()) + Expect(network1.Subnets[1].LeaseRange).To(BeNil()) + }) + + It("create bridge with ipv6 enabled and two ipv4 subnets", func() { + subnet1 := "10.100.0.0/24" + n1, _ := types.ParseCIDR(subnet1) + subnet2 := "10.200.0.0/24" + n2, _ := types.ParseCIDR(subnet2) + + network := types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: n1}, {Subnet: n2}, + }, + IPv6Enabled: true, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Subnets).To(HaveLen(3)) + Expect(network1.Subnets[0].Subnet.String()).To(Equal(subnet1)) + Expect(network1.Subnets[0].Gateway).ToNot(BeNil()) + Expect(network1.Subnets[0].LeaseRange).To(BeNil()) + Expect(network1.Subnets[1].Subnet.String()).To(Equal(subnet2)) + Expect(network1.Subnets[1].Gateway).ToNot(BeNil()) + Expect(network1.Subnets[1].LeaseRange).To(BeNil()) + Expect(network1.Subnets[2].Subnet.String()).To(ContainSubstring("::/64")) + Expect(network1.Subnets[2].Gateway).ToNot(BeNil()) + Expect(network1.Subnets[2].LeaseRange).To(BeNil()) + }) + + It("create bridge with subnet and gateway", func() { + subnet := "10.0.0.5/24" + n, _ := types.ParseCIDR(subnet) + gateway := "10.0.0.50" + g := net.ParseIP(gateway) + network := types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: n, Gateway: g}, + }, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Subnets).To(HaveLen(1)) + Expect(network1.Subnets[0].Subnet.String()).To(Equal("10.0.0.0/24")) + Expect(network1.Subnets[0].Gateway.String()).To(Equal(gateway)) + Expect(network1.Subnets[0].LeaseRange).To(BeNil()) + }) + + It("create bridge with subnet and gateway not in the same subnet", func() { + subnet := "10.0.0.0/24" + n, _ := types.ParseCIDR(subnet) + gateway := "10.10.0.50" + g := net.ParseIP(gateway) + network := types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: n, Gateway: g}, + }, + } + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not in subnet")) + }) + + It("create bridge with subnet and lease range", func() { + subnet := "10.0.0.0/24" + n, _ := types.ParseCIDR(subnet) + startIP := "10.0.0.10" + network := types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: n, LeaseRange: &types.LeaseRange{ + StartIP: net.ParseIP(startIP), + }}, + }, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Subnets).To(HaveLen(1)) + Expect(network1.Subnets[0].Subnet.String()).To(Equal(subnet)) + Expect(network1.Subnets[0].Gateway.String()).To(Equal("10.0.0.1")) + Expect(network1.Subnets[0].LeaseRange.StartIP.String()).To(Equal(startIP)) + + endIP := "10.0.0.10" + network = types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: n, LeaseRange: &types.LeaseRange{ + EndIP: net.ParseIP(endIP), + }}, + }, + } + network1, err = libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(filepath.Join(cniConfDir, network1.Name+".conflist")).To(BeARegularFile()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Subnets).To(HaveLen(1)) + Expect(network1.Subnets[0].Subnet.String()).To(Equal(subnet)) + Expect(network1.Subnets[0].Gateway.String()).To(Equal("10.0.0.1")) + Expect(network1.Subnets[0].LeaseRange.EndIP.String()).To(Equal(endIP)) + + network = types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: n, LeaseRange: &types.LeaseRange{ + StartIP: net.ParseIP(startIP), + EndIP: net.ParseIP(endIP), + }}, + }, + } + network1, err = libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(BeEmpty()) + Expect(network1.ID).ToNot(BeEmpty()) + Expect(network1.NetworkInterface).ToNot(BeEmpty()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Subnets).To(HaveLen(1)) + Expect(network1.Subnets[0].Subnet.String()).To(Equal(subnet)) + Expect(network1.Subnets[0].Gateway.String()).To(Equal("10.0.0.1")) + Expect(network1.Subnets[0].LeaseRange.StartIP.String()).To(Equal(startIP)) + Expect(network1.Subnets[0].LeaseRange.EndIP.String()).To(Equal(endIP)) + }) + + It("create bridge with subnet and invalid lease range", func() { + subnet := "10.0.0.0/24" + n, _ := types.ParseCIDR(subnet) + startIP := "10.0.1.2" + network := types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: n, LeaseRange: &types.LeaseRange{ + StartIP: net.ParseIP(startIP), + }}, + }, + } + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not in subnet")) + + endIP := "10.1.1.1" + network = types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: n, LeaseRange: &types.LeaseRange{ + EndIP: net.ParseIP(endIP), + }}, + }, + } + _, err = libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("not in subnet")) + }) + + It("create bridge with broken subnet", func() { + network := types.Network{ + Driver: "bridge", + Subnets: []types.Subnet{ + {Subnet: types.IPNet{}}, + }, + } + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("subnet invalid")) + }) + + It("create network with name", func() { + name := "myname" + network := types.Network{ + Name: name, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).To(Equal(name)) + Expect(network1.NetworkInterface).ToNot(Equal(name)) + Expect(network1.Driver).To(Equal("bridge")) + }) + + It("create network with invalid name", func() { + name := "myname@some" + network := types.Network{ + Name: name, + } + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + }) + + It("create network with name", func() { + name := "myname" + network := types.Network{ + Name: name, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).To(Equal(name)) + Expect(network1.NetworkInterface).ToNot(Equal(name)) + Expect(network1.Driver).To(Equal("bridge")) + }) + + It("create network with invalid name", func() { + name := "myname@some" + network := types.Network{ + Name: name, + } + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + }) + + It("create network with interface name", func() { + name := "myname" + network := types.Network{ + NetworkInterface: name, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).ToNot(Equal(name)) + Expect(network1.NetworkInterface).To(Equal(name)) + Expect(network1.Driver).To(Equal("bridge")) + }) + + It("create network with invalid interface name", func() { + name := "myname@some" + network := types.Network{ + NetworkInterface: name, + } + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + }) + + It("create network with ID should fail", func() { + id := "17f29b073143d8cd97b5bbe492bdeffec1c5fee55cc1fe2112c8b9335f8b6121" + network := types.Network{ + ID: id, + } + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("ID can not be set for network create")) + }) + + It("create bridge with dns", func() { + network := types.Network{ + Driver: "bridge", + DNSEnabled: true, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.DNSEnabled).To(BeTrue()) + path := filepath.Join(cniConfDir, network1.Name+".conflist") + Expect(path).To(BeARegularFile()) + grepInFile(path, `"type": "dnsname"`) + }) + + It("create bridge with internal", func() { + network := types.Network{ + Driver: "bridge", + Internal: true, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Subnets).To(HaveLen(1)) + Expect(network1.Subnets[0].Subnet.String()).ToNot(BeEmpty()) + Expect(network1.Subnets[0].Gateway).To(BeNil()) + Expect(network1.Internal).To(BeTrue()) + }) + + It("create network with labels", func() { + network := types.Network{ + Labels: map[string]string{ + "key": "value", + }, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Labels).ToNot(BeNil()) + Expect(network1.Labels).To(ContainElement("value")) + }) + + It("create network with mtu option", func() { + network := types.Network{ + Options: map[string]string{ + "mtu": "1500", + }, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Options).ToNot(BeNil()) + path := filepath.Join(cniConfDir, network1.Name+".conflist") + Expect(path).To(BeARegularFile()) + grepInFile(path, `"mtu": 1500,`) + Expect(network1.Options).To(HaveKeyWithValue("mtu", "1500")) + }) + + It("create network with invalid mtu option", func() { + network := types.Network{ + Options: map[string]string{ + "mtu": "abc", + }, + } + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(`parsing "abc": invalid syntax`)) + + network = types.Network{ + Options: map[string]string{ + "mtu": "-1", + }, + } + _, err = libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(`mtu -1 is less than zero`)) + }) + + It("create macvlan network with mtu option", func() { + network := types.Network{ + Driver: "macvlan", + Options: map[string]string{ + "mtu": "1500", + }, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Driver).To(Equal("macvlan")) + Expect(network1.Options).ToNot(BeNil()) + path := filepath.Join(cniConfDir, network1.Name+".conflist") + Expect(path).To(BeARegularFile()) + grepInFile(path, `"mtu": 1500`) + Expect(network1.Options).To(HaveKeyWithValue("mtu", "1500")) + }) + + It("create network with vlan option", func() { + network := types.Network{ + Options: map[string]string{ + "vlan": "5", + }, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Options).ToNot(BeNil()) + path := filepath.Join(cniConfDir, network1.Name+".conflist") + Expect(path).To(BeARegularFile()) + grepInFile(path, `"vlan": 5,`) + Expect(network1.Options).To(HaveKeyWithValue("vlan", "5")) + }) + + It("create network with invalid vlan option", func() { + network := types.Network{ + Options: map[string]string{ + "vlan": "abc", + }, + } + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(`parsing "abc": invalid syntax`)) + + network = types.Network{ + Options: map[string]string{ + "vlan": "-1", + }, + } + _, err = libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(`vlan ID -1 must be between 0 and 4094`)) + }) + + It("network create unsupported option", func() { + network := types.Network{Options: map[string]string{ + "someopt": "", + }} + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("unsupported network option someopt")) + }) + + It("network create unsupported driver", func() { + network := types.Network{ + Driver: "someDriver", + } + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("unsupported driver someDriver")) + }) + + It("network create internal and dns", func() { + network := types.Network{ + Driver: "bridge", + Internal: true, + DNSEnabled: true, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Driver).To(Equal("bridge")) + Expect(network1.Subnets).To(HaveLen(1)) + Expect(network1.Subnets[0].Subnet.String()).ToNot(BeEmpty()) + Expect(network1.Subnets[0].Gateway).To(BeNil()) + Expect(network1.Internal).To(BeTrue()) + // internal and dns does not work, dns should be disabled + Expect(network1.DNSEnabled).To(BeFalse()) + logString := logBuffer.String() + Expect(logString).To(ContainSubstring("dnsname and internal networks are incompatible")) + }) + + It("create config with podman machine plugin", func() { + libpodNet, err := getNetworkInterface(cniConfDir, true) + Expect(err).To(BeNil()) + + network := types.Network{} + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Driver).To(Equal("bridge")) + path := filepath.Join(cniConfDir, network1.Name+".conflist") + Expect(path).To(BeARegularFile()) + grepInFile(path, `"type": "podman-machine",`) + }) + + It("network inspect partial ID", func() { + network := types.Network{Name: "net4"} + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.ID).To(Equal("b44b7426c006839e7fe6f15d1faf64db58079d5233cba09b43be2257c1652cf5")) + network = types.Network{Name: "net5"} + network1, err = libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.ID).To(Equal("b67e86fb039828ad686aa13667975b9e51f192eb617044faf06cded9d31602af")) + + // Note ID is the sha256 from the name + // both net4 and net5 have an ID starting with b... + _, err = libpodNet.NetworkInspect("b") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("more than one result for network ID")) + }) + + It("network create two with same name", func() { + network := types.Network{Name: "net"} + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + Expect(network1.Name).To(Equal("net")) + network = types.Network{Name: "net"} + _, err = libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network name net already used")) + }) + + It("remove default network config should fail", func() { + err := libpodNet.NetworkRemove("podman") + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("default network podman cannot be removed")) + + network, err := libpodNet.NetworkInspect("podman") + Expect(err).To(BeNil()) + err = libpodNet.NetworkRemove(network.ID) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(Equal("default network podman cannot be removed")) + }) + + }) + + Context("network load valid existing ones", func() { + + BeforeEach(func() { + dir := "testfiles/valid" + files, err := ioutil.ReadDir(dir) + if err != nil { + Fail("Failed to read test directory") + } + for _, file := range files { + filename := file.Name() + data, err := ioutil.ReadFile(filepath.Join(dir, filename)) + if err != nil { + Fail("Failed to copy test files") + } + err = ioutil.WriteFile(filepath.Join(cniConfDir, filename), data, 0700) + if err != nil { + Fail("Failed to copy test files") + } + } + }) + + It("load networks from disk", func() { + nets, err := libpodNet.NetworkList() + Expect(err).To(BeNil()) + Expect(nets).To(HaveLen(9)) + // test the we do not show logrus warnings/errors + logString := logBuffer.String() + Expect(logString).To(BeEmpty()) + }) + + It("change network struct fields should not affect network struct in the backend", func() { + nets, err := libpodNet.NetworkList() + Expect(err).To(BeNil()) + Expect(nets).To(HaveLen(9)) + + nets[0].Name = "myname" + nets, err = libpodNet.NetworkList() + Expect(err).To(BeNil()) + Expect(nets).To(HaveLen(9)) + Expect(nets).ToNot(ContainElement(HaveNetworkName("myname"))) + + network, err := libpodNet.NetworkInspect("bridge") + Expect(err).To(BeNil()) + network.NetworkInterface = "abc" + + network, err = libpodNet.NetworkInspect("bridge") + Expect(err).To(BeNil()) + Expect(network.NetworkInterface).ToNot(Equal("abc")) + }) + + It("bridge network", func() { + network, err := libpodNet.NetworkInspect("bridge") + Expect(err).To(BeNil()) + Expect(network.Name).To(Equal("bridge")) + Expect(network.ID).To(HaveLen(64)) + Expect(network.NetworkInterface).To(Equal("cni-podman9")) + Expect(network.Driver).To(Equal("bridge")) + Expect(network.Subnets).To(HaveLen(1)) + Expect(network.Subnets[0].Subnet.String()).To(Equal("10.89.8.0/24")) + Expect(network.Subnets[0].Gateway.String()).To(Equal("10.89.8.1")) + Expect(network.Subnets[0].LeaseRange).ToNot(BeNil()) + Expect(network.Subnets[0].LeaseRange.StartIP.String()).To(Equal("10.89.8.20")) + Expect(network.Subnets[0].LeaseRange.EndIP.String()).To(Equal("10.89.8.50")) + Expect(network.Internal).To(BeFalse()) + }) + + It("macvlan network", func() { + network, err := libpodNet.NetworkInspect("macvlan") + Expect(err).To(BeNil()) + Expect(network.Name).To(Equal("macvlan")) + Expect(network.ID).To(HaveLen(64)) + Expect(network.NetworkInterface).To(Equal("lo")) + Expect(network.Driver).To(Equal("macvlan")) + Expect(network.Subnets).To(HaveLen(0)) + // DHCP + }) + + It("internal network", func() { + network, err := libpodNet.NetworkInspect("internal") + Expect(err).To(BeNil()) + Expect(network.Name).To(Equal("internal")) + Expect(network.ID).To(HaveLen(64)) + Expect(network.NetworkInterface).To(Equal("cni-podman8")) + Expect(network.Driver).To(Equal("bridge")) + Expect(network.Subnets).To(HaveLen(1)) + Expect(network.Subnets[0].Subnet.String()).To(Equal("10.89.7.0/24")) + Expect(network.Subnets[0].Gateway).To(BeNil()) + Expect(network.Internal).To(BeTrue()) + }) + + It("bridge network with mtu", func() { + network, err := libpodNet.NetworkInspect("mtu") + Expect(err).To(BeNil()) + Expect(network.Name).To(Equal("mtu")) + Expect(network.ID).To(HaveLen(64)) + Expect(network.NetworkInterface).To(Equal("cni-podman13")) + Expect(network.Driver).To(Equal("bridge")) + Expect(network.Subnets).To(HaveLen(1)) + Expect(network.Subnets[0].Subnet.String()).To(Equal("10.89.11.0/24")) + Expect(network.Subnets[0].Gateway.String()).To(Equal("10.89.11.1")) + Expect(network.Internal).To(BeFalse()) + Expect(network.Options).To(HaveLen(1)) + Expect(network.Options).To(HaveKeyWithValue("mtu", "1500")) + }) + + It("macvlan network with mtu", func() { + network, err := libpodNet.NetworkInspect("macvlan_mtu") + Expect(err).To(BeNil()) + Expect(network.Name).To(Equal("macvlan_mtu")) + Expect(network.ID).To(HaveLen(64)) + Expect(network.NetworkInterface).To(Equal("lo")) + Expect(network.Driver).To(Equal("macvlan")) + Expect(network.Subnets).To(HaveLen(0)) + Expect(network.Internal).To(BeFalse()) + Expect(network.Options).To(HaveLen(1)) + Expect(network.Options).To(HaveKeyWithValue("mtu", "1300")) + Expect(network.IPAMOptions).To(HaveLen(1)) + Expect(network.IPAMOptions).To(HaveKeyWithValue("driver", "dhcp")) + }) + + It("bridge network with vlan", func() { + network, err := libpodNet.NetworkInspect("vlan") + Expect(err).To(BeNil()) + Expect(network.Name).To(Equal("vlan")) + Expect(network.ID).To(HaveLen(64)) + Expect(network.NetworkInterface).To(Equal("cni-podman14")) + Expect(network.Driver).To(Equal("bridge")) + Expect(network.Subnets).To(HaveLen(1)) + Expect(network.Options).To(HaveLen(1)) + Expect(network.Options).To(HaveKeyWithValue("vlan", "5")) + }) + + It("bridge network with labels", func() { + network, err := libpodNet.NetworkInspect("label") + Expect(err).To(BeNil()) + Expect(network.Name).To(Equal("label")) + Expect(network.ID).To(HaveLen(64)) + Expect(network.NetworkInterface).To(Equal("cni-podman15")) + Expect(network.Driver).To(Equal("bridge")) + Expect(network.Subnets).To(HaveLen(1)) + Expect(network.Labels).To(HaveLen(1)) + Expect(network.Labels).To(HaveKeyWithValue("mykey", "value")) + }) + + It("dual stack network", func() { + network, err := libpodNet.NetworkInspect("dualstack") + Expect(err).To(BeNil()) + Expect(network.Name).To(Equal("dualstack")) + Expect(network.ID).To(HaveLen(64)) + Expect(network.NetworkInterface).To(Equal("cni-podman21")) + Expect(network.Driver).To(Equal("bridge")) + Expect(network.Subnets).To(HaveLen(2)) + + sub1, _ := types.ParseCIDR("fd10:88:a::/64") + sub2, _ := types.ParseCIDR("10.89.19.0/24") + Expect(network.Subnets).To(ContainElements( + types.Subnet{Subnet: sub1, Gateway: net.ParseIP("fd10:88:a::1")}, + types.Subnet{Subnet: sub2, Gateway: net.ParseIP("10.89.19.10").To4()}, + )) + }) + + It("network list with filters (name)", func() { + filters := map[string][]string{ + "name": {"internal", "bridge"}, + } + filterFuncs, err := util.GenerateNetworkFilters(filters) + Expect(err).To(BeNil()) + + networks, err := libpodNet.NetworkList(filterFuncs...) + Expect(err).To(BeNil()) + Expect(networks).To(HaveLen(2)) + Expect(networks).To(ConsistOf(HaveNetworkName("internal"), HaveNetworkName("bridge"))) + }) + + It("network list with filters (partial name)", func() { + filters := map[string][]string{ + "name": {"inte", "bri"}, + } + filterFuncs, err := util.GenerateNetworkFilters(filters) + Expect(err).To(BeNil()) + + networks, err := libpodNet.NetworkList(filterFuncs...) + Expect(err).To(BeNil()) + Expect(networks).To(HaveLen(2)) + Expect(networks).To(ConsistOf(HaveNetworkName("internal"), HaveNetworkName("bridge"))) + }) + + It("network list with filters (id)", func() { + filters := map[string][]string{ + "id": {"3bed2cb3a3acf7b6a8ef408420cc682d5520e26976d354254f528c965612054f", "17f29b073143d8cd97b5bbe492bdeffec1c5fee55cc1fe2112c8b9335f8b6121"}, + } + filterFuncs, err := util.GenerateNetworkFilters(filters) + Expect(err).To(BeNil()) + + networks, err := libpodNet.NetworkList(filterFuncs...) + Expect(err).To(BeNil()) + Expect(networks).To(HaveLen(2)) + Expect(networks).To(ConsistOf(HaveNetworkName("internal"), HaveNetworkName("bridge"))) + }) + + It("network list with filters (id)", func() { + filters := map[string][]string{ + "id": {"3bed2cb3a3acf7b6a8ef408420cc682d5520e26976d354254f528c965612054f", "17f29b073143d8cd97b5bbe492bdeffec1c5fee55cc1fe2112c8b9335f8b6121"}, + } + filterFuncs, err := util.GenerateNetworkFilters(filters) + Expect(err).To(BeNil()) + + networks, err := libpodNet.NetworkList(filterFuncs...) + Expect(err).To(BeNil()) + Expect(networks).To(HaveLen(2)) + Expect(networks).To(ConsistOf(HaveNetworkName("internal"), HaveNetworkName("bridge"))) + }) + + It("network list with filters (partial id)", func() { + filters := map[string][]string{ + "id": {"3bed2cb3a3acf7b6a8ef408420", "17f29b073143d8cd97b5bbe492bde"}, + } + filterFuncs, err := util.GenerateNetworkFilters(filters) + Expect(err).To(BeNil()) + + networks, err := libpodNet.NetworkList(filterFuncs...) + Expect(err).To(BeNil()) + Expect(networks).To(HaveLen(2)) + Expect(networks).To(ConsistOf(HaveNetworkName("internal"), HaveNetworkName("bridge"))) + }) + + It("network list with filters (driver)", func() { + filters := map[string][]string{ + "driver": {"bridge", "macvlan"}, + } + filterFuncs, err := util.GenerateNetworkFilters(filters) + Expect(err).To(BeNil()) + + networks, err := libpodNet.NetworkList(filterFuncs...) + Expect(err).To(BeNil()) + Expect(networks).To(HaveLen(9)) + Expect(networks).To(ConsistOf(HaveNetworkName("internal"), HaveNetworkName("bridge"), + HaveNetworkName("mtu"), HaveNetworkName("vlan"), HaveNetworkName("podman"), + HaveNetworkName("label"), HaveNetworkName("macvlan"), HaveNetworkName("macvlan_mtu"), HaveNetworkName("dualstack"))) + }) + + It("network list with filters (label)", func() { + filters := map[string][]string{ + "label": {"mykey"}, + } + filterFuncs, err := util.GenerateNetworkFilters(filters) + Expect(err).To(BeNil()) + + networks, err := libpodNet.NetworkList(filterFuncs...) + Expect(err).To(BeNil()) + Expect(networks).To(HaveLen(1)) + Expect(networks).To(ConsistOf(HaveNetworkName("label"))) + + filters = map[string][]string{ + "label": {"mykey=value"}, + } + filterFuncs, err = util.GenerateNetworkFilters(filters) + Expect(err).To(BeNil()) + + networks, err = libpodNet.NetworkList(filterFuncs...) + Expect(err).To(BeNil()) + Expect(networks).To(HaveLen(1)) + Expect(networks).To(ConsistOf(HaveNetworkName("label"))) + }) + + It("network list with filters", func() { + filters := map[string][]string{ + "driver": {"bridge"}, + "label": {"mykey"}, + } + filterFuncs, err := util.GenerateNetworkFilters(filters) + Expect(err).To(BeNil()) + Expect(filterFuncs).To(HaveLen(2)) + + networks, err := libpodNet.NetworkList(filterFuncs...) + Expect(err).To(BeNil()) + Expect(networks).To(HaveLen(1)) + Expect(networks).To(ConsistOf(HaveNetworkName("label"))) + + filters = map[string][]string{ + "driver": {"macvlan"}, + "label": {"mykey"}, + } + filterFuncs, err = util.GenerateNetworkFilters(filters) + Expect(err).To(BeNil()) + + networks, err = libpodNet.NetworkList(filterFuncs...) + Expect(err).To(BeNil()) + Expect(networks).To(HaveLen(0)) + }) + + It("crate bridge network with used interface name", func() { + network := types.Network{ + NetworkInterface: "cni-podman9", + } + _, err := libpodNet.NetworkCreate(network) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("bridge name cni-podman9 already in use")) + }) + }) + + Context("network load invalid existing ones", func() { + + BeforeEach(func() { + dir := "testfiles/invalid" + files, err := ioutil.ReadDir(dir) + if err != nil { + Fail("Failed to read test directory") + } + for _, file := range files { + filename := file.Name() + data, err := ioutil.ReadFile(filepath.Join(dir, filename)) + if err != nil { + Fail("Failed to copy test files") + } + err = ioutil.WriteFile(filepath.Join(cniConfDir, filename), data, 0700) + if err != nil { + Fail("Failed to copy test files") + } + } + }) + + It("load invalid networks from disk", func() { + nets, err := libpodNet.NetworkList() + Expect(err).To(BeNil()) + Expect(nets).To(HaveLen(2)) + logString := logBuffer.String() + Expect(logString).To(ContainSubstring("noname.conflist: error parsing configuration list: no name")) + Expect(logString).To(ContainSubstring("noplugin.conflist: error parsing configuration list: no plugins in list")) + Expect(logString).To(ContainSubstring("invalidname.conflist has invalid name, skipping: names must match")) + Expect(logString).To(ContainSubstring("has the same network name as")) + Expect(logString).To(ContainSubstring("broken.conflist: error parsing configuration list")) + Expect(logString).To(ContainSubstring("invalid_gateway.conflist could not be converted to a libpod config, skipping: failed to parse gateway ip 10.89.8")) + }) + + }) + +}) + +func grepInFile(path string, match string) { + data, err := ioutil.ReadFile(path) + ExpectWithOffset(1, err).To(BeNil()) + ExpectWithOffset(1, string(data)).To(ContainSubstring(match)) +} + +// HaveNetworkName is a custom GomegaMatcher to match a network name +func HaveNetworkName(name string) gomegaTypes.GomegaMatcher { + return WithTransform(func(e types.Network) string { + return e.Name + }, Equal(name)) +} diff --git a/libpod/network/cni/network.go b/libpod/network/cni/network.go new file mode 100644 index 000000000..fde08a0c6 --- /dev/null +++ b/libpod/network/cni/network.go @@ -0,0 +1,340 @@ +// +build linux + +package cni + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "net" + "os" + "strings" + + "github.com/containernetworking/cni/libcni" + "github.com/containers/podman/v3/libpod/define" + "github.com/containers/podman/v3/libpod/network/types" + "github.com/containers/podman/v3/libpod/network/util" + pkgutil "github.com/containers/podman/v3/pkg/util" + "github.com/containers/storage/pkg/lockfile" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +type cniNetwork struct { + // cniConfigDir is directory where the cni config files are stored. + cniConfigDir string + // cniPluginDirs is a list of directories where cni should look for the plugins. + cniPluginDirs []string + + cniConf *libcni.CNIConfig + + // defaultNetwork is the name for the default network. + defaultNetwork string + // defaultSubnet is the default subnet for the default network. + defaultSubnet types.IPNet + + // isMachine describes whenever podman runs in a podman machine environment. + isMachine bool + + // lock is a internal lock for critical operations + lock lockfile.Locker + + // networks is a map with loaded networks, the key is the network name + networks map[string]*network +} + +type network struct { + // filename is the full path to the cni config file on disk + filename string + libpodNet *types.Network + cniNet *libcni.NetworkConfigList +} + +type InitConfig struct { + // CNIConfigDir is directory where the cni config files are stored. + CNIConfigDir string + // CNIPluginDirs is a list of directories where cni should look for the plugins. + CNIPluginDirs []string + + // DefaultNetwork is the name for the default network. + DefaultNetwork string + // DefaultSubnet is the default subnet for the default network. + DefaultSubnet string + + // IsMachine describes whenever podman runs in a podman machine environment. + IsMachine bool + + // LockFile is the path to lock file. + LockFile string +} + +// NewCNINetworkInterface creates the ContainerNetwork interface for the CNI backend. +// Note: The networks are not loaded from disk until a method is called. +func NewCNINetworkInterface(conf InitConfig) (types.ContainerNetwork, error) { + // TODO: consider using a shared memory lock + lock, err := lockfile.GetLockfile(conf.LockFile) + if err != nil { + return nil, err + } + + defaultNetworkName := conf.DefaultNetwork + if defaultNetworkName == "" { + defaultNetworkName = types.DefaultNetworkName + } + + defaultSubnet := conf.DefaultSubnet + if defaultSubnet == "" { + defaultSubnet = types.DefaultSubnet + } + defaultNet, err := types.ParseCIDR(defaultSubnet) + if err != nil { + return nil, errors.Wrap(err, "failed to parse default subnet") + } + + cni := libcni.NewCNIConfig(conf.CNIPluginDirs, &cniExec{}) + n := &cniNetwork{ + cniConfigDir: conf.CNIConfigDir, + cniPluginDirs: conf.CNIPluginDirs, + cniConf: cni, + defaultNetwork: defaultNetworkName, + defaultSubnet: defaultNet, + isMachine: conf.IsMachine, + lock: lock, + } + + return n, nil +} + +func (n *cniNetwork) loadNetworks() error { + // skip loading networks if they are already loaded + if n.networks != nil { + return nil + } + // FIXME: do we have to support other file types as well, e.g. .conf? + files, err := libcni.ConfFiles(n.cniConfigDir, []string{".conflist"}) + if err != nil { + return err + } + networks := make(map[string]*network, len(files)) + for _, file := range files { + conf, err := libcni.ConfListFromFile(file) + if err != nil { + // do not log ENOENT errors + if !os.IsNotExist(err) { + logrus.Warnf("Error loading CNI config file %s: %v", file, err) + } + continue + } + + if !define.NameRegex.MatchString(conf.Name) { + logrus.Warnf("CNI config list %s has invalid name, skipping: %v", file, define.RegexError) + continue + } + + if _, err := n.cniConf.ValidateNetworkList(context.Background(), conf); err != nil { + logrus.Warnf("Error validating CNI config file %s: %v", file, err) + continue + } + + if val, ok := networks[conf.Name]; ok { + logrus.Warnf("CNI config list %s has the same network name as %s, skipping", file, val.filename) + continue + } + + net, err := createNetworkFromCNIConfigList(conf, file) + if err != nil { + logrus.Errorf("CNI config list %s could not be converted to a libpod config, skipping: %v", file, err) + continue + } + logrus.Tracef("Successfully loaded network %s: %v", net.Name, net) + networkInfo := network{ + filename: file, + cniNet: conf, + libpodNet: net, + } + networks[net.Name] = &networkInfo + } + + // create the default network in memory if it did not exists on disk + if networks[n.defaultNetwork] == nil { + networkInfo, err := n.createDefaultNetwork() + if err != nil { + return errors.Wrapf(err, "failed to create default network %s", n.defaultNetwork) + } + networks[n.defaultNetwork] = networkInfo + } + + logrus.Debugf("Successfully loaded %d networks", len(networks)) + n.networks = networks + return nil +} + +func (n *cniNetwork) createDefaultNetwork() (*network, error) { + net := types.Network{ + Name: n.defaultNetwork, + NetworkInterface: "cni-podman0", + Driver: types.BridgeNetworkDriver, + Subnets: []types.Subnet{ + {Subnet: n.defaultSubnet}, + }, + } + return n.networkCreate(net, false) +} + +// getNetwork will lookup a network by name or ID. It returns an +// error when no network was found or when more than one network +// with the given (partial) ID exists. +// getNetwork will read from the networks map, therefore the caller +// must ensure that n.lock is locked before using it. +func (n *cniNetwork) getNetwork(nameOrID string) (*network, error) { + // fast path check the map key, this will only work for names + if val, ok := n.networks[nameOrID]; ok { + return val, nil + } + // If there was no match we might got a full or partial ID. + var net *network + for _, val := range n.networks { + // This should not happen because we already looked up the map by name but check anyway. + if val.libpodNet.Name == nameOrID { + return val, nil + } + + if strings.HasPrefix(val.libpodNet.ID, nameOrID) { + if net != nil { + return nil, errors.Errorf("more than one result for network ID %s", nameOrID) + } + net = val + } + } + if net != nil { + return net, nil + } + return nil, errors.Wrapf(define.ErrNoSuchNetwork, "unable to find network with name or ID %s", nameOrID) +} + +// getNetworkIDFromName creates a network ID from the name. It is just the +// sha256 hash so it is not safe but it should be safe enough for our use case. +func getNetworkIDFromName(name string) string { + hash := sha256.Sum256([]byte(name)) + return hex.EncodeToString(hash[:]) +} + +// getFreeIPv6NetworkSubnet returns a unused ipv4 subnet +func (n *cniNetwork) getFreeIPv4NetworkSubnet() (*types.Subnet, error) { + networks, err := n.getUsedSubnets() + if err != nil { + return nil, err + } + + // the default podman network is 10.88.0.0/16 + // start locking for free /24 networks + network := &net.IPNet{ + IP: net.IP{10, 89, 0, 0}, + Mask: net.IPMask{255, 255, 255, 0}, + } + + // TODO: make sure to not use public subnets + for { + if intersectsConfig := util.NetworkIntersectsWithNetworks(network, networks); !intersectsConfig { + logrus.Debugf("found free ipv4 network subnet %s", network.String()) + return &types.Subnet{ + Subnet: types.IPNet{IPNet: *network}, + }, nil + } + network, err = util.NextSubnet(network) + if err != nil { + return nil, err + } + } +} + +// getFreeIPv6NetworkSubnet returns a unused ipv6 subnet +func (n *cniNetwork) getFreeIPv6NetworkSubnet() (*types.Subnet, error) { + networks, err := n.getUsedSubnets() + if err != nil { + return nil, err + } + + // FIXME: Is 10000 fine as limit? We should prevent an endless loop. + for i := 0; i < 10000; i++ { + // RFC4193: Choose the ipv6 subnet random and NOT sequentially. + network, err := util.GetRandomIPv6Subnet() + if err != nil { + return nil, err + } + if intersectsConfig := util.NetworkIntersectsWithNetworks(&network, networks); !intersectsConfig { + logrus.Debugf("found free ipv6 network subnet %s", network.String()) + return &types.Subnet{ + Subnet: types.IPNet{IPNet: network}, + }, nil + } + } + return nil, errors.New("failed to get random ipv6 subnet") +} + +// getUsedSubnets returns a list of all used subnets by network +// configs and interfaces on the host. +func (n *cniNetwork) getUsedSubnets() ([]*net.IPNet, error) { + // first, load all used subnets from network configs + subnets := make([]*net.IPNet, 0, len(n.networks)) + for _, val := range n.networks { + for _, subnet := range val.libpodNet.Subnets { + // nolint:exportloopref + subnets = append(subnets, &subnet.Subnet.IPNet) + } + } + // second, load networks from the current system + liveSubnets, err := util.GetLiveNetworkSubnets() + if err != nil { + return nil, err + } + return append(subnets, liveSubnets...), nil +} + +// getFreeDeviceName returns a free device name which can +// be used for new configs as name and bridge interface name +func (n *cniNetwork) getFreeDeviceName() (string, error) { + bridgeNames := n.getBridgeInterfaceNames() + netNames := n.getUsedNetworkNames() + liveInterfaces, err := util.GetLiveNetworkNames() + if err != nil { + return "", nil + } + names := make([]string, 0, len(bridgeNames)+len(netNames)+len(liveInterfaces)) + names = append(names, bridgeNames...) + names = append(names, netNames...) + names = append(names, liveInterfaces...) + // FIXME: Is a limit fine? + // Start by 1, 0 is reserved for the default network + for i := 1; i < 1000000; i++ { + deviceName := fmt.Sprintf("%s%d", cniDeviceName, i) + if !pkgutil.StringInSlice(deviceName, names) { + logrus.Debugf("found free device name %s", deviceName) + return deviceName, nil + } + } + return "", errors.New("could not find free device name, to many iterations") +} + +// getUsedNetworkNames returns all network names already used +// by network configs +func (n *cniNetwork) getUsedNetworkNames() []string { + names := make([]string, 0, len(n.networks)) + for _, val := range n.networks { + names = append(names, val.libpodNet.Name) + } + return names +} + +// getUsedNetworkNames returns all bridge device names already used +// by network configs +func (n *cniNetwork) getBridgeInterfaceNames() []string { + names := make([]string, 0, len(n.networks)) + for _, val := range n.networks { + if val.libpodNet.Driver == types.BridgeNetworkDriver { + names = append(names, val.libpodNet.NetworkInterface) + } + } + return names +} diff --git a/libpod/network/cni/run.go b/libpod/network/cni/run.go new file mode 100644 index 000000000..14634262c --- /dev/null +++ b/libpod/network/cni/run.go @@ -0,0 +1,309 @@ +// +build linux + +package cni + +import ( + "context" + "net" + "os" + "strings" + + "github.com/containernetworking/cni/libcni" + cnitypes "github.com/containernetworking/cni/pkg/types" + "github.com/containernetworking/cni/pkg/types/current" + "github.com/containernetworking/plugins/pkg/ns" + "github.com/containers/podman/v3/libpod/define" + "github.com/containers/podman/v3/libpod/network/types" + "github.com/hashicorp/go-multierror" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" +) + +// Setup will setup the container network namespace. It returns +// a map of StatusBlocks, the key is the network name. +func (n *cniNetwork) Setup(namespacePath string, options types.SetupOptions) (map[string]types.StatusBlock, error) { + n.lock.Lock() + defer n.lock.Unlock() + err := n.loadNetworks() + if err != nil { + return nil, err + } + + if namespacePath == "" { + return nil, errors.New("namespacePath is empty") + } + if options.ContainerID == "" { + return nil, errors.New("ContainerID is empty") + } + if len(options.Networks) == 0 { + return nil, errors.New("must specify at least one network") + } + for name, netOpts := range options.Networks { + network := n.networks[name] + if network == nil { + return nil, errors.Wrapf(define.ErrNoSuchNetwork, "network %s", name) + } + err := validatePerNetworkOpts(network, netOpts) + if err != nil { + return nil, err + } + } + + // set the loopback adapter up in the container netns + err = ns.WithNetNSPath(namespacePath, func(_ ns.NetNS) error { + link, err := netlink.LinkByName("lo") + if err == nil { + err = netlink.LinkSetUp(link) + } + return err + }) + if err != nil { + return nil, errors.Wrapf(err, "failed to set the loopback adapter up") + } + + var retErr error + teardownOpts := options + teardownOpts.Networks = map[string]types.PerNetworkOptions{} + // make sure to teardown the already connected networks on error + defer func() { + if retErr != nil { + if len(teardownOpts.Networks) > 0 { + err := n.teardown(namespacePath, types.TeardownOptions(teardownOpts)) + if err != nil { + logrus.Warn(err) + } + } + } + }() + + ports, err := convertSpecgenPortsToCNIPorts(options.PortMappings) + if err != nil { + return nil, err + } + + results := make(map[string]types.StatusBlock, len(options.Networks)) + for name, netOpts := range options.Networks { + network := n.networks[name] + rt := getRuntimeConfig(namespacePath, options.ContainerName, options.ContainerID, name, ports, netOpts) + + // If we have more than one static ip we need parse the ips via runtime config, + // make sure to add the ips capability to the first plugin otherwise it doesn't get the ips + if len(netOpts.StaticIPs) > 0 && !network.cniNet.Plugins[0].Network.Capabilities["ips"] { + caps := make(map[string]interface{}) + caps["capabilities"] = map[string]bool{"ips": true} + network.cniNet.Plugins[0], retErr = libcni.InjectConf(network.cniNet.Plugins[0], caps) + if retErr != nil { + return nil, retErr + } + } + + var res cnitypes.Result + res, retErr = n.cniConf.AddNetworkList(context.Background(), network.cniNet, rt) + // Add this network to teardown opts since it is now connected. + // Also add this if an errors was returned since we want to call teardown on this regardless. + teardownOpts.Networks[name] = netOpts + if retErr != nil { + return nil, retErr + } + + var cnires *current.Result + cnires, retErr = current.GetResult(res) + if retErr != nil { + return nil, retErr + } + logrus.Debugf("cni result for container %s network %s: %v", options.ContainerID, name, cnires) + var status types.StatusBlock + status, retErr = cniResultToStatus(cnires) + if retErr != nil { + return nil, retErr + } + results[name] = status + } + return results, nil +} + +// cniResultToStatus convert the cni result to status block +func cniResultToStatus(cniResult *current.Result) (types.StatusBlock, error) { + result := types.StatusBlock{} + nameservers := make([]net.IP, 0, len(cniResult.DNS.Nameservers)) + for _, nameserver := range cniResult.DNS.Nameservers { + ip := net.ParseIP(nameserver) + if ip == nil { + return result, errors.Errorf("failed to parse cni nameserver ip %s", nameserver) + } + nameservers = append(nameservers, ip) + } + result.DNSServerIPs = nameservers + result.DNSSearchDomains = cniResult.DNS.Search + + interfaces := make(map[string]types.NetInterface) + for _, ip := range cniResult.IPs { + if ip.Interface == nil { + // we do no expect ips without an interface + continue + } + if len(cniResult.Interfaces) <= *ip.Interface { + return result, errors.Errorf("invalid cni result, interface index %d out of range", *ip.Interface) + } + cniInt := cniResult.Interfaces[*ip.Interface] + netInt, ok := interfaces[cniInt.Name] + if ok { + netInt.Networks = append(netInt.Networks, types.NetAddress{ + Subnet: types.IPNet{IPNet: ip.Address}, + Gateway: ip.Gateway, + }) + interfaces[cniInt.Name] = netInt + } else { + mac, err := net.ParseMAC(cniInt.Mac) + if err != nil { + return result, err + } + interfaces[cniInt.Name] = types.NetInterface{ + MacAddress: mac, + Networks: []types.NetAddress{{ + Subnet: types.IPNet{IPNet: ip.Address}, + Gateway: ip.Gateway, + }}, + } + } + } + result.Interfaces = interfaces + return result, nil +} + +// validatePerNetworkOpts checks that all given static ips are in a subnet on this network +func validatePerNetworkOpts(network *network, netOpts types.PerNetworkOptions) error { + if netOpts.InterfaceName == "" { + return errors.Errorf("interface name on network %s is empty", network.libpodNet.Name) + } +outer: + for _, ip := range netOpts.StaticIPs { + for _, s := range network.libpodNet.Subnets { + if s.Subnet.Contains(ip) { + continue outer + } + } + return errors.Errorf("requested static ip %s not in any subnet on network %s", ip.String(), network.libpodNet.Name) + } + if len(netOpts.Aliases) > 0 && !network.libpodNet.DNSEnabled { + return errors.New("cannot set aliases on a network without dns enabled") + } + return nil +} + +func getRuntimeConfig(netns, conName, conID, networkName string, ports []cniPortMapEntry, opts types.PerNetworkOptions) *libcni.RuntimeConf { + rt := &libcni.RuntimeConf{ + ContainerID: conID, + NetNS: netns, + IfName: opts.InterfaceName, + Args: [][2]string{ + {"IgnoreUnknown", "1"}, + // FIXME: Should we set the K8S args? + //{"K8S_POD_NAMESPACE", conName}, + //{"K8S_POD_INFRA_CONTAINER_ID", conID}, + // K8S_POD_NAME is used by dnsname to get the container name + {"K8S_POD_NAME", conName}, + }, + CapabilityArgs: map[string]interface{}{}, + } + + // Propagate environment CNI_ARGS + for _, kvpairs := range strings.Split(os.Getenv("CNI_ARGS"), ";") { + if keyval := strings.SplitN(kvpairs, "=", 2); len(keyval) == 2 { + rt.Args = append(rt.Args, [2]string{keyval[0], keyval[1]}) + } + } + + // Add mac address to cni args + if len(opts.StaticMAC) > 0 { + rt.Args = append(rt.Args, [2]string{"MAC", opts.StaticMAC.String()}) + } + + if len(opts.StaticIPs) == 1 { + // Add a single IP to the args field. CNI plugins < 1.0.0 + // do not support multiple ips via capability args. + rt.Args = append(rt.Args, [2]string{"IP", opts.StaticIPs[0].String()}) + } else if len(opts.StaticIPs) > 1 { + // Set the static ips in the capability args + // to support more than one static ip per network. + rt.CapabilityArgs["ips"] = opts.StaticIPs + } + + // Set network aliases for the dnsname plugin. + if len(opts.Aliases) > 0 { + rt.CapabilityArgs["aliases"] = map[string][]string{ + networkName: opts.Aliases, + } + } + + // Set PortMappings in Capabilities + if len(ports) > 0 { + rt.CapabilityArgs["portMappings"] = ports + } + + return rt +} + +// Teardown will teardown the container network namespace. +func (n *cniNetwork) Teardown(namespacePath string, options types.TeardownOptions) error { + n.lock.Lock() + defer n.lock.Unlock() + err := n.loadNetworks() + if err != nil { + return err + } + return n.teardown(namespacePath, options) +} + +func (n *cniNetwork) teardown(namespacePath string, options types.TeardownOptions) error { + // Note: An empty namespacePath is allowed because some plugins + // still need teardown, for example ipam should remove used ip allocations. + + ports, err := convertSpecgenPortsToCNIPorts(options.PortMappings) + if err != nil { + return err + } + + var multiErr *multierror.Error + for name, netOpts := range options.Networks { + rt := getRuntimeConfig(namespacePath, options.ContainerName, options.ContainerID, name, ports, netOpts) + + cniConfList, newRt, err := getCachedNetworkConfig(n.cniConf, name, rt) + if err == nil { + rt = newRt + } else { + logrus.Warnf("failed to load cached network config: %v, falling back to loading network %s from disk", err, name) + network := n.networks[name] + if network == nil { + multiErr = multierror.Append(multiErr, errors.Wrapf(define.ErrNoSuchNetwork, "network %s", name)) + continue + } + cniConfList = network.cniNet + } + + err = n.cniConf.DelNetworkList(context.Background(), cniConfList, rt) + if err != nil { + multiErr = multierror.Append(multiErr, err) + } + } + return multiErr.ErrorOrNil() +} + +func getCachedNetworkConfig(cniConf *libcni.CNIConfig, name string, rt *libcni.RuntimeConf) (*libcni.NetworkConfigList, *libcni.RuntimeConf, error) { + cniConfList := &libcni.NetworkConfigList{ + Name: name, + } + confBytes, rt, err := cniConf.GetNetworkListCachedConfig(cniConfList, rt) + if err != nil { + return nil, nil, err + } else if confBytes == nil { + return nil, nil, errors.Errorf("network %s not found in CNI cache", name) + } + + cniConfList, err = libcni.ConfListFromBytes(confBytes) + if err != nil { + return nil, nil, err + } + return cniConfList, rt, nil +} diff --git a/libpod/network/cni/run_test.go b/libpod/network/cni/run_test.go new file mode 100644 index 000000000..32e88ca61 --- /dev/null +++ b/libpod/network/cni/run_test.go @@ -0,0 +1,1326 @@ +// +build linux + +package cni_test + +// The tests have to be run as root. +// For each test there will be two network namespaces created, +// netNSTest and netNSContainer. Each test must be run inside +// netNSTest to prevent leakage in the host netns, therefore +// it should use the following structure: +// It("test name", func() { +// runTest(func() { +// // add test logic here +// }) +// }) + +import ( + "bytes" + "io/ioutil" + "net" + "os" + "path/filepath" + "strconv" + "sync" + "time" + + "github.com/containernetworking/plugins/pkg/ns" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + "github.com/sirupsen/logrus" + "github.com/vishvananda/netlink" + "golang.org/x/sys/unix" + + "github.com/containers/podman/v3/libpod/network/types" + "github.com/containers/podman/v3/pkg/netns" + "github.com/containers/podman/v3/pkg/rootless" + "github.com/containers/storage/pkg/stringid" +) + +var _ = Describe("run CNI", func() { + var ( + libpodNet types.ContainerNetwork + cniConfDir string + logBuffer bytes.Buffer + netNSTest ns.NetNS + netNSContainer ns.NetNS + ) + const cniVarDir = "/var/lib/cni" + + // runTest is a helper function to run a test. It ensures that each test + // is run in its own netns. It also creates a mountns to mount a tmpfs to /var/lib/cni. + runTest := func(run func()) { + netNSTest.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + err := os.MkdirAll(cniVarDir, 0755) + Expect(err).To(BeNil(), "Failed to create cniVarDir") + err = unix.Unshare(unix.CLONE_NEWNS) + Expect(err).To(BeNil(), "Failed to create new mountns") + err = unix.Mount("tmpfs", cniVarDir, "tmpfs", unix.MS_NOEXEC|unix.MS_NOSUID|unix.MS_NODEV, "") + Expect(err).To(BeNil(), "Failed to mount tmpfs for cniVarDir") + defer unix.Unmount(cniVarDir, 0) + + // we have to setup the loopback adapter in this netns to use port forwarding + link, err := netlink.LinkByName("lo") + Expect(err).To(BeNil(), "Failed to get loopback adapter") + err = netlink.LinkSetUp(link) + Expect(err).To(BeNil(), "Failed to set loopback adapter up") + run() + return nil + }) + } + + BeforeEach(func() { + // The tests need root privileges. + // Technically we could work around that by using user namespaces and + // the rootless cni code but this is to much work to get it right for a unit test. + if rootless.IsRootless() { + Skip("this test needs to be run as root") + } + + var err error + cniConfDir, err = ioutil.TempDir("", "podman_cni_test") + if err != nil { + Fail("Failed to create tmpdir") + } + logBuffer = bytes.Buffer{} + logrus.SetOutput(&logBuffer) + + netNSTest, err = netns.NewNS() + if err != nil { + Fail("Failed to create netns") + } + + netNSContainer, err = netns.NewNS() + if err != nil { + Fail("Failed to create netns") + } + }) + + JustBeforeEach(func() { + var err error + libpodNet, err = getNetworkInterface(cniConfDir, false) + if err != nil { + Fail("Failed to create NewCNINetworkInterface") + } + }) + + AfterEach(func() { + os.RemoveAll(cniConfDir) + + netns.UnmountNS(netNSTest) + netNSTest.Close() + + netns.UnmountNS(netNSContainer) + netNSContainer.Close() + }) + + Context("network setup test", func() { + + It("run with default config", func() { + runTest(func() { + defNet := types.DefaultNetworkName + intName := "eth0" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + defNet: {InterfaceName: intName}, + }, + }, + } + res, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(BeNil()) + Expect(res).To(HaveLen(1)) + Expect(res).To(HaveKey(defNet)) + Expect(res[defNet].Interfaces).To(HaveKey(intName)) + Expect(res[defNet].Interfaces[intName].Networks).To(HaveLen(1)) + Expect(res[defNet].Interfaces[intName].Networks[0].Subnet.IP.String()).To(ContainSubstring("10.88.0.")) + Expect(res[defNet].Interfaces[intName].MacAddress).To(HaveLen(6)) + // default network has no dns + Expect(res[defNet].DNSServerIPs).To(BeEmpty()) + Expect(res[defNet].DNSSearchDomains).To(BeEmpty()) + + err = libpodNet.Teardown(netNSContainer.Path(), types.TeardownOptions(setupOpts)) + Expect(err).To(BeNil()) + }) + }) + + It("run with default config and static ip", func() { + runTest(func() { + defNet := types.DefaultNetworkName + intName := "eth0" + ip := net.ParseIP("10.88.5.5") + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + defNet: { + InterfaceName: intName, + StaticIPs: []net.IP{ip}, + }, + }, + }, + } + res, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(BeNil()) + Expect(res).To(HaveLen(1)) + Expect(res).To(HaveKey(defNet)) + Expect(res[defNet].Interfaces).To(HaveKey(intName)) + Expect(res[defNet].Interfaces[intName].Networks).To(HaveLen(1)) + Expect(res[defNet].Interfaces[intName].Networks[0].Subnet.IP).To(Equal(ip)) + Expect(res[defNet].Interfaces[intName].MacAddress).To(HaveLen(6)) + // default network has no dns + Expect(res[defNet].DNSServerIPs).To(BeEmpty()) + Expect(res[defNet].DNSSearchDomains).To(BeEmpty()) + + err = libpodNet.Teardown(netNSContainer.Path(), types.TeardownOptions(setupOpts)) + Expect(err).To(BeNil()) + }) + }) + + for _, proto := range []string{"tcp", "udp"} { + // copy proto to extra var to keep correct references in the goroutines + protocol := proto + It("run with exposed ports protocol "+protocol, func() { + runTest(func() { + testdata := stringid.GenerateNonCryptoID() + defNet := types.DefaultNetworkName + intName := "eth0" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + PortMappings: []types.PortMapping{{ + Protocol: protocol, + HostIP: "127.0.0.1", + HostPort: 5000, + ContainerPort: 5000, + }}, + Networks: map[string]types.PerNetworkOptions{ + defNet: {InterfaceName: intName}, + }, + }, + } + res, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(BeNil()) + Expect(res).To(HaveLen(1)) + Expect(res).To(HaveKey(defNet)) + Expect(res[defNet].Interfaces).To(HaveKey(intName)) + Expect(res[defNet].Interfaces[intName].Networks).To(HaveLen(1)) + Expect(res[defNet].Interfaces[intName].Networks[0].Subnet.IP.String()).To(ContainSubstring("10.88.0.")) + Expect(res[defNet].Interfaces[intName].MacAddress).To(HaveLen(6)) + // default network has no dns + Expect(res[defNet].DNSServerIPs).To(BeEmpty()) + Expect(res[defNet].DNSSearchDomains).To(BeEmpty()) + var wg sync.WaitGroup + wg.Add(1) + // start a listener in the container ns + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + runNetListener(&wg, protocol, "0.0.0.0", 5000, testdata) + return nil + }) + Expect(err).To(BeNil()) + + conn, err := net.Dial(protocol, "127.0.0.1:5000") + Expect(err).To(BeNil()) + _, err = conn.Write([]byte(testdata)) + Expect(err).To(BeNil()) + conn.Close() + + // wait for the listener to finish + wg.Wait() + + err = libpodNet.Teardown(netNSContainer.Path(), types.TeardownOptions(setupOpts)) + Expect(err).To(BeNil()) + }) + }) + + It("run with range ports protocol "+protocol, func() { + runTest(func() { + defNet := types.DefaultNetworkName + intName := "eth0" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + PortMappings: []types.PortMapping{{ + Protocol: protocol, + HostIP: "127.0.0.1", + HostPort: 5001, + ContainerPort: 5000, + Range: 3, + }}, + Networks: map[string]types.PerNetworkOptions{ + defNet: {InterfaceName: intName}, + }, + }, + } + res, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(BeNil()) + Expect(res).To(HaveLen(1)) + Expect(res).To(HaveKey(defNet)) + Expect(res[defNet].Interfaces).To(HaveKey(intName)) + Expect(res[defNet].Interfaces[intName].Networks).To(HaveLen(1)) + containerIP := res[defNet].Interfaces[intName].Networks[0].Subnet.IP.String() + Expect(containerIP).To(ContainSubstring("10.88.0.")) + Expect(res[defNet].Interfaces[intName].MacAddress).To(HaveLen(6)) + // default network has no dns + Expect(res[defNet].DNSServerIPs).To(BeEmpty()) + Expect(res[defNet].DNSSearchDomains).To(BeEmpty()) + + // loop over all ports + for p := 5001; p < 5004; p++ { + port := p + var wg sync.WaitGroup + wg.Add(1) + testdata := stringid.GenerateNonCryptoID() + // start a listener in the container ns + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + runNetListener(&wg, protocol, containerIP, port-1, testdata) + return nil + }) + Expect(err).To(BeNil()) + + conn, err := net.Dial(protocol, net.JoinHostPort("127.0.0.1", strconv.Itoa(port))) + Expect(err).To(BeNil()) + _, err = conn.Write([]byte(testdata)) + Expect(err).To(BeNil()) + conn.Close() + + // wait for the listener to finish + wg.Wait() + } + + err = libpodNet.Teardown(netNSContainer.Path(), types.TeardownOptions(setupOpts)) + Expect(err).To(BeNil()) + }) + }) + } + + It("run with comma separated port protocol", func() { + runTest(func() { + defNet := types.DefaultNetworkName + intName := "eth0" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + PortMappings: []types.PortMapping{{ + Protocol: "tcp,udp", + HostIP: "127.0.0.1", + HostPort: 5000, + ContainerPort: 5000, + }}, + Networks: map[string]types.PerNetworkOptions{ + defNet: {InterfaceName: intName}, + }, + }, + } + res, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(BeNil()) + Expect(res).To(HaveLen(1)) + Expect(res).To(HaveKey(defNet)) + Expect(res[defNet].Interfaces).To(HaveKey(intName)) + Expect(res[defNet].Interfaces[intName].Networks).To(HaveLen(1)) + Expect(res[defNet].Interfaces[intName].Networks[0].Subnet.IP.String()).To(ContainSubstring("10.88.0.")) + Expect(res[defNet].Interfaces[intName].MacAddress).To(HaveLen(6)) + + for _, proto := range []string{"tcp", "udp"} { + // copy proto to extra var to keep correct references in the goroutines + protocol := proto + + testdata := stringid.GenerateNonCryptoID() + var wg sync.WaitGroup + wg.Add(1) + // start tcp listener in the container ns + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + runNetListener(&wg, protocol, "0.0.0.0", 5000, testdata) + return nil + }) + Expect(err).To(BeNil()) + + conn, err := net.Dial(protocol, "127.0.0.1:5000") + Expect(err).To(BeNil()) + _, err = conn.Write([]byte(testdata)) + Expect(err).To(BeNil()) + conn.Close() + + // wait for the listener to finish + wg.Wait() + } + + err = libpodNet.Teardown(netNSContainer.Path(), types.TeardownOptions(setupOpts)) + Expect(err).To(BeNil()) + }) + }) + + It("call setup twice", func() { + runTest(func() { + network := types.Network{} + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + + intName1 := "eth0" + netName1 := network1.Name + + containerID := stringid.GenerateNonCryptoID() + + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: containerID, + Networks: map[string]types.PerNetworkOptions{ + netName1: { + InterfaceName: intName1, + }, + }, + }, + } + + res, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(BeNil()) + Expect(res).To(HaveLen(1)) + + Expect(res).To(HaveKey(netName1)) + Expect(res[netName1].Interfaces).To(HaveKey(intName1)) + Expect(res[netName1].Interfaces[intName1].Networks).To(HaveLen(1)) + ipInt1 := res[netName1].Interfaces[intName1].Networks[0].Subnet.IP + Expect(ipInt1).ToNot(BeEmpty()) + macInt1 := res[netName1].Interfaces[intName1].MacAddress + Expect(macInt1).To(HaveLen(6)) + + // check in the container namespace if the settings are applied + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + i, err := net.InterfaceByName(intName1) + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal(intName1)) + Expect(i.HardwareAddr).To(Equal(macInt1)) + addrs, err := i.Addrs() + Expect(err).To(BeNil()) + subnet := &net.IPNet{ + IP: ipInt1, + Mask: net.CIDRMask(24, 32), + } + Expect(addrs).To(ContainElements(subnet)) + + // check loopback adapter + i, err = net.InterfaceByName("lo") + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal("lo")) + Expect(i.Flags & net.FlagLoopback).To(Equal(net.FlagLoopback)) + Expect(i.Flags&net.FlagUp).To(Equal(net.FlagUp), "Loopback adapter should be up") + return nil + }) + Expect(err).To(BeNil()) + + network = types.Network{} + network2, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + + intName2 := "eth1" + netName2 := network2.Name + + setupOpts.Networks = map[string]types.PerNetworkOptions{ + netName2: { + InterfaceName: intName2, + }, + } + + res, err = libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(BeNil()) + Expect(res).To(HaveLen(1)) + + Expect(res).To(HaveKey(netName2)) + Expect(res[netName2].Interfaces).To(HaveKey(intName2)) + Expect(res[netName2].Interfaces[intName2].Networks).To(HaveLen(1)) + ipInt2 := res[netName2].Interfaces[intName2].Networks[0].Subnet.IP + Expect(ipInt2).ToNot(BeEmpty()) + macInt2 := res[netName2].Interfaces[intName2].MacAddress + Expect(macInt2).To(HaveLen(6)) + + // check in the container namespace if the settings are applied + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + i, err := net.InterfaceByName(intName1) + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal(intName1)) + Expect(i.HardwareAddr).To(Equal(macInt1)) + addrs, err := i.Addrs() + Expect(err).To(BeNil()) + subnet := &net.IPNet{ + IP: ipInt1, + Mask: net.CIDRMask(24, 32), + } + Expect(addrs).To(ContainElements(subnet)) + + i, err = net.InterfaceByName(intName2) + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal(intName2)) + Expect(i.HardwareAddr).To(Equal(macInt2)) + addrs, err = i.Addrs() + Expect(err).To(BeNil()) + subnet = &net.IPNet{ + IP: ipInt2, + Mask: net.CIDRMask(24, 32), + } + Expect(addrs).To(ContainElements(subnet)) + + // check loopback adapter + i, err = net.InterfaceByName("lo") + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal("lo")) + Expect(i.Flags & net.FlagLoopback).To(Equal(net.FlagLoopback)) + Expect(i.Flags&net.FlagUp).To(Equal(net.FlagUp), "Loopback adapter should be up") + return nil + }) + Expect(err).To(BeNil()) + + teatdownOpts := types.TeardownOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: containerID, + Networks: map[string]types.PerNetworkOptions{ + netName1: { + InterfaceName: intName1, + }, + netName2: { + InterfaceName: intName2, + }, + }, + }, + } + + err = libpodNet.Teardown(netNSContainer.Path(), teatdownOpts) + Expect(err).To(BeNil()) + logString := logBuffer.String() + Expect(logString).To(BeEmpty()) + + // check in the container namespace that the interface is removed + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + _, err := net.InterfaceByName(intName1) + Expect(err).To(HaveOccurred()) + _, err = net.InterfaceByName(intName2) + Expect(err).To(HaveOccurred()) + + // check that only the loopback adapter is left + ints, err := net.Interfaces() + Expect(err).To(BeNil()) + Expect(ints).To(HaveLen(1)) + Expect(ints[0].Name).To(Equal("lo")) + Expect(ints[0].Flags & net.FlagLoopback).To(Equal(net.FlagLoopback)) + Expect(ints[0].Flags&net.FlagUp).To(Equal(net.FlagUp), "Loopback adapter should be up") + + return nil + }) + Expect(err).To(BeNil()) + + err = libpodNet.NetworkRemove(netName1) + Expect(err).To(BeNil()) + err = libpodNet.NetworkRemove(netName2) + Expect(err).To(BeNil()) + + // check that the interfaces are removed in the host ns + _, err = net.InterfaceByName(network1.NetworkInterface) + Expect(err).To(HaveOccurred()) + _, err = net.InterfaceByName(network2.NetworkInterface) + Expect(err).To(HaveOccurred()) + }) + }) + + It("setup two networks with one setup call", func() { + runTest(func() { + subnet1, _ := types.ParseCIDR("192.168.0.0/24") + subnet2, _ := types.ParseCIDR("192.168.1.0/24") + network := types.Network{ + Subnets: []types.Subnet{ + {Subnet: subnet1}, + }, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + + network = types.Network{ + Subnets: []types.Subnet{ + {Subnet: subnet2}, + }, + } + network2, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + + intName1 := "eth0" + intName2 := "eth1" + netName1 := network1.Name + netName2 := network2.Name + + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + netName1: { + InterfaceName: intName1, + }, + netName2: { + InterfaceName: intName2, + }, + }, + }, + } + + res, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(BeNil()) + Expect(res).To(HaveLen(2)) + + Expect(res).To(HaveKey(netName1)) + Expect(res[netName1].Interfaces).To(HaveKey(intName1)) + Expect(res[netName1].Interfaces[intName1].Networks).To(HaveLen(1)) + ipInt1 := res[netName1].Interfaces[intName1].Networks[0].Subnet.IP + Expect(ipInt1.String()).To(ContainSubstring("192.168.0.")) + macInt1 := res[netName1].Interfaces[intName1].MacAddress + Expect(macInt1).To(HaveLen(6)) + + Expect(res).To(HaveKey(netName2)) + Expect(res[netName2].Interfaces).To(HaveKey(intName2)) + Expect(res[netName2].Interfaces[intName2].Networks).To(HaveLen(1)) + ipInt2 := res[netName2].Interfaces[intName2].Networks[0].Subnet.IP + Expect(ipInt2.String()).To(ContainSubstring("192.168.1.")) + macInt2 := res[netName2].Interfaces[intName2].MacAddress + Expect(macInt2).To(HaveLen(6)) + + // default network has no dns + Expect(res[netName1].DNSServerIPs).To(BeEmpty()) + Expect(res[netName1].DNSSearchDomains).To(BeEmpty()) + + // check in the container namespace if the settings are applied + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + i, err := net.InterfaceByName(intName1) + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal(intName1)) + Expect(i.HardwareAddr).To(Equal(macInt1)) + addrs, err := i.Addrs() + Expect(err).To(BeNil()) + subnet := &net.IPNet{ + IP: ipInt1, + Mask: net.CIDRMask(24, 32), + } + Expect(addrs).To(ContainElements(subnet)) + + i, err = net.InterfaceByName(intName2) + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal(intName2)) + Expect(i.HardwareAddr).To(Equal(macInt2)) + addrs, err = i.Addrs() + Expect(err).To(BeNil()) + subnet = &net.IPNet{ + IP: ipInt2, + Mask: net.CIDRMask(24, 32), + } + Expect(addrs).To(ContainElements(subnet)) + + // check loopback adapter + i, err = net.InterfaceByName("lo") + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal("lo")) + Expect(i.Flags & net.FlagLoopback).To(Equal(net.FlagLoopback)) + Expect(i.Flags&net.FlagUp).To(Equal(net.FlagUp), "Loopback adapter should be up") + return nil + }) + Expect(err).To(BeNil()) + + err = libpodNet.Teardown(netNSContainer.Path(), types.TeardownOptions(setupOpts)) + Expect(err).To(BeNil()) + logString := logBuffer.String() + Expect(logString).To(BeEmpty()) + + // check in the container namespace that the interface is removed + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + _, err := net.InterfaceByName(intName1) + Expect(err).To(HaveOccurred()) + _, err = net.InterfaceByName(intName2) + Expect(err).To(HaveOccurred()) + + // check that only the loopback adapter is left + ints, err := net.Interfaces() + Expect(err).To(BeNil()) + Expect(ints).To(HaveLen(1)) + Expect(ints[0].Name).To(Equal("lo")) + Expect(ints[0].Flags & net.FlagLoopback).To(Equal(net.FlagLoopback)) + Expect(ints[0].Flags&net.FlagUp).To(Equal(net.FlagUp), "Loopback adapter should be up") + + return nil + }) + Expect(err).To(BeNil()) + }) + + }) + + It("dual stack network with static ips", func() { + // Version checks for cni plugins are not possible, the plugins do not output + // version information and using the package manager does not work across distros. + // Fedora has the right version so we use this for now. + SkipIfNotFedora("requires cni plugins 1.0.0 or newer for multiple static ips") + runTest(func() { + subnet1, _ := types.ParseCIDR("192.168.0.0/24") + subnet2, _ := types.ParseCIDR("fd41:0a75:2ca0:48a9::/64") + network := types.Network{ + Subnets: []types.Subnet{ + {Subnet: subnet1}, {Subnet: subnet2}, + }, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + + mac, _ := net.ParseMAC("40:15:2f:d8:42:36") + interfaceName := "eth0" + + ip1 := net.ParseIP("192.168.0.5") + ip2 := net.ParseIP("fd41:0a75:2ca0:48a9::5") + + netName := network1.Name + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerName: "mycon", + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + netName: { + InterfaceName: interfaceName, + StaticIPs: []net.IP{ip1, ip2}, + StaticMAC: mac, + }, + }, + }, + } + + res, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(BeNil()) + Expect(res).To(HaveLen(1)) + Expect(res).To(HaveKey(netName)) + Expect(res[netName].Interfaces).To(HaveKey(interfaceName)) + Expect(res[netName].Interfaces[interfaceName].Networks).To(HaveLen(2)) + Expect(res[netName].Interfaces[interfaceName].Networks[0].Subnet.IP.String()).To(Equal(ip1.String())) + Expect(res[netName].Interfaces[interfaceName].Networks[0].Subnet.Mask).To(Equal(subnet1.Mask)) + Expect(res[netName].Interfaces[interfaceName].Networks[0].Gateway).To(Equal(net.ParseIP("192.168.0.1"))) + Expect(res[netName].Interfaces[interfaceName].Networks[1].Subnet.IP.String()).To(Equal(ip2.String())) + Expect(res[netName].Interfaces[interfaceName].Networks[1].Subnet.Mask).To(Equal(subnet2.Mask)) + Expect(res[netName].Interfaces[interfaceName].Networks[1].Gateway).To(Equal(net.ParseIP("fd41:0a75:2ca0:48a9::1"))) + Expect(res[netName].Interfaces[interfaceName].MacAddress).To(Equal(mac)) + // default network has no dns + Expect(res[netName].DNSServerIPs).To(BeEmpty()) + Expect(res[netName].DNSSearchDomains).To(BeEmpty()) + + // check in the container namespace if the settings are applied + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + i, err := net.InterfaceByName(interfaceName) + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal(interfaceName)) + Expect(i.HardwareAddr).To(Equal(mac)) + addrs, err := i.Addrs() + Expect(err).To(BeNil()) + subnet1 := &net.IPNet{ + IP: ip1, + Mask: net.CIDRMask(24, 32), + } + subnet2 := &net.IPNet{ + IP: ip2, + Mask: net.CIDRMask(64, 128), + } + Expect(addrs).To(ContainElements(subnet1, subnet2)) + + // check loopback adapter + i, err = net.InterfaceByName("lo") + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal("lo")) + Expect(i.Flags & net.FlagLoopback).To(Equal(net.FlagLoopback)) + Expect(i.Flags&net.FlagUp).To(Equal(net.FlagUp), "Loopback adapter should be up") + return nil + }) + Expect(err).To(BeNil()) + + err = libpodNet.Teardown(netNSContainer.Path(), types.TeardownOptions(setupOpts)) + Expect(err).To(BeNil()) + logString := logBuffer.String() + Expect(logString).To(BeEmpty()) + + // check in the container namespace that the interface is removed + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + _, err := net.InterfaceByName(interfaceName) + Expect(err).To(HaveOccurred()) + + // check that only the loopback adapter is left + ints, err := net.Interfaces() + Expect(err).To(BeNil()) + Expect(ints).To(HaveLen(1)) + Expect(ints[0].Name).To(Equal("lo")) + Expect(ints[0].Flags & net.FlagLoopback).To(Equal(net.FlagLoopback)) + Expect(ints[0].Flags&net.FlagUp).To(Equal(net.FlagUp), "Loopback adapter should be up") + + return nil + }) + Expect(err).To(BeNil()) + }) + }) + + It("CNI_ARGS from environment variable", func() { + runTest(func() { + subnet1, _ := types.ParseCIDR("172.16.1.0/24") + ip := "172.16.1.5" + network := types.Network{ + Subnets: []types.Subnet{ + {Subnet: subnet1}, + }, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + netName := network1.Name + intName := "eth0" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + netName: { + InterfaceName: intName, + }, + }, + }, + } + + os.Setenv("CNI_ARGS", "IP="+ip) + defer os.Unsetenv("CNI_ARGS") + + res, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(BeNil()) + Expect(res).To(HaveLen(1)) + Expect(res).To(HaveKey(netName)) + Expect(res[netName].Interfaces).To(HaveKey(intName)) + Expect(res[netName].Interfaces[intName].Networks).To(HaveLen(1)) + Expect(res[netName].Interfaces[intName].Networks[0].Subnet.IP.String()).To(Equal(ip)) + Expect(res[netName].Interfaces[intName].Networks[0].Subnet.Mask).To(Equal(net.CIDRMask(24, 32))) + + // check in the container namespace if the settings are applied + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + i, err := net.InterfaceByName(intName) + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal(intName)) + addrs, err := i.Addrs() + Expect(err).To(BeNil()) + subnet := &net.IPNet{ + IP: net.ParseIP(ip), + Mask: net.CIDRMask(24, 32), + } + Expect(addrs).To(ContainElements(subnet)) + + // check loopback adapter + i, err = net.InterfaceByName("lo") + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal("lo")) + Expect(i.Flags & net.FlagLoopback).To(Equal(net.FlagLoopback)) + Expect(i.Flags&net.FlagUp).To(Equal(net.FlagUp), "Loopback adapter should be up") + return nil + }) + Expect(err).To(BeNil()) + }) + }) + }) + + Context("network setup test with networks from disk", func() { + + BeforeEach(func() { + dir := "testfiles/valid" + files, err := ioutil.ReadDir(dir) + if err != nil { + Fail("Failed to read test directory") + } + for _, file := range files { + filename := file.Name() + data, err := ioutil.ReadFile(filepath.Join(dir, filename)) + if err != nil { + Fail("Failed to copy test files") + } + err = ioutil.WriteFile(filepath.Join(cniConfDir, filename), data, 0700) + if err != nil { + Fail("Failed to copy test files") + } + } + }) + + It("dualstack setup with static ip and dns", func() { + SkipIfNoDnsname() + // Version checks for cni plugins are not possible, the plugins do not output + // version information and using the package manager does not work across distros. + // Fedora has the right version so we use this for now. + SkipIfNotFedora("requires cni plugins 1.0.0 or newer for multiple static ips") + runTest(func() { + interfaceName := "eth0" + + ip1 := net.ParseIP("fd10:88:a::11") + ip2 := net.ParseIP("10.89.19.15") + + containerName := "myname" + aliases := []string{"aliasname"} + + netName := "dualstack" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + ContainerName: containerName, + Networks: map[string]types.PerNetworkOptions{ + netName: { + InterfaceName: interfaceName, + StaticIPs: []net.IP{ip1, ip2}, + Aliases: aliases, + }, + }, + }, + } + + network, err := libpodNet.NetworkInspect(netName) + Expect(err).To(BeNil()) + Expect(network.Name).To(Equal(netName)) + Expect(network.DNSEnabled).To(BeTrue()) + Expect(network.Subnets).To(HaveLen(2)) + gw1 := network.Subnets[0].Gateway + Expect(gw1).To(HaveLen(16)) + mask1 := network.Subnets[0].Subnet.Mask + Expect(mask1).To(HaveLen(16)) + gw2 := network.Subnets[1].Gateway + Expect(gw2).To(HaveLen(4)) + mask2 := network.Subnets[1].Subnet.Mask + Expect(mask2).To(HaveLen(4)) + + // because this net has dns we should always teardown otherwise we leak a dnsmasq process + defer libpodNet.Teardown(netNSContainer.Path(), types.TeardownOptions(setupOpts)) + res, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(BeNil()) + Expect(res).To(HaveLen(1)) + Expect(res).To(HaveKey(netName)) + Expect(res[netName].Interfaces).To(HaveKey(interfaceName)) + Expect(res[netName].Interfaces[interfaceName].Networks).To(HaveLen(2)) + Expect(res[netName].Interfaces[interfaceName].Networks[0].Subnet.IP.String()).To(Equal(ip1.String())) + Expect(res[netName].Interfaces[interfaceName].Networks[0].Subnet.Mask).To(Equal(mask1)) + Expect(res[netName].Interfaces[interfaceName].Networks[1].Subnet.IP.String()).To(Equal(ip2.String())) + Expect(res[netName].Interfaces[interfaceName].Networks[1].Subnet.Mask).To(Equal(mask2)) + // dualstack network dns + Expect(res[netName].DNSServerIPs).To(HaveLen(2)) + Expect(res[netName].DNSSearchDomains).To(HaveLen(1)) + Expect(res[netName].DNSSearchDomains).To(ConsistOf("dns.podman")) + + // check in the container namespace if the settings are applied + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + i, err := net.InterfaceByName(interfaceName) + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal(interfaceName)) + addrs, err := i.Addrs() + Expect(err).To(BeNil()) + subnet1 := &net.IPNet{ + IP: ip1, + Mask: net.CIDRMask(64, 128), + } + subnet2 := &net.IPNet{ + IP: ip2, + Mask: net.CIDRMask(24, 32), + } + Expect(addrs).To(ContainElements(subnet1, subnet2)) + + // check loopback adapter + i, err = net.InterfaceByName("lo") + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal("lo")) + Expect(i.Flags & net.FlagLoopback).To(Equal(net.FlagLoopback)) + Expect(i.Flags&net.FlagUp).To(Equal(net.FlagUp), "Loopback adapter should be up") + + return nil + }) + Expect(err).To(BeNil()) + + err = libpodNet.Teardown(netNSContainer.Path(), types.TeardownOptions(setupOpts)) + Expect(err).To(BeNil()) + logString := logBuffer.String() + Expect(logString).To(BeEmpty()) + + // check in the container namespace that the interface is removed + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + _, err := net.InterfaceByName(interfaceName) + Expect(err).To(HaveOccurred()) + + // check that only the loopback adapter is left + ints, err := net.Interfaces() + Expect(err).To(BeNil()) + Expect(ints).To(HaveLen(1)) + Expect(ints[0].Name).To(Equal("lo")) + Expect(ints[0].Flags & net.FlagLoopback).To(Equal(net.FlagLoopback)) + Expect(ints[0].Flags&net.FlagUp).To(Equal(net.FlagUp), "Loopback adapter should be up") + + return nil + }) + Expect(err).To(BeNil()) + }) + }) + + }) + + Context("invalid network setup test", func() { + + It("static ip not in subnet", func() { + runTest(func() { + defNet := types.DefaultNetworkName + intName := "eth0" + ip := "1.1.1.1" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + defNet: { + InterfaceName: intName, + StaticIPs: []net.IP{net.ParseIP(ip)}, + }, + }, + }, + } + _, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("requested static ip %s not in any subnet on network %s", ip, defNet)) + }) + }) + + It("setup without namespace path", func() { + runTest(func() { + defNet := types.DefaultNetworkName + intName := "eth0" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + defNet: { + InterfaceName: intName, + }, + }, + }, + } + _, err := libpodNet.Setup("", setupOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("namespacePath is empty")) + }) + }) + + It("setup with invalid namespace path", func() { + runTest(func() { + defNet := types.DefaultNetworkName + intName := "eth0" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + defNet: { + InterfaceName: intName, + }, + }, + }, + } + _, err := libpodNet.Setup("some path", setupOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring(`"some path": no such file or directory`)) + }) + }) + + It("setup without container ID", func() { + runTest(func() { + defNet := types.DefaultNetworkName + intName := "eth0" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: "", + Networks: map[string]types.PerNetworkOptions{ + defNet: { + InterfaceName: intName, + }, + }, + }, + } + _, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("ContainerID is empty")) + }) + }) + + It("setup with aliases but dns disabled", func() { + runTest(func() { + defNet := types.DefaultNetworkName + intName := "eth0" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + defNet: { + InterfaceName: intName, + Aliases: []string{"somealias"}, + }, + }, + }, + } + _, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("cannot set aliases on a network without dns enabled")) + }) + }) + + It("setup without networks", func() { + runTest(func() { + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + }, + } + _, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("must specify at least one network")) + }) + }) + + It("setup without interface name", func() { + runTest(func() { + defNet := types.DefaultNetworkName + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + defNet: { + InterfaceName: "", + }, + }, + }, + } + _, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("interface name on network %s is empty", defNet)) + }) + }) + + It("setup does teardown on failure", func() { + runTest(func() { + subnet1, _ := types.ParseCIDR("192.168.0.0/24") + network := types.Network{ + Subnets: []types.Subnet{ + {Subnet: subnet1}, + }, + } + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + + subnet2, _ := types.ParseCIDR("192.168.1.0/31") + network = types.Network{ + Subnets: []types.Subnet{ + {Subnet: subnet2}, + }, + } + network2, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + + intName1 := "eth0" + intName2 := "eth1" + netName1 := network1.Name + netName2 := network2.Name + + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + netName1: { + InterfaceName: intName1, + }, + netName2: { + InterfaceName: intName2, + }, + }, + }, + } + _, err = libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("Network 192.168.1.0/31 too small to allocate from")) + // Note: we call teardown on the failing net and log the error, it should be the same. + logString := logBuffer.String() + Expect(logString).To(ContainSubstring("Network 192.168.1.0/31 too small to allocate from")) + + // check in the container namespace that no interface is there + err = netNSContainer.Do(func(_ ns.NetNS) error { + defer GinkgoRecover() + _, err := net.InterfaceByName(intName1) + Expect(err).To(HaveOccurred()) + + // Note: We can check if intName2 is removed because + // the cni plugin fails before it removes the interface + + // check loopback adapter + i, err := net.InterfaceByName("lo") + Expect(err).To(BeNil()) + Expect(i.Name).To(Equal("lo")) + Expect(i.Flags & net.FlagLoopback).To(Equal(net.FlagLoopback)) + Expect(i.Flags&net.FlagUp).To(Equal(net.FlagUp), "Loopback adapter should be up") + return nil + }) + Expect(err).To(BeNil()) + }) + }) + + It("setup with exposed invalid port protocol", func() { + runTest(func() { + defNet := types.DefaultNetworkName + intName := "eth0" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + PortMappings: []types.PortMapping{{ + Protocol: "someproto", + HostIP: "127.0.0.1", + HostPort: 5000, + ContainerPort: 5000, + }}, + Networks: map[string]types.PerNetworkOptions{ + defNet: {InterfaceName: intName}, + }, + }, + } + _, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("unknown port protocol someproto")) + }) + }) + + It("setup with exposed empty port protocol", func() { + runTest(func() { + defNet := types.DefaultNetworkName + intName := "eth0" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + PortMappings: []types.PortMapping{{ + Protocol: "", + HostIP: "127.0.0.1", + HostPort: 5000, + ContainerPort: 5000, + }}, + Networks: map[string]types.PerNetworkOptions{ + defNet: {InterfaceName: intName}, + }, + }, + } + _, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("port protocol should not be empty")) + }) + }) + + It("setup with unknown network", func() { + runTest(func() { + defNet := "somenet" + intName := "eth0" + setupOpts := types.SetupOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + defNet: {InterfaceName: intName}, + }, + }, + } + _, err := libpodNet.Setup(netNSContainer.Path(), setupOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network somenet: network not found")) + }) + }) + + It("teardown with unknown network", func() { + runTest(func() { + interfaceName := "eth0" + netName := "somenet" + teardownOpts := types.TeardownOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + netName: { + InterfaceName: interfaceName, + }, + }, + }, + } + + err := libpodNet.Teardown(netNSContainer.Path(), teardownOpts) + Expect(err).To(HaveOccurred()) + Expect(err.Error()).To(ContainSubstring("network somenet: network not found")) + logString := logBuffer.String() + Expect(logString).To(ContainSubstring("failed to load cached network config")) + }) + }) + + It("teardown on not connected network", func() { + runTest(func() { + network := types.Network{} + network1, err := libpodNet.NetworkCreate(network) + Expect(err).To(BeNil()) + + interfaceName := "eth0" + netName := network1.Name + teardownOpts := types.TeardownOptions{ + NetworkOptions: types.NetworkOptions{ + ContainerID: stringid.GenerateNonCryptoID(), + Networks: map[string]types.PerNetworkOptions{ + netName: { + InterfaceName: interfaceName, + }, + }, + }, + } + + // Most CNI plugins do not error on teardown when there is nothing to do. + err = libpodNet.Teardown(netNSContainer.Path(), teardownOpts) + Expect(err).To(BeNil()) + logString := logBuffer.String() + Expect(logString).To(ContainSubstring("failed to load cached network config")) + }) + }) + }) +}) + +func runNetListener(wg *sync.WaitGroup, protocol, ip string, port int, expectedData string) { + switch protocol { + case "tcp": + ln, err := net.Listen(protocol, net.JoinHostPort(ip, strconv.Itoa(port))) + Expect(err).To(BeNil()) + // make sure to read in a separate goroutine to not block + go func() { + defer GinkgoRecover() + defer wg.Done() + conn, err := ln.Accept() + Expect(err).To(BeNil()) + conn.SetDeadline(time.Now().Add(1 * time.Second)) + data, err := ioutil.ReadAll(conn) + Expect(err).To(BeNil()) + Expect(string(data)).To(Equal(expectedData)) + conn.Close() + ln.Close() + }() + case "udp": + conn, err := net.ListenUDP("udp", &net.UDPAddr{ + IP: net.ParseIP(ip), + Port: port, + }) + Expect(err).To(BeNil()) + conn.SetDeadline(time.Now().Add(1 * time.Second)) + go func() { + defer GinkgoRecover() + defer wg.Done() + data := make([]byte, len(expectedData)) + i, err := conn.Read(data) + Expect(err).To(BeNil()) + Expect(i).To(Equal(len(expectedData))) + Expect(string(data)).To(Equal(expectedData)) + conn.Close() + }() + default: + Fail("unsupported protocol") + } +} diff --git a/libpod/network/cni/testfiles/invalid/broken.conflist b/libpod/network/cni/testfiles/invalid/broken.conflist new file mode 100644 index 000000000..e5bf48b39 --- /dev/null +++ b/libpod/network/cni/testfiles/invalid/broken.conflist @@ -0,0 +1,25 @@ +{ + "cniVersion": "0.4.0", + "name": "bridge", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman9", + "isGateway": true, + "ipMasq": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "10.89.8.0/24", + "gateway": "10.89.8.1" + } + ] + ] diff --git a/libpod/network/cni/testfiles/invalid/invalid_gateway.conflist b/libpod/network/cni/testfiles/invalid/invalid_gateway.conflist new file mode 100644 index 000000000..f03c1fde4 --- /dev/null +++ b/libpod/network/cni/testfiles/invalid/invalid_gateway.conflist @@ -0,0 +1,51 @@ +{ + "cniVersion": "0.4.0", + "name": "invalidgw", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman8", + "isGateway": true, + "ipMasq": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "10.89.8.0/24", + "gateway": "10.89.8", + "rangeStart": "10.89.8.20", + "rangeEnd": "10.89.8.50" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall", + "backend": "" + }, + { + "type": "tuning" + }, + { + "type": "dnsname", + "domainName": "dns.podman", + "capabilities": { + "aliases": true + } + } + ] +} diff --git a/libpod/network/cni/testfiles/invalid/invalidname.conflist b/libpod/network/cni/testfiles/invalid/invalidname.conflist new file mode 100644 index 000000000..e35be69db --- /dev/null +++ b/libpod/network/cni/testfiles/invalid/invalidname.conflist @@ -0,0 +1,49 @@ +{ + "cniVersion": "0.4.0", + "name": "bridge@123", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman9", + "isGateway": true, + "ipMasq": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "10.89.8.0/24", + "gateway": "10.89.8.1" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall", + "backend": "" + }, + { + "type": "tuning" + }, + { + "type": "dnsname", + "domainName": "dns.podman", + "capabilities": { + "aliases": true + } + } + ] +} diff --git a/libpod/network/cni/testfiles/invalid/noname.conflist b/libpod/network/cni/testfiles/invalid/noname.conflist new file mode 100644 index 000000000..865abadf8 --- /dev/null +++ b/libpod/network/cni/testfiles/invalid/noname.conflist @@ -0,0 +1,48 @@ +{ + "cniVersion": "0.4.0", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman9", + "isGateway": true, + "ipMasq": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "10.89.8.0/24", + "gateway": "10.89.8.1" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall", + "backend": "" + }, + { + "type": "tuning" + }, + { + "type": "dnsname", + "domainName": "dns.podman", + "capabilities": { + "aliases": true + } + } + ] +} diff --git a/libpod/network/cni/testfiles/invalid/noplugin.conflist b/libpod/network/cni/testfiles/invalid/noplugin.conflist new file mode 100644 index 000000000..af192adca --- /dev/null +++ b/libpod/network/cni/testfiles/invalid/noplugin.conflist @@ -0,0 +1,5 @@ +{ + "cniVersion": "0.4.0", + "name": "bridge", + "plugins": [] +} diff --git a/libpod/network/cni/testfiles/invalid/samename1.conflist b/libpod/network/cni/testfiles/invalid/samename1.conflist new file mode 100644 index 000000000..57b325264 --- /dev/null +++ b/libpod/network/cni/testfiles/invalid/samename1.conflist @@ -0,0 +1,49 @@ +{ + "cniVersion": "0.4.0", + "name": "bridge", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman9", + "isGateway": true, + "ipMasq": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "10.89.8.0/24", + "gateway": "10.89.8.1" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall", + "backend": "" + }, + { + "type": "tuning" + }, + { + "type": "dnsname", + "domainName": "dns.podman", + "capabilities": { + "aliases": true + } + } + ] +} diff --git a/libpod/network/cni/testfiles/invalid/samename2.conflist b/libpod/network/cni/testfiles/invalid/samename2.conflist new file mode 100644 index 000000000..57b325264 --- /dev/null +++ b/libpod/network/cni/testfiles/invalid/samename2.conflist @@ -0,0 +1,49 @@ +{ + "cniVersion": "0.4.0", + "name": "bridge", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman9", + "isGateway": true, + "ipMasq": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "10.89.8.0/24", + "gateway": "10.89.8.1" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall", + "backend": "" + }, + { + "type": "tuning" + }, + { + "type": "dnsname", + "domainName": "dns.podman", + "capabilities": { + "aliases": true + } + } + ] +} diff --git a/libpod/network/cni/testfiles/valid/87-podman.conflist b/libpod/network/cni/testfiles/valid/87-podman.conflist new file mode 100644 index 000000000..ef760a61b --- /dev/null +++ b/libpod/network/cni/testfiles/valid/87-podman.conflist @@ -0,0 +1,37 @@ +{ + "cniVersion": "0.4.0", + "name": "podman", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman0", + "isGateway": true, + "ipMasq": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "routes": [{ "dst": "0.0.0.0/0" }], + "ranges": [ + [ + { + "subnet": "10.88.0.0/16", + "gateway": "10.88.0.1" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall" + }, + { + "type": "tuning" + } + ] +} diff --git a/libpod/network/cni/testfiles/valid/bridge.conflist b/libpod/network/cni/testfiles/valid/bridge.conflist new file mode 100644 index 000000000..8952b50b7 --- /dev/null +++ b/libpod/network/cni/testfiles/valid/bridge.conflist @@ -0,0 +1,51 @@ +{ + "cniVersion": "0.4.0", + "name": "bridge", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman9", + "isGateway": true, + "ipMasq": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "10.89.8.0/24", + "gateway": "10.89.8.1", + "rangeStart": "10.89.8.20", + "rangeEnd": "10.89.8.50" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall", + "backend": "" + }, + { + "type": "tuning" + }, + { + "type": "dnsname", + "domainName": "dns.podman", + "capabilities": { + "aliases": true + } + } + ] +} diff --git a/libpod/network/cni/testfiles/valid/dualstack.conflist b/libpod/network/cni/testfiles/valid/dualstack.conflist new file mode 100644 index 000000000..dd08382f0 --- /dev/null +++ b/libpod/network/cni/testfiles/valid/dualstack.conflist @@ -0,0 +1,58 @@ +{ + "cniVersion": "0.4.0", + "name": "dualstack", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman21", + "isGateway": true, + "ipMasq": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "::/0" + }, + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "fd10:88:a::/64", + "gateway": "fd10:88:a::1" + } + ], + [ + { + "subnet": "10.89.19.0/24", + "gateway": "10.89.19.10" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall", + "backend": "" + }, + { + "type": "tuning" + }, + { + "type": "dnsname", + "domainName": "dns.podman", + "capabilities": { + "aliases": true + } + } + ] +} diff --git a/libpod/network/cni/testfiles/valid/internal.conflist b/libpod/network/cni/testfiles/valid/internal.conflist new file mode 100644 index 000000000..1b6f15a96 --- /dev/null +++ b/libpod/network/cni/testfiles/valid/internal.conflist @@ -0,0 +1,40 @@ +{ + "cniVersion": "0.4.0", + "name": "internal", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman8", + "isGateway": false, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "10.89.7.0/24" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall", + "backend": "" + }, + { + "type": "tuning" + } + ] +} diff --git a/libpod/network/cni/testfiles/valid/label.conflist b/libpod/network/cni/testfiles/valid/label.conflist new file mode 100644 index 000000000..1501f9bd7 --- /dev/null +++ b/libpod/network/cni/testfiles/valid/label.conflist @@ -0,0 +1,54 @@ +{ + "args": { + "podman_labels": { + "mykey": "value" + } + }, + "cniVersion": "0.4.0", + "name": "label", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman15", + "isGateway": true, + "ipMasq": true, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "10.89.13.0/24", + "gateway": "10.89.13.1" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall", + "backend": "" + }, + { + "type": "tuning" + }, + { + "type": "dnsname", + "domainName": "dns.podman", + "capabilities": { + "aliases": true + } + } + ] +} diff --git a/libpod/network/cni/testfiles/valid/macvlan.conflist b/libpod/network/cni/testfiles/valid/macvlan.conflist new file mode 100644 index 000000000..8f3692334 --- /dev/null +++ b/libpod/network/cni/testfiles/valid/macvlan.conflist @@ -0,0 +1,13 @@ +{ + "cniVersion": "0.4.0", + "name": "macvlan", + "plugins": [ + { + "type": "macvlan", + "master": "lo", + "ipam": { + "type": "dhcp" + } + } + ] +} diff --git a/libpod/network/cni/testfiles/valid/macvlan_mtu.conflist b/libpod/network/cni/testfiles/valid/macvlan_mtu.conflist new file mode 100644 index 000000000..2fd259117 --- /dev/null +++ b/libpod/network/cni/testfiles/valid/macvlan_mtu.conflist @@ -0,0 +1,14 @@ +{ + "cniVersion": "0.4.0", + "name": "macvlan_mtu", + "plugins": [ + { + "type": "macvlan", + "master": "lo", + "ipam": { + "type": "dhcp" + }, + "mtu": 1300 + } + ] +} diff --git a/libpod/network/cni/testfiles/valid/mtu.conflist b/libpod/network/cni/testfiles/valid/mtu.conflist new file mode 100644 index 000000000..db5f7e194 --- /dev/null +++ b/libpod/network/cni/testfiles/valid/mtu.conflist @@ -0,0 +1,49 @@ +{ + "cniVersion": "0.4.0", + "name": "mtu", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman13", + "isGateway": true, + "ipMasq": true, + "mtu": 1500, + "hairpinMode": true, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "10.89.11.0/24" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall", + "backend": "" + }, + { + "type": "tuning" + }, + { + "type": "dnsname", + "domainName": "dns.podman", + "capabilities": { + "aliases": true + } + } + ] +} diff --git a/libpod/network/cni/testfiles/valid/vlan.conflist b/libpod/network/cni/testfiles/valid/vlan.conflist new file mode 100644 index 000000000..75e8967f1 --- /dev/null +++ b/libpod/network/cni/testfiles/valid/vlan.conflist @@ -0,0 +1,50 @@ +{ + "cniVersion": "0.4.0", + "name": "vlan", + "plugins": [ + { + "type": "bridge", + "bridge": "cni-podman14", + "isGateway": true, + "ipMasq": true, + "hairpinMode": true, + "vlan": 5, + "ipam": { + "type": "host-local", + "routes": [ + { + "dst": "0.0.0.0/0" + } + ], + "ranges": [ + [ + { + "subnet": "10.89.12.0/24", + "gateway": "10.89.12.1" + } + ] + ] + } + }, + { + "type": "portmap", + "capabilities": { + "portMappings": true + } + }, + { + "type": "firewall", + "backend": "" + }, + { + "type": "tuning" + }, + { + "type": "dnsname", + "domainName": "dns.podman", + "capabilities": { + "aliases": true + } + } + ] +} diff --git a/libpod/network/types/const.go b/libpod/network/types/const.go new file mode 100644 index 000000000..be7ef03cf --- /dev/null +++ b/libpod/network/types/const.go @@ -0,0 +1,21 @@ +package types + +const ( + // BridgeNetworkDriver defines the bridge driver + BridgeNetworkDriver = "bridge" + // DefaultNetworkDriver is the default network type used + DefaultNetworkDriver = BridgeNetworkDriver + // MacVLANNetworkDriver defines the macvlan driver + MacVLANNetworkDriver = "macvlan" + + // IPAM drivers + // HostLocalIPAMDriver store the ip + HostLocalIPAMDriver = "host-local" + // DHCPIPAMDriver get subnet and ip from dhcp server + DHCPIPAMDriver = "dhcp" + + // DefaultSubnet is the name that will be used for the default CNI network. + DefaultNetworkName = "podman" + // DefaultSubnet is the subnet that will be used for the default CNI network. + DefaultSubnet = "10.88.0.0/16" +) diff --git a/libpod/network/types/network.go b/libpod/network/types/network.go new file mode 100644 index 000000000..c2c598f46 --- /dev/null +++ b/libpod/network/types/network.go @@ -0,0 +1,208 @@ +package types + +import ( + "net" + "time" +) + +type ContainerNetwork interface { + // NetworkCreate will take a partial filled Network and fill the + // missing fields. It creates the Network and returns the full Network. + NetworkCreate(Network) (Network, error) + // NetworkRemove will remove the Network with the given name or ID. + NetworkRemove(nameOrID string) error + // NetworkList will return all known Networks. Optionally you can + // supply a list of filter functions. Only if a network matches all + // functions it is returned. + NetworkList(...FilterFunc) ([]Network, error) + // NetworkInspect will return the Network with the given name or ID. + NetworkInspect(nameOrID string) (Network, error) + + // Setup will setup the container network namespace. It returns + // a map of StatusBlocks, the key is the network name. + Setup(namespacePath string, options SetupOptions) (map[string]StatusBlock, error) + // Teardown will teardown the container network namespace. + Teardown(namespacePath string, options TeardownOptions) error +} + +// Network describes the Network attributes. +type Network struct { + // Name of the Network. + Name string `json:"name,omitempty"` + // ID of the Network. + ID string `json:"id,omitempty"` + // Driver for this Network, e.g. bridge, macvlan... + Driver string `json:"driver,omitempty"` + // InterfaceName is the network interface name on the host. + NetworkInterface string `json:"network_interface,omitempty"` + // Created contains the timestamp when this network was created. + // This is not guaranteed to stay exactly the same. + Created time.Time + // Subnets to use. + Subnets []Subnet `json:"subnets,omitempty"` + // IPv6Enabled if set to true an ipv6 subnet should be created for this net. + IPv6Enabled bool `json:"ipv6_enabled"` + // Internal is whether the Network should not have external routes + // to public or other Networks. + Internal bool `json:"internal"` + // DNSEnabled is whether name resolution is active for container on + // this Network. + DNSEnabled bool `json:"dns_enabled"` + // Labels is a set of key-value labels that have been applied to the + // Network. + Labels map[string]string `json:"labels,omitempty"` + // Options is a set of key-value options that have been applied to + // the Network. + Options map[string]string `json:"options,omitempty"` + // IPAMOptions contains options used for the ip assignment. + IPAMOptions map[string]string `json:"ipam_options,omitempty"` +} + +// IPNet is used as custom net.IPNet type to add Marshal/Unmarshal methods. +type IPNet struct { + net.IPNet +} + +// ParseCIDR parse a string to IPNet +func ParseCIDR(cidr string) (IPNet, error) { + ip, net, err := net.ParseCIDR(cidr) + if err != nil { + return IPNet{}, err + } + // convert to 4 bytes if ipv4 + ipv4 := ip.To4() + if ipv4 != nil { + ip = ipv4 + } + net.IP = ip + return IPNet{*net}, err +} + +func (n *IPNet) MarshalText() ([]byte, error) { + return []byte(n.String()), nil +} + +func (n *IPNet) UnmarshalText(text []byte) error { + net, err := ParseCIDR(string(text)) + if err != nil { + return err + } + *n = net + return nil +} + +type Subnet struct { + // Subnet for this Network. + Subnet IPNet `json:"subnet,omitempty"` + // Gateway IP for this Network. + Gateway net.IP `json:"gateway,omitempty"` + // LeaseRange contains the range where IP are leased. Optional. + LeaseRange *LeaseRange `json:"lease_range,omitempty"` +} + +// LeaseRange contains the range where IP are leased. +type LeaseRange struct { + // StartIP first IP in the subnet which should be used to assign ips. + StartIP net.IP `json:"start_ip,omitempty"` + // EndIP last IP in the subnet which should be used to assign ips. + EndIP net.IP `json:"end_ip,omitempty"` +} + +// StatusBlock contains the network information about a container +// connected to one Network. +type StatusBlock struct { + // Interfaces contains the created network interface in the container. + // The map key is the interface name. + Interfaces map[string]NetInterface `json:"interfaces,omitempty"` + // DNSServerIPs nameserver addresses which should be added to + // the containers resolv.conf file. + DNSServerIPs []net.IP `json:"dns_server_ips,omitempty"` + // DNSSearchDomains search domains which should be added to + // the containers resolv.conf file. + DNSSearchDomains []string `json:"dns_search_domains,omitempty"` +} + +// NetInterface contains the settings for a given network interface. +type NetInterface struct { + // Networks list of assigned subnets with their gateway. + Networks []NetAddress `json:"networks,omitempty"` + // MacAddress for this Interface. + MacAddress net.HardwareAddr `json:"mac_address,omitempty"` +} + +// NetAddress contains the subnet and gatway. +type NetAddress struct { + // Subnet of this NetAddress. Note that the subnet contains the + // actual ip of the net interface and not the network address. + Subnet IPNet `json:"subnet,omitempty"` + // Gateway for the Subnet. This can be nil if there is no gateway, e.g. internal network. + Gateway net.IP `json:"gateway,omitempty"` +} + +// PerNetworkOptions are options which should be set on a per network basis. +type PerNetworkOptions struct { + // StaticIPv4 for this container. Optional. + StaticIPs []net.IP `json:"static_ips,omitempty"` + // Aliases contains a list of names which the dns server should resolve + // to this container. Can only be set when DNSEnabled is true on the Network. + // Optional. + Aliases []string `json:"aliases,omitempty"` + // StaticMac for this container. Optional. + StaticMAC net.HardwareAddr `json:"static_mac,omitempty"` + // InterfaceName for this container. Required. + InterfaceName string `json:"interface_name,omitempty"` +} + +// NetworkOptions for a given container. +type NetworkOptions struct { + // ContainerID is the container id, used for iptables comments and ipam allocation. + ContainerID string `json:"container_id,omitempty"` + // ContainerName is the container name, used as dns name. + ContainerName string `json:"container_name,omitempty"` + // PortMappings contains the port mappings for this container + PortMappings []PortMapping `json:"port_mappings,omitempty"` + // Networks contains all networks with the PerNetworkOptions. + // The map should contain at least one element. + Networks map[string]PerNetworkOptions `json:"networks,omitempty"` +} + +// PortMapping is one or more ports that will be mapped into the container. +type PortMapping struct { + // HostIP is the IP that we will bind to on the host. + // If unset, assumed to be 0.0.0.0 (all interfaces). + HostIP string `json:"host_ip,omitempty"` + // ContainerPort is the port number that will be exposed from the + // container. + // Mandatory. + ContainerPort uint16 `json:"container_port"` + // HostPort is the port number that will be forwarded from the host into + // the container. + // If omitted, a random port on the host (guaranteed to be over 1024) + // will be assigned. + HostPort uint16 `json:"host_port,omitempty"` + // Range is the number of ports that will be forwarded, starting at + // HostPort and ContainerPort and counting up. + // This is 1-indexed, so 1 is assumed to be a single port (only the + // Hostport:Containerport mapping will be added), 2 is two ports (both + // Hostport:Containerport and Hostport+1:Containerport+1), etc. + // If unset, assumed to be 1 (a single port). + // Both hostport + range and containerport + range must be less than + // 65536. + Range uint16 `json:"range,omitempty"` + // Protocol is the protocol forward. + // Must be either "tcp", "udp", and "sctp", or some combination of these + // separated by commas. + // If unset, assumed to be TCP. + Protocol string `json:"protocol,omitempty"` +} + +type SetupOptions struct { + NetworkOptions +} + +type TeardownOptions struct { + NetworkOptions +} + +// FilterFunc can be passed to NetworkList to filter the networks. +type FilterFunc func(Network) bool diff --git a/libpod/network/util/filters.go b/libpod/network/util/filters.go new file mode 100644 index 000000000..48e769196 --- /dev/null +++ b/libpod/network/util/filters.go @@ -0,0 +1,55 @@ +package util + +import ( + "strings" + + "github.com/containers/podman/v3/libpod/network/types" + "github.com/containers/podman/v3/pkg/util" + "github.com/pkg/errors" +) + +func GenerateNetworkFilters(filters map[string][]string) ([]types.FilterFunc, error) { + filterFuncs := make([]types.FilterFunc, 0, len(filters)) + for key, filterValues := range filters { + filterFunc, err := createFilterFuncs(key, filterValues) + if err != nil { + return nil, err + } + filterFuncs = append(filterFuncs, filterFunc) + } + return filterFuncs, nil +} + +func createFilterFuncs(key string, filterValues []string) (types.FilterFunc, error) { + switch strings.ToLower(key) { + case "name": + // matches one name, regex allowed + return func(net types.Network) bool { + return util.StringMatchRegexSlice(net.Name, filterValues) + }, nil + + case "label": + // matches all labels + return func(net types.Network) bool { + return util.MatchLabelFilters(filterValues, net.Labels) + }, nil + + case "driver": + // matches network driver + return func(net types.Network) bool { + return util.StringInSlice(net.Driver, filterValues) + }, nil + + case "id": + // matches part of one id + return func(net types.Network) bool { + return util.StringMatchRegexSlice(net.ID, filterValues) + }, nil + + // FIXME: What should we do with the old plugin filter + // TODO: add dangling, dns enabled, internal filter + + default: + return nil, errors.Errorf("invalid filter %q", key) + } +} diff --git a/libpod/network/util/interfaces.go b/libpod/network/util/interfaces.go new file mode 100644 index 000000000..dc2bd601d --- /dev/null +++ b/libpod/network/util/interfaces.go @@ -0,0 +1,34 @@ +package util + +import "net" + +// GetLiveNetworkSubnets returns a slice of subnets representing what the system +// has defined as network interfaces +func GetLiveNetworkSubnets() ([]*net.IPNet, error) { + addrs, err := net.InterfaceAddrs() + if err != nil { + return nil, err + } + nets := make([]*net.IPNet, 0, len(addrs)) + for _, address := range addrs { + _, n, err := net.ParseCIDR(address.String()) + if err != nil { + return nil, err + } + nets = append(nets, n) + } + return nets, nil +} + +// GetLiveNetworkNames returns a list of network interface names on the system +func GetLiveNetworkNames() ([]string, error) { + liveInterfaces, err := net.Interfaces() + if err != nil { + return nil, err + } + interfaceNames := make([]string, 0, len(liveInterfaces)) + for _, i := range liveInterfaces { + interfaceNames = append(interfaceNames, i.Name) + } + return interfaceNames, nil +} diff --git a/libpod/network/util/ip.go b/libpod/network/util/ip.go new file mode 100644 index 000000000..b2ba92735 --- /dev/null +++ b/libpod/network/util/ip.go @@ -0,0 +1,113 @@ +package util + +import ( + "crypto/rand" + "net" + + "github.com/pkg/errors" +) + +// IsIPv6 returns true if netIP is IPv6. +func IsIPv6(netIP net.IP) bool { + return netIP != nil && netIP.To4() == nil +} + +// IsIPv4 returns true if netIP is IPv4. +func IsIPv4(netIP net.IP) bool { + return netIP != nil && netIP.To4() != nil +} + +func incByte(subnet *net.IPNet, idx int, shift uint) error { + if idx < 0 { + return errors.New("no more subnets left") + } + if subnet.IP[idx] == 255 { + subnet.IP[idx] = 0 + return incByte(subnet, idx-1, 0) + } + subnet.IP[idx] += 1 << shift + return nil +} + +// NextSubnet returns subnet incremented by 1 +func NextSubnet(subnet *net.IPNet) (*net.IPNet, error) { + newSubnet := &net.IPNet{ + IP: subnet.IP, + Mask: subnet.Mask, + } + ones, bits := newSubnet.Mask.Size() + if ones == 0 { + return nil, errors.Errorf("%s has only one subnet", subnet.String()) + } + zeroes := uint(bits - ones) + shift := zeroes % 8 + idx := ones/8 - 1 + if idx < 0 { + idx = 0 + } + if err := incByte(newSubnet, idx, shift); err != nil { + return nil, err + } + return newSubnet, nil +} + +// LastIPInSubnet gets the last IP in a subnet +func LastIPInSubnet(addr *net.IPNet) (net.IP, error) { //nolint:interfacer + // re-parse to ensure clean network address + _, cidr, err := net.ParseCIDR(addr.String()) + if err != nil { + return nil, err + } + + ones, bits := cidr.Mask.Size() + if ones == bits { + return cidr.IP, nil + } + for i := range cidr.IP { + cidr.IP[i] = cidr.IP[i] | ^cidr.Mask[i] + } + return cidr.IP, nil +} + +// FirstIPInSubnet gets the first IP in a subnet +func FirstIPInSubnet(addr *net.IPNet) (net.IP, error) { //nolint:interfacer + // re-parse to ensure clean network address + _, cidr, err := net.ParseCIDR(addr.String()) + if err != nil { + return nil, err + } + ones, bits := cidr.Mask.Size() + if ones == bits { + return cidr.IP, nil + } + cidr.IP[len(cidr.IP)-1]++ + return cidr.IP, nil +} + +func NetworkIntersectsWithNetworks(n *net.IPNet, networklist []*net.IPNet) bool { + for _, nw := range networklist { + if networkIntersect(n, nw) { + return true + } + } + return false +} + +func networkIntersect(n1, n2 *net.IPNet) bool { + return n2.Contains(n1.IP) || n1.Contains(n2.IP) +} + +// GetRandomIPv6Subnet returns a random internal ipv6 subnet as described in RFC3879. +func GetRandomIPv6Subnet() (net.IPNet, error) { + ip := make(net.IP, 8, net.IPv6len) + // read 8 random bytes + _, err := rand.Read(ip) + if err != nil { + return net.IPNet{}, nil + } + // first byte must be FD as per RFC3879 + ip[0] = 0xfd + // add 8 zero bytes + ip = append(ip, make([]byte, 8)...) + return net.IPNet{IP: ip, Mask: net.CIDRMask(64, 128)}, nil +} diff --git a/libpod/network/util/ip_test.go b/libpod/network/util/ip_test.go new file mode 100644 index 000000000..c26ad140a --- /dev/null +++ b/libpod/network/util/ip_test.go @@ -0,0 +1,125 @@ +package util + +import ( + "fmt" + "net" + "reflect" + "testing" +) + +func parseCIDR(n string) *net.IPNet { + _, parsedNet, _ := net.ParseCIDR(n) + return parsedNet +} + +func TestNextSubnet(t *testing.T) { + type args struct { + subnet *net.IPNet + } + tests := []struct { + name string + args args + want *net.IPNet + wantErr bool + }{ + {"class b", args{subnet: parseCIDR("192.168.0.0/16")}, parseCIDR("192.169.0.0/16"), false}, + {"class c", args{subnet: parseCIDR("192.168.1.0/24")}, parseCIDR("192.168.2.0/24"), false}, + } + for _, tt := range tests { + test := tt + t.Run(test.name, func(t *testing.T) { + got, err := NextSubnet(test.args.subnet) + if (err != nil) != test.wantErr { + t.Errorf("NextSubnet() error = %v, wantErr %v", err, test.wantErr) + return + } + if !reflect.DeepEqual(got, test.want) { + t.Errorf("NextSubnet() got = %v, want %v", got, test.want) + } + }) + } +} + +func TestFirstIPInSubnet(t *testing.T) { + tests := []struct { + name string + args *net.IPNet + want net.IP + wantErr bool + }{ + {"class b", parseCIDR("192.168.0.0/16"), net.ParseIP("192.168.0.1"), false}, + {"class c", parseCIDR("192.168.1.0/24"), net.ParseIP("192.168.1.1"), false}, + {"cidr /23", parseCIDR("192.168.0.0/23"), net.ParseIP("192.168.0.1"), false}, + {"cidr /25", parseCIDR("192.168.1.0/25"), net.ParseIP("192.168.1.1"), false}, + {"cidr /26", parseCIDR("172.16.1.128/26"), net.ParseIP("172.16.1.129"), false}, + {"class a", parseCIDR("10.0.0.0/8"), net.ParseIP("10.0.0.1"), false}, + {"cidr /32", parseCIDR("192.168.255.4/32"), net.ParseIP("192.168.255.4"), false}, + {"cidr /31", parseCIDR("192.168.255.4/31"), net.ParseIP("192.168.255.5"), false}, + } + for _, tt := range tests { + test := tt + t.Run(test.name, func(t *testing.T) { + got, err := FirstIPInSubnet(test.args) + if (err != nil) != test.wantErr { + t.Errorf("FirstIPInSubnet() error = %v, wantErr %v", err, test.wantErr) + return + } + if !got.Equal(test.want) { + t.Errorf("FirstIPInSubnet() got = %v, want %v", got, test.want) + } + }) + } +} + +func TestLastIPInSubnet(t *testing.T) { + tests := []struct { + name string + args *net.IPNet + want net.IP + wantErr bool + }{ + {"class b", parseCIDR("192.168.0.0/16"), net.ParseIP("192.168.255.255"), false}, + {"class c", parseCIDR("192.168.1.0/24"), net.ParseIP("192.168.1.255"), false}, + {"cidr /23", parseCIDR("192.168.0.0/23"), net.ParseIP("192.168.1.255"), false}, + {"cidr /25", parseCIDR("192.168.1.0/25"), net.ParseIP("192.168.1.127"), false}, + {"cidr /26", parseCIDR("172.16.1.128/26"), net.ParseIP("172.16.1.191"), false}, + {"class a", parseCIDR("10.0.0.0/8"), net.ParseIP("10.255.255.255"), false}, + {"cidr /32", parseCIDR("192.168.255.4/32"), net.ParseIP("192.168.255.4"), false}, + {"cidr /31", parseCIDR("192.168.255.4/31"), net.ParseIP("192.168.255.5"), false}, + } + for _, tt := range tests { + test := tt + t.Run(test.name, func(t *testing.T) { + got, err := LastIPInSubnet(test.args) + if (err != nil) != test.wantErr { + t.Errorf("LastIPInSubnet() error = %v, wantErr %v", err, test.wantErr) + return + } + if !got.Equal(test.want) { + t.Errorf("LastIPInSubnet() got = %v, want %v", got, test.want) + } + }) + } +} + +func TestGetRandomIPv6Subnet(t *testing.T) { + for i := 0; i < 1000; i++ { + t.Run(fmt.Sprintf("GetRandomIPv6Subnet %d", i), func(t *testing.T) { + sub, err := GetRandomIPv6Subnet() + if err != nil { + t.Errorf("GetRandomIPv6Subnet() error should be nil: %v", err) + return + } + if sub.IP.To4() != nil { + t.Errorf("ip %s is not an ipv6 address", sub.IP) + } + if sub.IP[0] != 0xfd { + t.Errorf("ipv6 %s does not start with fd", sub.IP) + } + ones, bytes := sub.Mask.Size() + if ones != 64 || bytes != 128 { + t.Errorf("wrong network mask %v, it should be /64", sub.Mask) + } + }) + } +} diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go index 8e9b5997c..dbe2274d3 100644 --- a/libpod/networking_linux.go +++ b/libpod/networking_linux.go @@ -173,11 +173,27 @@ func (r *RootlessCNI) Do(toRun func() error) error { // the link target will be available in the mount ns. // see: https://github.com/containers/podman/issues/10855 resolvePath := "/etc/resolv.conf" - resolvePath, err = filepath.EvalSymlinks(resolvePath) - if err != nil { - return err + for i := 0; i < 255; i++ { + // Do not use filepath.EvalSymlinks, we only want the first symlink under /run. + // If /etc/resolv.conf has more than one symlink under /run, e.g. + // -> /run/systemd/resolve/stub-resolv.conf -> /run/systemd/resolve/resolv.conf + // we would put the netns resolv.conf file to the last path. However this will + // break dns because the second link does not exists in the mount ns. + // see https://github.com/containers/podman/issues/11222 + link, err := os.Readlink(resolvePath) + if err != nil { + // if there is no symlink exit + break + } + resolvePath = filepath.Join(filepath.Dir(resolvePath), link) + if strings.HasPrefix(resolvePath, "/run/") { + break + } + if i == 254 { + return errors.New("too many symlinks while resolving /etc/resolv.conf") + } } - logrus.Debugf("The actual path of /etc/resolv.conf on the host is %q", resolvePath) + logrus.Debugf("The path of /etc/resolv.conf in the mount ns is %q", resolvePath) // When /etc/resolv.conf on the host is a symlink to /run/systemd/resolve/stub-resolv.conf, // we have to mount an empty filesystem on /run/systemd/resolve in the child namespace, // so as to isolate the directory from the host mount namespace. @@ -999,7 +1015,7 @@ func (c *Container) getContainerNetworkInfo() (*define.InspectNetworkSettings, e } settings := new(define.InspectNetworkSettings) - settings.Ports = makeInspectPortBindings(c.config.PortMappings) + settings.Ports = makeInspectPortBindings(c.config.PortMappings, c.config.ExposedPorts) networks, isDefault, err := c.networks() if err != nil { @@ -1219,7 +1235,7 @@ func (c *Container) NetworkDisconnect(nameOrID, netName string, force bool) erro return err } - // OCICNI will set the loopback adpter down on teardown so we should set it up again + // OCICNI will set the loopback adapter down on teardown so we should set it up again err = c.state.NetNS.Do(func(_ ns.NetNS) error { link, err := netlink.LinkByName("lo") if err != nil { @@ -1229,7 +1245,7 @@ func (c *Container) NetworkDisconnect(nameOrID, netName string, force bool) erro return err }) if err != nil { - logrus.Warnf("failed to set loopback adpter up in the container: %v", err) + logrus.Warnf("failed to set loopback adapter up in the container: %v", err) } // Reload ports when there are still connected networks, maybe we removed the network interface with the child ip. // Reloading without connected networks does not make sense, so we can skip this step. diff --git a/libpod/oci_conmon_exec_linux.go b/libpod/oci_conmon_exec_linux.go index 05a4e19b0..469bc7d86 100644 --- a/libpod/oci_conmon_exec_linux.go +++ b/libpod/oci_conmon_exec_linux.go @@ -462,7 +462,7 @@ func (r *ConmonOCIRuntime) startExec(c *Container, sessionID string, options *Ex Setpgid: true, } - err = startCommandGivenSelinux(execCmd) + err = startCommandGivenSelinux(execCmd, c) // We don't need children pipes on the parent side errorhandling.CloseQuiet(childSyncPipe) diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go index 846d3815a..c14911980 100644 --- a/libpod/oci_conmon_linux.go +++ b/libpod/oci_conmon_linux.go @@ -364,11 +364,6 @@ func (r *ConmonOCIRuntime) StartContainer(ctr *Container) error { return err } env := []string{fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir)} - if ctr.config.SdNotifyMode == define.SdNotifyModeContainer { - if notify, ok := os.LookupEnv("NOTIFY_SOCKET"); ok { - env = append(env, fmt.Sprintf("NOTIFY_SOCKET=%s", notify)) - } - } if path, ok := os.LookupEnv("PATH"); ok { env = append(env, fmt.Sprintf("PATH=%s", path)) } @@ -630,9 +625,11 @@ func (r *ConmonOCIRuntime) HTTPAttach(ctr *Container, req *http.Request, w http. if err != nil { break } - _, err = httpBuf.Write([]byte("\n")) - if err != nil { - break + if !logLine.Partial() { + _, err = httpBuf.Write([]byte("\n")) + if err != nil { + break + } } err = httpBuf.Flush() if err != nil { @@ -1014,12 +1011,6 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co } } - if ctr.config.SdNotifyMode == define.SdNotifyModeIgnore { - if err := os.Unsetenv("NOTIFY_SOCKET"); err != nil { - logrus.Warnf("Error unsetting NOTIFY_SOCKET %v", err) - } - } - pidfile := ctr.config.PidFile if pidfile == "" { pidfile = filepath.Join(ctr.state.RunDir, "pidfile") @@ -1027,6 +1018,10 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co args := r.sharedConmonArgs(ctr, ctr.ID(), ctr.bundlePath(), pidfile, ctr.LogPath(), r.exitsDir, ociLog, ctr.LogDriver(), logTag) + if ctr.config.SdNotifyMode == define.SdNotifyModeContainer && ctr.notifySocket != "" { + args = append(args, fmt.Sprintf("--sdnotify-socket=%s", ctr.notifySocket)) + } + if ctr.config.Spec.Process.Terminal { args = append(args, "-t") } else if ctr.config.Stdin { @@ -1171,7 +1166,8 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co } } - err = startCommandGivenSelinux(cmd) + err = startCommandGivenSelinux(cmd, ctr) + // regardless of whether we errored or not, we no longer need the children pipes childSyncPipe.Close() childStartPipe.Close() @@ -1203,7 +1199,13 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co // conmon not having a pid file is a valid state, so don't set it if we don't have it logrus.Infof("Got Conmon PID as %d", conmonPID) ctr.state.ConmonPID = conmonPID - if ctr.config.SdNotifyMode != define.SdNotifyModeIgnore { + + // Send the MAINPID via sdnotify if needed. + switch ctr.config.SdNotifyMode { + case define.SdNotifyModeContainer, define.SdNotifyModeIgnore: + // Nothing to do or conmon takes care of it already. + + default: if sent, err := daemon.SdNotify(false, fmt.Sprintf("MAINPID=%d", conmonPID)); err != nil { logrus.Errorf("Error notifying systemd of Conmon PID: %v", err) } else if sent { @@ -1239,11 +1241,6 @@ func (r *ConmonOCIRuntime) configureConmonEnv(ctr *Container, runtimeDir string) } extraFiles := make([]*os.File, 0) - if ctr.config.SdNotifyMode == define.SdNotifyModeContainer { - if notify, ok := os.LookupEnv("NOTIFY_SOCKET"); ok { - env = append(env, fmt.Sprintf("NOTIFY_SOCKET=%s", notify)) - } - } if !r.sdNotify { if listenfds, ok := os.LookupEnv("LISTEN_FDS"); ok { env = append(env, fmt.Sprintf("LISTEN_FDS=%s", listenfds), "LISTEN_PID=1") @@ -1335,7 +1332,23 @@ func (r *ConmonOCIRuntime) sharedConmonArgs(ctr *Container, cuuid, bundlePath, p // startCommandGivenSelinux starts a container ensuring to set the labels of // the process to make sure SELinux doesn't block conmon communication, if SELinux is enabled -func startCommandGivenSelinux(cmd *exec.Cmd) error { +func startCommandGivenSelinux(cmd *exec.Cmd, ctr *Container) error { + // Make sure to unset the NOTIFY_SOCKET and reset if afterwards if needed. + switch ctr.config.SdNotifyMode { + case define.SdNotifyModeContainer, define.SdNotifyModeIgnore: + if ctr.notifySocket != "" { + if err := os.Unsetenv("NOTIFY_SOCKET"); err != nil { + logrus.Warnf("Error unsetting NOTIFY_SOCKET %v", err) + } + + defer func() { + if err := os.Setenv("NOTIFY_SOCKET", ctr.notifySocket); err != nil { + logrus.Errorf("Error resetting NOTIFY_SOCKET=%s", ctr.notifySocket) + } + }() + } + } + if !selinux.GetEnabled() { return cmd.Start() } diff --git a/libpod/options.go b/libpod/options.go index 071b085e7..0bcd1e3a6 100644 --- a/libpod/options.go +++ b/libpod/options.go @@ -268,7 +268,7 @@ func WithRegistriesConf(path string) RuntimeOption { logrus.Debugf("Setting custom registries.conf: %q", path) return func(rt *Runtime) error { if _, err := os.Stat(path); err != nil { - return errors.Wrap(err, "error locating specified registries.conf") + return errors.Wrap(err, "locating specified registries.conf") } if rt.imageContext == nil { rt.imageContext = &types.SystemContext{ @@ -956,8 +956,9 @@ func WithUserNSFrom(nsCtr *Container) CtrCreateOption { } ctr.config.UserNsCtr = nsCtr.ID() - ctr.config.IDMappings = nsCtr.config.IDMappings - + if err := JSONDeepCopy(nsCtr.IDMappings(), &ctr.config.IDMappings); err != nil { + return err + } g := generate.Generator{Config: ctr.config.Spec} g.ClearLinuxUIDMappings() @@ -968,7 +969,6 @@ func WithUserNSFrom(nsCtr *Container) CtrCreateOption { for _, gidmap := range nsCtr.config.IDMappings.GIDMap { g.AddLinuxGIDMapping(uint32(gidmap.HostID), uint32(gidmap.ContainerID), uint32(gidmap.Size)) } - ctr.config.IDMappings = nsCtr.config.IDMappings return nil } } @@ -1041,7 +1041,7 @@ func WithDependencyCtrs(ctrs []*Container) CtrCreateOption { // namespace with a minimal configuration. // An optional array of port mappings can be provided. // Conflicts with WithNetNSFrom(). -func WithNetNS(portMappings []ocicni.PortMapping, postConfigureNetNS bool, netmode string, networks []string) CtrCreateOption { +func WithNetNS(portMappings []ocicni.PortMapping, exposedPorts map[uint16][]string, postConfigureNetNS bool, netmode string, networks []string) CtrCreateOption { return func(ctr *Container) error { if ctr.valid { return define.ErrCtrFinalized @@ -1051,6 +1051,7 @@ func WithNetNS(portMappings []ocicni.PortMapping, postConfigureNetNS bool, netmo ctr.config.NetMode = namespaces.NetworkMode(netmode) ctr.config.CreateNetNS = true ctr.config.PortMappings = portMappings + ctr.config.ExposedPorts = exposedPorts ctr.config.Networks = networks @@ -1453,7 +1454,7 @@ func WithNamedVolumes(volumes []*ContainerNamedVolume) CtrCreateOption { for _, vol := range volumes { mountOpts, err := util.ProcessOptions(vol.Options, false, "") if err != nil { - return errors.Wrapf(err, "error processing options for named volume %q mounted at %q", vol.Name, vol.Dest) + return errors.Wrapf(err, "processing options for named volume %q mounted at %q", vol.Name, vol.Dest) } ctr.config.NamedVolumes = append(ctr.config.NamedVolumes, &ContainerNamedVolume{ @@ -2423,6 +2424,24 @@ func WithVolatile() CtrCreateOption { } ctr.config.Volatile = true + + return nil + } +} + +// WithPodUserns sets the userns for the infra container in a pod. +func WithPodUserns(userns specgen.Namespace) PodCreateOption { + return func(pod *Pod) error { + if pod.valid { + return define.ErrPodFinalized + } + + if !pod.config.InfraContainer.HasInfraContainer { + return errors.Wrapf(define.ErrInvalidArg, "cannot configure pod userns as no infra container is being created") + } + + pod.config.InfraContainer.Userns = userns + return nil } } diff --git a/libpod/pod.go b/libpod/pod.go index 0fef7f6f3..7df15df7b 100644 --- a/libpod/pod.go +++ b/libpod/pod.go @@ -117,6 +117,7 @@ type InfraContainerConfig struct { Slirp4netns bool `json:"slirp4netns,omitempty"` NetworkOptions map[string][]string `json:"network_options,omitempty"` ResourceLimits *specs.LinuxResources `json:"resource_limits,omitempty"` + Userns specgen.Namespace `json:"userns,omitempty"` } // ID retrieves the pod's ID diff --git a/libpod/pod_api.go b/libpod/pod_api.go index 90d67dbb0..53fb9538f 100644 --- a/libpod/pod_api.go +++ b/libpod/pod_api.go @@ -32,14 +32,14 @@ func (p *Pod) startInitContainers(ctx context.Context) error { if rc != 0 { return errors.Errorf("init container %s exited with code %d", initCon.ID(), rc) } - // If the container is an oneshot init container, we need to remove it + // If the container is a once init container, we need to remove it // after it runs if initCon.Config().InitContainerType == define.OneShotInitContainer { icLock := initCon.lock icLock.Lock() if err := p.runtime.removeContainer(ctx, initCon, false, false, true); err != nil { icLock.Unlock() - return errors.Wrapf(err, "failed to remove oneshot init container %s", initCon.ID()) + return errors.Wrapf(err, "failed to remove once init container %s", initCon.ID()) } // Removing a container this way requires an explicit call to clean up the db if err := p.runtime.state.RemoveContainerFromPod(p, initCon); err != nil { @@ -593,6 +593,7 @@ func (p *Pod) Inspect() (*define.InspectPodData, error) { infraConfig.CPUQuota = p.CPUQuota() infraConfig.CPUSetCPUs = p.ResourceLim().CPU.Cpus infraConfig.PidNS = p.PidMode() + infraConfig.UserNS = p.config.InfraContainer.Userns.String() if len(p.config.InfraContainer.DNSServer) > 0 { infraConfig.DNSServer = make([]string, 0, len(p.config.InfraContainer.DNSServer)) @@ -615,7 +616,7 @@ func (p *Pod) Inspect() (*define.InspectPodData, error) { infraConfig.Networks = append(infraConfig.Networks, p.config.InfraContainer.Networks...) } infraConfig.NetworkOptions = p.config.InfraContainer.NetworkOptions - infraConfig.PortBindings = makeInspectPortBindings(p.config.InfraContainer.PortBindings) + infraConfig.PortBindings = makeInspectPortBindings(p.config.InfraContainer.PortBindings, nil) } inspectData := define.InspectPodData{ diff --git a/libpod/runtime.go b/libpod/runtime.go index 30659a3d4..c5f5db531 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -30,6 +30,7 @@ import ( "github.com/containers/podman/v3/libpod/shutdown" "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/rootless" + "github.com/containers/podman/v3/pkg/systemd" "github.com/containers/podman/v3/pkg/util" "github.com/containers/storage" "github.com/containers/storage/pkg/unshare" @@ -500,6 +501,15 @@ func makeRuntime(ctx context.Context, runtime *Runtime) (retErr error) { // no containers running. Create immediately a namespace, as // we will need to access the storage. if needsUserns { + // warn users if mode is rootless and cgroup manager is systemd + // and no valid systemd session is present + // warn only whenever new namespace is created + if runtime.config.Engine.CgroupManager == config.SystemdCgroupsManager { + unified, _ := cgroups.IsCgroup2UnifiedMode() + if unified && rootless.IsRootless() && !systemd.IsSystemdSessionValid(rootless.GetRootlessUID()) { + logrus.Debug("Invalid systemd user session for current user") + } + } aliveLock.Unlock() // Unlock to avoid deadlock as BecomeRootInUserNS will reexec. pausePid, err := util.GetRootlessPauseProcessPidPathGivenDir(runtime.config.Engine.TmpDir) if err != nil { @@ -941,6 +951,11 @@ func (r *Runtime) GetOCIRuntimePath() string { return r.defaultOCIRuntime.Path() } +// DefaultOCIRuntime return copy of Default OCI Runtime +func (r *Runtime) DefaultOCIRuntime() OCIRuntime { + return r.defaultOCIRuntime +} + // StorageConfig retrieves the storage options for the container runtime func (r *Runtime) StorageConfig() storage.StoreOptions { return r.storageConfig diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 059f56798..52072b0f3 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -246,6 +246,20 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (_ *Contai ctr.config.Networks = netNames } + // https://github.com/containers/podman/issues/11285 + // normalize the networks aliases to use network names and never ids + if len(ctr.config.NetworkAliases) > 0 { + netAliases := make(map[string][]string, len(ctr.config.NetworkAliases)) + for nameOrID, aliases := range ctr.config.NetworkAliases { + netName, err := network.NormalizeName(r.config, nameOrID) + if err != nil { + return nil, err + } + netAliases[netName] = aliases + } + ctr.config.NetworkAliases = netAliases + } + // Inhibit shutdown until creation succeeds shutdown.Inhibit() defer shutdown.Uninhibit() @@ -448,8 +462,15 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (_ *Contai ctrNamedVolumes = append(ctrNamedVolumes, newVol) } - if ctr.config.LogPath == "" && ctr.config.LogDriver != define.JournaldLogging && ctr.config.LogDriver != define.NoLogging { - ctr.config.LogPath = filepath.Join(ctr.config.StaticDir, "ctr.log") + switch ctr.config.LogDriver { + case define.NoLogging: + break + case define.JournaldLogging: + ctr.initializeJournal(ctx) + default: + if ctr.config.LogPath == "" { + ctr.config.LogPath = filepath.Join(ctr.config.StaticDir, "ctr.log") + } } if !MountExists(ctr.config.Spec.Mounts, "/dev/shm") && ctr.config.ShmDir == "" { diff --git a/libpod/runtime_pod_infra_linux.go b/libpod/runtime_pod_infra_linux.go index d4f861118..9236fb1f5 100644 --- a/libpod/runtime_pod_infra_linux.go +++ b/libpod/runtime_pod_infra_linux.go @@ -8,7 +8,9 @@ import ( "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/libpod/define" + "github.com/containers/podman/v3/pkg/namespaces" "github.com/containers/podman/v3/pkg/rootless" + "github.com/containers/podman/v3/pkg/specgen" "github.com/containers/podman/v3/pkg/util" v1 "github.com/opencontainers/image-spec/specs-go/v1" spec "github.com/opencontainers/runtime-spec/specs-go" @@ -110,9 +112,8 @@ func (r *Runtime) makeInfraContainer(ctx context.Context, p *Pod, imgName, rawIm options = append(options, WithNetworkOptions(p.config.InfraContainer.NetworkOptions)) } } - // PostConfigureNetNS should not be set since user namespace sharing is not implemented - // and rootless networking no longer supports post configuration setup - options = append(options, WithNetNS(p.config.InfraContainer.PortBindings, false, netmode, p.config.InfraContainer.Networks)) + // FIXME allow pods to have exposed ports + options = append(options, WithNetNS(p.config.InfraContainer.PortBindings, nil, !p.config.InfraContainer.Userns.IsHost(), netmode, p.config.InfraContainer.Networks)) } // For each option in InfraContainerConfig - if set, pass into @@ -158,11 +159,39 @@ func (r *Runtime) makeInfraContainer(ctx context.Context, p *Pod, imgName, rawIm g.Config.Linux.Namespaces = newNS } } + + for _, ctl := range r.config.Containers.DefaultSysctls { + sysctl := strings.SplitN(ctl, "=", 2) + if len(sysctl) < 2 { + return nil, errors.Errorf("invalid default sysctl %s", ctl) + } + + // Ignore net sysctls if --net=host + if p.config.InfraContainer.HostNetwork && strings.HasPrefix(sysctl[0], "net.") { + logrus.Infof("Sysctl %s=%s ignored in containers.conf, since Network Namespace set to host", sysctl[0], sysctl[1]) + continue + } + + g.AddLinuxSysctl(sysctl[0], sysctl[1]) + } + g.SetRootReadonly(true) g.SetProcessArgs(infraCtrCommand) logrus.Debugf("Using %q as infra container command", infraCtrCommand) + mapopt, err := util.ParseIDMapping(namespaces.UsernsMode(p.config.InfraContainer.Userns.String()), []string{}, []string{}, "", "") + if err != nil { + return nil, err + } + user, err := specgen.SetupUserNS(mapopt, p.config.InfraContainer.Userns, &g) + if err != nil { + return nil, err + } + if user != "" { + options = append(options, WithUser(user)) + } + g.RemoveMount("/dev/shm") if isRootless { g.RemoveMount("/dev/pts") @@ -210,14 +239,15 @@ func (r *Runtime) makeInfraContainer(ctx context.Context, p *Pod, imgName, rawIm options = append(options, WithRootFSFromImage(imgID, imgName, rawImageName)) options = append(options, WithName(containerName)) options = append(options, withIsInfra()) + options = append(options, WithIDMappings(*mapopt)) if len(p.config.InfraContainer.ConmonPidFile) > 0 { options = append(options, WithConmonPidFile(p.config.InfraContainer.ConmonPidFile)) } newRes := new(spec.LinuxResources) newRes.CPU = new(spec.LinuxCPU) newRes.CPU = p.ResourceLim().CPU - g.Config.Linux.Resources.CPU = newRes.CPU + return r.newContainer(ctx, g.Config, options...) } diff --git a/libpod/runtime_volume_linux.go b/libpod/runtime_volume_linux.go index 40df98d7c..d1ea7d4fd 100644 --- a/libpod/runtime_volume_linux.go +++ b/libpod/runtime_volume_linux.go @@ -255,11 +255,6 @@ func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool) error // Set volume as invalid so it can no longer be used v.valid = false - // Remove the volume from the state - if err := r.state.RemoveVolume(v); err != nil { - return errors.Wrapf(err, "error removing volume %s", v.Name()) - } - var removalErr error // If we use a volume plugin, we need to remove from the plugin. @@ -287,11 +282,19 @@ func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool) error req := new(pluginapi.RemoveRequest) req.Name = v.Name() if err := v.plugin.RemoveVolume(req); err != nil { - removalErr = errors.Wrapf(err, "volume %s could not be removed from plugin %s, but it has been removed from Podman", v.Name(), v.Driver()) + return errors.Wrapf(err, "volume %s could not be removed from plugin %s", v.Name(), v.Driver()) } } } + // Remove the volume from the state + if err := r.state.RemoveVolume(v); err != nil { + if removalErr != nil { + logrus.Errorf("Error removing volume %s from plugin %s: %v", v.Name(), v.Driver(), removalErr) + } + return errors.Wrapf(err, "error removing volume %s", v.Name()) + } + // Free the volume's lock if err := v.lock.Free(); err != nil { if removalErr == nil { diff --git a/libpod/shutdown/handler.go b/libpod/shutdown/handler.go index 848b6729a..1e8a9ec3b 100644 --- a/libpod/shutdown/handler.go +++ b/libpod/shutdown/handler.go @@ -35,7 +35,7 @@ func Start() error { return nil } - sigChan = make(chan os.Signal, 1) + sigChan = make(chan os.Signal, 2) cancelChan = make(chan bool, 1) stopped = false diff --git a/libpod/util.go b/libpod/util.go index 3b32fb264..ed5c4e6c6 100644 --- a/libpod/util.go +++ b/libpod/util.go @@ -295,8 +295,8 @@ func writeHijackHeader(r *http.Request, conn io.Writer) { } // Convert OCICNI port bindings into Inspect-formatted port bindings. -func makeInspectPortBindings(bindings []ocicni.PortMapping) map[string][]define.InspectHostPort { - portBindings := make(map[string][]define.InspectHostPort) +func makeInspectPortBindings(bindings []ocicni.PortMapping, expose map[uint16][]string) map[string][]define.InspectHostPort { + portBindings := make(map[string][]define.InspectHostPort, len(bindings)) for _, port := range bindings { key := fmt.Sprintf("%d/%s", port.ContainerPort, port.Protocol) hostPorts := portBindings[key] @@ -309,6 +309,15 @@ func makeInspectPortBindings(bindings []ocicni.PortMapping) map[string][]define. }) portBindings[key] = hostPorts } + // add exposed ports without host port information to match docker + for port, protocols := range expose { + for _, protocol := range protocols { + key := fmt.Sprintf("%d/%s", port, protocol) + if _, ok := portBindings[key]; !ok { + portBindings[key] = nil + } + } + } return portBindings } diff --git a/libpod/volume.go b/libpod/volume.go index 8f3dc4fcc..90b423f1d 100644 --- a/libpod/volume.go +++ b/libpod/volume.go @@ -139,6 +139,17 @@ func (v *Volume) MountPoint() (string, error) { return v.mountPoint(), nil } +// MountCount returns the volume's mountcount on the host from state +// Useful in determining if volume is using plugin or a filesystem mount and its mount +func (v *Volume) MountCount() (uint, error) { + v.lock.Lock() + defer v.lock.Unlock() + if err := v.update(); err != nil { + return 0, err + } + return v.state.MountCount, nil +} + // Internal-only helper for volume mountpoint func (v *Volume) mountPoint() string { if v.UsesVolumeDriver() { diff --git a/pkg/api/handlers/compat/containers_logs.go b/pkg/api/handlers/compat/containers_logs.go index 50cdb1e65..a7cfe09ea 100644 --- a/pkg/api/handlers/compat/containers_logs.go +++ b/pkg/api/handlers/compat/containers_logs.go @@ -152,9 +152,7 @@ func LogsFromContainer(w http.ResponseWriter, r *http.Request) { } frame.WriteString(line.Msg) - // Log lines in the compat layer require adding EOL - // https://github.com/containers/podman/issues/8058 - if !utils.IsLibpodRequest(r) { + if !line.Partial() { frame.WriteString("\n") } diff --git a/pkg/api/handlers/compat/images_build.go b/pkg/api/handlers/compat/images_build.go index 08d1df4b8..0fcca1821 100644 --- a/pkg/api/handlers/compat/images_build.go +++ b/pkg/api/handlers/compat/images_build.go @@ -34,13 +34,16 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { contentType := hdr[0] switch contentType { case "application/tar": - logrus.Warnf("tar file content type is %s, should use \"application/x-tar\" content type", contentType) + logrus.Infof("tar file content type is %s, should use \"application/x-tar\" content type", contentType) case "application/x-tar": break default: - utils.BadRequest(w, "Content-Type", hdr[0], - fmt.Errorf("Content-Type: %s is not supported. Should be \"application/x-tar\"", hdr[0])) - return + if utils.IsLibpodRequest(r) { + utils.BadRequest(w, "Content-Type", hdr[0], + fmt.Errorf("Content-Type: %s is not supported. Should be \"application/x-tar\"", hdr[0])) + return + } + logrus.Infof("tar file content type is %s, should use \"application/x-tar\" content type", contentType) } } diff --git a/pkg/api/handlers/compat/info.go b/pkg/api/handlers/compat/info.go index d7cefd516..2c26c7bf8 100644 --- a/pkg/api/handlers/compat/info.go +++ b/pkg/api/handlers/compat/info.go @@ -102,14 +102,18 @@ func GetInfo(w http.ResponseWriter, r *http.Request) { OomKillDisable: sysInfo.OomKillDisable, OperatingSystem: infoData.Host.Distribution.Distribution, PidsLimit: sysInfo.PidsLimit, - Plugins: docker.PluginsInfo{}, - ProductLicense: "Apache-2.0", - RegistryConfig: new(registry.ServiceConfig), - RuncCommit: docker.Commit{}, - Runtimes: getRuntimes(configInfo), - SecurityOptions: getSecOpts(sysInfo), - ServerVersion: versionInfo.Version, - SwapLimit: sysInfo.SwapLimit, + Plugins: docker.PluginsInfo{ + Volume: infoData.Plugins.Volume, + Network: infoData.Plugins.Network, + Log: infoData.Plugins.Log, + }, + ProductLicense: "Apache-2.0", + RegistryConfig: new(registry.ServiceConfig), + RuncCommit: docker.Commit{}, + Runtimes: getRuntimes(configInfo), + SecurityOptions: getSecOpts(sysInfo), + ServerVersion: versionInfo.Version, + SwapLimit: sysInfo.SwapLimit, Swarm: swarm.Info{ LocalNodeState: swarm.LocalNodeStateInactive, }, diff --git a/pkg/api/handlers/compat/swagger.go b/pkg/api/handlers/compat/swagger.go index b773799ef..cfbdd1154 100644 --- a/pkg/api/handlers/compat/swagger.go +++ b/pkg/api/handlers/compat/swagger.go @@ -2,7 +2,6 @@ package compat import ( "github.com/containers/podman/v3/pkg/domain/entities" - "github.com/containers/storage/pkg/archive" "github.com/docker/docker/api/types" ) @@ -28,15 +27,6 @@ type swagCtrWaitResponse struct { } } -// Object Changes -// swagger:response Changes -type swagChangesResponse struct { - // in:body - Body struct { - Changes []archive.Change - } -} - // Network inspect // swagger:response CompatNetworkInspect type swagCompatNetworkInspect struct { diff --git a/pkg/api/handlers/compat/version.go b/pkg/api/handlers/compat/version.go index f1cd77a9a..a115cc885 100644 --- a/pkg/api/handlers/compat/version.go +++ b/pkg/api/handlers/compat/version.go @@ -13,20 +13,19 @@ import ( "github.com/containers/podman/v3/pkg/domain/entities/types" "github.com/containers/podman/v3/version" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) func VersionHandler(w http.ResponseWriter, r *http.Request) { - // 200 ok - // 500 internal runtime := r.Context().Value("runtime").(*libpod.Runtime) - versionInfo, err := define.GetVersion() + running, err := define.GetVersion() if err != nil { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, err) return } - infoData, err := runtime.Info() + info, err := runtime.Info() if err != nil { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrapf(err, "failed to obtain system memory info")) return @@ -34,20 +33,40 @@ func VersionHandler(w http.ResponseWriter, r *http.Request) { components := []types.ComponentVersion{{ Name: "Podman Engine", - Version: versionInfo.Version, + Version: running.Version, Details: map[string]string{ "APIVersion": version.APIVersion[version.Libpod][version.CurrentAPI].String(), "Arch": goRuntime.GOARCH, - "BuildTime": time.Unix(versionInfo.Built, 0).Format(time.RFC3339), - "Experimental": "true", - "GitCommit": versionInfo.GitCommit, - "GoVersion": versionInfo.GoVersion, - "KernelVersion": infoData.Host.Kernel, + "BuildTime": time.Unix(running.Built, 0).Format(time.RFC3339), + "Experimental": "false", + "GitCommit": running.GitCommit, + "GoVersion": running.GoVersion, + "KernelVersion": info.Host.Kernel, "MinAPIVersion": version.APIVersion[version.Libpod][version.MinimalAPI].String(), "Os": goRuntime.GOOS, }, }} + if conmon, oci, err := runtime.DefaultOCIRuntime().RuntimeInfo(); err != nil { + logrus.Warnf("Failed to retrieve Conmon and OCI Information: %q", err.Error()) + } else { + additional := []types.ComponentVersion{ + { + Name: "Conmon", + Version: conmon.Version, + Details: map[string]string{ + "Package": conmon.Package, + }}, + { + Name: fmt.Sprintf("OCI Runtime (%s)", oci.Name), + Version: oci.Version, + Details: map[string]string{ + "Package": oci.Package, + }}, + } + components = append(components, additional...) + } + apiVersion := version.APIVersion[version.Compat][version.CurrentAPI] minVersion := version.APIVersion[version.Compat][version.MinimalAPI] @@ -56,13 +75,13 @@ func VersionHandler(w http.ResponseWriter, r *http.Request) { Platform: struct { Name string }{ - Name: fmt.Sprintf("%s/%s/%s-%s", goRuntime.GOOS, goRuntime.GOARCH, infoData.Host.Distribution.Distribution, infoData.Host.Distribution.Version), + Name: fmt.Sprintf("%s/%s/%s-%s", goRuntime.GOOS, goRuntime.GOARCH, info.Host.Distribution.Distribution, info.Host.Distribution.Version), }, APIVersion: fmt.Sprintf("%d.%d", apiVersion.Major, apiVersion.Minor), Arch: components[0].Details["Arch"], BuildTime: components[0].Details["BuildTime"], Components: components, - Experimental: true, + Experimental: false, GitCommit: components[0].Details["GitCommit"], GoVersion: components[0].Details["GoVersion"], KernelVersion: components[0].Details["KernelVersion"], diff --git a/pkg/api/handlers/libpod/images_pull.go b/pkg/api/handlers/libpod/images_pull.go index 04b415638..3c13c6e20 100644 --- a/pkg/api/handlers/libpod/images_pull.go +++ b/pkg/api/handlers/libpod/images_pull.go @@ -33,6 +33,7 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) { TLSVerify bool `schema:"tlsVerify"` AllTags bool `schema:"allTags"` PullPolicy string `schema:"policy"` + Quiet bool `schema:"quiet"` }{ TLSVerify: true, PullPolicy: "always", @@ -116,8 +117,10 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) { select { case s := <-writer.Chan(): report.Stream = string(s) - if err := enc.Encode(report); err != nil { - logrus.Warnf("Failed to encode json: %v", err) + if !query.Quiet { + if err := enc.Encode(report); err != nil { + logrus.Warnf("Failed to encode json: %v", err) + } } flush() case <-runCtx.Done(): diff --git a/pkg/api/handlers/libpod/pods.go b/pkg/api/handlers/libpod/pods.go index 4dc8740e2..ff105bc48 100644 --- a/pkg/api/handlers/libpod/pods.go +++ b/pkg/api/handlers/libpod/pods.go @@ -30,6 +30,12 @@ func PodCreate(w http.ResponseWriter, r *http.Request) { utils.Error(w, "failed to decode specgen", http.StatusInternalServerError, errors.Wrap(err, "failed to decode specgen")) return } + // parse userns so we get the valid default value of userns + psg.Userns, err = specgen.ParseUserNamespace(psg.Userns.String()) + if err != nil { + utils.Error(w, "failed to parse userns", http.StatusInternalServerError, errors.Wrap(err, "failed to parse userns")) + return + } pod, err := generate.MakePod(&psg, runtime) if err != nil { httpCode := http.StatusInternalServerError diff --git a/pkg/api/handlers/swagger/swagger.go b/pkg/api/handlers/swagger/swagger.go index 83ff5914e..2296eea3a 100644 --- a/pkg/api/handlers/swagger/swagger.go +++ b/pkg/api/handlers/swagger/swagger.go @@ -152,13 +152,6 @@ type swagPodTopResponse struct { } } -// List processes in pod -// swagger:response DocsPodStatsResponse -type swagPodStatsResponse struct { - // in:body - Body []*entities.PodStatsReport -} - // Inspect container // swagger:response LibpodInspectContainerResponse type swagLibpodInspectContainerResponse struct { @@ -183,12 +176,3 @@ type swagInspectPodResponse struct { define.InspectPodData } } - -// Inspect volume -// swagger:response InspectVolumeResponse -type swagInspectVolumeResponse struct { - // in:body - Body struct { - define.InspectVolumeData - } -} diff --git a/pkg/api/handlers/types.go b/pkg/api/handlers/types.go index af5878798..b82c586ea 100644 --- a/pkg/api/handlers/types.go +++ b/pkg/api/handlers/types.go @@ -104,6 +104,7 @@ type ContainerWaitOKBody struct { } // CreateContainerConfig used when compatible endpoint creates a container +// swagger:model CreateContainerConfig type CreateContainerConfig struct { Name string // container name dockerContainer.Config // desired container configuration diff --git a/pkg/api/handlers/utils/images.go b/pkg/api/handlers/utils/images.go index 1e8edb6dd..1e3647a3e 100644 --- a/pkg/api/handlers/utils/images.go +++ b/pkg/api/handlers/utils/images.go @@ -27,7 +27,7 @@ func IsRegistryReference(name string) error { if imageRef.Transport().Name() == docker.Transport.Name() { return nil } - return errors.Errorf("unsupport transport %s in %q: only docker transport is supported", imageRef.Transport().Name(), name) + return errors.Errorf("unsupported transport %s in %q: only docker transport is supported", imageRef.Transport().Name(), name) } // ParseStorageReference parses the specified image name to a diff --git a/pkg/api/server/register_containers.go b/pkg/api/server/register_containers.go index 0ec4f95d9..b36cb75f1 100644 --- a/pkg/api/server/register_containers.go +++ b/pkg/api/server/register_containers.go @@ -21,6 +21,12 @@ func (s *APIServer) registerContainersHandlers(r *mux.Router) error { // name: name // type: string // description: container name + // - in: body + // name: body + // description: Container to create + // schema: + // $ref: "#/definitions/CreateContainerConfig" + // required: true // responses: // 201: // $ref: "#/responses/ContainerCreateResponse" diff --git a/pkg/api/server/register_images.go b/pkg/api/server/register_images.go index 2103c093c..2630acac2 100644 --- a/pkg/api/server/register_images.go +++ b/pkg/api/server/register_images.go @@ -25,6 +25,10 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // produces: // - application/json // parameters: + // - in: header + // name: X-Registry-Auth + // type: string + // description: A base64-encoded auth configuration. // - in: query // name: fromImage // type: string @@ -49,13 +53,8 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // name: platform // type: string // description: Platform in the format os[/arch[/variant]] - // default: "" - // - in: header - // name: X-Registry-Auth - // type: string - // description: A base64-encoded auth configuration. // - in: body - // name: request + // name: inputImage // schema: // type: string // format: binary @@ -472,6 +471,14 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // summary: Create image // description: Build an image from the given Dockerfile(s) // parameters: + // - in: header + // name: Content-Type + // type: string + // default: application/x-tar + // enum: ["application/x-tar"] + // - in: header + // name: X-Registry-Config + // type: string // - in: query // name: dockerfile // type: string @@ -653,6 +660,14 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // description: | // output configuration TBD // (As of version 1.xx) + // - in: body + // name: inputStream + // description: | + // A tar archive compressed with one of the following algorithms: + // identity (no compression), gzip, bzip2, xz. + // schema: + // type: string + // format: binary // produces: // - application/json // responses: @@ -852,6 +867,11 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // summary: Import image // description: Import a previously exported tarball as an image. // parameters: + // - in: header + // name: Content-Type + // type: string + // default: application/x-tar + // enum: ["application/x-tar"] // - in: query // name: changes // description: "Apply the following possible instructions to the created image: CMD | ENTRYPOINT | ENV | EXPOSE | LABEL | STOPSIGNAL | USER | VOLUME | WORKDIR. JSON encoded string" @@ -875,7 +895,8 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // required: true // description: tarball for imported image // schema: - // type: "string" + // type: string + // format: binary // produces: // - application/json // consumes: @@ -962,6 +983,15 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // description: "Mandatory reference to the image (e.g., quay.io/image/name:tag)" // type: string // - in: query + // name: quiet + // description: "silences extra stream data on pull" + // type: boolean + // default: false + // - in: query + // name: credentials + // description: "username:password for the registry" + // type: string + // - in: query // name: Arch // description: Pull image for the specified architecture. // type: string diff --git a/pkg/api/server/register_pods.go b/pkg/api/server/register_pods.go index 3bcc50ba4..de3669a0a 100644 --- a/pkg/api/server/register_pods.go +++ b/pkg/api/server/register_pods.go @@ -17,7 +17,18 @@ func (s *APIServer) registerPodsHandlers(r *mux.Router) error { // - in: query // name: filters // type: string - // description: needs description and plumbing for filters + // description: | + // JSON encoded value of the filters (a map[string][]string) to process on the pods list. Available filters: + // - `id=<pod-id>` Matches all of pod id. + // - `label=<key>` or `label=<key>:<value>` Matches pods based on the presence of a label alone or a label and a value. + // - `name=<pod-name>` Matches all of pod name. + // - `until=<timestamp>` List pods created before this timestamp. The `<timestamp>` can be Unix timestamps, date formatted timestamps, or Go duration strings (e.g. `10m`, `1h30m`) computed relative to the daemon machine’s time. + // - `status=<pod-status>` Pod's status: `stopped`, `running`, `paused`, `exited`, `dead`, `created`, `degraded`. + // - `network=<pod-network>` Name or full ID of network. + // - `ctr-names=<pod-ctr-names>` Container name within the pod. + // - `ctr-ids=<pod-ctr-ids>` Container ID within the pod. + // - `ctr-status=<pod-ctr-status>` Container status within the pod. + // - `ctr-number=<pod-ctr-number>` Number of containers in the pod. // responses: // 200: // $ref: "#/responses/ListPodsResponse" @@ -40,7 +51,7 @@ func (s *APIServer) registerPodsHandlers(r *mux.Router) error { // responses: // 201: // schema: - // $ref: "#/definitions/IdResponse" + // $ref: "#/definitions/IDResponse" // 400: // $ref: "#/responses/BadParamError" // 409: diff --git a/pkg/api/server/swagger.go b/pkg/api/server/swagger.go index d282edf23..0fd66652e 100644 --- a/pkg/api/server/swagger.go +++ b/pkg/api/server/swagger.go @@ -141,13 +141,6 @@ type swagImageSummary struct { Body []entities.ImageSummary } -// Registries summary -// swagger:response DocsRegistriesList -type swagRegistriesList struct { - // in:body - Body entities.ListRegistriesReport -} - // List Containers // swagger:response DocsListContainer type swagListContainers struct { diff --git a/pkg/bindings/connection.go b/pkg/bindings/connection.go index 62b1655ac..4127ad2f0 100644 --- a/pkg/bindings/connection.go +++ b/pkg/bindings/connection.go @@ -56,7 +56,7 @@ func NewConnection(ctx context.Context, uri string) (context.Context, error) { return NewConnectionWithIdentity(ctx, uri, "") } -// NewConnection takes a URI as a string and returns a context with the +// NewConnectionWithIdentity takes a URI as a string and returns a context with the // Connection embedded as a value. This context needs to be passed to each // endpoint to work correctly. // @@ -117,7 +117,7 @@ func NewConnectionWithIdentity(ctx context.Context, uri string, identity string) ctx = context.WithValue(ctx, clientKey, &connection) if err := pingNewConnection(ctx); err != nil { - return nil, errors.Wrap(err, "cannot connect to the Podman socket, please verify that Podman REST API service is running") + return nil, errors.Wrap(err, "cannot connect to the Podman socket, please verify the connection to the Linux system, or use `podman machine` to create/start a Linux VM.") } return ctx, nil } @@ -149,6 +149,7 @@ func pingNewConnection(ctx context.Context) error { if err != nil { return err } + defer response.Body.Close() if response.StatusCode == http.StatusOK { versionHdr := response.Header.Get("Libpod-API-Version") @@ -338,7 +339,7 @@ func (c *Connection) DoRequest(httpBody io.Reader, httpMethod, endpoint string, req.Header.Set(key, val) } // Give the Do three chances in the case of a comm/service hiccup - for i := 0; i < 3; i++ { + for i := 1; i <= 3; i++ { response, err = c.Client.Do(req) // nolint if err == nil { break @@ -358,7 +359,7 @@ func FiltersToString(filters map[string][]string) (string, error) { return jsoniter.MarshalToString(lowerCaseKeys) } -// IsInformation returns true if the response code is 1xx +// IsInformational returns true if the response code is 1xx func (h *APIResponse) IsInformational() bool { return h.Response.StatusCode/100 == 1 } diff --git a/pkg/bindings/containers/archive.go b/pkg/bindings/containers/archive.go index 52b73662b..876f5340b 100644 --- a/pkg/bindings/containers/archive.go +++ b/pkg/bindings/containers/archive.go @@ -27,6 +27,7 @@ func Stat(ctx context.Context, nameOrID string, path string) (*entities.Containe if err != nil { return nil, err } + defer response.Body.Close() var finalErr error if response.StatusCode == http.StatusNotFound { @@ -53,7 +54,9 @@ func CopyFromArchive(ctx context.Context, nameOrID string, path string, reader i return CopyFromArchiveWithOptions(ctx, nameOrID, path, reader, nil) } -// CopyFromArchiveWithOptions FIXME: remove this function and make CopyFromArchive accept the option as the last parameter in podman 4.0 +// CopyFromArchiveWithOptions copy files into container +// +// FIXME: remove this function and make CopyFromArchive accept the option as the last parameter in podman 4.0 func CopyFromArchiveWithOptions(ctx context.Context, nameOrID string, path string, reader io.Reader, options *CopyOptions) (entities.ContainerCopyFunc, error) { conn, err := bindings.GetClient(ctx) if err != nil { @@ -72,6 +75,7 @@ func CopyFromArchiveWithOptions(ctx context.Context, nameOrID string, path strin if err != nil { return err } + if response.StatusCode != http.StatusOK { return errors.New(response.Status) } @@ -79,6 +83,7 @@ func CopyFromArchiveWithOptions(ctx context.Context, nameOrID string, path strin }, nil } +// CopyToArchive copy files from container func CopyToArchive(ctx context.Context, nameOrID string, path string, writer io.Writer) (entities.ContainerCopyFunc, error) { conn, err := bindings.GetClient(ctx) if err != nil { @@ -91,11 +96,14 @@ func CopyToArchive(ctx context.Context, nameOrID string, path string, writer io. if err != nil { return nil, err } + if response.StatusCode != http.StatusOK { + defer response.Body.Close() return nil, response.Process(nil) } return func() error { + defer response.Body.Close() _, err := io.Copy(writer, response.Body) return err }, nil diff --git a/pkg/bindings/containers/attach.go b/pkg/bindings/containers/attach.go index 01c14d350..6efbcb57b 100644 --- a/pkg/bindings/containers/attach.go +++ b/pkg/bindings/containers/attach.go @@ -134,7 +134,9 @@ func Attach(ctx context.Context, nameOrID string, stdin io.Reader, stdout io.Wri if err != nil { return err } + if !(response.IsSuccess() || response.IsInformational()) { + defer response.Body.Close() return response.Process(nil) } @@ -207,7 +209,7 @@ func Attach(ctx context.Context, nameOrID string, stdin io.Reader, stdout io.Wri } } } else { - logrus.Debugf("Copying standard streams of container in non-terminal mode") + logrus.Debugf("Copying standard streams of container %q in non-terminal mode", ctnr.ID) for { // Read multiplexed channels and write to appropriate stream fd, l, err := DemuxHeader(socket, buffer) @@ -324,6 +326,8 @@ func resizeTTY(ctx context.Context, endpoint string, height *int, width *int) er if err != nil { return err } + defer rsp.Body.Close() + return rsp.Process(nil) } @@ -407,6 +411,7 @@ func ExecStartAndAttach(ctx context.Context, sessionID string, options *ExecStar if err != nil { return err } + defer resp.Body.Close() respStruct := new(define.InspectExecSession) if err := resp.Process(respStruct); err != nil { @@ -477,6 +482,8 @@ func ExecStartAndAttach(ctx context.Context, sessionID string, options *ExecStar if err != nil { return err } + defer response.Body.Close() + if !(response.IsSuccess() || response.IsInformational()) { return response.Process(nil) } diff --git a/pkg/bindings/containers/checkpoint.go b/pkg/bindings/containers/checkpoint.go index 440bf707d..7f7080f13 100644 --- a/pkg/bindings/containers/checkpoint.go +++ b/pkg/bindings/containers/checkpoint.go @@ -27,6 +27,8 @@ func Checkpoint(ctx context.Context, nameOrID string, options *CheckpointOptions if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } @@ -54,5 +56,7 @@ func Restore(ctx context.Context, nameOrID string, options *RestoreOptions) (*en if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } diff --git a/pkg/bindings/containers/commit.go b/pkg/bindings/containers/commit.go index 8ae61322e..a4adebb1f 100644 --- a/pkg/bindings/containers/commit.go +++ b/pkg/bindings/containers/commit.go @@ -28,5 +28,7 @@ func Commit(ctx context.Context, nameOrID string, options *CommitOptions) (handl if err != nil { return id, err } + defer response.Body.Close() + return id, response.Process(&id) } diff --git a/pkg/bindings/containers/containers.go b/pkg/bindings/containers/containers.go index bc7b0c8c9..aafb83f65 100644 --- a/pkg/bindings/containers/containers.go +++ b/pkg/bindings/containers/containers.go @@ -42,6 +42,8 @@ func List(ctx context.Context, options *ListOptions) ([]entities.ListContainer, if err != nil { return containers, err } + defer response.Body.Close() + return containers, response.Process(&containers) } @@ -66,6 +68,8 @@ func Prune(ctx context.Context, options *PruneOptions) ([]*reports.PruneReport, if err != nil { return nil, err } + defer response.Body.Close() + return reports, response.Process(&reports) } @@ -90,6 +94,8 @@ func Remove(ctx context.Context, nameOrID string, options *RemoveOptions) error if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } @@ -113,6 +119,8 @@ func Inspect(ctx context.Context, nameOrID string, options *InspectOptions) (*de if err != nil { return nil, err } + defer response.Body.Close() + inspect := define.InspectContainerData{} return &inspect, response.Process(&inspect) } @@ -136,6 +144,8 @@ func Kill(ctx context.Context, nameOrID string, options *KillOptions) error { if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } @@ -154,6 +164,8 @@ func Pause(ctx context.Context, nameOrID string, options *PauseOptions) error { if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } @@ -176,6 +188,8 @@ func Restart(ctx context.Context, nameOrID string, options *RestartOptions) erro if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } @@ -199,6 +213,8 @@ func Start(ctx context.Context, nameOrID string, options *StartOptions) error { if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } @@ -231,6 +247,7 @@ func Stats(ctx context.Context, containers []string, options *StatsOptions) (cha go func() { defer close(statsChan) + defer response.Body.Close() dec := json.NewDecoder(response.Body) doStream := true @@ -245,6 +262,7 @@ func Stats(ctx context.Context, containers []string, options *StatsOptions) (cha default: // fall through and do some work } + var report entities.ContainerStatsReport if err := dec.Decode(&report); err != nil { report = entities.ContainerStatsReport{Error: err} @@ -279,6 +297,7 @@ func Top(ctx context.Context, nameOrID string, options *TopOptions) ([]string, e if err != nil { return nil, err } + defer response.Body.Close() body := handlers.ContainerTopOKBody{} if err = response.Process(&body); err != nil { @@ -311,6 +330,8 @@ func Unpause(ctx context.Context, nameOrID string, options *UnpauseOptions) erro if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } @@ -334,6 +355,8 @@ func Wait(ctx context.Context, nameOrID string, options *WaitOptions) (int32, er if err != nil { return exitCode, err } + defer response.Body.Close() + return exitCode, response.Process(&exitCode) } @@ -353,6 +376,8 @@ func Exists(ctx context.Context, nameOrID string, options *ExistsOptions) (bool, if err != nil { return false, err } + defer response.Body.Close() + return response.IsSuccess(), nil } @@ -374,6 +399,8 @@ func Stop(ctx context.Context, nameOrID string, options *StopOptions) error { if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } @@ -393,6 +420,8 @@ func Export(ctx context.Context, nameOrID string, w io.Writer, options *ExportOp if err != nil { return err } + defer response.Body.Close() + if response.StatusCode/100 == 2 { _, err = io.Copy(w, response.Body) return err @@ -416,6 +445,8 @@ func ContainerInit(ctx context.Context, nameOrID string, options *InitOptions) e if err != nil { return err } + defer response.Body.Close() + if response.StatusCode == http.StatusNotModified { return errors.Wrapf(define.ErrCtrStateInvalid, "container %s has already been created in runtime", nameOrID) } @@ -435,5 +466,7 @@ func ShouldRestart(ctx context.Context, nameOrID string, options *ShouldRestartO if err != nil { return false, err } + defer response.Body.Close() + return response.IsSuccess(), nil } diff --git a/pkg/bindings/containers/create.go b/pkg/bindings/containers/create.go index 3efa9643d..c0b9538a6 100644 --- a/pkg/bindings/containers/create.go +++ b/pkg/bindings/containers/create.go @@ -30,5 +30,7 @@ func CreateWithSpec(ctx context.Context, s *specgen.SpecGenerator, options *Crea if err != nil { return ccr, err } + defer response.Body.Close() + return ccr, response.Process(&ccr) } diff --git a/pkg/bindings/containers/diff.go b/pkg/bindings/containers/diff.go index 7d20ae530..e4ec49809 100644 --- a/pkg/bindings/containers/diff.go +++ b/pkg/bindings/containers/diff.go @@ -26,6 +26,8 @@ func Diff(ctx context.Context, nameOrID string, options *DiffOptions) ([]archive if err != nil { return nil, err } + defer response.Body.Close() + var changes []archive.Change return changes, response.Process(&changes) } diff --git a/pkg/bindings/containers/exec.go b/pkg/bindings/containers/exec.go index 5ae6d1d71..12b31aba3 100644 --- a/pkg/bindings/containers/exec.go +++ b/pkg/bindings/containers/exec.go @@ -39,6 +39,7 @@ func ExecCreate(ctx context.Context, nameOrID string, config *handlers.ExecCreat if err != nil { return "", err } + defer resp.Body.Close() respStruct := new(handlers.ExecCreateResponse) if err := resp.Process(respStruct); err != nil { @@ -66,6 +67,7 @@ func ExecInspect(ctx context.Context, sessionID string, options *ExecInspectOpti if err != nil { return nil, err } + defer resp.Body.Close() respStruct := new(define.InspectExecSession) if err := resp.Process(respStruct); err != nil { @@ -103,6 +105,7 @@ func ExecStart(ctx context.Context, sessionID string, options *ExecStartOptions) if err != nil { return err } + defer resp.Body.Close() return resp.Process(nil) } diff --git a/pkg/bindings/containers/healthcheck.go b/pkg/bindings/containers/healthcheck.go index d6b721615..0e65a5a46 100644 --- a/pkg/bindings/containers/healthcheck.go +++ b/pkg/bindings/containers/healthcheck.go @@ -26,5 +26,7 @@ func RunHealthCheck(ctx context.Context, nameOrID string, options *HealthCheckOp if err != nil { return nil, err } + defer response.Body.Close() + return &status, response.Process(&status) } diff --git a/pkg/bindings/containers/logs.go b/pkg/bindings/containers/logs.go index a3100c697..67db94487 100644 --- a/pkg/bindings/containers/logs.go +++ b/pkg/bindings/containers/logs.go @@ -33,6 +33,7 @@ func Logs(ctx context.Context, nameOrID string, options *LogOptions, stdoutChan, if err != nil { return err } + defer response.Body.Close() buffer := make([]byte, 1024) for { diff --git a/pkg/bindings/containers/mount.go b/pkg/bindings/containers/mount.go index bb5c3bd67..c07998fd3 100644 --- a/pkg/bindings/containers/mount.go +++ b/pkg/bindings/containers/mount.go @@ -25,6 +25,8 @@ func Mount(ctx context.Context, nameOrID string, options *MountOptions) (string, if err != nil { return path, err } + defer response.Body.Close() + return path, response.Process(&path) } @@ -43,6 +45,8 @@ func Unmount(ctx context.Context, nameOrID string, options *UnmountOptions) erro if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } @@ -61,5 +65,7 @@ func GetMountedContainerPaths(ctx context.Context, options *MountedContainerPath if err != nil { return mounts, err } + defer response.Body.Close() + return mounts, response.Process(&mounts) } diff --git a/pkg/bindings/containers/rename.go b/pkg/bindings/containers/rename.go index 60d7fda73..172d7838a 100644 --- a/pkg/bindings/containers/rename.go +++ b/pkg/bindings/containers/rename.go @@ -24,5 +24,7 @@ func Rename(ctx context.Context, nameOrID string, options *RenameOptions) error if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } diff --git a/pkg/bindings/errors.go b/pkg/bindings/errors.go index 3339062a5..9c311d912 100644 --- a/pkg/bindings/errors.go +++ b/pkg/bindings/errors.go @@ -20,6 +20,8 @@ func handleError(data []byte) error { return e } +// Process drains the response body, and processes the HTTP status code +// Note: Closing the response.Body is left to the caller func (h APIResponse) Process(unmarshalInto interface{}) error { data, err := ioutil.ReadAll(h.Response.Body) if err != nil { diff --git a/pkg/bindings/generate/generate.go b/pkg/bindings/generate/generate.go index 7c904a6a8..742956515 100644 --- a/pkg/bindings/generate/generate.go +++ b/pkg/bindings/generate/generate.go @@ -26,10 +26,15 @@ func Systemd(ctx context.Context, nameOrID string, options *SystemdOptions) (*en if err != nil { return nil, err } + defer response.Body.Close() + report := &entities.GenerateSystemdReport{} return report, response.Process(&report.Units) } +// Kube generate Kubernetes YAML (v1 specification) +// +// Note: Caller is responsible for closing returned reader func Kube(ctx context.Context, nameOrIDs []string, options *KubeOptions) (*entities.GenerateKubeReport, error) { if options == nil { options = new(KubeOptions) diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go index e1aeae244..39e0fc5df 100644 --- a/pkg/bindings/images/build.go +++ b/pkg/bindings/images/build.go @@ -481,9 +481,9 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) { return nil // skip root dir } - name := strings.TrimPrefix(path, s+string(filepath.Separator)) + name := filepath.ToSlash(strings.TrimPrefix(path, s+string(filepath.Separator))) - excluded, err := pm.Matches(filepath.ToSlash(name)) // nolint:staticcheck + excluded, err := pm.Matches(name) // nolint:staticcheck if err != nil { return errors.Wrapf(err, "error checking if %q is excluded", name) } diff --git a/pkg/bindings/images/diff.go b/pkg/bindings/images/diff.go index 79b0df8c9..671b73089 100644 --- a/pkg/bindings/images/diff.go +++ b/pkg/bindings/images/diff.go @@ -23,6 +23,8 @@ func Diff(ctx context.Context, nameOrID string, options *DiffOptions) ([]archive if err != nil { return nil, err } + defer response.Body.Close() + var changes []archive.Change return changes, response.Process(&changes) } diff --git a/pkg/bindings/images/images.go b/pkg/bindings/images/images.go index 8680d6baa..959481e0d 100644 --- a/pkg/bindings/images/images.go +++ b/pkg/bindings/images/images.go @@ -27,6 +27,8 @@ func Exists(ctx context.Context, nameOrID string, options *ExistsOptions) (bool, if err != nil { return false, err } + defer response.Body.Close() + return response.IsSuccess(), nil } @@ -49,6 +51,8 @@ func List(ctx context.Context, options *ListOptions) ([]*entities.ImageSummary, if err != nil { return imageSummary, err } + defer response.Body.Close() + return imageSummary, response.Process(&imageSummary) } @@ -71,6 +75,8 @@ func GetImage(ctx context.Context, nameOrID string, options *GetOptions) (*entit if err != nil { return &inspectedData, err } + defer response.Body.Close() + return &inspectedData, response.Process(&inspectedData) } @@ -92,6 +98,8 @@ func Tree(ctx context.Context, nameOrID string, options *TreeOptions) (*entities if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } @@ -110,6 +118,8 @@ func History(ctx context.Context, nameOrID string, options *HistoryOptions) ([]* if err != nil { return history, err } + defer response.Body.Close() + return history, response.Process(&history) } @@ -123,6 +133,8 @@ func Load(ctx context.Context, r io.Reader) (*entities.ImageLoadReport, error) { if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } @@ -147,6 +159,7 @@ func Export(ctx context.Context, nameOrIDs []string, w io.Writer, options *Expor if err != nil { return err } + defer response.Body.Close() if response.StatusCode/100 == 2 || response.StatusCode/100 == 3 { _, err = io.Copy(w, response.Body) @@ -176,8 +189,9 @@ func Prune(ctx context.Context, options *PruneOptions) ([]*reports.PruneReport, if err != nil { return deleted, err } - err = response.Process(&deleted) - return deleted, err + defer response.Body.Close() + + return deleted, response.Process(&deleted) } // Tag adds an additional name to locally-stored image. Both the tag and repo parameters are required. @@ -197,6 +211,8 @@ func Tag(ctx context.Context, nameOrID, tag, repo string, options *TagOptions) e if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } @@ -217,10 +233,12 @@ func Untag(ctx context.Context, nameOrID, tag, repo string, options *UntagOption if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } -// Imports adds the given image to the local image store. This can be done by file and the given reader +// Import adds the given image to the local image store. This can be done by file and the given reader // or via the url parameter. Additional metadata can be associated with the image by using the changes and // message parameters. The image can also be tagged given a reference. One of url OR r must be provided. func Import(ctx context.Context, r io.Reader, options *ImportOptions) (*entities.ImageImportReport, error) { @@ -243,6 +261,8 @@ func Import(ctx context.Context, r io.Reader, options *ImportOptions) (*entities if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } @@ -269,8 +289,8 @@ func Push(ctx context.Context, source string, destination string, options *PushO if err != nil { return err } - //SkipTLSVerify is special. We need to delete the param added by - //toparams and change the key and flip the bool + // SkipTLSVerify is special. We need to delete the param added by + // toparams and change the key and flip the bool if options.SkipTLSVerify != nil { params.Del("SkipTLSVerify") params.Set("tlsVerify", strconv.FormatBool(!options.GetSkipTLSVerify())) @@ -282,6 +302,7 @@ func Push(ctx context.Context, source string, destination string, options *PushO if err != nil { return err } + defer response.Body.Close() return response.Process(err) } @@ -317,6 +338,7 @@ func Search(ctx context.Context, term string, options *SearchOptions) ([]entitie if err != nil { return nil, err } + defer response.Body.Close() results := []entities.ImageSearchReport{} if err := response.Process(&results); err != nil { diff --git a/pkg/bindings/images/rm.go b/pkg/bindings/images/rm.go index e45e583f4..461eb7729 100644 --- a/pkg/bindings/images/rm.go +++ b/pkg/bindings/images/rm.go @@ -36,6 +36,8 @@ func Remove(ctx context.Context, images []string, options *RemoveOptions) (*enti if err != nil { return nil, []error{err} } + defer response.Body.Close() + if err := response.Process(&report); err != nil { return nil, []error{err} } diff --git a/pkg/bindings/manifests/manifests.go b/pkg/bindings/manifests/manifests.go index 268ce3b19..6aa4961f1 100644 --- a/pkg/bindings/manifests/manifests.go +++ b/pkg/bindings/manifests/manifests.go @@ -46,10 +46,12 @@ func Create(ctx context.Context, names, images []string, options *CreateOptions) if err != nil { return "", err } + defer response.Body.Close() + return idr.ID, response.Process(&idr) } -// Exists returns true if a given maifest list exists +// Exists returns true if a given manifest list exists func Exists(ctx context.Context, name string, options *ExistsOptions) (bool, error) { conn, err := bindings.GetClient(ctx) if err != nil { @@ -59,6 +61,8 @@ func Exists(ctx context.Context, name string, options *ExistsOptions) (bool, err if err != nil { return false, err } + defer response.Body.Close() + return response.IsSuccess(), nil } @@ -77,6 +81,8 @@ func Inspect(ctx context.Context, name string, options *InspectOptions) (*manife if err != nil { return nil, err } + defer response.Body.Close() + return &list, response.Process(&list) } @@ -100,6 +106,8 @@ func Add(ctx context.Context, name string, options *AddOptions) (string, error) if err != nil { return "", err } + defer response.Body.Close() + return idr.ID, response.Process(&idr) } @@ -121,6 +129,8 @@ func Remove(ctx context.Context, name, digest string, options *RemoveOptions) (s if err != nil { return "", err } + defer response.Body.Close() + return idr.ID, response.Process(&idr) } @@ -145,18 +155,20 @@ func Push(ctx context.Context, name, destination string, options *images.PushOpt if err != nil { return "", err } - //SkipTLSVerify is special. We need to delete the param added by - //toparams and change the key and flip the bool + // SkipTLSVerify is special. We need to delete the param added by + // toparams and change the key and flip the bool if options.SkipTLSVerify != nil { params.Del("SkipTLSVerify") params.Set("tlsVerify", strconv.FormatBool(!options.GetSkipTLSVerify())) } params.Set("image", name) params.Set("destination", destination) - _, err = conn.DoRequest(nil, http.MethodPost, "/manifests/%s/push", params, nil, name) + response, err := conn.DoRequest(nil, http.MethodPost, "/manifests/%s/push", params, nil, name) if err != nil { return "", err } + defer response.Body.Close() + return idr.ID, err } @@ -179,5 +191,6 @@ func Push(ctx context.Context, name, destination string, options *images.PushOpt // if err != nil { // return "", err // } +// defer response.Body.Close() // return idr.ID, response.Process(&idr) //} diff --git a/pkg/bindings/network/network.go b/pkg/bindings/network/network.go index 17451c273..59207aa8d 100644 --- a/pkg/bindings/network/network.go +++ b/pkg/bindings/network/network.go @@ -34,6 +34,8 @@ func Create(ctx context.Context, options *CreateOptions) (*entities.NetworkCreat if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } @@ -53,6 +55,8 @@ func Inspect(ctx context.Context, nameOrID string, options *InspectOptions) ([]e if err != nil { return nil, err } + defer response.Body.Close() + return reports, response.Process(&reports[0]) } @@ -76,6 +80,8 @@ func Remove(ctx context.Context, nameOrID string, options *RemoveOptions) ([]*en if err != nil { return nil, err } + defer response.Body.Close() + return reports, response.Process(&reports) } @@ -99,6 +105,8 @@ func List(ctx context.Context, options *ListOptions) ([]*entities.NetworkListRep if err != nil { return netList, err } + defer response.Body.Close() + return netList, response.Process(&netList) } @@ -133,6 +141,8 @@ func Disconnect(ctx context.Context, networkName string, ContainerNameOrID strin if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } @@ -166,6 +176,8 @@ func Connect(ctx context.Context, networkName string, ContainerNameOrID string, if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } @@ -179,6 +191,8 @@ func Exists(ctx context.Context, nameOrID string, options *ExistsOptions) (bool, if err != nil { return false, err } + defer response.Body.Close() + return response.IsSuccess(), nil } @@ -203,5 +217,7 @@ func Prune(ctx context.Context, options *PruneOptions) ([]*entities.NetworkPrune if err != nil { return nil, err } + defer response.Body.Close() + return prunedNetworks, response.Process(&prunedNetworks) } diff --git a/pkg/bindings/play/play.go b/pkg/bindings/play/play.go index 4b735c821..8451cd533 100644 --- a/pkg/bindings/play/play.go +++ b/pkg/bindings/play/play.go @@ -48,6 +48,8 @@ func Kube(ctx context.Context, path string, options *KubeOptions) (*entities.Pla if err != nil { return nil, err } + defer response.Body.Close() + if err := response.Process(&report); err != nil { return nil, err } diff --git a/pkg/bindings/pods/pods.go b/pkg/bindings/pods/pods.go index eb7b273cf..9d3ff322e 100644 --- a/pkg/bindings/pods/pods.go +++ b/pkg/bindings/pods/pods.go @@ -34,6 +34,8 @@ func CreatePodFromSpec(ctx context.Context, s *specgen.PodSpecGenerator, options if err != nil { return nil, err } + defer response.Body.Close() + return &pcr, response.Process(&pcr) } @@ -47,6 +49,8 @@ func Exists(ctx context.Context, nameOrID string, options *ExistsOptions) (bool, if err != nil { return false, err } + defer response.Body.Close() + return response.IsSuccess(), nil } @@ -67,6 +71,8 @@ func Inspect(ctx context.Context, nameOrID string, options *InspectOptions) (*en if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } @@ -91,6 +97,8 @@ func Kill(ctx context.Context, nameOrID string, options *KillOptions) (*entities if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } @@ -109,6 +117,8 @@ func Pause(ctx context.Context, nameOrID string, options *PauseOptions) (*entiti if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } @@ -128,6 +138,8 @@ func Prune(ctx context.Context, options *PruneOptions) ([]*entities.PodPruneRepo if err != nil { return nil, err } + defer response.Body.Close() + return reports, response.Process(&reports) } @@ -152,6 +164,8 @@ func List(ctx context.Context, options *ListOptions) ([]*entities.ListPodsReport if err != nil { return podsReports, err } + defer response.Body.Close() + return podsReports, response.Process(&podsReports) } @@ -170,6 +184,8 @@ func Restart(ctx context.Context, nameOrID string, options *RestartOptions) (*en if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } @@ -192,6 +208,8 @@ func Remove(ctx context.Context, nameOrID string, options *RemoveOptions) (*enti if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } @@ -210,6 +228,8 @@ func Start(ctx context.Context, nameOrID string, options *StartOptions) (*entiti if err != nil { return nil, err } + defer response.Body.Close() + if response.StatusCode == http.StatusNotModified { report.Id = nameOrID return &report, nil @@ -236,6 +256,8 @@ func Stop(ctx context.Context, nameOrID string, options *StopOptions) (*entities if err != nil { return nil, err } + defer response.Body.Close() + if response.StatusCode == http.StatusNotModified { report.Id = nameOrID return &report, nil @@ -261,6 +283,7 @@ func Top(ctx context.Context, nameOrID string, options *TopOptions) ([]string, e if err != nil { return nil, err } + defer response.Body.Close() body := handlers.PodTopOKBody{} if err = response.Process(&body); err != nil { @@ -293,6 +316,8 @@ func Unpause(ctx context.Context, nameOrID string, options *UnpauseOptions) (*en if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } @@ -318,5 +343,7 @@ func Stats(ctx context.Context, namesOrIDs []string, options *StatsOptions) ([]* if err != nil { return nil, err } + defer response.Body.Close() + return reports, response.Process(&reports) } diff --git a/pkg/bindings/secrets/secrets.go b/pkg/bindings/secrets/secrets.go index 091d38e56..b741d3e5c 100644 --- a/pkg/bindings/secrets/secrets.go +++ b/pkg/bindings/secrets/secrets.go @@ -22,6 +22,8 @@ func List(ctx context.Context, options *ListOptions) ([]*entities.SecretInfoRepo if err != nil { return secrs, err } + defer response.Body.Close() + return secrs, response.Process(&secrs) } @@ -38,6 +40,8 @@ func Inspect(ctx context.Context, nameOrID string, options *InspectOptions) (*en if err != nil { return inspect, err } + defer response.Body.Close() + return inspect, response.Process(&inspect) } @@ -52,6 +56,8 @@ func Remove(ctx context.Context, nameOrID string) error { if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } @@ -74,5 +80,7 @@ func Create(ctx context.Context, reader io.Reader, options *CreateOptions) (*ent if err != nil { return nil, err } + defer response.Body.Close() + return create, response.Process(&create) } diff --git a/pkg/bindings/system/info.go b/pkg/bindings/system/info.go index 244f9643e..8a307a4ca 100644 --- a/pkg/bindings/system/info.go +++ b/pkg/bindings/system/info.go @@ -9,12 +9,7 @@ import ( ) // Info returns information about the libpod environment and its stores -func Info(ctx context.Context, options *InfoOptions) (*define.Info, error) { - if options == nil { - options = new(InfoOptions) - } - _ = options - info := define.Info{} +func Info(ctx context.Context, _ *InfoOptions) (*define.Info, error) { conn, err := bindings.GetClient(ctx) if err != nil { return nil, err @@ -23,5 +18,8 @@ func Info(ctx context.Context, options *InfoOptions) (*define.Info, error) { if err != nil { return nil, err } + defer response.Body.Close() + + info := define.Info{} return &info, response.Process(&info) } diff --git a/pkg/bindings/system/system.go b/pkg/bindings/system/system.go index 310bcef15..719cde52e 100644 --- a/pkg/bindings/system/system.go +++ b/pkg/bindings/system/system.go @@ -31,6 +31,8 @@ func Events(ctx context.Context, eventChan chan entities.Event, cancelChan chan if err != nil { return err } + defer response.Body.Close() + if cancelChan != nil { go func() { <-cancelChan @@ -75,6 +77,8 @@ func Prune(ctx context.Context, options *PruneOptions) (*entities.SystemPruneRep if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } @@ -101,6 +105,7 @@ func Version(ctx context.Context, options *VersionOptions) (*entities.SystemVers if err != nil { return nil, err } + defer response.Body.Close() if err = response.Process(&component); err != nil { return nil, err @@ -141,5 +146,7 @@ func DiskUsage(ctx context.Context, options *DiskOptions) (*entities.SystemDfRep if err != nil { return nil, err } + defer response.Body.Close() + return &report, response.Process(&report) } diff --git a/pkg/bindings/test/attach_test.go b/pkg/bindings/test/attach_test.go index fbdf18d44..5c3ec48e4 100644 --- a/pkg/bindings/test/attach_test.go +++ b/pkg/bindings/test/attach_test.go @@ -81,10 +81,9 @@ var _ = Describe("Podman containers attach", func() { tickTock := time.NewTimer(2 * time.Second) go func() { <-tickTock.C - timeout := uint(5) - err := containers.Stop(bt.conn, ctnr.ID, new(containers.StopOptions).WithTimeout(timeout)) + err := containers.Stop(bt.conn, ctnr.ID, new(containers.StopOptions).WithTimeout(uint(5))) if err != nil { - GinkgoWriter.Write([]byte(err.Error())) + fmt.Fprint(GinkgoWriter, err.Error()) } }() diff --git a/pkg/bindings/test/common_test.go b/pkg/bindings/test/common_test.go index 9bac4b620..91ebe21fc 100644 --- a/pkg/bindings/test/common_test.go +++ b/pkg/bindings/test/common_test.go @@ -8,6 +8,7 @@ import ( "os/exec" "path/filepath" "strings" + "time" "github.com/containers/podman/v3/libpod/define" . "github.com/containers/podman/v3/pkg/bindings" @@ -150,11 +151,21 @@ func createTempDirInTempDir() (string, error) { } func (b *bindingTest) startAPIService() *gexec.Session { - var ( - cmd []string - ) - cmd = append(cmd, "--log-level=debug", "--events-backend=file", "system", "service", "--timeout=0", b.sock) - return b.runPodman(cmd) + cmd := []string{"--log-level=debug", "--events-backend=file", "system", "service", "--timeout=0", b.sock} + session := b.runPodman(cmd) + + sock := strings.TrimPrefix(b.sock, "unix://") + for i := 0; i < 10; i++ { + if _, err := os.Stat(sock); err != nil { + if !os.IsNotExist(err) { + break + } + time.Sleep(time.Second) + continue + } + break + } + return session } func (b *bindingTest) cleanup() { diff --git a/pkg/bindings/test/resource_test.go b/pkg/bindings/test/resource_test.go new file mode 100644 index 000000000..b12d1ccd6 --- /dev/null +++ b/pkg/bindings/test/resource_test.go @@ -0,0 +1,116 @@ +package test_bindings + +import ( + "context" + "fmt" + "io/fs" + "os" + "os/exec" + "path/filepath" + "reflect" + "strconv" + "syscall" + + "github.com/containers/podman/v3/pkg/bindings" + "github.com/containers/podman/v3/pkg/bindings/containers" + "github.com/containers/podman/v3/pkg/bindings/images" + "github.com/containers/podman/v3/pkg/bindings/pods" + "github.com/containers/podman/v3/pkg/bindings/system" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gexec" +) + +var _ = Describe("Verify Podman resources", func() { + var ( + bt *bindingTest + s *Session + ) + + BeforeEach(func() { + bt = newBindingTest() + s = bt.startAPIService() + err := bt.NewConnection() + Expect(err).ShouldNot(HaveOccurred()) + }) + + AfterEach(func() { + s.Kill() + bt.cleanup() + }) + + It("no leaked connections", func() { + conn, err := bindings.NewConnection(context.Background(), bt.sock) + Expect(err).ShouldNot(HaveOccurred()) + + // Record details on open file descriptors before using API + buffer := lsof() + + // Record open fd from /proc + start, err := readProc() + Expect(err).ShouldNot(HaveOccurred()) + + // Run some operations + _, err = system.Info(conn, nil) + Expect(err).ShouldNot(HaveOccurred()) + _, err = images.List(conn, nil) + Expect(err).ShouldNot(HaveOccurred()) + _, err = containers.List(conn, nil) + Expect(err).ShouldNot(HaveOccurred()) + _, err = pods.List(conn, nil) + Expect(err).ShouldNot(HaveOccurred()) + + podman, _ := bindings.GetClient(conn) + podman.Client.CloseIdleConnections() + + // Record open fd from /proc + finished, err := readProc() + Expect(err).ShouldNot(HaveOccurred()) + if !reflect.DeepEqual(finished, start) { + fmt.Fprintf(GinkgoWriter, "Open FDs:\nlsof Before:\n%s\n", buffer) + + // Record details on open file descriptors after using API + buffer := lsof() + fmt.Fprintf(GinkgoWriter, "lsof After:\n%s\n", buffer) + + // We know test has failed. Easier to let ginkgo format output. + Expect(finished).Should(Equal(start)) + } + }) +}) + +func lsof() string { + lsof := exec.Command("lsof", "+E", "-p", strconv.Itoa(os.Getpid())) + buffer, err := lsof.Output() + Expect(err).ShouldNot(HaveOccurred()) + return string(buffer) +} + +func readProc() ([]string, error) { + syscall.Sync() + + names := make([]string, 0) + err := filepath.WalkDir(fmt.Sprintf("/proc/%d/fd", os.Getpid()), + func(path string, d fs.DirEntry, err error) error { + name := path + " -> " + + switch { + case d.IsDir(): + return nil + case err != nil: + name += err.Error() + case d.Type()&fs.ModeSymlink != 0: + n, err := os.Readlink(path) + if err != nil && !os.IsNotExist(err) { + return err + } + if n == "" { + n = d.Type().String() + } + name += n + } + names = append(names, name) + return nil + }) + return names, err +} diff --git a/pkg/bindings/volumes/volumes.go b/pkg/bindings/volumes/volumes.go index fb58a1d1f..56cf13ade 100644 --- a/pkg/bindings/volumes/volumes.go +++ b/pkg/bindings/volumes/volumes.go @@ -33,6 +33,8 @@ func Create(ctx context.Context, config entities.VolumeCreateOptions, options *C if err != nil { return nil, err } + defer response.Body.Close() + return &v, response.Process(&v) } @@ -53,6 +55,8 @@ func Inspect(ctx context.Context, nameOrID string, options *InspectOptions) (*en if err != nil { return &inspect, err } + defer response.Body.Close() + return &inspect, response.Process(&inspect) } @@ -74,6 +78,8 @@ func List(ctx context.Context, options *ListOptions) ([]*entities.VolumeListRepo if err != nil { return vols, err } + defer response.Body.Close() + return vols, response.Process(&vols) } @@ -94,6 +100,8 @@ func Prune(ctx context.Context, options *PruneOptions) ([]*reports.PruneReport, if err != nil { return nil, err } + defer response.Body.Close() + return pruned, response.Process(&pruned) } @@ -112,6 +120,8 @@ func Remove(ctx context.Context, nameOrID string, options *RemoveOptions) error if err != nil { return err } + defer response.Body.Close() + return response.Process(nil) } @@ -125,5 +135,7 @@ func Exists(ctx context.Context, nameOrID string, options *ExistsOptions) (bool, if err != nil { return false, err } + defer response.Body.Close() + return response.IsSuccess(), nil } diff --git a/pkg/domain/entities/containers.go b/pkg/domain/entities/containers.go index d2a7505a8..607e68256 100644 --- a/pkg/domain/entities/containers.go +++ b/pkg/domain/entities/containers.go @@ -8,6 +8,7 @@ import ( "github.com/containers/image/v5/types" "github.com/containers/podman/v3/libpod/define" + nettypes "github.com/containers/podman/v3/libpod/network/types" "github.com/containers/podman/v3/pkg/specgen" "github.com/containers/storage/pkg/archive" "github.com/cri-o/ocicni/pkg/ocicni" @@ -208,7 +209,7 @@ type RestoreOptions struct { Name string TCPEstablished bool ImportPrevious string - PublishPorts []specgen.PortMapping + PublishPorts []nettypes.PortMapping Pod string } diff --git a/pkg/domain/entities/engine.go b/pkg/domain/entities/engine.go index af996ad1e..a8023f7cf 100644 --- a/pkg/domain/entities/engine.go +++ b/pkg/domain/entities/engine.go @@ -39,6 +39,7 @@ type PodmanConfig struct { EngineMode EngineMode // ABI or Tunneling mode Identity string // ssh identity for connecting to server MaxWorks int // maximum number of parallel threads + MemoryProfile string // Hidden: Should memory profile be taken RegistriesConf string // allows for specifying a custom registries.conf Remote bool // Connection to Podman API Service will use RESTful API RuntimePath string // --runtime flag will set Engine.RuntimePath diff --git a/pkg/domain/entities/engine_container.go b/pkg/domain/entities/engine_container.go index d573e4704..5d3c9480e 100644 --- a/pkg/domain/entities/engine_container.go +++ b/pkg/domain/entities/engine_container.go @@ -92,6 +92,7 @@ type ContainerEngine interface { Version(ctx context.Context) (*SystemVersionReport, error) VolumeCreate(ctx context.Context, opts VolumeCreateOptions) (*IDOrNameResponse, error) VolumeExists(ctx context.Context, namesOrID string) (*BoolReport, error) + VolumeMounted(ctx context.Context, namesOrID string) (*BoolReport, error) VolumeInspect(ctx context.Context, namesOrIds []string, opts InspectOptions) ([]*VolumeInspectReport, []error, error) VolumeList(ctx context.Context, opts VolumeListOptions) ([]*VolumeListReport, error) VolumePrune(ctx context.Context, options VolumePruneOptions) ([]*reports.PruneReport, error) diff --git a/pkg/domain/entities/generate.go b/pkg/domain/entities/generate.go index 3ec713edf..8a437061f 100644 --- a/pkg/domain/entities/generate.go +++ b/pkg/domain/entities/generate.go @@ -35,6 +35,8 @@ type GenerateKubeOptions struct { } // GenerateKubeReport +// +// FIXME: Podman4.0 should change io.Reader to io.ReaderCloser type GenerateKubeReport struct { // Reader - the io.Reader to reader the generated YAML file. Reader io.Reader diff --git a/pkg/domain/entities/play.go b/pkg/domain/entities/play.go index 89dfc08e9..01de73ebe 100644 --- a/pkg/domain/entities/play.go +++ b/pkg/domain/entities/play.go @@ -10,6 +10,8 @@ import ( type PlayKubeOptions struct { // Authfile - path to an authentication file. Authfile string + // Indicator to build all images with Containerfile or Dockerfile + Build bool // CertDir - to a directory containing TLS certifications and keys. CertDir string // Username for authenticating against the registry. diff --git a/pkg/domain/entities/pods.go b/pkg/domain/entities/pods.go index 68e335f8d..c66bf96fc 100644 --- a/pkg/domain/entities/pods.go +++ b/pkg/domain/entities/pods.go @@ -122,6 +122,7 @@ type PodCreateOptions struct { Pid string Cpus float64 CpusetCpus string + Userns specgen.Namespace } type PodCreateReport struct { @@ -217,6 +218,7 @@ func (p *PodCreateOptions) ToPodSpecGen(s *specgen.PodSpecGenerator) error { s.CPUQuota = *cpuDat.Quota } } + s.Userns = p.Userns return nil } diff --git a/pkg/domain/entities/types.go b/pkg/domain/entities/types.go index 9e25b7bf8..db4c6bb8a 100644 --- a/pkg/domain/entities/types.go +++ b/pkg/domain/entities/types.go @@ -6,6 +6,7 @@ import ( buildahDefine "github.com/containers/buildah/define" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/libpod/events" + "github.com/containers/podman/v3/libpod/network/types" "github.com/containers/podman/v3/pkg/specgen" "github.com/containers/storage/pkg/archive" ) @@ -40,7 +41,7 @@ type NetOptions struct { DNSServers []net.IP Network specgen.Namespace NoHosts bool - PublishPorts []specgen.PortMapping + PublishPorts []types.PortMapping StaticIP *net.IP StaticMAC *net.HardwareAddr // NetworkOptions are additional options for each network diff --git a/pkg/domain/filters/containers.go b/pkg/domain/filters/containers.go index dc9fed2a4..269cd2d27 100644 --- a/pkg/domain/filters/containers.go +++ b/pkg/domain/filters/containers.go @@ -214,7 +214,7 @@ func GenerateContainerFilterFuncs(filter string, filterValues []string, r *libpo networkMode := c.NetworkMode() // support docker like `--filter network=container:<IDorName>` // check if networkMode is configured as `container:<ctr>` - // peform a match against filter `container:<IDorName>` + // perform a match against filter `container:<IDorName>` // networks is already going to be empty if `container:<ctr>` is configured as Mode if strings.HasPrefix(networkMode, "container:") { networkModeContainerPart := strings.SplitN(networkMode, ":", 2) diff --git a/pkg/domain/filters/pods.go b/pkg/domain/filters/pods.go index 9a1c7d19d..9a2f0a3ba 100644 --- a/pkg/domain/filters/pods.go +++ b/pkg/domain/filters/pods.go @@ -116,6 +116,17 @@ func GeneratePodFilterFunc(filter string, filterValues []string) ( labels := p.Labels() return util.MatchLabelFilters(filterValues, labels) }, nil + case "until": + return func(p *libpod.Pod) bool { + until, err := util.ComputeUntilTimestamp(filterValues) + if err != nil { + return false + } + if p.CreatedTime().Before(until) { + return true + } + return false + }, nil case "network": return func(p *libpod.Pod) bool { infra, err := p.InfraContainer() diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go index d257bad18..6224feff5 100644 --- a/pkg/domain/infra/abi/play.go +++ b/pkg/domain/infra/abi/play.go @@ -7,9 +7,11 @@ import ( "io" "io/ioutil" "os" + "path/filepath" "strconv" "strings" + buildahDefine "github.com/containers/buildah/define" "github.com/containers/common/libimage" "github.com/containers/common/pkg/config" "github.com/containers/image/v5/types" @@ -266,39 +268,69 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY } containers := make([]*libpod.Container, 0, len(podYAML.Spec.Containers)) + cwd, err := os.Getwd() + if err != nil { + return nil, err + } for _, container := range podYAML.Spec.Containers { // Contains all labels obtained from kube labels := make(map[string]string) - - // NOTE: set the pull policy to "newer". This will cover cases - // where the "latest" tag requires a pull and will also - // transparently handle "localhost/" prefixed files which *may* - // refer to a locally built image OR an image running a - // registry on localhost. - pullPolicy := config.PullPolicyNewer - if len(container.ImagePullPolicy) > 0 { - // Make sure to lower the strings since K8s pull policy - // may be capitalized (see bugzilla.redhat.com/show_bug.cgi?id=1985905). - rawPolicy := string(container.ImagePullPolicy) - pullPolicy, err = config.ParsePullPolicy(strings.ToLower(rawPolicy)) - if err != nil { - return nil, err - } + var pulledImage *libimage.Image + buildFile, err := getBuildFile(container.Image, cwd) + if err != nil { + return nil, err } - // This ensures the image is the image store - pullOptions := &libimage.PullOptions{} - pullOptions.AuthFilePath = options.Authfile - pullOptions.CertDirPath = options.CertDir - pullOptions.SignaturePolicyPath = options.SignaturePolicy - pullOptions.Writer = writer - pullOptions.Username = options.Username - pullOptions.Password = options.Password - pullOptions.InsecureSkipTLSVerify = options.SkipTLSVerify - - pulledImages, err := ic.Libpod.LibimageRuntime().Pull(ctx, container.Image, pullPolicy, pullOptions) + existsLocally, err := ic.Libpod.LibimageRuntime().Exists(container.Image) if err != nil { return nil, err } + if (len(buildFile) > 0 && !existsLocally) || (len(buildFile) > 0 && options.Build) { + buildOpts := new(buildahDefine.BuildOptions) + commonOpts := new(buildahDefine.CommonBuildOptions) + buildOpts.ConfigureNetwork = buildahDefine.NetworkDefault + buildOpts.Isolation = buildahDefine.IsolationChroot + buildOpts.CommonBuildOpts = commonOpts + buildOpts.Output = container.Image + if _, _, err := ic.Libpod.Build(ctx, *buildOpts, []string{buildFile}...); err != nil { + return nil, err + } + i, _, err := ic.Libpod.LibimageRuntime().LookupImage(container.Image, new(libimage.LookupImageOptions)) + if err != nil { + return nil, err + } + pulledImage = i + } else { + // NOTE: set the pull policy to "newer". This will cover cases + // where the "latest" tag requires a pull and will also + // transparently handle "localhost/" prefixed files which *may* + // refer to a locally built image OR an image running a + // registry on localhost. + pullPolicy := config.PullPolicyNewer + if len(container.ImagePullPolicy) > 0 { + // Make sure to lower the strings since K8s pull policy + // may be capitalized (see bugzilla.redhat.com/show_bug.cgi?id=1985905). + rawPolicy := string(container.ImagePullPolicy) + pullPolicy, err = config.ParsePullPolicy(strings.ToLower(rawPolicy)) + if err != nil { + return nil, err + } + } + // This ensures the image is the image store + pullOptions := &libimage.PullOptions{} + pullOptions.AuthFilePath = options.Authfile + pullOptions.CertDirPath = options.CertDir + pullOptions.SignaturePolicyPath = options.SignaturePolicy + pullOptions.Writer = writer + pullOptions.Username = options.Username + pullOptions.Password = options.Password + pullOptions.InsecureSkipTLSVerify = options.SkipTLSVerify + + pulledImages, err := ic.Libpod.LibimageRuntime().Pull(ctx, container.Image, pullPolicy, pullOptions) + if err != nil { + return nil, err + } + pulledImage = pulledImages[0] + } // Handle kube annotations for k, v := range annotations { @@ -318,7 +350,7 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY specgenOpts := kube.CtrSpecGenOptions{ Container: container, - Image: pulledImages[0], + Image: pulledImage, Volumes: volumes, PodID: pod.ID(), PodName: podName, @@ -509,3 +541,48 @@ func sortKubeKinds(documentList [][]byte) ([][]byte, error) { return sortedDocumentList, nil } +func imageNamePrefix(imageName string) string { + prefix := imageName + s := strings.Split(prefix, ":") + if len(s) > 0 { + prefix = s[0] + } + s = strings.Split(prefix, "/") + if len(s) > 0 { + prefix = s[len(s)-1] + } + s = strings.Split(prefix, "@") + if len(s) > 0 { + prefix = s[0] + } + return prefix +} + +func getBuildFile(imageName string, cwd string) (string, error) { + buildDirName := imageNamePrefix(imageName) + containerfilePath := filepath.Join(cwd, buildDirName, "Containerfile") + dockerfilePath := filepath.Join(cwd, buildDirName, "Dockerfile") + + _, err := os.Stat(filepath.Join(containerfilePath)) + if err == nil { + logrus.Debugf("building %s with %s", imageName, containerfilePath) + return containerfilePath, nil + } + // If the error is not because the file does not exist, take + // a mulligan and try Dockerfile. If that also fails, return that + // error + if err != nil && !os.IsNotExist(err) { + logrus.Errorf("%v: unable to check for %s", err, containerfilePath) + } + + _, err = os.Stat(filepath.Join(dockerfilePath)) + if err == nil { + logrus.Debugf("building %s with %s", imageName, dockerfilePath) + return dockerfilePath, nil + } + // Strike two + if os.IsNotExist(err) { + return "", nil + } + return "", err +} diff --git a/pkg/domain/infra/abi/terminal/sigproxy_linux.go b/pkg/domain/infra/abi/terminal/sigproxy_linux.go index 26e199aee..a9bd2d5fb 100644 --- a/pkg/domain/infra/abi/terminal/sigproxy_linux.go +++ b/pkg/domain/infra/abi/terminal/sigproxy_linux.go @@ -12,13 +12,17 @@ import ( "github.com/sirupsen/logrus" ) +// Make sure the signal buffer is sufficiently big. +// runc is using the same value. +const signalBufferSize = 2048 + // ProxySignals ... func ProxySignals(ctr *libpod.Container) { // Stop catching the shutdown signals (SIGINT, SIGTERM) - they're going // to the container now. shutdown.Stop() - sigBuffer := make(chan os.Signal, 128) + sigBuffer := make(chan os.Signal, signalBufferSize) signal.CatchAll(sigBuffer) logrus.Debugf("Enabling signal proxying") diff --git a/pkg/domain/infra/abi/volumes.go b/pkg/domain/infra/abi/volumes.go index e077b10ea..1610c0b48 100644 --- a/pkg/domain/infra/abi/volumes.go +++ b/pkg/domain/infra/abi/volumes.go @@ -162,3 +162,19 @@ func (ic *ContainerEngine) VolumeExists(ctx context.Context, nameOrID string) (* } return &entities.BoolReport{Value: exists}, nil } + +// Volumemounted check if a given volume using plugin or filesystem is mounted or not. +func (ic *ContainerEngine) VolumeMounted(ctx context.Context, nameOrID string) (*entities.BoolReport, error) { + vol, err := ic.Libpod.LookupVolume(nameOrID) + if err != nil { + return nil, err + } + mountCount, err := vol.MountCount() + if err != nil { + return &entities.BoolReport{Value: false}, nil + } + if mountCount > 0 { + return &entities.BoolReport{Value: true}, nil + } + return &entities.BoolReport{Value: false}, nil +} diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go index b638bfe24..81ddce42f 100644 --- a/pkg/domain/infra/tunnel/containers.go +++ b/pkg/domain/infra/tunnel/containers.go @@ -404,11 +404,11 @@ func (ic *ContainerEngine) ContainerLogs(_ context.Context, nameOrIDs []string, return err case line := <-stdoutCh: if opts.StdoutWriter != nil { - _, _ = io.WriteString(opts.StdoutWriter, line+"\n") + _, _ = io.WriteString(opts.StdoutWriter, line) } case line := <-stderrCh: if opts.StderrWriter != nil { - _, _ = io.WriteString(opts.StderrWriter, line+"\n") + _, _ = io.WriteString(opts.StderrWriter, line) } } } diff --git a/pkg/domain/infra/tunnel/generate.go b/pkg/domain/infra/tunnel/generate.go index 0e768b30b..3d3cd52be 100644 --- a/pkg/domain/infra/tunnel/generate.go +++ b/pkg/domain/infra/tunnel/generate.go @@ -16,6 +16,9 @@ func (ic *ContainerEngine) GenerateSystemd(ctx context.Context, nameOrID string, return generate.Systemd(ic.ClientCtx, nameOrID, options) } +// GenerateKube Kubernetes YAML (v1 specification) for nameOrIDs +// +// Note: Caller is responsible for closing returned Reader func (ic *ContainerEngine) GenerateKube(ctx context.Context, nameOrIDs []string, opts entities.GenerateKubeOptions) (*entities.GenerateKubeReport, error) { options := new(generate.KubeOptions).WithService(opts.Service) return generate.Kube(ic.ClientCtx, nameOrIDs, options) diff --git a/pkg/domain/infra/tunnel/volumes.go b/pkg/domain/infra/tunnel/volumes.go index 2d231bad6..2b2b2c2a1 100644 --- a/pkg/domain/infra/tunnel/volumes.go +++ b/pkg/domain/infra/tunnel/volumes.go @@ -91,3 +91,9 @@ func (ic *ContainerEngine) VolumeExists(ctx context.Context, nameOrID string) (* Value: exists, }, nil } + +// Volumemounted check if a given volume using plugin or filesystem is mounted or not. +// TODO: Not used and exposed to tunnel. Will be used by `export` command which is unavailable to `podman-remote` +func (ic *ContainerEngine) VolumeMounted(ctx context.Context, nameOrID string) (*entities.BoolReport, error) { + return nil, errors.New("not implemented") +} diff --git a/pkg/machine/fcos.go b/pkg/machine/fcos.go index 11936aee7..49ec01e67 100644 --- a/pkg/machine/fcos.go +++ b/pkg/machine/fcos.go @@ -3,14 +3,14 @@ package machine import ( - "crypto/sha256" - "io/ioutil" url2 "net/url" + "os" "path/filepath" "runtime" "strings" digest "github.com/opencontainers/go-digest" + "github.com/sirupsen/logrus" ) // These should eventually be moved into machine/qemu as @@ -91,24 +91,23 @@ func UpdateAvailable(d *Download) (bool, error) { // check the sha of the local image if it exists // get the sha of the remote image // == dont bother to pull - files, err := ioutil.ReadDir(filepath.Dir(d.LocalPath)) + if _, err := os.Stat(d.LocalPath); os.IsNotExist(err) { + return false, nil + } + fd, err := os.Open(d.LocalPath) if err != nil { return false, err } - for _, file := range files { - if filepath.Base(d.LocalPath) == file.Name() { - b, err := ioutil.ReadFile(d.LocalPath) - if err != nil { - return false, err - } - s := sha256.Sum256(b) - sum := digest.NewDigestFromBytes(digest.SHA256, s[:]) - if sum.Encoded() == d.Sha256sum { - return true, nil - } + defer func() { + if err := fd.Close(); err != nil { + logrus.Error(err) } + }() + sum, err := digest.SHA256.FromReader(fd) + if err != nil { + return false, err } - return false, nil + return sum.Encoded() == d.Sha256sum, nil } func getFcosArch() string { diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go index 7b1ebcb03..a92892957 100644 --- a/pkg/machine/qemu/machine.go +++ b/pkg/machine/qemu/machine.go @@ -605,10 +605,12 @@ func CheckActiveVM() (bool, string, error) { // startHostNetworking runs a binary on the host system that allows users // to setup port forwarding to the podman virtual machine func (v *MachineVM) startHostNetworking() error { - binary := filepath.Join("/usr/lib/podman/", machine.ForwarderBinaryName) - if _, err := os.Stat(binary); os.IsNotExist(err) { - return errors.Errorf("unable to find %s", binary) + // TODO we may wish to configure the directory in containers common + binary := filepath.Join("/usr/libexec/podman/", machine.ForwarderBinaryName) + if _, err := os.Stat(binary); err != nil { + return err } + // Listen on all at port 7777 for setting up and tearing // down forwarding listenSocket := "tcp://0.0.0.0:7777" diff --git a/pkg/rootless/rootless_linux.go b/pkg/rootless/rootless_linux.go index 9ef56acb4..c046ecde7 100644 --- a/pkg/rootless/rootless_linux.go +++ b/pkg/rootless/rootless_linux.go @@ -397,8 +397,6 @@ func becomeRootInUserNS(pausePid, fileToRead string, fileOutput *os.File) (_ boo return false, -1, errors.Wrapf(err, "error setting up the process") } - c := make(chan os.Signal, 1) - signals := []os.Signal{} for sig := 0; sig < numSig; sig++ { if sig == int(unix.SIGTSTP) { @@ -407,6 +405,7 @@ func becomeRootInUserNS(pausePid, fileToRead string, fileOutput *os.File) (_ boo signals = append(signals, unix.Signal(sig)) } + c := make(chan os.Signal, len(signals)) gosignal.Notify(c, signals...) defer gosignal.Reset() go func() { diff --git a/pkg/rootlessport/rootlessport_linux.go b/pkg/rootlessport/rootlessport_linux.go index ede216bfe..9a2f93f8e 100644 --- a/pkg/rootlessport/rootlessport_linux.go +++ b/pkg/rootlessport/rootlessport_linux.go @@ -20,7 +20,6 @@ import ( "net" "os" "os/exec" - "os/signal" "path/filepath" "github.com/containernetworking/plugins/pkg/ns" @@ -106,30 +105,6 @@ func parent() error { return err } - exitC := make(chan os.Signal, 1) - defer close(exitC) - - go func() { - sigC := make(chan os.Signal, 1) - signal.Notify(sigC, unix.SIGPIPE) - defer func() { - signal.Stop(sigC) - close(sigC) - }() - - select { - case s := <-sigC: - if s == unix.SIGPIPE { - if f, err := os.OpenFile("/dev/null", os.O_WRONLY, 0755); err == nil { - unix.Dup2(int(f.Fd()), 1) // nolint:errcheck - unix.Dup2(int(f.Fd()), 2) // nolint:errcheck - f.Close() - } - } - case <-exitC: - } - }() - socketDir := filepath.Join(cfg.TmpDir, "rp") err = os.MkdirAll(socketDir, 0700) if err != nil { @@ -251,8 +226,16 @@ outer: go serve(socket, driver) } - // write and close ReadyFD (convention is same as slirp4netns --ready-fd) logrus.Info("ready") + + // https://github.com/containers/podman/issues/11248 + // Copy /dev/null to stdout and stderr to prevent SIGPIPE errors + if f, err := os.OpenFile("/dev/null", os.O_WRONLY, 0755); err == nil { + unix.Dup2(int(f.Fd()), 1) // nolint:errcheck + unix.Dup2(int(f.Fd()), 2) // nolint:errcheck + f.Close() + } + // write and close ReadyFD (convention is same as slirp4netns --ready-fd) if _, err := readyW.Write([]byte("1")); err != nil { return err } diff --git a/pkg/specgen/generate/kube/kube.go b/pkg/specgen/generate/kube/kube.go index fb7eb99a2..04b4e5ab3 100644 --- a/pkg/specgen/generate/kube/kube.go +++ b/pkg/specgen/generate/kube/kube.go @@ -12,6 +12,7 @@ import ( "github.com/containers/common/pkg/parse" "github.com/containers/common/pkg/secrets" "github.com/containers/image/v5/manifest" + "github.com/containers/podman/v3/libpod/network/types" ann "github.com/containers/podman/v3/pkg/annotations" "github.com/containers/podman/v3/pkg/specgen" "github.com/containers/podman/v3/pkg/specgen/generate" @@ -303,6 +304,8 @@ func ToSpecGen(ctx context.Context, opts *CtrSpecGenOptions) (*specgen.SpecGener if opts.NetNSIsHost { s.NetNS.NSMode = specgen.Host } + // Always set the userns to host since k8s doesn't have support for userns yet + s.UserNS.NSMode = specgen.Host // Add labels that come from kube if len(s.Labels) == 0 { @@ -586,8 +589,8 @@ func envVarValue(env v1.EnvVar, opts *CtrSpecGenOptions) (string, error) { // getPodPorts converts a slice of kube container descriptions to an // array of portmapping -func getPodPorts(containers []v1.Container) []specgen.PortMapping { - var infraPorts []specgen.PortMapping +func getPodPorts(containers []v1.Container) []types.PortMapping { + var infraPorts []types.PortMapping for _, container := range containers { for _, p := range container.Ports { if p.HostPort != 0 && p.ContainerPort == 0 { @@ -596,7 +599,7 @@ func getPodPorts(containers []v1.Container) []specgen.PortMapping { if p.Protocol == "" { p.Protocol = "tcp" } - portBinding := specgen.PortMapping{ + portBinding := types.PortMapping{ HostPort: uint16(p.HostPort), ContainerPort: uint16(p.ContainerPort), Protocol: strings.ToLower(string(p.Protocol)), diff --git a/pkg/specgen/generate/namespaces.go b/pkg/specgen/generate/namespaces.go index f41186ae4..80790dcc1 100644 --- a/pkg/specgen/generate/namespaces.go +++ b/pkg/specgen/generate/namespaces.go @@ -175,6 +175,11 @@ func namespaceOptions(ctx context.Context, s *specgen.SpecGenerator, rt *libpod. if pod == nil || infraCtr == nil { return nil, errNoInfra } + // Inherit the user from the infra container if it is set and --user has not + // been set explicitly + if infraCtr.User() != "" && s.User == "" { + toReturn = append(toReturn, libpod.WithUser(infraCtr.User())) + } toReturn = append(toReturn, libpod.WithUserNSFrom(infraCtr)) case specgen.FromContainer: userCtr, err := rt.LookupContainer(s.UserNS.Value) @@ -184,7 +189,10 @@ func namespaceOptions(ctx context.Context, s *specgen.SpecGenerator, rt *libpod. toReturn = append(toReturn, libpod.WithUserNSFrom(userCtr)) } - if s.IDMappings != nil { + // This wipes the UserNS settings that get set from the infra container + // when we are inheritting from the pod. So only apply this if the container + // is not being created in a pod. + if s.IDMappings != nil && pod == nil { toReturn = append(toReturn, libpod.WithIDMappings(*s.IDMappings)) } if s.User != "" { @@ -234,7 +242,7 @@ func namespaceOptions(ctx context.Context, s *specgen.SpecGenerator, rt *libpod. } toReturn = append(toReturn, libpod.WithNetNSFrom(netCtr)) case specgen.Slirp: - portMappings, err := createPortMappings(ctx, s, imageData) + portMappings, expose, err := createPortMappings(ctx, s, imageData) if err != nil { return nil, err } @@ -242,15 +250,15 @@ func namespaceOptions(ctx context.Context, s *specgen.SpecGenerator, rt *libpod. if s.NetNS.Value != "" { val = fmt.Sprintf("slirp4netns:%s", s.NetNS.Value) } - toReturn = append(toReturn, libpod.WithNetNS(portMappings, postConfigureNetNS, val, nil)) + toReturn = append(toReturn, libpod.WithNetNS(portMappings, expose, postConfigureNetNS, val, nil)) case specgen.Private: fallthrough case specgen.Bridge: - portMappings, err := createPortMappings(ctx, s, imageData) + portMappings, expose, err := createPortMappings(ctx, s, imageData) if err != nil { return nil, err } - toReturn = append(toReturn, libpod.WithNetNS(portMappings, postConfigureNetNS, "bridge", s.CNINetworks)) + toReturn = append(toReturn, libpod.WithNetNS(portMappings, expose, postConfigureNetNS, "bridge", s.CNINetworks)) } if s.UseImageHosts { @@ -379,46 +387,8 @@ func specConfigureNamespaces(s *specgen.SpecGenerator, g *generate.Generator, rt } // User - switch s.UserNS.NSMode { - case specgen.Path: - if _, err := os.Stat(s.UserNS.Value); err != nil { - return errors.Wrap(err, "cannot find specified user namespace path") - } - if err := g.AddOrReplaceLinuxNamespace(string(spec.UserNamespace), s.UserNS.Value); err != nil { - return err - } - // runc complains if no mapping is specified, even if we join another ns. So provide a dummy mapping - g.AddLinuxUIDMapping(uint32(0), uint32(0), uint32(1)) - g.AddLinuxGIDMapping(uint32(0), uint32(0), uint32(1)) - case specgen.Host: - if err := g.RemoveLinuxNamespace(string(spec.UserNamespace)); err != nil { - return err - } - case specgen.KeepID: - var ( - err error - uid, gid int - ) - s.IDMappings, uid, gid, err = util.GetKeepIDMapping() - if err != nil { - return err - } - g.SetProcessUID(uint32(uid)) - g.SetProcessGID(uint32(gid)) - fallthrough - case specgen.Private: - if err := g.AddOrReplaceLinuxNamespace(string(spec.UserNamespace), ""); err != nil { - return err - } - if s.IDMappings == nil || (len(s.IDMappings.UIDMap) == 0 && len(s.IDMappings.GIDMap) == 0) { - return errors.Errorf("must provide at least one UID or GID mapping to configure a user namespace") - } - for _, uidmap := range s.IDMappings.UIDMap { - g.AddLinuxUIDMapping(uint32(uidmap.HostID), uint32(uidmap.ContainerID), uint32(uidmap.Size)) - } - for _, gidmap := range s.IDMappings.GIDMap { - g.AddLinuxGIDMapping(uint32(gidmap.HostID), uint32(gidmap.ContainerID), uint32(gidmap.Size)) - } + if _, err := specgen.SetupUserNS(s.IDMappings, s.UserNS, g); err != nil { + return err } // Cgroup @@ -474,7 +444,7 @@ func specConfigureNamespaces(s *specgen.SpecGenerator, g *generate.Generator, rt // GetNamespaceOptions transforms a slice of kernel namespaces // into a slice of pod create options. Currently, not all // kernel namespaces are supported, and they will be returned in an error -func GetNamespaceOptions(ns []string) ([]libpod.PodCreateOption, error) { +func GetNamespaceOptions(ns []string, netnsIsHost bool) ([]libpod.PodCreateOption, error) { var options []libpod.PodCreateOption var erroredOptions []libpod.PodCreateOption if ns == nil { @@ -486,7 +456,10 @@ func GetNamespaceOptions(ns []string) ([]libpod.PodCreateOption, error) { case "cgroup": options = append(options, libpod.WithPodCgroups()) case "net": - options = append(options, libpod.WithPodNet()) + // share the netns setting with other containers in the pod only when it is not set to host + if !netnsIsHost { + options = append(options, libpod.WithPodNet()) + } case "mnt": return erroredOptions, errors.Errorf("Mount sharing functionality not supported on pod level") case "pid": diff --git a/pkg/specgen/generate/pod_create.go b/pkg/specgen/generate/pod_create.go index aab29499e..426cf1b6d 100644 --- a/pkg/specgen/generate/pod_create.go +++ b/pkg/specgen/generate/pod_create.go @@ -27,11 +27,16 @@ func createPodOptions(p *specgen.PodSpecGenerator, rt *libpod.Runtime) ([]libpod ) if !p.NoInfra { options = append(options, libpod.WithInfraContainer()) - nsOptions, err := GetNamespaceOptions(p.SharedNamespaces) + nsOptions, err := GetNamespaceOptions(p.SharedNamespaces, p.NetNS.IsHost()) if err != nil { return nil, err } options = append(options, nsOptions...) + // Use pod user and infra userns only when --userns is not set to host + if !p.Userns.IsHost() { + options = append(options, libpod.WithPodUser()) + options = append(options, libpod.WithPodUserns(p.Userns)) + } // Make our exit command storageConfig := rt.StorageConfig() @@ -154,5 +159,6 @@ func createPodOptions(p *specgen.PodSpecGenerator, rt *libpod.Runtime) ([]libpod if len(p.InfraConmonPidFile) > 0 { options = append(options, libpod.WithInfraConmonPidFile(p.InfraConmonPidFile)) } + return options, nil } diff --git a/pkg/specgen/generate/ports.go b/pkg/specgen/generate/ports.go index c00ad19fb..a300f8014 100644 --- a/pkg/specgen/generate/ports.go +++ b/pkg/specgen/generate/ports.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/containers/common/libimage" + "github.com/containers/podman/v3/libpod/network/types" "github.com/containers/podman/v3/utils" "github.com/containers/podman/v3/pkg/specgen" @@ -24,7 +25,7 @@ const ( // Parse port maps to OCICNI port mappings. // Returns a set of OCICNI port mappings, and maps of utilized container and // host ports. -func ParsePortMapping(portMappings []specgen.PortMapping) ([]ocicni.PortMapping, map[string]map[string]map[uint16]uint16, map[string]map[string]map[uint16]uint16, error) { +func ParsePortMapping(portMappings []types.PortMapping) ([]ocicni.PortMapping, map[string]map[string]map[uint16]uint16, map[string]map[string]map[uint16]uint16, error) { // First, we need to validate the ports passed in the specgen, and then // convert them into CNI port mappings. type tempMapping struct { @@ -253,17 +254,15 @@ func ParsePortMapping(portMappings []specgen.PortMapping) ([]ocicni.PortMapping, } // Make final port mappings for the container -func createPortMappings(ctx context.Context, s *specgen.SpecGenerator, imageData *libimage.ImageData) ([]ocicni.PortMapping, error) { +func createPortMappings(ctx context.Context, s *specgen.SpecGenerator, imageData *libimage.ImageData) ([]ocicni.PortMapping, map[uint16][]string, error) { finalMappings, containerPortValidate, hostPortValidate, err := ParsePortMapping(s.PortMappings) if err != nil { - return nil, err + return nil, nil, err } - // If not publishing exposed ports, or if we are publishing and there is - // nothing to publish - then just return the port mappings we've made so - // far. - if !s.PublishExposedPorts || (len(s.Expose) == 0 && imageData == nil) { - return finalMappings, nil + // No exposed ports so return the port mappings we've made so far. + if len(s.Expose) == 0 && imageData == nil { + return finalMappings, nil, nil } logrus.Debugf("Adding exposed ports") @@ -272,7 +271,7 @@ func createPortMappings(ctx context.Context, s *specgen.SpecGenerator, imageData if imageData != nil { expose, err = GenExposedPorts(imageData.Config.ExposedPorts) if err != nil { - return nil, err + return nil, nil, err } } @@ -288,11 +287,11 @@ func createPortMappings(ctx context.Context, s *specgen.SpecGenerator, imageData // Validate protocol first protocols, err := checkProtocol(proto, false) if err != nil { - return nil, errors.Wrapf(err, "error validating protocols for exposed port %d", port) + return nil, nil, errors.Wrapf(err, "error validating protocols for exposed port %d", port) } if port == 0 { - return nil, errors.Errorf("cannot expose 0 as it is not a valid port number") + return nil, nil, errors.Errorf("cannot expose 0 as it is not a valid port number") } // Check to see if the port is already present in existing @@ -316,6 +315,11 @@ func createPortMappings(ctx context.Context, s *specgen.SpecGenerator, imageData } } + // If not publishing exposed ports return mappings and exposed ports. + if !s.PublishExposedPorts { + return finalMappings, toExpose, nil + } + // We now have a final list of ports that we want exposed. // Let's find empty, unallocated host ports for them. for port, protocols := range toExpose { @@ -331,7 +335,7 @@ func createPortMappings(ctx context.Context, s *specgen.SpecGenerator, imageData // unfortunate for the UDP case. candidate, err := utils.GetRandomPort() if err != nil { - return nil, err + return nil, nil, err } // Check if the host port is already bound @@ -362,12 +366,12 @@ func createPortMappings(ctx context.Context, s *specgen.SpecGenerator, imageData } if tries == 0 && hostPort == 0 { // We failed to find an open port. - return nil, errors.Errorf("failed to find an open port to expose container port %d on the host", port) + return nil, nil, errors.Errorf("failed to find an open port to expose container port %d on the host", port) } } } - return finalMappings, nil + return finalMappings, nil, nil } // Check a string to ensure it is a comma-separated set of valid protocols diff --git a/pkg/specgen/generate/storage.go b/pkg/specgen/generate/storage.go index 13f336594..de655ad7d 100644 --- a/pkg/specgen/generate/storage.go +++ b/pkg/specgen/generate/storage.go @@ -10,6 +10,7 @@ import ( "github.com/containers/common/libimage" "github.com/containers/common/pkg/config" + "github.com/containers/common/pkg/parse" "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/specgen" @@ -59,6 +60,9 @@ func finalizeMounts(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Ru for _, m := range s.Mounts { // Ensure that mount dest is clean, so that it can be // compared against named volumes and avoid duplicate mounts. + if err = parse.ValidateVolumeCtrDir(m.Destination); err != nil { + return nil, nil, nil, err + } cleanDestination := filepath.Clean(m.Destination) if _, ok := unifiedMounts[cleanDestination]; ok { return nil, nil, nil, errors.Wrapf(errDuplicateDest, "conflict in specified mounts - multiple mounts at %q", cleanDestination) @@ -67,34 +71,54 @@ func finalizeMounts(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Ru } for _, m := range commonMounts { - if _, ok := unifiedMounts[m.Destination]; !ok { - unifiedMounts[m.Destination] = m + if err = parse.ValidateVolumeCtrDir(m.Destination); err != nil { + return nil, nil, nil, err + } + cleanDestination := filepath.Clean(m.Destination) + if _, ok := unifiedMounts[cleanDestination]; !ok { + unifiedMounts[cleanDestination] = m } } for _, v := range s.Volumes { - if _, ok := unifiedVolumes[v.Dest]; ok { - return nil, nil, nil, errors.Wrapf(errDuplicateDest, "conflict in specified volumes - multiple volumes at %q", v.Dest) + if err = parse.ValidateVolumeCtrDir(v.Dest); err != nil { + return nil, nil, nil, err } - unifiedVolumes[v.Dest] = v + cleanDestination := filepath.Clean(v.Dest) + if _, ok := unifiedVolumes[cleanDestination]; ok { + return nil, nil, nil, errors.Wrapf(errDuplicateDest, "conflict in specified volumes - multiple volumes at %q", cleanDestination) + } + unifiedVolumes[cleanDestination] = v } for _, v := range commonVolumes { - if _, ok := unifiedVolumes[v.Dest]; !ok { - unifiedVolumes[v.Dest] = v + if err = parse.ValidateVolumeCtrDir(v.Dest); err != nil { + return nil, nil, nil, err + } + cleanDestination := filepath.Clean(v.Dest) + if _, ok := unifiedVolumes[cleanDestination]; !ok { + unifiedVolumes[cleanDestination] = v } } for _, v := range s.OverlayVolumes { - if _, ok := unifiedOverlays[v.Destination]; ok { - return nil, nil, nil, errors.Wrapf(errDuplicateDest, "conflict in specified volumes - multiple volumes at %q", v.Destination) + if err = parse.ValidateVolumeCtrDir(v.Destination); err != nil { + return nil, nil, nil, err } - unifiedOverlays[v.Destination] = v + cleanDestination := filepath.Clean(v.Destination) + if _, ok := unifiedOverlays[cleanDestination]; ok { + return nil, nil, nil, errors.Wrapf(errDuplicateDest, "conflict in specified volumes - multiple volumes at %q", cleanDestination) + } + unifiedOverlays[cleanDestination] = v } for _, v := range commonOverlayVolumes { - if _, ok := unifiedOverlays[v.Destination]; ok { - unifiedOverlays[v.Destination] = v + if err = parse.ValidateVolumeCtrDir(v.Destination); err != nil { + return nil, nil, nil, err + } + cleanDestination := filepath.Clean(v.Destination) + if _, ok := unifiedOverlays[cleanDestination]; !ok { + unifiedOverlays[cleanDestination] = v } } @@ -190,6 +214,9 @@ func getImageVolumes(ctx context.Context, img *libimage.Image, s *specgen.SpecGe } for volume := range inspect.Config.Volumes { logrus.Debugf("Image has volume at %q", volume) + if err = parse.ValidateVolumeCtrDir(volume); err != nil { + return nil, nil, err + } cleanDest := filepath.Clean(volume) switch mode { case "", "anonymous": @@ -304,9 +331,13 @@ func getVolumesFrom(volumesFrom []string, runtime *libpod.Runtime) (map[string]s if _, ok := finalMounts[namedVol.Dest]; ok { logrus.Debugf("Overriding named volume mount to %s with new named volume from container %s", namedVol.Dest, ctr.ID()) } + if err = parse.ValidateVolumeCtrDir(namedVol.Dest); err != nil { + return nil, nil, err + } + cleanDest := filepath.Clean(namedVol.Dest) newVol := new(specgen.NamedVolume) - newVol.Dest = namedVol.Dest + newVol.Dest = cleanDest newVol.Options = namedVol.Options newVol.Name = namedVol.Name diff --git a/pkg/specgen/namespaces.go b/pkg/specgen/namespaces.go index 76fa66bc7..2f4c48811 100644 --- a/pkg/specgen/namespaces.go +++ b/pkg/specgen/namespaces.go @@ -1,10 +1,16 @@ package specgen import ( + "fmt" + "os" "strings" "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/rootless" + "github.com/containers/podman/v3/pkg/util" + "github.com/containers/storage" + spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-tools/generate" "github.com/pkg/errors" ) @@ -103,6 +109,13 @@ func (n *Namespace) IsKeepID() bool { return n.NSMode == KeepID } +func (n *Namespace) String() string { + if n.Value != "" { + return fmt.Sprintf("%s:%s", n.NSMode, n.Value) + } + return string(n.NSMode) +} + func validateUserNS(n *Namespace) error { if n == nil { return nil @@ -323,3 +336,48 @@ func ParseNetworkString(network string) (Namespace, []string, map[string][]strin } return ns, cniNets, networkOptions, nil } + +func SetupUserNS(idmappings *storage.IDMappingOptions, userns Namespace, g *generate.Generator) (string, error) { + // User + var user string + switch userns.NSMode { + case Path: + if _, err := os.Stat(userns.Value); err != nil { + return user, errors.Wrap(err, "cannot find specified user namespace path") + } + if err := g.AddOrReplaceLinuxNamespace(string(spec.UserNamespace), userns.Value); err != nil { + return user, err + } + // runc complains if no mapping is specified, even if we join another ns. So provide a dummy mapping + g.AddLinuxUIDMapping(uint32(0), uint32(0), uint32(1)) + g.AddLinuxGIDMapping(uint32(0), uint32(0), uint32(1)) + case Host: + if err := g.RemoveLinuxNamespace(string(spec.UserNamespace)); err != nil { + return user, err + } + case KeepID: + mappings, uid, gid, err := util.GetKeepIDMapping() + if err != nil { + return user, err + } + idmappings = mappings + g.SetProcessUID(uint32(uid)) + g.SetProcessGID(uint32(gid)) + user = fmt.Sprintf("%d:%d", uid, gid) + fallthrough + case Private: + if err := g.AddOrReplaceLinuxNamespace(string(spec.UserNamespace), ""); err != nil { + return user, err + } + if idmappings == nil || (len(idmappings.UIDMap) == 0 && len(idmappings.GIDMap) == 0) { + return user, errors.Errorf("must provide at least one UID or GID mapping to configure a user namespace") + } + for _, uidmap := range idmappings.UIDMap { + g.AddLinuxUIDMapping(uint32(uidmap.HostID), uint32(uidmap.ContainerID), uint32(uidmap.Size)) + } + for _, gidmap := range idmappings.GIDMap { + g.AddLinuxGIDMapping(uint32(gidmap.HostID), uint32(gidmap.ContainerID), uint32(gidmap.Size)) + } + } + return user, nil +} diff --git a/pkg/specgen/podspecgen.go b/pkg/specgen/podspecgen.go index 02237afe9..386571d11 100644 --- a/pkg/specgen/podspecgen.go +++ b/pkg/specgen/podspecgen.go @@ -3,6 +3,7 @@ package specgen import ( "net" + "github.com/containers/podman/v3/libpod/network/types" spec "github.com/opencontainers/runtime-spec/specs-go" ) @@ -67,6 +68,10 @@ type PodBasicConfig struct { // Optional (defaults to private if unset). This sets the PID namespace of the infra container // This configuration will then be shared with the entire pod if PID namespace sharing is enabled via --share Pid Namespace `json:"pid,omitempty:"` + // Userns is used to indicate which kind of Usernamespace to enter. + // Any containers created within the pod will inherit the pod's userns settings. + // Optional + Userns Namespace `json:"userns,omitempty"` } // PodNetworkConfig contains networking configuration for a pod. @@ -98,7 +103,7 @@ type PodNetworkConfig struct { // container, this will forward the ports to the entire pod. // Only available if NetNS is set to Bridge or Slirp. // Optional. - PortMappings []PortMapping `json:"portmappings,omitempty"` + PortMappings []types.PortMapping `json:"portmappings,omitempty"` // CNINetworks is a list of CNI networks that the infra container will // join. As, by default, containers share their network with the infra // container, these networks will effectively be joined by the diff --git a/pkg/specgen/specgen.go b/pkg/specgen/specgen.go index fc647227e..0c30c498a 100644 --- a/pkg/specgen/specgen.go +++ b/pkg/specgen/specgen.go @@ -5,6 +5,7 @@ import ( "syscall" "github.com/containers/image/v5/manifest" + nettypes "github.com/containers/podman/v3/libpod/network/types" "github.com/containers/storage/types" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -184,7 +185,7 @@ type ContainerBasicConfig struct { // Optional. EnvSecrets map[string]string `json:"secret_env,omitempty"` // InitContainerType describes if this container is an init container - // and if so, what type: always or oneshot + // and if so, what type: always or once InitContainerType string `json:"init_container_type"` // Personality allows users to configure different execution domains. // Execution domains tell Linux how to map signal numbers into signal actions. @@ -393,7 +394,7 @@ type ContainerNetworkConfig struct { // PortBindings is a set of ports to map into the container. // Only available if NetNS is set to bridge or slirp. // Optional. - PortMappings []PortMapping `json:"portmappings,omitempty"` + PortMappings []nettypes.PortMapping `json:"portmappings,omitempty"` // PublishExposedPorts will publish ports specified in the image to // random unused ports (guaranteed to be above 1024) on the host. // This is based on ports set in Expose below, and any ports specified @@ -506,36 +507,6 @@ type SpecGenerator struct { ContainerHealthCheckConfig } -// PortMapping is one or more ports that will be mapped into the container. -type PortMapping struct { - // HostIP is the IP that we will bind to on the host. - // If unset, assumed to be 0.0.0.0 (all interfaces). - HostIP string `json:"host_ip,omitempty"` - // ContainerPort is the port number that will be exposed from the - // container. - // Mandatory. - ContainerPort uint16 `json:"container_port"` - // HostPort is the port number that will be forwarded from the host into - // the container. - // If omitted, a random port on the host (guaranteed to be over 1024) - // will be assigned. - HostPort uint16 `json:"host_port,omitempty"` - // Range is the number of ports that will be forwarded, starting at - // HostPort and ContainerPort and counting up. - // This is 1-indexed, so 1 is assumed to be a single port (only the - // Hostport:Containerport mapping will be added), 2 is two ports (both - // Hostport:Containerport and Hostport+1:Containerport+1), etc. - // If unset, assumed to be 1 (a single port). - // Both hostport + range and containerport + range must be less than - // 65536. - Range uint16 `json:"range,omitempty"` - // Protocol is the protocol forward. - // Must be either "tcp", "udp", and "sctp", or some combination of these - // separated by commas. - // If unset, assumed to be TCP. - Protocol string `json:"protocol,omitempty"` -} - type Secret struct { Source string UID uint32 diff --git a/pkg/specgen/volumes.go b/pkg/specgen/volumes.go index d85d2bdd1..eca8c0c35 100644 --- a/pkg/specgen/volumes.go +++ b/pkg/specgen/volumes.go @@ -1,7 +1,6 @@ package specgen import ( - "path/filepath" "strings" "github.com/containers/common/pkg/parse" @@ -93,11 +92,6 @@ func GenVolumeMounts(volumeFlag []string) (map[string]spec.Mount, map[string]*Na return nil, nil, nil, errors.New("host directory cannot be empty") } } - if err := parse.ValidateVolumeCtrDir(dest); err != nil { - return nil, nil, nil, err - } - - cleanDest := filepath.Clean(dest) if strings.HasPrefix(src, "/") || strings.HasPrefix(src, ".") { // This is not a named volume @@ -120,7 +114,7 @@ func GenVolumeMounts(volumeFlag []string) (map[string]spec.Mount, map[string]*Na if overlayFlag { // This is a overlay volume newOverlayVol := new(OverlayVolume) - newOverlayVol.Destination = cleanDest + newOverlayVol.Destination = dest newOverlayVol.Source = src newOverlayVol.Options = options @@ -130,7 +124,7 @@ func GenVolumeMounts(volumeFlag []string) (map[string]spec.Mount, map[string]*Na overlayVolumes[newOverlayVol.Destination] = newOverlayVol } else { newMount := spec.Mount{ - Destination: cleanDest, + Destination: dest, Type: "bind", Source: src, Options: options, @@ -144,7 +138,7 @@ func GenVolumeMounts(volumeFlag []string) (map[string]spec.Mount, map[string]*Na // This is a named volume newNamedVol := new(NamedVolume) newNamedVol.Name = src - newNamedVol.Dest = cleanDest + newNamedVol.Dest = dest newNamedVol.Options = options if _, ok := volumes[newNamedVol.Dest]; ok { diff --git a/pkg/systemd/dbus.go b/pkg/systemd/dbus.go index 718082526..c49f537b6 100644 --- a/pkg/systemd/dbus.go +++ b/pkg/systemd/dbus.go @@ -9,8 +9,106 @@ import ( "github.com/containers/podman/v3/pkg/rootless" "github.com/coreos/go-systemd/v22/dbus" godbus "github.com/godbus/dbus/v5" + "github.com/sirupsen/logrus" ) +// IsSystemdSessionValid checks if sessions is valid for provided rootless uid. +func IsSystemdSessionValid(uid int) bool { + var conn *godbus.Conn + var err error + var object godbus.BusObject + var seat0Path godbus.ObjectPath + dbusDest := "org.freedesktop.login1" + dbusInterface := "org.freedesktop.login1.Manager" + dbusPath := "/org/freedesktop/login1" + + if rootless.IsRootless() { + conn, err = GetLogindConnection(rootless.GetRootlessUID()) + object = conn.Object(dbusDest, godbus.ObjectPath(dbusPath)) + if err != nil { + //unable to fetch systemd object for logind + logrus.Debugf("systemd-logind: %s", err) + return false + } + object = conn.Object(dbusDest, godbus.ObjectPath(dbusPath)) + if err := object.Call(dbusInterface+".GetSeat", 0, "seat0").Store(&seat0Path); err != nil { + //unable to get seat0 path. + logrus.Debugf("systemd-logind: %s", err) + return false + } + seat0Obj := conn.Object(dbusDest, seat0Path) + activeSession, err := seat0Obj.GetProperty(dbusDest + ".Seat.ActiveSession") + if err != nil { + //unable to get active sessions. + logrus.Debugf("systemd-logind: %s", err) + return false + } + activeSessionMap, ok := activeSession.Value().([]interface{}) + if !ok || len(activeSessionMap) < 2 { + //unable to get active session map. + logrus.Debugf("systemd-logind: %s", err) + return false + } + activeSessionPath, ok := activeSessionMap[1].(godbus.ObjectPath) + if !ok { + //unable to fetch active session path. + logrus.Debugf("systemd-logind: %s", err) + return false + } + activeSessionObj := conn.Object(dbusDest, activeSessionPath) + sessionUser, err := activeSessionObj.GetProperty(dbusDest + ".Session.User") + if err != nil { + //unable to fetch session user from activeSession path. + logrus.Debugf("systemd-logind: %s", err) + return false + } + dbusUser, ok := sessionUser.Value().([]interface{}) + if !ok { + // not a valid user. + return false + } + if len(dbusUser) < 2 { + // not a valid session user. + return false + } + activeUID, ok := dbusUser[0].(uint32) + if !ok { + return false + } + //active session found which belongs to following rootless user + if activeUID == uint32(uid) { + return true + } + return false + } + return true +} + +// GetDbusConnection returns an user connection to D-BUS +func GetLogindConnection(uid int) (*godbus.Conn, error) { + return dbusAuthConnectionLogind(uid) +} + +func dbusAuthConnectionLogind(uid int) (*godbus.Conn, error) { + var conn *godbus.Conn + var err error + conn, err = godbus.SystemBusPrivate() + if err != nil { + return nil, err + } + methods := []godbus.Auth{godbus.AuthExternal(strconv.Itoa(uid))} + if err = conn.Auth(methods); err != nil { + conn.Close() + return nil, err + } + err = conn.Hello() + if err != nil { + conn.Close() + return nil, err + } + return conn, nil +} + func dbusAuthRootlessConnection(createBus func(opts ...godbus.ConnOption) (*godbus.Conn, error)) (*godbus.Conn, error) { conn, err := createBus() if err != nil { diff --git a/pkg/systemd/generate/common.go b/pkg/systemd/generate/common.go index 45e12014a..49465fb30 100644 --- a/pkg/systemd/generate/common.go +++ b/pkg/systemd/generate/common.go @@ -71,11 +71,12 @@ func filterCommonContainerFlags(command []string, argCount int) []string { case s == "--rm": // Boolean flags support --flag and --flag={true,false}. continue - case s == "--sdnotify", s == "--cgroups": + case s == "--sdnotify", s == "--cgroups", s == "--cidfile": i++ continue case strings.HasPrefix(s, "--rm="), - strings.HasPrefix(s, "--cgroups="): + strings.HasPrefix(s, "--cgroups="), + strings.HasPrefix(s, "--cidfile="): continue } processed = append(processed, s) diff --git a/pkg/systemd/generate/common_test.go b/pkg/systemd/generate/common_test.go index 3e2ac015f..80abebb26 100644 --- a/pkg/systemd/generate/common_test.go +++ b/pkg/systemd/generate/common_test.go @@ -103,12 +103,12 @@ func TestFilterCommonContainerFlags(t *testing.T) { }, { []string{"podman", "run", "--cidfile", "foo", "alpine"}, - []string{"podman", "run", "--cidfile", "foo", "alpine"}, + []string{"podman", "run", "alpine"}, 1, }, { []string{"podman", "run", "--cidfile=foo", "alpine"}, - []string{"podman", "run", "--cidfile=foo", "alpine"}, + []string{"podman", "run", "alpine"}, 1, }, { diff --git a/pkg/systemd/generate/containers.go b/pkg/systemd/generate/containers.go index 78b81b54b..931f13972 100644 --- a/pkg/systemd/generate/containers.go +++ b/pkg/systemd/generate/containers.go @@ -233,9 +233,10 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst info.Type = "notify" info.NotifyAccess = "all" info.PIDFile = "" - info.ContainerIDFile = "" - info.ExecStop = "" - info.ExecStopPost = "" + info.ContainerIDFile = "%t/%n.ctr-id" + info.ExecStartPre = "/bin/rm -f {{{{.ContainerIDFile}}}}" + info.ExecStop = "{{{{.Executable}}}} stop --ignore --cidfile={{{{.ContainerIDFile}}}}" + info.ExecStopPost = "{{{{.Executable}}}} rm -f --ignore --cidfile={{{{.ContainerIDFile}}}}" // The create command must at least have three arguments: // /usr/bin/podman run $IMAGE index := 0 @@ -258,6 +259,7 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst } startCommand = append(startCommand, "run", + "--cidfile={{{{.ContainerIDFile}}}}", "--cgroups=no-conmon", "--rm", ) diff --git a/pkg/systemd/generate/containers_test.go b/pkg/systemd/generate/containers_test.go index 6141950d0..c60c301cc 100644 --- a/pkg/systemd/generate/containers_test.go +++ b/pkg/systemd/generate/containers_test.go @@ -130,7 +130,10 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=70 -ExecStart=/usr/bin/podman container run --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN "foo=arg \"with \" space" +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman container run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN "foo=arg \"with \" space" +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -152,7 +155,10 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=70 -ExecStart=/usr/bin/podman container run --cgroups=no-conmon --rm -d --replace --sdnotify=container --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN "foo=arg \"with \" space" +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman container run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm -d --replace --sdnotify=container --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN "foo=arg \"with \" space" +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -174,7 +180,10 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=70 -ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon --replace -d --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --sdnotify=conmon --replace -d --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -196,7 +205,10 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=70 -ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --pod-id-file %t/pod-foobar.pod-id-file --sdnotify=conmon --replace -d --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --pod-id-file %t/pod-foobar.pod-id-file --sdnotify=conmon --replace -d --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -218,7 +230,10 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=70 -ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon --replace --detach --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --sdnotify=conmon --replace --detach --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -240,7 +255,10 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=70 -ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d awesome-image:latest +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --sdnotify=conmon -d awesome-image:latest +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -263,9 +281,12 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=102 -ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon ` + +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --sdnotify=conmon ` + detachparam + ` awesome-image:latest +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -289,7 +310,10 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=102 -ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name test -p 80:80 awesome-image:latest somecmd --detach=false +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name test -p 80:80 awesome-image:latest somecmd --detach=false +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -311,7 +335,10 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=102 -ExecStart=/usr/bin/podman --events-backend none --runroot /root run --cgroups=no-conmon --rm --sdnotify=conmon -d awesome-image:latest +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman --events-backend none --runroot /root run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --sdnotify=conmon -d awesome-image:latest +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -333,7 +360,10 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=70 -ExecStart=/usr/bin/podman container run --cgroups=no-conmon --rm --sdnotify=conmon -d awesome-image:latest +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman container run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --sdnotify=conmon -d awesome-image:latest +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -355,7 +385,10 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=70 -ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name test --log-driver=journald --log-opt=tag={{.Name}} awesome-image:latest +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name test --log-driver=journald --log-opt=tag={{.Name}} awesome-image:latest +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -377,7 +410,10 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=70 -ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name test awesome-image:latest sh -c "kill $$$$ && echo %%\\" +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name test awesome-image:latest sh -c "kill $$$$ && echo %%\\" +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -399,7 +435,10 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=70 -ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --conmon-pidfile=foo --cidfile=foo awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo alpine +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --sdnotify=conmon -d --conmon-pidfile=foo awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo alpine +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -421,7 +460,10 @@ RequiresMountsFor=/var/run/containers/storage Environment=PODMAN_SYSTEMD_UNIT=%n Restart=always TimeoutStopSec=70 -ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --pod-id-file %t/pod-foobar.pod-id-file --sdnotify=conmon -d --conmon-pidfile=foo --cidfile=foo awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo --pod-id-file /tmp/pod-foobar.pod-id-file alpine +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --pod-id-file %t/pod-foobar.pod-id-file --sdnotify=conmon -d --conmon-pidfile=foo awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo --pod-id-file /tmp/pod-foobar.pod-id-file alpine +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all @@ -444,7 +486,10 @@ Environment=PODMAN_SYSTEMD_UNIT=%n Environment=FOO=abc "BAR=my test" USER=%%a Restart=always TimeoutStopSec=70 -ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --env FOO --env=BAR --env=MYENV=2 -e USER awesome-image:latest +ExecStartPre=/bin/rm -f %t/%n.ctr-id +ExecStart=/usr/bin/podman run --cidfile=%t/%n.ctr-id --cgroups=no-conmon --rm --sdnotify=conmon -d --env FOO --env=BAR --env=MYENV=2 -e USER awesome-image:latest +ExecStop=/usr/bin/podman stop --ignore --cidfile=%t/%n.ctr-id +ExecStopPost=/usr/bin/podman rm -f --ignore --cidfile=%t/%n.ctr-id Type=notify NotifyAccess=all diff --git a/pkg/util/utils.go b/pkg/util/utils.go index 37a00c25c..208d815d9 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -621,6 +621,12 @@ func ValidateSysctls(strSlice []string) (map[string]string, error) { if len(arr) < 2 { return nil, errors.Errorf("%s is invalid, sysctl values must be in the form of KEY=VALUE", val) } + + trimmed := fmt.Sprintf("%s=%s", strings.TrimSpace(arr[0]), strings.TrimSpace(arr[1])) + if trimmed != val { + return nil, errors.Errorf("'%s' is invalid, extra spaces found", val) + } + if validSysctlMap[arr[0]] { sysctl[arr[0]] = arr[1] continue diff --git a/pkg/util/utils_test.go b/pkg/util/utils_test.go index 35322c7ea..3d74d4c78 100644 --- a/pkg/util/utils_test.go +++ b/pkg/util/utils_test.go @@ -1,6 +1,7 @@ package util import ( + "fmt" "testing" "time" @@ -259,6 +260,28 @@ func TestValidateSysctlBadSysctl(t *testing.T) { assert.Error(t, err) } +func TestValidateSysctlBadSysctlWithExtraSpaces(t *testing.T) { + expectedError := "'%s' is invalid, extra spaces found" + + // should fail fast on first sysctl + strSlice1 := []string{ + "net.ipv4.ping_group_range = 0 0", + "net.ipv4.ping_group_range=0 0 ", + } + _, err := ValidateSysctls(strSlice1) + assert.Error(t, err) + assert.Equal(t, err.Error(), fmt.Sprintf(expectedError, strSlice1[0])) + + // should fail on second sysctl + strSlice2 := []string{ + "net.ipv4.ping_group_range=0 0", + "net.ipv4.ping_group_range=0 0 ", + } + _, err = ValidateSysctls(strSlice2) + assert.Error(t, err) + assert.Equal(t, err.Error(), fmt.Sprintf(expectedError, strSlice2[1])) +} + func TestCoresToPeriodAndQuota(t *testing.T) { cores := 1.0 expectedPeriod := DefaultCPUPeriod diff --git a/test/apiv2/10-images.at b/test/apiv2/10-images.at index 195b11ff0..abc8d44b7 100644 --- a/test/apiv2/10-images.at +++ b/test/apiv2/10-images.at @@ -173,7 +173,7 @@ curl -XPOST --data-binary @<(cat $CONTAINERFILE_TAR) \ BUILD_TEST_ERROR="" if ! grep -q '200 OK' "${TMPD}/headers.txt"; then - echo -e "${red}NOK: Image build from tar failed response was not 200 OK" + echo -e "${red}NOK: Image build from tar failed response was not 200 OK (application/x-tar)" BUILD_TEST_ERROR="1" fi @@ -182,6 +182,38 @@ if ! grep -q 'quay.io/libpod/alpine_labels' "${TMPD}/response.txt"; then BUILD_TEST_ERROR="1" fi +curl -XPOST --data-binary @<(cat $CONTAINERFILE_TAR) \ + -H "content-type: application/tar" \ + --dump-header "${TMPD}/headers.txt" \ + -o /dev/null \ + "http://$HOST:$PORT/v1.40/libpod/build?dockerfile=containerfile" &> /dev/null +if ! grep -q '200 OK' "${TMPD}/headers.txt"; then + echo -e "${red}NOK: Image build from tar failed response was not 200 OK (application/tar)" + BUILD_TEST_ERROR="1" +fi + +# Yes, this is very un-RESTful re: Content-Type header ignored when compatibility endpoint used +# See https://github.com/containers/podman/issues/11012 +curl -XPOST --data-binary @<(cat $CONTAINERFILE_TAR) \ + -H "content-type: application/json" \ + --dump-header "${TMPD}/headers.txt" \ + -o /dev/null \ + "http://$HOST:$PORT/v1.40/build?dockerfile=containerfile" &> /dev/null +if ! grep -q '200 OK' "${TMPD}/headers.txt"; then + echo -e "${red}NOK: Image build from tar failed response was not 200 OK (application/tar)" + BUILD_TEST_ERROR="1" +fi + +curl -XPOST --data-binary @<(cat $CONTAINERFILE_TAR) \ + -H "content-type: application/json" \ + --dump-header "${TMPD}/headers.txt" \ + -o /dev/null \ + "http://$HOST:$PORT/v1.40/libpod/build?dockerfile=containerfile" &> /dev/null +if ! grep -q '400 Bad Request' "${TMPD}/headers.txt"; then + echo -e "${red}NOK: Image build should have failed with 400 (wrong Content-Type)" + BUILD_TEST_ERROR="1" +fi + cleanBuildTest if [[ "${BUILD_TEST_ERROR}" ]]; then exit 1 diff --git a/test/apiv2/20-containers.at b/test/apiv2/20-containers.at index 610d3e36d..e2eb94233 100644 --- a/test/apiv2/20-containers.at +++ b/test/apiv2/20-containers.at @@ -356,3 +356,14 @@ t GET containers/$cid/json 200 \ .HostConfig.NetworkMode="bridge" t DELETE containers/$cid?v=true 204 + +# Test Compat Create with healthcheck, check default values +t POST containers/create Image=$IMAGE Cmd='["top"]' Healthcheck='{"Test":["true"]}' 201 \ + .Id~[0-9a-f]\\{64\\} +cid=$(jq -r '.Id' <<<"$output") +t GET containers/$cid/json 200 \ + .Config.Healthcheck.Interval=30000000000 \ + .Config.Healthcheck.Timeout=30000000000 \ + .Config.Healthcheck.Retries=3 + +t DELETE containers/$cid?v=true 204 diff --git a/test/apiv2/40-pods.at b/test/apiv2/40-pods.at index 94c72dbaa..985b26411 100644 --- a/test/apiv2/40-pods.at +++ b/test/apiv2/40-pods.at @@ -19,6 +19,9 @@ t GET libpod/pods/json 200 \ .[0].Id=$pod_id \ .[0].Containers\|length=1 +t GET libpod/pods/json?filters='{"until":["500000"]}' 200 length=0 +t GET libpod/pods/json?filters='{"until":["5000000000"]}' 200 length=1 + # Cannot create a dup pod with the same name t POST "libpod/pods/create (dup pod)" name=foo 409 \ .cause="pod already exists" diff --git a/test/apiv2/python/rest_api/test_v2_0_0_image.py b/test/apiv2/python/rest_api/test_v2_0_0_image.py index 3e8ecb1ef..bcacaa935 100644 --- a/test/apiv2/python/rest_api/test_v2_0_0_image.py +++ b/test/apiv2/python/rest_api/test_v2_0_0_image.py @@ -87,6 +87,27 @@ class ImageTestCase(APITestCase): self.assertTrue(keys["images"], "Expected to find images stanza") self.assertTrue(keys["stream"], "Expected to find stream progress stanza's") + r = requests.post(self.uri("/images/pull?reference=alpine&quiet=true"), timeout=15) + self.assertEqual(r.status_code, 200, r.status_code) + text = r.text + keys = { + "error": False, + "id": False, + "images": False, + "stream": False, + } + # Read and record stanza's from pull + for line in str.splitlines(text): + obj = json.loads(line) + key_list = list(obj.keys()) + for k in key_list: + keys[k] = True + + self.assertFalse(keys["error"], "Expected no errors") + self.assertTrue(keys["id"], "Expected to find id stanza") + self.assertTrue(keys["images"], "Expected to find images stanza") + self.assertFalse(keys["stream"], "Expected to find stream progress stanza's") + def test_create(self): r = requests.post( self.podman_url + "/v1.40/images/create?fromImage=alpine&platform=linux/amd64/v8", diff --git a/test/apiv2/python/rest_api/test_v2_0_0_system.py b/test/apiv2/python/rest_api/test_v2_0_0_system.py index 3628b5af1..3dfd08525 100644 --- a/test/apiv2/python/rest_api/test_v2_0_0_system.py +++ b/test/apiv2/python/rest_api/test_v2_0_0_system.py @@ -70,6 +70,15 @@ class SystemTestCase(APITestCase): r = requests.get(self.uri("/version")) self.assertEqual(r.status_code, 200, r.text) + body = r.json() + names = [d.get("Name", "") for d in body["Components"]] + + self.assertIn("Conmon", names) + for n in names: + if n.startswith("OCI Runtime"): + oci_name = n + self.assertIsNotNone(oci_name, "OCI Runtime not found in version components.") + def test_df(self): r = requests.get(self.podman_url + "/v1.40/system/df") self.assertEqual(r.status_code, 200, r.text) diff --git a/test/buildah-bud/apply-podman-deltas b/test/buildah-bud/apply-podman-deltas index 18b3d56f9..44a33b0b8 100755 --- a/test/buildah-bud/apply-podman-deltas +++ b/test/buildah-bud/apply-podman-deltas @@ -165,14 +165,6 @@ skip "FIXME FIXME FIXME: this passes on Ed's laptop, fails in CI??" \ skip "buildah runs with --cgroup-manager=cgroupfs, podman with systemd" \ "bud with --cgroup-parent" -# see https://github.com/containers/podman/pull/10829 -skip "FIXME FIXME FIXME - requires updated CI images (#10829)" \ - "bud with --runtime and --runtime-flag" - -############################################################################### -# BEGIN tests which are skipped due to actual podman bugs. - - ############################################################################### # BEGIN tests which are skipped because they make no sense under podman-remote diff --git a/test/compose/mount_and_label/docker-compose.yml b/test/compose/mount_and_label/docker-compose.yml index 112d7e134..81fda2512 100644 --- a/test/compose/mount_and_label/docker-compose.yml +++ b/test/compose/mount_and_label/docker-compose.yml @@ -6,5 +6,7 @@ services: - '5000:5000' volumes: - /tmp/data:/data:ro + security_opt: + - label=disable labels: - "io.podman=the_best" diff --git a/test/e2e/commit_test.go b/test/e2e/commit_test.go index 0a368b10f..fbd4068f8 100644 --- a/test/e2e/commit_test.go +++ b/test/e2e/commit_test.go @@ -329,4 +329,40 @@ var _ = Describe("Podman commit", func() { session.WaitWithDefaultTimeout() Expect(session.OutputToString()).To(Not(ContainSubstring(secretsString))) }) + + It("podman commit adds exposed ports", func() { + name := "testcon" + s := podmanTest.Podman([]string{"run", "--name", name, "-p", "8080:80", ALPINE, "true"}) + s.WaitWithDefaultTimeout() + Expect(s).Should(Exit(0)) + + newImageName := "newimage" + c := podmanTest.Podman([]string{"commit", name, newImageName}) + c.WaitWithDefaultTimeout() + Expect(c).Should(Exit(0)) + + inspect := podmanTest.Podman([]string{"inspect", newImageName}) + inspect.WaitWithDefaultTimeout() + Expect(inspect).Should(Exit(0)) + images := inspect.InspectImageJSON() + Expect(images).To(HaveLen(1)) + Expect(images[0].Config.ExposedPorts).To(HaveKey("80/tcp")) + + name = "testcon2" + s = podmanTest.Podman([]string{"run", "--name", name, "-d", nginx}) + s.WaitWithDefaultTimeout() + Expect(s).Should(Exit(0)) + + newImageName = "newimage2" + c = podmanTest.Podman([]string{"commit", name, newImageName}) + c.WaitWithDefaultTimeout() + Expect(c).Should(Exit(0)) + + inspect = podmanTest.Podman([]string{"inspect", newImageName}) + inspect.WaitWithDefaultTimeout() + Expect(inspect).Should(Exit(0)) + images = inspect.InspectImageJSON() + Expect(images).To(HaveLen(1)) + Expect(images[0].Config.ExposedPorts).To(HaveKey("80/tcp")) + }) }) diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go index 2e48e1763..20ed72c59 100644 --- a/test/e2e/common_test.go +++ b/test/e2e/common_test.go @@ -264,6 +264,11 @@ func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration { if rootless.IsRootless() { storageFs = ROOTLESS_STORAGE_FS } + if os.Getenv("STORAGE_FS") != "" { + storageFs = os.Getenv("STORAGE_FS") + storageOptions = "--storage-driver " + storageFs + } + p := &PodmanTestIntegration{ PodmanTest: PodmanTest{ PodmanBinary: podmanBinary, @@ -645,9 +650,13 @@ func isRootless() bool { return os.Geteuid() != 0 } +func isCgroupsV1() bool { + return !CGROUPSV2 +} + func SkipIfCgroupV1(reason string) { checkReason(reason) - if !CGROUPSV2 { + if isCgroupsV1() { Skip(reason) } } @@ -841,3 +850,18 @@ func (p *PodmanTestIntegration) buildImage(dockerfile, imageName string, layers output := session.OutputToStringArray() return output[len(output)-1] } + +func writeYaml(content string, fileName string) error { + f, err := os.Create(fileName) + if err != nil { + return err + } + defer f.Close() + + _, err = f.WriteString(content) + if err != nil { + return err + } + + return nil +} diff --git a/test/e2e/container_inspect_test.go b/test/e2e/container_inspect_test.go index 9a95a275a..7d05b09fb 100644 --- a/test/e2e/container_inspect_test.go +++ b/test/e2e/container_inspect_test.go @@ -3,6 +3,7 @@ package integration import ( "os" + "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/annotations" . "github.com/containers/podman/v3/test/utils" . "github.com/onsi/ginkgo" @@ -43,4 +44,28 @@ var _ = Describe("Podman container inspect", func() { Expect(data[0].Config.Annotations[annotations.ContainerManager]). To(Equal(annotations.ContainerManagerLibpod)) }) + + It("podman inspect shows exposed ports", func() { + name := "testcon" + session := podmanTest.Podman([]string{"run", "-d", "--stop-timeout", "0", "--expose", "8080/udp", "--name", name, ALPINE, "sleep", "inf"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + data := podmanTest.InspectContainer(name) + + Expect(data).To(HaveLen(1)) + Expect(data[0].NetworkSettings.Ports). + To(Equal(map[string][]define.InspectHostPort{"8080/udp": nil})) + }) + + It("podman inspect shows exposed ports on image", func() { + name := "testcon" + session := podmanTest.Podman([]string{"run", "-d", "--expose", "8080", "--name", name, nginx}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + data := podmanTest.InspectContainer(name) + Expect(data).To(HaveLen(1)) + Expect(data[0].NetworkSettings.Ports). + To(Equal(map[string][]define.InspectHostPort{"80/tcp": nil, "8080/tcp": nil})) + }) }) diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go index 975596dee..32d98c2a9 100644 --- a/test/e2e/create_test.go +++ b/test/e2e/create_test.go @@ -60,10 +60,24 @@ var _ = Describe("Podman create", func() { }) It("podman container create container based on a remote image", func() { - session := podmanTest.Podman([]string{"container", "create", BB_GLIBC, "ls"}) + containerCreate := podmanTest.Podman([]string{"container", "create", BB_GLIBC, "ls"}) + containerCreate.WaitWithDefaultTimeout() + Expect(containerCreate).Should(Exit(0)) + + lock := GetPortLock("5000") + defer lock.Unlock() + session := podmanTest.Podman([]string{"run", "-d", "--name", "registry", "-p", "5000:5000", registry, "/entrypoint.sh", "/etc/docker/registry/config.yml"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(podmanTest.NumberOfContainers()).To(Equal(1)) + + if !WaitContainerReady(podmanTest, "registry", "listening on", 20, 1) { + Skip("Cannot start docker registry.") + } + + create := podmanTest.Podman([]string{"container", "create", "--tls-verify=false", ALPINE}) + create.WaitWithDefaultTimeout() + Expect(create).Should(Exit(0)) + Expect(podmanTest.NumberOfContainers()).To(Equal(3)) }) It("podman create using short options", func() { @@ -609,7 +623,7 @@ var _ = Describe("Podman create", func() { Expect(session).Should(ExitWithError()) }) - It("create container in pod ppublish ports should fail", func() { + It("create container in pod publish ports should fail", func() { name := "createwithpublishports" pod := podmanTest.RunTopContainerInPod("", "new:"+name) pod.WaitWithDefaultTimeout() diff --git a/test/e2e/info_test.go b/test/e2e/info_test.go index 8ac538dd2..bc3ae4443 100644 --- a/test/e2e/info_test.go +++ b/test/e2e/info_test.go @@ -77,6 +77,15 @@ var _ = Describe("Podman Info", func() { Expect(session.OutputToString()).To(ContainSubstring("registry")) }) + It("podman info --format GO template plugins", func() { + session := podmanTest.Podman([]string{"info", "--format", "{{.Plugins}}"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + Expect(session.OutputToString()).To(ContainSubstring("local")) + Expect(session.OutputToString()).To(ContainSubstring("journald")) + Expect(session.OutputToString()).To(ContainSubstring("bridge")) + }) + It("podman info rootless storage path", func() { SkipIfNotRootless("test of rootless_storage_path is only meaningful as rootless") SkipIfRemote("Only tests storage on local client") diff --git a/test/e2e/login_logout_test.go b/test/e2e/login_logout_test.go index 7ad1fc1f2..d8ca9cbd9 100644 --- a/test/e2e/login_logout_test.go +++ b/test/e2e/login_logout_test.go @@ -79,9 +79,9 @@ var _ = Describe("Podman login and logout", func() { session = podmanTest.Podman([]string{"run", "-d", "-p", strings.Join([]string{strconv.Itoa(port), strconv.Itoa(port)}, ":"), "-e", strings.Join([]string{"REGISTRY_HTTP_ADDR=0.0.0.0", strconv.Itoa(port)}, ":"), "--name", "registry", "-v", - strings.Join([]string{authPath, "/auth"}, ":"), "-e", "REGISTRY_AUTH=htpasswd", "-e", + strings.Join([]string{authPath, "/auth:Z"}, ":"), "-e", "REGISTRY_AUTH=htpasswd", "-e", "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm", "-e", "REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd", - "-v", strings.Join([]string{certPath, "/certs"}, ":"), "-e", "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt", + "-v", strings.Join([]string{certPath, "/certs:Z"}, ":"), "-e", "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt", "-e", "REGISTRY_HTTP_TLS_KEY=/certs/domain.key", "registry:2.6"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) @@ -235,10 +235,13 @@ var _ = Describe("Podman login and logout", func() { setup.WaitWithDefaultTimeout() defer os.RemoveAll(certDir) + // N/B: This second registry container shares the same auth and cert dirs + // as the registry started from BeforeEach(). Since this one starts + // second, re-labeling the volumes should keep SELinux happy. session := podmanTest.Podman([]string{"run", "-d", "-p", "9001:9001", "-e", "REGISTRY_HTTP_ADDR=0.0.0.0:9001", "--name", "registry1", "-v", - strings.Join([]string{authPath, "/auth"}, ":"), "-e", "REGISTRY_AUTH=htpasswd", "-e", + strings.Join([]string{authPath, "/auth:z"}, ":"), "-e", "REGISTRY_AUTH=htpasswd", "-e", "REGISTRY_AUTH_HTPASSWD_REALM=Registry Realm", "-e", "REGISTRY_AUTH_HTPASSWD_PATH=/auth/htpasswd", - "-v", strings.Join([]string{certPath, "/certs"}, ":"), "-e", "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt", + "-v", strings.Join([]string{certPath, "/certs:z"}, ":"), "-e", "REGISTRY_HTTP_TLS_CERTIFICATE=/certs/domain.crt", "-e", "REGISTRY_HTTP_TLS_KEY=/certs/domain.key", "registry:2.6"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) diff --git a/test/e2e/network_connect_disconnect_test.go b/test/e2e/network_connect_disconnect_test.go index b1f3607ab..217efdeec 100644 --- a/test/e2e/network_connect_disconnect_test.go +++ b/test/e2e/network_connect_disconnect_test.go @@ -236,8 +236,6 @@ var _ = Describe("Podman network connect and disconnect", func() { }) It("podman network connect and run with network ID", func() { - SkipIfRemote("remote flakes to much I will fix this in another PR") - SkipIfRootless("network connect and disconnect are only rootful") netName := "ID" + stringid.GenerateNonCryptoID() session := podmanTest.Podman([]string{"network", "create", netName}) session.WaitWithDefaultTimeout() @@ -249,7 +247,7 @@ var _ = Describe("Podman network connect and disconnect", func() { Expect(session).Should(Exit(0)) netID := session.OutputToString() - ctr := podmanTest.Podman([]string{"run", "-dt", "--name", "test", "--network", netID, ALPINE, "top"}) + ctr := podmanTest.Podman([]string{"run", "-dt", "--name", "test", "--network", netID, "--network-alias", "somealias", ALPINE, "top"}) ctr.WaitWithDefaultTimeout() Expect(ctr).Should(Exit(0)) @@ -269,7 +267,7 @@ var _ = Describe("Podman network connect and disconnect", func() { Expect(session).Should(Exit(0)) newNetID := session.OutputToString() - connect := podmanTest.Podman([]string{"network", "connect", newNetID, "test"}) + connect := podmanTest.Podman([]string{"network", "connect", "--alias", "secondalias", newNetID, "test"}) connect.WaitWithDefaultTimeout() Expect(connect).Should(Exit(0)) @@ -324,8 +322,6 @@ var _ = Describe("Podman network connect and disconnect", func() { }) It("podman network disconnect and run with network ID", func() { - SkipIfRemote("remote flakes to much I will fix this in another PR") - SkipIfRootless("network connect and disconnect are only rootful") netName := "aliasTest" + stringid.GenerateNonCryptoID() session := podmanTest.Podman([]string{"network", "create", netName}) session.WaitWithDefaultTimeout() diff --git a/test/e2e/play_build_test.go b/test/e2e/play_build_test.go new file mode 100644 index 000000000..16f2687f3 --- /dev/null +++ b/test/e2e/play_build_test.go @@ -0,0 +1,243 @@ +// +build !remote + +// build for play kube is not supported on remote yet. + +package integration + +import ( + "os" + "path/filepath" + + . "github.com/containers/podman/v3/test/utils" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gexec" +) + +var _ = Describe("Podman play kube with build", func() { + var ( + tempdir string + err error + podmanTest *PodmanTestIntegration + ) + + BeforeEach(func() { + tempdir, err = CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest = PodmanTestCreate(tempdir) + podmanTest.Setup() + podmanTest.SeedImages() + }) + + AfterEach(func() { + podmanTest.Cleanup() + f := CurrentGinkgoTestDescription() + processTestResult(f) + + }) + + var testYAML = ` +apiVersion: v1 +kind: Pod +metadata: + creationTimestamp: "2021-08-05T17:55:51Z" + labels: + app: foobar + name: top_pod +spec: + containers: + - command: + - top + env: + - name: PATH + value: /usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin + - name: TERM + value: xterm + - name: container + value: podman + image: foobar + name: foobar + resources: {} + securityContext: + allowPrivilegeEscalation: true + capabilities: + drop: + - CAP_MKNOD + - CAP_NET_RAW + - CAP_AUDIT_WRITE + privileged: false + readOnlyRootFilesystem: false + seLinuxOptions: {} + tty: true + workingDir: / + dnsConfig: {} +status: {} +` + + var playBuildFile = ` +FROM quay.io/libpod/alpine_nginx:latest +RUN apk update && apk add strace +LABEL homer=dad +` + var prebuiltImage = ` +FROM quay.io/libpod/alpine_nginx:latest +RUN apk update && apk add strace +LABEL marge=mom +` + It("Check that image is built using Dockerfile", func() { + // Setup + yamlDir := filepath.Join(tempdir, RandomString(12)) + err := os.Mkdir(yamlDir, 0755) + err = writeYaml(testYAML, filepath.Join(yamlDir, "top.yaml")) + Expect(err).To(BeNil()) + app1Dir := filepath.Join(yamlDir, "foobar") + err = os.Mkdir(app1Dir, 0755) + Expect(err).To(BeNil()) + err = writeYaml(playBuildFile, filepath.Join(app1Dir, "Dockerfile")) + Expect(err).To(BeNil()) + + // Switch to temp dir and restore it afterwards + cwd, err := os.Getwd() + Expect(err).To(BeNil()) + Expect(os.Chdir(yamlDir)).To(BeNil()) + defer func() { (Expect(os.Chdir(cwd)).To(BeNil())) }() + + session := podmanTest.Podman([]string{"play", "kube", "top.yaml"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + exists := podmanTest.Podman([]string{"image", "exists", "foobar"}) + exists.WaitWithDefaultTimeout() + Expect(exists).Should(Exit(0)) + + inspect := podmanTest.Podman([]string{"container", "inspect", "top_pod-foobar"}) + inspect.WaitWithDefaultTimeout() + Expect(inspect).Should(Exit(0)) + inspectData := inspect.InspectContainerToJSON() + Expect(len(inspectData)).To(BeNumerically(">", 0)) + Expect(inspectData[0].Config.Labels["homer"]).To(Equal("dad")) + }) + + It("Check that image is built using Containerfile", func() { + // Setup + yamlDir := filepath.Join(tempdir, RandomString(12)) + err := os.Mkdir(yamlDir, 0755) + err = writeYaml(testYAML, filepath.Join(yamlDir, "top.yaml")) + Expect(err).To(BeNil()) + app1Dir := filepath.Join(yamlDir, "foobar") + err = os.Mkdir(app1Dir, 0755) + Expect(err).To(BeNil()) + err = writeYaml(playBuildFile, filepath.Join(app1Dir, "Containerfile")) + Expect(err).To(BeNil()) + + // Switch to temp dir and restore it afterwards + cwd, err := os.Getwd() + Expect(err).To(BeNil()) + Expect(os.Chdir(yamlDir)).To(BeNil()) + defer func() { (Expect(os.Chdir(cwd)).To(BeNil())) }() + + session := podmanTest.Podman([]string{"play", "kube", "top.yaml"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + exists := podmanTest.Podman([]string{"image", "exists", "foobar"}) + exists.WaitWithDefaultTimeout() + Expect(exists).Should(Exit(0)) + + inspect := podmanTest.Podman([]string{"container", "inspect", "top_pod-foobar"}) + inspect.WaitWithDefaultTimeout() + Expect(inspect).Should(Exit(0)) + inspectData := inspect.InspectContainerToJSON() + Expect(len(inspectData)).To(BeNumerically(">", 0)) + Expect(inspectData[0].Config.Labels["homer"]).To(Equal("dad")) + }) + + It("Do not build image if already in the local store", func() { + // Setup + yamlDir := filepath.Join(tempdir, RandomString(12)) + err := os.Mkdir(yamlDir, 0755) + err = writeYaml(testYAML, filepath.Join(yamlDir, "top.yaml")) + Expect(err).To(BeNil()) + + // build an image called foobar but make sure it doesnt have + // the same label as the yaml buildfile, so we can check that + // the image is NOT rebuilt. + err = writeYaml(prebuiltImage, filepath.Join(yamlDir, "Containerfile")) + Expect(err).To(BeNil()) + + app1Dir := filepath.Join(yamlDir, "foobar") + err = os.Mkdir(app1Dir, 0755) + Expect(err).To(BeNil()) + err = writeYaml(playBuildFile, filepath.Join(app1Dir, "Containerfile")) + Expect(err).To(BeNil()) + + // Switch to temp dir and restore it afterwards + cwd, err := os.Getwd() + Expect(err).To(BeNil()) + Expect(os.Chdir(yamlDir)).To(BeNil()) + defer func() { (Expect(os.Chdir(cwd)).To(BeNil())) }() + + // Build the image into the local store + build := podmanTest.Podman([]string{"build", "-t", "foobar", "-f", "Containerfile"}) + build.WaitWithDefaultTimeout() + Expect(build).Should(Exit(0)) + + session := podmanTest.Podman([]string{"play", "kube", "top.yaml"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + inspect := podmanTest.Podman([]string{"container", "inspect", "top_pod-foobar"}) + inspect.WaitWithDefaultTimeout() + Expect(inspect).Should(Exit(0)) + inspectData := inspect.InspectContainerToJSON() + Expect(len(inspectData)).To(BeNumerically(">", 0)) + Expect(inspectData[0].Config.Labels["homer"]).To(Equal("")) + Expect(inspectData[0].Config.Labels["marge"]).To(Equal("mom")) + }) + + It("--build should override image in store", func() { + // Setup + yamlDir := filepath.Join(tempdir, RandomString(12)) + err := os.Mkdir(yamlDir, 0755) + err = writeYaml(testYAML, filepath.Join(yamlDir, "top.yaml")) + Expect(err).To(BeNil()) + + // build an image called foobar but make sure it doesnt have + // the same label as the yaml buildfile, so we can check that + // the image is NOT rebuilt. + err = writeYaml(prebuiltImage, filepath.Join(yamlDir, "Containerfile")) + Expect(err).To(BeNil()) + + app1Dir := filepath.Join(yamlDir, "foobar") + err = os.Mkdir(app1Dir, 0755) + Expect(err).To(BeNil()) + err = writeYaml(playBuildFile, filepath.Join(app1Dir, "Containerfile")) + Expect(err).To(BeNil()) + + // Switch to temp dir and restore it afterwards + cwd, err := os.Getwd() + Expect(err).To(BeNil()) + Expect(os.Chdir(yamlDir)).To(BeNil()) + defer func() { (Expect(os.Chdir(cwd)).To(BeNil())) }() + + // Build the image into the local store + build := podmanTest.Podman([]string{"build", "-t", "foobar", "-f", "Containerfile"}) + build.WaitWithDefaultTimeout() + Expect(build).Should(Exit(0)) + + session := podmanTest.Podman([]string{"play", "kube", "--build", "top.yaml"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + inspect := podmanTest.Podman([]string{"container", "inspect", "top_pod-foobar"}) + inspect.WaitWithDefaultTimeout() + Expect(inspect).Should(Exit(0)) + inspectData := inspect.InspectContainerToJSON() + Expect(len(inspectData)).To(BeNumerically(">", 0)) + Expect(inspectData[0].Config.Labels["homer"]).To(Equal("dad")) + Expect(inspectData[0].Config.Labels["marge"]).To(Equal("")) + }) + +}) diff --git a/test/e2e/play_kube_test.go b/test/e2e/play_kube_test.go index 66bfdefe7..eec4b43a5 100644 --- a/test/e2e/play_kube_test.go +++ b/test/e2e/play_kube_test.go @@ -512,21 +512,6 @@ var ( defaultSecret = []byte(`{"FOO":"Zm9v","BAR":"YmFy"}`) ) -func writeYaml(content string, fileName string) error { - f, err := os.Create(fileName) - if err != nil { - return err - } - defer f.Close() - - _, err = f.WriteString(content) - if err != nil { - return err - } - - return nil -} - // getKubeYaml returns a kubernetes YAML document. func getKubeYaml(kind string, object interface{}) (string, error) { var yamlTemplate string @@ -1114,7 +1099,7 @@ var _ = Describe("Podman play kube", func() { }) It("podman play kube should share ipc,net,uts when shareProcessNamespace is set", func() { - SkipIfRootless("Requires root priviledges for sharing few namespaces") + SkipIfRootless("Requires root privileges for sharing few namespaces") err := writeYaml(sharedNamespacePodYaml, kubeYaml) Expect(err).To(BeNil()) diff --git a/test/e2e/pod_create_test.go b/test/e2e/pod_create_test.go index 4c6788b9d..c961bfc32 100644 --- a/test/e2e/pod_create_test.go +++ b/test/e2e/pod_create_test.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "os" + "os/user" "path/filepath" "strconv" "strings" @@ -120,6 +121,21 @@ var _ = Describe("Podman pod create", func() { Expect(check).Should(Exit(0)) }) + It("podman create pod with id file with network portbindings", func() { + file := filepath.Join(podmanTest.TempDir, "pod.id") + name := "test" + session := podmanTest.Podman([]string{"pod", "create", "--name", name, "--pod-id-file", file, "-p", "8080:80"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + webserver := podmanTest.Podman([]string{"run", "--pod-id-file", file, "-dt", nginx}) + webserver.WaitWithDefaultTimeout() + Expect(webserver).Should(Exit(0)) + + check := SystemExec("nc", []string{"-z", "localhost", "8080"}) + Expect(check).Should(Exit(0)) + }) + It("podman create pod with no infra but portbindings should fail", func() { name := "test" session := podmanTest.Podman([]string{"pod", "create", "--infra=false", "--name", name, "-p", "80:80"}) @@ -621,4 +637,223 @@ ENTRYPOINT ["sleep","99999"] Expect(podCreate).Should(ExitWithError()) }) + + It("podman pod create with --userns=keep-id", func() { + if os.Geteuid() == 0 { + Skip("Test only runs without root") + } + + podName := "testPod" + podCreate := podmanTest.Podman([]string{"pod", "create", "--userns", "keep-id", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + + session := podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "id", "-u"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + uid := fmt.Sprintf("%d", os.Geteuid()) + ok, _ := session.GrepString(uid) + Expect(ok).To(BeTrue()) + + // Check passwd + session = podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "id", "-un"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + u, err := user.Current() + Expect(err).To(BeNil()) + ok, _ = session.GrepString(u.Name) + Expect(ok).To(BeTrue()) + + // root owns /usr + session = podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "stat", "-c%u", "/usr"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + Expect(session.OutputToString()).To(Equal("0")) + + // fail if --pod and --userns set together + session = podmanTest.Podman([]string{"run", "--pod", podName, "--userns", "keep-id", ALPINE, "id", "-u"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(125)) + }) + + It("podman pod create with --userns=keep-id can add users", func() { + if os.Geteuid() == 0 { + Skip("Test only runs without root") + } + + podName := "testPod" + podCreate := podmanTest.Podman([]string{"pod", "create", "--userns", "keep-id", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + + ctrName := "ctr-name" + session := podmanTest.Podman([]string{"run", "--pod", podName, "-d", "--stop-signal", "9", "--name", ctrName, fedoraMinimal, "sleep", "600"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + // container inside pod inherits user form infra container if --user is not set + // etc/passwd entry will look like 1000:*:1000:1000:container user:/:/bin/sh + exec1 := podmanTest.Podman([]string{"exec", ctrName, "cat", "/etc/passwd"}) + exec1.WaitWithDefaultTimeout() + Expect(exec1).Should(Exit(0)) + Expect(exec1.OutputToString()).To(ContainSubstring("container")) + + exec2 := podmanTest.Podman([]string{"exec", ctrName, "useradd", "testuser"}) + exec2.WaitWithDefaultTimeout() + Expect(exec2).Should(Exit(0)) + + exec3 := podmanTest.Podman([]string{"exec", ctrName, "cat", "/etc/passwd"}) + exec3.WaitWithDefaultTimeout() + Expect(exec3).Should(Exit(0)) + Expect(exec3.OutputToString()).To(ContainSubstring("testuser")) + }) + + It("podman pod create with --userns=auto", func() { + u, err := user.Current() + Expect(err).To(BeNil()) + name := u.Name + if name == "root" { + name = "containers" + } + + content, err := ioutil.ReadFile("/etc/subuid") + if err != nil { + Skip("cannot read /etc/subuid") + } + if !strings.Contains(string(content), name) { + Skip("cannot find mappings for the current user") + } + + m := make(map[string]string) + for i := 0; i < 5; i++ { + podName := "testPod" + strconv.Itoa(i) + podCreate := podmanTest.Podman([]string{"pod", "create", "--userns=auto", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + + session := podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/uid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + l := session.OutputToString() + Expect(strings.Contains(l, "1024")).To(BeTrue()) + m[l] = l + } + // check for no duplicates + Expect(len(m)).To(Equal(5)) + }) + + It("podman pod create --userns=auto:size=%d", func() { + u, err := user.Current() + Expect(err).To(BeNil()) + + name := u.Name + if name == "root" { + name = "containers" + } + + content, err := ioutil.ReadFile("/etc/subuid") + if err != nil { + Skip("cannot read /etc/subuid") + } + if !strings.Contains(string(content), name) { + Skip("cannot find mappings for the current user") + } + + podName := "testPod" + podCreate := podmanTest.Podman([]string{"pod", "create", "--userns=auto:size=500", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + session := podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/uid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + ok, _ := session.GrepString("500") + + podName = "testPod-1" + podCreate = podmanTest.Podman([]string{"pod", "create", "--userns=auto:size=3000", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + session = podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/uid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + ok, _ = session.GrepString("3000") + + Expect(ok).To(BeTrue()) + }) + + It("podman pod create --userns=auto:uidmapping=", func() { + u, err := user.Current() + Expect(err).To(BeNil()) + + name := u.Name + if name == "root" { + name = "containers" + } + + content, err := ioutil.ReadFile("/etc/subuid") + if err != nil { + Skip("cannot read /etc/subuid") + } + if !strings.Contains(string(content), name) { + Skip("cannot find mappings for the current user") + } + + podName := "testPod" + podCreate := podmanTest.Podman([]string{"pod", "create", "--userns=auto:uidmapping=0:0:1", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + session := podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/uid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + output := session.OutputToString() + Expect(output).To(MatchRegexp("\\s0\\s0\\s1")) + + podName = "testPod-1" + podCreate = podmanTest.Podman([]string{"pod", "create", "--userns=auto:size=8192,uidmapping=0:0:1", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + session = podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/uid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + ok, _ := session.GrepString("8191") + Expect(ok).To(BeTrue()) + }) + + It("podman pod create --userns=auto:gidmapping=", func() { + u, err := user.Current() + Expect(err).To(BeNil()) + + name := u.Name + if name == "root" { + name = "containers" + } + + content, err := ioutil.ReadFile("/etc/subuid") + if err != nil { + Skip("cannot read /etc/subuid") + } + if !strings.Contains(string(content), name) { + Skip("cannot find mappings for the current user") + } + + podName := "testPod" + podCreate := podmanTest.Podman([]string{"pod", "create", "--userns=auto:gidmapping=0:0:1", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + session := podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/gid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + output := session.OutputToString() + Expect(output).To(MatchRegexp("\\s0\\s0\\s1")) + + podName = "testPod-1" + podCreate = podmanTest.Podman([]string{"pod", "create", "--userns=auto:size=8192,gidmapping=0:0:1", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + session = podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/gid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + ok, _ := session.GrepString("8191") + Expect(ok).To(BeTrue()) + }) + }) diff --git a/test/e2e/pod_initcontainers_test.go b/test/e2e/pod_initcontainers_test.go index 606294f51..11e7ca400 100644 --- a/test/e2e/pod_initcontainers_test.go +++ b/test/e2e/pod_initcontainers_test.go @@ -98,10 +98,10 @@ var _ = Describe("Podman init containers", func() { Expect(checkLog.OutputToString()).To(Equal(content)) }) - It("podman make sure oneshot container is removed", func() { + It("podman make sure once container is removed", func() { filename := filepath.Join("/dev/shm", RandomString(12)) content := RandomString(16) - session := podmanTest.Podman([]string{"create", "--init-ctr", "oneshot", "--pod", "new:foobar", ALPINE, "bin/sh", "-c", fmt.Sprintf("echo %s > %s", content, filename)}) + session := podmanTest.Podman([]string{"create", "--init-ctr", "once", "--pod", "new:foobar", ALPINE, "bin/sh", "-c", fmt.Sprintf("echo %s > %s", content, filename)}) session.WaitWithDefaultTimeout() initContainerID := session.OutputToString() Expect(session).Should(Exit(0)) diff --git a/test/e2e/pod_ps_test.go b/test/e2e/pod_ps_test.go index c27539d6f..b4a0df904 100644 --- a/test/e2e/pod_ps_test.go +++ b/test/e2e/pod_ps_test.go @@ -108,6 +108,22 @@ var _ = Describe("Podman ps", func() { Expect(result).Should(Exit(0)) }) + It("podman pod ps --filter until", func() { + name := "mypod" + _, ec, _ := podmanTest.CreatePod(map[string][]string{"--name": {name}}) + Expect(ec).To(Equal(0)) + + result := podmanTest.Podman([]string{"pod", "ps", "--filter", "until=50"}) + result.WaitWithDefaultTimeout() + Expect(result).Should(Exit(0)) + Expect(result.OutputToString()).To(Not(ContainSubstring(name))) + + result = podmanTest.Podman([]string{"pod", "ps", "--filter", "until=5000000000"}) + result.WaitWithDefaultTimeout() + Expect(result).Should(Exit(0)) + Expect(result.OutputToString()).To(ContainSubstring(name)) + }) + It("podman pod ps filter name regexp", func() { _, ec, podid := podmanTest.CreatePod(map[string][]string{"--name": {"mypod"}}) Expect(ec).To(Equal(0)) diff --git a/test/e2e/run_cgroup_parent_test.go b/test/e2e/run_cgroup_parent_test.go index 3e261961b..82b6c3057 100644 --- a/test/e2e/run_cgroup_parent_test.go +++ b/test/e2e/run_cgroup_parent_test.go @@ -64,6 +64,7 @@ var _ = Describe("Podman run with --cgroup-parent", func() { }) Specify("always honor --cgroup-parent", func() { + Skip("https://github.com/containers/podman/issues/11165") SkipIfCgroupV1("test not supported in cgroups v1") if Containerized() || podmanTest.CgroupManager == "cgroupfs" { Skip("Requires Systemd cgroup manager support") diff --git a/test/e2e/run_device_test.go b/test/e2e/run_device_test.go index 40de1d50d..08905aed2 100644 --- a/test/e2e/run_device_test.go +++ b/test/e2e/run_device_test.go @@ -41,36 +41,35 @@ var _ = Describe("Podman run device", func() { }) It("podman run device test", func() { - session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg", ALPINE, "ls", "--color=never", "/dev/kmsg"}) + session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg", ALPINE, "test", "-c", "/dev/kmsg"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(session.OutputToString()).To(Equal("/dev/kmsg")) }) It("podman run device rename test", func() { - session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:/dev/kmsg1", ALPINE, "ls", "--color=never", "/dev/kmsg1"}) + // TODO: Confirm absence of /dev/kmsg in container + session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:/dev/kmsg1", ALPINE, "test", "-c", "/dev/kmsg1"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(session.OutputToString()).To(Equal("/dev/kmsg1")) }) It("podman run device permission test", func() { - session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:r", ALPINE, "ls", "--color=never", "/dev/kmsg"}) + // TODO: Confirm write-permission failure + session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:r", ALPINE, "test", "-r", "/dev/kmsg"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(session.OutputToString()).To(Equal("/dev/kmsg")) }) It("podman run device rename and permission test", func() { - session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:/dev/kmsg1:r", ALPINE, "ls", "--color=never", "/dev/kmsg1"}) + // TODO: Confirm write-permission failure + session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:/dev/kmsg1:r", ALPINE, "test", "-r", "/dev/kmsg1"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(session.OutputToString()).To(Equal("/dev/kmsg1")) }) It("podman run device rename and bad permission test", func() { - session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:/dev/kmsg1:rd", ALPINE, "ls", "--color=never", "/dev/kmsg1"}) + session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:/dev/kmsg1:rd", ALPINE, "true"}) session.WaitWithDefaultTimeout() - Expect(session).To(ExitWithError()) + Expect(session).Should(Exit(125)) }) It("podman run device host device and container device parameter are directories", func() { @@ -89,12 +88,13 @@ var _ = Describe("Podman run device", func() { }) It("podman run device host device with --privileged", func() { - if _, err := os.Stat("/dev/kvm"); err != nil { - Skip("/dev/kvm not available") - } - session := podmanTest.Podman([]string{"run", "--privileged", ALPINE, "ls", "/dev/kvm"}) + session := podmanTest.Podman([]string{"run", "--privileged", ALPINE, "test", "-c", "/dev/kmsg"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) + // verify --privileged is required + session2 := podmanTest.Podman([]string{"run", ALPINE, "test", "-c", "/dev/kmsg"}) + session2.WaitWithDefaultTimeout() + Expect(session2).Should((Exit(1))) }) It("podman run CDI device test", func() { @@ -109,14 +109,13 @@ var _ = Describe("Podman run device", func() { err = cmd.Run() Expect(err).To(BeNil()) - session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "myKmsg", ALPINE, "ls", "--color=never", "/dev/kmsg1"}) + session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "myKmsg", ALPINE, "test", "-c", "/dev/kmsg1"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(session.OutputToString()).To(Equal("/dev/kmsg1")) }) It("podman run --gpus noop", func() { - session := podmanTest.Podman([]string{"run", "--gpus", "all", ALPINE, "ls", "/"}) + session := podmanTest.Podman([]string{"run", "--gpus", "all", ALPINE, "true"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) }) diff --git a/test/e2e/run_networking_test.go b/test/e2e/run_networking_test.go index 92388b099..8eabeba97 100644 --- a/test/e2e/run_networking_test.go +++ b/test/e2e/run_networking_test.go @@ -764,7 +764,6 @@ var _ = Describe("Podman run networking", func() { }) It("podman run check dnsname adds dns search domain", func() { - Skip("needs dnsname#57") net := "dnsname" + stringid.GenerateNonCryptoID() session := podmanTest.Podman([]string{"network", "create", net}) session.WaitWithDefaultTimeout() diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go index d68aa6ac4..846da283d 100644 --- a/test/e2e/run_test.go +++ b/test/e2e/run_test.go @@ -166,9 +166,25 @@ var _ = Describe("Podman run", func() { }) It("podman run a container based on remote image", func() { - session := podmanTest.Podman([]string{"run", "-dt", BB_GLIBC, "ls"}) + // Changing session to rsession + rsession := podmanTest.Podman([]string{"run", "-dt", ALPINE, "ls"}) + rsession.WaitWithDefaultTimeout() + Expect(rsession).Should(Exit(0)) + + lock := GetPortLock("5000") + defer lock.Unlock() + session := podmanTest.Podman([]string{"run", "-d", "--name", "registry", "-p", "5000:5000", registry, "/entrypoint.sh", "/etc/docker/registry/config.yml"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) + + if !WaitContainerReady(podmanTest, "registry", "listening on", 20, 1) { + Skip("Cannot start docker registry.") + } + + run := podmanTest.Podman([]string{"run", "--tls-verify=false", ALPINE}) + run.WaitWithDefaultTimeout() + Expect(run).Should(Exit(0)) + Expect(podmanTest.NumberOfContainers()).To(Equal(3)) }) It("podman run a container with a --rootfs", func() { @@ -946,7 +962,7 @@ USER mail`, BB) Expect(err).To(BeNil()) mountpoint := "/myvol/" - session := podmanTest.Podman([]string{"create", "--volume", vol + ":" + mountpoint, ALPINE, "cat", mountpoint + filename}) + session := podmanTest.Podman([]string{"create", "--volume", vol + ":" + mountpoint + ":z", ALPINE, "cat", mountpoint + filename}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) ctrID := session.OutputToString() @@ -1315,10 +1331,10 @@ USER mail`, BB) } curCgroupsBytes, err := ioutil.ReadFile("/proc/self/cgroup") - Expect(err).To(BeNil()) - var curCgroups string = string(curCgroupsBytes) + Expect(err).ShouldNot(HaveOccurred()) + var curCgroups = string(curCgroupsBytes) fmt.Printf("Output:\n%s\n", curCgroups) - Expect(curCgroups).To(Not(Equal(""))) + Expect(curCgroups).ToNot(Equal("")) ctrName := "testctr" container := podmanTest.Podman([]string{"run", "--name", ctrName, "-d", "--cgroups=disabled", ALPINE, "top"}) @@ -1329,14 +1345,14 @@ USER mail`, BB) inspectOut := podmanTest.InspectContainer(ctrName) Expect(len(inspectOut)).To(Equal(1)) pid := inspectOut[0].State.Pid - Expect(pid).To(Not(Equal(0))) + Expect(pid).ToNot(Equal(0)) Expect(inspectOut[0].HostConfig.CgroupParent).To(Equal("")) ctrCgroupsBytes, err := ioutil.ReadFile(fmt.Sprintf("/proc/%d/cgroup", pid)) - Expect(err).To(BeNil()) - var ctrCgroups string = string(ctrCgroupsBytes) + Expect(err).ShouldNot(HaveOccurred()) + var ctrCgroups = string(ctrCgroupsBytes) fmt.Printf("Output\n:%s\n", ctrCgroups) - Expect(curCgroups).To(Equal(ctrCgroups)) + Expect(ctrCgroups).To(Equal(curCgroups)) }) It("podman run with cgroups=enabled makes cgroups", func() { diff --git a/test/e2e/stats_test.go b/test/e2e/stats_test.go index a0be5d462..c0d56fdbc 100644 --- a/test/e2e/stats_test.go +++ b/test/e2e/stats_test.go @@ -22,6 +22,9 @@ var _ = Describe("Podman stats", func() { BeforeEach(func() { SkipIfRootlessCgroupsV1("stats not supported on cgroupv1 for rootless users") + if isContainerized() { + SkipIfCgroupV1("stats not supported inside cgroupv1 container environment") + } var err error tempdir, err = CreateTempDirInTempDir() if err != nil { diff --git a/test/e2e/systemd_test.go b/test/e2e/systemd_test.go index bb51d6ac2..3213a839a 100644 --- a/test/e2e/systemd_test.go +++ b/test/e2e/systemd_test.go @@ -6,7 +6,6 @@ import ( "strings" "time" - "github.com/containers/podman/v3/pkg/rootless" . "github.com/containers/podman/v3/test/utils" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -118,11 +117,13 @@ WantedBy=multi-user.target Expect(len(conData)).To(Equal(1)) Expect(conData[0].Config.SystemdMode).To(BeTrue()) - if CGROUPSV2 || !rootless.IsRootless() { - stats := podmanTest.Podman([]string{"stats", "--no-stream", ctrName}) - stats.WaitWithDefaultTimeout() - Expect(stats).Should(Exit(0)) + // stats not supported w/ CGv1 rootless or containerized + if isCgroupsV1() && (isRootless() || isContainerized()) { + return } + stats := podmanTest.Podman([]string{"stats", "--no-stream", ctrName}) + stats.WaitWithDefaultTimeout() + Expect(stats).Should(Exit(0)) }) It("podman create container with systemd entrypoint triggers systemd mode", func() { diff --git a/test/e2e/volume_create_test.go b/test/e2e/volume_create_test.go index 51005d177..3be1486d8 100644 --- a/test/e2e/volume_create_test.go +++ b/test/e2e/volume_create_test.go @@ -60,6 +60,69 @@ var _ = Describe("Podman volume create", func() { Expect(len(check.OutputToStringArray())).To(Equal(1)) }) + It("podman create and export volume", func() { + if podmanTest.RemoteTest { + Skip("Volume export check does not work with a remote client") + } + + session := podmanTest.Podman([]string{"volume", "create", "myvol"}) + session.WaitWithDefaultTimeout() + volName := session.OutputToString() + Expect(session).Should(Exit(0)) + + session = podmanTest.Podman([]string{"run", "--volume", volName + ":/data", ALPINE, "sh", "-c", "echo hello >> " + "/data/test"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + check := podmanTest.Podman([]string{"volume", "export", volName}) + check.WaitWithDefaultTimeout() + Expect(check.OutputToString()).To(ContainSubstring("hello")) + }) + + It("podman create and import volume", func() { + if podmanTest.RemoteTest { + Skip("Volume export check does not work with a remote client") + } + + session := podmanTest.Podman([]string{"volume", "create", "my_vol"}) + session.WaitWithDefaultTimeout() + volName := session.OutputToString() + Expect(session).Should(Exit(0)) + + session = podmanTest.Podman([]string{"run", "--volume", volName + ":/data", ALPINE, "sh", "-c", "echo hello >> " + "/data/test"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + session = podmanTest.Podman([]string{"volume", "export", volName, "--output=hello.tar"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + session = podmanTest.Podman([]string{"volume", "create", "my_vol2"}) + session.WaitWithDefaultTimeout() + volName = session.OutputToString() + Expect(session).Should(Exit(0)) + + session = podmanTest.Podman([]string{"volume", "import", "my_vol2", "hello.tar"}) + session.WaitWithDefaultTimeout() + volName = session.OutputToString() + Expect(session).Should(Exit(0)) + + session = podmanTest.Podman([]string{"run", "--volume", "my_vol2:/data", ALPINE, "cat", "/data/test"}) + session.WaitWithDefaultTimeout() + Expect(session.OutputToString()).To(ContainSubstring("hello")) + }) + + It("podman import volume should fail", func() { + // try import on volume or source which does not exists + if podmanTest.RemoteTest { + Skip("Volume export check does not work with a remote client") + } + + session := podmanTest.Podman([]string{"volume", "import", "notfound", "notfound.tar"}) + session.WaitWithDefaultTimeout() + Expect(session).To(ExitWithError()) + }) + It("podman create volume with bad volume option", func() { session := podmanTest.Podman([]string{"volume", "create", "--opt", "badOpt=bad"}) session.WaitWithDefaultTimeout() diff --git a/test/python/requirements.txt b/test/python/requirements.txt index ee85bf1d1..f177f76fc 100644 --- a/test/python/requirements.txt +++ b/test/python/requirements.txt @@ -1,5 +1,5 @@ docker~=4.4.3 - +requests-mock~=1.9.3 requests~=2.20.0 setuptools~=50.3.2 python-dateutil~=2.8.1 diff --git a/test/system/005-info.bats b/test/system/005-info.bats index 50c3ceb30..96ca2c1bd 100644 --- a/test/system/005-info.bats +++ b/test/system/005-info.bats @@ -47,6 +47,7 @@ store.configFile | $expr_path store.graphDriverName | [a-z0-9]\\\+\\\$ store.graphRoot | $expr_path store.imageStore.number | 1 +host.slirp4netns.executable | $expr_path " parse_table "$tests" | while read field expect; do diff --git a/test/system/070-build.bats b/test/system/070-build.bats index 26113e45c..0f58b2784 100644 --- a/test/system/070-build.bats +++ b/test/system/070-build.bats @@ -29,6 +29,27 @@ EOF run_podman rmi -f build_test } +@test "podman buildx - basic test" { + rand_filename=$(random_string 20) + rand_content=$(random_string 50) + + tmpdir=$PODMAN_TMPDIR/build-test + mkdir -p $tmpdir + dockerfile=$tmpdir/Dockerfile + cat >$dockerfile <<EOF +FROM $IMAGE +RUN echo $rand_content > /$rand_filename +EOF + + run_podman buildx build --load -t build_test --format=docker $tmpdir + is "$output" ".*COMMIT" "COMMIT seen in log" + + run_podman run --rm build_test cat /$rand_filename + is "$output" "$rand_content" "reading generated file in image" + + run_podman rmi -f build_test +} + @test "podman build test -f -" { rand_filename=$(random_string 20) rand_content=$(random_string 50) diff --git a/test/system/130-kill.bats b/test/system/130-kill.bats index 3770eac27..d85f0a6a9 100644 --- a/test/system/130-kill.bats +++ b/test/system/130-kill.bats @@ -33,7 +33,7 @@ load helpers exec 5<$fifo # First container emits READY when ready; wait for it. - read -t 10 -u 5 ready + read -t 60 -u 5 ready is "$ready" "READY" "first log message from container" # Helper function: send the given signal, verify that it's received. @@ -42,7 +42,7 @@ load helpers local signum=${2:-$1} # e.g. if signal=HUP, we expect to see '1' run_podman kill -s $signal $cid - read -t 10 -u 5 actual || die "Timed out: no ACK for kill -s $signal" + read -t 60 -u 5 actual || die "Timed out: no ACK for kill -s $signal" is "$actual" "got: $signum" "Signal $signal handled by container" } diff --git a/test/system/160-volumes.bats b/test/system/160-volumes.bats index 9a852db89..f6dc3f0af 100644 --- a/test/system/160-volumes.bats +++ b/test/system/160-volumes.bats @@ -186,6 +186,22 @@ EOF } +# Podman volume import test +@test "podman volume import test" { + skip_if_remote "volumes import is not applicable on podman-remote" + run_podman volume create my_vol + run_podman run --rm -v my_vol:/data $IMAGE sh -c "echo hello >> /data/test" + run_podman volume create my_vol2 + run_podman volume export my_vol --output=hello.tar + # we want to use `run_podman volume export my_vol` but run_podman is wrapping EOF + cat hello.tar | run_podman volume import my_vol2 - + run_podman run --rm -v my_vol2:/data $IMAGE sh -c "cat /data/test" + is "$output" "hello" "output from second container" + run_podman volume rm my_vol + run_podman volume rm my_vol2 +} + + # Confirm that container sees the correct id @test "podman volume with --userns=keep-id" { is_rootless || skip "only meaningful when run rootless" diff --git a/test/system/250-systemd.bats b/test/system/250-systemd.bats index ee951ff21..5d4ae4cb1 100644 --- a/test/system/250-systemd.bats +++ b/test/system/250-systemd.bats @@ -46,11 +46,20 @@ function service_setup() { # Helper to stop a systemd service running a container function service_cleanup() { + local status=$1 run systemctl stop "$SERVICE_NAME" if [ $status -ne 0 ]; then die "Error stopping systemd unit $SERVICE_NAME, output: $output" fi + if [[ -z "$status" ]]; then + run systemctl is-active "$SERVICE_NAME" + if [ $status -ne 0 ]; then + die "Error checking stauts of systemd unit $SERVICE_NAME, output: $output" + fi + is "$output" "$status" "$SERVICE_NAME not in expected state" + fi + rm -f "$UNIT_FILE" systemctl daemon-reload } @@ -60,7 +69,8 @@ function service_cleanup() { @test "podman generate - systemd - basic" { cname=$(random_string) # See #7407 for --pull=always. - run_podman create --pull=always --name $cname --label "io.containers.autoupdate=registry" $IMAGE top + run_podman create --pull=always --name $cname --label "io.containers.autoupdate=registry" $IMAGE \ + sh -c "trap 'echo Received SIGTERM, finishing; exit' SIGTERM; echo WAITING; while :; do sleep 0.1; done" # Start systemd service to run this container service_setup @@ -68,7 +78,7 @@ function service_cleanup() { # Give container time to start; make sure output looks top-like sleep 2 run_podman logs $cname - is "$output" ".*Load average:.*" "running container 'top'-like output" + is "$output" ".*WAITING.*" "running is waiting for signal" # Exercise `podman auto-update`. # TODO: this will at least run auto-update code but won't perform an update @@ -77,7 +87,8 @@ function service_cleanup() { run_podman auto-update # All good. Stop service, clean up. - service_cleanup + # Also make sure the service is in the `inactive` state (see #11304). + service_cleanup inactive } @test "podman autoupdate local" { diff --git a/test/system/255-auto-update.bats b/test/system/255-auto-update.bats index 4e242e1f1..7766ca3f9 100644 --- a/test/system/255-auto-update.bats +++ b/test/system/255-auto-update.bats @@ -102,7 +102,7 @@ function _wait_service_ready() { let timeout=$timeout-1 done - # Print serivce status as debug information before failed the case + # Print service status as debug information before failed the case systemctl status $sname die "Timed out waiting for $sname to start" } @@ -304,7 +304,7 @@ EOF fi done - # Only check the last service is started. Previous services should already actived. + # Only check that the last service is started. Previous services should already be activated. _wait_service_ready container-$cname.service run_podman commit --change CMD=/bin/bash $local_cname quay.io/libpod/localtest:latest # Exit code is expected, due to invalid 'fakevalue' diff --git a/test/system/260-sdnotify.bats b/test/system/260-sdnotify.bats index acb30de47..b5d3f9b86 100644 --- a/test/system/260-sdnotify.bats +++ b/test/system/260-sdnotify.bats @@ -130,6 +130,8 @@ function _assert_mainpid_is_conmon() { _stop_socat } +# These tests can fail in dev. environment because of SELinux. +# quick fix: chcon -t container_runtime_exec_t ./bin/podman @test "sdnotify : container" { # Sigh... we need to pull a humongous image because it has systemd-notify. # (IMPORTANT: fedora:32 and above silently removed systemd-notify; this @@ -150,7 +152,7 @@ function _assert_mainpid_is_conmon() { wait_for_ready $cid run_podman logs $cid - is "${lines[0]}" "/.*/container\.sock/notify" "NOTIFY_SOCKET is passed to container" + is "${lines[0]}" "/run/notify/notify.sock" "NOTIFY_SOCKET is passed to container" # With container, READY=1 isn't necessarily the last message received; # just look for it anywhere in received messages diff --git a/test/system/330-corrupt-images.bats b/test/system/330-corrupt-images.bats index 2ee5eee9c..eeffff3ec 100644 --- a/test/system/330-corrupt-images.bats +++ b/test/system/330-corrupt-images.bats @@ -13,7 +13,8 @@ if [ -z "${PODMAN_CORRUPT_TEST_WORKDIR}" ]; then export PODMAN_CORRUPT_TEST_WORKDIR=$(mktemp -d --tmpdir=${BATS_TMPDIR:-${TMPDIR:-/tmp}} podman_corrupt_test.XXXXXX) fi -PODMAN_CORRUPT_TEST_IMAGE_FQIN=quay.io/libpod/alpine@sha256:634a8f35b5f16dcf4aaa0822adc0b1964bb786fca12f6831de8ddc45e5986a00 +PODMAN_CORRUPT_TEST_IMAGE_CANONICAL_FQIN=quay.io/libpod/alpine@sha256:634a8f35b5f16dcf4aaa0822adc0b1964bb786fca12f6831de8ddc45e5986a00 +PODMAN_CORRUPT_TEST_IMAGE_TAGGED_FQIN=${PODMAN_CORRUPT_TEST_IMAGE_CANONICAL_FQIN%%@sha256:*}:test PODMAN_CORRUPT_TEST_IMAGE_ID=961769676411f082461f9ef46626dd7a2d1e2b2a38e6a44364bcbecf51e66dd4 # All tests in this file (and ONLY in this file) run with a custom rootdir @@ -59,7 +60,7 @@ function _corrupt_image_test() { run_podman load -i ${PODMAN_CORRUPT_TEST_WORKDIR}/img.tar # "podman load" restores it without a tag, which (a) causes rmi-by-name # to fail, and (b) causes "podman images" to exit 0 instead of 125 - run_podman tag ${PODMAN_CORRUPT_TEST_IMAGE_ID} ${PODMAN_CORRUPT_TEST_IMAGE_FQIN} + run_podman tag ${PODMAN_CORRUPT_TEST_IMAGE_ID} ${PODMAN_CORRUPT_TEST_IMAGE_TAGGED_FQIN} # shortcut variable name local id=${PODMAN_CORRUPT_TEST_IMAGE_ID} @@ -91,9 +92,9 @@ function _corrupt_image_test() { @test "podman corrupt images - initialize" { # Pull once, save cached copy. - run_podman pull $PODMAN_CORRUPT_TEST_IMAGE_FQIN + run_podman pull $PODMAN_CORRUPT_TEST_IMAGE_CANONICAL_FQIN run_podman save -o ${PODMAN_CORRUPT_TEST_WORKDIR}/img.tar \ - $PODMAN_CORRUPT_TEST_IMAGE_FQIN + $PODMAN_CORRUPT_TEST_IMAGE_CANONICAL_FQIN } # END first "test" does a one-time pull of our desired image @@ -104,8 +105,8 @@ function _corrupt_image_test() { _corrupt_image_test "rmi -f ${PODMAN_CORRUPT_TEST_IMAGE_ID}" } -@test "podman corrupt images - rmi -f <image-name>" { - _corrupt_image_test "rmi -f ${PODMAN_CORRUPT_TEST_IMAGE_FQIN}" +@test "podman corrupt images - rmi -f <image-tagged-name>" { + _corrupt_image_test "rmi -f ${PODMAN_CORRUPT_TEST_IMAGE_TAGGED_FQIN}" } @test "podman corrupt images - rmi -f -a" { diff --git a/test/system/500-networking.bats b/test/system/500-networking.bats index 6ffee7eaf..3ebe45e63 100644 --- a/test/system/500-networking.bats +++ b/test/system/500-networking.bats @@ -210,6 +210,9 @@ load helpers $IMAGE nc -l -n -v -p $myport cid="$output" + # check that dns is working inside the container + run_podman exec $cid nslookup google.com + # emit random string, and check it teststring=$(random_string 30) echo "$teststring" | nc 127.0.0.1 $myport diff --git a/test/system/700-play.bats b/test/system/700-play.bats index 3e6961b08..498956b9a 100644 --- a/test/system/700-play.bats +++ b/test/system/700-play.bats @@ -94,9 +94,9 @@ RELABEL="system_u:object_r:container_file_t:s0" mkdir -p $TESTDIR echo "$testYaml" | sed "s|TESTDIR|${TESTDIR}|g" > $PODMAN_TMPDIR/test.yaml run_podman 125 play kube --network bridge $PODMAN_TMPDIR/test.yaml - is "$output" ".*invalid value passed to --network: bridge or host networking must be configured in YAML" "podman plan-network should fail wth --network host" + is "$output" ".*invalid value passed to --network: bridge or host networking must be configured in YAML" "podman plan-network should fail with --network host" run_podman 125 play kube --network host $PODMAN_TMPDIR/test.yaml - is "$output" ".*invalid value passed to --network: bridge or host networking must be configured in YAML" "podman plan-network should fail wth --network host" + is "$output" ".*invalid value passed to --network: bridge or host networking must be configured in YAML" "podman plan-network should fail with --network host" run_podman play kube --network slirp4netns:port_handler=slirp4netns $PODMAN_TMPDIR/test.yaml run_podman pod rm -f test_pod } diff --git a/troubleshooting.md b/troubleshooting.md index 24dcb8e35..dcf1d8715 100644 --- a/troubleshooting.md +++ b/troubleshooting.md @@ -746,3 +746,138 @@ an Infra container image for CNI-in-slirp4netns must be created. The instructions for building the Infra container image can be found for v2.2.1 [here](https://github.com/containers/podman/tree/v2.2.1-rhel/contrib/rootless-cni-infra), and for v3.0.1 [here](https://github.com/containers/podman/tree/v3.0.1-rhel/contrib/rootless-cni-infra). +### 29) Container related firewall rules are lost after reloading firewalld +Container network can't be reached after `firewall-cmd --reload` and `systemctl restart firewalld` Running `podman network reload` will fix it but it has to be done manually. + +#### Symptom +The firewall rules created by podman are lost when the firewall is reloaded. + +#### Solution +[@ranjithrajaram](https://github.com/containers/podman/issues/5431#issuecomment-847758377) has created a systemd-hook to fix this issue + +1) For "firewall-cmd --reload", create a systemd unit file with the following +``` +[Unit] +Description=firewalld reload hook - run a hook script on firewalld reload +Wants=dbus.service +After=dbus.service + +[Service] +Type=simple +ExecStart=/bin/bash -c '/bin/busctl monitor --system --match "interface=org.fedoraproject.FirewallD1,member=Reloaded" --match "interface=org.fedoraproject.FirewallD1,member=PropertiesChanged" | while read -r line ; do podman network reload --all ; done' + +[Install] +WantedBy=multi-user.target +``` +2) For "systemctl restart firewalld", create a systemd unit file with the following +``` +[Unit] +Description=podman network reload +Wants=firewalld.service +After=firewalld.service +PartOf=firewalld.service + +[Service] +Type=simple +RemainAfterExit=yes +ExecStart=/usr/bin/podman network reload --all + +[Install] +WantedBy=multi-user.target +``` +However, If you use busctl monitor then you can't get machine-readable output on `RHEL 8`. +Since it doesn't have `busctl -j` as mentioned here by [@yrro](https://github.com/containers/podman/issues/5431#issuecomment-896943018). + +For RHEL 8, you can use the following one-liner bash script. +``` +[Unit] +Description=Redo podman NAT rules after firewalld starts or reloads +Wants=dbus.service +After=dbus.service +Requires=firewalld.service + +[Service] +Type=simple +ExecStart=/bin/bash -c "dbus-monitor --profile --system 'type=signal,sender=org.freedesktop.DBus,path=/org/freedesktop/DBus,interface=org.freedesktop.DBus,member=NameAcquired,arg0=org.fedoraproject.FirewallD1' 'type=signal,path=/org/fedoraproject/FirewallD1,interface=org.fedoraproject.FirewallD1,member=Reloaded' | sed -u '/^#/d' | while read -r type timestamp serial sender destination path interface member _junk; do if [[ $type = '#'* ]]; then continue; elif [[ $interface = org.freedesktop.DBus && $member = NameAcquired ]]; then echo 'firewalld started'; podman network reload --all; elif [[ $interface = org.fedoraproject.FirewallD1 && $member = Reloaded ]]; then echo 'firewalld reloaded'; podman network reload --all; fi; done" +Restart=Always + +[Install] +WantedBy=multi-user.target +``` +`busctl-monitor` is almost usable in `RHEL 8`, except that it always outputs two bogus events when it starts up, +one of which is (in its only machine-readable format) indistinguishable from the `NameOwnerChanged` that you get when firewalld starts up. +This means you would get an extra `podman network reload --all` when this unit starts. + +Apart from this, you can use the following systemd service with the python3 code. + +``` +[Unit] +Description=Redo podman NAT rules after firewalld starts or reloads +Wants=dbus.service +Requires=firewalld.service +After=dbus.service + +[Service] +Type=simple +ExecStart=/usr/bin/python /path/to/python/code/podman-redo-nat.py +Restart=always + +[Install] +WantedBy=multi-user.target +``` +The code reloads podman network twice when you use `systemctl restart firewalld`. +``` +import dbus +from gi.repository import GLib +from dbus.mainloop.glib import DBusGMainLoop +import subprocess +import sys + +# I'm a bit confused on the return values in the code +# Not sure if they are needed. + +def reload_podman_network(): + try: + subprocess.run(["podman","network","reload","--all"],timeout=90) + # I'm not sure about this part + sys.stdout.write("podman network reload done\n") + sys.stdout.flush() + except subprocess.TimeoutExpired as t: + sys.stderr.write(f"Podman reload failed due to Timeout {t}") + except subprocess.CalledProcessError as e: + sys.stderr.write(f"Podman reload failed due to {e}") + except Exception as e: + sys.stderr.write(f"Podman reload failed with an Unhandled Exception {e}") + + return False + +def signal_handler(*args, **kwargs): + if kwargs.get('member') == "Reloaded": + reload_podman_network() + elif kwargs.get('member') == "NameOwnerChanged": + reload_podman_network() + else: + return None + return None + +def signal_listener(): + try: + DBusGMainLoop(set_as_default=True)# Define the loop. + loop = GLib.MainLoop() + system_bus = dbus.SystemBus() + # Listens to systemctl restart firewalld with a filter added, will cause podman network to be reloaded twice + system_bus.add_signal_receiver(signal_handler,dbus_interface='org.freedesktop.DBus',arg0='org.fedoraproject.FirewallD1',member_keyword='member') + # Listens to firewall-cmd --reload + system_bus.add_signal_receiver(signal_handler,dbus_interface='org.fedoraproject.FirewallD1',signal_name='Reloaded',member_keyword='member') + loop.run() + except KeyboardInterrupt: + loop.quit() + sys.exit(0) + except Exception as e: + loop.quit() + sys.stderr.write(f"Error occured {e}") + sys.exit(1) + +if __name__ == "__main__": + signal_listener() +``` diff --git a/utils/utils.go b/utils/utils.go index a2268a30b..2e415130e 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -107,6 +107,16 @@ func UntarToFileSystem(dest string, tarball *os.File, options *archive.TarOption return archive.Untar(tarball, dest, options) } +// Creates a new tar file and wrties bytes from io.ReadCloser +func CreateTarFromSrc(source string, dest string) error { + file, err := os.Create(dest) + if err != nil { + return errors.Wrapf(err, "Could not create tarball file '%s'", dest) + } + defer file.Close() + return TarToFilesystem(source, file) +} + // TarToFilesystem creates a tarball from source and writes to an os.file // provided func TarToFilesystem(source string, tarball *os.File) error { diff --git a/utils/utils_supported.go b/utils/utils_supported.go index 8668e3fba..ebc870d26 100644 --- a/utils/utils_supported.go +++ b/utils/utils_supported.go @@ -36,6 +36,7 @@ func RunUnderSystemdScope(pid int, slice string, unitName string) error { return err } } + defer conn.Close() properties = append(properties, systemdDbus.PropSlice(slice)) properties = append(properties, newProp("PIDs", []uint32{uint32(pid)})) properties = append(properties, newProp("Delegate", true)) @@ -54,7 +55,6 @@ func RunUnderSystemdScope(pid int, slice string, unitName string) error { } return err } - defer conn.Close() // Block until job is started <-ch diff --git a/vendor/github.com/containers/buildah/.cirrus.yml b/vendor/github.com/containers/buildah/.cirrus.yml index 860d258b2..9a13725d3 100644 --- a/vendor/github.com/containers/buildah/.cirrus.yml +++ b/vendor/github.com/containers/buildah/.cirrus.yml @@ -6,7 +6,7 @@ env: #### Global variables used for all tasks #### # Name of the ultimate destination branch for this CI run, PR or post-merge. - DEST_BRANCH: "main" + DEST_BRANCH: "release-1.22" GOPATH: "/var/tmp/go" GOSRC: "${GOPATH}/src/github.com/containers/buildah" # Overrides default location (/tmp/cirrus) for repo clone @@ -120,7 +120,7 @@ vendor_task: # Runs within Cirrus's "community cluster" container: - image: docker.io/library/golang:1.13 + image: docker.io/library/golang:1.16 cpu: 1 memory: 1 diff --git a/vendor/github.com/containers/buildah/CHANGELOG.md b/vendor/github.com/containers/buildah/CHANGELOG.md index 8b92ddbe1..ab926d496 100644 --- a/vendor/github.com/containers/buildah/CHANGELOG.md +++ b/vendor/github.com/containers/buildah/CHANGELOG.md @@ -1,225 +1,17 @@  # Changelog -## v1.21.0 (2021-05-19) - Don't blow up if cpp detects errors - Vendor in containers/common v0.38.4 - Remove 'buildah run --security-opt' from completion - update c/common - Fix handling of --default-mounts-file - update vendor of containers/storage v1.31.1 - Bump github.com/containers/storage from 1.30.3 to 1.31.0 - Send logrus messages back to caller when building - github: Fix bad repo. ref in workflow config - Check earlier for bad image tags name - buildah bud: fix containers/podman/issues/10307 - Bump github.com/containers/storage from 1.30.1 to 1.30.3 - Cirrus: Support [CI:DOCS] test skipping - Notification email for cirrus-cron build failures - Bump github.com/opencontainers/runc from 1.0.0-rc93 to 1.0.0-rc94 - Fix race condition - Fix copy race while walking paths - Preserve ownership of lower directory when doing an overlay mount - Bump github.com/onsi/gomega from 1.11.0 to 1.12.0 - Update nix pin with `make nixpkgs` - codespell cleanup - Multi-arch github-action workflow unification - Bump github.com/containers/image/v5 from 5.11.1 to 5.12.0 - Bump github.com/onsi/ginkgo from 1.16.1 to 1.16.2 - imagebuildah: ignore signatures when tagging images - update to latest libimage - Bump github.com/containers/common from 0.37.0 to 0.37.1 - Bump github.com/containers/storage from 1.30.0 to 1.30.1 - Upgrade to GitHub-native Dependabot - Document location of auth.json file if XDG_RUNTIME_DIR is not set - run.bats: fix flake in run-user test - Cirrus: Update F34beta -> F34 - pr-should-include-tests: try to make work in buildah - runUsingRuntime: when relaying error from the runtime, mention that - Run(): avoid Mkdir() into the rootfs - imagebuildah: replace archive with chrootarchive - imagebuildah.StageExecutor.volumeCacheSaveVFS(): set up bind mounts - conformance: use :Z with transient mounts when SELinux is enabled - bud.bats: fix a bats warning - imagebuildah: create volume directories when using overlays - imagebuildah: drop resolveSymlink() - namespaces test - refactoring and cleanup - Refactor 'idmapping' system test - Cirrus: Update Ubuntu images to 21.04 - Tiny fixes in bud system tests - Add compabitility wrappers for removed packages - Fix expected message at pulling image - Fix system tests of 'bud' subcommand - [CI:DOCS] Update steps for CentOS runc users - Add support for secret mounts - Add buildah manifest rm command - restore push/pull and util API - [CI:DOCS] Remove older distro docs - Rename rhel secrets to subscriptions - vendor in openshift/imagebuilder - Remove buildah bud --loglevel ... - use new containers/common/libimage package - Fix copier when using globs - Test namespace flags of 'bud' subcommand - Add system test of 'bud' subcommand - Output names of multiple tags in buildah bud - push to docker test: don't get fooled by podman - copier: add Remove() - build(deps): bump github.com/containers/image/v5 from 5.10.5 to 5.11.1 - Restore log timestamps - Add system test of 'buildah help' with a tiny fix - tests: copy.bats: fix infinite hang - Do not force hard code to crun in rootless mode - build(deps): bump github.com/openshift/imagebuilder from 1.2.0 to 1.2.1 - build(deps): bump github.com/containers/ocicrypt from 1.1.0 to 1.1.1 - build(deps): bump github.com/containers/common from 0.35.4 to 0.36.0 - Fix arg missing warning in bud - Check without flag in 'from --cgroup-parent' test - Minor fixes to Buildah as a library tutorial documentation - Add system test of 'buildah version' for packaged buildah - Add a few system tests of 'buildah from' - Log the final error with %+v at logging level "trace" - copier: add GetOptions.NoCrossDevice - Update nix pin with `make nixpkgs` - Bump to v1.20.2-dev +## v1.22.3 (2021-08-20) + * [release-1.22] bump to v1.22.3 -## v1.20.1 (2021-04-13) - Run container with isolation type set at 'from' - bats helpers.bash - minor refactoring - Bump containers/storage vendor to v1.29.0 - build(deps): bump github.com/onsi/ginkgo from 1.16.0 to 1.16.1 - Cirrus: Update VMs w/ F34beta - CLI add/copy: add a --from option - build(deps): bump github.com/onsi/ginkgo from 1.15.2 to 1.16.0 - Add authentication system tests for 'commit' and 'bud' - fix local image lookup for custom platform - Double-check existence of OCI runtimes - Cirrus: Make use of shared get_ci_vm container - Add system tests of "buildah run" - Update nix pin with `make nixpkgs` - Remove some stuttering on returns errors - Setup alias for --tty to --terminal - Add conformance tests for COPY /... - Put a few more minutes on the clock for the CI conformance test - Add a conformance test for COPY --from $symlink - Add conformance tests for COPY "" - Check for symlink in builtin volume - Sort all mounts by destination directory - System-test cleanup - Export parse.Platform string to be used by podman-remote - blobcache: fix sequencing error - build(deps): bump github.com/containers/common from 0.35.3 to 0.35.4 - Fix URL in demos/buildah_multi_stage.sh - Add a few system tests - [NO TESTS NEEDED] Use --recurse-modules when building git context - Bump to v1.20.1-dev +## v1.22.2 (2021-08-19) + * [release-1.22] bump c/image to v5.15.2 -## v1.20.0 (2021-03-25) - * vendor in containers/storage v1.28.1 - * build(deps): bump github.com/containers/common from 0.35.2 to 0.35.3 - * tests: prefetch: use buildah, not podman, for pulls - * Use faster way to check image tag existence during multi-arch build - * Add information about multi-arch images to the Readme - * COPY --chown: expand the conformance test - * pkg/chrootuser: use a bufio.Scanner - * [CI:DOCS] Fix rootful typo in docs - * build(deps): bump github.com/onsi/ginkgo from 1.15.1 to 1.15.2 - * Add documentation and testing for .containerignore - * build(deps): bump github.com/sirupsen/logrus from 1.8.0 to 1.8.1 - * build(deps): bump github.com/hashicorp/go-multierror from 1.1.0 to 1.1.1 - * Lookup Containerfile if user specifies a directory - * Add Tag format placeholder to docs - * copier: ignore sockets - * image: propagate errors from extractRootfs - * Remove system test of 'buildah containers -a' - * Clarify userns options are usable only as root in man pages - * Fix system test of 'containers -a' - * Remove duplicated code in addcopy - * build(deps): bump github.com/onsi/ginkgo from 1.15.0 to 1.15.1 - * build(deps): bump github.com/onsi/gomega from 1.10.5 to 1.11.0 - * build(deps): bump github.com/fsouza/go-dockerclient from 1.7.1 to 1.7.2 - * Update multi-arch buildah build setup with new logic - * Update nix pin with `make nixpkgs` - * overlay.bats: fix the "overlay source permissions" test - * imagebuildah: use overlay for volumes when using overlay - * Make PolicyMap and PullPolicy names align - * copier: add GetOptions.IgnoreUnreadable - * Check local image to match system context - * fix: Containerfiles - smaller set of userns u/gids - * Set upperdir permissions based on source - * Shrink the vendoring size of pkc/cli - * Clarify image name match failure message - * ADD/COPY: create the destination directory first, chroot to it - * copier.GetOptions: add NoDerefSymLinks - * copier: add an Eval function - * Update system test for 'from --cap-add/drop' - * copier: fix a renaming bug - * copier: return child process stderr if we can't JSON decode the response - * Add some system tests - * build(deps): bump github.com/containers/storage from 1.26.0 to 1.27.0 - * complement add/copy --chmod documentation - * buildah login and logout, do not need to enter user namespace - * Add multi-arch image build - * chmod/chown added/fixed in bash completions - * OWNERS: add @lsm5 - * buildah add/copy --chmod dockerfile implementation - * bump github.com/openshift/imagebuilder from 1.1.8 to 1.2.0 - * buildah add/copy --chmod cli implementation for files and urls - * Make sure we set the buildah version label - * Isolation strings, should match user input - * [CI:DOCS] buildah-from.md: remove dup arch,os - * build(deps): bump github.com/containers/image/v5 from 5.10.2 to 5.10.3 - * Cirrus: Temp. disable prior-fedora (F32) testing - * pr-should-include-tests: recognized "renamed" tests - * build(deps): bump github.com/sirupsen/logrus from 1.7.0 to 1.8.0 - * build(deps): bump github.com/fsouza/go-dockerclient from 1.7.0 to 1.7.1 - * build(deps): bump github.com/containers/common from 0.34.2 to 0.35.0 - * Fix reaping of stages with no instructions - * add stale bot - * Add base image name to comment - * build(deps): bump github.com/spf13/cobra from 1.1.1 to 1.1.3 - * Don't fail copy to emptydir - * buildah: use volatile containers - * vendor: update containers/storage - * Eliminate the use of containers/building import in pkg subdirs - * Add more support for removing config - * Improve messages about --cache-from not being supported - * Revert patch to allow COPY/ADD of empty dirs. - * Don't fail copy to emptydir - * Fix tutorial for rootless mode - * Fix caching layers with build args - * Vendor in containers/image v5.10.2 - * build(deps): bump github.com/containers/common from 0.34.0 to 0.34.2 - * build(deps): bump github.com/onsi/ginkgo from 1.14.2 to 1.15.0 - * 'make validate': require PRs to include tests - * build(deps): bump github.com/onsi/gomega from 1.10.4 to 1.10.5 - * build(deps): bump github.com/containers/storage from 1.24.5 to 1.25.0 - * Use chown function for U volume flag from containers/common repository - * --iidfile: print hash prefix - * bump containernetworking/cni to v0.8.1 - fix for CVE-2021-20206 - * run: fix check for host pid namespace - * Finish plumbing for buildah bud --manifest - * buildah manifest add localimage should work - * Stop testing directory permissions with latest docker - * Fix build arg check - * build(deps): bump github.com/containers/ocicrypt from 1.0.3 to 1.1.0 - * [ci:docs] Fix man page for buildah push - * Update nix pin with `make nixpkgs` - * Bump to containers/image v5.10.1 - * Rebuild layer if a change in ARG is detected - * Bump golang.org/x/crypto to the latest - * Add Ashley and Urvashi to Approvers - * local image lookup by digest - * Use build-arg ENV val from local environment if set - * Pick default OCI Runtime from containers.conf - * Added required devel packages - * Cirrus: Native OSX Build - * Cirrus: Two minor cleanup items - * Workaround for RHEL gating test failure - * build(deps): bump github.com/stretchr/testify from 1.6.1 to 1.7.0 - * build(deps): bump github.com/mattn/go-shellwords from 1.0.10 to 1.0.11 - * Reset upstream branch to dev version - * If destination does not exists, do not throw error +## v1.22.1 (2021-08-17) + * [release-1.22] Bump c/storage to v1.34.1 + * Post-branch commit + * [release-1.22] Accept repositories on login/logout + * [CI:DOCS][release-1.22] Fix CHANGELOG.md ## v1.22.0 (2021-08-02) c/image, c/storage, c/common vendor before Podman 3.3 release diff --git a/vendor/github.com/containers/buildah/Makefile b/vendor/github.com/containers/buildah/Makefile index 2fa5020ee..b1a95685d 100644 --- a/vendor/github.com/containers/buildah/Makefile +++ b/vendor/github.com/containers/buildah/Makefile @@ -171,7 +171,7 @@ test-unit: tests/testreport/testreport $(GO_TEST) -v -tags "$(STORAGETAGS) $(SECURITYTAGS)" -cover -race ./cmd/buildah -args --root $$tmp/root --runroot $$tmp/runroot --storage-driver vfs --signature-policy $(shell pwd)/tests/policy.json --registries-conf $(shell pwd)/tests/registries.conf vendor-in-container: - podman run --privileged --rm --env HOME=/root -v `pwd`:/src -w /src docker.io/library/golang:1.13 make vendor + podman run --privileged --rm --env HOME=/root -v `pwd`:/src -w /src docker.io/library/golang:1.16 make vendor .PHONY: vendor vendor: diff --git a/vendor/github.com/containers/buildah/changelog.txt b/vendor/github.com/containers/buildah/changelog.txt index 6755535b1..066fca0f2 100644 --- a/vendor/github.com/containers/buildah/changelog.txt +++ b/vendor/github.com/containers/buildah/changelog.txt @@ -1,3 +1,15 @@ +- Changelog for v1.22.3 (2021-08-20) + * [release-1.22] bump to v1.22.3 + +- Changelog for v1.22.2 (2021-08-19) + * [release-1.22] bump c/image to v5.15.2 + +- Changelog for v1.22.1 (2021-08-17) + * [release-1.22] Bump c/storage to v1.34.1 + * Post-branch commit + * [release-1.22] Accept repositories on login/logout + * [CI:DOCS][release-1.22] Fix CHANGELOG.md + - Changelog for v1.22.0 (2021-08-02) * c/image, c/storage, c/common vendor before Podman 3.3 release * WIP: tests: new assert() diff --git a/vendor/github.com/containers/buildah/define/types.go b/vendor/github.com/containers/buildah/define/types.go index 27f536a89..8676a574c 100644 --- a/vendor/github.com/containers/buildah/define/types.go +++ b/vendor/github.com/containers/buildah/define/types.go @@ -28,7 +28,7 @@ const ( Package = "buildah" // Version for the Package. Bump version in contrib/rpm/buildah.spec // too. - Version = "1.22.0" + Version = "1.22.3" // DefaultRuntime if containers.conf fails. DefaultRuntime = "runc" diff --git a/vendor/github.com/containers/buildah/go.mod b/vendor/github.com/containers/buildah/go.mod index a8e3e96a3..bb5db94f1 100644 --- a/vendor/github.com/containers/buildah/go.mod +++ b/vendor/github.com/containers/buildah/go.mod @@ -5,9 +5,9 @@ go 1.12 require ( github.com/containernetworking/cni v0.8.1 github.com/containers/common v0.42.1 - github.com/containers/image/v5 v5.15.0 + github.com/containers/image/v5 v5.15.2 github.com/containers/ocicrypt v1.1.2 - github.com/containers/storage v1.33.1 + github.com/containers/storage v1.34.1 github.com/docker/distribution v2.7.1+incompatible github.com/docker/go-units v0.4.0 github.com/docker/libnetwork v0.8.0-dev.2.0.20190625141545-5a177b73e316 @@ -24,7 +24,7 @@ require ( github.com/opencontainers/runc v1.0.1 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 github.com/opencontainers/runtime-tools v0.9.0 - github.com/opencontainers/selinux v1.8.2 + github.com/opencontainers/selinux v1.8.4 github.com/openshift/imagebuilder v1.2.2-0.20210415181909-87f3e48c2656 github.com/pkg/errors v0.9.1 github.com/seccomp/libseccomp-golang v0.9.2-0.20200616122406-847368b35ebf diff --git a/vendor/github.com/containers/buildah/go.sum b/vendor/github.com/containers/buildah/go.sum index 055b8a386..1a3a8043a 100644 --- a/vendor/github.com/containers/buildah/go.sum +++ b/vendor/github.com/containers/buildah/go.sum @@ -53,8 +53,9 @@ github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= +github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= @@ -226,8 +227,8 @@ github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRD github.com/containers/common v0.42.1 h1:ADOZrVAS8ZY5hBAvr/GoRoPv5Z7TBkxWgxQEXQjlqac= github.com/containers/common v0.42.1/go.mod h1:AaF3ipZfgezsctDuhzLkq4Vl+LkEy7J74ikh2HSXDsg= github.com/containers/image/v5 v5.14.0/go.mod h1:SxiBKOcKuT+4yTjD0AskjO+UwFvNcVOJ9qlAw1HNSPU= -github.com/containers/image/v5 v5.15.0 h1:NduhN20ptHNlf0uRny5iTJa2OodB9SLMEB4hKKbzBBs= -github.com/containers/image/v5 v5.15.0/go.mod h1:gzdBcooi6AFdiqfzirUqv90hUyHyI0MMdaqKzACKr2s= +github.com/containers/image/v5 v5.15.2 h1:DKicmVr0h1HGkzs9muoErX+fVbV9sV9W5TyMy5perLE= +github.com/containers/image/v5 v5.15.2/go.mod h1:8jejVSzTDfyPwr/HXp9rri34n/vbdavYk6IzTiB3TBw= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= @@ -236,9 +237,9 @@ github.com/containers/ocicrypt v1.1.1/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B github.com/containers/ocicrypt v1.1.2 h1:Ez+GAMP/4GLix5Ywo/fL7O0nY771gsBIigiqUm1aXz0= github.com/containers/ocicrypt v1.1.2/go.mod h1:Dm55fwWm1YZAjYRaJ94z2mfZikIyIN4B0oB3dj3jFxY= github.com/containers/storage v1.32.6/go.mod h1:mdB+b89p+jU8zpzLTVXA0gWMmIo0WrkfGMh1R8O2IQw= -github.com/containers/storage v1.33.0/go.mod h1:FUZPF4nJijX8ixdhByZJXf02cvbyLi6dyDwXdIe8QVY= -github.com/containers/storage v1.33.1 h1:RHUPZ7vQxwoeOoMoKUDsVun4f9Wi8BTXmr/wQiruBYU= github.com/containers/storage v1.33.1/go.mod h1:FUZPF4nJijX8ixdhByZJXf02cvbyLi6dyDwXdIe8QVY= +github.com/containers/storage v1.34.1 h1:PsBGMH7hwuQ3MOr7qTgPznFrE8ebfIbwQbg2gKvg0lE= +github.com/containers/storage v1.34.1/go.mod h1:FY2TcbfgCLMU4lYoKnlZeZXeH353TOTbpDEA+sAcqAY= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= @@ -514,8 +515,9 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= +github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -649,8 +651,9 @@ github.com/opencontainers/runtime-tools v0.9.0 h1:FYgwVsKRI/H9hU32MJ/4MLOzXWodKK github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= -github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/opencontainers/selinux v1.8.4 h1:krlgQ6/j9CkCXT5oW0yVXdQFOME3NjKuuAZXuR6O7P4= +github.com/opencontainers/selinux v1.8.4/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= github.com/openshift/imagebuilder v1.2.2-0.20210415181909-87f3e48c2656 h1:WaxyNFpmIDu4i6so9r6LVFIbSaXqsj8oitMitt86ae4= github.com/openshift/imagebuilder v1.2.2-0.20210415181909-87f3e48c2656/go.mod h1:9aJRczxCH0mvT6XQ+5STAQaPWz7OsWcU5/mRkt8IWeo= github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913 h1:TnbXhKzrTOyuvWrjI8W6pcoI9XPbLHFXCdN2dtUw7Rw= diff --git a/vendor/github.com/containers/common/libimage/image.go b/vendor/github.com/containers/common/libimage/image.go index c47e63339..b4623a870 100644 --- a/vendor/github.com/containers/common/libimage/image.go +++ b/vendor/github.com/containers/common/libimage/image.go @@ -448,14 +448,24 @@ func (i *Image) removeRecursive(ctx context.Context, rmMap map[string]*RemoveIma return parent.removeRecursive(ctx, rmMap, processedIDs, "", options) } +var errTagDigest = errors.New("tag by digest not supported") + // Tag the image with the specified name and store it in the local containers // storage. The name is normalized according to the rules of NormalizeName. func (i *Image) Tag(name string) error { + if strings.HasPrefix(name, "sha256:") { // ambiguous input + return errors.Wrap(errTagDigest, name) + } + ref, err := NormalizeName(name) if err != nil { return errors.Wrapf(err, "error normalizing name %q", name) } + if _, isDigested := ref.(reference.Digested); isDigested { + return errors.Wrap(errTagDigest, name) + } + logrus.Debugf("Tagging image %s with %q", i.ID(), ref.String()) if i.runtime.eventChannel != nil { defer i.runtime.writeEvent(&Event{ID: i.ID(), Name: name, Time: time.Now(), Type: EventTypeImageTag}) @@ -480,7 +490,7 @@ var errUntagDigest = errors.New("untag by digest not supported") // the local containers storage. The name is normalized according to the rules // of NormalizeName. func (i *Image) Untag(name string) error { - if strings.HasPrefix(name, "sha256:") { + if strings.HasPrefix(name, "sha256:") { // ambiguous input return errors.Wrap(errUntagDigest, name) } @@ -488,6 +498,17 @@ func (i *Image) Untag(name string) error { if err != nil { return errors.Wrapf(err, "error normalizing name %q", name) } + + // FIXME: this is breaking Podman CI but must be re-enabled once + // c/storage supports alterting the digests of an image. Then, + // Podman will do the right thing. + // + // !!! Also make sure to re-enable the tests !!! + // + // if _, isDigested := ref.(reference.Digested); isDigested { + // return errors.Wrap(errUntagDigest, name) + // } + name = ref.String() logrus.Debugf("Untagging %q from image %s", ref.String(), i.ID()) diff --git a/vendor/github.com/containers/common/pkg/auth/auth.go b/vendor/github.com/containers/common/pkg/auth/auth.go index 093da0299..0934b155f 100644 --- a/vendor/github.com/containers/common/pkg/auth/auth.go +++ b/vendor/github.com/containers/common/pkg/auth/auth.go @@ -104,7 +104,6 @@ func Login(ctx context.Context, systemContext *types.SystemContext, opts *LoginO return errors.Wrap(err, "get credentials for repository") } } else { - // nolint: staticcheck authConfig, err = config.GetCredentials(systemContext, registry) if err != nil { return errors.Wrap(err, "get credentials") @@ -321,7 +320,6 @@ func Logout(systemContext *types.SystemContext, opts *LogoutOptions, args []stri return errors.Wrap(err, "get credentials for repository") } } else { - // nolint: staticcheck authConfig, err = config.GetCredentials(systemContext, registry) if err != nil { return errors.Wrap(err, "get credentials") diff --git a/vendor/github.com/containers/common/pkg/config/config.go b/vendor/github.com/containers/common/pkg/config/config.go index 008cfb642..e554bac70 100644 --- a/vendor/github.com/containers/common/pkg/config/config.go +++ b/vendor/github.com/containers/common/pkg/config/config.go @@ -274,6 +274,9 @@ type EngineConfig struct { // MachineEnabled indicates if Podman is running in a podman-machine VM MachineEnabled bool `toml:"machine_enabled,omitempty"` + // MachineImage is the image used when creating a podman-machine VM + MachineImage string `toml:"machine_image,omitempty"` + // MultiImageArchive - if true, the container engine allows for storing // archives (e.g., of the docker-archive transport) with multiple // images. By default, Podman creates single-image archives. @@ -691,8 +694,8 @@ func (c *Config) Validate() error { } func (c *EngineConfig) findRuntime() string { - // Search for crun first followed by runc and kata - for _, name := range []string{"crun", "runc", "kata"} { + // Search for crun first followed by runc, kata, runsc + for _, name := range []string{"crun", "runc", "kata", "runsc"} { for _, v := range c.OCIRuntimes[name] { if _, err := os.Stat(v); err == nil { return name diff --git a/vendor/github.com/containers/common/pkg/config/containers.conf b/vendor/github.com/containers/common/pkg/config/containers.conf index a83aa9407..0068a9a17 100644 --- a/vendor/github.com/containers/common/pkg/config/containers.conf +++ b/vendor/github.com/containers/common/pkg/config/containers.conf @@ -61,7 +61,7 @@ default_capabilities = [ # A list of sysctls to be set in containers by default, # specified as "name=value", -# for example:"net.ipv4.ping_group_range = 0 0". +# for example:"net.ipv4.ping_group_range=0 0". # default_sysctls = [ "net.ipv4.ping_group_range=0 0", @@ -381,6 +381,10 @@ default_sysctls = [ # #machine_enabled = false +# The image used when creating a podman-machine VM. +# +#machine_image = "testing" + # MultiImageArchive - if true, the container engine allows for storing archives # (e.g., of the docker-archive transport) with multiple images. By default, # Podman creates single-image archives. diff --git a/vendor/github.com/containers/common/pkg/config/default.go b/vendor/github.com/containers/common/pkg/config/default.go index a16dd0e02..66531a2ba 100644 --- a/vendor/github.com/containers/common/pkg/config/default.go +++ b/vendor/github.com/containers/common/pkg/config/default.go @@ -105,8 +105,6 @@ const ( DefaultApparmorProfile = apparmor.Profile // SystemdCgroupsManager represents systemd native cgroup manager SystemdCgroupsManager = "systemd" - // DefaultLogDriver is the default type of log files - DefaultLogDriver = "k8s-file" // DefaultLogSizeMax is the default value for the maximum log size // allowed for a container. Negative values mean that no limit is imposed. DefaultLogSizeMax = -1 @@ -339,6 +337,7 @@ func defaultConfigFromMemory() (*EngineConfig, error) { // constants. c.LockType = "shm" c.MachineEnabled = false + c.MachineImage = "testing" c.ChownCopiedFiles = true @@ -549,6 +548,7 @@ func (c *Config) LogDriver() string { return c.Containers.LogDriver } +// MachineEnabled returns if podman is running inside a VM or not func (c *Config) MachineEnabled() bool { return c.Engine.MachineEnabled } @@ -558,3 +558,9 @@ func (c *Config) MachineEnabled() bool { func (c *Config) RootlessNetworking() string { return c.Containers.RootlessNetworking } + +// MachineImage returns the image to be +// used when creating a podman-machine VM +func (c *Config) MachineImage() string { + return c.Engine.MachineImage +} diff --git a/vendor/github.com/containers/common/pkg/config/nosystemd.go b/vendor/github.com/containers/common/pkg/config/nosystemd.go index 6e39a6ccd..2a3b6fb35 100644 --- a/vendor/github.com/containers/common/pkg/config/nosystemd.go +++ b/vendor/github.com/containers/common/pkg/config/nosystemd.go @@ -1,7 +1,12 @@ -// +build !systemd +// +build !systemd !cgo package config +const ( + // DefaultLogDriver is the default type of log files + DefaultLogDriver = "k8s-file" +) + func defaultCgroupManager() string { return CgroupfsCgroupsManager } diff --git a/vendor/github.com/containers/common/pkg/config/systemd.go b/vendor/github.com/containers/common/pkg/config/systemd.go index ed014126b..fab3ea437 100644 --- a/vendor/github.com/containers/common/pkg/config/systemd.go +++ b/vendor/github.com/containers/common/pkg/config/systemd.go @@ -1,4 +1,4 @@ -// +build systemd +// +build systemd,cgo package config @@ -9,11 +9,19 @@ import ( "github.com/containers/common/pkg/cgroupv2" "github.com/containers/storage/pkg/unshare" + "github.com/coreos/go-systemd/v22/sdjournal" ) var ( - systemdOnce sync.Once - usesSystemd bool + systemdOnce sync.Once + usesSystemd bool + journaldOnce sync.Once + usesJournald bool +) + +const ( + // DefaultLogDriver is the default type of log files + DefaultLogDriver = "journald" ) func defaultCgroupManager() string { @@ -29,20 +37,17 @@ func defaultCgroupManager() string { } func defaultEventsLogger() string { - if useSystemd() { + if useJournald() { return "journald" } return "file" } func defaultLogDriver() string { - // If we decide to change the default for logdriver, it should be done here. - if useSystemd() { - return DefaultLogDriver + if useJournald() { + return "journald" } - - return DefaultLogDriver - + return "k8s-file" } func useSystemd() bool { @@ -56,3 +61,19 @@ func useSystemd() bool { }) return usesSystemd } + +func useJournald() bool { + journaldOnce.Do(func() { + if !useSystemd() { + return + } + journal, err := sdjournal.NewJournal() + if err != nil { + return + } + journal.Close() + usesJournald = true + return + }) + return usesJournald +} diff --git a/vendor/github.com/containers/common/pkg/parse/parse.go b/vendor/github.com/containers/common/pkg/parse/parse.go index 1a25957d6..02e670c50 100644 --- a/vendor/github.com/containers/common/pkg/parse/parse.go +++ b/vendor/github.com/containers/common/pkg/parse/parse.go @@ -5,6 +5,7 @@ package parse import ( "os" + "path" "path/filepath" "strings" @@ -155,7 +156,7 @@ func ValidateVolumeCtrDir(ctrDir string) error { if ctrDir == "" { return errors.New("container directory cannot be empty") } - if !filepath.IsAbs(ctrDir) { + if !path.IsAbs(ctrDir) { return errors.Errorf("invalid container path %q, must be an absolute path", ctrDir) } return nil diff --git a/vendor/github.com/containers/common/version/version.go b/vendor/github.com/containers/common/version/version.go index 1e9e48f33..572fe9bbd 100644 --- a/vendor/github.com/containers/common/version/version.go +++ b/vendor/github.com/containers/common/version/version.go @@ -1,4 +1,4 @@ package version // Version is the version of the build. -const Version = "0.42.1" +const Version = "0.43.2" diff --git a/vendor/github.com/containers/image/v5/version/version.go b/vendor/github.com/containers/image/v5/version/version.go index 8936ec087..478a03b05 100644 --- a/vendor/github.com/containers/image/v5/version/version.go +++ b/vendor/github.com/containers/image/v5/version/version.go @@ -8,7 +8,7 @@ const ( // VersionMinor is for functionality in a backwards-compatible manner VersionMinor = 15 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 0 + VersionPatch = 2 // VersionDev indicates development branch. Releases will be empty string. VersionDev = "" diff --git a/vendor/github.com/containers/storage/.cirrus.yml b/vendor/github.com/containers/storage/.cirrus.yml index 12f6f10c6..20bede452 100644 --- a/vendor/github.com/containers/storage/.cirrus.yml +++ b/vendor/github.com/containers/storage/.cirrus.yml @@ -25,7 +25,7 @@ env: # GCE project where images live IMAGE_PROJECT: "libpod-218412" # VM Image built in containers/automation_images - _BUILT_IMAGE_SUFFIX: "c6032583541653504" + _BUILT_IMAGE_SUFFIX: "c6248193773010944" FEDORA_CACHE_IMAGE_NAME: "fedora-${_BUILT_IMAGE_SUFFIX}" PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${_BUILT_IMAGE_SUFFIX}" UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${_BUILT_IMAGE_SUFFIX}" diff --git a/vendor/github.com/containers/storage/Makefile b/vendor/github.com/containers/storage/Makefile index 581961fed..dbc1f7c99 100644 --- a/vendor/github.com/containers/storage/Makefile +++ b/vendor/github.com/containers/storage/Makefile @@ -29,7 +29,7 @@ GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) GIT_BRANCH_CLEAN := $(shell echo $(GIT_BRANCH) | sed -e "s/[^[:alnum:]]/-/g") EPOCH_TEST_COMMIT := 0418ebf59f9e1f564831c0ba9378b7f8e40a1c73 NATIVETAGS := -AUTOTAGS := $(shell ./hack/btrfs_tag.sh) $(shell ./hack/libdm_tag.sh) +AUTOTAGS := $(shell ./hack/btrfs_tag.sh) $(shell ./hack/libdm_tag.sh) $(shell ./hack/libsubid_tag.sh) BUILDFLAGS := -tags "$(AUTOTAGS) $(TAGS)" $(FLAGS) GO ?= go TESTFLAGS := $(shell go test -race $(BUILDFLAGS) ./pkg/stringutils 2>&1 > /dev/null && echo -race) @@ -108,7 +108,7 @@ install.docs: docs install: install.docs lint: install.tools - tests/tools/build/golangci-lint run + tests/tools/build/golangci-lint run --build-tags="$(AUTOTAGS) $(TAGS)" help: ## this help @awk 'BEGIN {FS = ":.*?## "} /^[a-z A-Z_-]+:.*?## / {gsub(" ",",",$$1);gsub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-21s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index 02261bead..a95a46d9f 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.33.1 +1.34.1 diff --git a/vendor/github.com/containers/storage/drivers/overlay/check_115.go b/vendor/github.com/containers/storage/drivers/overlay/check_115.go new file mode 100644 index 000000000..9ad1b863d --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/overlay/check_115.go @@ -0,0 +1,42 @@ +// +build !go1.16 + +package overlay + +import ( + "os" + "path/filepath" + "strings" + + "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/system" +) + +func scanForMountProgramIndicators(home string) (detected bool, err error) { + err = filepath.Walk(home, func(path string, info os.FileInfo, err error) error { + if detected { + return filepath.SkipDir + } + if err != nil { + return err + } + basename := filepath.Base(path) + if strings.HasPrefix(basename, archive.WhiteoutPrefix) { + detected = true + return filepath.SkipDir + } + if info.IsDir() { + xattrs, err := system.Llistxattr(path) + if err != nil { + return err + } + for _, xattr := range xattrs { + if strings.HasPrefix(xattr, "user.fuseoverlayfs.") || strings.HasPrefix(xattr, "user.containers.") { + detected = true + return filepath.SkipDir + } + } + } + return nil + }) + return detected, err +} diff --git a/vendor/github.com/containers/storage/drivers/overlay/check_116.go b/vendor/github.com/containers/storage/drivers/overlay/check_116.go new file mode 100644 index 000000000..6d7913cbf --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/overlay/check_116.go @@ -0,0 +1,42 @@ +// +build go1.16 + +package overlay + +import ( + "io/fs" + "path/filepath" + "strings" + + "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/system" +) + +func scanForMountProgramIndicators(home string) (detected bool, err error) { + err = filepath.WalkDir(home, func(path string, d fs.DirEntry, err error) error { + if detected { + return fs.SkipDir + } + if err != nil { + return err + } + basename := filepath.Base(path) + if strings.HasPrefix(basename, archive.WhiteoutPrefix) { + detected = true + return fs.SkipDir + } + if d.IsDir() { + xattrs, err := system.Llistxattr(path) + if err != nil { + return err + } + for _, xattr := range xattrs { + if strings.HasPrefix(xattr, "user.fuseoverlayfs.") || strings.HasPrefix(xattr, "user.containers.") { + detected = true + return fs.SkipDir + } + } + } + return nil + }) + return detected, err +} diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index ecfbae916..f546f9b10 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -266,9 +266,8 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) } if opts.mountProgram != "" { - f, err := os.Create(getMountProgramFlagFile(home)) - if err == nil { - f.Close() + if err := ioutil.WriteFile(getMountProgramFlagFile(home), []byte("true"), 0600); err != nil { + return nil, err } } else { // check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs @@ -542,9 +541,29 @@ func SupportsNativeOverlay(graphroot, rundir string) (bool, error) { home := filepath.Join(graphroot, "overlay") runhome := filepath.Join(rundir, "overlay") - if _, err := os.Stat(getMountProgramFlagFile(home)); err == nil { + var contents string + flagContent, err := ioutil.ReadFile(getMountProgramFlagFile(home)) + if err == nil { + contents = strings.TrimSpace(string(flagContent)) + } + switch contents { + case "true": logrus.Debugf("overlay storage already configured with a mount-program") return false, nil + default: + needsMountProgram, err := scanForMountProgramIndicators(home) + if err != nil && !os.IsNotExist(err) { + return false, err + } + if err := ioutil.WriteFile(getMountProgramFlagFile(home), []byte(fmt.Sprintf("%t", needsMountProgram)), 0600); err != nil && !os.IsNotExist(err) { + return false, err + } + if needsMountProgram { + return false, nil + } + // fall through to check if we find ourselves needing to use a + // mount program now + case "false": } for _, dir := range []string{home, runhome} { @@ -1922,7 +1941,7 @@ func (al *additionalLayer) Info() (io.ReadCloser, error) { return os.Open(filepath.Join(al.path, "info")) } -// Blob returns a reader of the raw contents of this leyer. +// Blob returns a reader of the raw contents of this layer. func (al *additionalLayer) Blob() (io.ReadCloser, error) { return os.Open(filepath.Join(al.path, "blob")) } diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index e4f484d6b..d2d438d93 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -3,14 +3,14 @@ go 1.14 module github.com/containers/storage require ( - github.com/BurntSushi/toml v0.3.1 + github.com/BurntSushi/toml v0.4.1 github.com/Microsoft/go-winio v0.5.0 github.com/Microsoft/hcsshim v0.8.20 github.com/docker/go-units v0.4.0 github.com/google/go-intervals v0.0.2 github.com/hashicorp/go-multierror v1.1.1 github.com/json-iterator/go v1.1.11 - github.com/klauspost/compress v1.13.1 + github.com/klauspost/compress v1.13.4 github.com/klauspost/pgzip v1.2.5 github.com/mattn/go-shellwords v1.0.12 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible @@ -18,7 +18,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/runc v1.0.1 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 - github.com/opencontainers/selinux v1.8.2 + github.com/opencontainers/selinux v1.8.4 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 diff --git a/vendor/github.com/containers/storage/go.sum b/vendor/github.com/containers/storage/go.sum index 2607dbc9b..da7a8f53e 100644 --- a/vendor/github.com/containers/storage/go.sum +++ b/vendor/github.com/containers/storage/go.sum @@ -34,8 +34,9 @@ github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= +github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= @@ -387,8 +388,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ= -github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= +github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -480,8 +481,9 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.m github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= -github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/opencontainers/selinux v1.8.4 h1:krlgQ6/j9CkCXT5oW0yVXdQFOME3NjKuuAZXuR6O7P4= +github.com/opencontainers/selinux v1.8.4/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index fa0dce033..b85ff7e70 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -27,6 +27,7 @@ import ( digest "github.com/opencontainers/go-digest" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" + "github.com/sirupsen/logrus" "github.com/vbatts/tar-split/archive/tar" "github.com/vbatts/tar-split/tar/asm" "github.com/vbatts/tar-split/tar/storage" @@ -1407,7 +1408,7 @@ func (r *layerStore) Diff(from, to string, options *DiffOptions) (io.ReadCloser, if ad, ok := r.driver.(drivers.AdditionalLayerStoreDriver); ok { if aLayer, err := ad.LookupAdditionalLayerByID(to); err == nil { - // This is an additional layer. We leverage blob API for aquiring the reproduced raw blob. + // This is an additional layer. We leverage blob API for acquiring the reproduced raw blob. info, err := aLayer.Info() if err != nil { aLayer.Release() @@ -1529,6 +1530,9 @@ func (r *layerStore) ApplyDiff(to string, diff io.Reader) (size int64, err error if err != nil { compressor = pgzip.NewWriter(&tsdata) } + if err := compressor.SetConcurrency(1024*1024, 1); err != nil { // 1024*1024 is the hard-coded default; we're not changing that + logrus.Infof("error setting compression concurrency threads to 1: %v; ignoring", err) + } metadata := storage.NewJSONPacker(compressor) uncompressed, err := archive.DecompressStream(defragmented) if err != nil { diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools.go b/vendor/github.com/containers/storage/pkg/idtools/idtools.go index 34345d145..83bc8c34f 100644 --- a/vendor/github.com/containers/storage/pkg/idtools/idtools.go +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools.go @@ -146,11 +146,11 @@ type IDMappings struct { // using the data from /etc/sub{uid,gid} ranges, creates the // proper uid and gid remapping ranges for that user/group pair func NewIDMappings(username, groupname string) (*IDMappings, error) { - subuidRanges, err := parseSubuid(username) + subuidRanges, err := readSubuid(username) if err != nil { return nil, err } - subgidRanges, err := parseSubgid(groupname) + subgidRanges, err := readSubgid(groupname) if err != nil { return nil, err } @@ -244,14 +244,6 @@ func createIDMap(subidRanges ranges) []IDMap { return idMap } -func parseSubuid(username string) (ranges, error) { - return parseSubidFile(subuidFileName, username) -} - -func parseSubgid(username string) (ranges, error) { - return parseSubidFile(subgidFileName, username) -} - // parseSubidFile will read the appropriate file (/etc/subuid or /etc/subgid) // and return all found ranges for a specified username. If the special value // "ALL" is supplied for username, then all ranges in the file will be returned diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools_supported.go b/vendor/github.com/containers/storage/pkg/idtools/idtools_supported.go new file mode 100644 index 000000000..db50a62e4 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools_supported.go @@ -0,0 +1,61 @@ +// +build linux,cgo,libsubid + +package idtools + +import ( + "unsafe" + + "github.com/pkg/errors" +) + +/* +#cgo LDFLAGS: -l subid +#include <shadow/subid.h> +#include <stdlib.h> +const char *Prog = "storage"; +struct subid_range get_range(struct subid_range *ranges, int i) +{ + return ranges[i]; +} +*/ +import "C" + +func readSubid(username string, isUser bool) (ranges, error) { + var ret ranges + if username == "ALL" { + return nil, errors.New("username ALL not supported") + } + + cUsername := C.CString(username) + defer C.free(unsafe.Pointer(cUsername)) + + var nRanges C.int + var cRanges *C.struct_subid_range + if isUser { + nRanges = C.get_subuid_ranges(cUsername, &cRanges) + } else { + nRanges = C.get_subgid_ranges(cUsername, &cRanges) + } + if nRanges < 0 { + return nil, errors.New("cannot read subids") + } + defer C.free(unsafe.Pointer(cRanges)) + + for i := 0; i < int(nRanges); i++ { + r := C.get_range(cRanges, C.int(i)) + newRange := subIDRange{ + Start: int(r.start), + Length: int(r.count), + } + ret = append(ret, newRange) + } + return ret, nil +} + +func readSubuid(username string) (ranges, error) { + return readSubid(username, true) +} + +func readSubgid(username string) (ranges, error) { + return readSubid(username, false) +} diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools_unsupported.go b/vendor/github.com/containers/storage/pkg/idtools/idtools_unsupported.go new file mode 100644 index 000000000..84da1b764 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools_unsupported.go @@ -0,0 +1,11 @@ +// +build !linux !libsubid !cgo + +package idtools + +func readSubuid(username string) (ranges, error) { + return parseSubidFile(subuidFileName, username) +} + +func readSubgid(username string) (ranges, error) { + return parseSubidFile(subgidFileName, username) +} diff --git a/vendor/github.com/containers/storage/pkg/idtools/usergroupadd_linux.go b/vendor/github.com/containers/storage/pkg/idtools/usergroupadd_linux.go index 9da7975e2..3dd7bf210 100644 --- a/vendor/github.com/containers/storage/pkg/idtools/usergroupadd_linux.go +++ b/vendor/github.com/containers/storage/pkg/idtools/usergroupadd_linux.go @@ -91,7 +91,7 @@ func createSubordinateRanges(name string) error { // first, we should verify that ranges weren't automatically created // by the distro tooling - ranges, err := parseSubuid(name) + ranges, err := readSubuid(name) if err != nil { return fmt.Errorf("Error while looking for subuid ranges for user %q: %v", name, err) } @@ -107,7 +107,7 @@ func createSubordinateRanges(name string) error { } } - ranges, err = parseSubgid(name) + ranges, err = readSubgid(name) if err != nil { return fmt.Errorf("Error while looking for subgid ranges for user %q: %v", name, err) } @@ -126,7 +126,7 @@ func createSubordinateRanges(name string) error { } func findNextUIDRange() (int, error) { - ranges, err := parseSubuid("ALL") + ranges, err := readSubuid("ALL") if err != nil { return -1, fmt.Errorf("Couldn't parse all ranges in /etc/subuid file: %v", err) } @@ -135,7 +135,7 @@ func findNextUIDRange() (int, error) { } func findNextGIDRange() (int, error) { - ranges, err := parseSubgid("ALL") + ranges, err := readSubgid("ALL") if err != nil { return -1, fmt.Errorf("Couldn't parse all ranges in /etc/subgid file: %v", err) } diff --git a/vendor/github.com/containers/storage/types/utils.go b/vendor/github.com/containers/storage/types/utils.go index 4d62b151a..b7ab07342 100644 --- a/vendor/github.com/containers/storage/types/utils.go +++ b/vendor/github.com/containers/storage/types/utils.go @@ -155,8 +155,14 @@ func getRootlessUID() int { } func expandEnvPath(path string, rootlessUID int) (string, error) { + var err error path = strings.Replace(path, "$UID", strconv.Itoa(rootlessUID), -1) - return filepath.Clean(os.ExpandEnv(path)), nil + path = os.ExpandEnv(path) + newpath, err := filepath.EvalSymlinks(path) + if err != nil { + newpath = filepath.Clean(path) + } + return newpath, nil } func DefaultConfigFile(rootless bool) (string, error) { diff --git a/vendor/github.com/fsnotify/fsnotify/.mailmap b/vendor/github.com/fsnotify/fsnotify/.mailmap new file mode 100644 index 000000000..a04f2907f --- /dev/null +++ b/vendor/github.com/fsnotify/fsnotify/.mailmap @@ -0,0 +1,2 @@ +Chris Howey <howeyc@gmail.com> <chris@howey.me> +Nathan Youngman <git@nathany.com> <4566+nathany@users.noreply.github.com> diff --git a/vendor/github.com/fsnotify/fsnotify/.travis.yml b/vendor/github.com/fsnotify/fsnotify/.travis.yml deleted file mode 100644 index a9c30165c..000000000 --- a/vendor/github.com/fsnotify/fsnotify/.travis.yml +++ /dev/null @@ -1,36 +0,0 @@ -sudo: false -language: go - -go: - - "stable" - - "1.11.x" - - "1.10.x" - - "1.9.x" - -matrix: - include: - - go: "stable" - env: GOLINT=true - allow_failures: - - go: tip - fast_finish: true - - -before_install: - - if [ ! -z "${GOLINT}" ]; then go get -u golang.org/x/lint/golint; fi - -script: - - go test --race ./... - -after_script: - - test -z "$(gofmt -s -l -w . | tee /dev/stderr)" - - if [ ! -z "${GOLINT}" ]; then echo running golint; golint --set_exit_status ./...; else echo skipping golint; fi - - go vet ./... - -os: - - linux - - osx - - windows - -notifications: - email: false diff --git a/vendor/github.com/fsnotify/fsnotify/AUTHORS b/vendor/github.com/fsnotify/fsnotify/AUTHORS index 5ab5d41c5..6cbabe5ef 100644 --- a/vendor/github.com/fsnotify/fsnotify/AUTHORS +++ b/vendor/github.com/fsnotify/fsnotify/AUTHORS @@ -4,35 +4,44 @@ # You can update this list using the following command: # -# $ git shortlog -se | awk '{print $2 " " $3 " " $4}' +# $ (head -n10 AUTHORS && git shortlog -se | sed -E 's/^\s+[0-9]+\t//') | tee AUTHORS # Please keep the list sorted. Aaron L <aaron@bettercoder.net> Adrien Bustany <adrien@bustany.org> +Alexey Kazakov <alkazako@redhat.com> Amit Krishnan <amit.krishnan@oracle.com> Anmol Sethi <me@anmol.io> Bjørn Erik Pedersen <bjorn.erik.pedersen@gmail.com> +Brian Goff <cpuguy83@gmail.com> Bruno Bigras <bigras.bruno@gmail.com> Caleb Spare <cespare@gmail.com> Case Nelson <case@teammating.com> -Chris Howey <chris@howey.me> <howeyc@gmail.com> +Chris Howey <howeyc@gmail.com> Christoffer Buchholz <christoffer.buchholz@gmail.com> Daniel Wagner-Hall <dawagner@gmail.com> Dave Cheney <dave@cheney.net> +Eric Lin <linxiulei@gmail.com> Evan Phoenix <evan@fallingsnow.net> Francisco Souza <f@souza.cc> +Gautam Dey <gautam.dey77@gmail.com> Hari haran <hariharan.uno@gmail.com> -John C Barstow +Ichinose Shogo <shogo82148@gmail.com> +Johannes Ebke <johannes@ebke.org> +John C Barstow <jbowtie@amathaine.com> Kelvin Fo <vmirage@gmail.com> Ken-ichirou MATSUZAWA <chamas@h4.dion.ne.jp> Matt Layher <mdlayher@gmail.com> +Matthias Stone <matthias@bellstone.ca> Nathan Youngman <git@nathany.com> Nickolai Zeldovich <nickolai@csail.mit.edu> +Oliver Bristow <evilumbrella+github@gmail.com> Patrick <patrick@dropbox.com> Paul Hammond <paul@paulhammond.org> Pawel Knap <pawelknap88@gmail.com> Pieter Droogendijk <pieter@binky.org.uk> +Pratik Shinde <pratikshinde320@gmail.com> Pursuit92 <JoshChase@techpursuit.net> Riku Voipio <riku.voipio@linaro.org> Rob Figueiredo <robfig@gmail.com> @@ -41,6 +50,7 @@ Slawek Ligus <root@ooz.ie> Soge Zhang <zhssoge@gmail.com> Tiffany Jernigan <tiffany.jernigan@intel.com> Tilak Sharma <tilaks@google.com> +Tobias Klauser <tobias.klauser@gmail.com> Tom Payne <twpayne@gmail.com> Travis Cline <travis.cline@gmail.com> Tudor Golubenco <tudor.g@gmail.com> diff --git a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md index be4d7ea2c..a438fe4b4 100644 --- a/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md +++ b/vendor/github.com/fsnotify/fsnotify/CHANGELOG.md @@ -1,6 +1,28 @@ # Changelog -## v1.4.7 / 2018-01-09 +All notable changes to this project will be documented in this file. + +The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), +and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). + +## [Unreleased] + +## [1.5.1] - 2021-08-24 + +* Revert Add AddRaw to not follow symlinks + +## [1.5.0] - 2021-08-20 + +* Go: Increase minimum required version to Go 1.12 [#381](https://github.com/fsnotify/fsnotify/pull/381) +* Feature: Add AddRaw method which does not follow symlinks when adding a watch [#289](https://github.com/fsnotify/fsnotify/pull/298) +* Windows: Follow symlinks by default like on all other systems [#289](https://github.com/fsnotify/fsnotify/pull/289) +* CI: Use GitHub Actions for CI and cover go 1.12-1.17 + [#378](https://github.com/fsnotify/fsnotify/pull/378) + [#381](https://github.com/fsnotify/fsnotify/pull/381) + [#385](https://github.com/fsnotify/fsnotify/pull/385) +* Go 1.14+: Fix unsafe pointer conversion [#325](https://github.com/fsnotify/fsnotify/pull/325) + +## [1.4.7] - 2018-01-09 * BSD/macOS: Fix possible deadlock on closing the watcher on kqueue (thanks @nhooyr and @glycerine) * Tests: Fix missing verb on format string (thanks @rchiossi) @@ -10,62 +32,62 @@ * Linux: Properly handle inotify's IN_Q_OVERFLOW event (thanks @zeldovich) * Docs: replace references to OS X with macOS -## v1.4.2 / 2016-10-10 +## [1.4.2] - 2016-10-10 * Linux: use InotifyInit1 with IN_CLOEXEC to stop leaking a file descriptor to a child process when using fork/exec [#178](https://github.com/fsnotify/fsnotify/pull/178) (thanks @pattyshack) -## v1.4.1 / 2016-10-04 +## [1.4.1] - 2016-10-04 * Fix flaky inotify stress test on Linux [#177](https://github.com/fsnotify/fsnotify/pull/177) (thanks @pattyshack) -## v1.4.0 / 2016-10-01 +## [1.4.0] - 2016-10-01 * add a String() method to Event.Op [#165](https://github.com/fsnotify/fsnotify/pull/165) (thanks @oozie) -## v1.3.1 / 2016-06-28 +## [1.3.1] - 2016-06-28 * Windows: fix for double backslash when watching the root of a drive [#151](https://github.com/fsnotify/fsnotify/issues/151) (thanks @brunoqc) -## v1.3.0 / 2016-04-19 +## [1.3.0] - 2016-04-19 * Support linux/arm64 by [patching](https://go-review.googlesource.com/#/c/21971/) x/sys/unix and switching to to it from syscall (thanks @suihkulokki) [#135](https://github.com/fsnotify/fsnotify/pull/135) -## v1.2.10 / 2016-03-02 +## [1.2.10] - 2016-03-02 * Fix golint errors in windows.go [#121](https://github.com/fsnotify/fsnotify/pull/121) (thanks @tiffanyfj) -## v1.2.9 / 2016-01-13 +## [1.2.9] - 2016-01-13 kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsnotify/pull/111) (thanks @bep) -## v1.2.8 / 2015-12-17 +## [1.2.8] - 2015-12-17 * kqueue: fix race condition in Close [#105](https://github.com/fsnotify/fsnotify/pull/105) (thanks @djui for reporting the issue and @ppknap for writing a failing test) * inotify: fix race in test * enable race detection for continuous integration (Linux, Mac, Windows) -## v1.2.5 / 2015-10-17 +## [1.2.5] - 2015-10-17 * inotify: use epoll_create1 for arm64 support (requires Linux 2.6.27 or later) [#100](https://github.com/fsnotify/fsnotify/pull/100) (thanks @suihkulokki) * inotify: fix path leaks [#73](https://github.com/fsnotify/fsnotify/pull/73) (thanks @chamaken) * kqueue: watch for rename events on subdirectories [#83](https://github.com/fsnotify/fsnotify/pull/83) (thanks @guotie) * kqueue: avoid infinite loops from symlinks cycles [#101](https://github.com/fsnotify/fsnotify/pull/101) (thanks @illicitonion) -## v1.2.1 / 2015-10-14 +## [1.2.1] - 2015-10-14 * kqueue: don't watch named pipes [#98](https://github.com/fsnotify/fsnotify/pull/98) (thanks @evanphx) -## v1.2.0 / 2015-02-08 +## [1.2.0] - 2015-02-08 * inotify: use epoll to wake up readEvents [#66](https://github.com/fsnotify/fsnotify/pull/66) (thanks @PieterD) * inotify: closing watcher should now always shut down goroutine [#63](https://github.com/fsnotify/fsnotify/pull/63) (thanks @PieterD) * kqueue: close kqueue after removing watches, fixes [#59](https://github.com/fsnotify/fsnotify/issues/59) -## v1.1.1 / 2015-02-05 +## [1.1.1] - 2015-02-05 * inotify: Retry read on EINTR [#61](https://github.com/fsnotify/fsnotify/issues/61) (thanks @PieterD) -## v1.1.0 / 2014-12-12 +## [1.1.0] - 2014-12-12 * kqueue: rework internals [#43](https://github.com/fsnotify/fsnotify/pull/43) * add low-level functions @@ -77,22 +99,22 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn * kqueue: fix regression in rework causing subdirectories to be watched [#48](https://github.com/fsnotify/fsnotify/issues/48) * kqueue: cleanup internal watch before sending remove event [#51](https://github.com/fsnotify/fsnotify/issues/51) -## v1.0.4 / 2014-09-07 +## [1.0.4] - 2014-09-07 * kqueue: add dragonfly to the build tags. * Rename source code files, rearrange code so exported APIs are at the top. * Add done channel to example code. [#37](https://github.com/fsnotify/fsnotify/pull/37) (thanks @chenyukang) -## v1.0.3 / 2014-08-19 +## [1.0.3] - 2014-08-19 * [Fix] Windows MOVED_TO now translates to Create like on BSD and Linux. [#36](https://github.com/fsnotify/fsnotify/issues/36) -## v1.0.2 / 2014-08-17 +## [1.0.2] - 2014-08-17 * [Fix] Missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) * [Fix] Make ./path and path equivalent. (thanks @zhsso) -## v1.0.0 / 2014-08-15 +## [1.0.0] - 2014-08-15 * [API] Remove AddWatch on Windows, use Add. * Improve documentation for exported identifiers. [#30](https://github.com/fsnotify/fsnotify/issues/30) @@ -146,51 +168,51 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn * no tests for the current implementation * not fully implemented on Windows [#93](https://github.com/howeyc/fsnotify/issues/93#issuecomment-39285195) -## v0.9.3 / 2014-12-31 +## [0.9.3] - 2014-12-31 * kqueue: cleanup internal watch before sending remove event [#51](https://github.com/fsnotify/fsnotify/issues/51) -## v0.9.2 / 2014-08-17 +## [0.9.2] - 2014-08-17 * [Backport] Fix missing create events on macOS. [#14](https://github.com/fsnotify/fsnotify/issues/14) (thanks @zhsso) -## v0.9.1 / 2014-06-12 +## [0.9.1] - 2014-06-12 * Fix data race on kevent buffer (thanks @tilaks) [#98](https://github.com/howeyc/fsnotify/pull/98) -## v0.9.0 / 2014-01-17 +## [0.9.0] - 2014-01-17 * IsAttrib() for events that only concern a file's metadata [#79][] (thanks @abustany) * [Fix] kqueue: fix deadlock [#77][] (thanks @cespare) * [NOTICE] Development has moved to `code.google.com/p/go.exp/fsnotify` in preparation for inclusion in the Go standard library. -## v0.8.12 / 2013-11-13 +## [0.8.12] - 2013-11-13 * [API] Remove FD_SET and friends from Linux adapter -## v0.8.11 / 2013-11-02 +## [0.8.11] - 2013-11-02 * [Doc] Add Changelog [#72][] (thanks @nathany) * [Doc] Spotlight and double modify events on macOS [#62][] (reported by @paulhammond) -## v0.8.10 / 2013-10-19 +## [0.8.10] - 2013-10-19 * [Fix] kqueue: remove file watches when parent directory is removed [#71][] (reported by @mdwhatcott) * [Fix] kqueue: race between Close and readEvents [#70][] (reported by @bernerdschaefer) * [Doc] specify OS-specific limits in README (thanks @debrando) -## v0.8.9 / 2013-09-08 +## [0.8.9] - 2013-09-08 * [Doc] Contributing (thanks @nathany) * [Doc] update package path in example code [#63][] (thanks @paulhammond) * [Doc] GoCI badge in README (Linux only) [#60][] * [Doc] Cross-platform testing with Vagrant [#59][] (thanks @nathany) -## v0.8.8 / 2013-06-17 +## [0.8.8] - 2013-06-17 * [Fix] Windows: handle `ERROR_MORE_DATA` on Windows [#49][] (thanks @jbowtie) -## v0.8.7 / 2013-06-03 +## [0.8.7] - 2013-06-03 * [API] Make syscall flags internal * [Fix] inotify: ignore event changes @@ -198,74 +220,74 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn * [Fix] tests on Windows * lower case error messages -## v0.8.6 / 2013-05-23 +## [0.8.6] - 2013-05-23 * kqueue: Use EVT_ONLY flag on Darwin * [Doc] Update README with full example -## v0.8.5 / 2013-05-09 +## [0.8.5] - 2013-05-09 * [Fix] inotify: allow monitoring of "broken" symlinks (thanks @tsg) -## v0.8.4 / 2013-04-07 +## [0.8.4] - 2013-04-07 * [Fix] kqueue: watch all file events [#40][] (thanks @ChrisBuchholz) -## v0.8.3 / 2013-03-13 +## [0.8.3] - 2013-03-13 * [Fix] inoitfy/kqueue memory leak [#36][] (reported by @nbkolchin) * [Fix] kqueue: use fsnFlags for watching a directory [#33][] (reported by @nbkolchin) -## v0.8.2 / 2013-02-07 +## [0.8.2] - 2013-02-07 * [Doc] add Authors * [Fix] fix data races for map access [#29][] (thanks @fsouza) -## v0.8.1 / 2013-01-09 +## [0.8.1] - 2013-01-09 * [Fix] Windows path separators * [Doc] BSD License -## v0.8.0 / 2012-11-09 +## [0.8.0] - 2012-11-09 * kqueue: directory watching improvements (thanks @vmirage) * inotify: add `IN_MOVED_TO` [#25][] (requested by @cpisto) * [Fix] kqueue: deleting watched directory [#24][] (reported by @jakerr) -## v0.7.4 / 2012-10-09 +## [0.7.4] - 2012-10-09 * [Fix] inotify: fixes from https://codereview.appspot.com/5418045/ (ugorji) * [Fix] kqueue: preserve watch flags when watching for delete [#21][] (reported by @robfig) * [Fix] kqueue: watch the directory even if it isn't a new watch (thanks @robfig) * [Fix] kqueue: modify after recreation of file -## v0.7.3 / 2012-09-27 +## [0.7.3] - 2012-09-27 * [Fix] kqueue: watch with an existing folder inside the watched folder (thanks @vmirage) * [Fix] kqueue: no longer get duplicate CREATE events -## v0.7.2 / 2012-09-01 +## [0.7.2] - 2012-09-01 * kqueue: events for created directories -## v0.7.1 / 2012-07-14 +## [0.7.1] - 2012-07-14 * [Fix] for renaming files -## v0.7.0 / 2012-07-02 +## [0.7.0] - 2012-07-02 * [Feature] FSNotify flags * [Fix] inotify: Added file name back to event path -## v0.6.0 / 2012-06-06 +## [0.6.0] - 2012-06-06 * kqueue: watch files after directory created (thanks @tmc) -## v0.5.1 / 2012-05-22 +## [0.5.1] - 2012-05-22 * [Fix] inotify: remove all watches before Close() -## v0.5.0 / 2012-05-03 +## [0.5.0] - 2012-05-03 * [API] kqueue: return errors during watch instead of sending over channel * kqueue: match symlink behavior on Linux @@ -273,22 +295,22 @@ kqueue: Fix logic for CREATE after REMOVE [#111](https://github.com/fsnotify/fsn * [Fix] kqueue: handle EINTR (reported by @robfig) * [Doc] Godoc example [#1][] (thanks @davecheney) -## v0.4.0 / 2012-03-30 +## [0.4.0] - 2012-03-30 * Go 1 released: build with go tool * [Feature] Windows support using winfsnotify * Windows does not have attribute change notifications * Roll attribute notifications into IsModify -## v0.3.0 / 2012-02-19 +## [0.3.0] - 2012-02-19 * kqueue: add files when watch directory -## v0.2.0 / 2011-12-30 +## [0.2.0] - 2011-12-30 * update to latest Go weekly code -## v0.1.0 / 2011-10-19 +## [0.1.0] - 2011-10-19 * kqueue: add watch on file creation to match inotify * kqueue: create file event diff --git a/vendor/github.com/fsnotify/fsnotify/README.md b/vendor/github.com/fsnotify/fsnotify/README.md index b2629e522..df57b1b28 100644 --- a/vendor/github.com/fsnotify/fsnotify/README.md +++ b/vendor/github.com/fsnotify/fsnotify/README.md @@ -12,9 +12,9 @@ Cross platform: Windows, Linux, BSD and macOS. | Adapter | OS | Status | | --------------------- | -------------------------------- | ------------------------------------------------------------------------------------------------------------------------------- | -| inotify | Linux 2.6.27 or later, Android\* | Supported [](https://travis-ci.org/fsnotify/fsnotify) | -| kqueue | BSD, macOS, iOS\* | Supported [](https://travis-ci.org/fsnotify/fsnotify) | -| ReadDirectoryChangesW | Windows | Supported [](https://travis-ci.org/fsnotify/fsnotify) | +| inotify | Linux 2.6.27 or later, Android\* | Supported | +| kqueue | BSD, macOS, iOS\* | Supported | +| ReadDirectoryChangesW | Windows | Supported | | FSEvents | macOS | [Planned](https://github.com/fsnotify/fsnotify/issues/11) | | FEN | Solaris 11 | [In Progress](https://github.com/fsnotify/fsnotify/issues/12) | | fanotify | Linux 2.6.37+ | [Planned](https://github.com/fsnotify/fsnotify/issues/114) | diff --git a/vendor/github.com/fsnotify/fsnotify/fen.go b/vendor/github.com/fsnotify/fsnotify/fen.go index ced39cb88..b3ac3d8f5 100644 --- a/vendor/github.com/fsnotify/fsnotify/fen.go +++ b/vendor/github.com/fsnotify/fsnotify/fen.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build solaris // +build solaris package fsnotify diff --git a/vendor/github.com/fsnotify/fsnotify/fsnotify.go b/vendor/github.com/fsnotify/fsnotify/fsnotify.go index 89cab046d..0f4ee52e8 100644 --- a/vendor/github.com/fsnotify/fsnotify/fsnotify.go +++ b/vendor/github.com/fsnotify/fsnotify/fsnotify.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !plan9 // +build !plan9 // Package fsnotify provides a platform-independent interface for file system notifications. diff --git a/vendor/github.com/fsnotify/fsnotify/go.mod b/vendor/github.com/fsnotify/fsnotify/go.mod index ff11e13f2..54089e48b 100644 --- a/vendor/github.com/fsnotify/fsnotify/go.mod +++ b/vendor/github.com/fsnotify/fsnotify/go.mod @@ -2,4 +2,6 @@ module github.com/fsnotify/fsnotify go 1.13 -require golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9 +require golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c + +retract v1.5.0 diff --git a/vendor/github.com/fsnotify/fsnotify/go.sum b/vendor/github.com/fsnotify/fsnotify/go.sum index f60af9855..0f478630c 100644 --- a/vendor/github.com/fsnotify/fsnotify/go.sum +++ b/vendor/github.com/fsnotify/fsnotify/go.sum @@ -1,2 +1,2 @@ -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9 h1:L2auWcuQIvxz9xSEqzESnV/QN/gNRXNApHi3fYwl2w0= -golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/vendor/github.com/fsnotify/fsnotify/inotify.go b/vendor/github.com/fsnotify/fsnotify/inotify.go index d9fd1b88a..eb87699b5 100644 --- a/vendor/github.com/fsnotify/fsnotify/inotify.go +++ b/vendor/github.com/fsnotify/fsnotify/inotify.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build linux // +build linux package fsnotify @@ -272,7 +273,7 @@ func (w *Watcher) readEvents() { if nameLen > 0 { // Point "bytes" at the first byte of the filename - bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent])) + bytes := (*[unix.PathMax]byte)(unsafe.Pointer(&buf[offset+unix.SizeofInotifyEvent]))[:nameLen:nameLen] // The filename is padded with NULL bytes. TrimRight() gets rid of those. name += "/" + strings.TrimRight(string(bytes[0:nameLen]), "\000") } diff --git a/vendor/github.com/fsnotify/fsnotify/inotify_poller.go b/vendor/github.com/fsnotify/fsnotify/inotify_poller.go index b33f2b4d4..e9ff9439f 100644 --- a/vendor/github.com/fsnotify/fsnotify/inotify_poller.go +++ b/vendor/github.com/fsnotify/fsnotify/inotify_poller.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build linux // +build linux package fsnotify diff --git a/vendor/github.com/fsnotify/fsnotify/kqueue.go b/vendor/github.com/fsnotify/fsnotify/kqueue.go index 86e76a3d6..368f5b790 100644 --- a/vendor/github.com/fsnotify/fsnotify/kqueue.go +++ b/vendor/github.com/fsnotify/fsnotify/kqueue.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build freebsd || openbsd || netbsd || dragonfly || darwin // +build freebsd openbsd netbsd dragonfly darwin package fsnotify diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go b/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go index 2306c4620..36cc3845b 100644 --- a/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go +++ b/vendor/github.com/fsnotify/fsnotify/open_mode_bsd.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build freebsd || openbsd || netbsd || dragonfly // +build freebsd openbsd netbsd dragonfly package fsnotify diff --git a/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go b/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go index 870c4d6d1..98cd8476f 100644 --- a/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go +++ b/vendor/github.com/fsnotify/fsnotify/open_mode_darwin.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build darwin // +build darwin package fsnotify diff --git a/vendor/github.com/fsnotify/fsnotify/windows.go b/vendor/github.com/fsnotify/fsnotify/windows.go index 09436f31d..c02b75f7c 100644 --- a/vendor/github.com/fsnotify/fsnotify/windows.go +++ b/vendor/github.com/fsnotify/fsnotify/windows.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build windows // +build windows package fsnotify diff --git a/vendor/github.com/klauspost/compress/.gitattributes b/vendor/github.com/klauspost/compress/.gitattributes new file mode 100644 index 000000000..402433593 --- /dev/null +++ b/vendor/github.com/klauspost/compress/.gitattributes @@ -0,0 +1,2 @@ +* -text +*.bin -text -diff diff --git a/vendor/github.com/klauspost/compress/.gitignore b/vendor/github.com/klauspost/compress/.gitignore new file mode 100644 index 000000000..b35f8449b --- /dev/null +++ b/vendor/github.com/klauspost/compress/.gitignore @@ -0,0 +1,25 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof +/s2/cmd/_s2sx/sfx-exe diff --git a/vendor/github.com/klauspost/compress/.goreleaser.yml b/vendor/github.com/klauspost/compress/.goreleaser.yml new file mode 100644 index 000000000..c9014ce1d --- /dev/null +++ b/vendor/github.com/klauspost/compress/.goreleaser.yml @@ -0,0 +1,137 @@ +# This is an example goreleaser.yaml file with some sane defaults. +# Make sure to check the documentation at http://goreleaser.com +before: + hooks: + - ./gen.sh + +builds: + - + id: "s2c" + binary: s2c + main: ./s2/cmd/s2c/main.go + flags: + - -trimpath + env: + - CGO_ENABLED=0 + goos: + - aix + - linux + - freebsd + - netbsd + - windows + - darwin + goarch: + - 386 + - amd64 + - arm + - arm64 + - ppc64 + - ppc64le + - mips64 + - mips64le + goarm: + - 7 + - + id: "s2d" + binary: s2d + main: ./s2/cmd/s2d/main.go + flags: + - -trimpath + env: + - CGO_ENABLED=0 + goos: + - aix + - linux + - freebsd + - netbsd + - windows + - darwin + goarch: + - 386 + - amd64 + - arm + - arm64 + - ppc64 + - ppc64le + - mips64 + - mips64le + goarm: + - 7 + - + id: "s2sx" + binary: s2sx + main: ./s2/cmd/_s2sx/main.go + flags: + - -modfile=s2sx.mod + - -trimpath + env: + - CGO_ENABLED=0 + goos: + - aix + - linux + - freebsd + - netbsd + - windows + - darwin + goarch: + - 386 + - amd64 + - arm + - arm64 + - ppc64 + - ppc64le + - mips64 + - mips64le + goarm: + - 7 + +archives: + - + id: s2-binaries + name_template: "s2-{{ .Os }}_{{ .Arch }}_{{ .Version }}" + replacements: + aix: AIX + darwin: OSX + linux: Linux + windows: Windows + 386: i386 + amd64: x86_64 + freebsd: FreeBSD + netbsd: NetBSD + format_overrides: + - goos: windows + format: zip + files: + - unpack/* + - s2/LICENSE + - s2/README.md +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ .Tag }}-next" +changelog: + sort: asc + filters: + exclude: + - '^doc:' + - '^docs:' + - '^test:' + - '^tests:' + - '^Update\sREADME.md' + +nfpms: + - + file_name_template: "s2_package_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + vendor: Klaus Post + homepage: https://github.com/klauspost/compress + maintainer: Klaus Post <klauspost@gmail.com> + description: S2 Compression Tool + license: BSD 3-Clause + formats: + - deb + - rpm + replacements: + darwin: Darwin + linux: Linux + freebsd: FreeBSD + amd64: x86_64 diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md new file mode 100644 index 000000000..d6a26466c --- /dev/null +++ b/vendor/github.com/klauspost/compress/README.md @@ -0,0 +1,426 @@ +# compress
+
+This package provides various compression algorithms.
+
+* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go.
+* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy.
+* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib).
+* [snappy](https://github.com/klauspost/compress/tree/master/snappy) is a drop-in replacement for `github.com/golang/snappy` offering better compression and concurrent streams.
+* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding.
+* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped requests efficiently.
+* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation.
+* [fuzz package](https://github.com/klauspost/compress-fuzz) for fuzz testing all compressors/decompressors here.
+
+[](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories)
+[](https://github.com/klauspost/compress/actions/workflows/go.yml)
+[](https://sourcegraph.com/github.com/klauspost/compress?badge)
+
+# changelog
+
+* Aug 3, 2021 (v1.13.3)
+
+ * zstd: Improve Best compression [#404](https://github.com/klauspost/compress/pull/404)
+ * zstd: Fix WriteTo error forwarding [#411](https://github.com/klauspost/compress/pull/411)
+ * gzhttp: Return http.HandlerFunc instead of http.Handler. Unlikely breaking change. [#406](https://github.com/klauspost/compress/pull/406)
+ * s2sx: Fix max size error [#399](https://github.com/klauspost/compress/pull/399)
+ * zstd: Add optional stream content size on reset [#401](https://github.com/klauspost/compress/pull/401)
+ * zstd: use SpeedBestCompression for level >= 10 [#410](https://github.com/klauspost/compress/pull/410)
+
+* Jun 14, 2021 (v1.13.1)
+
+ * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396)
+ * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394)
+ * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389)
+ * s2: Improve speed with bigger output margin [#395](https://github.com/klauspost/compress/pull/395)
+
+* Jun 3, 2021 (v1.13.0)
+ * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors.
+ * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382)
+ * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380)
+
+* May 25, 2021 (v1.12.3)
+ * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374)
+ * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375)
+ * zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373)
+
+* Apr 27, 2021 (v1.12.2)
+ * zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365)
+ * zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363)
+ * deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367)
+ * s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358)
+ * s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362)
+ * s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368)
+
+* Apr 14, 2021 (v1.12.1)
+ * snappy package removed. Upstream added as dependency.
+ * s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353)
+ * s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352)
+ * s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348)
+ * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352)
+ * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346)
+ * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349)
+
+* Mar 26, 2021 (v1.11.13)
+ * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345)
+ * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336)
+ * deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338)
+ * s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341)
+
+* Mar 5, 2021 (v1.11.12)
+ * s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives).
+ * s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328)
+
+* Mar 1, 2021 (v1.11.9)
+ * s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324)
+ * s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325)
+ * s2: Fix binaries.
+
+* Feb 25, 2021 (v1.11.8)
+ * s2: Fixed occational out-of-bounds write on amd64. Upgrade recommended.
+ * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315)
+ * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322)
+ * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314)
+ * zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313)
+
+* Jan 14, 2021 (v1.11.7)
+ * Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309)
+ * s2: Add 'best' compression option. [#310](https://github.com/klauspost/compress/pull/310)
+ * s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311)
+ * s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308)
+ * s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312)
+
+* Jan 7, 2021 (v1.11.6)
+ * zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306)
+ * zstd: Free Decoder resources when Reset is called with a nil io.Reader [#305](https://github.com/klauspost/compress/pull/305)
+
+* Dec 20, 2020 (v1.11.4)
+ * zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304)
+ * Add header decoder [#299](https://github.com/klauspost/compress/pull/299)
+ * s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297)
+ * Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300)
+ * zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303)
+
+* Nov 15, 2020 (v1.11.3)
+ * inflate: 10-15% faster decompression [#293](https://github.com/klauspost/compress/pull/293)
+ * zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295)
+
+* Oct 11, 2020 (v1.11.2)
+ * s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291)
+
+* Oct 1, 2020 (v1.11.1)
+ * zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286)
+
+* Sept 8, 2020 (v1.11.0)
+ * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281)
+ * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282)
+ * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274)
+
+<details>
+ <summary>See changes prior to v1.11.0</summary>
+
+* July 8, 2020 (v1.10.11)
+ * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278)
+ * huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275)
+
+* June 23, 2020 (v1.10.10)
+ * zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270)
+
+* June 16, 2020 (v1.10.9):
+ * zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268)
+ * zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266)
+ * Fuzzit tests removed. The service has been purchased and is no longer available.
+
+* June 5, 2020 (v1.10.8):
+ * 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265)
+
+* June 1, 2020 (v1.10.7):
+ * Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries)
+ * Increase zstd decompression speed up to 1.19x. [#259](https://github.com/klauspost/compress/pull/259)
+ * Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263)
+
+* May 21, 2020: (v1.10.6)
+ * zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252)
+ * zstd: Stricter decompression checks.
+
+* April 12, 2020: (v1.10.5)
+ * s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239)
+
+* Apr 8, 2020: (v1.10.4)
+ * zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251), [#250](https://github.com/klauspost/compress/pull/250), [#249](https://github.com/klauspost/compress/pull/249), [#247](https://github.com/klauspost/compress/pull/247)
+* Mar 11, 2020: (v1.10.3)
+ * s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245)
+ * s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244)
+ * zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240)
+ * zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241)
+ * zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238)
+
+* Feb 27, 2020: (v1.10.2)
+ * Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232)
+ * Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227)
+
+* Feb 18, 2020: (v1.10.1)
+ * Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226)
+ * deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224)
+ * Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224)
+
+* Feb 4, 2020: (v1.10.0)
+ * Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216)
+ * Fix buffer overflow on repeated small block deflate. [#218](https://github.com/klauspost/compress/pull/218)
+ * Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214)
+ * Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s. [#186](https://github.com/klauspost/compress/pull/186)
+
+</details>
+
+<details>
+ <summary>See changes prior to v1.10.0</summary>
+
+* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056), [#206](https://github.com/klauspost/compress/pull/206).
+* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204)
+* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed.
+* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases.
+* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192)
+* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder.
+* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199)
+* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features
+* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197)
+* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198)
+* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit.
+* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191)
+* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188)
+* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187)
+* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines.
+* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate.
+* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184)
+* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate.
+* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180)
+* Nov 11, 2019: Set default [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB.
+* Nov 11, 2019: Reduce inflate memory use by 1KB.
+* Nov 10, 2019: Less allocations in deflate bit writer.
+* Nov 10, 2019: Fix inconsistent error returned by zstd decoder.
+* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174)
+* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173)
+* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172)
+* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105)
+
+</details>
+
+<details>
+ <summary>See changes prior to v1.9.0</summary>
+
+* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169)
+* Oct 3, 2019: Fix inconsistent results on broken zstd streams.
+* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools)
+* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools).
+* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip).
+* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes).
+* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option.
+* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables.
+* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode.
+* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding.
+* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy.
+* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing.
+* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing.
+* Aug 14, 2019: zstd: Skip incompressible data 2x faster. [#147](https://github.com/klauspost/compress/pull/147)
+* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146)
+* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144)
+* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142)
+* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder.
+* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder.
+* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content.
+* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix.
+* June 17, 2019: zstd decompression bugfix.
+* June 17, 2019: fix 32 bit builds.
+* June 17, 2019: Easier use in modules (less dependencies).
+* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio.
+* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression.
+* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels.
+* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression!
+* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels.
+* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added.
+* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression).
+* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below.
+* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0).
+* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change.
+* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change.
+* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function.
+* May 28, 2017: Reduce allocations when resetting decoder.
+* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7.
+* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625).
+* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before.
+* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update.
+* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level.
+* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression.
+* Mar 24, 2016: Small speedup for level 1-3.
+* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster.
+* Feb 19, 2016: Handle small payloads faster in level 1-3.
+* Feb 19, 2016: Added faster level 2 + 3 compression modes.
+* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progresssion in terms of compression. New default level is 5.
+* Feb 14, 2016: Snappy: Merge upstream changes.
+* Feb 14, 2016: Snappy: Fix aggressive skipping.
+* Feb 14, 2016: Snappy: Update benchmark.
+* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression.
+* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%.
+* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content.
+* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup.
+* Jan 16, 2016: Optimization on deflate level 1,2,3 compression.
+* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives.
+* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs.
+* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms.
+* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update!
+* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet).
+* Nov 20 2015: Small optimization to bit writer on 64 bit systems.
+* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15).
+* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate.
+* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file
+* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x.
+
+</details>
+
+# deflate usage
+
+* [High Throughput Benchmark](http://blog.klauspost.com/go-gzipdeflate-benchmarks/).
+* [Small Payload/Webserver Benchmarks](http://blog.klauspost.com/gzip-performance-for-go-webservers/).
+* [Linear Time Compression](http://blog.klauspost.com/constant-time-gzipzip-compression/).
+* [Re-balancing Deflate Compression Levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/)
+
+The packages are drop-in replacements for standard libraries. Simply replace the import path to use them:
+
+| old import | new import | Documentation
+|--------------------|-----------------------------------------|--------------------|
+| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc)
+| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc)
+| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc)
+| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc)
+
+* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib).
+
+You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages.
+
+The packages contains the same as the standard library, so you can use the godoc for that: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/).
+
+Currently there is only minor speedup on decompression (mostly CRC32 calculation).
+
+Memory usage is typically 1MB for a Writer. stdlib is in the same range.
+If you expect to have a lot of concurrently allocated Writers consider using
+the stateless compress described below.
+
+# Stateless compression
+
+This package offers stateless compression as a special option for gzip/deflate.
+It will do compression but without maintaining any state between Write calls.
+
+This means there will be no memory kept between Write calls, but compression and speed will be suboptimal.
+
+This is only relevant in cases where you expect to run many thousands of compressors concurrently,
+but with very little activity. This is *not* intended for regular web servers serving individual requests.
+
+Because of this, the size of actual Write calls will affect output size.
+
+In gzip, specify level `-3` / `gzip.StatelessCompression` to enable.
+
+For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter)
+
+A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer:
+
+```
+ // replace 'ioutil.Discard' with your output.
+ gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression)
+ if err != nil {
+ return err
+ }
+ defer gzw.Close()
+
+ w := bufio.NewWriterSize(gzw, 4096)
+ defer w.Flush()
+
+ // Write to 'w'
+```
+
+This will only use up to 4KB in memory when the writer is idle.
+
+Compression is almost always worse than the fastest compression level
+and each write will allocate (a little) memory.
+
+# Performance Update 2018
+
+It has been a while since we have been looking at the speed of this package compared to the standard library, so I thought I would re-do my tests and give some overall recommendations based on the current state. All benchmarks have been performed with Go 1.10 on my Desktop Intel(R) Core(TM) i7-2600 CPU @3.40GHz. Since I last ran the tests, I have gotten more RAM, which means tests with big files are no longer limited by my SSD.
+
+The raw results are in my [updated spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). Due to cgo changes and upstream updates i could not get the cgo version of gzip to compile. Instead I included the [zstd](https://github.com/datadog/zstd) cgo implementation. If I get cgo gzip to work again, I might replace the results in the sheet.
+
+The columns to take note of are: *MB/s* - the throughput. *Reduction* - the data size reduction in percent of the original. *Rel Speed* relative speed compared to the standard library at the same level. *Smaller* - how many percent smaller is the compressed output compared to stdlib. Negative means the output was bigger. *Loss* means the loss (or gain) in compression as a percentage difference of the input.
+
+The `gzstd` (standard library gzip) and `gzkp` (this package gzip) only uses one CPU core. [`pgzip`](https://github.com/klauspost/pgzip), [`bgzf`](https://github.com/biogo/hts/tree/master/bgzf) uses all 4 cores. [`zstd`](https://github.com/DataDog/zstd) uses one core, and is a beast (but not Go, yet).
+
+
+## Overall differences.
+
+There appears to be a roughly 5-10% speed advantage over the standard library when comparing at similar compression levels.
+
+The biggest difference you will see is the result of [re-balancing](https://blog.klauspost.com/rebalancing-deflate-compression-levels/) the compression levels. I wanted by library to give a smoother transition between the compression levels than the standard library.
+
+This package attempts to provide a more smooth transition, where "1" is taking a lot of shortcuts, "5" is the reasonable trade-off and "9" is the "give me the best compression", and the values in between gives something reasonable in between. The standard library has big differences in levels 1-4, but levels 5-9 having no significant gains - often spending a lot more time than can be justified by the achieved compression.
+
+There are links to all the test data in the [spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing) in the top left field on each tab.
+
+## Web Content
+
+This test set aims to emulate typical use in a web server. The test-set is 4GB data in 53k files, and is a mixture of (mostly) HTML, JS, CSS.
+
+Since level 1 and 9 are close to being the same code, they are quite close. But looking at the levels in-between the differences are quite big.
+
+Looking at level 6, this package is 88% faster, but will output about 6% more data. For a web server, this means you can serve 88% more data, but have to pay for 6% more bandwidth. You can draw your own conclusions on what would be the most expensive for your case.
+
+## Object files
+
+This test is for typical data files stored on a server. In this case it is a collection of Go precompiled objects. They are very compressible.
+
+The picture is similar to the web content, but with small differences since this is very compressible. Levels 2-3 offer good speed, but is sacrificing quite a bit of compression.
+
+The standard library seems suboptimal on level 3 and 4 - offering both worse compression and speed than level 6 & 7 of this package respectively.
+
+## Highly Compressible File
+
+This is a JSON file with very high redundancy. The reduction starts at 95% on level 1, so in real life terms we are dealing with something like a highly redundant stream of data, etc.
+
+It is definitely visible that we are dealing with specialized content here, so the results are very scattered. This package does not do very well at levels 1-4, but picks up significantly at level 5 and levels 7 and 8 offering great speed for the achieved compression.
+
+So if you know you content is extremely compressible you might want to go slightly higher than the defaults. The standard library has a huge gap between levels 3 and 4 in terms of speed (2.75x slowdown), so it offers little "middle ground".
+
+## Medium-High Compressible
+
+This is a pretty common test corpus: [enwik9](http://mattmahoney.net/dc/textdata.html). It contains the first 10^9 bytes of the English Wikipedia dump on Mar. 3, 2006. This is a very good test of typical text based compression and more data heavy streams.
+
+We see a similar picture here as in "Web Content". On equal levels some compression is sacrificed for more speed. Level 5 seems to be the best trade-off between speed and size, beating stdlib level 3 in both.
+
+## Medium Compressible
+
+I will combine two test sets, one [10GB file set](http://mattmahoney.net/dc/10gb.html) and a VM disk image (~8GB). Both contain different data types and represent a typical backup scenario.
+
+The most notable thing is how quickly the standard library drops to very low compression speeds around level 5-6 without any big gains in compression. Since this type of data is fairly common, this does not seem like good behavior.
+
+
+## Un-compressible Content
+
+This is mainly a test of how good the algorithms are at detecting un-compressible input. The standard library only offers this feature with very conservative settings at level 1. Obviously there is no reason for the algorithms to try to compress input that cannot be compressed. The only downside is that it might skip some compressible data on false detections.
+
+
+## Huffman only compression
+
+This compression library adds a special compression level, named `HuffmanOnly`, which allows near linear time compression. This is done by completely disabling matching of previous data, and only reduce the number of bits to represent each character.
+
+This means that often used characters, like 'e' and ' ' (space) in text use the fewest bits to represent, and rare characters like '¤' takes more bits to represent. For more information see [wikipedia](https://en.wikipedia.org/wiki/Huffman_coding) or this nice [video](https://youtu.be/ZdooBTdW5bM).
+
+Since this type of compression has much less variance, the compression speed is mostly unaffected by the input data, and is usually more than *180MB/s* for a single core.
+
+The downside is that the compression ratio is usually considerably worse than even the fastest conventional compression. The compression ratio can never be better than 8:1 (12.5%).
+
+The linear time compression can be used as a "better than nothing" mode, where you cannot risk the encoder to slow down on some content. For comparison, the size of the "Twain" text is *233460 bytes* (+29% vs. level 1) and encode speed is 144MB/s (4.5x level 1). So in this case you trade a 30% size increase for a 4 times speedup.
+
+For more information see my blog post on [Fast Linear Time Compression](http://blog.klauspost.com/constant-time-gzipzip-compression/).
+
+This is implemented on Go 1.7 as "Huffman Only" mode, though not exposed for gzip.
+
+
+# license
+
+This code is licensed under the same conditions as the original Go code. See LICENSE file.
diff --git a/vendor/github.com/klauspost/compress/compressible.go b/vendor/github.com/klauspost/compress/compressible.go new file mode 100644 index 000000000..ea5a692d5 --- /dev/null +++ b/vendor/github.com/klauspost/compress/compressible.go @@ -0,0 +1,85 @@ +package compress + +import "math" + +// Estimate returns a normalized compressibility estimate of block b. +// Values close to zero are likely uncompressible. +// Values above 0.1 are likely to be compressible. +// Values above 0.5 are very compressible. +// Very small lengths will return 0. +func Estimate(b []byte) float64 { + if len(b) < 16 { + return 0 + } + + // Correctly predicted order 1 + hits := 0 + lastMatch := false + var o1 [256]byte + var hist [256]int + c1 := byte(0) + for _, c := range b { + if c == o1[c1] { + // We only count a hit if there was two correct predictions in a row. + if lastMatch { + hits++ + } + lastMatch = true + } else { + lastMatch = false + } + o1[c1] = c + c1 = c + hist[c]++ + } + + // Use x^0.6 to give better spread + prediction := math.Pow(float64(hits)/float64(len(b)), 0.6) + + // Calculate histogram distribution + variance := float64(0) + avg := float64(len(b)) / 256 + + for _, v := range hist { + Δ := float64(v) - avg + variance += Δ * Δ + } + + stddev := math.Sqrt(float64(variance)) / float64(len(b)) + exp := math.Sqrt(1 / float64(len(b))) + + // Subtract expected stddev + stddev -= exp + if stddev < 0 { + stddev = 0 + } + stddev *= 1 + exp + + // Use x^0.4 to give better spread + entropy := math.Pow(stddev, 0.4) + + // 50/50 weight between prediction and histogram distribution + return math.Pow((prediction+entropy)/2, 0.9) +} + +// ShannonEntropyBits returns the number of bits minimum required to represent +// an entropy encoding of the input bytes. +// https://en.wiktionary.org/wiki/Shannon_entropy +func ShannonEntropyBits(b []byte) int { + if len(b) == 0 { + return 0 + } + var hist [256]int + for _, c := range b { + hist[c]++ + } + shannon := float64(0) + invTotal := 1.0 / float64(len(b)) + for _, v := range hist[:] { + if v > 0 { + n := float64(v) + shannon += math.Ceil(-math.Log2(n*invTotal) * n) + } + } + return int(math.Ceil(shannon)) +} diff --git a/vendor/github.com/klauspost/compress/gen.sh b/vendor/github.com/klauspost/compress/gen.sh new file mode 100644 index 000000000..aff942205 --- /dev/null +++ b/vendor/github.com/klauspost/compress/gen.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +cd s2/cmd/_s2sx/ || exit 1 +go generate . diff --git a/vendor/github.com/klauspost/compress/go.mod b/vendor/github.com/klauspost/compress/go.mod new file mode 100644 index 000000000..2a4f0a41f --- /dev/null +++ b/vendor/github.com/klauspost/compress/go.mod @@ -0,0 +1,5 @@ +module github.com/klauspost/compress + +go 1.13 + +require github.com/golang/snappy v0.0.3 // indirect diff --git a/vendor/github.com/klauspost/compress/go.sum b/vendor/github.com/klauspost/compress/go.sum new file mode 100644 index 000000000..73204cafa --- /dev/null +++ b/vendor/github.com/klauspost/compress/go.sum @@ -0,0 +1,2 @@ +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go index 0823c928c..8323dc053 100644 --- a/vendor/github.com/klauspost/compress/huff0/compress.go +++ b/vendor/github.com/klauspost/compress/huff0/compress.go @@ -161,6 +161,70 @@ func compress(in []byte, s *Scratch, compressor func(src []byte) ([]byte, error) return s.Out, false, nil } +// EstimateSizes will estimate the data sizes +func EstimateSizes(in []byte, s *Scratch) (tableSz, dataSz, reuseSz int, err error) { + s, err = s.prepare(in) + if err != nil { + return 0, 0, 0, err + } + + // Create histogram, if none was provided. + tableSz, dataSz, reuseSz = -1, -1, -1 + maxCount := s.maxCount + var canReuse = false + if maxCount == 0 { + maxCount, canReuse = s.countSimple(in) + } else { + canReuse = s.canUseTable(s.prevTable) + } + + // We want the output size to be less than this: + wantSize := len(in) + if s.WantLogLess > 0 { + wantSize -= wantSize >> s.WantLogLess + } + + // Reset for next run. + s.clearCount = true + s.maxCount = 0 + if maxCount >= len(in) { + if maxCount > len(in) { + return 0, 0, 0, fmt.Errorf("maxCount (%d) > length (%d)", maxCount, len(in)) + } + if len(in) == 1 { + return 0, 0, 0, ErrIncompressible + } + // One symbol, use RLE + return 0, 0, 0, ErrUseRLE + } + if maxCount == 1 || maxCount < (len(in)>>7) { + // Each symbol present maximum once or too well distributed. + return 0, 0, 0, ErrIncompressible + } + + // Calculate new table. + err = s.buildCTable() + if err != nil { + return 0, 0, 0, err + } + + if false && !s.canUseTable(s.cTable) { + panic("invalid table generated") + } + + tableSz, err = s.cTable.estTableSize(s) + if err != nil { + return 0, 0, 0, err + } + if canReuse { + reuseSz = s.prevTable.estimateSize(s.count[:s.symbolLen]) + } + dataSz = s.cTable.estimateSize(s.count[:s.symbolLen]) + + // Restore + return tableSz, dataSz, reuseSz, nil +} + func (s *Scratch) compress1X(src []byte) ([]byte, error) { return s.compress1xDo(s.Out, src) } diff --git a/vendor/github.com/klauspost/compress/huff0/huff0.go b/vendor/github.com/klauspost/compress/huff0/huff0.go index 7ec2022b6..3ee00ecb4 100644 --- a/vendor/github.com/klauspost/compress/huff0/huff0.go +++ b/vendor/github.com/klauspost/compress/huff0/huff0.go @@ -245,6 +245,68 @@ func (c cTable) write(s *Scratch) error { return nil } +func (c cTable) estTableSize(s *Scratch) (sz int, err error) { + var ( + // precomputed conversion table + bitsToWeight [tableLogMax + 1]byte + huffLog = s.actualTableLog + // last weight is not saved. + maxSymbolValue = uint8(s.symbolLen - 1) + huffWeight = s.huffWeight[:256] + ) + const ( + maxFSETableLog = 6 + ) + // convert to weight + bitsToWeight[0] = 0 + for n := uint8(1); n < huffLog+1; n++ { + bitsToWeight[n] = huffLog + 1 - n + } + + // Acquire histogram for FSE. + hist := s.fse.Histogram() + hist = hist[:256] + for i := range hist[:16] { + hist[i] = 0 + } + for n := uint8(0); n < maxSymbolValue; n++ { + v := bitsToWeight[c[n].nBits] & 15 + huffWeight[n] = v + hist[v]++ + } + + // FSE compress if feasible. + if maxSymbolValue >= 2 { + huffMaxCnt := uint32(0) + huffMax := uint8(0) + for i, v := range hist[:16] { + if v == 0 { + continue + } + huffMax = byte(i) + if v > huffMaxCnt { + huffMaxCnt = v + } + } + s.fse.HistogramFinished(huffMax, int(huffMaxCnt)) + s.fse.TableLog = maxFSETableLog + b, err := fse.Compress(huffWeight[:maxSymbolValue], s.fse) + if err == nil && len(b) < int(s.symbolLen>>1) { + sz += 1 + len(b) + return sz, nil + } + // Unable to compress (RLE/uncompressible) + } + // write raw values as 4-bits (max : 15) + if maxSymbolValue > (256 - 128) { + // should not happen : likely means source cannot be compressed + return 0, ErrIncompressible + } + // special case, pack weights 4 bits/weight. + sz += 1 + int(maxSymbolValue/2) + return sz, nil +} + // estimateSize returns the estimated size in bytes of the input represented in the // histogram supplied. func (c cTable) estimateSize(hist []uint32) int { diff --git a/vendor/github.com/klauspost/compress/s2sx.mod b/vendor/github.com/klauspost/compress/s2sx.mod new file mode 100644 index 000000000..a9faf7a02 --- /dev/null +++ b/vendor/github.com/klauspost/compress/s2sx.mod @@ -0,0 +1,5 @@ +module github.com/klauspost/compress + +go 1.16 + +require github.com/golang/snappy v0.0.3 // indirect diff --git a/vendor/github.com/klauspost/compress/s2sx.sum b/vendor/github.com/klauspost/compress/s2sx.sum new file mode 100644 index 000000000..73204cafa --- /dev/null +++ b/vendor/github.com/klauspost/compress/s2sx.sum @@ -0,0 +1,2 @@ +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= diff --git a/vendor/github.com/klauspost/compress/zstd/README.md b/vendor/github.com/klauspost/compress/zstd/README.md index 787813fa9..c8f0f16fc 100644 --- a/vendor/github.com/klauspost/compress/zstd/README.md +++ b/vendor/github.com/klauspost/compress/zstd/README.md @@ -152,7 +152,7 @@ file out level insize outsize millis mb/s silesia.tar zskp 1 211947520 73101992 643 313.87 silesia.tar zskp 2 211947520 67504318 969 208.38 silesia.tar zskp 3 211947520 64595893 2007 100.68 -silesia.tar zskp 4 211947520 60995370 7691 26.28 +silesia.tar zskp 4 211947520 60995370 8825 22.90 cgo zstd: silesia.tar zstd 1 211947520 73605392 543 371.56 @@ -162,7 +162,7 @@ silesia.tar zstd 9 211947520 60212393 5063 39.92 gzip, stdlib/this package: silesia.tar gzstd 1 211947520 80007735 1654 122.21 -silesia.tar gzkp 1 211947520 80369488 1168 173.06 +silesia.tar gzkp 1 211947520 80136201 1152 175.45 GOB stream of binary data. Highly compressible. https://files.klauspost.com/compress/gob-stream.7z @@ -171,13 +171,15 @@ file out level insize outsize millis mb/s gob-stream zskp 1 1911399616 235022249 3088 590.30 gob-stream zskp 2 1911399616 205669791 3786 481.34 gob-stream zskp 3 1911399616 175034659 9636 189.17 -gob-stream zskp 4 1911399616 167273881 29337 62.13 +gob-stream zskp 4 1911399616 165609838 50369 36.19 + gob-stream zstd 1 1911399616 249810424 2637 691.26 gob-stream zstd 3 1911399616 208192146 3490 522.31 gob-stream zstd 6 1911399616 193632038 6687 272.56 gob-stream zstd 9 1911399616 177620386 16175 112.70 + gob-stream gzstd 1 1911399616 357382641 10251 177.82 -gob-stream gzkp 1 1911399616 362156523 5695 320.08 +gob-stream gzkp 1 1911399616 359753026 5438 335.20 The test data for the Large Text Compression Benchmark is the first 10^9 bytes of the English Wikipedia dump on Mar. 3, 2006. @@ -187,11 +189,13 @@ file out level insize outsize millis mb/s enwik9 zskp 1 1000000000 343848582 3609 264.18 enwik9 zskp 2 1000000000 317276632 5746 165.97 enwik9 zskp 3 1000000000 292243069 12162 78.41 -enwik9 zskp 4 1000000000 275241169 36430 26.18 +enwik9 zskp 4 1000000000 262183768 82837 11.51 + enwik9 zstd 1 1000000000 358072021 3110 306.65 enwik9 zstd 3 1000000000 313734672 4784 199.35 enwik9 zstd 6 1000000000 295138875 10290 92.68 enwik9 zstd 9 1000000000 278348700 28549 33.40 + enwik9 gzstd 1 1000000000 382578136 9604 99.30 enwik9 gzkp 1 1000000000 383825945 6544 145.73 @@ -202,13 +206,15 @@ file out level insize outsize millis mb/s github-june-2days-2019.json zskp 1 6273951764 699045015 10620 563.40 github-june-2days-2019.json zskp 2 6273951764 617881763 11687 511.96 github-june-2days-2019.json zskp 3 6273951764 524340691 34043 175.75 -github-june-2days-2019.json zskp 4 6273951764 503314661 93811 63.78 +github-june-2days-2019.json zskp 4 6273951764 470320075 170190 35.16 + github-june-2days-2019.json zstd 1 6273951764 766284037 8450 708.00 github-june-2days-2019.json zstd 3 6273951764 661889476 10927 547.57 github-june-2days-2019.json zstd 6 6273951764 642756859 22996 260.18 github-june-2days-2019.json zstd 9 6273951764 601974523 52413 114.16 + github-june-2days-2019.json gzstd 1 6273951764 1164400847 29948 199.79 -github-june-2days-2019.json gzkp 1 6273951764 1128755542 19236 311.03 +github-june-2days-2019.json gzkp 1 6273951764 1125417694 21788 274.61 VM Image, Linux mint with a few installed applications: https://files.klauspost.com/compress/rawstudio-mint14.7z @@ -217,13 +223,15 @@ file out level insize outsize millis mb/s rawstudio-mint14.tar zskp 1 8558382592 3667489370 20210 403.84 rawstudio-mint14.tar zskp 2 8558382592 3364592300 31873 256.07 rawstudio-mint14.tar zskp 3 8558382592 3158085214 77675 105.08 -rawstudio-mint14.tar zskp 4 8558382592 3020370044 404956 20.16 +rawstudio-mint14.tar zskp 4 8558382592 2965110639 857750 9.52 + rawstudio-mint14.tar zstd 1 8558382592 3609250104 17136 476.27 rawstudio-mint14.tar zstd 3 8558382592 3341679997 29262 278.92 rawstudio-mint14.tar zstd 6 8558382592 3235846406 77904 104.77 rawstudio-mint14.tar zstd 9 8558382592 3160778861 140946 57.91 + rawstudio-mint14.tar gzstd 1 8558382592 3926257486 57722 141.40 -rawstudio-mint14.tar gzkp 1 8558382592 3970463184 41749 195.49 +rawstudio-mint14.tar gzkp 1 8558382592 3962605659 45113 180.92 CSV data: https://files.klauspost.com/compress/nyc-taxi-data-10M.csv.zst @@ -232,13 +240,15 @@ file out level insize outsize millis mb/s nyc-taxi-data-10M.csv zskp 1 3325605752 641339945 8925 355.35 nyc-taxi-data-10M.csv zskp 2 3325605752 591748091 11268 281.44 nyc-taxi-data-10M.csv zskp 3 3325605752 530289687 25239 125.66 -nyc-taxi-data-10M.csv zskp 4 3325605752 490907191 65939 48.10 +nyc-taxi-data-10M.csv zskp 4 3325605752 476268884 135958 23.33 + nyc-taxi-data-10M.csv zstd 1 3325605752 687399637 8233 385.18 nyc-taxi-data-10M.csv zstd 3 3325605752 598514411 10065 315.07 nyc-taxi-data-10M.csv zstd 6 3325605752 570522953 20038 158.27 nyc-taxi-data-10M.csv zstd 9 3325605752 517554797 64565 49.12 + nyc-taxi-data-10M.csv gzstd 1 3325605752 928656485 23876 132.83 -nyc-taxi-data-10M.csv gzkp 1 3325605752 924718719 16388 193.53 +nyc-taxi-data-10M.csv gzkp 1 3325605752 922257165 16780 189.00 ``` ## Decompressor diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 4d984c3b2..f430f58b5 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -260,9 +260,10 @@ func (d *Decoder) WriteTo(w io.Writer) (int64, error) { if len(d.current.b) > 0 { n2, err2 := w.Write(d.current.b) n += int64(n2) - if err2 != nil && d.current.err == nil { + if err2 != nil && (d.current.err == nil || d.current.err == io.EOF) { d.current.err = err2 - break + } else if n2 != len(d.current.b) { + d.current.err = io.ErrShortWrite } } if d.current.err != nil { diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go index 60f298648..295cd602a 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_base.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go @@ -38,8 +38,8 @@ func (e *fastBase) AppendCRC(dst []byte) []byte { // WindowSize returns the window size of the encoder, // or a window size small enough to contain the input size, if > 0. -func (e *fastBase) WindowSize(size int) int32 { - if size > 0 && size < int(e.maxMatchOff) { +func (e *fastBase) WindowSize(size int64) int32 { + if size > 0 && size < int64(e.maxMatchOff) { b := int32(1) << uint(bits.Len(uint(size))) // Keep minimum window. if b < 1024 { diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go index b7d4b9004..96028ecd8 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_best.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go @@ -5,22 +5,61 @@ package zstd import ( + "bytes" "fmt" - "math/bits" + + "github.com/klauspost/compress" ) const ( - bestLongTableBits = 20 // Bits used in the long match table + bestLongTableBits = 22 // Bits used in the long match table bestLongTableSize = 1 << bestLongTableBits // Size of the table + bestLongLen = 8 // Bytes used for table hash // Note: Increasing the short table bits or making the hash shorter // can actually lead to compression degradation since it will 'steal' more from the // long match table and match offsets are quite big. // This greatly depends on the type of input. - bestShortTableBits = 16 // Bits used in the short match table + bestShortTableBits = 18 // Bits used in the short match table bestShortTableSize = 1 << bestShortTableBits // Size of the table + bestShortLen = 4 // Bytes used for table hash + ) +type match struct { + offset int32 + s int32 + length int32 + rep int32 + est int32 +} + +const highScore = 25000 + +// estBits will estimate output bits from predefined tables. +func (m *match) estBits(bitsPerByte int32) { + mlc := mlCode(uint32(m.length - zstdMinMatch)) + var ofc uint8 + if m.rep < 0 { + ofc = ofCode(uint32(m.s-m.offset) + 3) + } else { + ofc = ofCode(uint32(m.rep)) + } + // Cost, excluding + ofTT, mlTT := fsePredefEnc[tableOffsets].ct.symbolTT[ofc], fsePredefEnc[tableMatchLengths].ct.symbolTT[mlc] + + // Add cost of match encoding... + m.est = int32(ofTT.outBits + mlTT.outBits) + m.est += int32(ofTT.deltaNbBits>>16 + mlTT.deltaNbBits>>16) + // Subtract savings compared to literal encoding... + m.est -= (m.length * bitsPerByte) >> 10 + if m.est > 0 { + // Unlikely gain.. + m.length = 0 + m.est = highScore + } +} + // bestFastEncoder uses 2 tables, one for short matches (5 bytes) and one for long matches. // The long match table contains the previous entry with the same hash, // effectively making it a "chain" of length 2. @@ -109,6 +148,14 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) { return } + // Use this to estimate literal cost. + // Scaled by 10 bits. + bitsPerByte := int32((compress.ShannonEntropyBits(src) * 1024) / len(src)) + // Huffman can never go < 1 bit/byte + if bitsPerByte < 1024 { + bitsPerByte = 1024 + } + // Override src src = e.hist sLimit := int32(len(src)) - inputMargin @@ -145,51 +192,49 @@ encodeLoop: panic("offset0 was 0") } - type match struct { - offset int32 - s int32 - length int32 - rep int32 - } - matchAt := func(offset int32, s int32, first uint32, rep int32) match { - if s-offset >= e.maxMatchOff || load3232(src, offset) != first { - return match{offset: offset, s: s} - } - return match{offset: offset, s: s, length: 4 + e.matchlen(s+4, offset+4, src), rep: rep} - } - bestOf := func(a, b match) match { - aScore := b.s - a.s + a.length - bScore := a.s - b.s + b.length - if a.rep < 0 { - aScore = aScore - int32(bits.Len32(uint32(a.offset)))/8 - } - if b.rep < 0 { - bScore = bScore - int32(bits.Len32(uint32(b.offset)))/8 - } - if aScore >= bScore { + if a.est+(a.s-b.s)*bitsPerByte>>10 < b.est+(b.s-a.s)*bitsPerByte>>10 { return a } return b } const goodEnough = 100 - nextHashL := hash8(cv, bestLongTableBits) - nextHashS := hash4x64(cv, bestShortTableBits) + nextHashL := hashLen(cv, bestLongTableBits, bestLongLen) + nextHashS := hashLen(cv, bestShortTableBits, bestShortLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] + matchAt := func(offset int32, s int32, first uint32, rep int32) match { + if s-offset >= e.maxMatchOff || load3232(src, offset) != first { + return match{s: s, est: highScore} + } + if debugAsserts { + if !bytes.Equal(src[s:s+4], src[offset:offset+4]) { + panic(fmt.Sprintf("first match mismatch: %v != %v, first: %08x", src[s:s+4], src[offset:offset+4], first)) + } + } + m := match{offset: offset, s: s, length: 4 + e.matchlen(s+4, offset+4, src), rep: rep} + m.estBits(bitsPerByte) + return m + } + best := bestOf(matchAt(candidateL.offset-e.cur, s, uint32(cv), -1), matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)) best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)) best = bestOf(best, matchAt(candidateS.prev-e.cur, s, uint32(cv), -1)) + if canRepeat && best.length < goodEnough { - best = bestOf(best, matchAt(s-offset1+1, s+1, uint32(cv>>8), 1)) - best = bestOf(best, matchAt(s-offset2+1, s+1, uint32(cv>>8), 2)) - best = bestOf(best, matchAt(s-offset3+1, s+1, uint32(cv>>8), 3)) + cv32 := uint32(cv >> 8) + spp := s + 1 + best = bestOf(best, matchAt(spp-offset1, spp, cv32, 1)) + best = bestOf(best, matchAt(spp-offset2, spp, cv32, 2)) + best = bestOf(best, matchAt(spp-offset3, spp, cv32, 3)) if best.length > 0 { - best = bestOf(best, matchAt(s-offset1+3, s+3, uint32(cv>>24), 1)) - best = bestOf(best, matchAt(s-offset2+3, s+3, uint32(cv>>24), 2)) - best = bestOf(best, matchAt(s-offset3+3, s+3, uint32(cv>>24), 3)) + cv32 = uint32(cv >> 24) + spp += 2 + best = bestOf(best, matchAt(spp-offset1, spp, cv32, 1)) + best = bestOf(best, matchAt(spp-offset2, spp, cv32, 2)) + best = bestOf(best, matchAt(spp-offset3, spp, cv32, 3)) } } // Load next and check... @@ -209,22 +254,28 @@ encodeLoop: } s++ - candidateS = e.table[hash4x64(cv>>8, bestShortTableBits)] + candidateS = e.table[hashLen(cv>>8, bestShortTableBits, bestShortLen)] cv = load6432(src, s) cv2 := load6432(src, s+1) - candidateL = e.longTable[hash8(cv, bestLongTableBits)] - candidateL2 := e.longTable[hash8(cv2, bestLongTableBits)] + candidateL = e.longTable[hashLen(cv, bestLongTableBits, bestLongLen)] + candidateL2 := e.longTable[hashLen(cv2, bestLongTableBits, bestLongLen)] + // Short at s+1 best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)) + // Long at s+1, s+2 best = bestOf(best, matchAt(candidateL.offset-e.cur, s, uint32(cv), -1)) best = bestOf(best, matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)) best = bestOf(best, matchAt(candidateL2.offset-e.cur, s+1, uint32(cv2), -1)) best = bestOf(best, matchAt(candidateL2.prev-e.cur, s+1, uint32(cv2), -1)) - + if false { + // Short at s+3. + // Too often worse... + best = bestOf(best, matchAt(e.table[hashLen(cv2>>8, bestShortTableBits, bestShortLen)].offset-e.cur, s+2, uint32(cv2>>8), -1)) + } // See if we can find a better match by checking where the current best ends. // Use that offset to see if we can find a better full match. if sAt := best.s + best.length; sAt < sLimit { - nextHashL := hash8(load6432(src, sAt), bestLongTableBits) + nextHashL := hashLen(load6432(src, sAt), bestLongTableBits, bestLongLen) candidateEnd := e.longTable[nextHashL] if pos := candidateEnd.offset - e.cur - best.length; pos >= 0 { bestEnd := bestOf(best, matchAt(pos, best.s, load3232(src, best.s), -1)) @@ -236,6 +287,12 @@ encodeLoop: } } + if debugAsserts { + if !bytes.Equal(src[best.s:best.s+best.length], src[best.offset:best.offset+best.length]) { + panic(fmt.Sprintf("match mismatch: %v != %v", src[best.s:best.s+best.length], src[best.offset:best.offset+best.length])) + } + } + // We have a match, we can store the forward value if best.rep > 0 { s = best.s @@ -284,8 +341,8 @@ encodeLoop: off := index0 + e.cur for index0 < s-1 { cv0 := load6432(src, index0) - h0 := hash8(cv0, bestLongTableBits) - h1 := hash4x64(cv0, bestShortTableBits) + h0 := hashLen(cv0, bestLongTableBits, bestLongLen) + h1 := hashLen(cv0, bestShortTableBits, bestShortLen) e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} e.table[h1] = prevEntry{offset: off, prev: e.table[h1].offset} off++ @@ -311,7 +368,7 @@ encodeLoop: panic(fmt.Sprintf("s (%d) <= t (%d)", s, t)) } - if debugAsserts && canRepeat && int(offset1) > len(src) { + if debugAsserts && int(offset1) > len(src) { panic("invalid offset") } @@ -352,8 +409,8 @@ encodeLoop: // every entry for index0 < s-1 { cv0 := load6432(src, index0) - h0 := hash8(cv0, bestLongTableBits) - h1 := hash4x64(cv0, bestShortTableBits) + h0 := hashLen(cv0, bestLongTableBits, bestLongLen) + h1 := hashLen(cv0, bestShortTableBits, bestShortLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} e.table[h1] = prevEntry{offset: off, prev: e.table[h1].offset} @@ -374,8 +431,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hash4x64(cv, bestShortTableBits) - nextHashL := hash8(cv, bestLongTableBits) + nextHashS := hashLen(cv, bestShortTableBits, bestShortLen) + nextHashL := hashLen(cv, bestLongTableBits, bestLongLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -425,7 +482,7 @@ func (e *bestFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { e.Encode(blk, src) } -// ResetDict will reset and set a dictionary if not nil +// Reset will reset and set a dictionary if not nil func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { e.resetBase(d, singleBlock) if d == nil { @@ -441,10 +498,10 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { const hashLog = bestShortTableBits cv := load6432(d.content, i-e.maxMatchOff) - nextHash := hash4x64(cv, hashLog) // 0 -> 4 - nextHash1 := hash4x64(cv>>8, hashLog) // 1 -> 5 - nextHash2 := hash4x64(cv>>16, hashLog) // 2 -> 6 - nextHash3 := hash4x64(cv>>24, hashLog) // 3 -> 7 + nextHash := hashLen(cv, hashLog, bestShortLen) // 0 -> 4 + nextHash1 := hashLen(cv>>8, hashLog, bestShortLen) // 1 -> 5 + nextHash2 := hashLen(cv>>16, hashLog, bestShortLen) // 2 -> 6 + nextHash3 := hashLen(cv>>24, hashLog, bestShortLen) // 3 -> 7 e.dictTable[nextHash] = prevEntry{ prev: e.dictTable[nextHash].offset, offset: i, @@ -472,7 +529,7 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { } if len(d.content) >= 8 { cv := load6432(d.content, 0) - h := hash8(cv, bestLongTableBits) + h := hashLen(cv, bestLongTableBits, bestLongLen) e.dictLongTable[h] = prevEntry{ offset: e.maxMatchOff, prev: e.dictLongTable[h].offset, @@ -482,7 +539,7 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { off := 8 // First to read for i := e.maxMatchOff + 1; i < end; i++ { cv = cv>>8 | (uint64(d.content[off]) << 56) - h := hash8(cv, bestLongTableBits) + h := hashLen(cv, bestLongTableBits, bestLongLen) e.dictLongTable[h] = prevEntry{ offset: i, prev: e.dictLongTable[h].offset, diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index eab7b5083..602c05ee0 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -9,6 +9,7 @@ import "fmt" const ( betterLongTableBits = 19 // Bits used in the long match table betterLongTableSize = 1 << betterLongTableBits // Size of the table + betterLongLen = 8 // Bytes used for table hash // Note: Increasing the short table bits or making the hash shorter // can actually lead to compression degradation since it will 'steal' more from the @@ -16,6 +17,7 @@ const ( // This greatly depends on the type of input. betterShortTableBits = 13 // Bits used in the short match table betterShortTableSize = 1 << betterShortTableBits // Size of the table + betterShortLen = 5 // Bytes used for table hash betterLongTableShardCnt = 1 << (betterLongTableBits - dictShardBits) // Number of shards in the table betterLongTableShardSize = betterLongTableSize / betterLongTableShardCnt // Size of an individual shard @@ -154,8 +156,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hash5(cv, betterShortTableBits) - nextHashL := hash8(cv, betterLongTableBits) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) + nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -214,10 +216,10 @@ encodeLoop: for index0 < s-1 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hash8(cv0, betterLongTableBits) + h0 := hashLen(cv0, betterLongTableBits, betterLongLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[hash5(cv1, betterShortTableBits)] = tableEntry{offset: off + 1, val: uint32(cv1)} + e.table[hashLen(cv1, betterShortTableBits, betterShortLen)] = tableEntry{offset: off + 1, val: uint32(cv1)} index0 += 2 } cv = load6432(src, s) @@ -275,10 +277,10 @@ encodeLoop: for index0 < s-1 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hash8(cv0, betterLongTableBits) + h0 := hashLen(cv0, betterLongTableBits, betterLongLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[hash5(cv1, betterShortTableBits)] = tableEntry{offset: off + 1, val: uint32(cv1)} + e.table[hashLen(cv1, betterShortTableBits, betterShortLen)] = tableEntry{offset: off + 1, val: uint32(cv1)} index0 += 2 } cv = load6432(src, s) @@ -353,7 +355,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hash8(cv, betterLongTableBits) + nextHashL = hashLen(cv, betterLongTableBits, betterLongLen) candidateL = e.longTable[nextHashL] coffsetL = candidateL.offset - e.cur @@ -413,8 +415,8 @@ encodeLoop: } // Try to find a better match by searching for a long match at the end of the current best match - if true && s+matched < sLimit { - nextHashL := hash8(load6432(src, s+matched), betterLongTableBits) + if s+matched < sLimit { + nextHashL := hashLen(load6432(src, s+matched), betterLongTableBits, betterLongLen) cv := load3232(src, s) candidateL := e.longTable[nextHashL] coffsetL := candidateL.offset - e.cur - matched @@ -495,10 +497,10 @@ encodeLoop: for index0 < s-1 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hash8(cv0, betterLongTableBits) + h0 := hashLen(cv0, betterLongTableBits, betterLongLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[hash5(cv1, betterShortTableBits)] = tableEntry{offset: off + 1, val: uint32(cv1)} + e.table[hashLen(cv1, betterShortTableBits, betterShortLen)] = tableEntry{offset: off + 1, val: uint32(cv1)} index0 += 2 } @@ -516,8 +518,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hash5(cv, betterShortTableBits) - nextHashL := hash8(cv, betterLongTableBits) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) + nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -672,8 +674,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hash5(cv, betterShortTableBits) - nextHashL := hash8(cv, betterLongTableBits) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) + nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -734,11 +736,11 @@ encodeLoop: for index0 < s-1 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hash8(cv0, betterLongTableBits) + h0 := hashLen(cv0, betterLongTableBits, betterLongLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} e.markLongShardDirty(h0) - h1 := hash5(cv1, betterShortTableBits) + h1 := hashLen(cv1, betterShortTableBits, betterShortLen) e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} e.markShortShardDirty(h1) index0 += 2 @@ -798,11 +800,11 @@ encodeLoop: for index0 < s-1 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hash8(cv0, betterLongTableBits) + h0 := hashLen(cv0, betterLongTableBits, betterLongLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} e.markLongShardDirty(h0) - h1 := hash5(cv1, betterShortTableBits) + h1 := hashLen(cv1, betterShortTableBits, betterShortLen) e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} e.markShortShardDirty(h1) index0 += 2 @@ -879,7 +881,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hash8(cv, betterLongTableBits) + nextHashL = hashLen(cv, betterLongTableBits, betterLongLen) candidateL = e.longTable[nextHashL] coffsetL = candidateL.offset - e.cur @@ -940,7 +942,7 @@ encodeLoop: } // Try to find a better match by searching for a long match at the end of the current best match if s+matched < sLimit { - nextHashL := hash8(load6432(src, s+matched), betterLongTableBits) + nextHashL := hashLen(load6432(src, s+matched), betterLongTableBits, betterLongLen) cv := load3232(src, s) candidateL := e.longTable[nextHashL] coffsetL := candidateL.offset - e.cur - matched @@ -1021,11 +1023,11 @@ encodeLoop: for index0 < s-1 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hash8(cv0, betterLongTableBits) + h0 := hashLen(cv0, betterLongTableBits, betterLongLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} e.markLongShardDirty(h0) - h1 := hash5(cv1, betterShortTableBits) + h1 := hashLen(cv1, betterShortTableBits, betterShortLen) e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} e.markShortShardDirty(h1) index0 += 2 @@ -1045,8 +1047,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hash5(cv, betterShortTableBits) - nextHashL := hash8(cv, betterLongTableBits) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) + nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -1113,10 +1115,10 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { const hashLog = betterShortTableBits cv := load6432(d.content, i-e.maxMatchOff) - nextHash := hash5(cv, hashLog) // 0 -> 4 - nextHash1 := hash5(cv>>8, hashLog) // 1 -> 5 - nextHash2 := hash5(cv>>16, hashLog) // 2 -> 6 - nextHash3 := hash5(cv>>24, hashLog) // 3 -> 7 + nextHash := hashLen(cv, hashLog, betterShortLen) // 0 -> 4 + nextHash1 := hashLen(cv>>8, hashLog, betterShortLen) // 1 -> 5 + nextHash2 := hashLen(cv>>16, hashLog, betterShortLen) // 2 -> 6 + nextHash3 := hashLen(cv>>24, hashLog, betterShortLen) // 3 -> 7 e.dictTable[nextHash] = tableEntry{ val: uint32(cv), offset: i, @@ -1145,7 +1147,7 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { } if len(d.content) >= 8 { cv := load6432(d.content, 0) - h := hash8(cv, betterLongTableBits) + h := hashLen(cv, betterLongTableBits, betterLongLen) e.dictLongTable[h] = prevEntry{ offset: e.maxMatchOff, prev: e.dictLongTable[h].offset, @@ -1155,7 +1157,7 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { off := 8 // First to read for i := e.maxMatchOff + 1; i < end; i++ { cv = cv>>8 | (uint64(d.content[off]) << 56) - h := hash8(cv, betterLongTableBits) + h := hashLen(cv, betterLongTableBits, betterLongLen) e.dictLongTable[h] = prevEntry{ offset: i, prev: e.dictLongTable[h].offset, diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index 96b21b90e..d6b310424 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -10,6 +10,7 @@ const ( dFastLongTableBits = 17 // Bits used in the long match table dFastLongTableSize = 1 << dFastLongTableBits // Size of the table dFastLongTableMask = dFastLongTableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. + dFastLongLen = 8 // Bytes used for table hash dLongTableShardCnt = 1 << (dFastLongTableBits - dictShardBits) // Number of shards in the table dLongTableShardSize = dFastLongTableSize / tableShardCnt // Size of an individual shard @@ -17,6 +18,8 @@ const ( dFastShortTableBits = tableBits // Bits used in the short match table dFastShortTableSize = 1 << dFastShortTableBits // Size of the table dFastShortTableMask = dFastShortTableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. + dFastShortLen = 5 // Bytes used for table hash + ) type doubleFastEncoder struct { @@ -124,8 +127,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hash5(cv, dFastShortTableBits) - nextHashL := hash8(cv, dFastLongTableBits) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -208,7 +211,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hash8(cv, dFastLongTableBits) + nextHashL = hashLen(cv, dFastLongTableBits, dFastLongLen) candidateL = e.longTable[nextHashL] coffsetL = s - (candidateL.offset - e.cur) + checkAt @@ -304,16 +307,16 @@ encodeLoop: cv1 := load6432(src, index1) te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} - e.longTable[hash8(cv0, dFastLongTableBits)] = te0 - e.longTable[hash8(cv1, dFastLongTableBits)] = te1 + e.longTable[hashLen(cv0, dFastLongTableBits, dFastLongLen)] = te0 + e.longTable[hashLen(cv1, dFastLongTableBits, dFastLongLen)] = te1 cv0 >>= 8 cv1 >>= 8 te0.offset++ te1.offset++ te0.val = uint32(cv0) te1.val = uint32(cv1) - e.table[hash5(cv0, dFastShortTableBits)] = te0 - e.table[hash5(cv1, dFastShortTableBits)] = te1 + e.table[hashLen(cv0, dFastShortTableBits, dFastShortLen)] = te0 + e.table[hashLen(cv1, dFastShortTableBits, dFastShortLen)] = te1 cv = load6432(src, s) @@ -330,8 +333,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hash5(cv, dFastShortTableBits) - nextHashL := hash8(cv, dFastLongTableBits) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -436,8 +439,8 @@ encodeLoop: var t int32 for { - nextHashS := hash5(cv, dFastShortTableBits) - nextHashL := hash8(cv, dFastLongTableBits) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -521,7 +524,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hash8(cv, dFastLongTableBits) + nextHashL = hashLen(cv, dFastLongTableBits, dFastLongLen) candidateL = e.longTable[nextHashL] coffsetL = s - (candidateL.offset - e.cur) + checkAt @@ -614,16 +617,16 @@ encodeLoop: cv1 := load6432(src, index1) te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} - e.longTable[hash8(cv0, dFastLongTableBits)] = te0 - e.longTable[hash8(cv1, dFastLongTableBits)] = te1 + e.longTable[hashLen(cv0, dFastLongTableBits, dFastLongLen)] = te0 + e.longTable[hashLen(cv1, dFastLongTableBits, dFastLongLen)] = te1 cv0 >>= 8 cv1 >>= 8 te0.offset++ te1.offset++ te0.val = uint32(cv0) te1.val = uint32(cv1) - e.table[hash5(cv0, dFastShortTableBits)] = te0 - e.table[hash5(cv1, dFastShortTableBits)] = te1 + e.table[hashLen(cv0, dFastShortTableBits, dFastShortLen)] = te0 + e.table[hashLen(cv1, dFastShortTableBits, dFastShortLen)] = te1 cv = load6432(src, s) @@ -640,8 +643,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hash5(cv1>>8, dFastShortTableBits) - nextHashL := hash8(cv, dFastLongTableBits) + nextHashS := hashLen(cv1>>8, dFastShortTableBits, dFastShortLen) + nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -782,8 +785,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hash5(cv, dFastShortTableBits) - nextHashL := hash8(cv, dFastLongTableBits) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -868,7 +871,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hash8(cv, dFastLongTableBits) + nextHashL = hashLen(cv, dFastLongTableBits, dFastLongLen) candidateL = e.longTable[nextHashL] coffsetL = s - (candidateL.offset - e.cur) + checkAt @@ -965,8 +968,8 @@ encodeLoop: cv1 := load6432(src, index1) te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} - longHash1 := hash8(cv0, dFastLongTableBits) - longHash2 := hash8(cv0, dFastLongTableBits) + longHash1 := hashLen(cv0, dFastLongTableBits, dFastLongLen) + longHash2 := hashLen(cv0, dFastLongTableBits, dFastLongLen) e.longTable[longHash1] = te0 e.longTable[longHash2] = te1 e.markLongShardDirty(longHash1) @@ -977,8 +980,8 @@ encodeLoop: te1.offset++ te0.val = uint32(cv0) te1.val = uint32(cv1) - hashVal1 := hash5(cv0, dFastShortTableBits) - hashVal2 := hash5(cv1, dFastShortTableBits) + hashVal1 := hashLen(cv0, dFastShortTableBits, dFastShortLen) + hashVal2 := hashLen(cv1, dFastShortTableBits, dFastShortLen) e.table[hashVal1] = te0 e.markShardDirty(hashVal1) e.table[hashVal2] = te1 @@ -999,8 +1002,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hash5(cv, dFastShortTableBits) - nextHashL := hash8(cv, dFastLongTableBits) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -1071,14 +1074,14 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { } if len(d.content) >= 8 { cv := load6432(d.content, 0) - e.dictLongTable[hash8(cv, dFastLongTableBits)] = tableEntry{ + e.dictLongTable[hashLen(cv, dFastLongTableBits, dFastLongLen)] = tableEntry{ val: uint32(cv), offset: e.maxMatchOff, } end := int32(len(d.content)) - 8 + e.maxMatchOff for i := e.maxMatchOff + 1; i < end; i++ { cv = cv>>8 | (uint64(d.content[i-e.maxMatchOff+7]) << 56) - e.dictLongTable[hash8(cv, dFastLongTableBits)] = tableEntry{ + e.dictLongTable[hashLen(cv, dFastLongTableBits, dFastLongLen)] = tableEntry{ val: uint32(cv), offset: i, } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index 2246d286d..f2502629b 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -11,12 +11,13 @@ import ( ) const ( - tableBits = 15 // Bits used in the table - tableSize = 1 << tableBits // Size of the table - tableShardCnt = 1 << (tableBits - dictShardBits) // Number of shards in the table - tableShardSize = tableSize / tableShardCnt // Size of an individual shard - tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. - maxMatchLength = 131074 + tableBits = 15 // Bits used in the table + tableSize = 1 << tableBits // Size of the table + tableShardCnt = 1 << (tableBits - dictShardBits) // Number of shards in the table + tableShardSize = tableSize / tableShardCnt // Size of an individual shard + tableFastHashLen = 6 + tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. + maxMatchLength = 131074 ) type tableEntry struct { @@ -122,8 +123,8 @@ encodeLoop: panic("offset0 was 0") } - nextHash := hash6(cv, hashLog) - nextHash2 := hash6(cv>>8, hashLog) + nextHash := hashLen(cv, hashLog, tableFastHashLen) + nextHash2 := hashLen(cv>>8, hashLog, tableFastHashLen) candidate := e.table[nextHash] candidate2 := e.table[nextHash2] repIndex := s - offset1 + 2 @@ -301,7 +302,7 @@ encodeLoop: } // Store this, since we have it. - nextHash := hash6(cv, hashLog) + nextHash := hashLen(cv, hashLog, tableFastHashLen) e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} seq.matchLen = uint32(l) - zstdMinMatch seq.litLen = 0 @@ -405,8 +406,8 @@ encodeLoop: // By not using them for the first 3 matches for { - nextHash := hash6(cv, hashLog) - nextHash2 := hash6(cv>>8, hashLog) + nextHash := hashLen(cv, hashLog, tableFastHashLen) + nextHash2 := hashLen(cv>>8, hashLog, tableFastHashLen) candidate := e.table[nextHash] candidate2 := e.table[nextHash2] repIndex := s - offset1 + 2 @@ -589,7 +590,7 @@ encodeLoop: } // Store this, since we have it. - nextHash := hash6(cv, hashLog) + nextHash := hashLen(cv, hashLog, tableFastHashLen) e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} seq.matchLen = uint32(l) - zstdMinMatch seq.litLen = 0 @@ -715,8 +716,8 @@ encodeLoop: panic("offset0 was 0") } - nextHash := hash6(cv, hashLog) - nextHash2 := hash6(cv>>8, hashLog) + nextHash := hashLen(cv, hashLog, tableFastHashLen) + nextHash2 := hashLen(cv>>8, hashLog, tableFastHashLen) candidate := e.table[nextHash] candidate2 := e.table[nextHash2] repIndex := s - offset1 + 2 @@ -896,7 +897,7 @@ encodeLoop: } // Store this, since we have it. - nextHash := hash6(cv, hashLog) + nextHash := hashLen(cv, hashLog, tableFastHashLen) e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} e.markShardDirty(nextHash) seq.matchLen = uint32(l) - zstdMinMatch @@ -957,9 +958,9 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { const hashLog = tableBits cv := load6432(d.content, i-e.maxMatchOff) - nextHash := hash6(cv, hashLog) // 0 -> 5 - nextHash1 := hash6(cv>>8, hashLog) // 1 -> 6 - nextHash2 := hash6(cv>>16, hashLog) // 2 -> 7 + nextHash := hashLen(cv, hashLog, tableFastHashLen) // 0 -> 5 + nextHash1 := hashLen(cv>>8, hashLog, tableFastHashLen) // 1 -> 6 + nextHash2 := hashLen(cv>>16, hashLog, tableFastHashLen) // 2 -> 7 e.dictTable[nextHash] = tableEntry{ val: uint32(cv), offset: i, diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index ea85548fc..e6e315969 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -33,7 +33,7 @@ type encoder interface { Block() *blockEnc CRC() *xxhash.Digest AppendCRC([]byte) []byte - WindowSize(size int) int32 + WindowSize(size int64) int32 UseBlock(*blockEnc) Reset(d *dict, singleBlock bool) } @@ -48,6 +48,8 @@ type encoderState struct { err error writeErr error nWritten int64 + nInput int64 + frameContentSize int64 headerWritten bool eofWritten bool fullFrameWritten bool @@ -120,7 +122,21 @@ func (e *Encoder) Reset(w io.Writer) { s.w = w s.err = nil s.nWritten = 0 + s.nInput = 0 s.writeErr = nil + s.frameContentSize = 0 +} + +// ResetContentSize will reset and set a content size for the next stream. +// If the bytes written does not match the size given an error will be returned +// when calling Close(). +// This is removed when Reset is called. +// Sizes <= 0 results in no content size set. +func (e *Encoder) ResetContentSize(w io.Writer, size int64) { + e.Reset(w) + if size >= 0 { + e.state.frameContentSize = size + } } // Write data to the encoder. @@ -190,6 +206,7 @@ func (e *Encoder) nextBlock(final bool) error { return s.err } s.nWritten += int64(n2) + s.nInput += int64(len(s.filling)) s.current = s.current[:0] s.filling = s.filling[:0] s.headerWritten = true @@ -200,8 +217,8 @@ func (e *Encoder) nextBlock(final bool) error { var tmp [maxHeaderSize]byte fh := frameHeader{ - ContentSize: 0, - WindowSize: uint32(s.encoder.WindowSize(0)), + ContentSize: uint64(s.frameContentSize), + WindowSize: uint32(s.encoder.WindowSize(s.frameContentSize)), SingleSegment: false, Checksum: e.o.crc, DictID: e.o.dict.ID(), @@ -243,6 +260,7 @@ func (e *Encoder) nextBlock(final bool) error { // Move blocks forward. s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current + s.nInput += int64(len(s.current)) s.wg.Add(1) go func(src []byte) { if debugEncoder { @@ -394,6 +412,11 @@ func (e *Encoder) Close() error { if err != nil { return err } + if s.frameContentSize > 0 { + if s.nInput != s.frameContentSize { + return fmt.Errorf("frame content size %d given, but %d bytes was written", s.frameContentSize, s.nInput) + } + } if e.state.fullFrameWritten { return s.err } @@ -470,7 +493,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } fh := frameHeader{ ContentSize: uint64(len(src)), - WindowSize: uint32(enc.WindowSize(len(src))), + WindowSize: uint32(enc.WindowSize(int64(len(src)))), SingleSegment: single, Checksum: e.o.crc, DictID: e.o.dict.ID(), diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index 16d4ab63c..7d29e1d68 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -189,7 +189,7 @@ func EncoderLevelFromZstd(level int) EncoderLevel { case level >= 6 && level < 10: return SpeedBetterCompression case level >= 10: - return SpeedBetterCompression + return SpeedBestCompression } return SpeedDefault } diff --git a/vendor/github.com/klauspost/compress/zstd/hash.go b/vendor/github.com/klauspost/compress/zstd/hash.go index 4a752067f..cf33f29a1 100644 --- a/vendor/github.com/klauspost/compress/zstd/hash.go +++ b/vendor/github.com/klauspost/compress/zstd/hash.go @@ -13,24 +13,24 @@ const ( prime8bytes = 0xcf1bbcdcb7a56463 ) -// hashLen returns a hash of the lowest l bytes of u for a size size of h bytes. -// l must be >=4 and <=8. Any other value will return hash for 4 bytes. -// h should always be <32. -// Preferably h and l should be a constant. -// FIXME: This does NOT get resolved, if 'mls' is constant, -// so this cannot be used. -func hashLen(u uint64, hashLog, mls uint8) uint32 { +// hashLen returns a hash of the lowest mls bytes of with length output bits. +// mls must be >=3 and <=8. Any other value will return hash for 4 bytes. +// length should always be < 32. +// Preferably length and mls should be a constant for inlining. +func hashLen(u uint64, length, mls uint8) uint32 { switch mls { + case 3: + return (uint32(u<<8) * prime3bytes) >> (32 - length) case 5: - return hash5(u, hashLog) + return uint32(((u << (64 - 40)) * prime5bytes) >> (64 - length)) case 6: - return hash6(u, hashLog) + return uint32(((u << (64 - 48)) * prime6bytes) >> (64 - length)) case 7: - return hash7(u, hashLog) + return uint32(((u << (64 - 56)) * prime7bytes) >> (64 - length)) case 8: - return hash8(u, hashLog) + return uint32((u * prime8bytes) >> (64 - length)) default: - return hash4x64(u, hashLog) + return (uint32(u) * prime4bytes) >> (32 - length) } } @@ -39,39 +39,3 @@ func hashLen(u uint64, hashLog, mls uint8) uint32 { func hash3(u uint32, h uint8) uint32 { return ((u << (32 - 24)) * prime3bytes) >> ((32 - h) & 31) } - -// hash4 returns the hash of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <32. -func hash4(u uint32, h uint8) uint32 { - return (u * prime4bytes) >> ((32 - h) & 31) -} - -// hash4x64 returns the hash of the lowest 4 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <32. -func hash4x64(u uint64, h uint8) uint32 { - return (uint32(u) * prime4bytes) >> ((32 - h) & 31) -} - -// hash5 returns the hash of the lowest 5 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <64. -func hash5(u uint64, h uint8) uint32 { - return uint32(((u << (64 - 40)) * prime5bytes) >> ((64 - h) & 63)) -} - -// hash6 returns the hash of the lowest 6 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <64. -func hash6(u uint64, h uint8) uint32 { - return uint32(((u << (64 - 48)) * prime6bytes) >> ((64 - h) & 63)) -} - -// hash7 returns the hash of the lowest 7 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <64. -func hash7(u uint64, h uint8) uint32 { - return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & 63)) -} - -// hash8 returns the hash of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <64. -func hash8(u uint64, h uint8) uint32 { - return uint32((u * prime8bytes) >> ((64 - h) & 63)) -} diff --git a/vendor/github.com/onsi/gomega/CHANGELOG.md b/vendor/github.com/onsi/gomega/CHANGELOG.md index 3486f3582..18190e8b9 100644 --- a/vendor/github.com/onsi/gomega/CHANGELOG.md +++ b/vendor/github.com/onsi/gomega/CHANGELOG.md @@ -1,3 +1,11 @@ +## 1.16.0 + +### Features +- feat: HaveHTTPStatus multiple expected values (#465) [aa69f1b] +- feat: HaveHTTPHeaderWithValue() matcher (#463) [dd83a96] +- feat: HaveHTTPBody matcher (#462) [504e1f2] +- feat: formatter for HTTP responses (#461) [e5b3157] + ## 1.15.0 ### Fixes diff --git a/vendor/github.com/onsi/gomega/go.mod b/vendor/github.com/onsi/gomega/go.mod index 62b8f396c..7fea4ac07 100644 --- a/vendor/github.com/onsi/gomega/go.mod +++ b/vendor/github.com/onsi/gomega/go.mod @@ -1,6 +1,6 @@ module github.com/onsi/gomega -go 1.14 +go 1.16 require ( github.com/golang/protobuf v1.5.2 diff --git a/vendor/github.com/onsi/gomega/go.sum b/vendor/github.com/onsi/gomega/go.sum index 177d5e876..56f1b44e2 100644 --- a/vendor/github.com/onsi/gomega/go.sum +++ b/vendor/github.com/onsi/gomega/go.sum @@ -1,4 +1,5 @@ github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= @@ -20,6 +21,7 @@ github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -30,13 +32,19 @@ github.com/onsi/ginkgo v1.16.4 h1:29JGrr5oVBm5ulCWet69zQkzWipVXIol6ygQUe/EzNc= github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/yuin/goldmark v1.2.1 h1:ruQGxdhGHe7FWOJPT0mKs5+pD2Xs1Bm/kdGlHO04FmM= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= @@ -47,6 +55,7 @@ golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG0 golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 h1:SQFwaSi55rU7vdNs9Yr0Z324VNlrF+0wMqRXT4St8ck= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -60,6 +69,7 @@ golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da h1:b3NXsE2LusjYGGjL5bxEVZZORm/YEFFrWFjR8eFrw/c= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= @@ -85,6 +95,7 @@ google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/l google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 h1:uRGJdciOHaEIrze2W8Q3AKkepLTh2hOroT7a+7czfdQ= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= diff --git a/vendor/github.com/onsi/gomega/gomega_dsl.go b/vendor/github.com/onsi/gomega/gomega_dsl.go index 6c7f1d9b7..84775142c 100644 --- a/vendor/github.com/onsi/gomega/gomega_dsl.go +++ b/vendor/github.com/onsi/gomega/gomega_dsl.go @@ -22,7 +22,7 @@ import ( "github.com/onsi/gomega/types" ) -const GOMEGA_VERSION = "1.15.0" +const GOMEGA_VERSION = "1.16.0" const nilGomegaPanic = `You are trying to make an assertion, but haven't registered Gomega's fail handler. If you're using Ginkgo then you probably forgot to put your assertion in an It(). diff --git a/vendor/github.com/onsi/gomega/matchers.go b/vendor/github.com/onsi/gomega/matchers.go index 667160ade..223f6ef53 100644 --- a/vendor/github.com/onsi/gomega/matchers.go +++ b/vendor/github.com/onsi/gomega/matchers.go @@ -423,10 +423,29 @@ func BeADirectory() types.GomegaMatcher { //Expected must be either an int or a string. // Expect(resp).Should(HaveHTTPStatus(http.StatusOK)) // asserts that resp.StatusCode == 200 // Expect(resp).Should(HaveHTTPStatus("404 Not Found")) // asserts that resp.Status == "404 Not Found" -func HaveHTTPStatus(expected interface{}) types.GomegaMatcher { +// Expect(resp).Should(HaveHTTPStatus(http.StatusOK, http.StatusNoContent)) // asserts that resp.StatusCode == 200 || resp.StatusCode == 204 +func HaveHTTPStatus(expected ...interface{}) types.GomegaMatcher { return &matchers.HaveHTTPStatusMatcher{Expected: expected} } +// HaveHTTPHeaderWithValue succeeds if the header is found and the value matches. +// Actual must be either a *http.Response or *httptest.ResponseRecorder. +// Expected must be a string header name, followed by a header value which +// can be a string, or another matcher. +func HaveHTTPHeaderWithValue(header string, value interface{}) types.GomegaMatcher { + return &matchers.HaveHTTPHeaderWithValueMatcher{ + Header: header, + Value: value, + } +} + +// HaveHTTPBody matches if the body matches. +// Actual must be either a *http.Response or *httptest.ResponseRecorder. +// Expected must be either a string, []byte, or other matcher +func HaveHTTPBody(expected interface{}) types.GomegaMatcher { + return &matchers.HaveHTTPBodyMatcher{Expected: expected} +} + //And succeeds only if all of the given matchers succeed. //The matchers are tried in order, and will fail-fast if one doesn't succeed. // Expect("hi").To(And(HaveLen(2), Equal("hi")) diff --git a/vendor/github.com/onsi/gomega/matchers/have_http_body_matcher.go b/vendor/github.com/onsi/gomega/matchers/have_http_body_matcher.go new file mode 100644 index 000000000..66cbb254a --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/have_http_body_matcher.go @@ -0,0 +1,101 @@ +package matchers + +import ( + "fmt" + "io/ioutil" + "net/http" + "net/http/httptest" + + "github.com/onsi/gomega/format" + "github.com/onsi/gomega/types" +) + +type HaveHTTPBodyMatcher struct { + Expected interface{} + cachedBody []byte +} + +func (matcher *HaveHTTPBodyMatcher) Match(actual interface{}) (bool, error) { + body, err := matcher.body(actual) + if err != nil { + return false, err + } + + switch e := matcher.Expected.(type) { + case string: + return (&EqualMatcher{Expected: e}).Match(string(body)) + case []byte: + return (&EqualMatcher{Expected: e}).Match(body) + case types.GomegaMatcher: + return e.Match(body) + default: + return false, fmt.Errorf("HaveHTTPBody matcher expects string, []byte, or GomegaMatcher. Got:\n%s", format.Object(matcher.Expected, 1)) + } +} + +func (matcher *HaveHTTPBodyMatcher) FailureMessage(actual interface{}) (message string) { + body, err := matcher.body(actual) + if err != nil { + return fmt.Sprintf("failed to read body: %s", err) + } + + switch e := matcher.Expected.(type) { + case string: + return (&EqualMatcher{Expected: e}).FailureMessage(string(body)) + case []byte: + return (&EqualMatcher{Expected: e}).FailureMessage(body) + case types.GomegaMatcher: + return e.FailureMessage(body) + default: + return fmt.Sprintf("HaveHTTPBody matcher expects string, []byte, or GomegaMatcher. Got:\n%s", format.Object(matcher.Expected, 1)) + } +} + +func (matcher *HaveHTTPBodyMatcher) NegatedFailureMessage(actual interface{}) (message string) { + body, err := matcher.body(actual) + if err != nil { + return fmt.Sprintf("failed to read body: %s", err) + } + + switch e := matcher.Expected.(type) { + case string: + return (&EqualMatcher{Expected: e}).NegatedFailureMessage(string(body)) + case []byte: + return (&EqualMatcher{Expected: e}).NegatedFailureMessage(body) + case types.GomegaMatcher: + return e.NegatedFailureMessage(body) + default: + return fmt.Sprintf("HaveHTTPBody matcher expects string, []byte, or GomegaMatcher. Got:\n%s", format.Object(matcher.Expected, 1)) + } +} + +// body returns the body. It is cached because once we read it in Match() +// the Reader is closed and it is not readable again in FailureMessage() +// or NegatedFailureMessage() +func (matcher *HaveHTTPBodyMatcher) body(actual interface{}) ([]byte, error) { + if matcher.cachedBody != nil { + return matcher.cachedBody, nil + } + + body := func(a *http.Response) ([]byte, error) { + if a.Body != nil { + defer a.Body.Close() + var err error + matcher.cachedBody, err = ioutil.ReadAll(a.Body) + if err != nil { + return nil, fmt.Errorf("error reading response body: %w", err) + } + } + return matcher.cachedBody, nil + } + + switch a := actual.(type) { + case *http.Response: + return body(a) + case *httptest.ResponseRecorder: + return body(a.Result()) + default: + return nil, fmt.Errorf("HaveHTTPBody matcher expects *http.Response or *httptest.ResponseRecorder. Got:\n%s", format.Object(actual, 1)) + } + +} diff --git a/vendor/github.com/onsi/gomega/matchers/have_http_header_with_value_matcher.go b/vendor/github.com/onsi/gomega/matchers/have_http_header_with_value_matcher.go new file mode 100644 index 000000000..c256f452e --- /dev/null +++ b/vendor/github.com/onsi/gomega/matchers/have_http_header_with_value_matcher.go @@ -0,0 +1,81 @@ +package matchers + +import ( + "fmt" + "net/http" + "net/http/httptest" + + "github.com/onsi/gomega/format" + "github.com/onsi/gomega/types" +) + +type HaveHTTPHeaderWithValueMatcher struct { + Header string + Value interface{} +} + +func (matcher *HaveHTTPHeaderWithValueMatcher) Match(actual interface{}) (success bool, err error) { + headerValue, err := matcher.extractHeader(actual) + if err != nil { + return false, err + } + + headerMatcher, err := matcher.getSubMatcher() + if err != nil { + return false, err + } + + return headerMatcher.Match(headerValue) +} + +func (matcher *HaveHTTPHeaderWithValueMatcher) FailureMessage(actual interface{}) string { + headerValue, err := matcher.extractHeader(actual) + if err != nil { + panic(err) // protected by Match() + } + + headerMatcher, err := matcher.getSubMatcher() + if err != nil { + panic(err) // protected by Match() + } + + diff := format.IndentString(headerMatcher.FailureMessage(headerValue), 1) + return fmt.Sprintf("HTTP header %q:\n%s", matcher.Header, diff) +} + +func (matcher *HaveHTTPHeaderWithValueMatcher) NegatedFailureMessage(actual interface{}) (message string) { + headerValue, err := matcher.extractHeader(actual) + if err != nil { + panic(err) // protected by Match() + } + + headerMatcher, err := matcher.getSubMatcher() + if err != nil { + panic(err) // protected by Match() + } + + diff := format.IndentString(headerMatcher.NegatedFailureMessage(headerValue), 1) + return fmt.Sprintf("HTTP header %q:\n%s", matcher.Header, diff) +} + +func (matcher *HaveHTTPHeaderWithValueMatcher) getSubMatcher() (types.GomegaMatcher, error) { + switch m := matcher.Value.(type) { + case string: + return &EqualMatcher{Expected: matcher.Value}, nil + case types.GomegaMatcher: + return m, nil + default: + return nil, fmt.Errorf("HaveHTTPHeaderWithValue matcher must be passed a string or a GomegaMatcher. Got:\n%s", format.Object(matcher.Value, 1)) + } +} + +func (matcher *HaveHTTPHeaderWithValueMatcher) extractHeader(actual interface{}) (string, error) { + switch r := actual.(type) { + case *http.Response: + return r.Header.Get(matcher.Header), nil + case *httptest.ResponseRecorder: + return r.Result().Header.Get(matcher.Header), nil + default: + return "", fmt.Errorf("HaveHTTPHeaderWithValue matcher expects *http.Response or *httptest.ResponseRecorder. Got:\n%s", format.Object(actual, 1)) + } +} diff --git a/vendor/github.com/onsi/gomega/matchers/have_http_status_matcher.go b/vendor/github.com/onsi/gomega/matchers/have_http_status_matcher.go index 3ce4800b7..70f54899a 100644 --- a/vendor/github.com/onsi/gomega/matchers/have_http_status_matcher.go +++ b/vendor/github.com/onsi/gomega/matchers/have_http_status_matcher.go @@ -2,14 +2,17 @@ package matchers import ( "fmt" + "io/ioutil" "net/http" "net/http/httptest" + "reflect" + "strings" "github.com/onsi/gomega/format" ) type HaveHTTPStatusMatcher struct { - Expected interface{} + Expected []interface{} } func (matcher *HaveHTTPStatusMatcher) Match(actual interface{}) (success bool, err error) { @@ -23,20 +26,71 @@ func (matcher *HaveHTTPStatusMatcher) Match(actual interface{}) (success bool, e return false, fmt.Errorf("HaveHTTPStatus matcher expects *http.Response or *httptest.ResponseRecorder. Got:\n%s", format.Object(actual, 1)) } - switch e := matcher.Expected.(type) { - case int: - return resp.StatusCode == e, nil - case string: - return resp.Status == e, nil + if len(matcher.Expected) == 0 { + return false, fmt.Errorf("HaveHTTPStatus matcher must be passed an int or a string. Got nothing") } - return false, fmt.Errorf("HaveHTTPStatus matcher must be passed an int or a string. Got:\n%s", format.Object(matcher.Expected, 1)) + for _, expected := range matcher.Expected { + switch e := expected.(type) { + case int: + if resp.StatusCode == e { + return true, nil + } + case string: + if resp.Status == e { + return true, nil + } + default: + return false, fmt.Errorf("HaveHTTPStatus matcher must be passed int or string types. Got:\n%s", format.Object(expected, 1)) + } + } + + return false, nil } func (matcher *HaveHTTPStatusMatcher) FailureMessage(actual interface{}) (message string) { - return format.Message(actual, "to have HTTP status", matcher.Expected) + return fmt.Sprintf("Expected\n%s\n%s\n%s", formatHttpResponse(actual), "to have HTTP status", matcher.expectedString()) } func (matcher *HaveHTTPStatusMatcher) NegatedFailureMessage(actual interface{}) (message string) { - return format.Message(actual, "not to have HTTP status", matcher.Expected) + return fmt.Sprintf("Expected\n%s\n%s\n%s", formatHttpResponse(actual), "not to have HTTP status", matcher.expectedString()) +} + +func (matcher *HaveHTTPStatusMatcher) expectedString() string { + var lines []string + for _, expected := range matcher.Expected { + lines = append(lines, format.Object(expected, 1)) + } + return strings.Join(lines, "\n") +} + +func formatHttpResponse(input interface{}) string { + var resp *http.Response + switch r := input.(type) { + case *http.Response: + resp = r + case *httptest.ResponseRecorder: + resp = r.Result() + default: + return "cannot format invalid HTTP response" + } + + body := "<nil>" + if resp.Body != nil { + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + data = []byte("<error reading body>") + } + body = format.Object(string(data), 0) + } + + var s strings.Builder + s.WriteString(fmt.Sprintf("%s<%s>: {\n", format.Indent, reflect.TypeOf(input))) + s.WriteString(fmt.Sprintf("%s%sStatus: %s\n", format.Indent, format.Indent, format.Object(resp.Status, 0))) + s.WriteString(fmt.Sprintf("%s%sStatusCode: %s\n", format.Indent, format.Indent, format.Object(resp.StatusCode, 0))) + s.WriteString(fmt.Sprintf("%s%sBody: %s\n", format.Indent, format.Indent, body)) + s.WriteString(fmt.Sprintf("%s}", format.Indent)) + + return s.String() } diff --git a/vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_linux.go b/vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_linux.go index a1e7f0afd..5ea9d940c 100644 --- a/vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_linux.go +++ b/vendor/github.com/opencontainers/runc/libcontainer/configs/cgroup_linux.go @@ -131,4 +131,16 @@ type Resources struct { // // NOTE it is impossible to start a container which has this flag set. SkipDevices bool `json:"-"` + + // SkipFreezeOnSet is a flag for cgroup manager to skip the cgroup + // freeze when setting resources. Only applicable to systemd legacy + // (i.e. cgroup v1) manager (which uses freeze by default to avoid + // spurious permission errors caused by systemd inability to update + // device rules in a non-disruptive manner). + // + // If not set, a few methods (such as looking into cgroup's + // devices.list and querying the systemd unit properties) are used + // during Set() to figure out whether the freeze is required. Those + // methods may be relatively slow, thus this flag. + SkipFreezeOnSet bool `json:"-"` } diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/rchcon.go b/vendor/github.com/opencontainers/selinux/go-selinux/rchcon.go new file mode 100644 index 000000000..897ecbac4 --- /dev/null +++ b/vendor/github.com/opencontainers/selinux/go-selinux/rchcon.go @@ -0,0 +1,22 @@ +// +build linux,go1.16 + +package selinux + +import ( + "errors" + "io/fs" + "os" + + "github.com/opencontainers/selinux/pkg/pwalkdir" +) + +func rchcon(fpath, label string) error { + return pwalkdir.Walk(fpath, func(p string, _ fs.DirEntry, _ error) error { + e := setFileLabel(p, label) + // Walk a file tree can race with removal, so ignore ENOENT. + if errors.Is(e, os.ErrNotExist) { + return nil + } + return e + }) +} diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/rchcon_go115.go b/vendor/github.com/opencontainers/selinux/go-selinux/rchcon_go115.go new file mode 100644 index 000000000..2c8b033ce --- /dev/null +++ b/vendor/github.com/opencontainers/selinux/go-selinux/rchcon_go115.go @@ -0,0 +1,21 @@ +// +build linux,!go1.16 + +package selinux + +import ( + "errors" + "os" + + "github.com/opencontainers/selinux/pkg/pwalk" +) + +func rchcon(fpath, label string) error { + return pwalk.Walk(fpath, func(p string, _ os.FileInfo, _ error) error { + e := setFileLabel(p, label) + // Walk a file tree can race with removal, so ignore ENOENT. + if errors.Is(e, os.ErrNotExist) { + return nil + } + return e + }) +} diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go index 62df82a34..a804473e4 100644 --- a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go +++ b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go @@ -18,7 +18,6 @@ import ( "sync" "github.com/bits-and-blooms/bitset" - "github.com/opencontainers/selinux/pkg/pwalk" "golang.org/x/sys/unix" ) @@ -1048,17 +1047,10 @@ func chcon(fpath string, label string, recurse bool) error { } if !recurse { - return SetFileLabel(fpath, label) + return setFileLabel(fpath, label) } - return pwalk.Walk(fpath, func(p string, info os.FileInfo, err error) error { - e := SetFileLabel(p, label) - // Walk a file tree can race with removal, so ignore ENOENT - if errors.Is(e, os.ErrNotExist) { - return nil - } - return e - }) + return rchcon(fpath, label) } // dupSecOpt takes an SELinux process label and returns security options that diff --git a/vendor/github.com/opencontainers/selinux/pkg/pwalk/README.md b/vendor/github.com/opencontainers/selinux/pkg/pwalk/README.md index 16c4dfd3e..7e78dce01 100644 --- a/vendor/github.com/opencontainers/selinux/pkg/pwalk/README.md +++ b/vendor/github.com/opencontainers/selinux/pkg/pwalk/README.md @@ -8,6 +8,12 @@ By default, it utilizes 2\*runtime.NumCPU() goroutines for callbacks. This can be changed by using WalkN function which has the additional parameter, specifying the number of goroutines (concurrency). +### pwalk vs pwalkdir + +This package is deprecated in favor of +[pwalkdir](https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalkdir), +which is faster, but requires at least Go 1.16. + ### Caveats Please note the following limitations of this code: diff --git a/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go b/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go index a8088a196..011fe862a 100644 --- a/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go +++ b/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go @@ -19,7 +19,7 @@ type WalkFunc = filepath.WalkFunc // // Note that this implementation only supports primitive error handling: // -// - no errors are ever passed to WalkFn; +// - no errors are ever passed to walkFn; // // - once a walkFn returns any error, all further processing stops // and the error is returned to the caller of Walk; @@ -95,7 +95,7 @@ func WalkN(root string, walkFn WalkFunc, num int) error { return err } -// walkArgs holds the arguments that were passed to the Walk or WalkLimit +// walkArgs holds the arguments that were passed to the Walk or WalkN // functions. type walkArgs struct { path string diff --git a/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/README.md b/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/README.md new file mode 100644 index 000000000..068ac4005 --- /dev/null +++ b/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/README.md @@ -0,0 +1,54 @@ +## pwalkdir: parallel implementation of filepath.WalkDir + +This is a wrapper for [filepath.WalkDir](https://pkg.go.dev/path/filepath#WalkDir) +which may speed it up by calling multiple callback functions (WalkDirFunc) +in parallel, utilizing goroutines. + +By default, it utilizes 2\*runtime.NumCPU() goroutines for callbacks. +This can be changed by using WalkN function which has the additional +parameter, specifying the number of goroutines (concurrency). + +### pwalk vs pwalkdir + +This package is very similar to +[pwalk](https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalkdir), +but utilizes `filepath.WalkDir` (added to Go 1.16), which does not call stat(2) +on every entry and is therefore faster (up to 3x, depending on usage scenario). + +Users who are OK with requiring Go 1.16+ should switch to this +implementation. + +### Caveats + +Please note the following limitations of this code: + +* Unlike filepath.WalkDir, the order of calls is non-deterministic; + +* Only primitive error handling is supported: + + * fs.SkipDir is not supported; + + * no errors are ever passed to WalkDirFunc; + + * once any error is returned from any walkDirFunc instance, no more calls + to WalkDirFunc are made, and the error is returned to the caller of WalkDir; + + * if more than one WalkDirFunc instance will return an error, only one + of such errors will be propagated to and returned by WalkDir, others + will be silently discarded. + +### Documentation + +For the official documentation, see +https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalkdir + +### Benchmarks + +For a WalkDirFunc that consists solely of the return statement, this +implementation is about 15% slower than the standard library's +filepath.WalkDir. + +Otherwise (if a WalkDirFunc is actually doing something) this is usually +faster, except when the WalkDirN(..., 1) is used. Run `go test -bench .` +to see how different operations can benefit from it, as well as how the +level of paralellism affects the speed. diff --git a/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/pwalkdir.go b/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/pwalkdir.go new file mode 100644 index 000000000..222820750 --- /dev/null +++ b/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/pwalkdir.go @@ -0,0 +1,103 @@ +// +build go1.16 + +package pwalkdir + +import ( + "fmt" + "io/fs" + "path/filepath" + "runtime" + "sync" +) + +// Walk is a wrapper for filepath.WalkDir which can call multiple walkFn +// in parallel, allowing to handle each item concurrently. A maximum of +// twice the runtime.NumCPU() walkFn will be called at any one time. +// If you want to change the maximum, use WalkN instead. +// +// The order of calls is non-deterministic. +// +// Note that this implementation only supports primitive error handling: +// +// - no errors are ever passed to walkFn; +// +// - once a walkFn returns any error, all further processing stops +// and the error is returned to the caller of Walk; +// +// - filepath.SkipDir is not supported; +// +// - if more than one walkFn instance will return an error, only one +// of such errors will be propagated and returned by Walk, others +// will be silently discarded. +func Walk(root string, walkFn fs.WalkDirFunc) error { + return WalkN(root, walkFn, runtime.NumCPU()*2) +} + +// WalkN is a wrapper for filepath.WalkDir which can call multiple walkFn +// in parallel, allowing to handle each item concurrently. A maximum of +// num walkFn will be called at any one time. +// +// Please see Walk documentation for caveats of using this function. +func WalkN(root string, walkFn fs.WalkDirFunc, num int) error { + // make sure limit is sensible + if num < 1 { + return fmt.Errorf("walk(%q): num must be > 0", root) + } + + files := make(chan *walkArgs, 2*num) + errCh := make(chan error, 1) // Get the first error, ignore others. + + // Start walking a tree asap. + var ( + err error + wg sync.WaitGroup + ) + wg.Add(1) + go func() { + err = filepath.WalkDir(root, func(p string, entry fs.DirEntry, err error) error { + if err != nil { + close(files) + return err + } + // Add a file to the queue unless a callback sent an error. + select { + case e := <-errCh: + close(files) + return e + default: + files <- &walkArgs{path: p, entry: entry} + return nil + } + }) + if err == nil { + close(files) + } + wg.Done() + }() + + wg.Add(num) + for i := 0; i < num; i++ { + go func() { + for file := range files { + if e := walkFn(file.path, file.entry, nil); e != nil { + select { + case errCh <- e: // sent ok + default: // buffer full + } + } + } + wg.Done() + }() + } + + wg.Wait() + + return err +} + +// walkArgs holds the arguments that were passed to the Walk or WalkN +// functions. +type walkArgs struct { + path string + entry fs.DirEntry +} diff --git a/vendor/golang.org/x/sys/unix/ztypes_linux.go b/vendor/golang.org/x/sys/unix/ztypes_linux.go index 72887abe5..c9d7eb41e 100644 --- a/vendor/golang.org/x/sys/unix/ztypes_linux.go +++ b/vendor/golang.org/x/sys/unix/ztypes_linux.go @@ -1773,6 +1773,8 @@ const ( NFPROTO_NUMPROTO = 0xd ) +const SO_ORIGINAL_DST = 0x50 + type Nfgenmsg struct { Nfgen_family uint8 Version uint8 diff --git a/vendor/golang.org/x/sys/windows/types_windows.go b/vendor/golang.org/x/sys/windows/types_windows.go index 1f733398e..17f03312d 100644 --- a/vendor/golang.org/x/sys/windows/types_windows.go +++ b/vendor/golang.org/x/sys/windows/types_windows.go @@ -680,7 +680,7 @@ const ( WTD_CHOICE_CERT = 5 WTD_STATEACTION_IGNORE = 0x00000000 - WTD_STATEACTION_VERIFY = 0x00000010 + WTD_STATEACTION_VERIFY = 0x00000001 WTD_STATEACTION_CLOSE = 0x00000002 WTD_STATEACTION_AUTO_CACHE = 0x00000003 WTD_STATEACTION_AUTO_CACHE_FLUSH = 0x00000004 diff --git a/vendor/modules.txt b/vendor/modules.txt index 189ffbf0f..634930d17 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -78,7 +78,7 @@ github.com/containernetworking/plugins/pkg/utils/hwaddr github.com/containernetworking/plugins/pkg/utils/sysctl github.com/containernetworking/plugins/plugins/ipam/host-local/backend github.com/containernetworking/plugins/plugins/ipam/host-local/backend/allocator -# github.com/containers/buildah v1.22.0 +# github.com/containers/buildah v1.22.3 github.com/containers/buildah github.com/containers/buildah/bind github.com/containers/buildah/chroot @@ -94,7 +94,7 @@ github.com/containers/buildah/pkg/overlay github.com/containers/buildah/pkg/parse github.com/containers/buildah/pkg/rusage github.com/containers/buildah/util -# github.com/containers/common v0.42.1 +# github.com/containers/common v0.43.2 github.com/containers/common/libimage github.com/containers/common/libimage/manifests github.com/containers/common/pkg/apparmor @@ -126,7 +126,7 @@ github.com/containers/common/pkg/umask github.com/containers/common/version # github.com/containers/conmon v2.0.20+incompatible github.com/containers/conmon/runner/config -# github.com/containers/image/v5 v5.15.0 +# github.com/containers/image/v5 v5.15.2 github.com/containers/image/v5/copy github.com/containers/image/v5/directory github.com/containers/image/v5/directory/explicitfilepath @@ -198,7 +198,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.33.1 +# github.com/containers/storage v1.34.1 github.com/containers/storage github.com/containers/storage/drivers github.com/containers/storage/drivers/aufs @@ -337,7 +337,7 @@ github.com/docker/libnetwork/resolvconf/dns github.com/docker/libnetwork/types # github.com/dtylman/scp v0.0.0-20181017070807-f3000a34aef4 github.com/dtylman/scp -# github.com/fsnotify/fsnotify v1.4.9 +# github.com/fsnotify/fsnotify v1.5.1 github.com/fsnotify/fsnotify # github.com/fsouza/go-dockerclient v1.7.3 github.com/fsouza/go-dockerclient @@ -405,7 +405,8 @@ github.com/json-iterator/go # github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a github.com/juju/ansiterm github.com/juju/ansiterm/tabwriter -# github.com/klauspost/compress v1.13.1 +# github.com/klauspost/compress v1.13.4 +github.com/klauspost/compress github.com/klauspost/compress/flate github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 @@ -488,7 +489,7 @@ github.com/onsi/ginkgo/reporters/stenographer github.com/onsi/ginkgo/reporters/stenographer/support/go-colorable github.com/onsi/ginkgo/reporters/stenographer/support/go-isatty github.com/onsi/ginkgo/types -# github.com/onsi/gomega v1.15.0 +# github.com/onsi/gomega v1.16.0 github.com/onsi/gomega github.com/onsi/gomega/format github.com/onsi/gomega/gbytes @@ -505,7 +506,7 @@ github.com/opencontainers/go-digest # github.com/opencontainers/image-spec v1.0.2-0.20190823105129-775207bd45b6 github.com/opencontainers/image-spec/specs-go github.com/opencontainers/image-spec/specs-go/v1 -# github.com/opencontainers/runc v1.0.1 +# github.com/opencontainers/runc v1.0.2 github.com/opencontainers/runc/libcontainer/apparmor github.com/opencontainers/runc/libcontainer/cgroups github.com/opencontainers/runc/libcontainer/configs @@ -522,10 +523,11 @@ github.com/opencontainers/runtime-tools/generate github.com/opencontainers/runtime-tools/generate/seccomp github.com/opencontainers/runtime-tools/specerror github.com/opencontainers/runtime-tools/validate -# github.com/opencontainers/selinux v1.8.3 +# github.com/opencontainers/selinux v1.8.4 github.com/opencontainers/selinux/go-selinux github.com/opencontainers/selinux/go-selinux/label github.com/opencontainers/selinux/pkg/pwalk +github.com/opencontainers/selinux/pkg/pwalkdir # github.com/openshift/imagebuilder v1.2.2-0.20210415181909-87f3e48c2656 github.com/openshift/imagebuilder github.com/openshift/imagebuilder/dockerfile/command @@ -555,7 +557,7 @@ github.com/prometheus/procfs/internal/fs github.com/prometheus/procfs/internal/util # github.com/rivo/uniseg v0.2.0 github.com/rivo/uniseg -# github.com/rootless-containers/rootlesskit v0.14.4 +# github.com/rootless-containers/rootlesskit v0.14.5 github.com/rootless-containers/rootlesskit/pkg/api github.com/rootless-containers/rootlesskit/pkg/msgutil github.com/rootless-containers/rootlesskit/pkg/port @@ -672,7 +674,7 @@ golang.org/x/net/proxy golang.org/x/net/trace # golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sync/semaphore -# golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 +# golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c golang.org/x/sys/cpu golang.org/x/sys/execabs golang.org/x/sys/internal/unsafeheader @@ -796,10 +798,10 @@ gopkg.in/tomb.v1 gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b gopkg.in/yaml.v3 -# k8s.io/api v0.22.0 +# k8s.io/api v0.22.1 k8s.io/api/apps/v1 k8s.io/api/core/v1 -# k8s.io/apimachinery v0.22.0 +# k8s.io/apimachinery v0.22.1 k8s.io/apimachinery/pkg/api/resource k8s.io/apimachinery/pkg/apis/meta/v1 k8s.io/apimachinery/pkg/conversion |