diff options
67 files changed, 724 insertions, 118 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index f044b98ab..e56ba9086 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -26,11 +26,11 @@ env: #### FEDORA_NAME: "fedora-34beta" PRIOR_FEDORA_NAME: "fedora-33" - UBUNTU_NAME: "ubuntu-2010" - PRIOR_UBUNTU_NAME: "ubuntu-2004" + UBUNTU_NAME: "ubuntu-2104" + PRIOR_UBUNTU_NAME: "ubuntu-2010" # Google-cloud VM Images - IMAGE_SUFFIX: "c5032481331085312" + IMAGE_SUFFIX: "c6731272010596352" FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}" PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}" UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}" diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index 1a7153848..b8ba48819 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -67,7 +67,8 @@ Briefly describe the problem you are having in a few paragraphs. (paste your output here) ``` -**Have you tested with the latest version of Podman and have you checked the Podman Troubleshooting Guide?** +**Have you tested with the latest version of Podman and have you checked the Podman Troubleshooting Guide? (https://github.com/containers/podman/blob/master/troubleshooting.md)** + Yes/No diff --git a/.github/workflows/multi-arch-build.yaml b/.github/workflows/multi-arch-build.yaml index 41ef3f3a6..e4ab88544 100644 --- a/.github/workflows/multi-arch-build.yaml +++ b/.github/workflows/multi-arch-build.yaml @@ -77,7 +77,7 @@ jobs: docker://$PODMAN_QUAY_REGISTRY/stable | \ jq -r '.Tags[]') PUSH="false" - if fgrep -qx "$VERSION" <<<"$ALLTAGS"; then + if ! fgrep -qx "$VERSION" <<<"$ALLTAGS"; then PUSH="true" fi @@ -85,7 +85,7 @@ jobs: # Only push if version tag does not exist if [[ "$PUSH" == "true" ]]; then echo "Will push $FQIN" - echo "::set-output name=podman_push::${PUSH}" + echo "::set-output name=podman_push::true" echo "::set-output name=podman_fqin::${FQIN}" else echo "Not pushing, $FQIN already exists." @@ -97,7 +97,7 @@ jobs: docker://$CONTAINERS_QUAY_REGISTRY/podman | \ jq -r '.Tags[]') PUSH="false" - if fgrep -qx "$VERSION" <<<"$ALLTAGS"; then + if ! fgrep -qx "$VERSION" <<<"$ALLTAGS"; then PUSH="true" fi @@ -105,7 +105,7 @@ jobs: # Only push if version tag does not exist if [[ "$PUSH" == "true" ]]; then echo "Will push $FQIN" - echo "::set-output name=containers_push::${PUSH}" + echo "::set-output name=containers_push::true" echo "::set-output name=containers_fqin::$FQIN" else echo "Not pushing, $FQIN already exists." @@ -425,8 +425,8 @@ pkg/api/swagger.yaml: .gopathok make -C pkg/api $(MANPAGES): %: %.md .install.md2man docdir - @sed -e 's/\((podman.*\.md)\)//' -e 's/\[\(podman.*\)\]/\1/' \ - -e 's;<\(/\)\?\(a[^>]*\|sup\)>;;g' $< | \ + @sed -e 's/\((podman[^)]*\.md)\)//g' -e 's/\[\(podman[^]]*\)\]/\1/g' \ + -e 's;<\(/\)\?\(a\|a\s\+[^>]*\|sup\)>;;g' $< | \ $(GOMD2MAN) -in /dev/stdin -out $(subst source/markdown,build/man,$@) .PHONY: docdir diff --git a/cmd/podman/common/create.go b/cmd/podman/common/create.go index d496ae308..c3d00d293 100644 --- a/cmd/podman/common/create.go +++ b/cmd/podman/common/create.go @@ -651,7 +651,7 @@ func DefineCreateFlags(cmd *cobra.Command, cf *ContainerCLIOpts) { createFlags.UintVar( &cf.StopTimeout, stopTimeoutFlagName, containerConfig.Engine.StopTimeout, - "Timeout (in seconds) to stop a container. Default is 10", + "Timeout (in seconds) that containers stopped by user command have to exit. If exceeded, the container will be forcibly stopped via SIGKILL.", ) _ = cmd.RegisterFlagCompletionFunc(stopTimeoutFlagName, completion.AutocompleteNone) @@ -697,6 +697,14 @@ func DefineCreateFlags(cmd *cobra.Command, cf *ContainerCLIOpts) { ) _ = cmd.RegisterFlagCompletionFunc(systemdFlagName, AutocompleteSystemdFlag) + timeoutFlagName := "timeout" + createFlags.UintVar( + &cf.Timeout, + timeoutFlagName, 0, + "Maximum length of time a container is allowed to run. The container will be killed automatically after the time expires.", + ) + _ = cmd.RegisterFlagCompletionFunc(timeoutFlagName, completion.AutocompleteNone) + tmpfsFlagName := "tmpfs" createFlags.StringArrayVar( &cf.TmpFS, diff --git a/cmd/podman/common/create_opts.go b/cmd/podman/common/create_opts.go index 983b9e5ca..ca36d751e 100644 --- a/cmd/podman/common/create_opts.go +++ b/cmd/podman/common/create_opts.go @@ -108,6 +108,7 @@ type ContainerCLIOpts struct { SubGIDName string Sysctl []string Systemd string + Timeout uint TmpFS []string TTY bool Timezone string diff --git a/cmd/podman/common/specgen.go b/cmd/podman/common/specgen.go index 310a07a00..f889a0169 100644 --- a/cmd/podman/common/specgen.go +++ b/cmd/podman/common/specgen.go @@ -641,6 +641,7 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *ContainerCLIOpts, args []string } s.Remove = c.Rm s.StopTimeout = &c.StopTimeout + s.Timeout = c.Timeout s.Timezone = c.Timezone s.Umask = c.Umask s.Secrets = c.Secrets diff --git a/cmd/podman/containers/stop.go b/cmd/podman/containers/stop.go index bed45f374..62ce9b036 100644 --- a/cmd/podman/containers/stop.go +++ b/cmd/podman/containers/stop.go @@ -72,7 +72,8 @@ func stopFlags(cmd *cobra.Command) { _ = flags.MarkHidden("cidfile") _ = flags.MarkHidden("ignore") } - flags.SetNormalizeFunc(utils.AliasFlags) + + flags.SetNormalizeFunc(utils.TimeoutAliasFlags) } func init() { diff --git a/cmd/podman/generate/systemd.go b/cmd/podman/generate/systemd.go index 72b2e6335..8a8f5016a 100644 --- a/cmd/podman/generate/systemd.go +++ b/cmd/podman/generate/systemd.go @@ -74,7 +74,7 @@ func init() { flags.StringVar(&format, formatFlagName, "", "Print the created units in specified format (json)") _ = systemdCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(nil)) - flags.SetNormalizeFunc(utils.AliasFlags) + flags.SetNormalizeFunc(utils.TimeoutAliasFlags) } func systemd(cmd *cobra.Command, args []string) error { diff --git a/cmd/podman/utils/alias.go b/cmd/podman/utils/alias.go index 8d089920b..306e610d9 100644 --- a/cmd/podman/utils/alias.go +++ b/cmd/podman/utils/alias.go @@ -17,8 +17,6 @@ func AliasFlags(f *pflag.FlagSet, name string) pflag.NormalizedName { name = "health-timeout" case "net": name = "network" - case "timeout": - name = "time" case "namespace": name = "ns" case "storage": @@ -34,3 +32,12 @@ func AliasFlags(f *pflag.FlagSet, name string) pflag.NormalizedName { } return pflag.NormalizedName(name) } + +// TimeoutAliasFlags is a function to handle backwards compatibility with old timeout flags +func TimeoutAliasFlags(f *pflag.FlagSet, name string) pflag.NormalizedName { + switch name { + case "timeout": + name = "time" + } + return pflag.NormalizedName(name) +} diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 2cd28e34a..16eb6735a 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -10,27 +10,17 @@ set -a # handling of the (otherwise) default shell setup is non-uniform. Rather # than attempt to workaround differences, simply force-load/set required # items every time this library is utilized. -_waserrexit=0 -if [[ "$SHELLOPTS" =~ errexit ]]; then _waserrexit=1; fi -set +e # Assumed in F33 for setting global vars -if [[ -r "/etc/automation_environment" ]]; then - source /etc/automation_environment -else # prior to automation library v2.0, this was necessary - source /etc/profile - source /etc/environment -fi -if [[ -r "/etc/ci_environment" ]]; then source /etc/ci_environment; fi USER="$(whoami)" HOME="$(getent passwd $USER | cut -d : -f 6)" # Some platforms set and make this read-only [[ -n "$UID" ]] || \ UID=$(getent passwd $USER | cut -d : -f 3) -if ((_waserrexit)); then set -e; fi -# During VM Image build, the 'containers/automation' installation -# was performed. The final step of installation sets the library -# location $AUTOMATION_LIB_PATH in /etc/environment or in the -# default shell profile depending on distribution. +# Automation library installed at image-build time, +# defining $AUTOMATION_LIB_PATH in this file. +if [[ -r "/etc/automation_environment" ]]; then + source /etc/automation_environment +fi # shellcheck disable=SC2154 if [[ -n "$AUTOMATION_LIB_PATH" ]]; then # shellcheck source=/usr/share/automation/lib/common_lib.sh @@ -43,6 +33,9 @@ else ) > /dev/stderr fi +# Managed by setup_environment.sh; holds task-specific definitions. +if [[ -r "/etc/ci_environment" ]]; then source /etc/ci_environment; fi + OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" # GCE image-name compatible string representation of distribution _major_ version OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | tr -d '.')" diff --git a/docs/source/conf.py b/docs/source/conf.py index 883e1240e..52869baf4 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -14,6 +14,8 @@ # import sys # sys.path.insert(0, os.path.abspath('.')) +import re +from recommonmark.transform import AutoStructify # -- Project information ----------------------------------------------------- @@ -59,3 +61,27 @@ html_css_files = [ ] # -- Extension configuration ------------------------------------------------- + +def convert_markdown_title(app, docname, source): + # Process markdown files only + docpath = app.env.doc2path(docname) + if docpath.endswith(".md"): + # Convert pandoc title line into eval_rst block for recommonmark + source[0] = re.sub( + r"^% (.*)", + r"```eval_rst\n.. title:: \g<1>\n```", + source[0]) + + +def setup(app): + app.connect("source-read", convert_markdown_title) + + app.add_config_value( + "recommonmark_config", { + "enable_eval_rst": True, + "enable_auto_doc_ref": False, + "enable_auto_toc_tree": False, + "enable_math": False, + "enable_inline_math": False, + }, True) + app.add_transform(AutoStructify) diff --git a/docs/source/markdown/podman-container-cleanup.1.md b/docs/source/markdown/podman-container-cleanup.1.md index bd650c6af..9f9b90fc2 100644 --- a/docs/source/markdown/podman-container-cleanup.1.md +++ b/docs/source/markdown/podman-container-cleanup.1.md @@ -48,7 +48,7 @@ After cleanup, remove the image entirely. `podman container cleanup --latest` ## SEE ALSO -podman(1), podman-container(1) +**podman**(1), **podman-container**(1), **conmon**(8). ## HISTORY Jun 2018, Originally compiled by Dan Walsh <dwalsh@redhat.com> diff --git a/docs/source/markdown/podman-create.1.md b/docs/source/markdown/podman-create.1.md index 229bb82f5..ff3f86ec9 100644 --- a/docs/source/markdown/podman-create.1.md +++ b/docs/source/markdown/podman-create.1.md @@ -951,6 +951,12 @@ The `container_manage_cgroup` boolean must be enabled for this to be allowed on `setsebool -P container_manage_cgroup true` +#### **\-\-timeout**=*seconds* + +Maximimum time a container is allowed to run before conmon sends it the kill +signal. By default containers will run until they exit or are stopped by +`podman stop`. + #### **\-\-tmpfs**=*fs* Create a tmpfs mount @@ -1412,6 +1418,20 @@ $ podman start --attach ctr b ``` +## CONMON + +When Podman starts a container it actually executes the conmon program, which +then executes the OCI Runtime. Conmon is the container monitor. It is a small +program whose job is to watch the primary process of the container, and if the +container dies, save the exit code. It also holds open the tty of the +container, so that it can be attached to later. This is what allows Podman to +run in detached mode (backgrounded), so Podman can exit but conmon continues to +run. Each container has their own instance of conmon. Conmon waits for the +container to exit, gathers and saves the exit code, and then launches a Podman +process to complete the container cleanup, by shutting down the network and +storage. For more information on conmon, please reference the conmon(8) man +page. + ## FILES **/etc/subuid** @@ -1421,7 +1441,7 @@ NOTE: Use the environment variable `TMPDIR` to change the temporary storage loca ## SEE ALSO **podman**(1), **podman-secret**(1), **podman-save**(1), **podman-ps**(1), **podman-attach**(1), **podman-pod-create**(1), **podman-port**(1), **podman-start*(1), **podman-kill**(1), **podman-stop**(1), -**podman-generate-systemd**(1) **podman-rm**(1), **subgid**(5), **subuid**(5), **containers.conf**(5), **systemd.unit**(5), **setsebool**(8), **slirp4netns**(1), **fuse-overlayfs**(1), **proc**(5)**. +**podman-generate-systemd**(1) **podman-rm**(1), **subgid**(5), **subuid**(5), **containers.conf**(5), **systemd.unit**(5), **setsebool**(8), **slirp4netns**(1), **fuse-overlayfs**(1), **proc**(5), **conmon**(8). ## HISTORY October 2017, converted from Docker documentation to Podman by Dan Walsh for Podman `<dwalsh@redhat.com>` diff --git a/docs/source/markdown/podman-generate-systemd.1.md b/docs/source/markdown/podman-generate-systemd.1.md index 431163d56..1d427d35b 100644 --- a/docs/source/markdown/podman-generate-systemd.1.md +++ b/docs/source/markdown/podman-generate-systemd.1.md @@ -233,7 +233,7 @@ CONTAINER ID IMAGE COMMAND CREATED STATUS bb310a0780ae docker.io/library/alpine:latest /bin/sh 3 minutes ago Created busy_moser ``` ## SEE ALSO -[podman(1)](podman.1.md), [podman-container(1)](podman-container.1.md), systemctl(1), systemd.unit(5), systemd.service(5) +[**podman**(1)](podman.1.md), [**podman-container**(1)](podman-container.1.md), **systemctl**(1), **systemd.unit**(5), **systemd.service**(5), **conmon**(8). ## HISTORY April 2020, Updated details and added usecase to use generated .service files as root and non-root, by Sujil Shah (sushah at redhat dot com) diff --git a/docs/source/markdown/podman-run.1.md b/docs/source/markdown/podman-run.1.md index 2e6d97a05..a41938ff6 100644 --- a/docs/source/markdown/podman-run.1.md +++ b/docs/source/markdown/podman-run.1.md @@ -1024,6 +1024,12 @@ The **container_manage_cgroup** boolean must be enabled for this to be allowed o setsebool -P container_manage_cgroup true ``` +#### **\-\-timeout**=*seconds* + +Maximimum time a container is allowed to run before conmon sends it the kill +signal. By default containers will run until they exit or are stopped by +`podman stop`. + #### **\-\-tmpfs**=*fs* Create a tmpfs mount. @@ -1763,6 +1769,20 @@ $ podman run --env ENV*****=b alpine printenv ENV***** b ``` +## CONMON + +When Podman starts a container it actually executes the conmon program, which +then executes the OCI Runtime. Conmon is the container monitor. It is a small +program whose job is to watch the primary process of the container, and if the +container dies, save the exit code. It also holds open the tty of the +container, so that it can be attached to later. This is what allows Podman to +run in detached mode (backgrounded), so Podman can exit but conmon continues to +run. Each container has their own instance of conmon. Conmon waits for the +container to exit, gathers and saves the exit code, and then launches a Podman +process to complete the container cleanup, by shutting down the network and +storage. For more information on conmon, please reference the conmon(8) man +page. + ## FILES **/etc/subuid** @@ -1773,7 +1793,7 @@ NOTE: Use the environment variable `TMPDIR` to change the temporary storage loca ## SEE ALSO **podman**(1), **podman-save**(1), **podman-ps**(1), **podman-attach**(1), **podman-pod-create**(1), **podman-port**(1), **podman-start**(1), **podman-kill**(1), **podman-stop**(1), -**podman-generate-systemd**(1) **podman-rm**(1), **subgid**(5), **subuid**(5), **containers.conf**(5), **systemd.unit**(5), **setsebool**(8), **slirp4netns**(1), **fuse-overlayfs**(1), **proc**(5)**. +**podman-generate-systemd**(1) **podman-rm**(1), **subgid**(5), **subuid**(5), **containers.conf**(5), **systemd.unit**(5), **setsebool**(8), **slirp4netns**(1), **fuse-overlayfs**(1), **proc**(5), **conmon**(8). ## HISTORY September 2018, updated by Kunal Kushwaha `<kushwaha_kunal_v7@lab.ntt.co.jp>` diff --git a/docs/source/markdown/podman.1.md b/docs/source/markdown/podman.1.md index 87bcd8802..3b90a0922 100644 --- a/docs/source/markdown/podman.1.md +++ b/docs/source/markdown/podman.1.md @@ -343,7 +343,7 @@ The Network File System (NFS) and other distributed file systems (for example: L For more information, please refer to the [Podman Troubleshooting Page](https://github.com/containers/podman/blob/master/troubleshooting.md). ## SEE ALSO -`containers-mounts.conf(5)`, `containers-registries.conf(5)`, `containers-storage.conf(5)`, `buildah(1)`, `containers.conf(5)`, `oci-hooks(5)`, `containers-policy.json(5)`, `crun(8)`, `runc(8)`, `subuid(5)`, `subgid(5)`, `slirp4netns(1)` +**containers-mounts.conf**(5), **containers-registries.conf**(5), **containers-storage.conf**(5), **buildah**(1), **containers.conf**(5), **oci-hooks**(5), **containers-policy.json**(5), **crun**(8), **runc**(8), **subuid**(5), **subgid**(5), **slirp4netns**(1), **conmon**(8). ## HISTORY Dec 2016, Originally compiled by Dan Walsh <dwalsh@redhat.com> @@ -17,7 +17,7 @@ require ( github.com/containers/image/v5 v5.11.1 github.com/containers/ocicrypt v1.1.1 github.com/containers/psgo v1.5.2 - github.com/containers/storage v1.30.0 + github.com/containers/storage v1.30.1 github.com/coreos/go-systemd/v22 v22.3.1 github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3 github.com/cri-o/ocicni v0.2.1-0.20210301205850-541cf7c703cf @@ -38,7 +38,7 @@ require ( github.com/gorilla/schema v1.2.0 github.com/hashicorp/go-multierror v1.1.1 github.com/hpcloud/tail v1.0.0 - github.com/json-iterator/go v1.1.10 + github.com/json-iterator/go v1.1.11 github.com/mattn/go-colorable v0.1.8 // indirect github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 github.com/mrunalp/fileutils v0.5.0 @@ -216,8 +216,9 @@ github.com/containers/storage v1.24.8/go.mod h1:YC+2pY8SkfEAcZkwycxYbpK8EiRbx5so github.com/containers/storage v1.28.0/go.mod h1:ixAwO7Bj31cigqPEG7aCz+PYmxkDxbIFdUFioYdxbzI= github.com/containers/storage v1.28.1/go.mod h1:5bwiMh2LkrN3AWIfDFMH7A/xbVNLcve+oeXYvHvW8cc= github.com/containers/storage v1.29.0/go.mod h1:u84RU4CCufGeJBNTRNwMB+FoE+AiFeFw4SsMoqAOeCM= -github.com/containers/storage v1.30.0 h1:KS6zmoPyy0Qcx1HCCiseQ0ysSckRvtiuoVpIGh9iwQA= github.com/containers/storage v1.30.0/go.mod h1:M/xn0pg6ReYFrLtWl5YELI/a4Xjq+Z3e5GJxQrJCcDI= +github.com/containers/storage v1.30.1 h1:+87sZDoUp0uNsP45dWypHTWTEoy0eNDgFYjTU1XIRVQ= +github.com/containers/storage v1.30.1/go.mod h1:NDJkiwxnSHD1Is+4DGcyR3SIEYSDOa0xnAW+uGQFx9E= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= @@ -485,8 +486,9 @@ github.com/jsimonetti/rtnetlink v0.0.0-20201009170750-9c6f07d100c1/go.mod h1:hqo github.com/jsimonetti/rtnetlink v0.0.0-20201110080708-d2c240429e6c/go.mod h1:huN4d1phzjhlOsNIjFsw2SVRbwIHj3fJDMEU2SDPTmg= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= -github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.11 h1:uVUAXhF2To8cbw/3xN3pxj6kk7TYKs98NIrTqPlMWAQ= +github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= @@ -503,8 +505,9 @@ github.com/klauspost/compress v1.11.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.12.1 h1:/+xsCsk06wE38cyiqOR/o7U2fSftcH72xD+BQXmja/g= github.com/klauspost/compress v1.12.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8= +github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= diff --git a/libpod/container_config.go b/libpod/container_config.go index d0572fbc2..ede6b1aab 100644 --- a/libpod/container_config.go +++ b/libpod/container_config.go @@ -298,6 +298,8 @@ type ContainerMiscConfig struct { StopSignal uint `json:"stopSignal,omitempty"` // StopTimeout is the signal that will be used to stop the container StopTimeout uint `json:"stopTimeout,omitempty"` + // Timeout is maximimum time a container will run before getting the kill signal + Timeout uint `json:"timeout,omitempty"` // Time container was created CreatedTime time.Time `json:"createdTime"` // CgroupManager is the cgroup manager used to create this container. diff --git a/libpod/container_inspect.go b/libpod/container_inspect.go index 61cc43314..5b2103c92 100644 --- a/libpod/container_inspect.go +++ b/libpod/container_inspect.go @@ -304,6 +304,8 @@ func (c *Container) generateInspectContainerConfig(spec *spec.Spec) *define.Insp ctrConfig.WorkingDir = spec.Process.Cwd } + ctrConfig.StopTimeout = c.config.StopTimeout + ctrConfig.Timeout = c.config.Timeout ctrConfig.OpenStdin = c.config.Stdin ctrConfig.Image = c.config.RootfsImageName ctrConfig.SystemdMode = c.config.Systemd diff --git a/libpod/define/container_inspect.go b/libpod/define/container_inspect.go index 1a38f5b0a..c236f35b0 100644 --- a/libpod/define/container_inspect.go +++ b/libpod/define/container_inspect.go @@ -64,6 +64,10 @@ type InspectContainerConfig struct { Umask string `json:"Umask,omitempty"` // Secrets are the secrets mounted in the container Secrets []*InspectSecret `json:"Secrets,omitempty"` + // Timeout is time before container is killed by conmon + Timeout uint `json:"Timeout"` + // StopTimeout is time before container is stoped when calling stop + StopTimeout uint `json:"StopTimeout"` } // InspectRestartPolicy holds information about the container's restart policy. diff --git a/libpod/define/info.go b/libpod/define/info.go index 00146da48..87935be2d 100644 --- a/libpod/define/info.go +++ b/libpod/define/info.go @@ -17,6 +17,7 @@ type SecurityInfo struct { DefaultCapabilities string `json:"capabilities"` Rootless bool `json:"rootless"` SECCOMPEnabled bool `json:"seccompEnabled"` + SECCOMPProfilePath string `json:"seccompProfilePath"` SELinuxEnabled bool `json:"selinuxEnabled"` } diff --git a/libpod/image/prune.go b/libpod/image/prune.go index 0e41fde44..e0480d3d1 100644 --- a/libpod/image/prune.go +++ b/libpod/image/prune.go @@ -30,7 +30,7 @@ func generatePruneFilterFuncs(filter, filterValue string) (ImageFilter, error) { return nil, err } return func(i *Image) bool { - if !until.IsZero() && i.Created().After((until)) { + if !until.IsZero() && i.Created().Before(until) { return true } return false diff --git a/libpod/image/pull_test.go b/libpod/image/pull_test.go index 2e1464ad3..d2930451c 100644 --- a/libpod/image/pull_test.go +++ b/libpod/image/pull_test.go @@ -308,6 +308,12 @@ func TestPullGoalFromPossiblyUnqualifiedName(t *testing.T) { sc.UserShortNameAliasConfPath = aliasesConf.Name() sc.SystemRegistriesConfPath = registriesConf.Name() + // Make sure to not sure the system's registries.conf.d + dir, err := ioutil.TempDir("", "example") + require.NoError(t, err) + sc.SystemRegistriesConfDirPath = dir + defer os.RemoveAll(dir) // clean up + for _, c := range []struct { input string expected []pullRefStrings diff --git a/libpod/info.go b/libpod/info.go index ef0c83a2a..7a28a4cf7 100644 --- a/libpod/info.go +++ b/libpod/info.go @@ -87,6 +87,12 @@ func (r *Runtime) hostInfo() (*define.HostInfo, error) { if err != nil { return nil, errors.Wrapf(err, "error getting hostname") } + + seccompProfilePath, err := DefaultSeccompPath() + if err != nil { + return nil, errors.Wrapf(err, "error getting Seccomp profile path") + } + info := define.HostInfo{ Arch: runtime.GOARCH, BuildahVersion: buildah.Version, @@ -106,6 +112,7 @@ func (r *Runtime) hostInfo() (*define.HostInfo, error) { DefaultCapabilities: strings.Join(r.config.Containers.DefaultCapabilities, ","), Rootless: rootless.IsRootless(), SECCOMPEnabled: seccomp.IsEnabled(), + SECCOMPProfilePath: seccompProfilePath, SELinuxEnabled: selinux.GetEnabled(), }, Slirp4NetNS: define.SlirpInfo{}, diff --git a/libpod/kube.go b/libpod/kube.go index 11ccaeadc..adcfe92c9 100644 --- a/libpod/kube.go +++ b/libpod/kube.go @@ -1,6 +1,7 @@ package libpod import ( + "fmt" "math/rand" "os" "strconv" @@ -539,11 +540,17 @@ func libpodMountsToKubeVolumeMounts(c *Container) ([]v1.VolumeMount, []v1.Volume namedVolumes, mounts := c.sortUserVolumes(c.config.Spec) vms := make([]v1.VolumeMount, 0, len(mounts)) vos := make([]v1.Volume, 0, len(mounts)) - for _, m := range mounts { + + var suffix string + for index, m := range mounts { vm, vo, err := generateKubeVolumeMount(m) if err != nil { return vms, vos, err } + // Name will be the same, so use the index as suffix + suffix = fmt.Sprintf("-%d", index) + vm.Name += suffix + vo.Name += suffix vms = append(vms, vm) vos = append(vos, vo) } diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go index 6e2c2880f..cfed5a1f2 100644 --- a/libpod/networking_linux.go +++ b/libpod/networking_linux.go @@ -577,7 +577,7 @@ func (r *Runtime) setupRootlessNetNS(ctr *Container) error { // set up port forwarder for CNI-in-slirp4netns netnsPath := ctr.state.NetNS.Path() // TODO: support slirp4netns port forwarder as well - return r.setupRootlessPortMappingViaRLK(ctr, netnsPath) + return r.setupRootlessPortMappingViaRLK(ctr, netnsPath, "") } return nil } diff --git a/libpod/networking_slirp4netns.go b/libpod/networking_slirp4netns.go index 72ab3c919..c46dc6972 100644 --- a/libpod/networking_slirp4netns.go +++ b/libpod/networking_slirp4netns.go @@ -312,7 +312,7 @@ func (r *Runtime) setupSlirp4netns(ctr *Container) error { if netOptions.isSlirpHostForward { return r.setupRootlessPortMappingViaSlirp(ctr, cmd, apiSocket) } - return r.setupRootlessPortMappingViaRLK(ctr, netnsPath) + return r.setupRootlessPortMappingViaRLK(ctr, netnsPath, netOptions.cidr) } return nil } @@ -363,7 +363,7 @@ func waitForSync(syncR *os.File, cmd *exec.Cmd, logFile io.ReadSeeker, timeout t return nil } -func (r *Runtime) setupRootlessPortMappingViaRLK(ctr *Container, netnsPath string) error { +func (r *Runtime) setupRootlessPortMappingViaRLK(ctr *Container, netnsPath, slirp4CIDR string) error { syncR, syncW, err := os.Pipe() if err != nil { return errors.Wrapf(err, "failed to open pipe") @@ -391,6 +391,16 @@ func (r *Runtime) setupRootlessPortMappingViaRLK(ctr *Container, netnsPath strin } childIP := slirp4netnsIP + // set the correct childIP when a custom cidr is set + if slirp4CIDR != "" { + _, cidr, err := net.ParseCIDR(slirp4CIDR) + if err != nil { + return errors.Wrap(err, "failed to parse slirp4netns cidr") + } + // the slirp container ip is always the hundredth ip in the subnet + cidr.IP[len(cidr.IP)-1] = cidr.IP[len(cidr.IP)-1] + 100 + childIP = cidr.IP.String() + } outer: for _, r := range ctr.state.NetworkStatus { for _, i := range r.IPs { diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go index c1acec977..1b1d4ad59 100644 --- a/libpod/oci_conmon_linux.go +++ b/libpod/oci_conmon_linux.go @@ -1024,6 +1024,10 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co args = append(args, "-i") } + if ctr.config.Timeout > 0 { + args = append(args, fmt.Sprintf("--timeout=%d", ctr.config.Timeout)) + } + if !r.enableKeyring { args = append(args, "--no-new-keyring") } diff --git a/libpod/options.go b/libpod/options.go index 103a9a80a..39415a817 100644 --- a/libpod/options.go +++ b/libpod/options.go @@ -769,6 +769,19 @@ func WithStopTimeout(timeout uint) CtrCreateOption { } } +// WithTimeout sets the maximum time a container is allowed to run" +func WithTimeout(timeout uint) CtrCreateOption { + return func(ctr *Container) error { + if ctr.valid { + return define.ErrCtrFinalized + } + + ctr.config.Timeout = timeout + + return nil + } +} + // WithIDMappings sets the idmappings for the container func WithIDMappings(idmappings storage.IDMappingOptions) CtrCreateOption { return func(ctr *Container) error { diff --git a/libpod/util.go b/libpod/util.go index b75c9179a..7f4a01f28 100644 --- a/libpod/util.go +++ b/libpod/util.go @@ -194,7 +194,15 @@ func programVersion(mountProgram string) (string, error) { // if it exists, first it checks OverrideSeccomp and then default. // If neither exist function returns "" func DefaultSeccompPath() (string, error) { - _, err := os.Stat(config.SeccompOverridePath) + def, err := config.Default() + if err != nil { + return "", err + } + if def.Containers.SeccompProfile != "" { + return def.Containers.SeccompProfile, nil + } + + _, err = os.Stat(config.SeccompOverridePath) if err == nil { return config.SeccompOverridePath, nil } diff --git a/pkg/api/handlers/compat/events.go b/pkg/api/handlers/compat/events.go index dd0a9e7a9..405e616c5 100644 --- a/pkg/api/handlers/compat/events.go +++ b/pkg/api/handlers/compat/events.go @@ -89,6 +89,10 @@ func GetEvents(w http.ResponseWriter, r *http.Request) { } e := entities.ConvertToEntitiesEvent(*evt) + if !utils.IsLibpodRequest(r) && e.Status == "died" { + e.Status = "die" + } + if err := coder.Encode(e); err != nil { logrus.Errorf("unable to write json: %q", err) } diff --git a/pkg/api/handlers/compat/volumes.go b/pkg/api/handlers/compat/volumes.go index d86fc1e19..1ff1468e7 100644 --- a/pkg/api/handlers/compat/volumes.go +++ b/pkg/api/handlers/compat/volumes.go @@ -96,11 +96,17 @@ func CreateVolume(w http.ResponseWriter, r *http.Request) { return } - // See if the volume exists already - existingVolume, err := runtime.GetVolume(input.Name) - if err != nil && errors.Cause(err) != define.ErrNoSuchVolume { - utils.InternalServerError(w, err) - return + var ( + existingVolume *libpod.Volume + err error + ) + if len(input.Name) != 0 { + // See if the volume exists already + existingVolume, err = runtime.GetVolume(input.Name) + if err != nil && errors.Cause(err) != define.ErrNoSuchVolume { + utils.InternalServerError(w, err) + return + } } // if using the compat layer and the volume already exists, we diff --git a/pkg/domain/filters/containers.go b/pkg/domain/filters/containers.go index 45791cd84..9ac72e415 100644 --- a/pkg/domain/filters/containers.go +++ b/pkg/domain/filters/containers.go @@ -83,7 +83,19 @@ func GenerateContainerFilterFuncs(filter string, filterValues []string, r *libpo return func(c *libpod.Container) bool { for _, filterValue := range filterValues { containerConfig := c.Config() - if strings.Contains(containerConfig.RootfsImageID, filterValue) || strings.Contains(containerConfig.RootfsImageName, filterValue) { + var imageTag string + var imageNameWithoutTag string + // Compare with ImageID, ImageName + // Will match ImageName if running image has tag latest for other tags exact complete filter must be given + imageNameSlice := strings.SplitN(containerConfig.RootfsImageName, ":", 2) + if len(imageNameSlice) == 2 { + imageNameWithoutTag = imageNameSlice[0] + imageTag = imageNameSlice[1] + } + + if (containerConfig.RootfsImageID == filterValue) || + (containerConfig.RootfsImageName == filterValue) || + (imageNameWithoutTag == filterValue && imageTag == "latest") { return true } } diff --git a/pkg/machine/ignition.go b/pkg/machine/ignition.go index cc5c01de6..00068a136 100644 --- a/pkg/machine/ignition.go +++ b/pkg/machine/ignition.go @@ -168,6 +168,22 @@ func getFiles(usrName string) []File { }, FileEmbedded1: FileEmbedded1{Mode: intToPtr(420)}, }) + + // Set machine_enabled to true to indicate we're in a VM + files = append(files, File{ + Node: Node{ + Group: getNodeGrp("root"), + Path: "/etc/containers/containers.conf", + User: getNodeUsr("root"), + }, + FileEmbedded1: FileEmbedded1{ + Append: nil, + Contents: Resource{ + Source: strToPtr("data:,%5Bengine%5D%0Amachine_enabled%3Dtrue%0A"), + }, + Mode: intToPtr(420), + }, + }) return files } diff --git a/pkg/rootless/rootless_linux.go b/pkg/rootless/rootless_linux.go index fdfeed854..6eff25eb9 100644 --- a/pkg/rootless/rootless_linux.go +++ b/pkg/rootless/rootless_linux.go @@ -116,7 +116,7 @@ func tryMappingTool(uid bool, pid int, hostID int, mappings []idtools.IDMap) err } path, err := exec.LookPath(tool) if err != nil { - return errors.Wrapf(err, "cannot find %s", tool) + return errors.Wrapf(err, "command required for rootless mode with multiple IDs") } appendTriplet := func(l []string, a, b, c int) []string { diff --git a/pkg/specgen/generate/container_create.go b/pkg/specgen/generate/container_create.go index 2f623bf10..277435ef1 100644 --- a/pkg/specgen/generate/container_create.go +++ b/pkg/specgen/generate/container_create.go @@ -325,6 +325,9 @@ func createContainerOptions(ctx context.Context, rt *libpod.Runtime, s *specgen. if s.StopTimeout != nil { options = append(options, libpod.WithStopTimeout(*s.StopTimeout)) } + if s.Timeout != 0 { + options = append(options, libpod.WithTimeout(s.Timeout)) + } if s.LogConfiguration != nil { if len(s.LogConfiguration.Path) > 0 { options = append(options, libpod.WithLogPath(s.LogConfiguration.Path)) diff --git a/pkg/specgen/specgen.go b/pkg/specgen/specgen.go index e3d4b1436..fdcb7a0e0 100644 --- a/pkg/specgen/specgen.go +++ b/pkg/specgen/specgen.go @@ -83,6 +83,11 @@ type ContainerBasicConfig struct { // instead. // Optional. StopTimeout *uint `json:"stop_timeout,omitempty"` + // Timeout is a maximum time in seconds the container will run before + // main process is sent SIGKILL. + // If 0 is used, signal will not be sent. Container can run indefinitely + // Optional. + Timeout uint `json:"timeout,omitempty"` // LogConfiguration describes the logging for a container including // driver, path, and options. // Optional diff --git a/test/apiv2/10-images.at b/test/apiv2/10-images.at index f854d38ab..a08393668 100644 --- a/test/apiv2/10-images.at +++ b/test/apiv2/10-images.at @@ -126,6 +126,20 @@ t DELETE libpod/images/test:test 200 t GET images/json?filters='{"label":["xyz"]}' 200 length=0 t GET libpod/images/json?filters='{"label":["xyz"]}' 200 length=0 + +# to be used in prune until filter tests +podman image build -t test1:latest -<<EOF +from alpine +RUN >file3 +EOF + +# image should not be deleted +t GET images/json?filters='{"reference":["test1"]}' 200 length=1 +t POST images/prune?filters='{"until":["500000"]}' 200 +t GET images/json?filters='{"reference":["test1"]}' 200 length=1 + +t DELETE libpod/images/test1:latest 200 + # Export more than one image # FIXME FIXME FIXME, this doesn't work: # not ok 64 [10-images] GET images/get?names=alpine,busybox : status diff --git a/test/apiv2/30-volumes.at b/test/apiv2/30-volumes.at index 623e691e3..ed606134a 100644 --- a/test/apiv2/30-volumes.at +++ b/test/apiv2/30-volumes.at @@ -13,6 +13,13 @@ t POST libpod/volumes/create name=foo1 201 \ .CreatedAt~[0-9]\\{4\\}-[0-9]\\{2\\}-[0-9]\\{2\\}.* \ .Labels={} \ .Options={} +t POST volumes/create 201 \ + .Name~[0-9a-f]\\{64\\} + .Driver=local \ + .Mountpoint=$volumepath/~[0-9a-f]\\{64\\}/_data \ + .CreatedAt~[0-9]\\{4\\}-[0-9]\\{2\\}-[0-9]\\{2\\}.* \ + .Labels={} \ + .Options={} t POST libpod/volumes/create 201 t POST libpod/volumes/create \ Name=foo2 \ diff --git a/test/e2e/containers_conf_test.go b/test/e2e/containers_conf_test.go index 803124de1..a354de3b2 100644 --- a/test/e2e/containers_conf_test.go +++ b/test/e2e/containers_conf_test.go @@ -353,4 +353,23 @@ var _ = Describe("Podman run", func() { Expect(session.ExitCode()).To(Equal(0)) Expect(session.OutputToString()).To(ContainSubstring("test")) }) + + It("podman info seccomp profile path", func() { + configPath := filepath.Join(podmanTest.TempDir, "containers.conf") + os.Setenv("CONTAINERS_CONF", configPath) + + profile := filepath.Join(podmanTest.TempDir, "seccomp.json") + containersConf := []byte(fmt.Sprintf("[containers]\nseccomp_profile=\"%s\"", profile)) + err = ioutil.WriteFile(configPath, containersConf, os.ModePerm) + Expect(err).To(BeNil()) + + if IsRemote() { + podmanTest.RestartRemoteService() + } + + session := podmanTest.Podman([]string{"info", "--format", "{{.Host.Security.SECCOMPProfilePath}}"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(Equal(profile)) + }) }) diff --git a/test/e2e/generate_kube_test.go b/test/e2e/generate_kube_test.go index c3586d9b6..611e8ddac 100644 --- a/test/e2e/generate_kube_test.go +++ b/test/e2e/generate_kube_test.go @@ -496,6 +496,29 @@ var _ = Describe("Podman generate kube", func() { Expect(inspect.OutputToString()).To(ContainSubstring(vol1)) }) + It("podman generate kube when bind-mounting '/' and '/root' at the same time ", func() { + // Fixes https://github.com/containers/podman/issues/9764 + + ctrName := "mount-root-ctr" + session1 := podmanTest.Podman([]string{"run", "-d", "--pod", "new:mount-root-conflict", "--name", ctrName, + "-v", "/:/volume1/", + "-v", "/root:/volume2/", + "alpine", "top"}) + session1.WaitWithDefaultTimeout() + Expect(session1.ExitCode()).To(Equal(0)) + + kube := podmanTest.Podman([]string{"generate", "kube", "mount-root-conflict"}) + kube.WaitWithDefaultTimeout() + Expect(kube.ExitCode()).To(Equal(0)) + + pod := new(v1.Pod) + err := yaml.Unmarshal(kube.Out.Contents(), pod) + Expect(err).To(BeNil()) + + Expect(len(pod.Spec.Volumes)).To(Equal(2)) + + }) + It("podman generate kube with persistent volume claim", func() { vol := "vol-test-persistent-volume-claim" diff --git a/test/e2e/generate_systemd_test.go b/test/e2e/generate_systemd_test.go index 3a1da5d8c..75d778f10 100644 --- a/test/e2e/generate_systemd_test.go +++ b/test/e2e/generate_systemd_test.go @@ -242,7 +242,7 @@ var _ = Describe("Podman generate systemd", func() { n.WaitWithDefaultTimeout() Expect(n.ExitCode()).To(Equal(0)) - session := podmanTest.Podman([]string{"generate", "systemd", "--timeout", "42", "--name", "--new", "foo"}) + session := podmanTest.Podman([]string{"generate", "systemd", "--time", "42", "--name", "--new", "foo"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) diff --git a/test/e2e/ps_test.go b/test/e2e/ps_test.go index 37b6516c1..d5269f415 100644 --- a/test/e2e/ps_test.go +++ b/test/e2e/ps_test.go @@ -269,6 +269,12 @@ var _ = Describe("Podman ps", func() { result.WaitWithDefaultTimeout() Expect(result.ExitCode()).To(Equal(0)) Expect(result.OutputToString()).To(Equal(cid)) + + // Query by trunctated image name should not match ( should return empty output ) + result = podmanTest.Podman([]string{"ps", "-q", "--no-trunc", "-a", "--filter", "ancestor=quay.io/libpod/alpi"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(result.OutputToString()).To(Equal("")) }) It("podman ps id filter flag", func() { diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go index 93505d742..74bdfce2c 100644 --- a/test/e2e/run_test.go +++ b/test/e2e/run_test.go @@ -582,6 +582,9 @@ USER bin`, BB) if _, err := os.Stat("/sys/fs/cgroup/io.stat"); os.IsNotExist(err) { Skip("Kernel does not have io.stat") } + if _, err := os.Stat("/sys/fs/cgroup/system.slice/io.bfq.weight"); os.IsNotExist(err) { + Skip("Kernel does not support BFQ IO scheduler") + } session := podmanTest.Podman([]string{"run", "--rm", "--blkio-weight=15", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/io.bfq.weight"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) diff --git a/test/system/030-run.bats b/test/system/030-run.bats index b2999a9e7..2b83fa56e 100644 --- a/test/system/030-run.bats +++ b/test/system/030-run.bats @@ -501,6 +501,7 @@ json-file | f run_podman inspect --format '{{.OCIRuntime}}' $cid is "$output" "$new_runtime" "podman inspect shows configured runtime" run_podman kill $cid + run_podman wait $cid run_podman rm $cid rm -f $new_runtime } @@ -668,4 +669,27 @@ json-file | f is "$output" ".*HOME=/.*" } +@test "podman run --timeout - basic test" { + cid=timeouttest + t0=$SECONDS + run_podman 255 run --name $cid --timeout 10 $IMAGE sleep 60 + t1=$SECONDS + # Confirm that container is stopped. Podman-remote unfortunately + # cannot tell the difference between "stopped" and "exited", and + # spits them out interchangeably, so we need to recognize either. + run_podman inspect --format '{{.State.Status}} {{.State.ExitCode}}' $cid + is "$output" "\\(stopped\|exited\\) \-1" \ + "Status and exit code of stopped container" + + # This operation should take + # exactly 10 seconds. Give it some leeway. + delta_t=$(( $t1 - $t0 )) + [ $delta_t -gt 8 ] ||\ + die "podman stop: ran too quickly! ($delta_t seconds; expected >= 10)" + [ $delta_t -le 14 ] ||\ + die "podman stop: took too long ($delta_t seconds; expected ~10)" + + run_podman rm $cid +} + # vim: filetype=sh diff --git a/test/system/500-networking.bats b/test/system/500-networking.bats index 8da864798..788dc4cd1 100644 --- a/test/system/500-networking.bats +++ b/test/system/500-networking.bats @@ -77,35 +77,48 @@ load helpers # FIXME: randomize port, and create second random host port myport=54321 - # Container will exit as soon as 'nc' receives input - # We use '-n -v' to give us log messages showing an incoming connection - # and its IP address; the purpose of that is guaranteeing that the - # remote IP is not 127.0.0.1 (podman PR #9052). - # We could get more parseable output by using $NCAT_REMOTE_ADDR, - # but busybox nc doesn't support that. - run_podman run -d --userns=keep-id -p 127.0.0.1:$myport:$myport \ - $IMAGE nc -l -n -v -p $myport - cid="$output" - - # emit random string, and check it - teststring=$(random_string 30) - echo "$teststring" | nc 127.0.0.1 $myport - - run_podman logs $cid - # Sigh. We can't check line-by-line, because 'nc' output order is - # unreliable. We usually get the 'connect to' line before the random - # string, but sometimes we get it after. So, just do substring checks. - is "$output" ".*listening on \[::\]:$myport .*" "nc -v shows right port" - - # This is the truly important check: make sure the remote IP is - # in the 10.X range, not 127.X. - is "$output" \ - ".*connect to \[::ffff:10\..*\]:$myport from \[::ffff:10\..*\]:.*" \ - "nc -v shows remote IP address in 10.X space (not 127.0.0.1)" - is "$output" ".*${teststring}.*" "test string received on container" - - # Clean up - run_podman rm $cid + for cidr in "" "$(random_rfc1918_subnet).0/24"; do + myport=$(( myport + 1 )) + if [[ -z $cidr ]]; then + # regex to match that we are in 10.X subnet + match="10\..*" + else + # Issue #9828 make sure a custom slir4netns cidr also works + network_arg="--network slirp4netns:cidr=$cidr" + # slirp4netns interface ip is always .100 + match="${cidr%.*}.100" + fi + + # Container will exit as soon as 'nc' receives input + # We use '-n -v' to give us log messages showing an incoming connection + # and its IP address; the purpose of that is guaranteeing that the + # remote IP is not 127.0.0.1 (podman PR #9052). + # We could get more parseable output by using $NCAT_REMOTE_ADDR, + # but busybox nc doesn't support that. + run_podman run -d --userns=keep-id $network_arg -p 127.0.0.1:$myport:$myport \ + $IMAGE nc -l -n -v -p $myport + cid="$output" + + # emit random string, and check it + teststring=$(random_string 30) + echo "$teststring" | nc 127.0.0.1 $myport + + run_podman logs $cid + # Sigh. We can't check line-by-line, because 'nc' output order is + # unreliable. We usually get the 'connect to' line before the random + # string, but sometimes we get it after. So, just do substring checks. + is "$output" ".*listening on \[::\]:$myport .*" "nc -v shows right port" + + # This is the truly important check: make sure the remote IP is not 127.X. + is "$output" \ + ".*connect to \[::ffff:$match*\]:$myport from \[::ffff:$match\]:.*" \ + "nc -v shows remote IP address is not 127.0.0.1" + is "$output" ".*${teststring}.*" "test string received on container" + + # Clean up + run_podman wait $cid + run_podman rm $cid + done } # "network create" now works rootless, with the help of a special container @@ -128,7 +141,7 @@ load helpers is "$output" ".* inet ${mysubnet}\.2/24 brd ${mysubnet}\.255 " \ "sdfsdf" - run_podman run --rm -d --network $mynetname -p 127.0.0.1:$myport:$myport \ + run_podman run -d --network $mynetname -p 127.0.0.1:$myport:$myport \ $IMAGE nc -l -n -v -p $myport cid="$output" @@ -154,6 +167,7 @@ load helpers is "$output" "Error: the network name $mynetname is already used" \ "Trying to create an already-existing network" + run_podman rm $cid run_podman network rm $mynetname run_podman 1 network rm $mynetname } diff --git a/troubleshooting.md b/troubleshooting.md index 077e342cd..1e21edab4 100644 --- a/troubleshooting.md +++ b/troubleshooting.md @@ -23,7 +23,7 @@ touch: cannot touch '/content/file': Permission denied #### Solution -This is usually caused by SELinux. +This is sometimes caused by SELinux, and sometimes by user namespaces. Labeling systems like SELinux require that proper labels are placed on volume content mounted into a container. Without a label, the security system might @@ -47,6 +47,14 @@ will disable SELinux separation for the container. $ podman run --security-opt label=disable -v ~:/home/user fedora touch /home/user/file +In cases where the container image runs as a specific, non-root user, though, the +solution is to fix the user namespace. This would include container images such as +the Jupyter Notebook image (which runs as "jovyan") and the Postgres image (which runs +as "postgres"). In either case, use the `--userns` switch to map user namespaces, +most of the time by using keep_id option. + +$ podman run -v "$PWD":/home/jovyan/work --userns=keep_id jupyter/scipy-notebook + --- ### 3) No such image or Bare keys cannot contain ':' @@ -203,12 +211,12 @@ Rootless Podman requires the newuidmap and newgidmap programs to be installed. #### Symptom -If you are running Podman or buildah as a not root user, you get an error complaining about +If you are running Podman or Buildah as a rootless user, you get an error complaining about a missing newuidmap executable. ``` podman run -ti fedora sh -cannot find newuidmap: exec: "newuidmap": executable file not found in $PATH +command required for rootless mode with multiple IDs: exec: "newuidmap": executable file not found in $PATH ``` #### Solution diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index 034552a83..7f3c3affd 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.30.0 +1.30.1 diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index cbf31d353..26f4b0a55 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -594,7 +594,8 @@ func supportsOverlay(home string, homeMagic graphdriver.FsMagic, rootUID, rootGI _ = idtools.MkdirAs(upperDir, 0700, rootUID, rootGID) _ = idtools.MkdirAs(workDir, 0700, rootUID, rootGID) flags := fmt.Sprintf("lowerdir=%s:%s,upperdir=%s,workdir=%s", lower1Dir, lower2Dir, upperDir, workDir) - if selinux.GetEnabled() { + if selinux.GetEnabled() && + selinux.SecurityCheckContext(selinuxLabelTest) == nil { // Linux 5.11 introduced unprivileged overlay mounts but it has an issue // when used together with selinux labels. // Check that overlay supports selinux labels as well. diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index 6a34d06c6..c035e2979 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -9,7 +9,7 @@ require ( github.com/docker/go-units v0.4.0 github.com/google/go-intervals v0.0.2 github.com/hashicorp/go-multierror v1.1.1 - github.com/klauspost/compress v1.12.1 + github.com/klauspost/compress v1.12.2 github.com/klauspost/pgzip v1.2.5 github.com/mattn/go-shellwords v1.0.11 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible diff --git a/vendor/github.com/containers/storage/go.sum b/vendor/github.com/containers/storage/go.sum index 28edd4a7e..f73f225f2 100644 --- a/vendor/github.com/containers/storage/go.sum +++ b/vendor/github.com/containers/storage/go.sum @@ -339,8 +339,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.12.1 h1:/+xsCsk06wE38cyiqOR/o7U2fSftcH72xD+BQXmja/g= -github.com/klauspost/compress v1.12.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.12.2 h1:2KCfW3I9M7nSc5wOqXAlW2v2U6v+w6cbjvbfp+OykW8= +github.com/klauspost/compress v1.12.2/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index feb931133..25767b7a2 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -2115,6 +2115,37 @@ func (s *store) SetNames(id string, names []string) error { return ristore.SetNames(id, deduped) } + // Check is id refers to a RO Store + ristores, err := s.ROImageStores() + if err != nil { + return err + } + for _, s := range ristores { + store := s + store.RLock() + defer store.Unlock() + if modified, err := store.Modified(); modified || err != nil { + if err = store.Load(); err != nil { + return err + } + } + if i, err := store.Get(id); err == nil { + // Unlock R/O lock and lock with R/W lock + // Previous defer.Unlock() will free the new lock. + ristore.Unlock() + ristore.Lock() + if len(deduped) > 1 { + // Do not want to create image name in R/W storage + deduped = deduped[1:] + } + _, err := ristore.Create(id, deduped, i.TopLayer, i.Metadata, i.Created, i.Digest) + if err == nil { + return ristore.Save() + } + return err + } + } + rcstore, err := s.ContainerStore() if err != nil { return err diff --git a/vendor/github.com/json-iterator/go/go.sum b/vendor/github.com/json-iterator/go/go.sum index d778b5a14..be00a6df9 100644 --- a/vendor/github.com/json-iterator/go/go.sum +++ b/vendor/github.com/json-iterator/go/go.sum @@ -9,6 +9,7 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742 h1:Esafd1046DLD github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/stretchr/objx v0.1.0 h1:4G4v2dO3VZwixGIRoQ5Lfboy6nUhCyYzaqnIAPPhYs4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= diff --git a/vendor/github.com/json-iterator/go/iter_float.go b/vendor/github.com/json-iterator/go/iter_float.go index b9754638e..8a3d8b6fb 100644 --- a/vendor/github.com/json-iterator/go/iter_float.go +++ b/vendor/github.com/json-iterator/go/iter_float.go @@ -288,6 +288,9 @@ non_decimal_loop: return iter.readFloat64SlowPath() } value = (value << 3) + (value << 1) + uint64(ind) + if value > maxFloat64 { + return iter.readFloat64SlowPath() + } } } return iter.readFloat64SlowPath() diff --git a/vendor/github.com/json-iterator/go/iter_int.go b/vendor/github.com/json-iterator/go/iter_int.go index 214232035..d786a89fe 100644 --- a/vendor/github.com/json-iterator/go/iter_int.go +++ b/vendor/github.com/json-iterator/go/iter_int.go @@ -9,6 +9,7 @@ var intDigits []int8 const uint32SafeToMultiply10 = uint32(0xffffffff)/10 - 1 const uint64SafeToMultiple10 = uint64(0xffffffffffffffff)/10 - 1 +const maxFloat64 = 1<<53 - 1 func init() { intDigits = make([]int8, 256) @@ -339,7 +340,7 @@ func (iter *Iterator) readUint64(c byte) (ret uint64) { } func (iter *Iterator) assertInteger() { - if iter.head < len(iter.buf) && iter.buf[iter.head] == '.' { + if iter.head < iter.tail && iter.buf[iter.head] == '.' { iter.ReportError("assertInteger", "can not decode float as int") } } diff --git a/vendor/github.com/json-iterator/go/reflect.go b/vendor/github.com/json-iterator/go/reflect.go index 74974ba74..39acb320a 100644 --- a/vendor/github.com/json-iterator/go/reflect.go +++ b/vendor/github.com/json-iterator/go/reflect.go @@ -65,7 +65,7 @@ func (iter *Iterator) ReadVal(obj interface{}) { decoder := iter.cfg.getDecoderFromCache(cacheKey) if decoder == nil { typ := reflect2.TypeOf(obj) - if typ.Kind() != reflect.Ptr { + if typ == nil || typ.Kind() != reflect.Ptr { iter.ReportError("ReadVal", "can only unmarshal into pointer") return } diff --git a/vendor/github.com/json-iterator/go/reflect_json_raw_message.go b/vendor/github.com/json-iterator/go/reflect_json_raw_message.go index f2619936c..eba434f2f 100644 --- a/vendor/github.com/json-iterator/go/reflect_json_raw_message.go +++ b/vendor/github.com/json-iterator/go/reflect_json_raw_message.go @@ -33,11 +33,19 @@ type jsonRawMessageCodec struct { } func (codec *jsonRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { - *((*json.RawMessage)(ptr)) = json.RawMessage(iter.SkipAndReturnBytes()) + if iter.ReadNil() { + *((*json.RawMessage)(ptr)) = nil + } else { + *((*json.RawMessage)(ptr)) = iter.SkipAndReturnBytes() + } } func (codec *jsonRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteRaw(string(*((*json.RawMessage)(ptr)))) + if *((*json.RawMessage)(ptr)) == nil { + stream.WriteNil() + } else { + stream.WriteRaw(string(*((*json.RawMessage)(ptr)))) + } } func (codec *jsonRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { @@ -48,11 +56,19 @@ type jsoniterRawMessageCodec struct { } func (codec *jsoniterRawMessageCodec) Decode(ptr unsafe.Pointer, iter *Iterator) { - *((*RawMessage)(ptr)) = RawMessage(iter.SkipAndReturnBytes()) + if iter.ReadNil() { + *((*RawMessage)(ptr)) = nil + } else { + *((*RawMessage)(ptr)) = iter.SkipAndReturnBytes() + } } func (codec *jsoniterRawMessageCodec) Encode(ptr unsafe.Pointer, stream *Stream) { - stream.WriteRaw(string(*((*RawMessage)(ptr)))) + if *((*RawMessage)(ptr)) == nil { + stream.WriteNil() + } else { + stream.WriteRaw(string(*((*RawMessage)(ptr)))) + } } func (codec *jsoniterRawMessageCodec) IsEmpty(ptr unsafe.Pointer) bool { diff --git a/vendor/github.com/json-iterator/go/reflect_struct_decoder.go b/vendor/github.com/json-iterator/go/reflect_struct_decoder.go index d7eb0eb5c..92ae912dc 100644 --- a/vendor/github.com/json-iterator/go/reflect_struct_decoder.go +++ b/vendor/github.com/json-iterator/go/reflect_struct_decoder.go @@ -1075,6 +1075,11 @@ type stringModeNumberDecoder struct { } func (decoder *stringModeNumberDecoder) Decode(ptr unsafe.Pointer, iter *Iterator) { + if iter.WhatIsNext() == NilValue { + decoder.elemDecoder.Decode(ptr, iter) + return + } + c := iter.nextToken() if c != '"' { iter.ReportError("stringModeNumberDecoder", `expect ", but found `+string([]byte{c})) diff --git a/vendor/github.com/klauspost/compress/flate/level5.go b/vendor/github.com/klauspost/compress/flate/level5.go index d513f1ffd..293a3a320 100644 --- a/vendor/github.com/klauspost/compress/flate/level5.go +++ b/vendor/github.com/klauspost/compress/flate/level5.go @@ -182,12 +182,27 @@ func (e *fastEncL5) Encode(dst *tokens, src []byte) { // match. But, prior to the match, src[nextEmit:s] are unmatched. Emit // them as literal bytes. - // Extend the 4-byte match as long as possible. if l == 0 { + // Extend the 4-byte match as long as possible. l = e.matchlenLong(s+4, t+4, src) + 4 } else if l == maxMatchLength { l += e.matchlenLong(s+l, t+l, src) } + + // Try to locate a better match by checking the end of best match... + if sAt := s + l; l < 30 && sAt < sLimit { + eLong := e.bTable[hash7(load6432(src, sAt), tableBits)].Cur.offset + // Test current + t2 := eLong - e.cur - l + off := s - t2 + if t2 >= 0 && off < maxMatchOffset && off > 0 { + if l2 := e.matchlenLong(s, t2, src); l2 > l { + t = t2 + l = l2 + } + } + } + // Extend backwards for t > 0 && s > nextEmit && src[t-1] == src[s-1] { s-- diff --git a/vendor/github.com/klauspost/compress/flate/level6.go b/vendor/github.com/klauspost/compress/flate/level6.go index a52c80ea4..a709977ec 100644 --- a/vendor/github.com/klauspost/compress/flate/level6.go +++ b/vendor/github.com/klauspost/compress/flate/level6.go @@ -211,6 +211,31 @@ func (e *fastEncL6) Encode(dst *tokens, src []byte) { l += e.matchlenLong(s+l, t+l, src) } + // Try to locate a better match by checking the end-of-match... + if sAt := s + l; sAt < sLimit { + eLong := &e.bTable[hash7(load6432(src, sAt), tableBits)] + // Test current + t2 := eLong.Cur.offset - e.cur - l + off := s - t2 + if off < maxMatchOffset { + if off > 0 && t2 >= 0 { + if l2 := e.matchlenLong(s, t2, src); l2 > l { + t = t2 + l = l2 + } + } + // Test next: + t2 = eLong.Prev.offset - e.cur - l + off := s - t2 + if off > 0 && off < maxMatchOffset && t2 >= 0 { + if l2 := e.matchlenLong(s, t2, src); l2 > l { + t = t2 + l = l2 + } + } + } + } + // Extend backwards for t > 0 && s > nextEmit && src[t-1] == src[s-1] { s-- diff --git a/vendor/github.com/klauspost/compress/zstd/README.md b/vendor/github.com/klauspost/compress/zstd/README.md index 7680bfe1d..e7d7eb0ae 100644 --- a/vendor/github.com/klauspost/compress/zstd/README.md +++ b/vendor/github.com/klauspost/compress/zstd/README.md @@ -152,8 +152,8 @@ This package: file out level insize outsize millis mb/s silesia.tar zskp 1 211947520 73101992 643 313.87 silesia.tar zskp 2 211947520 67504318 969 208.38 -silesia.tar zskp 3 211947520 65177448 1899 106.44 -silesia.tar zskp 4 211947520 61381950 8115 24.91 +silesia.tar zskp 3 211947520 64595893 2007 100.68 +silesia.tar zskp 4 211947520 60995370 7691 26.28 cgo zstd: silesia.tar zstd 1 211947520 73605392 543 371.56 @@ -171,8 +171,8 @@ https://files.klauspost.com/compress/gob-stream.7z file out level insize outsize millis mb/s gob-stream zskp 1 1911399616 235022249 3088 590.30 gob-stream zskp 2 1911399616 205669791 3786 481.34 -gob-stream zskp 3 1911399616 185792019 9324 195.48 -gob-stream zskp 4 1911399616 171537212 32113 56.76 +gob-stream zskp 3 1911399616 175034659 9636 189.17 +gob-stream zskp 4 1911399616 167273881 29337 62.13 gob-stream zstd 1 1911399616 249810424 2637 691.26 gob-stream zstd 3 1911399616 208192146 3490 522.31 gob-stream zstd 6 1911399616 193632038 6687 272.56 @@ -187,8 +187,8 @@ http://mattmahoney.net/dc/textdata.html file out level insize outsize millis mb/s enwik9 zskp 1 1000000000 343848582 3609 264.18 enwik9 zskp 2 1000000000 317276632 5746 165.97 -enwik9 zskp 3 1000000000 294540704 11725 81.34 -enwik9 zskp 4 1000000000 276609671 44029 21.66 +enwik9 zskp 3 1000000000 292243069 12162 78.41 +enwik9 zskp 4 1000000000 275241169 36430 26.18 enwik9 zstd 1 1000000000 358072021 3110 306.65 enwik9 zstd 3 1000000000 313734672 4784 199.35 enwik9 zstd 6 1000000000 295138875 10290 92.68 @@ -202,8 +202,8 @@ https://files.klauspost.com/compress/github-june-2days-2019.json.zst file out level insize outsize millis mb/s github-june-2days-2019.json zskp 1 6273951764 699045015 10620 563.40 github-june-2days-2019.json zskp 2 6273951764 617881763 11687 511.96 -github-june-2days-2019.json zskp 3 6273951764 537511906 29252 204.54 -github-june-2days-2019.json zskp 4 6273951764 512796117 97791 61.18 +github-june-2days-2019.json zskp 3 6273951764 524340691 34043 175.75 +github-june-2days-2019.json zskp 4 6273951764 503314661 93811 63.78 github-june-2days-2019.json zstd 1 6273951764 766284037 8450 708.00 github-june-2days-2019.json zstd 3 6273951764 661889476 10927 547.57 github-june-2days-2019.json zstd 6 6273951764 642756859 22996 260.18 @@ -217,8 +217,8 @@ https://files.klauspost.com/compress/rawstudio-mint14.7z file out level insize outsize millis mb/s rawstudio-mint14.tar zskp 1 8558382592 3667489370 20210 403.84 rawstudio-mint14.tar zskp 2 8558382592 3364592300 31873 256.07 -rawstudio-mint14.tar zskp 3 8558382592 3224594213 71751 113.75 -rawstudio-mint14.tar zskp 4 8558382592 3027332295 486243 16.79 +rawstudio-mint14.tar zskp 3 8558382592 3158085214 77675 105.08 +rawstudio-mint14.tar zskp 4 8558382592 3020370044 404956 20.16 rawstudio-mint14.tar zstd 1 8558382592 3609250104 17136 476.27 rawstudio-mint14.tar zstd 3 8558382592 3341679997 29262 278.92 rawstudio-mint14.tar zstd 6 8558382592 3235846406 77904 104.77 @@ -232,8 +232,8 @@ https://files.klauspost.com/compress/nyc-taxi-data-10M.csv.zst file out level insize outsize millis mb/s nyc-taxi-data-10M.csv zskp 1 3325605752 641339945 8925 355.35 nyc-taxi-data-10M.csv zskp 2 3325605752 591748091 11268 281.44 -nyc-taxi-data-10M.csv zskp 3 3325605752 538490114 19880 159.53 -nyc-taxi-data-10M.csv zskp 4 3325605752 495986829 89368 35.49 +nyc-taxi-data-10M.csv zskp 3 3325605752 530289687 25239 125.66 +nyc-taxi-data-10M.csv zskp 4 3325605752 490907191 65939 48.10 nyc-taxi-data-10M.csv zstd 1 3325605752 687399637 8233 385.18 nyc-taxi-data-10M.csv zstd 3 3325605752 598514411 10065 315.07 nyc-taxi-data-10M.csv zstd 6 3325605752 570522953 20038 158.27 diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go index fe3625c5f..dc1eed5f0 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_best.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go @@ -220,6 +220,20 @@ encodeLoop: best = bestOf(best, matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)) best = bestOf(best, matchAt(candidateL2.offset-e.cur, s+1, uint32(cv2), -1)) best = bestOf(best, matchAt(candidateL2.prev-e.cur, s+1, uint32(cv2), -1)) + + // See if we can find a better match by checking where the current best ends. + // Use that offset to see if we can find a better full match. + if sAt := best.s + best.length; sAt < sLimit { + nextHashL := hash8(load6432(src, sAt), bestLongTableBits) + candidateEnd := e.longTable[nextHashL] + if pos := candidateEnd.offset - e.cur - best.length; pos >= 0 { + bestEnd := bestOf(best, matchAt(pos, best.s, load3232(src, best.s), -1)) + if pos := candidateEnd.prev - e.cur - best.length; pos >= 0 { + bestEnd = bestOf(bestEnd, matchAt(pos, best.s, load3232(src, best.s), -1)) + } + best = bestEnd + } + } } // We have a match, we can store the forward value diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index c2ce4a2ba..604954290 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -412,8 +412,41 @@ encodeLoop: cv = load6432(src, s) } - // A 4-byte match has been found. Update recent offsets. - // We'll later see if more than 4 bytes. + // Try to find a better match by searching for a long match at the end of the current best match + if true && s+matched < sLimit { + nextHashL := hash8(load6432(src, s+matched), betterLongTableBits) + cv := load3232(src, s) + candidateL := e.longTable[nextHashL] + coffsetL := candidateL.offset - e.cur - matched + if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { + // Found a long match, at least 4 bytes. + matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4 + if matchedNext > matched { + t = coffsetL + matched = matchedNext + if debugMatches { + println("long match at end-of-match") + } + } + } + + // Check prev long... + if true { + coffsetL = candidateL.prev - e.cur - matched + if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { + // Found a long match, at least 4 bytes. + matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4 + if matchedNext > matched { + t = coffsetL + matched = matchedNext + if debugMatches { + println("prev long match at end-of-match") + } + } + } + } + } + // A match has been found. Update recent offsets. offset2 = offset1 offset1 = s - t @@ -905,9 +938,41 @@ encodeLoop: } cv = load6432(src, s) } + // Try to find a better match by searching for a long match at the end of the current best match + if s+matched < sLimit { + nextHashL := hash8(load6432(src, s+matched), betterLongTableBits) + cv := load3232(src, s) + candidateL := e.longTable[nextHashL] + coffsetL := candidateL.offset - e.cur - matched + if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { + // Found a long match, at least 4 bytes. + matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4 + if matchedNext > matched { + t = coffsetL + matched = matchedNext + if debugMatches { + println("long match at end-of-match") + } + } + } - // A 4-byte match has been found. Update recent offsets. - // We'll later see if more than 4 bytes. + // Check prev long... + if true { + coffsetL = candidateL.prev - e.cur - matched + if coffsetL >= 0 && coffsetL < s && s-coffsetL < e.maxMatchOff && cv == load3232(src, coffsetL) { + // Found a long match, at least 4 bytes. + matchedNext := e.matchlen(s+4, coffsetL+4, src) + 4 + if matchedNext > matched { + t = coffsetL + matched = matchedNext + if debugMatches { + println("prev long match at end-of-match") + } + } + } + } + } + // A match has been found. Update recent offsets. offset2 = offset1 offset1 = s - t diff --git a/vendor/github.com/klauspost/compress/zstd/zip.go b/vendor/github.com/klauspost/compress/zstd/zip.go new file mode 100644 index 000000000..e35a0a2f8 --- /dev/null +++ b/vendor/github.com/klauspost/compress/zstd/zip.go @@ -0,0 +1,120 @@ +// Copyright 2019+ Klaus Post. All rights reserved. +// License information can be found in the LICENSE file. + +package zstd + +import ( + "errors" + "io" + "sync" +) + +// ZipMethodWinZip is the method for Zstandard compressed data inside Zip files for WinZip. +// See https://www.winzip.com/win/en/comp_info.html +const ZipMethodWinZip = 93 + +// ZipMethodPKWare is the method number used by PKWARE to indicate Zstandard compression. +// See https://pkware.cachefly.net/webdocs/APPNOTE/APPNOTE-6.3.7.TXT +const ZipMethodPKWare = 20 + +var zipReaderPool sync.Pool + +// newZipReader cannot be used since we would leak goroutines... +func newZipReader(r io.Reader) io.ReadCloser { + dec, ok := zipReaderPool.Get().(*Decoder) + if ok { + dec.Reset(r) + } else { + d, err := NewReader(r, WithDecoderConcurrency(1), WithDecoderLowmem(true)) + if err != nil { + panic(err) + } + dec = d + } + return &pooledZipReader{dec: dec} +} + +type pooledZipReader struct { + mu sync.Mutex // guards Close and Read + dec *Decoder +} + +func (r *pooledZipReader) Read(p []byte) (n int, err error) { + r.mu.Lock() + defer r.mu.Unlock() + if r.dec == nil { + return 0, errors.New("Read after Close") + } + dec, err := r.dec.Read(p) + + return dec, err +} + +func (r *pooledZipReader) Close() error { + r.mu.Lock() + defer r.mu.Unlock() + var err error + if r.dec != nil { + err = r.dec.Reset(nil) + zipReaderPool.Put(r.dec) + r.dec = nil + } + return err +} + +type pooledZipWriter struct { + mu sync.Mutex // guards Close and Read + enc *Encoder +} + +func (w *pooledZipWriter) Write(p []byte) (n int, err error) { + w.mu.Lock() + defer w.mu.Unlock() + if w.enc == nil { + return 0, errors.New("Write after Close") + } + return w.enc.Write(p) +} + +func (w *pooledZipWriter) Close() error { + w.mu.Lock() + defer w.mu.Unlock() + var err error + if w.enc != nil { + err = w.enc.Close() + zipReaderPool.Put(w.enc) + w.enc = nil + } + return err +} + +// ZipCompressor returns a compressor that can be registered with zip libraries. +// The provided encoder options will be used on all encodes. +func ZipCompressor(opts ...EOption) func(w io.Writer) (io.WriteCloser, error) { + var pool sync.Pool + return func(w io.Writer) (io.WriteCloser, error) { + enc, ok := pool.Get().(*Encoder) + if ok { + enc.Reset(w) + } else { + var err error + enc, err = NewWriter(w, opts...) + if err != nil { + return nil, err + } + } + return &pooledZipWriter{enc: enc}, nil + } +} + +// ZipDecompressor returns a decompressor that can be registered with zip libraries. +// See ZipCompressor for example. +func ZipDecompressor() func(r io.Reader) io.ReadCloser { + return func(r io.Reader) io.ReadCloser { + d, err := NewReader(r, WithDecoderConcurrency(1), WithDecoderLowmem(true)) + if err != nil { + panic(err) + } + return d.IOReadCloser() + } +} diff --git a/vendor/modules.txt b/vendor/modules.txt index b0658df5b..d59b6b731 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -191,7 +191,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.30.0 +# github.com/containers/storage v1.30.1 github.com/containers/storage github.com/containers/storage/drivers github.com/containers/storage/drivers/aufs @@ -384,12 +384,12 @@ github.com/imdario/mergo github.com/inconshreveable/mousetrap # github.com/ishidawataru/sctp v0.0.0-20191218070446-00ab2ac2db07 github.com/ishidawataru/sctp -# github.com/json-iterator/go v1.1.10 +# github.com/json-iterator/go v1.1.11 github.com/json-iterator/go # github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a github.com/juju/ansiterm github.com/juju/ansiterm/tabwriter -# github.com/klauspost/compress v1.12.1 +# github.com/klauspost/compress v1.12.2 github.com/klauspost/compress/flate github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 |