diff options
107 files changed, 2343 insertions, 470 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index b6ef69a96..ef6ac5e8b 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -378,8 +378,7 @@ osx_alt_build_task: always: *binary_artifacts -# This task is a stub: In the future it will be used to verify -# podman is compatible with the docker python-module. +# Verify podman is compatible with the docker python-module. docker-py_test_task: name: Docker-py Compat. alias: docker-py_test @@ -90,8 +90,7 @@ else ISODATE ?= $(shell date --iso-8601) endif LIBPOD := ${PROJECT}/v3/libpod -GCFLAGS ?= all=-trimpath=$(CURDIR) -ASMFLAGS ?= all=-trimpath=$(CURDIR) +GOFLAGS ?= -trimpath LDFLAGS_PODMAN ?= \ -X $(LIBPOD)/define.gitCommit=$(GIT_COMMIT) \ -X $(LIBPOD)/define.buildInfo=$(BUILD_INFO) \ @@ -258,7 +257,7 @@ test/goecho/goecho: .gopathok $(wildcard test/goecho/*.go) .PHONY: codespell codespell: - codespell -S bin,vendor,.git,go.sum,changelog.txt,.cirrus.yml,"RELEASE_NOTES.md,*.xz,*.gz,*.tar,*.tgz,bin2img,*ico,*.png,*.1,*.5,copyimg,*.orig,apidoc.go" -L uint,iff,od,seeked,splitted,marge,ERRO,hist,ether -w + codespell -S bin,vendor,.git,go.sum,changelog.txt,.cirrus.yml,"RELEASE_NOTES.md,*.xz,*.gz,*.ps1,*.tar,*.tgz,bin2img,*ico,*.png,*.1,*.5,copyimg,*.orig,apidoc.go" -L uint,iff,od,seeked,splitted,marge,ERRO,hist,ether -w .PHONY: validate validate: gofmt lint .gitvalidation validate.completions man-page-check swagger-check tests-included tests-expect-exit @@ -295,8 +294,6 @@ endif CGO_ENABLED=$(CGO_ENABLED) \ $(GO) build \ $(BUILDFLAGS) \ - -gcflags '$(GCFLAGS)' \ - -asmflags '$(ASMFLAGS)' \ -ldflags '$(LDFLAGS_PODMAN)' \ -tags "$(BUILDTAGS)" \ -o $@ ./cmd/podman @@ -310,8 +307,6 @@ $(SRCBINDIR)/podman$(BINSFX): $(SRCBINDIR) .gopathok $(SOURCES) go.mod go.sum GOOS=$(GOOS) \ $(GO) build \ $(BUILDFLAGS) \ - -gcflags '$(GCFLAGS)' \ - -asmflags '$(ASMFLAGS)' \ -ldflags '$(LDFLAGS_PODMAN)' \ -tags "${REMOTETAGS}" \ -o $@ ./cmd/podman @@ -321,8 +316,6 @@ $(SRCBINDIR)/podman-remote-static: $(SRCBINDIR) .gopathok $(SOURCES) go.mod go.s GOOS=$(GOOS) \ $(GO) build \ $(BUILDFLAGS) \ - -gcflags '$(GCFLAGS)' \ - -asmflags '$(ASMFLAGS)' \ -ldflags '$(LDFLAGS_PODMAN_STATIC)' \ -tags "${REMOTETAGS}" \ -o $@ ./cmd/podman @@ -376,8 +369,6 @@ bin/podman.cross.%: .gopathok CGO_ENABLED=0 \ $(GO) build \ $(BUILDFLAGS) \ - -gcflags '$(GCFLAGS)' \ - -asmflags '$(ASMFLAGS)' \ -ldflags '$(LDFLAGS_PODMAN)' \ -tags '$(BUILDTAGS_CROSS)' \ -o "$@" ./cmd/podman @@ -502,10 +493,12 @@ validate.completions: if [ -x /bin/zsh ]; then /bin/zsh completions/zsh/_podman; fi if [ -x /bin/fish ]; then /bin/fish completions/fish/podman.fish; fi +# Note: Assumes test/python/requirements.txt is installed & available .PHONY: run-docker-py-tests run-docker-py-tests: - $(eval testLogs=$(shell mktemp podman_tmp_XXXX)) - ./bin/podman run --rm --security-opt label=disable --privileged -v $(testLogs):/testLogs --net=host -e DOCKER_HOST=tcp://localhost:8080 $(DOCKERPY_IMAGE) sh -c "pytest $(DOCKERPY_TEST) " + touch test/__init__.py + pytest test/python/docker/ + -rm test/__init__.py .PHONY: localunit localunit: test/goecho/goecho @@ -850,11 +843,13 @@ clean: ## Clean all make artifacts build \ test/checkseccomp/checkseccomp \ test/goecho/goecho \ + test/__init__.py \ test/testdata/redis-image \ libpod/container_ffjson.go \ libpod/pod_ffjson.go \ libpod/container_easyjson.go \ libpod/pod_easyjson.go \ .install.goimports \ - docs/build + docs/build \ + venv make -C docs clean diff --git a/cmd/podman/common/completion.go b/cmd/podman/common/completion.go index 08b2f6235..9a4524b46 100644 --- a/cmd/podman/common/completion.go +++ b/cmd/podman/common/completion.go @@ -323,6 +323,18 @@ func prefixSlice(pre string, slice []string) []string { return slice } +func suffixCompSlice(suf string, slice []string) []string { + for i := range slice { + split := strings.SplitN(slice[i], "\t", 2) + if len(split) > 1 { + slice[i] = split[0] + suf + "\t" + split[1] + } else { + slice[i] = slice[i] + suf + } + } + return slice +} + func completeKeyValues(toComplete string, k keyValueCompletion) ([]string, cobra.ShellCompDirective) { suggestions := make([]string, 0, len(k)) directive := cobra.ShellCompDirectiveNoFileComp @@ -664,6 +676,42 @@ func AutocompleteSystemConnections(cmd *cobra.Command, args []string, toComplete return suggestions, cobra.ShellCompDirectiveNoFileComp } +// AutocompleteScp returns a list of connections, images, or both, depending on the amount of arguments +func AutocompleteScp(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + if !validCurrentCmdLine(cmd, args, toComplete) { + return nil, cobra.ShellCompDirectiveNoFileComp + } + switch len(args) { + case 0: + split := strings.SplitN(toComplete, "::", 2) + if len(split) > 1 { + imageSuggestions, _ := getImages(cmd, split[1]) + return prefixSlice(split[0]+"::", imageSuggestions), cobra.ShellCompDirectiveNoFileComp + } + connectionSuggestions, _ := AutocompleteSystemConnections(cmd, args, toComplete) + imageSuggestions, _ := getImages(cmd, toComplete) + totalSuggestions := append(suffixCompSlice("::", connectionSuggestions), imageSuggestions...) + directive := cobra.ShellCompDirectiveNoFileComp + // if we have connections do not add a space after the completion + if len(connectionSuggestions) > 0 { + directive = cobra.ShellCompDirectiveNoFileComp | cobra.ShellCompDirectiveNoSpace + } + return totalSuggestions, directive + case 1: + split := strings.SplitN(args[0], "::", 2) + if len(split) > 1 { + if len(split[1]) > 0 { + return nil, cobra.ShellCompDirectiveNoFileComp + } + imageSuggestions, _ := getImages(cmd, toComplete) + return imageSuggestions, cobra.ShellCompDirectiveNoFileComp + } + connectionSuggestions, _ := AutocompleteSystemConnections(cmd, args, toComplete) + return suffixCompSlice("::", connectionSuggestions), cobra.ShellCompDirectiveNoFileComp + } + return nil, cobra.ShellCompDirectiveNoFileComp +} + /* -------------- Flags ----------------- */ // AutocompleteDetachKeys - Autocomplete detach-keys options. diff --git a/cmd/podman/common/create_opts.go b/cmd/podman/common/create_opts.go index 0a969bfd2..0fdf3ce08 100644 --- a/cmd/podman/common/create_opts.go +++ b/cmd/podman/common/create_opts.go @@ -356,51 +356,55 @@ func ContainerCreateToContainerCLIOpts(cc handlers.CreateContainerConfig, rtc *c CPUSetMems: cc.HostConfig.CpusetMems, // Detach: false, // don't need // DetachKeys: "", // don't need - Devices: devices, - DeviceCGroupRule: nil, - DeviceReadBPs: readBps, - DeviceReadIOPs: readIops, - DeviceWriteBPs: writeBps, - DeviceWriteIOPs: writeIops, - Entrypoint: entrypoint, - Env: cc.Config.Env, - Expose: expose, - GroupAdd: cc.HostConfig.GroupAdd, - Hostname: cc.Config.Hostname, - ImageVolume: "bind", - Init: init, - Interactive: cc.Config.OpenStdin, - IPC: string(cc.HostConfig.IpcMode), - Label: stringMaptoArray(cc.Config.Labels), - LogDriver: cc.HostConfig.LogConfig.Type, - LogOptions: stringMaptoArray(cc.HostConfig.LogConfig.Config), - Name: cc.Name, - OOMScoreAdj: cc.HostConfig.OomScoreAdj, - Arch: "", - OS: "", - Variant: "", - PID: string(cc.HostConfig.PidMode), - PIDsLimit: cc.HostConfig.PidsLimit, - Privileged: cc.HostConfig.Privileged, - PublishAll: cc.HostConfig.PublishAllPorts, - Quiet: false, - ReadOnly: cc.HostConfig.ReadonlyRootfs, - ReadOnlyTmpFS: true, // podman default - Rm: cc.HostConfig.AutoRemove, - SecurityOpt: cc.HostConfig.SecurityOpt, - StopSignal: cc.Config.StopSignal, - StorageOpt: stringMaptoArray(cc.HostConfig.StorageOpt), - Sysctl: stringMaptoArray(cc.HostConfig.Sysctls), - Systemd: "true", // podman default - TmpFS: parsedTmp, - TTY: cc.Config.Tty, - User: cc.Config.User, - UserNS: string(cc.HostConfig.UsernsMode), - UTS: string(cc.HostConfig.UTSMode), - Mount: mounts, - VolumesFrom: cc.HostConfig.VolumesFrom, - Workdir: cc.Config.WorkingDir, - Net: &netInfo, + Devices: devices, + DeviceCGroupRule: nil, + DeviceReadBPs: readBps, + DeviceReadIOPs: readIops, + DeviceWriteBPs: writeBps, + DeviceWriteIOPs: writeIops, + Entrypoint: entrypoint, + Env: cc.Config.Env, + Expose: expose, + GroupAdd: cc.HostConfig.GroupAdd, + Hostname: cc.Config.Hostname, + ImageVolume: "bind", + Init: init, + Interactive: cc.Config.OpenStdin, + IPC: string(cc.HostConfig.IpcMode), + Label: stringMaptoArray(cc.Config.Labels), + LogDriver: cc.HostConfig.LogConfig.Type, + LogOptions: stringMaptoArray(cc.HostConfig.LogConfig.Config), + Name: cc.Name, + OOMScoreAdj: cc.HostConfig.OomScoreAdj, + Arch: "", + OS: "", + Variant: "", + PID: string(cc.HostConfig.PidMode), + PIDsLimit: cc.HostConfig.PidsLimit, + Privileged: cc.HostConfig.Privileged, + PublishAll: cc.HostConfig.PublishAllPorts, + Quiet: false, + ReadOnly: cc.HostConfig.ReadonlyRootfs, + ReadOnlyTmpFS: true, // podman default + Rm: cc.HostConfig.AutoRemove, + SecurityOpt: cc.HostConfig.SecurityOpt, + StopSignal: cc.Config.StopSignal, + StorageOpt: stringMaptoArray(cc.HostConfig.StorageOpt), + Sysctl: stringMaptoArray(cc.HostConfig.Sysctls), + Systemd: "true", // podman default + TmpFS: parsedTmp, + TTY: cc.Config.Tty, + User: cc.Config.User, + UserNS: string(cc.HostConfig.UsernsMode), + UTS: string(cc.HostConfig.UTSMode), + Mount: mounts, + VolumesFrom: cc.HostConfig.VolumesFrom, + Workdir: cc.Config.WorkingDir, + Net: &netInfo, + HealthInterval: DefaultHealthCheckInterval, + HealthRetries: DefaultHealthCheckRetries, + HealthTimeout: DefaultHealthCheckTimeout, + HealthStartPeriod: DefaultHealthCheckStartPeriod, } if !rootless.IsRootless() { var ulimits []string @@ -527,10 +531,18 @@ func ContainerCreateToContainerCLIOpts(cc handlers.CreateContainerConfig, rtc *c finCmd = finCmd[:len(finCmd)-1] } cliOpts.HealthCmd = finCmd - cliOpts.HealthInterval = cc.Config.Healthcheck.Interval.String() - cliOpts.HealthRetries = uint(cc.Config.Healthcheck.Retries) - cliOpts.HealthStartPeriod = cc.Config.Healthcheck.StartPeriod.String() - cliOpts.HealthTimeout = cc.Config.Healthcheck.Timeout.String() + if cc.Config.Healthcheck.Interval > 0 { + cliOpts.HealthInterval = cc.Config.Healthcheck.Interval.String() + } + if cc.Config.Healthcheck.Retries > 0 { + cliOpts.HealthRetries = uint(cc.Config.Healthcheck.Retries) + } + if cc.Config.Healthcheck.StartPeriod > 0 { + cliOpts.HealthStartPeriod = cc.Config.Healthcheck.StartPeriod.String() + } + if cc.Config.Healthcheck.Timeout > 0 { + cliOpts.HealthTimeout = cc.Config.Healthcheck.Timeout.String() + } } // specgen assumes the image name is arg[0] diff --git a/cmd/podman/common/specgen.go b/cmd/podman/common/specgen.go index 8d6a21cb7..59d32f568 100644 --- a/cmd/podman/common/specgen.go +++ b/cmd/podman/common/specgen.go @@ -685,7 +685,7 @@ func makeHealthCheckFromCli(inCmd, interval string, retries uint, timeout, start concat := "" if cmdArr[0] == "CMD" || cmdArr[0] == "none" { // this is for compat, we are already split properly for most compat cases cmdArr = strings.Fields(inCmd) - } else if cmdArr[0] != "CMD-SHELL" { // this is for podman side of things, wont contain the keywords + } else if cmdArr[0] != "CMD-SHELL" { // this is for podman side of things, won't contain the keywords if isArr && len(cmdArr) > 1 { // an array of consecutive commands cmdArr = append([]string{"CMD"}, cmdArr...) } else { // one singular command diff --git a/cmd/podman/containers/create.go b/cmd/podman/containers/create.go index 895736144..a57488af2 100644 --- a/cmd/podman/containers/create.go +++ b/cmd/podman/containers/create.go @@ -13,6 +13,7 @@ import ( "github.com/containers/podman/v3/cmd/podman/common" "github.com/containers/podman/v3/cmd/podman/registry" "github.com/containers/podman/v3/cmd/podman/utils" + "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/specgen" "github.com/containers/podman/v3/pkg/util" @@ -105,8 +106,8 @@ func create(cmd *cobra.Command, args []string) error { if !cmd.Flags().Changed("pod") { return errors.New("must specify pod value with init-ctr") } - if !util.StringInSlice(initctr, []string{"always", "oneshot"}) { - return errors.New("init-ctr value must be 'always' or 'oneshot'") + if !util.StringInSlice(initctr, []string{define.AlwaysInitContainer, define.OneShotInitContainer}) { + return errors.Errorf("init-ctr value must be '%s' or '%s'", define.AlwaysInitContainer, define.OneShotInitContainer) } cliVals.InitContainerType = initctr } @@ -184,6 +185,9 @@ func createInit(c *cobra.Command) error { if c.Flag("cpu-quota").Changed && c.Flag("cpus").Changed { return errors.Errorf("--cpu-quota and --cpus cannot be set together") } + if c.Flag("pod").Changed && !strings.HasPrefix(c.Flag("pod").Value.String(), "new:") && c.Flag("userns").Changed { + return errors.Errorf("--userns and --pod cannot be set together") + } noHosts, err := c.Flags().GetBool("no-hosts") if err != nil { @@ -309,6 +313,12 @@ func createPodIfNecessary(s *specgen.SpecGenerator, netOpts *entities.NetOptions if len(podName) < 1 { return nil, errors.Errorf("new pod name must be at least one character") } + + userns, err := specgen.ParseUserNamespace(cliVals.UserNS) + if err != nil { + return nil, err + } + createOptions := entities.PodCreateOptions{ Name: podName, Infra: true, @@ -318,6 +328,7 @@ func createPodIfNecessary(s *specgen.SpecGenerator, netOpts *entities.NetOptions Cpus: cliVals.CPUS, CpusetCpus: cliVals.CPUSetCPUs, Pid: cliVals.PID, + Userns: userns, } // Unset config values we passed to the pod to prevent them being used twice for the container and pod. s.ContainerBasicConfig.Hostname = "" diff --git a/cmd/podman/images/build.go b/cmd/podman/images/build.go index 3aeba6fb0..a1a28b809 100644 --- a/cmd/podman/images/build.go +++ b/cmd/podman/images/build.go @@ -67,6 +67,18 @@ var ( podman image build --layers --force-rm --tag imageName .`, } + buildxBuildCmd = &cobra.Command{ + Args: buildCmd.Args, + Use: buildCmd.Use, + Short: buildCmd.Short, + Long: buildCmd.Long, + RunE: buildCmd.RunE, + ValidArgsFunction: buildCmd.ValidArgsFunction, + Example: `podman buildx build . + podman buildx build --creds=username:password -t imageName -f Containerfile.simple . + podman buildx build --layers --force-rm --tag imageName .`, + } + buildOpts = buildFlagsWrapper{} ) @@ -91,11 +103,24 @@ func init() { Parent: imageCmd, }) buildFlags(imageBuildCmd) + registry.Commands = append(registry.Commands, registry.CliCommand{ + Command: buildxBuildCmd, + Parent: buildxCmd, + }) + buildFlags(buildxBuildCmd) } func buildFlags(cmd *cobra.Command) { flags := cmd.Flags() + // buildx build --load ignored, but added for compliance + flags.Bool("load", false, "buildx --load") + _ = flags.MarkHidden("load") + + // buildx build --progress ignored, but added for compliance + flags.String("progress", "auto", "buildx --progress") + _ = flags.MarkHidden("progress") + // Podman flags flags.BoolVarP(&buildOpts.SquashAll, "squash-all", "", false, "Squash all layers into a single layer") diff --git a/cmd/podman/images/buildx.go b/cmd/podman/images/buildx.go new file mode 100644 index 000000000..5c8e5aaa0 --- /dev/null +++ b/cmd/podman/images/buildx.go @@ -0,0 +1,29 @@ +package images + +import ( + "github.com/containers/podman/v3/cmd/podman/registry" + "github.com/containers/podman/v3/cmd/podman/validate" + "github.com/spf13/cobra" +) + +var ( + // Command: podman _buildx_ + // This is a hidden command, which was added to make converting + // from Docker to Podman easier. + // For now podman buildx build just calls into podman build + // If we are adding new buildx features, we will add them by default + // to podman build. + buildxCmd = &cobra.Command{ + Use: "buildx", + Short: "Build images", + Long: "Build images", + RunE: validate.SubCommandExists, + Hidden: true, + } +) + +func init() { + registry.Commands = append(registry.Commands, registry.CliCommand{ + Command: buildxCmd, + }) +} diff --git a/cmd/podman/images/scp.go b/cmd/podman/images/scp.go index a47d01995..176563440 100644 --- a/cmd/podman/images/scp.go +++ b/cmd/podman/images/scp.go @@ -33,7 +33,7 @@ var ( Short: "securely copy images", RunE: scp, Args: cobra.RangeArgs(1, 2), - ValidArgsFunction: common.AutocompleteImages, + ValidArgsFunction: common.AutocompleteScp, Example: `podman image scp myimage:latest otherhost::`, } ) diff --git a/cmd/podman/pods/create.go b/cmd/podman/pods/create.go index abc47164b..bf5b9e350 100644 --- a/cmd/podman/pods/create.go +++ b/cmd/podman/pods/create.go @@ -48,6 +48,7 @@ var ( podIDFile string replace bool share string + userns string ) func init() { @@ -72,6 +73,10 @@ func init() { flags.StringVar(&createOptions.CGroupParent, cgroupParentflagName, "", "Set parent cgroup for the pod") _ = createCommand.RegisterFlagCompletionFunc(cgroupParentflagName, completion.AutocompleteDefault) + usernsFlagName := "userns" + flags.StringVar(&userns, usernsFlagName, os.Getenv("PODMAN_USERNS"), "User namespace to use") + _ = createCommand.RegisterFlagCompletionFunc(usernsFlagName, common.AutocompleteUserNamespace) + flags.BoolVar(&createOptions.Infra, "infra", true, "Create an infra container associated with the pod to share namespaces with") infraConmonPidfileFlagName := "infra-conmon-pidfile" @@ -178,6 +183,11 @@ func create(cmd *cobra.Command, args []string) error { } } + createOptions.Userns, err = specgen.ParseUserNamespace(userns) + if err != nil { + return err + } + if cmd.Flag("pod-id-file").Changed { podIDFD, err = util.OpenExclusiveFile(podIDFile) if err != nil && os.IsExist(err) { diff --git a/cmd/podman/registry/config.go b/cmd/podman/registry/config.go index b512ba341..50e488b02 100644 --- a/cmd/podman/registry/config.go +++ b/cmd/podman/registry/config.go @@ -89,7 +89,12 @@ func newPodmanConfig() { // use for the containers.conf configuration file. func setXdgDirs() error { if !rootless.IsRootless() { - return nil + // unset XDG_RUNTIME_DIR for root + // Sometimes XDG_RUNTIME_DIR is set to /run/user/0 sometimes it is unset, + // the inconsistency is causing issues for the dnsname plugin. + // It is already set to an empty string for conmon so lets do the same + // for podman. see #10806 and #10745 + return os.Unsetenv("XDG_RUNTIME_DIR") } // Setup XDG_RUNTIME_DIR diff --git a/cmd/podman/shell_completion_test.go b/cmd/podman/shell_completion_test.go index 9bd821d8d..792beeb19 100644 --- a/cmd/podman/shell_completion_test.go +++ b/cmd/podman/shell_completion_test.go @@ -33,7 +33,9 @@ func TestShellCompletionFunctions(t *testing.T) { func checkCommand(t *testing.T, cmd *cobra.Command) { if cmd.HasSubCommands() { for _, childCmd := range cmd.Commands() { - checkCommand(t, childCmd) + if !childCmd.Hidden { + checkCommand(t, childCmd) + } } // if not check if completion for that command is provided diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh index da43ffb0a..c1972b90f 100755 --- a/contrib/cirrus/runner.sh +++ b/contrib/cirrus/runner.sh @@ -88,7 +88,8 @@ function _run_bindings() { } function _run_docker-py() { - msg "This is docker-py stub, it is only a stub" + source venv/bin/activate + make run-docker-py-tests } function _run_endpoint() { diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 86b8d78ef..3f2176fd6 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -191,7 +191,19 @@ case "$TEST_FLAVOR" in bigto dnf install -y glibc-minimal-langpack rpm-build fi ;& - docker-py) ;& + docker-py) + remove_packaged_podman_files + make install PREFIX=/usr ETCDIR=/etc + + # TODO: Don't install stuff at test runtime! Do this from + # cache_images/fedora_packaging.sh in containers/automation_images + # and STRONGLY prefer installing RPMs vs pip packages in venv + dnf install -y python3-virtualenv python3-pytest4 + virtualenv venv + source venv/bin/activate + pip install --upgrade pip + pip install --requirement $GOSRC/test/python/requirements.txt + ;; build) make clean ;; unit) ;; apiv2) ;& # use next item diff --git a/contrib/podmanimage/README.md b/contrib/podmanimage/README.md index 6effec38b..b7be328c7 100644 --- a/contrib/podmanimage/README.md +++ b/contrib/podmanimage/README.md @@ -17,10 +17,10 @@ default to `/`. The container images are: * `quay.io/containers/podman:<version>` and `quay.io/podman/stable:<version>` - - These images are built when a new Podman version becomes available in - Fedora. These images are intended to be unchanging and stable, they will - never be updated by automation once they've been pushed. For build details, - please [see the configuration file](stable/Dockerfile). + These images are built daily. They are intended to contain an unchanging + and stable version of podman. Though for the most recent `<version>` tag, + image contents will be updated to incorporate (especially) security upgrades. + For build details, please [see the configuration file](stable/Dockerfile). * `quay.io/containers/podman:latest` and `quay.io/podman/stable:latest` - Built daily using the same Dockerfile as above. The Podman version will remain the "latest" available in Fedora, however the other image diff --git a/docs/source/markdown/podman-build.1.md b/docs/source/markdown/podman-build.1.md index c65fbccb9..a2ed35f89 100644 --- a/docs/source/markdown/podman-build.1.md +++ b/docs/source/markdown/podman-build.1.md @@ -43,6 +43,8 @@ containers can be left in container storage. Use the `podman ps --all --storage` command to see these containers. External containers can be removed with the `podman rm --storage` command. +`podman buildx build` command is an alias of `podman build`. Not all `buildx build` features are available in Podman. The `buildx build` option is provided for scripting compatibility. + ## OPTIONS #### **--add-host**=*host* diff --git a/docs/source/markdown/podman-create.1.md b/docs/source/markdown/podman-create.1.md index 51f51c10a..b73f6c05a 100644 --- a/docs/source/markdown/podman-create.1.md +++ b/docs/source/markdown/podman-create.1.md @@ -453,10 +453,9 @@ When using pods, create an init style container, which is run after the infra co but before regular pod containers are started. Init containers are useful for running setup operations for the pod's applications. -Valid values for `init-ctr` type are *always* or *oneshot*. The *always* value -means the container will run with each and every `pod start`, whereas the *oneshot* -value means is will ony run once when the pod is started and then the container is -removed. +Valid values for `init-ctr` type are *always* or *once*. The *always* value +means the container will run with each and every `pod start`, whereas the *once* +value means the container will only run once when the pod is started and then the container is removed. Init containers are only run on pod `start`. Restarting a pod will not execute any init containers should they be present. Furthermore, init containers can only be created in a @@ -661,13 +660,13 @@ Set the network mode for the container. Invalid if using **--dns**, **--dns-opt* Valid _mode_ values are: -- **bridge**: create a network stack on the default bridge; -- **none**: no networking; -- **container:**_id_: reuse another container's network stack; -- **host**: use the Podman host network stack. Note: the host mode gives the container full access to local system services such as D-bus and is therefore considered insecure; -- _network-id_: connect to a user-defined network, multiple networks should be comma-separated; -- **ns:**_path_: path to a network namespace to join; -- **private**: create a new namespace for the container (default) +- **bridge**: Create a network stack on the default bridge. This is the default for rootfull containers. +- **none**: Create a network namespace for the container but do not configure network interfaces for it, thus the container has no network connectivity. +- **container:**_id_: Reuse another container's network stack. +- **host**: Do not create a network namespace, the container will use the host's network. Note: The host mode gives the container full access to local system services such as D-bus and is therefore considered insecure. +- **network**: Connect to a user-defined network, multiple networks should be comma-separated. +- **ns:**_path_: Path to a network namespace to join. +- **private**: Create a new namespace for the container. This will use the **bridge** mode for rootfull containers and **slirp4netns** for rootless ones. - **slirp4netns[:OPTIONS,...]**: use **slirp4netns**(1) to create a user network stack. This is the default for rootless containers. It is possible to specify these additional options: - **allow_host_loopback=true|false**: Allow the slirp4netns to reach the host loopback IP (`10.0.2.2`, which is added to `/etc/hosts` as `host.containers.internal` for your convenience). Default is false. - **mtu=MTU**: Specify the MTU to use for this network. (Default is `65520`). @@ -678,7 +677,8 @@ Valid _mode_ values are: - **outbound_addr6=INTERFACE**: Specify the outbound interface slirp should bind to (ipv6 traffic only). - **outbound_addr6=IPv6**: Specify the outbound ipv6 address slirp should bind to. - **port_handler=rootlesskit**: Use rootlesskit for port forwarding. Default. - - **port_handler=slirp4netns**: Use the slirp4netns port forwarding. + Note: Rootlesskit changes the source IP address of incoming packets to a IP address in the container network namespace, usually `10.0.2.100`. If your application requires the real source IP address, e.g. web server logs, use the slirp4netns port handler. The rootlesskit port handler is also used for rootless containers when connected to user-defined networks. + - **port_handler=slirp4netns**: Use the slirp4netns port forwarding, it is slower than rootlesskit but preserves the correct source IP address. This port handler cannot be used for user-defined networks. #### **--network-alias**=*alias* @@ -1123,9 +1123,9 @@ Podman allocates unique ranges of UIDs and GIDs from the `containers` subpordina Valid `auto`options: - - *gidmapping*=_HOST_GID:CONTAINER_GID:SIZE_: to force a GID mapping to be present in the user namespace. + - *gidmapping*=_CONTAINER_GID:HOST_GID:SIZE_: to force a GID mapping to be present in the user namespace. - *size*=_SIZE_: to specify an explicit size for the automatic user namespace. e.g. `--userns=auto:size=8192`. If `size` is not specified, `auto` will estimate a size for the user namespace. - - *uidmapping*=_HOST_UID:CONTAINER_UID:SIZE_: to force a UID mapping to be present in the user namespace. + - *uidmapping*=_CONTAINER_UID:HOST_UID:SIZE_: to force a UID mapping to be present in the user namespace. **container:**_id_: join the user namespace of the specified container. diff --git a/docs/source/markdown/podman-pod-create.1.md b/docs/source/markdown/podman-pod-create.1.md index 441995cb6..4e822dca5 100644 --- a/docs/source/markdown/podman-pod-create.1.md +++ b/docs/source/markdown/podman-pod-create.1.md @@ -101,12 +101,15 @@ Assign a name to the pod. #### **--network**=*mode*, **--net** -Set network mode for the pod. Supported values are +Set network mode for the pod. Supported values are: - **bridge**: Create a network stack on the default bridge. This is the default for rootfull containers. +- **none**: Create a network namespace for the container but do not configure network interfaces for it, thus the container has no network connectivity. - **host**: Do not create a network namespace, all containers in the pod will use the host's network. Note: the host mode gives the container full access to local system services such as D-bus and is therefore considered insecure. -- Comma-separated list of the names of CNI networks the pod should join. -- **slirp4netns[:OPTIONS,...]**: use slirp4netns to create a user network stack. This is the default for rootless containers. It is possible to specify these additional options: - - **allow_host_loopback=true|false**: Allow the slirp4netns to reach the host loopback IP (`10.0.2.2`). Default is false. +- **network**: Connect to a user-defined network, multiple networks should be comma-separated. +- **private**: Create a new namespace for the container. This will use the **bridge** mode for rootfull containers and **slirp4netns** for rootless ones. +- **slirp4netns[:OPTIONS,...]**: use **slirp4netns**(1) to create a user network stack. This is the default for rootless containers. It is possible to specify these additional options: + - **allow_host_loopback=true|false**: Allow the slirp4netns to reach the host loopback IP (`10.0.2.2`, which is added to `/etc/hosts` as `host.containers.internal` for your convenience). Default is false. + - **mtu=MTU**: Specify the MTU to use for this network. (Default is `65520`). - **cidr=CIDR**: Specify ip range to use for this network. (Default is `10.0.2.0/24`). - **enable_ipv6=true|false**: Enable IPv6. Default is false. (Required for `outbound_addr6`). - **outbound_addr=INTERFACE**: Specify the outbound interface slirp should bind to (ipv4 traffic only). @@ -114,7 +117,8 @@ Set network mode for the pod. Supported values are - **outbound_addr6=INTERFACE**: Specify the outbound interface slirp should bind to (ipv6 traffic only). - **outbound_addr6=IPv6**: Specify the outbound ipv6 address slirp should bind to. - **port_handler=rootlesskit**: Use rootlesskit for port forwarding. Default. - - **port_handler=slirp4netns**: Use the slirp4netns port forwarding. + Note: Rootlesskit changes the source IP address of incoming packets to a IP address in the container network namespace, usually `10.0.2.100`. If your application requires the real source IP address, e.g. web server logs, use the slirp4netns port handler. The rootlesskit port handler is also used for rootless containers when connected to user-defined networks. + - **port_handler=slirp4netns**: Use the slirp4netns port forwarding, it is slower than rootlesskit but preserves the correct source IP address. This port handler cannot be used for user-defined networks. #### **--network-alias**=strings @@ -164,6 +168,19 @@ podman generates a UUID for each pod, and if a name is not assigned to the container with **--name** then a random string name will be generated for it. The name is useful any place you need to identify a pod. +#### **--userns**=*mode* + +Set the user namespace mode for all the containers in a pod. It defaults to the **PODMAN_USERNS** environment variable. An empty value ("") means user namespaces are disabled. + +Valid _mode_ values are: + +- *auto[:*_OPTIONS,..._*]*: automatically create a namespace. It is possible to specify these options to `auto`: + - *gidmapping=*_CONTAINER_GID:HOST_GID:SIZE_ to force a GID mapping to be present in the user namespace. + - *size=*_SIZE_: to specify an explicit size for the automatic user namespace. e.g. `--userns=auto:size=8192`. If `size` is not specified, `auto` will estimate a size for the user namespace. + - *uidmapping=*_CONTAINER_UID:HOST_UID:SIZE_ to force a UID mapping to be present in the user namespace. +- *host*: run in the user namespace of the caller. The processes running in the container will have the same privileges on the host as any other process launched by the calling user (default). +- *keep-id*: creates a user namespace where the current rootless user's UID:GID are mapped to the same values in the container. This option is ignored for containers created by the root user. + ## EXAMPLES ``` diff --git a/docs/source/markdown/podman-pod-ps.1.md b/docs/source/markdown/podman-pod-ps.1.md index 156adccaa..ed0789e93 100644 --- a/docs/source/markdown/podman-pod-ps.1.md +++ b/docs/source/markdown/podman-pod-ps.1.md @@ -98,6 +98,7 @@ Valid filters are listed below: | id | [ID] Pod's ID (accepts regex) | | name | [Name] Pod's name (accepts regex) | | label | [Key] or [Key=Value] Label assigned to a container | +| until | Only list pods created before given timestamp | | status | Pod's status: `stopped`, `running`, `paused`, `exited`, `dead`, `created`, `degraded` | | network | [Network] name or full ID of network | | ctr-names | Container name within the pod (accepts regex) | diff --git a/docs/source/markdown/podman-run.1.md b/docs/source/markdown/podman-run.1.md index 38ed44582..afee64775 100644 --- a/docs/source/markdown/podman-run.1.md +++ b/docs/source/markdown/podman-run.1.md @@ -680,13 +680,13 @@ Set the network mode for the container. Invalid if using **--dns**, **--dns-opt* Valid _mode_ values are: -- **bridge**: create a network stack on the default bridge; -- **none**: no networking; -- **container:**_id_: reuse another container's network stack; -- **host**: use the Podman host network stack. Note: the host mode gives the container full access to local system services such as D-bus and is therefore considered insecure; -- _network-id_: connect to a user-defined network, multiple networks should be comma-separated; -- **ns:**_path_: path to a network namespace to join; -- **private**: create a new namespace for the container (default) +- **bridge**: Create a network stack on the default bridge. This is the default for rootfull containers. +- **none**: Create a network namespace for the container but do not configure network interfaces for it, thus the container has no network connectivity. +- **container:**_id_: Reuse another container's network stack. +- **host**: Do not create a network namespace, the container will use the host's network. Note: The host mode gives the container full access to local system services such as D-bus and is therefore considered insecure. +- **network**: Connect to a user-defined network, multiple networks should be comma-separated. +- **ns:**_path_: Path to a network namespace to join. +- **private**: Create a new namespace for the container. This will use the **bridge** mode for rootfull containers and **slirp4netns** for rootless ones. - **slirp4netns[:OPTIONS,...]**: use **slirp4netns**(1) to create a user network stack. This is the default for rootless containers. It is possible to specify these additional options: - **allow_host_loopback=true|false**: Allow the slirp4netns to reach the host loopback IP (`10.0.2.2`, which is added to `/etc/hosts` as `host.containers.internal` for your convenience). Default is false. - **mtu=MTU**: Specify the MTU to use for this network. (Default is `65520`). @@ -697,7 +697,8 @@ Valid _mode_ values are: - **outbound_addr6=INTERFACE**: Specify the outbound interface slirp should bind to (ipv6 traffic only). - **outbound_addr6=IPv6**: Specify the outbound ipv6 address slirp should bind to. - **port_handler=rootlesskit**: Use rootlesskit for port forwarding. Default. - - **port_handler=slirp4netns**: Use the slirp4netns port forwarding. + Note: Rootlesskit changes the source IP address of incoming packets to a IP address in the container network namespace, usually `10.0.2.100`. If your application requires the real source IP address, e.g. web server logs, use the slirp4netns port handler. The rootlesskit port handler is also used for rootless containers when connected to user-defined networks. + - **port_handler=slirp4netns**: Use the slirp4netns port forwarding, it is slower than rootlesskit but preserves the correct source IP address. This port handler cannot be used for user-defined networks. #### **--network-alias**=*alias* @@ -1181,9 +1182,9 @@ Podman allocates unique ranges of UIDs and GIDs from the `containers` subpordina Valid `auto`options: - - *gidmapping*=_HOST_GID:CONTAINER_GID:SIZE_: to force a GID mapping to be present in the user namespace. + - *gidmapping*=_CONTAINER_GID:HOST_GID:SIZE_: to force a GID mapping to be present in the user namespace. - *size*=_SIZE_: to specify an explicit size for the automatic user namespace. e.g. `--userns=auto:size=8192`. If `size` is not specified, `auto` will estimate a size for the user namespace. - - *uidmapping*=_HOST_UID:CONTAINER_UID:SIZE_: to force a UID mapping to be present in the user namespace. + - *uidmapping*=_CONTAINER_UID:HOST_UID:SIZE_: to force a UID mapping to be present in the user namespace. **container:**_id_: join the user namespace of the specified container. @@ -17,7 +17,7 @@ require ( github.com/containers/image/v5 v5.15.0 github.com/containers/ocicrypt v1.1.2 github.com/containers/psgo v1.5.2 - github.com/containers/storage v1.33.1 + github.com/containers/storage v1.34.0 github.com/coreos/go-systemd/v22 v22.3.2 github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3 github.com/cri-o/ocicni v0.2.1-0.20210621164014-d0acc7862283 @@ -50,7 +50,7 @@ require ( github.com/opencontainers/runc v1.0.1 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 github.com/opencontainers/runtime-tools v0.9.0 - github.com/opencontainers/selinux v1.8.3 + github.com/opencontainers/selinux v1.8.4 github.com/pkg/errors v0.9.1 github.com/pmezard/go-difflib v1.0.0 github.com/rootless-containers/rootlesskit v0.14.4 @@ -259,8 +259,9 @@ github.com/containers/psgo v1.5.2/go.mod h1:2ubh0SsreMZjSXW1Hif58JrEcFudQyIy9EzP github.com/containers/storage v1.23.5/go.mod h1:ha26Q6ngehFNhf3AWoXldvAvwI4jFe3ETQAf/CeZPyM= github.com/containers/storage v1.32.6/go.mod h1:mdB+b89p+jU8zpzLTVXA0gWMmIo0WrkfGMh1R8O2IQw= github.com/containers/storage v1.33.0/go.mod h1:FUZPF4nJijX8ixdhByZJXf02cvbyLi6dyDwXdIe8QVY= -github.com/containers/storage v1.33.1 h1:RHUPZ7vQxwoeOoMoKUDsVun4f9Wi8BTXmr/wQiruBYU= github.com/containers/storage v1.33.1/go.mod h1:FUZPF4nJijX8ixdhByZJXf02cvbyLi6dyDwXdIe8QVY= +github.com/containers/storage v1.34.0 h1:39MhQe+3knl2G6WcaYf24Fpqqz6gbdLK/52Ms5wV+II= +github.com/containers/storage v1.34.0/go.mod h1:t6I+hTgPU0/tVxQ75vw406wDi/TXwYBqZp4QZV9N7b8= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -581,8 +582,9 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ= github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.3 h1:BtAvtV1+h0YwSVwWoYXMREPpYu9VzTJ9QDI1TEg/iQQ= +github.com/klauspost/compress v1.13.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -749,8 +751,9 @@ github.com/opencontainers/selinux v1.5.1/go.mod h1:yTcKuYAh6R95iDpefGLQaPaRwJFwy github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= -github.com/opencontainers/selinux v1.8.3 h1:tzZR7AuKB5gU1+53uBkoG4XdIFGZzvJTOVoNbRQI8/4= github.com/opencontainers/selinux v1.8.3/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= +github.com/opencontainers/selinux v1.8.4 h1:krlgQ6/j9CkCXT5oW0yVXdQFOME3NjKuuAZXuR6O7P4= +github.com/opencontainers/selinux v1.8.4/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= github.com/openshift/imagebuilder v1.2.2-0.20210415181909-87f3e48c2656 h1:WaxyNFpmIDu4i6so9r6LVFIbSaXqsj8oitMitt86ae4= github.com/openshift/imagebuilder v1.2.2-0.20210415181909-87f3e48c2656/go.mod h1:9aJRczxCH0mvT6XQ+5STAQaPWz7OsWcU5/mRkt8IWeo= github.com/ostreedev/ostree-go v0.0.0-20190702140239-759a8c1ac913 h1:TnbXhKzrTOyuvWrjI8W6pcoI9XPbLHFXCdN2dtUw7Rw= diff --git a/libpod/container.go b/libpod/container.go index f3f4b27b7..80fd35c09 100644 --- a/libpod/container.go +++ b/libpod/container.go @@ -240,7 +240,7 @@ type ContainerImageVolume struct { type ContainerSecret struct { // Secret is the secret *secrets.Secret - // UID is tbe UID of the secret file + // UID is the UID of the secret file UID uint32 // GID is the GID of the secret file GID uint32 @@ -1020,8 +1020,8 @@ func (c *Container) RWSize() (int64, error) { } // IDMappings returns the UID/GID mapping used for the container -func (c *Container) IDMappings() (storage.IDMappingOptions, error) { - return c.config.IDMappings, nil +func (c *Container) IDMappings() storage.IDMappingOptions { + return c.config.IDMappings } // RootUID returns the root user mapping from container diff --git a/libpod/container_config.go b/libpod/container_config.go index 72a969fe6..e15030c15 100644 --- a/libpod/container_config.go +++ b/libpod/container_config.go @@ -376,6 +376,6 @@ type ContainerMiscConfig struct { // EnvSecrets are secrets that are set as environment variables EnvSecrets map[string]*secrets.Secret `json:"secret_env,omitempty"` // InitContainerType specifies if the container is an initcontainer - // and if so, what type: always or oneshot are possible non-nil entries + // and if so, what type: always or once are possible non-nil entries InitContainerType string `json:"init_container_type,omitempty"` } diff --git a/libpod/container_internal.go b/libpod/container_internal.go index 8ffcccf4c..3f7a4807d 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -367,6 +367,12 @@ func (c *Container) setupStorageMapping(dest, from *storage.IDMappingOptions) { return } *dest = *from + // If we are creating a container inside a pod, we always want to inherit the + // userns settings from the infra container. So clear the auto userns settings + // so that we don't request storage for a new uid/gid map. + if c.PodID() != "" && !c.IsInfra() { + dest.AutoUserNs = false + } if dest.AutoUserNs { overrides := c.getUserOverrides() dest.AutoUserNsOpts.PasswdFile = overrides.ContainerEtcPasswdPath diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index f30f622ac..f21aebb09 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -659,7 +659,7 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { } } - if c.config.IDMappings.AutoUserNs { + if c.config.UserNsCtr == "" && c.config.IDMappings.AutoUserNs { if err := g.AddOrReplaceLinuxNamespace(string(spec.UserNamespace), ""); err != nil { return nil, err } @@ -1782,7 +1782,7 @@ func (c *Container) generateResolvConf() (string, error) { cniResponse := c.state.NetworkStatus for _, i := range cniResponse { for _, ip := range i.IPs { - // Note: only using To16() does not work since it also returns a vaild ip for ipv4 + // Note: only using To16() does not work since it also returns a valid ip for ipv4 if ip.Address.IP.To4() == nil && ip.Address.IP.To16() != nil { ipv6 = true } @@ -1884,7 +1884,7 @@ func (c *Container) generateResolvConf() (string, error) { return "", err } - return filepath.Join(c.state.RunDir, "resolv.conf"), nil + return destPath, nil } // generateHosts creates a containers hosts file diff --git a/libpod/container_log_linux.go b/libpod/container_log_linux.go index d4afaa52a..11f1be7f9 100644 --- a/libpod/container_log_linux.go +++ b/libpod/container_log_linux.go @@ -79,7 +79,7 @@ func (c *Container) readFromJournal(ctx context.Context, options *logs.LogOption break } if cursorError != nil { - return errors.Wrap(cursorError, "inital journal cursor") + return errors.Wrap(cursorError, "initial journal cursor") } // We need the container's events in the same journal to guarantee diff --git a/libpod/define/container.go b/libpod/define/container.go index f0aca92aa..bb44a6a4a 100644 --- a/libpod/define/container.go +++ b/libpod/define/container.go @@ -34,5 +34,5 @@ const ( AlwaysInitContainer = "always" // OneShotInitContainer is a container that only runs as init once // and is then deleted. - OneShotInitContainer = "oneshot" + OneShotInitContainer = "once" ) diff --git a/libpod/define/pod_inspect.go b/libpod/define/pod_inspect.go index a17304875..f91fd198d 100644 --- a/libpod/define/pod_inspect.go +++ b/libpod/define/pod_inspect.go @@ -105,6 +105,8 @@ type InspectPodInfraConfig struct { CPUSetCPUs string `json:"cpuset_cpus,omitempty"` // Pid is the PID namespace mode of the pod's infra container PidNS string `json:"pid_ns,omitempty"` + // UserNS is the usernamespace that all the containers in the pod will join. + UserNS string `json:"userns,omitempty"` } // InspectPodContainerInfo contains information on a container in a pod. diff --git a/libpod/info.go b/libpod/info.go index cdc73780f..2b48ea590 100644 --- a/libpod/info.go +++ b/libpod/info.go @@ -141,19 +141,24 @@ func (r *Runtime) hostInfo() (*define.HostInfo, error) { } info.CGroupsVersion = cgroupVersion - if rootless.IsRootless() { - if path, err := exec.LookPath("slirp4netns"); err == nil { - version, err := programVersion(path) - if err != nil { - logrus.Warnf("Failed to retrieve program version for %s: %v", path, err) - } - program := define.SlirpInfo{ - Executable: path, - Package: packageVersion(path), - Version: version, - } - info.Slirp4NetNS = program + slirp4netnsPath := r.config.Engine.NetworkCmdPath + if slirp4netnsPath == "" { + slirp4netnsPath, _ = exec.LookPath("slirp4netns") + } + if slirp4netnsPath != "" { + version, err := programVersion(slirp4netnsPath) + if err != nil { + logrus.Warnf("Failed to retrieve program version for %s: %v", slirp4netnsPath, err) + } + program := define.SlirpInfo{ + Executable: slirp4netnsPath, + Package: packageVersion(slirp4netnsPath), + Version: version, } + info.Slirp4NetNS = program + } + + if rootless.IsRootless() { uidmappings, err := rootless.ReadMappingsProc("/proc/self/uid_map") if err != nil { return nil, errors.Wrapf(err, "error reading uid mappings") diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go index 8e9b5997c..2ed2bb01b 100644 --- a/libpod/networking_linux.go +++ b/libpod/networking_linux.go @@ -173,11 +173,27 @@ func (r *RootlessCNI) Do(toRun func() error) error { // the link target will be available in the mount ns. // see: https://github.com/containers/podman/issues/10855 resolvePath := "/etc/resolv.conf" - resolvePath, err = filepath.EvalSymlinks(resolvePath) - if err != nil { - return err + for i := 0; i < 255; i++ { + // Do not use filepath.EvalSymlinks, we only want the first symlink under /run. + // If /etc/resolv.conf has more than one symlink under /run, e.g. + // -> /run/systemd/resolve/stub-resolv.conf -> /run/systemd/resolve/resolv.conf + // we would put the netns resolv.conf file to the last path. However this will + // break dns because the second link does not exists in the mount ns. + // see https://github.com/containers/podman/issues/11222 + link, err := os.Readlink(resolvePath) + if err != nil { + // if there is no symlink exit + break + } + resolvePath = filepath.Join(filepath.Dir(resolvePath), link) + if strings.HasPrefix(resolvePath, "/run/") { + break + } + if i == 254 { + return errors.New("too many symlinks while resolving /etc/resolv.conf") + } } - logrus.Debugf("The actual path of /etc/resolv.conf on the host is %q", resolvePath) + logrus.Debugf("The path of /etc/resolv.conf in the mount ns is %q", resolvePath) // When /etc/resolv.conf on the host is a symlink to /run/systemd/resolve/stub-resolv.conf, // we have to mount an empty filesystem on /run/systemd/resolve in the child namespace, // so as to isolate the directory from the host mount namespace. @@ -1219,7 +1235,7 @@ func (c *Container) NetworkDisconnect(nameOrID, netName string, force bool) erro return err } - // OCICNI will set the loopback adpter down on teardown so we should set it up again + // OCICNI will set the loopback adapter down on teardown so we should set it up again err = c.state.NetNS.Do(func(_ ns.NetNS) error { link, err := netlink.LinkByName("lo") if err != nil { @@ -1229,7 +1245,7 @@ func (c *Container) NetworkDisconnect(nameOrID, netName string, force bool) erro return err }) if err != nil { - logrus.Warnf("failed to set loopback adpter up in the container: %v", err) + logrus.Warnf("failed to set loopback adapter up in the container: %v", err) } // Reload ports when there are still connected networks, maybe we removed the network interface with the child ip. // Reloading without connected networks does not make sense, so we can skip this step. diff --git a/libpod/options.go b/libpod/options.go index 071b085e7..b94ef88ba 100644 --- a/libpod/options.go +++ b/libpod/options.go @@ -956,8 +956,9 @@ func WithUserNSFrom(nsCtr *Container) CtrCreateOption { } ctr.config.UserNsCtr = nsCtr.ID() - ctr.config.IDMappings = nsCtr.config.IDMappings - + if err := JSONDeepCopy(nsCtr.IDMappings(), &ctr.config.IDMappings); err != nil { + return err + } g := generate.Generator{Config: ctr.config.Spec} g.ClearLinuxUIDMappings() @@ -968,7 +969,6 @@ func WithUserNSFrom(nsCtr *Container) CtrCreateOption { for _, gidmap := range nsCtr.config.IDMappings.GIDMap { g.AddLinuxGIDMapping(uint32(gidmap.HostID), uint32(gidmap.ContainerID), uint32(gidmap.Size)) } - ctr.config.IDMappings = nsCtr.config.IDMappings return nil } } @@ -2423,6 +2423,24 @@ func WithVolatile() CtrCreateOption { } ctr.config.Volatile = true + + return nil + } +} + +// WithPodUserns sets the userns for the infra container in a pod. +func WithPodUserns(userns specgen.Namespace) PodCreateOption { + return func(pod *Pod) error { + if pod.valid { + return define.ErrPodFinalized + } + + if !pod.config.InfraContainer.HasInfraContainer { + return errors.Wrapf(define.ErrInvalidArg, "cannot configure pod userns as no infra container is being created") + } + + pod.config.InfraContainer.Userns = userns + return nil } } diff --git a/libpod/pod.go b/libpod/pod.go index 0fef7f6f3..7df15df7b 100644 --- a/libpod/pod.go +++ b/libpod/pod.go @@ -117,6 +117,7 @@ type InfraContainerConfig struct { Slirp4netns bool `json:"slirp4netns,omitempty"` NetworkOptions map[string][]string `json:"network_options,omitempty"` ResourceLimits *specs.LinuxResources `json:"resource_limits,omitempty"` + Userns specgen.Namespace `json:"userns,omitempty"` } // ID retrieves the pod's ID diff --git a/libpod/pod_api.go b/libpod/pod_api.go index 90d67dbb0..716eb2e5b 100644 --- a/libpod/pod_api.go +++ b/libpod/pod_api.go @@ -32,14 +32,14 @@ func (p *Pod) startInitContainers(ctx context.Context) error { if rc != 0 { return errors.Errorf("init container %s exited with code %d", initCon.ID(), rc) } - // If the container is an oneshot init container, we need to remove it + // If the container is a once init container, we need to remove it // after it runs if initCon.Config().InitContainerType == define.OneShotInitContainer { icLock := initCon.lock icLock.Lock() if err := p.runtime.removeContainer(ctx, initCon, false, false, true); err != nil { icLock.Unlock() - return errors.Wrapf(err, "failed to remove oneshot init container %s", initCon.ID()) + return errors.Wrapf(err, "failed to remove once init container %s", initCon.ID()) } // Removing a container this way requires an explicit call to clean up the db if err := p.runtime.state.RemoveContainerFromPod(p, initCon); err != nil { @@ -593,6 +593,7 @@ func (p *Pod) Inspect() (*define.InspectPodData, error) { infraConfig.CPUQuota = p.CPUQuota() infraConfig.CPUSetCPUs = p.ResourceLim().CPU.Cpus infraConfig.PidNS = p.PidMode() + infraConfig.UserNS = p.config.InfraContainer.Userns.String() if len(p.config.InfraContainer.DNSServer) > 0 { infraConfig.DNSServer = make([]string, 0, len(p.config.InfraContainer.DNSServer)) diff --git a/libpod/runtime_pod_infra_linux.go b/libpod/runtime_pod_infra_linux.go index d4f861118..49213032e 100644 --- a/libpod/runtime_pod_infra_linux.go +++ b/libpod/runtime_pod_infra_linux.go @@ -8,7 +8,9 @@ import ( "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/libpod/define" + "github.com/containers/podman/v3/pkg/namespaces" "github.com/containers/podman/v3/pkg/rootless" + "github.com/containers/podman/v3/pkg/specgen" "github.com/containers/podman/v3/pkg/util" v1 "github.com/opencontainers/image-spec/specs-go/v1" spec "github.com/opencontainers/runtime-spec/specs-go" @@ -110,9 +112,7 @@ func (r *Runtime) makeInfraContainer(ctx context.Context, p *Pod, imgName, rawIm options = append(options, WithNetworkOptions(p.config.InfraContainer.NetworkOptions)) } } - // PostConfigureNetNS should not be set since user namespace sharing is not implemented - // and rootless networking no longer supports post configuration setup - options = append(options, WithNetNS(p.config.InfraContainer.PortBindings, false, netmode, p.config.InfraContainer.Networks)) + options = append(options, WithNetNS(p.config.InfraContainer.PortBindings, !p.config.InfraContainer.Userns.IsHost(), netmode, p.config.InfraContainer.Networks)) } // For each option in InfraContainerConfig - if set, pass into @@ -158,11 +158,39 @@ func (r *Runtime) makeInfraContainer(ctx context.Context, p *Pod, imgName, rawIm g.Config.Linux.Namespaces = newNS } } + + for _, ctl := range r.config.Containers.DefaultSysctls { + sysctl := strings.SplitN(ctl, "=", 2) + if len(sysctl) < 2 { + return nil, errors.Errorf("invalid default sysctl %s", ctl) + } + + // Ignore net sysctls if --net=host + if p.config.InfraContainer.HostNetwork && strings.HasPrefix(sysctl[0], "net.") { + logrus.Infof("Sysctl %s=%s ignored in containers.conf, since Network Namespace set to host", sysctl[0], sysctl[1]) + continue + } + + g.AddLinuxSysctl(sysctl[0], sysctl[1]) + } + g.SetRootReadonly(true) g.SetProcessArgs(infraCtrCommand) logrus.Debugf("Using %q as infra container command", infraCtrCommand) + mapopt, err := util.ParseIDMapping(namespaces.UsernsMode(p.config.InfraContainer.Userns.String()), []string{}, []string{}, "", "") + if err != nil { + return nil, err + } + user, err := specgen.SetupUserNS(mapopt, p.config.InfraContainer.Userns, &g) + if err != nil { + return nil, err + } + if user != "" { + options = append(options, WithUser(user)) + } + g.RemoveMount("/dev/shm") if isRootless { g.RemoveMount("/dev/pts") @@ -210,14 +238,15 @@ func (r *Runtime) makeInfraContainer(ctx context.Context, p *Pod, imgName, rawIm options = append(options, WithRootFSFromImage(imgID, imgName, rawImageName)) options = append(options, WithName(containerName)) options = append(options, withIsInfra()) + options = append(options, WithIDMappings(*mapopt)) if len(p.config.InfraContainer.ConmonPidFile) > 0 { options = append(options, WithConmonPidFile(p.config.InfraContainer.ConmonPidFile)) } newRes := new(spec.LinuxResources) newRes.CPU = new(spec.LinuxCPU) newRes.CPU = p.ResourceLim().CPU - g.Config.Linux.Resources.CPU = newRes.CPU + return r.newContainer(ctx, g.Config, options...) } diff --git a/pkg/api/handlers/compat/images_build.go b/pkg/api/handlers/compat/images_build.go index 08d1df4b8..0fcca1821 100644 --- a/pkg/api/handlers/compat/images_build.go +++ b/pkg/api/handlers/compat/images_build.go @@ -34,13 +34,16 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { contentType := hdr[0] switch contentType { case "application/tar": - logrus.Warnf("tar file content type is %s, should use \"application/x-tar\" content type", contentType) + logrus.Infof("tar file content type is %s, should use \"application/x-tar\" content type", contentType) case "application/x-tar": break default: - utils.BadRequest(w, "Content-Type", hdr[0], - fmt.Errorf("Content-Type: %s is not supported. Should be \"application/x-tar\"", hdr[0])) - return + if utils.IsLibpodRequest(r) { + utils.BadRequest(w, "Content-Type", hdr[0], + fmt.Errorf("Content-Type: %s is not supported. Should be \"application/x-tar\"", hdr[0])) + return + } + logrus.Infof("tar file content type is %s, should use \"application/x-tar\" content type", contentType) } } diff --git a/pkg/api/handlers/libpod/pods.go b/pkg/api/handlers/libpod/pods.go index 4dc8740e2..ff105bc48 100644 --- a/pkg/api/handlers/libpod/pods.go +++ b/pkg/api/handlers/libpod/pods.go @@ -30,6 +30,12 @@ func PodCreate(w http.ResponseWriter, r *http.Request) { utils.Error(w, "failed to decode specgen", http.StatusInternalServerError, errors.Wrap(err, "failed to decode specgen")) return } + // parse userns so we get the valid default value of userns + psg.Userns, err = specgen.ParseUserNamespace(psg.Userns.String()) + if err != nil { + utils.Error(w, "failed to parse userns", http.StatusInternalServerError, errors.Wrap(err, "failed to parse userns")) + return + } pod, err := generate.MakePod(&psg, runtime) if err != nil { httpCode := http.StatusInternalServerError diff --git a/pkg/api/handlers/utils/images.go b/pkg/api/handlers/utils/images.go index 1e8edb6dd..1e3647a3e 100644 --- a/pkg/api/handlers/utils/images.go +++ b/pkg/api/handlers/utils/images.go @@ -27,7 +27,7 @@ func IsRegistryReference(name string) error { if imageRef.Transport().Name() == docker.Transport.Name() { return nil } - return errors.Errorf("unsupport transport %s in %q: only docker transport is supported", imageRef.Transport().Name(), name) + return errors.Errorf("unsupported transport %s in %q: only docker transport is supported", imageRef.Transport().Name(), name) } // ParseStorageReference parses the specified image name to a diff --git a/pkg/api/server/register_pods.go b/pkg/api/server/register_pods.go index 3bcc50ba4..58234005e 100644 --- a/pkg/api/server/register_pods.go +++ b/pkg/api/server/register_pods.go @@ -17,7 +17,18 @@ func (s *APIServer) registerPodsHandlers(r *mux.Router) error { // - in: query // name: filters // type: string - // description: needs description and plumbing for filters + // description: | + // JSON encoded value of the filters (a map[string][]string) to process on the pods list. Available filters: + // - `id=<pod-id>` Matches all of pod id. + // - `label=<key>` or `label=<key>:<value>` Matches pods based on the presence of a label alone or a label and a value. + // - `name=<pod-name>` Matches all of pod name. + // - `until=<timestamp>` List pods created before this timestamp. The `<timestamp>` can be Unix timestamps, date formatted timestamps, or Go duration strings (e.g. `10m`, `1h30m`) computed relative to the daemon machine’s time. + // - `status=<pod-status>` Pod's status: `stopped`, `running`, `paused`, `exited`, `dead`, `created`, `degraded`. + // - `network=<pod-network>` Name or full ID of network. + // - `ctr-names=<pod-ctr-names>` Container name within the pod. + // - `ctr-ids=<pod-ctr-ids>` Container ID within the pod. + // - `ctr-status=<pod-ctr-status>` Container status within the pod. + // - `ctr-number=<pod-ctr-number>` Number of containers in the pod. // responses: // 200: // $ref: "#/responses/ListPodsResponse" diff --git a/pkg/domain/entities/pods.go b/pkg/domain/entities/pods.go index 68e335f8d..c66bf96fc 100644 --- a/pkg/domain/entities/pods.go +++ b/pkg/domain/entities/pods.go @@ -122,6 +122,7 @@ type PodCreateOptions struct { Pid string Cpus float64 CpusetCpus string + Userns specgen.Namespace } type PodCreateReport struct { @@ -217,6 +218,7 @@ func (p *PodCreateOptions) ToPodSpecGen(s *specgen.PodSpecGenerator) error { s.CPUQuota = *cpuDat.Quota } } + s.Userns = p.Userns return nil } diff --git a/pkg/domain/filters/containers.go b/pkg/domain/filters/containers.go index dc9fed2a4..269cd2d27 100644 --- a/pkg/domain/filters/containers.go +++ b/pkg/domain/filters/containers.go @@ -214,7 +214,7 @@ func GenerateContainerFilterFuncs(filter string, filterValues []string, r *libpo networkMode := c.NetworkMode() // support docker like `--filter network=container:<IDorName>` // check if networkMode is configured as `container:<ctr>` - // peform a match against filter `container:<IDorName>` + // perform a match against filter `container:<IDorName>` // networks is already going to be empty if `container:<ctr>` is configured as Mode if strings.HasPrefix(networkMode, "container:") { networkModeContainerPart := strings.SplitN(networkMode, ":", 2) diff --git a/pkg/domain/filters/pods.go b/pkg/domain/filters/pods.go index 9a1c7d19d..9a2f0a3ba 100644 --- a/pkg/domain/filters/pods.go +++ b/pkg/domain/filters/pods.go @@ -116,6 +116,17 @@ func GeneratePodFilterFunc(filter string, filterValues []string) ( labels := p.Labels() return util.MatchLabelFilters(filterValues, labels) }, nil + case "until": + return func(p *libpod.Pod) bool { + until, err := util.ComputeUntilTimestamp(filterValues) + if err != nil { + return false + } + if p.CreatedTime().Before(until) { + return true + } + return false + }, nil case "network": return func(p *libpod.Pod) bool { infra, err := p.InfraContainer() diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go index 7b1ebcb03..a92892957 100644 --- a/pkg/machine/qemu/machine.go +++ b/pkg/machine/qemu/machine.go @@ -605,10 +605,12 @@ func CheckActiveVM() (bool, string, error) { // startHostNetworking runs a binary on the host system that allows users // to setup port forwarding to the podman virtual machine func (v *MachineVM) startHostNetworking() error { - binary := filepath.Join("/usr/lib/podman/", machine.ForwarderBinaryName) - if _, err := os.Stat(binary); os.IsNotExist(err) { - return errors.Errorf("unable to find %s", binary) + // TODO we may wish to configure the directory in containers common + binary := filepath.Join("/usr/libexec/podman/", machine.ForwarderBinaryName) + if _, err := os.Stat(binary); err != nil { + return err } + // Listen on all at port 7777 for setting up and tearing // down forwarding listenSocket := "tcp://0.0.0.0:7777" diff --git a/pkg/specgen/generate/kube/kube.go b/pkg/specgen/generate/kube/kube.go index fb7eb99a2..d6d479c67 100644 --- a/pkg/specgen/generate/kube/kube.go +++ b/pkg/specgen/generate/kube/kube.go @@ -303,6 +303,8 @@ func ToSpecGen(ctx context.Context, opts *CtrSpecGenOptions) (*specgen.SpecGener if opts.NetNSIsHost { s.NetNS.NSMode = specgen.Host } + // Always set the userns to host since k8s doesn't have support for userns yet + s.UserNS.NSMode = specgen.Host // Add labels that come from kube if len(s.Labels) == 0 { diff --git a/pkg/specgen/generate/namespaces.go b/pkg/specgen/generate/namespaces.go index f41186ae4..1d7373093 100644 --- a/pkg/specgen/generate/namespaces.go +++ b/pkg/specgen/generate/namespaces.go @@ -175,6 +175,11 @@ func namespaceOptions(ctx context.Context, s *specgen.SpecGenerator, rt *libpod. if pod == nil || infraCtr == nil { return nil, errNoInfra } + // Inherit the user from the infra container if it is set and --user has not + // been set explicitly + if infraCtr.User() != "" && s.User == "" { + toReturn = append(toReturn, libpod.WithUser(infraCtr.User())) + } toReturn = append(toReturn, libpod.WithUserNSFrom(infraCtr)) case specgen.FromContainer: userCtr, err := rt.LookupContainer(s.UserNS.Value) @@ -184,7 +189,10 @@ func namespaceOptions(ctx context.Context, s *specgen.SpecGenerator, rt *libpod. toReturn = append(toReturn, libpod.WithUserNSFrom(userCtr)) } - if s.IDMappings != nil { + // This wipes the UserNS settings that get set from the infra container + // when we are inheritting from the pod. So only apply this if the container + // is not being created in a pod. + if s.IDMappings != nil && pod == nil { toReturn = append(toReturn, libpod.WithIDMappings(*s.IDMappings)) } if s.User != "" { @@ -379,46 +387,8 @@ func specConfigureNamespaces(s *specgen.SpecGenerator, g *generate.Generator, rt } // User - switch s.UserNS.NSMode { - case specgen.Path: - if _, err := os.Stat(s.UserNS.Value); err != nil { - return errors.Wrap(err, "cannot find specified user namespace path") - } - if err := g.AddOrReplaceLinuxNamespace(string(spec.UserNamespace), s.UserNS.Value); err != nil { - return err - } - // runc complains if no mapping is specified, even if we join another ns. So provide a dummy mapping - g.AddLinuxUIDMapping(uint32(0), uint32(0), uint32(1)) - g.AddLinuxGIDMapping(uint32(0), uint32(0), uint32(1)) - case specgen.Host: - if err := g.RemoveLinuxNamespace(string(spec.UserNamespace)); err != nil { - return err - } - case specgen.KeepID: - var ( - err error - uid, gid int - ) - s.IDMappings, uid, gid, err = util.GetKeepIDMapping() - if err != nil { - return err - } - g.SetProcessUID(uint32(uid)) - g.SetProcessGID(uint32(gid)) - fallthrough - case specgen.Private: - if err := g.AddOrReplaceLinuxNamespace(string(spec.UserNamespace), ""); err != nil { - return err - } - if s.IDMappings == nil || (len(s.IDMappings.UIDMap) == 0 && len(s.IDMappings.GIDMap) == 0) { - return errors.Errorf("must provide at least one UID or GID mapping to configure a user namespace") - } - for _, uidmap := range s.IDMappings.UIDMap { - g.AddLinuxUIDMapping(uint32(uidmap.HostID), uint32(uidmap.ContainerID), uint32(uidmap.Size)) - } - for _, gidmap := range s.IDMappings.GIDMap { - g.AddLinuxGIDMapping(uint32(gidmap.HostID), uint32(gidmap.ContainerID), uint32(gidmap.Size)) - } + if _, err := specgen.SetupUserNS(s.IDMappings, s.UserNS, g); err != nil { + return err } // Cgroup @@ -474,7 +444,7 @@ func specConfigureNamespaces(s *specgen.SpecGenerator, g *generate.Generator, rt // GetNamespaceOptions transforms a slice of kernel namespaces // into a slice of pod create options. Currently, not all // kernel namespaces are supported, and they will be returned in an error -func GetNamespaceOptions(ns []string) ([]libpod.PodCreateOption, error) { +func GetNamespaceOptions(ns []string, netnsIsHost bool) ([]libpod.PodCreateOption, error) { var options []libpod.PodCreateOption var erroredOptions []libpod.PodCreateOption if ns == nil { @@ -486,7 +456,10 @@ func GetNamespaceOptions(ns []string) ([]libpod.PodCreateOption, error) { case "cgroup": options = append(options, libpod.WithPodCgroups()) case "net": - options = append(options, libpod.WithPodNet()) + // share the netns setting with other containers in the pod only when it is not set to host + if !netnsIsHost { + options = append(options, libpod.WithPodNet()) + } case "mnt": return erroredOptions, errors.Errorf("Mount sharing functionality not supported on pod level") case "pid": diff --git a/pkg/specgen/generate/pod_create.go b/pkg/specgen/generate/pod_create.go index aab29499e..426cf1b6d 100644 --- a/pkg/specgen/generate/pod_create.go +++ b/pkg/specgen/generate/pod_create.go @@ -27,11 +27,16 @@ func createPodOptions(p *specgen.PodSpecGenerator, rt *libpod.Runtime) ([]libpod ) if !p.NoInfra { options = append(options, libpod.WithInfraContainer()) - nsOptions, err := GetNamespaceOptions(p.SharedNamespaces) + nsOptions, err := GetNamespaceOptions(p.SharedNamespaces, p.NetNS.IsHost()) if err != nil { return nil, err } options = append(options, nsOptions...) + // Use pod user and infra userns only when --userns is not set to host + if !p.Userns.IsHost() { + options = append(options, libpod.WithPodUser()) + options = append(options, libpod.WithPodUserns(p.Userns)) + } // Make our exit command storageConfig := rt.StorageConfig() @@ -154,5 +159,6 @@ func createPodOptions(p *specgen.PodSpecGenerator, rt *libpod.Runtime) ([]libpod if len(p.InfraConmonPidFile) > 0 { options = append(options, libpod.WithInfraConmonPidFile(p.InfraConmonPidFile)) } + return options, nil } diff --git a/pkg/specgen/namespaces.go b/pkg/specgen/namespaces.go index 76fa66bc7..2f4c48811 100644 --- a/pkg/specgen/namespaces.go +++ b/pkg/specgen/namespaces.go @@ -1,10 +1,16 @@ package specgen import ( + "fmt" + "os" "strings" "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/rootless" + "github.com/containers/podman/v3/pkg/util" + "github.com/containers/storage" + spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/opencontainers/runtime-tools/generate" "github.com/pkg/errors" ) @@ -103,6 +109,13 @@ func (n *Namespace) IsKeepID() bool { return n.NSMode == KeepID } +func (n *Namespace) String() string { + if n.Value != "" { + return fmt.Sprintf("%s:%s", n.NSMode, n.Value) + } + return string(n.NSMode) +} + func validateUserNS(n *Namespace) error { if n == nil { return nil @@ -323,3 +336,48 @@ func ParseNetworkString(network string) (Namespace, []string, map[string][]strin } return ns, cniNets, networkOptions, nil } + +func SetupUserNS(idmappings *storage.IDMappingOptions, userns Namespace, g *generate.Generator) (string, error) { + // User + var user string + switch userns.NSMode { + case Path: + if _, err := os.Stat(userns.Value); err != nil { + return user, errors.Wrap(err, "cannot find specified user namespace path") + } + if err := g.AddOrReplaceLinuxNamespace(string(spec.UserNamespace), userns.Value); err != nil { + return user, err + } + // runc complains if no mapping is specified, even if we join another ns. So provide a dummy mapping + g.AddLinuxUIDMapping(uint32(0), uint32(0), uint32(1)) + g.AddLinuxGIDMapping(uint32(0), uint32(0), uint32(1)) + case Host: + if err := g.RemoveLinuxNamespace(string(spec.UserNamespace)); err != nil { + return user, err + } + case KeepID: + mappings, uid, gid, err := util.GetKeepIDMapping() + if err != nil { + return user, err + } + idmappings = mappings + g.SetProcessUID(uint32(uid)) + g.SetProcessGID(uint32(gid)) + user = fmt.Sprintf("%d:%d", uid, gid) + fallthrough + case Private: + if err := g.AddOrReplaceLinuxNamespace(string(spec.UserNamespace), ""); err != nil { + return user, err + } + if idmappings == nil || (len(idmappings.UIDMap) == 0 && len(idmappings.GIDMap) == 0) { + return user, errors.Errorf("must provide at least one UID or GID mapping to configure a user namespace") + } + for _, uidmap := range idmappings.UIDMap { + g.AddLinuxUIDMapping(uint32(uidmap.HostID), uint32(uidmap.ContainerID), uint32(uidmap.Size)) + } + for _, gidmap := range idmappings.GIDMap { + g.AddLinuxGIDMapping(uint32(gidmap.HostID), uint32(gidmap.ContainerID), uint32(gidmap.Size)) + } + } + return user, nil +} diff --git a/pkg/specgen/podspecgen.go b/pkg/specgen/podspecgen.go index 02237afe9..893ebf675 100644 --- a/pkg/specgen/podspecgen.go +++ b/pkg/specgen/podspecgen.go @@ -67,6 +67,10 @@ type PodBasicConfig struct { // Optional (defaults to private if unset). This sets the PID namespace of the infra container // This configuration will then be shared with the entire pod if PID namespace sharing is enabled via --share Pid Namespace `json:"pid,omitempty:"` + // Userns is used to indicate which kind of Usernamespace to enter. + // Any containers created within the pod will inherit the pod's userns settings. + // Optional + Userns Namespace `json:"userns,omitempty"` } // PodNetworkConfig contains networking configuration for a pod. diff --git a/pkg/specgen/specgen.go b/pkg/specgen/specgen.go index fc647227e..2252ef405 100644 --- a/pkg/specgen/specgen.go +++ b/pkg/specgen/specgen.go @@ -184,7 +184,7 @@ type ContainerBasicConfig struct { // Optional. EnvSecrets map[string]string `json:"secret_env,omitempty"` // InitContainerType describes if this container is an init container - // and if so, what type: always or oneshot + // and if so, what type: always or once InitContainerType string `json:"init_container_type"` // Personality allows users to configure different execution domains. // Execution domains tell Linux how to map signal numbers into signal actions. diff --git a/test/apiv2/10-images.at b/test/apiv2/10-images.at index 195b11ff0..abc8d44b7 100644 --- a/test/apiv2/10-images.at +++ b/test/apiv2/10-images.at @@ -173,7 +173,7 @@ curl -XPOST --data-binary @<(cat $CONTAINERFILE_TAR) \ BUILD_TEST_ERROR="" if ! grep -q '200 OK' "${TMPD}/headers.txt"; then - echo -e "${red}NOK: Image build from tar failed response was not 200 OK" + echo -e "${red}NOK: Image build from tar failed response was not 200 OK (application/x-tar)" BUILD_TEST_ERROR="1" fi @@ -182,6 +182,38 @@ if ! grep -q 'quay.io/libpod/alpine_labels' "${TMPD}/response.txt"; then BUILD_TEST_ERROR="1" fi +curl -XPOST --data-binary @<(cat $CONTAINERFILE_TAR) \ + -H "content-type: application/tar" \ + --dump-header "${TMPD}/headers.txt" \ + -o /dev/null \ + "http://$HOST:$PORT/v1.40/libpod/build?dockerfile=containerfile" &> /dev/null +if ! grep -q '200 OK' "${TMPD}/headers.txt"; then + echo -e "${red}NOK: Image build from tar failed response was not 200 OK (application/tar)" + BUILD_TEST_ERROR="1" +fi + +# Yes, this is very un-RESTful re: Content-Type header ignored when compatibility endpoint used +# See https://github.com/containers/podman/issues/11012 +curl -XPOST --data-binary @<(cat $CONTAINERFILE_TAR) \ + -H "content-type: application/json" \ + --dump-header "${TMPD}/headers.txt" \ + -o /dev/null \ + "http://$HOST:$PORT/v1.40/build?dockerfile=containerfile" &> /dev/null +if ! grep -q '200 OK' "${TMPD}/headers.txt"; then + echo -e "${red}NOK: Image build from tar failed response was not 200 OK (application/tar)" + BUILD_TEST_ERROR="1" +fi + +curl -XPOST --data-binary @<(cat $CONTAINERFILE_TAR) \ + -H "content-type: application/json" \ + --dump-header "${TMPD}/headers.txt" \ + -o /dev/null \ + "http://$HOST:$PORT/v1.40/libpod/build?dockerfile=containerfile" &> /dev/null +if ! grep -q '400 Bad Request' "${TMPD}/headers.txt"; then + echo -e "${red}NOK: Image build should have failed with 400 (wrong Content-Type)" + BUILD_TEST_ERROR="1" +fi + cleanBuildTest if [[ "${BUILD_TEST_ERROR}" ]]; then exit 1 diff --git a/test/apiv2/20-containers.at b/test/apiv2/20-containers.at index 610d3e36d..e2eb94233 100644 --- a/test/apiv2/20-containers.at +++ b/test/apiv2/20-containers.at @@ -356,3 +356,14 @@ t GET containers/$cid/json 200 \ .HostConfig.NetworkMode="bridge" t DELETE containers/$cid?v=true 204 + +# Test Compat Create with healthcheck, check default values +t POST containers/create Image=$IMAGE Cmd='["top"]' Healthcheck='{"Test":["true"]}' 201 \ + .Id~[0-9a-f]\\{64\\} +cid=$(jq -r '.Id' <<<"$output") +t GET containers/$cid/json 200 \ + .Config.Healthcheck.Interval=30000000000 \ + .Config.Healthcheck.Timeout=30000000000 \ + .Config.Healthcheck.Retries=3 + +t DELETE containers/$cid?v=true 204 diff --git a/test/apiv2/40-pods.at b/test/apiv2/40-pods.at index 94c72dbaa..985b26411 100644 --- a/test/apiv2/40-pods.at +++ b/test/apiv2/40-pods.at @@ -19,6 +19,9 @@ t GET libpod/pods/json 200 \ .[0].Id=$pod_id \ .[0].Containers\|length=1 +t GET libpod/pods/json?filters='{"until":["500000"]}' 200 length=0 +t GET libpod/pods/json?filters='{"until":["5000000000"]}' 200 length=1 + # Cannot create a dup pod with the same name t POST "libpod/pods/create (dup pod)" name=foo 409 \ .cause="pod already exists" diff --git a/test/e2e/play_kube_test.go b/test/e2e/play_kube_test.go index 66bfdefe7..e3096d932 100644 --- a/test/e2e/play_kube_test.go +++ b/test/e2e/play_kube_test.go @@ -1114,7 +1114,7 @@ var _ = Describe("Podman play kube", func() { }) It("podman play kube should share ipc,net,uts when shareProcessNamespace is set", func() { - SkipIfRootless("Requires root priviledges for sharing few namespaces") + SkipIfRootless("Requires root privileges for sharing few namespaces") err := writeYaml(sharedNamespacePodYaml, kubeYaml) Expect(err).To(BeNil()) diff --git a/test/e2e/pod_create_test.go b/test/e2e/pod_create_test.go index 4c6788b9d..f6f532ce9 100644 --- a/test/e2e/pod_create_test.go +++ b/test/e2e/pod_create_test.go @@ -4,6 +4,7 @@ import ( "fmt" "io/ioutil" "os" + "os/user" "path/filepath" "strconv" "strings" @@ -621,4 +622,223 @@ ENTRYPOINT ["sleep","99999"] Expect(podCreate).Should(ExitWithError()) }) + + It("podman pod create with --userns=keep-id", func() { + if os.Geteuid() == 0 { + Skip("Test only runs without root") + } + + podName := "testPod" + podCreate := podmanTest.Podman([]string{"pod", "create", "--userns", "keep-id", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + + session := podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "id", "-u"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + uid := fmt.Sprintf("%d", os.Geteuid()) + ok, _ := session.GrepString(uid) + Expect(ok).To(BeTrue()) + + // Check passwd + session = podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "id", "-un"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + u, err := user.Current() + Expect(err).To(BeNil()) + ok, _ = session.GrepString(u.Name) + Expect(ok).To(BeTrue()) + + // root owns /usr + session = podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "stat", "-c%u", "/usr"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + Expect(session.OutputToString()).To(Equal("0")) + + // fail if --pod and --userns set together + session = podmanTest.Podman([]string{"run", "--pod", podName, "--userns", "keep-id", ALPINE, "id", "-u"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(125)) + }) + + It("podman pod create with --userns=keep-id can add users", func() { + if os.Geteuid() == 0 { + Skip("Test only runs without root") + } + + podName := "testPod" + podCreate := podmanTest.Podman([]string{"pod", "create", "--userns", "keep-id", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + + ctrName := "ctr-name" + session := podmanTest.Podman([]string{"run", "--pod", podName, "-d", "--stop-signal", "9", "--name", ctrName, fedoraMinimal, "sleep", "600"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + // container inside pod inherits user form infra container if --user is not set + // etc/passwd entry will look like 1000:*:1000:1000:container user:/:/bin/sh + exec1 := podmanTest.Podman([]string{"exec", ctrName, "cat", "/etc/passwd"}) + exec1.WaitWithDefaultTimeout() + Expect(exec1).Should(Exit(0)) + Expect(exec1.OutputToString()).To(ContainSubstring("container")) + + exec2 := podmanTest.Podman([]string{"exec", ctrName, "useradd", "testuser"}) + exec2.WaitWithDefaultTimeout() + Expect(exec2).Should(Exit(0)) + + exec3 := podmanTest.Podman([]string{"exec", ctrName, "cat", "/etc/passwd"}) + exec3.WaitWithDefaultTimeout() + Expect(exec3).Should(Exit(0)) + Expect(exec3.OutputToString()).To(ContainSubstring("testuser")) + }) + + It("podman pod create with --userns=auto", func() { + u, err := user.Current() + Expect(err).To(BeNil()) + name := u.Name + if name == "root" { + name = "containers" + } + + content, err := ioutil.ReadFile("/etc/subuid") + if err != nil { + Skip("cannot read /etc/subuid") + } + if !strings.Contains(string(content), name) { + Skip("cannot find mappings for the current user") + } + + m := make(map[string]string) + for i := 0; i < 5; i++ { + podName := "testPod" + strconv.Itoa(i) + podCreate := podmanTest.Podman([]string{"pod", "create", "--userns=auto", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + + session := podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/uid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + l := session.OutputToString() + Expect(strings.Contains(l, "1024")).To(BeTrue()) + m[l] = l + } + // check for no duplicates + Expect(len(m)).To(Equal(5)) + }) + + It("podman pod create --userns=auto:size=%d", func() { + u, err := user.Current() + Expect(err).To(BeNil()) + + name := u.Name + if name == "root" { + name = "containers" + } + + content, err := ioutil.ReadFile("/etc/subuid") + if err != nil { + Skip("cannot read /etc/subuid") + } + if !strings.Contains(string(content), name) { + Skip("cannot find mappings for the current user") + } + + podName := "testPod" + podCreate := podmanTest.Podman([]string{"pod", "create", "--userns=auto:size=500", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + session := podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/uid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + ok, _ := session.GrepString("500") + + podName = "testPod-1" + podCreate = podmanTest.Podman([]string{"pod", "create", "--userns=auto:size=3000", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + session = podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/uid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + ok, _ = session.GrepString("3000") + + Expect(ok).To(BeTrue()) + }) + + It("podman pod create --userns=auto:uidmapping=", func() { + u, err := user.Current() + Expect(err).To(BeNil()) + + name := u.Name + if name == "root" { + name = "containers" + } + + content, err := ioutil.ReadFile("/etc/subuid") + if err != nil { + Skip("cannot read /etc/subuid") + } + if !strings.Contains(string(content), name) { + Skip("cannot find mappings for the current user") + } + + podName := "testPod" + podCreate := podmanTest.Podman([]string{"pod", "create", "--userns=auto:uidmapping=0:0:1", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + session := podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/uid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + output := session.OutputToString() + Expect(output).To(MatchRegexp("\\s0\\s0\\s1")) + + podName = "testPod-1" + podCreate = podmanTest.Podman([]string{"pod", "create", "--userns=auto:size=8192,uidmapping=0:0:1", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + session = podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/uid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + ok, _ := session.GrepString("8191") + Expect(ok).To(BeTrue()) + }) + + It("podman pod create --userns=auto:gidmapping=", func() { + u, err := user.Current() + Expect(err).To(BeNil()) + + name := u.Name + if name == "root" { + name = "containers" + } + + content, err := ioutil.ReadFile("/etc/subuid") + if err != nil { + Skip("cannot read /etc/subuid") + } + if !strings.Contains(string(content), name) { + Skip("cannot find mappings for the current user") + } + + podName := "testPod" + podCreate := podmanTest.Podman([]string{"pod", "create", "--userns=auto:gidmapping=0:0:1", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + session := podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/gid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + output := session.OutputToString() + Expect(output).To(MatchRegexp("\\s0\\s0\\s1")) + + podName = "testPod-1" + podCreate = podmanTest.Podman([]string{"pod", "create", "--userns=auto:size=8192,gidmapping=0:0:1", "--name", podName}) + podCreate.WaitWithDefaultTimeout() + Expect(podCreate).Should(Exit(0)) + session = podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/proc/self/gid_map"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + ok, _ := session.GrepString("8191") + Expect(ok).To(BeTrue()) + }) + }) diff --git a/test/e2e/pod_initcontainers_test.go b/test/e2e/pod_initcontainers_test.go index 606294f51..11e7ca400 100644 --- a/test/e2e/pod_initcontainers_test.go +++ b/test/e2e/pod_initcontainers_test.go @@ -98,10 +98,10 @@ var _ = Describe("Podman init containers", func() { Expect(checkLog.OutputToString()).To(Equal(content)) }) - It("podman make sure oneshot container is removed", func() { + It("podman make sure once container is removed", func() { filename := filepath.Join("/dev/shm", RandomString(12)) content := RandomString(16) - session := podmanTest.Podman([]string{"create", "--init-ctr", "oneshot", "--pod", "new:foobar", ALPINE, "bin/sh", "-c", fmt.Sprintf("echo %s > %s", content, filename)}) + session := podmanTest.Podman([]string{"create", "--init-ctr", "once", "--pod", "new:foobar", ALPINE, "bin/sh", "-c", fmt.Sprintf("echo %s > %s", content, filename)}) session.WaitWithDefaultTimeout() initContainerID := session.OutputToString() Expect(session).Should(Exit(0)) diff --git a/test/e2e/pod_ps_test.go b/test/e2e/pod_ps_test.go index c27539d6f..b4a0df904 100644 --- a/test/e2e/pod_ps_test.go +++ b/test/e2e/pod_ps_test.go @@ -108,6 +108,22 @@ var _ = Describe("Podman ps", func() { Expect(result).Should(Exit(0)) }) + It("podman pod ps --filter until", func() { + name := "mypod" + _, ec, _ := podmanTest.CreatePod(map[string][]string{"--name": {name}}) + Expect(ec).To(Equal(0)) + + result := podmanTest.Podman([]string{"pod", "ps", "--filter", "until=50"}) + result.WaitWithDefaultTimeout() + Expect(result).Should(Exit(0)) + Expect(result.OutputToString()).To(Not(ContainSubstring(name))) + + result = podmanTest.Podman([]string{"pod", "ps", "--filter", "until=5000000000"}) + result.WaitWithDefaultTimeout() + Expect(result).Should(Exit(0)) + Expect(result.OutputToString()).To(ContainSubstring(name)) + }) + It("podman pod ps filter name regexp", func() { _, ec, podid := podmanTest.CreatePod(map[string][]string{"--name": {"mypod"}}) Expect(ec).To(Equal(0)) diff --git a/test/e2e/run_cgroup_parent_test.go b/test/e2e/run_cgroup_parent_test.go index 3e261961b..82b6c3057 100644 --- a/test/e2e/run_cgroup_parent_test.go +++ b/test/e2e/run_cgroup_parent_test.go @@ -64,6 +64,7 @@ var _ = Describe("Podman run with --cgroup-parent", func() { }) Specify("always honor --cgroup-parent", func() { + Skip("https://github.com/containers/podman/issues/11165") SkipIfCgroupV1("test not supported in cgroups v1") if Containerized() || podmanTest.CgroupManager == "cgroupfs" { Skip("Requires Systemd cgroup manager support") diff --git a/test/e2e/run_device_test.go b/test/e2e/run_device_test.go index 40de1d50d..08905aed2 100644 --- a/test/e2e/run_device_test.go +++ b/test/e2e/run_device_test.go @@ -41,36 +41,35 @@ var _ = Describe("Podman run device", func() { }) It("podman run device test", func() { - session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg", ALPINE, "ls", "--color=never", "/dev/kmsg"}) + session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg", ALPINE, "test", "-c", "/dev/kmsg"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(session.OutputToString()).To(Equal("/dev/kmsg")) }) It("podman run device rename test", func() { - session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:/dev/kmsg1", ALPINE, "ls", "--color=never", "/dev/kmsg1"}) + // TODO: Confirm absence of /dev/kmsg in container + session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:/dev/kmsg1", ALPINE, "test", "-c", "/dev/kmsg1"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(session.OutputToString()).To(Equal("/dev/kmsg1")) }) It("podman run device permission test", func() { - session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:r", ALPINE, "ls", "--color=never", "/dev/kmsg"}) + // TODO: Confirm write-permission failure + session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:r", ALPINE, "test", "-r", "/dev/kmsg"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(session.OutputToString()).To(Equal("/dev/kmsg")) }) It("podman run device rename and permission test", func() { - session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:/dev/kmsg1:r", ALPINE, "ls", "--color=never", "/dev/kmsg1"}) + // TODO: Confirm write-permission failure + session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:/dev/kmsg1:r", ALPINE, "test", "-r", "/dev/kmsg1"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(session.OutputToString()).To(Equal("/dev/kmsg1")) }) It("podman run device rename and bad permission test", func() { - session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:/dev/kmsg1:rd", ALPINE, "ls", "--color=never", "/dev/kmsg1"}) + session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "/dev/kmsg:/dev/kmsg1:rd", ALPINE, "true"}) session.WaitWithDefaultTimeout() - Expect(session).To(ExitWithError()) + Expect(session).Should(Exit(125)) }) It("podman run device host device and container device parameter are directories", func() { @@ -89,12 +88,13 @@ var _ = Describe("Podman run device", func() { }) It("podman run device host device with --privileged", func() { - if _, err := os.Stat("/dev/kvm"); err != nil { - Skip("/dev/kvm not available") - } - session := podmanTest.Podman([]string{"run", "--privileged", ALPINE, "ls", "/dev/kvm"}) + session := podmanTest.Podman([]string{"run", "--privileged", ALPINE, "test", "-c", "/dev/kmsg"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) + // verify --privileged is required + session2 := podmanTest.Podman([]string{"run", ALPINE, "test", "-c", "/dev/kmsg"}) + session2.WaitWithDefaultTimeout() + Expect(session2).Should((Exit(1))) }) It("podman run CDI device test", func() { @@ -109,14 +109,13 @@ var _ = Describe("Podman run device", func() { err = cmd.Run() Expect(err).To(BeNil()) - session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "myKmsg", ALPINE, "ls", "--color=never", "/dev/kmsg1"}) + session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "myKmsg", ALPINE, "test", "-c", "/dev/kmsg1"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(session.OutputToString()).To(Equal("/dev/kmsg1")) }) It("podman run --gpus noop", func() { - session := podmanTest.Podman([]string{"run", "--gpus", "all", ALPINE, "ls", "/"}) + session := podmanTest.Podman([]string{"run", "--gpus", "all", ALPINE, "true"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) }) diff --git a/test/python/requirements.txt b/test/python/requirements.txt index ee85bf1d1..f177f76fc 100644 --- a/test/python/requirements.txt +++ b/test/python/requirements.txt @@ -1,5 +1,5 @@ docker~=4.4.3 - +requests-mock~=1.9.3 requests~=2.20.0 setuptools~=50.3.2 python-dateutil~=2.8.1 diff --git a/test/system/005-info.bats b/test/system/005-info.bats index 50c3ceb30..96ca2c1bd 100644 --- a/test/system/005-info.bats +++ b/test/system/005-info.bats @@ -47,6 +47,7 @@ store.configFile | $expr_path store.graphDriverName | [a-z0-9]\\\+\\\$ store.graphRoot | $expr_path store.imageStore.number | 1 +host.slirp4netns.executable | $expr_path " parse_table "$tests" | while read field expect; do diff --git a/test/system/070-build.bats b/test/system/070-build.bats index 26113e45c..0f58b2784 100644 --- a/test/system/070-build.bats +++ b/test/system/070-build.bats @@ -29,6 +29,27 @@ EOF run_podman rmi -f build_test } +@test "podman buildx - basic test" { + rand_filename=$(random_string 20) + rand_content=$(random_string 50) + + tmpdir=$PODMAN_TMPDIR/build-test + mkdir -p $tmpdir + dockerfile=$tmpdir/Dockerfile + cat >$dockerfile <<EOF +FROM $IMAGE +RUN echo $rand_content > /$rand_filename +EOF + + run_podman buildx build --load -t build_test --format=docker $tmpdir + is "$output" ".*COMMIT" "COMMIT seen in log" + + run_podman run --rm build_test cat /$rand_filename + is "$output" "$rand_content" "reading generated file in image" + + run_podman rmi -f build_test +} + @test "podman build test -f -" { rand_filename=$(random_string 20) rand_content=$(random_string 50) diff --git a/test/system/255-auto-update.bats b/test/system/255-auto-update.bats index 4e242e1f1..8bb32b5b7 100644 --- a/test/system/255-auto-update.bats +++ b/test/system/255-auto-update.bats @@ -102,7 +102,7 @@ function _wait_service_ready() { let timeout=$timeout-1 done - # Print serivce status as debug information before failed the case + # Print service status as debug information before failed the case systemctl status $sname die "Timed out waiting for $sname to start" } @@ -221,6 +221,7 @@ function _confirm_update() { } @test "podman auto-update - label io.containers.autoupdate=local with rollback" { + skip "This test flakes way too often, see #11175" # sdnotify fails with runc 1.0.0-3-dev2 on Ubuntu. Let's just # assume that we work only with crun, nothing else. # [copied from 260-sdnotify.bats] @@ -304,7 +305,7 @@ EOF fi done - # Only check the last service is started. Previous services should already actived. + # Only check that the last service is started. Previous services should already be activated. _wait_service_ready container-$cname.service run_podman commit --change CMD=/bin/bash $local_cname quay.io/libpod/localtest:latest # Exit code is expected, due to invalid 'fakevalue' diff --git a/test/system/500-networking.bats b/test/system/500-networking.bats index 6ffee7eaf..3ebe45e63 100644 --- a/test/system/500-networking.bats +++ b/test/system/500-networking.bats @@ -210,6 +210,9 @@ load helpers $IMAGE nc -l -n -v -p $myport cid="$output" + # check that dns is working inside the container + run_podman exec $cid nslookup google.com + # emit random string, and check it teststring=$(random_string 30) echo "$teststring" | nc 127.0.0.1 $myport diff --git a/test/system/700-play.bats b/test/system/700-play.bats index 3e6961b08..498956b9a 100644 --- a/test/system/700-play.bats +++ b/test/system/700-play.bats @@ -94,9 +94,9 @@ RELABEL="system_u:object_r:container_file_t:s0" mkdir -p $TESTDIR echo "$testYaml" | sed "s|TESTDIR|${TESTDIR}|g" > $PODMAN_TMPDIR/test.yaml run_podman 125 play kube --network bridge $PODMAN_TMPDIR/test.yaml - is "$output" ".*invalid value passed to --network: bridge or host networking must be configured in YAML" "podman plan-network should fail wth --network host" + is "$output" ".*invalid value passed to --network: bridge or host networking must be configured in YAML" "podman plan-network should fail with --network host" run_podman 125 play kube --network host $PODMAN_TMPDIR/test.yaml - is "$output" ".*invalid value passed to --network: bridge or host networking must be configured in YAML" "podman plan-network should fail wth --network host" + is "$output" ".*invalid value passed to --network: bridge or host networking must be configured in YAML" "podman plan-network should fail with --network host" run_podman play kube --network slirp4netns:port_handler=slirp4netns $PODMAN_TMPDIR/test.yaml run_podman pod rm -f test_pod } diff --git a/vendor/github.com/containers/storage/.cirrus.yml b/vendor/github.com/containers/storage/.cirrus.yml index 12f6f10c6..20bede452 100644 --- a/vendor/github.com/containers/storage/.cirrus.yml +++ b/vendor/github.com/containers/storage/.cirrus.yml @@ -25,7 +25,7 @@ env: # GCE project where images live IMAGE_PROJECT: "libpod-218412" # VM Image built in containers/automation_images - _BUILT_IMAGE_SUFFIX: "c6032583541653504" + _BUILT_IMAGE_SUFFIX: "c6248193773010944" FEDORA_CACHE_IMAGE_NAME: "fedora-${_BUILT_IMAGE_SUFFIX}" PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${_BUILT_IMAGE_SUFFIX}" UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${_BUILT_IMAGE_SUFFIX}" diff --git a/vendor/github.com/containers/storage/Makefile b/vendor/github.com/containers/storage/Makefile index 581961fed..dbc1f7c99 100644 --- a/vendor/github.com/containers/storage/Makefile +++ b/vendor/github.com/containers/storage/Makefile @@ -29,7 +29,7 @@ GIT_BRANCH := $(shell git rev-parse --abbrev-ref HEAD 2>/dev/null) GIT_BRANCH_CLEAN := $(shell echo $(GIT_BRANCH) | sed -e "s/[^[:alnum:]]/-/g") EPOCH_TEST_COMMIT := 0418ebf59f9e1f564831c0ba9378b7f8e40a1c73 NATIVETAGS := -AUTOTAGS := $(shell ./hack/btrfs_tag.sh) $(shell ./hack/libdm_tag.sh) +AUTOTAGS := $(shell ./hack/btrfs_tag.sh) $(shell ./hack/libdm_tag.sh) $(shell ./hack/libsubid_tag.sh) BUILDFLAGS := -tags "$(AUTOTAGS) $(TAGS)" $(FLAGS) GO ?= go TESTFLAGS := $(shell go test -race $(BUILDFLAGS) ./pkg/stringutils 2>&1 > /dev/null && echo -race) @@ -108,7 +108,7 @@ install.docs: docs install: install.docs lint: install.tools - tests/tools/build/golangci-lint run + tests/tools/build/golangci-lint run --build-tags="$(AUTOTAGS) $(TAGS)" help: ## this help @awk 'BEGIN {FS = ":.*?## "} /^[a-z A-Z_-]+:.*?## / {gsub(" ",",",$$1);gsub("\\\\n",sprintf("\n%22c"," "), $$2);printf "\033[36m%-21s\033[0m %s\n", $$1, $$2}' $(MAKEFILE_LIST) diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index 02261bead..2b17ffd50 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.33.1 +1.34.0 diff --git a/vendor/github.com/containers/storage/drivers/overlay/check_115.go b/vendor/github.com/containers/storage/drivers/overlay/check_115.go new file mode 100644 index 000000000..9ad1b863d --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/overlay/check_115.go @@ -0,0 +1,42 @@ +// +build !go1.16 + +package overlay + +import ( + "os" + "path/filepath" + "strings" + + "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/system" +) + +func scanForMountProgramIndicators(home string) (detected bool, err error) { + err = filepath.Walk(home, func(path string, info os.FileInfo, err error) error { + if detected { + return filepath.SkipDir + } + if err != nil { + return err + } + basename := filepath.Base(path) + if strings.HasPrefix(basename, archive.WhiteoutPrefix) { + detected = true + return filepath.SkipDir + } + if info.IsDir() { + xattrs, err := system.Llistxattr(path) + if err != nil { + return err + } + for _, xattr := range xattrs { + if strings.HasPrefix(xattr, "user.fuseoverlayfs.") || strings.HasPrefix(xattr, "user.containers.") { + detected = true + return filepath.SkipDir + } + } + } + return nil + }) + return detected, err +} diff --git a/vendor/github.com/containers/storage/drivers/overlay/check_116.go b/vendor/github.com/containers/storage/drivers/overlay/check_116.go new file mode 100644 index 000000000..6d7913cbf --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/overlay/check_116.go @@ -0,0 +1,42 @@ +// +build go1.16 + +package overlay + +import ( + "io/fs" + "path/filepath" + "strings" + + "github.com/containers/storage/pkg/archive" + "github.com/containers/storage/pkg/system" +) + +func scanForMountProgramIndicators(home string) (detected bool, err error) { + err = filepath.WalkDir(home, func(path string, d fs.DirEntry, err error) error { + if detected { + return fs.SkipDir + } + if err != nil { + return err + } + basename := filepath.Base(path) + if strings.HasPrefix(basename, archive.WhiteoutPrefix) { + detected = true + return fs.SkipDir + } + if d.IsDir() { + xattrs, err := system.Llistxattr(path) + if err != nil { + return err + } + for _, xattr := range xattrs { + if strings.HasPrefix(xattr, "user.fuseoverlayfs.") || strings.HasPrefix(xattr, "user.containers.") { + detected = true + return fs.SkipDir + } + } + } + return nil + }) + return detected, err +} diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index ecfbae916..abb9ab71d 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -266,9 +266,8 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) } if opts.mountProgram != "" { - f, err := os.Create(getMountProgramFlagFile(home)) - if err == nil { - f.Close() + if err := ioutil.WriteFile(getMountProgramFlagFile(home), []byte("true"), 0600); err != nil { + return nil, err } } else { // check if they are running over btrfs, aufs, zfs, overlay, or ecryptfs @@ -542,9 +541,29 @@ func SupportsNativeOverlay(graphroot, rundir string) (bool, error) { home := filepath.Join(graphroot, "overlay") runhome := filepath.Join(rundir, "overlay") - if _, err := os.Stat(getMountProgramFlagFile(home)); err == nil { + var contents string + flagContent, err := ioutil.ReadFile(getMountProgramFlagFile(home)) + if err == nil { + contents = strings.TrimSpace(string(flagContent)) + } + switch contents { + case "true": logrus.Debugf("overlay storage already configured with a mount-program") return false, nil + default: + needsMountProgram, err := scanForMountProgramIndicators(home) + if err != nil && !os.IsNotExist(err) { + return false, err + } + if err := ioutil.WriteFile(getMountProgramFlagFile(home), []byte(fmt.Sprintf("%t", needsMountProgram)), 0600); err != nil && !os.IsNotExist(err) { + return false, err + } + if needsMountProgram { + return false, nil + } + // fall through to check if we find ourselves needing to use a + // mount program now + case "false": } for _, dir := range []string{home, runhome} { diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index e4f484d6b..ff14799a2 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -3,14 +3,14 @@ go 1.14 module github.com/containers/storage require ( - github.com/BurntSushi/toml v0.3.1 + github.com/BurntSushi/toml v0.4.1 github.com/Microsoft/go-winio v0.5.0 github.com/Microsoft/hcsshim v0.8.20 github.com/docker/go-units v0.4.0 github.com/google/go-intervals v0.0.2 github.com/hashicorp/go-multierror v1.1.1 github.com/json-iterator/go v1.1.11 - github.com/klauspost/compress v1.13.1 + github.com/klauspost/compress v1.13.3 github.com/klauspost/pgzip v1.2.5 github.com/mattn/go-shellwords v1.0.12 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible @@ -18,7 +18,7 @@ require ( github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/runc v1.0.1 github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 - github.com/opencontainers/selinux v1.8.2 + github.com/opencontainers/selinux v1.8.3 github.com/pkg/errors v0.9.1 github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.7.0 diff --git a/vendor/github.com/containers/storage/go.sum b/vendor/github.com/containers/storage/go.sum index 2607dbc9b..71ababfb2 100644 --- a/vendor/github.com/containers/storage/go.sum +++ b/vendor/github.com/containers/storage/go.sum @@ -34,8 +34,9 @@ github.com/Azure/go-autorest/autorest/mocks v0.4.0/go.mod h1:LTp+uSrOhSkaKrUy935 github.com/Azure/go-autorest/autorest/mocks v0.4.1/go.mod h1:LTp+uSrOhSkaKrUy935gNZuuIPPVsHlr9DSOxSayd+k= github.com/Azure/go-autorest/logger v0.2.0/go.mod h1:T9E3cAhj2VqvPOtCYAvby9aBXkZmbF5NWuPV8+WeEW8= github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= +github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= +github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= @@ -387,8 +388,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.13.1 h1:wXr2uRxZTJXHLly6qhJabee5JqIhTRoLBhDOA74hDEQ= -github.com/klauspost/compress v1.13.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= +github.com/klauspost/compress v1.13.3 h1:BtAvtV1+h0YwSVwWoYXMREPpYu9VzTJ9QDI1TEg/iQQ= +github.com/klauspost/compress v1.13.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -480,8 +481,9 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.m github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/selinux v1.6.0/go.mod h1:VVGKuOLlE7v4PJyT6h7mNWvq1rzqiriPsEqVhc+svHE= github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= -github.com/opencontainers/selinux v1.8.2 h1:c4ca10UMgRcvZ6h0K4HtS15UaVSBEaE+iln2LVpAuGc= github.com/opencontainers/selinux v1.8.2/go.mod h1:MUIHuUEvKB1wtJjQdOyYRgOnLD2xAPP8dBsCoU0KuF8= +github.com/opencontainers/selinux v1.8.3 h1:tzZR7AuKB5gU1+53uBkoG4XdIFGZzvJTOVoNbRQI8/4= +github.com/opencontainers/selinux v1.8.3/go.mod h1:HTvjPFoGMbpQsG886e3lQwnsRWtE4TC1OF3OUvG9FAo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.8.1/go.mod h1:T2/BmBdy8dvIRq1a/8aqjN41wvWlN4lrapLU/GW4pbc= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools.go b/vendor/github.com/containers/storage/pkg/idtools/idtools.go index 34345d145..83bc8c34f 100644 --- a/vendor/github.com/containers/storage/pkg/idtools/idtools.go +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools.go @@ -146,11 +146,11 @@ type IDMappings struct { // using the data from /etc/sub{uid,gid} ranges, creates the // proper uid and gid remapping ranges for that user/group pair func NewIDMappings(username, groupname string) (*IDMappings, error) { - subuidRanges, err := parseSubuid(username) + subuidRanges, err := readSubuid(username) if err != nil { return nil, err } - subgidRanges, err := parseSubgid(groupname) + subgidRanges, err := readSubgid(groupname) if err != nil { return nil, err } @@ -244,14 +244,6 @@ func createIDMap(subidRanges ranges) []IDMap { return idMap } -func parseSubuid(username string) (ranges, error) { - return parseSubidFile(subuidFileName, username) -} - -func parseSubgid(username string) (ranges, error) { - return parseSubidFile(subgidFileName, username) -} - // parseSubidFile will read the appropriate file (/etc/subuid or /etc/subgid) // and return all found ranges for a specified username. If the special value // "ALL" is supplied for username, then all ranges in the file will be returned diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools_supported.go b/vendor/github.com/containers/storage/pkg/idtools/idtools_supported.go new file mode 100644 index 000000000..db50a62e4 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools_supported.go @@ -0,0 +1,61 @@ +// +build linux,cgo,libsubid + +package idtools + +import ( + "unsafe" + + "github.com/pkg/errors" +) + +/* +#cgo LDFLAGS: -l subid +#include <shadow/subid.h> +#include <stdlib.h> +const char *Prog = "storage"; +struct subid_range get_range(struct subid_range *ranges, int i) +{ + return ranges[i]; +} +*/ +import "C" + +func readSubid(username string, isUser bool) (ranges, error) { + var ret ranges + if username == "ALL" { + return nil, errors.New("username ALL not supported") + } + + cUsername := C.CString(username) + defer C.free(unsafe.Pointer(cUsername)) + + var nRanges C.int + var cRanges *C.struct_subid_range + if isUser { + nRanges = C.get_subuid_ranges(cUsername, &cRanges) + } else { + nRanges = C.get_subgid_ranges(cUsername, &cRanges) + } + if nRanges < 0 { + return nil, errors.New("cannot read subids") + } + defer C.free(unsafe.Pointer(cRanges)) + + for i := 0; i < int(nRanges); i++ { + r := C.get_range(cRanges, C.int(i)) + newRange := subIDRange{ + Start: int(r.start), + Length: int(r.count), + } + ret = append(ret, newRange) + } + return ret, nil +} + +func readSubuid(username string) (ranges, error) { + return readSubid(username, true) +} + +func readSubgid(username string) (ranges, error) { + return readSubid(username, false) +} diff --git a/vendor/github.com/containers/storage/pkg/idtools/idtools_unsupported.go b/vendor/github.com/containers/storage/pkg/idtools/idtools_unsupported.go new file mode 100644 index 000000000..84da1b764 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/idtools/idtools_unsupported.go @@ -0,0 +1,11 @@ +// +build !linux !libsubid !cgo + +package idtools + +func readSubuid(username string) (ranges, error) { + return parseSubidFile(subuidFileName, username) +} + +func readSubgid(username string) (ranges, error) { + return parseSubidFile(subgidFileName, username) +} diff --git a/vendor/github.com/containers/storage/pkg/idtools/usergroupadd_linux.go b/vendor/github.com/containers/storage/pkg/idtools/usergroupadd_linux.go index 9da7975e2..3dd7bf210 100644 --- a/vendor/github.com/containers/storage/pkg/idtools/usergroupadd_linux.go +++ b/vendor/github.com/containers/storage/pkg/idtools/usergroupadd_linux.go @@ -91,7 +91,7 @@ func createSubordinateRanges(name string) error { // first, we should verify that ranges weren't automatically created // by the distro tooling - ranges, err := parseSubuid(name) + ranges, err := readSubuid(name) if err != nil { return fmt.Errorf("Error while looking for subuid ranges for user %q: %v", name, err) } @@ -107,7 +107,7 @@ func createSubordinateRanges(name string) error { } } - ranges, err = parseSubgid(name) + ranges, err = readSubgid(name) if err != nil { return fmt.Errorf("Error while looking for subgid ranges for user %q: %v", name, err) } @@ -126,7 +126,7 @@ func createSubordinateRanges(name string) error { } func findNextUIDRange() (int, error) { - ranges, err := parseSubuid("ALL") + ranges, err := readSubuid("ALL") if err != nil { return -1, fmt.Errorf("Couldn't parse all ranges in /etc/subuid file: %v", err) } @@ -135,7 +135,7 @@ func findNextUIDRange() (int, error) { } func findNextGIDRange() (int, error) { - ranges, err := parseSubgid("ALL") + ranges, err := readSubgid("ALL") if err != nil { return -1, fmt.Errorf("Couldn't parse all ranges in /etc/subgid file: %v", err) } diff --git a/vendor/github.com/containers/storage/types/utils.go b/vendor/github.com/containers/storage/types/utils.go index 4d62b151a..5dbbb4403 100644 --- a/vendor/github.com/containers/storage/types/utils.go +++ b/vendor/github.com/containers/storage/types/utils.go @@ -155,8 +155,17 @@ func getRootlessUID() int { } func expandEnvPath(path string, rootlessUID int) (string, error) { + var err error path = strings.Replace(path, "$UID", strconv.Itoa(rootlessUID), -1) - return filepath.Clean(os.ExpandEnv(path)), nil + path = os.ExpandEnv(path) + newpath, err := filepath.EvalSymlinks(path) + if err != nil { + if !os.IsNotExist(err) { + return "", err + } + newpath = filepath.Clean(path) + } + return newpath, nil } func DefaultConfigFile(rootless bool) (string, error) { diff --git a/vendor/github.com/klauspost/compress/.gitattributes b/vendor/github.com/klauspost/compress/.gitattributes new file mode 100644 index 000000000..402433593 --- /dev/null +++ b/vendor/github.com/klauspost/compress/.gitattributes @@ -0,0 +1,2 @@ +* -text +*.bin -text -diff diff --git a/vendor/github.com/klauspost/compress/.gitignore b/vendor/github.com/klauspost/compress/.gitignore new file mode 100644 index 000000000..b35f8449b --- /dev/null +++ b/vendor/github.com/klauspost/compress/.gitignore @@ -0,0 +1,25 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test +*.prof +/s2/cmd/_s2sx/sfx-exe diff --git a/vendor/github.com/klauspost/compress/.goreleaser.yml b/vendor/github.com/klauspost/compress/.goreleaser.yml new file mode 100644 index 000000000..c9014ce1d --- /dev/null +++ b/vendor/github.com/klauspost/compress/.goreleaser.yml @@ -0,0 +1,137 @@ +# This is an example goreleaser.yaml file with some sane defaults. +# Make sure to check the documentation at http://goreleaser.com +before: + hooks: + - ./gen.sh + +builds: + - + id: "s2c" + binary: s2c + main: ./s2/cmd/s2c/main.go + flags: + - -trimpath + env: + - CGO_ENABLED=0 + goos: + - aix + - linux + - freebsd + - netbsd + - windows + - darwin + goarch: + - 386 + - amd64 + - arm + - arm64 + - ppc64 + - ppc64le + - mips64 + - mips64le + goarm: + - 7 + - + id: "s2d" + binary: s2d + main: ./s2/cmd/s2d/main.go + flags: + - -trimpath + env: + - CGO_ENABLED=0 + goos: + - aix + - linux + - freebsd + - netbsd + - windows + - darwin + goarch: + - 386 + - amd64 + - arm + - arm64 + - ppc64 + - ppc64le + - mips64 + - mips64le + goarm: + - 7 + - + id: "s2sx" + binary: s2sx + main: ./s2/cmd/_s2sx/main.go + flags: + - -modfile=s2sx.mod + - -trimpath + env: + - CGO_ENABLED=0 + goos: + - aix + - linux + - freebsd + - netbsd + - windows + - darwin + goarch: + - 386 + - amd64 + - arm + - arm64 + - ppc64 + - ppc64le + - mips64 + - mips64le + goarm: + - 7 + +archives: + - + id: s2-binaries + name_template: "s2-{{ .Os }}_{{ .Arch }}_{{ .Version }}" + replacements: + aix: AIX + darwin: OSX + linux: Linux + windows: Windows + 386: i386 + amd64: x86_64 + freebsd: FreeBSD + netbsd: NetBSD + format_overrides: + - goos: windows + format: zip + files: + - unpack/* + - s2/LICENSE + - s2/README.md +checksum: + name_template: 'checksums.txt' +snapshot: + name_template: "{{ .Tag }}-next" +changelog: + sort: asc + filters: + exclude: + - '^doc:' + - '^docs:' + - '^test:' + - '^tests:' + - '^Update\sREADME.md' + +nfpms: + - + file_name_template: "s2_package_{{ .Version }}_{{ .Os }}_{{ .Arch }}" + vendor: Klaus Post + homepage: https://github.com/klauspost/compress + maintainer: Klaus Post <klauspost@gmail.com> + description: S2 Compression Tool + license: BSD 3-Clause + formats: + - deb + - rpm + replacements: + darwin: Darwin + linux: Linux + freebsd: FreeBSD + amd64: x86_64 diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md new file mode 100644 index 000000000..48851e0ce --- /dev/null +++ b/vendor/github.com/klauspost/compress/README.md @@ -0,0 +1,416 @@ +# compress
+
+This package provides various compression algorithms.
+
+* [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and decompression in pure Go.
+* [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) is a high performance replacement for Snappy.
+* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib).
+* [huff0](https://github.com/klauspost/compress/tree/master/huff0) and [FSE](https://github.com/klauspost/compress/tree/master/fse) implementations for raw entropy encoding.
+* [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp) Provides client and server wrappers for handling gzipped requests efficiently.
+* [pgzip](https://github.com/klauspost/pgzip) is a separate package that provides a very fast parallel gzip implementation.
+* [fuzz package](https://github.com/klauspost/compress-fuzz) for fuzz testing all compressors/decompressors here.
+
+[](https://pkg.go.dev/github.com/klauspost/compress?tab=subdirectories)
+[](https://github.com/klauspost/compress/actions/workflows/go.yml)
+[](https://sourcegraph.com/github.com/klauspost/compress?badge)
+
+# changelog
+
+* Jun 14, 2021 (v1.13.1)
+
+ * s2: Add full Snappy output support [#396](https://github.com/klauspost/compress/pull/396)
+ * zstd: Add configurable [Decoder window](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithDecoderMaxWindow) size [#394](https://github.com/klauspost/compress/pull/394)
+ * gzhttp: Add header to skip compression [#389](https://github.com/klauspost/compress/pull/389)
+ * s2: Improve speed with bigger output margin [#395](https://github.com/klauspost/compress/pull/395)
+
+* Jun 3, 2021 (v1.13.0)
+ * Added [gzhttp](https://github.com/klauspost/compress/tree/master/gzhttp#gzip-handler) which allows wrapping HTTP servers and clients with GZIP compressors.
+ * zstd: Detect short invalid signatures [#382](https://github.com/klauspost/compress/pull/382)
+ * zstd: Spawn decoder goroutine only if needed. [#380](https://github.com/klauspost/compress/pull/380)
+
+* May 25, 2021 (v1.12.3)
+ * deflate: Better/faster Huffman encoding [#374](https://github.com/klauspost/compress/pull/374)
+ * deflate: Allocate less for history. [#375](https://github.com/klauspost/compress/pull/375)
+ * zstd: Forward read errors [#373](https://github.com/klauspost/compress/pull/373)
+
+* Apr 27, 2021 (v1.12.2)
+ * zstd: Improve better/best compression [#360](https://github.com/klauspost/compress/pull/360) [#364](https://github.com/klauspost/compress/pull/364) [#365](https://github.com/klauspost/compress/pull/365)
+ * zstd: Add helpers to compress/decompress zstd inside zip files [#363](https://github.com/klauspost/compress/pull/363)
+ * deflate: Improve level 5+6 compression [#367](https://github.com/klauspost/compress/pull/367)
+ * s2: Improve better/best compression [#358](https://github.com/klauspost/compress/pull/358) [#359](https://github.com/klauspost/compress/pull/358)
+ * s2: Load after checking src limit on amd64. [#362](https://github.com/klauspost/compress/pull/362)
+ * s2sx: Limit max executable size [#368](https://github.com/klauspost/compress/pull/368)
+
+* Apr 14, 2021 (v1.12.1)
+ * snappy package removed. Upstream added as dependency.
+ * s2: Better compression in "best" mode [#353](https://github.com/klauspost/compress/pull/353)
+ * s2sx: Add stdin input and detect pre-compressed from signature [#352](https://github.com/klauspost/compress/pull/352)
+ * s2c/s2d: Add http as possible input [#348](https://github.com/klauspost/compress/pull/348)
+ * s2c/s2d/s2sx: Always truncate when writing files [#352](https://github.com/klauspost/compress/pull/352)
+ * zstd: Reduce memory usage further when using [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) [#346](https://github.com/klauspost/compress/pull/346)
+ * s2: Fix potential problem with amd64 assembly and profilers [#349](https://github.com/klauspost/compress/pull/349)
+
+* Mar 26, 2021 (v1.11.13)
+ * zstd: Big speedup on small dictionary encodes [#344](https://github.com/klauspost/compress/pull/344) [#345](https://github.com/klauspost/compress/pull/345)
+ * zstd: Add [WithLowerEncoderMem](https://pkg.go.dev/github.com/klauspost/compress/zstd#WithLowerEncoderMem) encoder option [#336](https://github.com/klauspost/compress/pull/336)
+ * deflate: Improve entropy compression [#338](https://github.com/klauspost/compress/pull/338)
+ * s2: Clean up and minor performance improvement in best [#341](https://github.com/klauspost/compress/pull/341)
+
+* Mar 5, 2021 (v1.11.12)
+ * s2: Add `s2sx` binary that creates [self extracting archives](https://github.com/klauspost/compress/tree/master/s2#s2sx-self-extracting-archives).
+ * s2: Speed up decompression on non-assembly platforms [#328](https://github.com/klauspost/compress/pull/328)
+
+* Mar 1, 2021 (v1.11.9)
+ * s2: Add ARM64 decompression assembly. Around 2x output speed. [#324](https://github.com/klauspost/compress/pull/324)
+ * s2: Improve "better" speed and efficiency. [#325](https://github.com/klauspost/compress/pull/325)
+ * s2: Fix binaries.
+
+* Feb 25, 2021 (v1.11.8)
+ * s2: Fixed occational out-of-bounds write on amd64. Upgrade recommended.
+ * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315)
+ * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322)
+ * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314)
+ * zip: Fix zip64 headers. [#313](https://github.com/klauspost/compress/pull/313)
+
+* Jan 14, 2021 (v1.11.7)
+ * Use Bytes() interface to get bytes across packages. [#309](https://github.com/klauspost/compress/pull/309)
+ * s2: Add 'best' compression option. [#310](https://github.com/klauspost/compress/pull/310)
+ * s2: Add ReaderMaxBlockSize, changes `s2.NewReader` signature to include varargs. [#311](https://github.com/klauspost/compress/pull/311)
+ * s2: Fix crash on small better buffers. [#308](https://github.com/klauspost/compress/pull/308)
+ * s2: Clean up decoder. [#312](https://github.com/klauspost/compress/pull/312)
+
+* Jan 7, 2021 (v1.11.6)
+ * zstd: Make decoder allocations smaller [#306](https://github.com/klauspost/compress/pull/306)
+ * zstd: Free Decoder resources when Reset is called with a nil io.Reader [#305](https://github.com/klauspost/compress/pull/305)
+
+* Dec 20, 2020 (v1.11.4)
+ * zstd: Add Best compression mode [#304](https://github.com/klauspost/compress/pull/304)
+ * Add header decoder [#299](https://github.com/klauspost/compress/pull/299)
+ * s2: Add uncompressed stream option [#297](https://github.com/klauspost/compress/pull/297)
+ * Simplify/speed up small blocks with known max size. [#300](https://github.com/klauspost/compress/pull/300)
+ * zstd: Always reset literal dict encoder [#303](https://github.com/klauspost/compress/pull/303)
+
+* Nov 15, 2020 (v1.11.3)
+ * inflate: 10-15% faster decompression [#293](https://github.com/klauspost/compress/pull/293)
+ * zstd: Tweak DecodeAll default allocation [#295](https://github.com/klauspost/compress/pull/295)
+
+* Oct 11, 2020 (v1.11.2)
+ * s2: Fix out of bounds read in "better" block compression [#291](https://github.com/klauspost/compress/pull/291)
+
+* Oct 1, 2020 (v1.11.1)
+ * zstd: Set allLitEntropy true in default configuration [#286](https://github.com/klauspost/compress/pull/286)
+
+* Sept 8, 2020 (v1.11.0)
+ * zstd: Add experimental compression [dictionaries](https://github.com/klauspost/compress/tree/master/zstd#dictionaries) [#281](https://github.com/klauspost/compress/pull/281)
+ * zstd: Fix mixed Write and ReadFrom calls [#282](https://github.com/klauspost/compress/pull/282)
+ * inflate/gz: Limit variable shifts, ~5% faster decompression [#274](https://github.com/klauspost/compress/pull/274)
+
+<details>
+ <summary>See changes prior to v1.11.0</summary>
+
+* July 8, 2020 (v1.10.11)
+ * zstd: Fix extra block when compressing with ReadFrom. [#278](https://github.com/klauspost/compress/pull/278)
+ * huff0: Also populate compression table when reading decoding table. [#275](https://github.com/klauspost/compress/pull/275)
+
+* June 23, 2020 (v1.10.10)
+ * zstd: Skip entropy compression in fastest mode when no matches. [#270](https://github.com/klauspost/compress/pull/270)
+
+* June 16, 2020 (v1.10.9):
+ * zstd: API change for specifying dictionaries. See [#268](https://github.com/klauspost/compress/pull/268)
+ * zip: update CreateHeaderRaw to handle zip64 fields. [#266](https://github.com/klauspost/compress/pull/266)
+ * Fuzzit tests removed. The service has been purchased and is no longer available.
+
+* June 5, 2020 (v1.10.8):
+ * 1.15x faster zstd block decompression. [#265](https://github.com/klauspost/compress/pull/265)
+
+* June 1, 2020 (v1.10.7):
+ * Added zstd decompression [dictionary support](https://github.com/klauspost/compress/tree/master/zstd#dictionaries)
+ * Increase zstd decompression speed up to 1.19x. [#259](https://github.com/klauspost/compress/pull/259)
+ * Remove internal reset call in zstd compression and reduce allocations. [#263](https://github.com/klauspost/compress/pull/263)
+
+* May 21, 2020: (v1.10.6)
+ * zstd: Reduce allocations while decoding. [#258](https://github.com/klauspost/compress/pull/258), [#252](https://github.com/klauspost/compress/pull/252)
+ * zstd: Stricter decompression checks.
+
+* April 12, 2020: (v1.10.5)
+ * s2-commands: Flush output when receiving SIGINT. [#239](https://github.com/klauspost/compress/pull/239)
+
+* Apr 8, 2020: (v1.10.4)
+ * zstd: Minor/special case optimizations. [#251](https://github.com/klauspost/compress/pull/251), [#250](https://github.com/klauspost/compress/pull/250), [#249](https://github.com/klauspost/compress/pull/249), [#247](https://github.com/klauspost/compress/pull/247)
+* Mar 11, 2020: (v1.10.3)
+ * s2: Use S2 encoder in pure Go mode for Snappy output as well. [#245](https://github.com/klauspost/compress/pull/245)
+ * s2: Fix pure Go block encoder. [#244](https://github.com/klauspost/compress/pull/244)
+ * zstd: Added "better compression" mode. [#240](https://github.com/klauspost/compress/pull/240)
+ * zstd: Improve speed of fastest compression mode by 5-10% [#241](https://github.com/klauspost/compress/pull/241)
+ * zstd: Skip creating encoders when not needed. [#238](https://github.com/klauspost/compress/pull/238)
+
+* Feb 27, 2020: (v1.10.2)
+ * Close to 50% speedup in inflate (gzip/zip decompression). [#236](https://github.com/klauspost/compress/pull/236) [#234](https://github.com/klauspost/compress/pull/234) [#232](https://github.com/klauspost/compress/pull/232)
+ * Reduce deflate level 1-6 memory usage up to 59%. [#227](https://github.com/klauspost/compress/pull/227)
+
+* Feb 18, 2020: (v1.10.1)
+ * Fix zstd crash when resetting multiple times without sending data. [#226](https://github.com/klauspost/compress/pull/226)
+ * deflate: Fix dictionary use on level 1-6. [#224](https://github.com/klauspost/compress/pull/224)
+ * Remove deflate writer reference when closing. [#224](https://github.com/klauspost/compress/pull/224)
+
+* Feb 4, 2020: (v1.10.0)
+ * Add optional dictionary to [stateless deflate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc#StatelessDeflate). Breaking change, send `nil` for previous behaviour. [#216](https://github.com/klauspost/compress/pull/216)
+ * Fix buffer overflow on repeated small block deflate. [#218](https://github.com/klauspost/compress/pull/218)
+ * Allow copying content from an existing ZIP file without decompressing+compressing. [#214](https://github.com/klauspost/compress/pull/214)
+ * Added [S2](https://github.com/klauspost/compress/tree/master/s2#s2-compression) AMD64 assembler and various optimizations. Stream speed >10GB/s. [#186](https://github.com/klauspost/compress/pull/186)
+
+</details>
+
+<details>
+ <summary>See changes prior to v1.10.0</summary>
+
+* Jan 20,2020 (v1.9.8) Optimize gzip/deflate with better size estimates and faster table generation. [#207](https://github.com/klauspost/compress/pull/207) by [luyu6056](https://github.com/luyu6056), [#206](https://github.com/klauspost/compress/pull/206).
+* Jan 11, 2020: S2 Encode/Decode will use provided buffer if capacity is big enough. [#204](https://github.com/klauspost/compress/pull/204)
+* Jan 5, 2020: (v1.9.7) Fix another zstd regression in v1.9.5 - v1.9.6 removed.
+* Jan 4, 2020: (v1.9.6) Regression in v1.9.5 fixed causing corrupt zstd encodes in rare cases.
+* Jan 4, 2020: Faster IO in [s2c + s2d commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) compression/decompression. [#192](https://github.com/klauspost/compress/pull/192)
+* Dec 29, 2019: Removed v1.9.5 since fuzz tests showed a compatibility problem with the reference zstandard decoder.
+* Dec 29, 2019: (v1.9.5) zstd: 10-20% faster block compression. [#199](https://github.com/klauspost/compress/pull/199)
+* Dec 29, 2019: [zip](https://godoc.org/github.com/klauspost/compress/zip) package updated with latest Go features
+* Dec 29, 2019: zstd: Single segment flag condintions tweaked. [#197](https://github.com/klauspost/compress/pull/197)
+* Dec 18, 2019: s2: Faster compression when ReadFrom is used. [#198](https://github.com/klauspost/compress/pull/198)
+* Dec 10, 2019: s2: Fix repeat length output when just above at 16MB limit.
+* Dec 10, 2019: zstd: Add function to get decoder as io.ReadCloser. [#191](https://github.com/klauspost/compress/pull/191)
+* Dec 3, 2019: (v1.9.4) S2: limit max repeat length. [#188](https://github.com/klauspost/compress/pull/188)
+* Dec 3, 2019: Add [WithNoEntropyCompression](https://godoc.org/github.com/klauspost/compress/zstd#WithNoEntropyCompression) to zstd [#187](https://github.com/klauspost/compress/pull/187)
+* Dec 3, 2019: Reduce memory use for tests. Check for leaked goroutines.
+* Nov 28, 2019 (v1.9.3) Less allocations in stateless deflate.
+* Nov 28, 2019: 5-20% Faster huff0 decode. Impacts zstd as well. [#184](https://github.com/klauspost/compress/pull/184)
+* Nov 12, 2019 (v1.9.2) Added [Stateless Compression](#stateless-compression) for gzip/deflate.
+* Nov 12, 2019: Fixed zstd decompression of large single blocks. [#180](https://github.com/klauspost/compress/pull/180)
+* Nov 11, 2019: Set default [s2c](https://github.com/klauspost/compress/tree/master/s2#commandline-tools) block size to 4MB.
+* Nov 11, 2019: Reduce inflate memory use by 1KB.
+* Nov 10, 2019: Less allocations in deflate bit writer.
+* Nov 10, 2019: Fix inconsistent error returned by zstd decoder.
+* Oct 28, 2019 (v1.9.1) ztsd: Fix crash when compressing blocks. [#174](https://github.com/klauspost/compress/pull/174)
+* Oct 24, 2019 (v1.9.0) zstd: Fix rare data corruption [#173](https://github.com/klauspost/compress/pull/173)
+* Oct 24, 2019 zstd: Fix huff0 out of buffer write [#171](https://github.com/klauspost/compress/pull/171) and always return errors [#172](https://github.com/klauspost/compress/pull/172)
+* Oct 10, 2019: Big deflate rewrite, 30-40% faster with better compression [#105](https://github.com/klauspost/compress/pull/105)
+
+</details>
+
+<details>
+ <summary>See changes prior to v1.9.0</summary>
+
+* Oct 10, 2019: (v1.8.6) zstd: Allow partial reads to get flushed data. [#169](https://github.com/klauspost/compress/pull/169)
+* Oct 3, 2019: Fix inconsistent results on broken zstd streams.
+* Sep 25, 2019: Added `-rm` (remove source files) and `-q` (no output except errors) to `s2c` and `s2d` [commands](https://github.com/klauspost/compress/tree/master/s2#commandline-tools)
+* Sep 16, 2019: (v1.8.4) Add `s2c` and `s2d` [commandline tools](https://github.com/klauspost/compress/tree/master/s2#commandline-tools).
+* Sep 10, 2019: (v1.8.3) Fix s2 decoder [Skip](https://godoc.org/github.com/klauspost/compress/s2#Reader.Skip).
+* Sep 7, 2019: zstd: Added [WithWindowSize](https://godoc.org/github.com/klauspost/compress/zstd#WithWindowSize), contributed by [ianwilkes](https://github.com/ianwilkes).
+* Sep 5, 2019: (v1.8.2) Add [WithZeroFrames](https://godoc.org/github.com/klauspost/compress/zstd#WithZeroFrames) which adds full zero payload block encoding option.
+* Sep 5, 2019: Lazy initialization of zstandard predefined en/decoder tables.
+* Aug 26, 2019: (v1.8.1) S2: 1-2% compression increase in "better" compression mode.
+* Aug 26, 2019: zstd: Check maximum size of Huffman 1X compressed literals while decoding.
+* Aug 24, 2019: (v1.8.0) Added [S2 compression](https://github.com/klauspost/compress/tree/master/s2#s2-compression), a high performance replacement for Snappy.
+* Aug 21, 2019: (v1.7.6) Fixed minor issues found by fuzzer. One could lead to zstd not decompressing.
+* Aug 18, 2019: Add [fuzzit](https://fuzzit.dev/) continuous fuzzing.
+* Aug 14, 2019: zstd: Skip incompressible data 2x faster. [#147](https://github.com/klauspost/compress/pull/147)
+* Aug 4, 2019 (v1.7.5): Better literal compression. [#146](https://github.com/klauspost/compress/pull/146)
+* Aug 4, 2019: Faster zstd compression. [#143](https://github.com/klauspost/compress/pull/143) [#144](https://github.com/klauspost/compress/pull/144)
+* Aug 4, 2019: Faster zstd decompression. [#145](https://github.com/klauspost/compress/pull/145) [#143](https://github.com/klauspost/compress/pull/143) [#142](https://github.com/klauspost/compress/pull/142)
+* July 15, 2019 (v1.7.4): Fix double EOF block in rare cases on zstd encoder.
+* July 15, 2019 (v1.7.3): Minor speedup/compression increase in default zstd encoder.
+* July 14, 2019: zstd decoder: Fix decompression error on multiple uses with mixed content.
+* July 7, 2019 (v1.7.2): Snappy update, zstd decoder potential race fix.
+* June 17, 2019: zstd decompression bugfix.
+* June 17, 2019: fix 32 bit builds.
+* June 17, 2019: Easier use in modules (less dependencies).
+* June 9, 2019: New stronger "default" [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression mode. Matches zstd default compression ratio.
+* June 5, 2019: 20-40% throughput in [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression and better compression.
+* June 5, 2019: deflate/gzip compression: Reduce memory usage of lower compression levels.
+* June 2, 2019: Added [zstandard](https://github.com/klauspost/compress/tree/master/zstd#zstd) compression!
+* May 25, 2019: deflate/gzip: 10% faster bit writer, mostly visible in lower levels.
+* Apr 22, 2019: [zstd](https://github.com/klauspost/compress/tree/master/zstd#zstd) decompression added.
+* Aug 1, 2018: Added [huff0 README](https://github.com/klauspost/compress/tree/master/huff0#huff0-entropy-compression).
+* Jul 8, 2018: Added [Performance Update 2018](#performance-update-2018) below.
+* Jun 23, 2018: Merged [Go 1.11 inflate optimizations](https://go-review.googlesource.com/c/go/+/102235). Go 1.9 is now required. Backwards compatible version tagged with [v1.3.0](https://github.com/klauspost/compress/releases/tag/v1.3.0).
+* Apr 2, 2018: Added [huff0](https://godoc.org/github.com/klauspost/compress/huff0) en/decoder. Experimental for now, API may change.
+* Mar 4, 2018: Added [FSE Entropy](https://godoc.org/github.com/klauspost/compress/fse) en/decoder. Experimental for now, API may change.
+* Nov 3, 2017: Add compression [Estimate](https://godoc.org/github.com/klauspost/compress#Estimate) function.
+* May 28, 2017: Reduce allocations when resetting decoder.
+* Apr 02, 2017: Change back to official crc32, since changes were merged in Go 1.7.
+* Jan 14, 2017: Reduce stack pressure due to array copies. See [Issue #18625](https://github.com/golang/go/issues/18625).
+* Oct 25, 2016: Level 2-4 have been rewritten and now offers significantly better performance than before.
+* Oct 20, 2016: Port zlib changes from Go 1.7 to fix zlib writer issue. Please update.
+* Oct 16, 2016: Go 1.7 changes merged. Apples to apples this package is a few percent faster, but has a significantly better balance between speed and compression per level.
+* Mar 24, 2016: Always attempt Huffman encoding on level 4-7. This improves base 64 encoded data compression.
+* Mar 24, 2016: Small speedup for level 1-3.
+* Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster.
+* Feb 19, 2016: Handle small payloads faster in level 1-3.
+* Feb 19, 2016: Added faster level 2 + 3 compression modes.
+* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progresssion in terms of compression. New default level is 5.
+* Feb 14, 2016: Snappy: Merge upstream changes.
+* Feb 14, 2016: Snappy: Fix aggressive skipping.
+* Feb 14, 2016: Snappy: Update benchmark.
+* Feb 13, 2016: Deflate: Fixed assembler problem that could lead to sub-optimal compression.
+* Feb 12, 2016: Snappy: Added AMD64 SSE 4.2 optimizations to matching, which makes easy to compress material run faster. Typical speedup is around 25%.
+* Feb 9, 2016: Added Snappy package fork. This version is 5-7% faster, much more on hard to compress content.
+* Jan 30, 2016: Optimize level 1 to 3 by not considering static dictionary or storing uncompressed. ~4-5% speedup.
+* Jan 16, 2016: Optimization on deflate level 1,2,3 compression.
+* Jan 8 2016: Merge [CL 18317](https://go-review.googlesource.com/#/c/18317): fix reading, writing of zip64 archives.
+* Dec 8 2015: Make level 1 and -2 deterministic even if write size differs.
+* Dec 8 2015: Split encoding functions, so hashing and matching can potentially be inlined. 1-3% faster on AMD64. 5% faster on other platforms.
+* Dec 8 2015: Fixed rare [one byte out-of bounds read](https://github.com/klauspost/compress/issues/20). Please update!
+* Nov 23 2015: Optimization on token writer. ~2-4% faster. Contributed by [@dsnet](https://github.com/dsnet).
+* Nov 20 2015: Small optimization to bit writer on 64 bit systems.
+* Nov 17 2015: Fixed out-of-bound errors if the underlying Writer returned an error. See [#15](https://github.com/klauspost/compress/issues/15).
+* Nov 12 2015: Added [io.WriterTo](https://golang.org/pkg/io/#WriterTo) support to gzip/inflate.
+* Nov 11 2015: Merged [CL 16669](https://go-review.googlesource.com/#/c/16669/4): archive/zip: enable overriding (de)compressors per file
+* Oct 15 2015: Added skipping on uncompressible data. Random data speed up >5x.
+
+</details>
+
+# deflate usage
+
+* [High Throughput Benchmark](http://blog.klauspost.com/go-gzipdeflate-benchmarks/).
+* [Small Payload/Webserver Benchmarks](http://blog.klauspost.com/gzip-performance-for-go-webservers/).
+* [Linear Time Compression](http://blog.klauspost.com/constant-time-gzipzip-compression/).
+* [Re-balancing Deflate Compression Levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/)
+
+The packages are drop-in replacements for standard libraries. Simply replace the import path to use them:
+
+| old import | new import | Documentation
+|--------------------|-----------------------------------------|--------------------|
+| `compress/gzip` | `github.com/klauspost/compress/gzip` | [gzip](https://pkg.go.dev/github.com/klauspost/compress/gzip?tab=doc)
+| `compress/zlib` | `github.com/klauspost/compress/zlib` | [zlib](https://pkg.go.dev/github.com/klauspost/compress/zlib?tab=doc)
+| `archive/zip` | `github.com/klauspost/compress/zip` | [zip](https://pkg.go.dev/github.com/klauspost/compress/zip?tab=doc)
+| `compress/flate` | `github.com/klauspost/compress/flate` | [flate](https://pkg.go.dev/github.com/klauspost/compress/flate?tab=doc)
+
+* Optimized [deflate](https://godoc.org/github.com/klauspost/compress/flate) packages which can be used as a dropin replacement for [gzip](https://godoc.org/github.com/klauspost/compress/gzip), [zip](https://godoc.org/github.com/klauspost/compress/zip) and [zlib](https://godoc.org/github.com/klauspost/compress/zlib).
+
+You may also be interested in [pgzip](https://github.com/klauspost/pgzip), which is a drop in replacement for gzip, which support multithreaded compression on big files and the optimized [crc32](https://github.com/klauspost/crc32) package used by these packages.
+
+The packages contains the same as the standard library, so you can use the godoc for that: [gzip](http://golang.org/pkg/compress/gzip/), [zip](http://golang.org/pkg/archive/zip/), [zlib](http://golang.org/pkg/compress/zlib/), [flate](http://golang.org/pkg/compress/flate/).
+
+Currently there is only minor speedup on decompression (mostly CRC32 calculation).
+
+Memory usage is typically 1MB for a Writer. stdlib is in the same range.
+If you expect to have a lot of concurrently allocated Writers consider using
+the stateless compress described below.
+
+# Stateless compression
+
+This package offers stateless compression as a special option for gzip/deflate.
+It will do compression but without maintaining any state between Write calls.
+
+This means there will be no memory kept between Write calls, but compression and speed will be suboptimal.
+
+This is only relevant in cases where you expect to run many thousands of compressors concurrently,
+but with very little activity. This is *not* intended for regular web servers serving individual requests.
+
+Because of this, the size of actual Write calls will affect output size.
+
+In gzip, specify level `-3` / `gzip.StatelessCompression` to enable.
+
+For direct deflate use, NewStatelessWriter and StatelessDeflate are available. See [documentation](https://godoc.org/github.com/klauspost/compress/flate#NewStatelessWriter)
+
+A `bufio.Writer` can of course be used to control write sizes. For example, to use a 4KB buffer:
+
+```
+ // replace 'ioutil.Discard' with your output.
+ gzw, err := gzip.NewWriterLevel(ioutil.Discard, gzip.StatelessCompression)
+ if err != nil {
+ return err
+ }
+ defer gzw.Close()
+
+ w := bufio.NewWriterSize(gzw, 4096)
+ defer w.Flush()
+
+ // Write to 'w'
+```
+
+This will only use up to 4KB in memory when the writer is idle.
+
+Compression is almost always worse than the fastest compression level
+and each write will allocate (a little) memory.
+
+# Performance Update 2018
+
+It has been a while since we have been looking at the speed of this package compared to the standard library, so I thought I would re-do my tests and give some overall recommendations based on the current state. All benchmarks have been performed with Go 1.10 on my Desktop Intel(R) Core(TM) i7-2600 CPU @3.40GHz. Since I last ran the tests, I have gotten more RAM, which means tests with big files are no longer limited by my SSD.
+
+The raw results are in my [updated spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing). Due to cgo changes and upstream updates i could not get the cgo version of gzip to compile. Instead I included the [zstd](https://github.com/datadog/zstd) cgo implementation. If I get cgo gzip to work again, I might replace the results in the sheet.
+
+The columns to take note of are: *MB/s* - the throughput. *Reduction* - the data size reduction in percent of the original. *Rel Speed* relative speed compared to the standard library at the same level. *Smaller* - how many percent smaller is the compressed output compared to stdlib. Negative means the output was bigger. *Loss* means the loss (or gain) in compression as a percentage difference of the input.
+
+The `gzstd` (standard library gzip) and `gzkp` (this package gzip) only uses one CPU core. [`pgzip`](https://github.com/klauspost/pgzip), [`bgzf`](https://github.com/biogo/hts/tree/master/bgzf) uses all 4 cores. [`zstd`](https://github.com/DataDog/zstd) uses one core, and is a beast (but not Go, yet).
+
+
+## Overall differences.
+
+There appears to be a roughly 5-10% speed advantage over the standard library when comparing at similar compression levels.
+
+The biggest difference you will see is the result of [re-balancing](https://blog.klauspost.com/rebalancing-deflate-compression-levels/) the compression levels. I wanted by library to give a smoother transition between the compression levels than the standard library.
+
+This package attempts to provide a more smooth transition, where "1" is taking a lot of shortcuts, "5" is the reasonable trade-off and "9" is the "give me the best compression", and the values in between gives something reasonable in between. The standard library has big differences in levels 1-4, but levels 5-9 having no significant gains - often spending a lot more time than can be justified by the achieved compression.
+
+There are links to all the test data in the [spreadsheet](https://docs.google.com/spreadsheets/d/1nuNE2nPfuINCZJRMt6wFWhKpToF95I47XjSsc-1rbPQ/edit?usp=sharing) in the top left field on each tab.
+
+## Web Content
+
+This test set aims to emulate typical use in a web server. The test-set is 4GB data in 53k files, and is a mixture of (mostly) HTML, JS, CSS.
+
+Since level 1 and 9 are close to being the same code, they are quite close. But looking at the levels in-between the differences are quite big.
+
+Looking at level 6, this package is 88% faster, but will output about 6% more data. For a web server, this means you can serve 88% more data, but have to pay for 6% more bandwidth. You can draw your own conclusions on what would be the most expensive for your case.
+
+## Object files
+
+This test is for typical data files stored on a server. In this case it is a collection of Go precompiled objects. They are very compressible.
+
+The picture is similar to the web content, but with small differences since this is very compressible. Levels 2-3 offer good speed, but is sacrificing quite a bit of compression.
+
+The standard library seems suboptimal on level 3 and 4 - offering both worse compression and speed than level 6 & 7 of this package respectively.
+
+## Highly Compressible File
+
+This is a JSON file with very high redundancy. The reduction starts at 95% on level 1, so in real life terms we are dealing with something like a highly redundant stream of data, etc.
+
+It is definitely visible that we are dealing with specialized content here, so the results are very scattered. This package does not do very well at levels 1-4, but picks up significantly at level 5 and levels 7 and 8 offering great speed for the achieved compression.
+
+So if you know you content is extremely compressible you might want to go slightly higher than the defaults. The standard library has a huge gap between levels 3 and 4 in terms of speed (2.75x slowdown), so it offers little "middle ground".
+
+## Medium-High Compressible
+
+This is a pretty common test corpus: [enwik9](http://mattmahoney.net/dc/textdata.html). It contains the first 10^9 bytes of the English Wikipedia dump on Mar. 3, 2006. This is a very good test of typical text based compression and more data heavy streams.
+
+We see a similar picture here as in "Web Content". On equal levels some compression is sacrificed for more speed. Level 5 seems to be the best trade-off between speed and size, beating stdlib level 3 in both.
+
+## Medium Compressible
+
+I will combine two test sets, one [10GB file set](http://mattmahoney.net/dc/10gb.html) and a VM disk image (~8GB). Both contain different data types and represent a typical backup scenario.
+
+The most notable thing is how quickly the standard library drops to very low compression speeds around level 5-6 without any big gains in compression. Since this type of data is fairly common, this does not seem like good behavior.
+
+
+## Un-compressible Content
+
+This is mainly a test of how good the algorithms are at detecting un-compressible input. The standard library only offers this feature with very conservative settings at level 1. Obviously there is no reason for the algorithms to try to compress input that cannot be compressed. The only downside is that it might skip some compressible data on false detections.
+
+
+## Huffman only compression
+
+This compression library adds a special compression level, named `HuffmanOnly`, which allows near linear time compression. This is done by completely disabling matching of previous data, and only reduce the number of bits to represent each character.
+
+This means that often used characters, like 'e' and ' ' (space) in text use the fewest bits to represent, and rare characters like '¤' takes more bits to represent. For more information see [wikipedia](https://en.wikipedia.org/wiki/Huffman_coding) or this nice [video](https://youtu.be/ZdooBTdW5bM).
+
+Since this type of compression has much less variance, the compression speed is mostly unaffected by the input data, and is usually more than *180MB/s* for a single core.
+
+The downside is that the compression ratio is usually considerably worse than even the fastest conventional compression. The compression ratio can never be better than 8:1 (12.5%).
+
+The linear time compression can be used as a "better than nothing" mode, where you cannot risk the encoder to slow down on some content. For comparison, the size of the "Twain" text is *233460 bytes* (+29% vs. level 1) and encode speed is 144MB/s (4.5x level 1). So in this case you trade a 30% size increase for a 4 times speedup.
+
+For more information see my blog post on [Fast Linear Time Compression](http://blog.klauspost.com/constant-time-gzipzip-compression/).
+
+This is implemented on Go 1.7 as "Huffman Only" mode, though not exposed for gzip.
+
+
+# license
+
+This code is licensed under the same conditions as the original Go code. See LICENSE file.
diff --git a/vendor/github.com/klauspost/compress/compressible.go b/vendor/github.com/klauspost/compress/compressible.go new file mode 100644 index 000000000..ea5a692d5 --- /dev/null +++ b/vendor/github.com/klauspost/compress/compressible.go @@ -0,0 +1,85 @@ +package compress + +import "math" + +// Estimate returns a normalized compressibility estimate of block b. +// Values close to zero are likely uncompressible. +// Values above 0.1 are likely to be compressible. +// Values above 0.5 are very compressible. +// Very small lengths will return 0. +func Estimate(b []byte) float64 { + if len(b) < 16 { + return 0 + } + + // Correctly predicted order 1 + hits := 0 + lastMatch := false + var o1 [256]byte + var hist [256]int + c1 := byte(0) + for _, c := range b { + if c == o1[c1] { + // We only count a hit if there was two correct predictions in a row. + if lastMatch { + hits++ + } + lastMatch = true + } else { + lastMatch = false + } + o1[c1] = c + c1 = c + hist[c]++ + } + + // Use x^0.6 to give better spread + prediction := math.Pow(float64(hits)/float64(len(b)), 0.6) + + // Calculate histogram distribution + variance := float64(0) + avg := float64(len(b)) / 256 + + for _, v := range hist { + Δ := float64(v) - avg + variance += Δ * Δ + } + + stddev := math.Sqrt(float64(variance)) / float64(len(b)) + exp := math.Sqrt(1 / float64(len(b))) + + // Subtract expected stddev + stddev -= exp + if stddev < 0 { + stddev = 0 + } + stddev *= 1 + exp + + // Use x^0.4 to give better spread + entropy := math.Pow(stddev, 0.4) + + // 50/50 weight between prediction and histogram distribution + return math.Pow((prediction+entropy)/2, 0.9) +} + +// ShannonEntropyBits returns the number of bits minimum required to represent +// an entropy encoding of the input bytes. +// https://en.wiktionary.org/wiki/Shannon_entropy +func ShannonEntropyBits(b []byte) int { + if len(b) == 0 { + return 0 + } + var hist [256]int + for _, c := range b { + hist[c]++ + } + shannon := float64(0) + invTotal := 1.0 / float64(len(b)) + for _, v := range hist[:] { + if v > 0 { + n := float64(v) + shannon += math.Ceil(-math.Log2(n*invTotal) * n) + } + } + return int(math.Ceil(shannon)) +} diff --git a/vendor/github.com/klauspost/compress/gen.sh b/vendor/github.com/klauspost/compress/gen.sh new file mode 100644 index 000000000..aff942205 --- /dev/null +++ b/vendor/github.com/klauspost/compress/gen.sh @@ -0,0 +1,4 @@ +#!/bin/sh + +cd s2/cmd/_s2sx/ || exit 1 +go generate . diff --git a/vendor/github.com/klauspost/compress/go.mod b/vendor/github.com/klauspost/compress/go.mod new file mode 100644 index 000000000..2a4f0a41f --- /dev/null +++ b/vendor/github.com/klauspost/compress/go.mod @@ -0,0 +1,5 @@ +module github.com/klauspost/compress + +go 1.13 + +require github.com/golang/snappy v0.0.3 // indirect diff --git a/vendor/github.com/klauspost/compress/go.sum b/vendor/github.com/klauspost/compress/go.sum new file mode 100644 index 000000000..73204cafa --- /dev/null +++ b/vendor/github.com/klauspost/compress/go.sum @@ -0,0 +1,2 @@ +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go index 0823c928c..8323dc053 100644 --- a/vendor/github.com/klauspost/compress/huff0/compress.go +++ b/vendor/github.com/klauspost/compress/huff0/compress.go @@ -161,6 +161,70 @@ func compress(in []byte, s *Scratch, compressor func(src []byte) ([]byte, error) return s.Out, false, nil } +// EstimateSizes will estimate the data sizes +func EstimateSizes(in []byte, s *Scratch) (tableSz, dataSz, reuseSz int, err error) { + s, err = s.prepare(in) + if err != nil { + return 0, 0, 0, err + } + + // Create histogram, if none was provided. + tableSz, dataSz, reuseSz = -1, -1, -1 + maxCount := s.maxCount + var canReuse = false + if maxCount == 0 { + maxCount, canReuse = s.countSimple(in) + } else { + canReuse = s.canUseTable(s.prevTable) + } + + // We want the output size to be less than this: + wantSize := len(in) + if s.WantLogLess > 0 { + wantSize -= wantSize >> s.WantLogLess + } + + // Reset for next run. + s.clearCount = true + s.maxCount = 0 + if maxCount >= len(in) { + if maxCount > len(in) { + return 0, 0, 0, fmt.Errorf("maxCount (%d) > length (%d)", maxCount, len(in)) + } + if len(in) == 1 { + return 0, 0, 0, ErrIncompressible + } + // One symbol, use RLE + return 0, 0, 0, ErrUseRLE + } + if maxCount == 1 || maxCount < (len(in)>>7) { + // Each symbol present maximum once or too well distributed. + return 0, 0, 0, ErrIncompressible + } + + // Calculate new table. + err = s.buildCTable() + if err != nil { + return 0, 0, 0, err + } + + if false && !s.canUseTable(s.cTable) { + panic("invalid table generated") + } + + tableSz, err = s.cTable.estTableSize(s) + if err != nil { + return 0, 0, 0, err + } + if canReuse { + reuseSz = s.prevTable.estimateSize(s.count[:s.symbolLen]) + } + dataSz = s.cTable.estimateSize(s.count[:s.symbolLen]) + + // Restore + return tableSz, dataSz, reuseSz, nil +} + func (s *Scratch) compress1X(src []byte) ([]byte, error) { return s.compress1xDo(s.Out, src) } diff --git a/vendor/github.com/klauspost/compress/huff0/huff0.go b/vendor/github.com/klauspost/compress/huff0/huff0.go index 7ec2022b6..3ee00ecb4 100644 --- a/vendor/github.com/klauspost/compress/huff0/huff0.go +++ b/vendor/github.com/klauspost/compress/huff0/huff0.go @@ -245,6 +245,68 @@ func (c cTable) write(s *Scratch) error { return nil } +func (c cTable) estTableSize(s *Scratch) (sz int, err error) { + var ( + // precomputed conversion table + bitsToWeight [tableLogMax + 1]byte + huffLog = s.actualTableLog + // last weight is not saved. + maxSymbolValue = uint8(s.symbolLen - 1) + huffWeight = s.huffWeight[:256] + ) + const ( + maxFSETableLog = 6 + ) + // convert to weight + bitsToWeight[0] = 0 + for n := uint8(1); n < huffLog+1; n++ { + bitsToWeight[n] = huffLog + 1 - n + } + + // Acquire histogram for FSE. + hist := s.fse.Histogram() + hist = hist[:256] + for i := range hist[:16] { + hist[i] = 0 + } + for n := uint8(0); n < maxSymbolValue; n++ { + v := bitsToWeight[c[n].nBits] & 15 + huffWeight[n] = v + hist[v]++ + } + + // FSE compress if feasible. + if maxSymbolValue >= 2 { + huffMaxCnt := uint32(0) + huffMax := uint8(0) + for i, v := range hist[:16] { + if v == 0 { + continue + } + huffMax = byte(i) + if v > huffMaxCnt { + huffMaxCnt = v + } + } + s.fse.HistogramFinished(huffMax, int(huffMaxCnt)) + s.fse.TableLog = maxFSETableLog + b, err := fse.Compress(huffWeight[:maxSymbolValue], s.fse) + if err == nil && len(b) < int(s.symbolLen>>1) { + sz += 1 + len(b) + return sz, nil + } + // Unable to compress (RLE/uncompressible) + } + // write raw values as 4-bits (max : 15) + if maxSymbolValue > (256 - 128) { + // should not happen : likely means source cannot be compressed + return 0, ErrIncompressible + } + // special case, pack weights 4 bits/weight. + sz += 1 + int(maxSymbolValue/2) + return sz, nil +} + // estimateSize returns the estimated size in bytes of the input represented in the // histogram supplied. func (c cTable) estimateSize(hist []uint32) int { diff --git a/vendor/github.com/klauspost/compress/s2sx.mod b/vendor/github.com/klauspost/compress/s2sx.mod new file mode 100644 index 000000000..a9faf7a02 --- /dev/null +++ b/vendor/github.com/klauspost/compress/s2sx.mod @@ -0,0 +1,5 @@ +module github.com/klauspost/compress + +go 1.16 + +require github.com/golang/snappy v0.0.3 // indirect diff --git a/vendor/github.com/klauspost/compress/s2sx.sum b/vendor/github.com/klauspost/compress/s2sx.sum new file mode 100644 index 000000000..73204cafa --- /dev/null +++ b/vendor/github.com/klauspost/compress/s2sx.sum @@ -0,0 +1,2 @@ +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= diff --git a/vendor/github.com/klauspost/compress/zstd/README.md b/vendor/github.com/klauspost/compress/zstd/README.md index 787813fa9..c8f0f16fc 100644 --- a/vendor/github.com/klauspost/compress/zstd/README.md +++ b/vendor/github.com/klauspost/compress/zstd/README.md @@ -152,7 +152,7 @@ file out level insize outsize millis mb/s silesia.tar zskp 1 211947520 73101992 643 313.87 silesia.tar zskp 2 211947520 67504318 969 208.38 silesia.tar zskp 3 211947520 64595893 2007 100.68 -silesia.tar zskp 4 211947520 60995370 7691 26.28 +silesia.tar zskp 4 211947520 60995370 8825 22.90 cgo zstd: silesia.tar zstd 1 211947520 73605392 543 371.56 @@ -162,7 +162,7 @@ silesia.tar zstd 9 211947520 60212393 5063 39.92 gzip, stdlib/this package: silesia.tar gzstd 1 211947520 80007735 1654 122.21 -silesia.tar gzkp 1 211947520 80369488 1168 173.06 +silesia.tar gzkp 1 211947520 80136201 1152 175.45 GOB stream of binary data. Highly compressible. https://files.klauspost.com/compress/gob-stream.7z @@ -171,13 +171,15 @@ file out level insize outsize millis mb/s gob-stream zskp 1 1911399616 235022249 3088 590.30 gob-stream zskp 2 1911399616 205669791 3786 481.34 gob-stream zskp 3 1911399616 175034659 9636 189.17 -gob-stream zskp 4 1911399616 167273881 29337 62.13 +gob-stream zskp 4 1911399616 165609838 50369 36.19 + gob-stream zstd 1 1911399616 249810424 2637 691.26 gob-stream zstd 3 1911399616 208192146 3490 522.31 gob-stream zstd 6 1911399616 193632038 6687 272.56 gob-stream zstd 9 1911399616 177620386 16175 112.70 + gob-stream gzstd 1 1911399616 357382641 10251 177.82 -gob-stream gzkp 1 1911399616 362156523 5695 320.08 +gob-stream gzkp 1 1911399616 359753026 5438 335.20 The test data for the Large Text Compression Benchmark is the first 10^9 bytes of the English Wikipedia dump on Mar. 3, 2006. @@ -187,11 +189,13 @@ file out level insize outsize millis mb/s enwik9 zskp 1 1000000000 343848582 3609 264.18 enwik9 zskp 2 1000000000 317276632 5746 165.97 enwik9 zskp 3 1000000000 292243069 12162 78.41 -enwik9 zskp 4 1000000000 275241169 36430 26.18 +enwik9 zskp 4 1000000000 262183768 82837 11.51 + enwik9 zstd 1 1000000000 358072021 3110 306.65 enwik9 zstd 3 1000000000 313734672 4784 199.35 enwik9 zstd 6 1000000000 295138875 10290 92.68 enwik9 zstd 9 1000000000 278348700 28549 33.40 + enwik9 gzstd 1 1000000000 382578136 9604 99.30 enwik9 gzkp 1 1000000000 383825945 6544 145.73 @@ -202,13 +206,15 @@ file out level insize outsize millis mb/s github-june-2days-2019.json zskp 1 6273951764 699045015 10620 563.40 github-june-2days-2019.json zskp 2 6273951764 617881763 11687 511.96 github-june-2days-2019.json zskp 3 6273951764 524340691 34043 175.75 -github-june-2days-2019.json zskp 4 6273951764 503314661 93811 63.78 +github-june-2days-2019.json zskp 4 6273951764 470320075 170190 35.16 + github-june-2days-2019.json zstd 1 6273951764 766284037 8450 708.00 github-june-2days-2019.json zstd 3 6273951764 661889476 10927 547.57 github-june-2days-2019.json zstd 6 6273951764 642756859 22996 260.18 github-june-2days-2019.json zstd 9 6273951764 601974523 52413 114.16 + github-june-2days-2019.json gzstd 1 6273951764 1164400847 29948 199.79 -github-june-2days-2019.json gzkp 1 6273951764 1128755542 19236 311.03 +github-june-2days-2019.json gzkp 1 6273951764 1125417694 21788 274.61 VM Image, Linux mint with a few installed applications: https://files.klauspost.com/compress/rawstudio-mint14.7z @@ -217,13 +223,15 @@ file out level insize outsize millis mb/s rawstudio-mint14.tar zskp 1 8558382592 3667489370 20210 403.84 rawstudio-mint14.tar zskp 2 8558382592 3364592300 31873 256.07 rawstudio-mint14.tar zskp 3 8558382592 3158085214 77675 105.08 -rawstudio-mint14.tar zskp 4 8558382592 3020370044 404956 20.16 +rawstudio-mint14.tar zskp 4 8558382592 2965110639 857750 9.52 + rawstudio-mint14.tar zstd 1 8558382592 3609250104 17136 476.27 rawstudio-mint14.tar zstd 3 8558382592 3341679997 29262 278.92 rawstudio-mint14.tar zstd 6 8558382592 3235846406 77904 104.77 rawstudio-mint14.tar zstd 9 8558382592 3160778861 140946 57.91 + rawstudio-mint14.tar gzstd 1 8558382592 3926257486 57722 141.40 -rawstudio-mint14.tar gzkp 1 8558382592 3970463184 41749 195.49 +rawstudio-mint14.tar gzkp 1 8558382592 3962605659 45113 180.92 CSV data: https://files.klauspost.com/compress/nyc-taxi-data-10M.csv.zst @@ -232,13 +240,15 @@ file out level insize outsize millis mb/s nyc-taxi-data-10M.csv zskp 1 3325605752 641339945 8925 355.35 nyc-taxi-data-10M.csv zskp 2 3325605752 591748091 11268 281.44 nyc-taxi-data-10M.csv zskp 3 3325605752 530289687 25239 125.66 -nyc-taxi-data-10M.csv zskp 4 3325605752 490907191 65939 48.10 +nyc-taxi-data-10M.csv zskp 4 3325605752 476268884 135958 23.33 + nyc-taxi-data-10M.csv zstd 1 3325605752 687399637 8233 385.18 nyc-taxi-data-10M.csv zstd 3 3325605752 598514411 10065 315.07 nyc-taxi-data-10M.csv zstd 6 3325605752 570522953 20038 158.27 nyc-taxi-data-10M.csv zstd 9 3325605752 517554797 64565 49.12 + nyc-taxi-data-10M.csv gzstd 1 3325605752 928656485 23876 132.83 -nyc-taxi-data-10M.csv gzkp 1 3325605752 924718719 16388 193.53 +nyc-taxi-data-10M.csv gzkp 1 3325605752 922257165 16780 189.00 ``` ## Decompressor diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 4d984c3b2..f430f58b5 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -260,9 +260,10 @@ func (d *Decoder) WriteTo(w io.Writer) (int64, error) { if len(d.current.b) > 0 { n2, err2 := w.Write(d.current.b) n += int64(n2) - if err2 != nil && d.current.err == nil { + if err2 != nil && (d.current.err == nil || d.current.err == io.EOF) { d.current.err = err2 - break + } else if n2 != len(d.current.b) { + d.current.err = io.ErrShortWrite } } if d.current.err != nil { diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go index 60f298648..295cd602a 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_base.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go @@ -38,8 +38,8 @@ func (e *fastBase) AppendCRC(dst []byte) []byte { // WindowSize returns the window size of the encoder, // or a window size small enough to contain the input size, if > 0. -func (e *fastBase) WindowSize(size int) int32 { - if size > 0 && size < int(e.maxMatchOff) { +func (e *fastBase) WindowSize(size int64) int32 { + if size > 0 && size < int64(e.maxMatchOff) { b := int32(1) << uint(bits.Len(uint(size))) // Keep minimum window. if b < 1024 { diff --git a/vendor/github.com/klauspost/compress/zstd/enc_best.go b/vendor/github.com/klauspost/compress/zstd/enc_best.go index b7d4b9004..41025d62b 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_best.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_best.go @@ -6,21 +6,59 @@ package zstd import ( "fmt" - "math/bits" + + "github.com/klauspost/compress" ) const ( - bestLongTableBits = 20 // Bits used in the long match table + bestLongTableBits = 22 // Bits used in the long match table bestLongTableSize = 1 << bestLongTableBits // Size of the table + bestLongLen = 8 // Bytes used for table hash // Note: Increasing the short table bits or making the hash shorter // can actually lead to compression degradation since it will 'steal' more from the // long match table and match offsets are quite big. // This greatly depends on the type of input. - bestShortTableBits = 16 // Bits used in the short match table + bestShortTableBits = 18 // Bits used in the short match table bestShortTableSize = 1 << bestShortTableBits // Size of the table + bestShortLen = 4 // Bytes used for table hash + ) +type match struct { + offset int32 + s int32 + length int32 + rep int32 + est int32 +} + +const highScore = 25000 + +// estBits will estimate output bits from predefined tables. +func (m *match) estBits(bitsPerByte int32) { + mlc := mlCode(uint32(m.length - zstdMinMatch)) + var ofc uint8 + if m.rep < 0 { + ofc = ofCode(uint32(m.s-m.offset) + 3) + } else { + ofc = ofCode(uint32(m.rep)) + } + // Cost, excluding + ofTT, mlTT := fsePredefEnc[tableOffsets].ct.symbolTT[ofc], fsePredefEnc[tableMatchLengths].ct.symbolTT[mlc] + + // Add cost of match encoding... + m.est = int32(ofTT.outBits + mlTT.outBits) + m.est += int32(ofTT.deltaNbBits>>16 + mlTT.deltaNbBits>>16) + // Subtract savings compared to literal encoding... + m.est -= (m.length * bitsPerByte) >> 10 + if m.est > 0 { + // Unlikely gain.. + m.length = 0 + m.est = highScore + } +} + // bestFastEncoder uses 2 tables, one for short matches (5 bytes) and one for long matches. // The long match table contains the previous entry with the same hash, // effectively making it a "chain" of length 2. @@ -109,6 +147,14 @@ func (e *bestFastEncoder) Encode(blk *blockEnc, src []byte) { return } + // Use this to estimate literal cost. + // Scaled by 10 bits. + bitsPerByte := int32((compress.ShannonEntropyBits(src) * 1024) / len(src)) + // Huffman can never go < 1 bit/byte + if bitsPerByte < 1024 { + bitsPerByte = 1024 + } + // Override src src = e.hist sLimit := int32(len(src)) - inputMargin @@ -145,51 +191,44 @@ encodeLoop: panic("offset0 was 0") } - type match struct { - offset int32 - s int32 - length int32 - rep int32 - } - matchAt := func(offset int32, s int32, first uint32, rep int32) match { - if s-offset >= e.maxMatchOff || load3232(src, offset) != first { - return match{offset: offset, s: s} - } - return match{offset: offset, s: s, length: 4 + e.matchlen(s+4, offset+4, src), rep: rep} - } - bestOf := func(a, b match) match { - aScore := b.s - a.s + a.length - bScore := a.s - b.s + b.length - if a.rep < 0 { - aScore = aScore - int32(bits.Len32(uint32(a.offset)))/8 - } - if b.rep < 0 { - bScore = bScore - int32(bits.Len32(uint32(b.offset)))/8 - } - if aScore >= bScore { + if a.est+(a.s-b.s)*bitsPerByte>>10 < b.est+(b.s-a.s)*bitsPerByte>>10 { return a } return b } const goodEnough = 100 - nextHashL := hash8(cv, bestLongTableBits) - nextHashS := hash4x64(cv, bestShortTableBits) + nextHashL := hashLen(cv, bestLongTableBits, bestLongLen) + nextHashS := hashLen(cv, bestShortTableBits, bestShortLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] + matchAt := func(offset int32, s int32, first uint32, rep int32) match { + if s-offset >= e.maxMatchOff || load3232(src, offset) != first { + return match{s: s, est: highScore} + } + m := match{offset: offset, s: s, length: 4 + e.matchlen(s+4, offset+4, src), rep: rep} + m.estBits(bitsPerByte) + return m + } + best := bestOf(matchAt(candidateL.offset-e.cur, s, uint32(cv), -1), matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)) best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)) best = bestOf(best, matchAt(candidateS.prev-e.cur, s, uint32(cv), -1)) + if canRepeat && best.length < goodEnough { - best = bestOf(best, matchAt(s-offset1+1, s+1, uint32(cv>>8), 1)) - best = bestOf(best, matchAt(s-offset2+1, s+1, uint32(cv>>8), 2)) - best = bestOf(best, matchAt(s-offset3+1, s+1, uint32(cv>>8), 3)) + cv := uint32(cv >> 8) + spp := s + 1 + best = bestOf(best, matchAt(spp-offset1, spp, cv, 1)) + best = bestOf(best, matchAt(spp-offset2, spp, cv, 2)) + best = bestOf(best, matchAt(spp-offset3, spp, cv, 3)) if best.length > 0 { - best = bestOf(best, matchAt(s-offset1+3, s+3, uint32(cv>>24), 1)) - best = bestOf(best, matchAt(s-offset2+3, s+3, uint32(cv>>24), 2)) - best = bestOf(best, matchAt(s-offset3+3, s+3, uint32(cv>>24), 3)) + cv >>= 16 + spp += 2 + best = bestOf(best, matchAt(spp-offset1, spp, cv, 1)) + best = bestOf(best, matchAt(spp-offset2, spp, cv, 2)) + best = bestOf(best, matchAt(spp-offset3, spp, cv, 3)) } } // Load next and check... @@ -209,22 +248,28 @@ encodeLoop: } s++ - candidateS = e.table[hash4x64(cv>>8, bestShortTableBits)] + candidateS = e.table[hashLen(cv>>8, bestShortTableBits, bestShortLen)] cv = load6432(src, s) cv2 := load6432(src, s+1) - candidateL = e.longTable[hash8(cv, bestLongTableBits)] - candidateL2 := e.longTable[hash8(cv2, bestLongTableBits)] + candidateL = e.longTable[hashLen(cv, bestLongTableBits, bestLongLen)] + candidateL2 := e.longTable[hashLen(cv2, bestLongTableBits, bestLongLen)] + // Short at s+1 best = bestOf(best, matchAt(candidateS.offset-e.cur, s, uint32(cv), -1)) + // Long at s+1, s+2 best = bestOf(best, matchAt(candidateL.offset-e.cur, s, uint32(cv), -1)) best = bestOf(best, matchAt(candidateL.prev-e.cur, s, uint32(cv), -1)) best = bestOf(best, matchAt(candidateL2.offset-e.cur, s+1, uint32(cv2), -1)) best = bestOf(best, matchAt(candidateL2.prev-e.cur, s+1, uint32(cv2), -1)) - + if false { + // Short at s+3. + // Too often worse... + best = bestOf(best, matchAt(e.table[hashLen(cv2>>8, bestShortTableBits, bestShortLen)].offset-e.cur, s+2, uint32(cv2>>8), -1)) + } // See if we can find a better match by checking where the current best ends. // Use that offset to see if we can find a better full match. if sAt := best.s + best.length; sAt < sLimit { - nextHashL := hash8(load6432(src, sAt), bestLongTableBits) + nextHashL := hashLen(load6432(src, sAt), bestLongTableBits, bestLongLen) candidateEnd := e.longTable[nextHashL] if pos := candidateEnd.offset - e.cur - best.length; pos >= 0 { bestEnd := bestOf(best, matchAt(pos, best.s, load3232(src, best.s), -1)) @@ -284,8 +329,8 @@ encodeLoop: off := index0 + e.cur for index0 < s-1 { cv0 := load6432(src, index0) - h0 := hash8(cv0, bestLongTableBits) - h1 := hash4x64(cv0, bestShortTableBits) + h0 := hashLen(cv0, bestLongTableBits, bestLongLen) + h1 := hashLen(cv0, bestShortTableBits, bestShortLen) e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} e.table[h1] = prevEntry{offset: off, prev: e.table[h1].offset} off++ @@ -352,8 +397,8 @@ encodeLoop: // every entry for index0 < s-1 { cv0 := load6432(src, index0) - h0 := hash8(cv0, bestLongTableBits) - h1 := hash4x64(cv0, bestShortTableBits) + h0 := hashLen(cv0, bestLongTableBits, bestLongLen) + h1 := hashLen(cv0, bestShortTableBits, bestShortLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} e.table[h1] = prevEntry{offset: off, prev: e.table[h1].offset} @@ -374,8 +419,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hash4x64(cv, bestShortTableBits) - nextHashL := hash8(cv, bestLongTableBits) + nextHashS := hashLen(cv, bestShortTableBits, bestShortLen) + nextHashL := hashLen(cv, bestLongTableBits, bestLongLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -425,7 +470,7 @@ func (e *bestFastEncoder) EncodeNoHist(blk *blockEnc, src []byte) { e.Encode(blk, src) } -// ResetDict will reset and set a dictionary if not nil +// Reset will reset and set a dictionary if not nil func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { e.resetBase(d, singleBlock) if d == nil { @@ -441,10 +486,10 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { const hashLog = bestShortTableBits cv := load6432(d.content, i-e.maxMatchOff) - nextHash := hash4x64(cv, hashLog) // 0 -> 4 - nextHash1 := hash4x64(cv>>8, hashLog) // 1 -> 5 - nextHash2 := hash4x64(cv>>16, hashLog) // 2 -> 6 - nextHash3 := hash4x64(cv>>24, hashLog) // 3 -> 7 + nextHash := hashLen(cv, hashLog, bestShortLen) // 0 -> 4 + nextHash1 := hashLen(cv>>8, hashLog, bestShortLen) // 1 -> 5 + nextHash2 := hashLen(cv>>16, hashLog, bestShortLen) // 2 -> 6 + nextHash3 := hashLen(cv>>24, hashLog, bestShortLen) // 3 -> 7 e.dictTable[nextHash] = prevEntry{ prev: e.dictTable[nextHash].offset, offset: i, @@ -472,7 +517,7 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { } if len(d.content) >= 8 { cv := load6432(d.content, 0) - h := hash8(cv, bestLongTableBits) + h := hashLen(cv, bestLongTableBits, bestLongLen) e.dictLongTable[h] = prevEntry{ offset: e.maxMatchOff, prev: e.dictLongTable[h].offset, @@ -482,7 +527,7 @@ func (e *bestFastEncoder) Reset(d *dict, singleBlock bool) { off := 8 // First to read for i := e.maxMatchOff + 1; i < end; i++ { cv = cv>>8 | (uint64(d.content[off]) << 56) - h := hash8(cv, bestLongTableBits) + h := hashLen(cv, bestLongTableBits, bestLongLen) e.dictLongTable[h] = prevEntry{ offset: i, prev: e.dictLongTable[h].offset, diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index eab7b5083..602c05ee0 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -9,6 +9,7 @@ import "fmt" const ( betterLongTableBits = 19 // Bits used in the long match table betterLongTableSize = 1 << betterLongTableBits // Size of the table + betterLongLen = 8 // Bytes used for table hash // Note: Increasing the short table bits or making the hash shorter // can actually lead to compression degradation since it will 'steal' more from the @@ -16,6 +17,7 @@ const ( // This greatly depends on the type of input. betterShortTableBits = 13 // Bits used in the short match table betterShortTableSize = 1 << betterShortTableBits // Size of the table + betterShortLen = 5 // Bytes used for table hash betterLongTableShardCnt = 1 << (betterLongTableBits - dictShardBits) // Number of shards in the table betterLongTableShardSize = betterLongTableSize / betterLongTableShardCnt // Size of an individual shard @@ -154,8 +156,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hash5(cv, betterShortTableBits) - nextHashL := hash8(cv, betterLongTableBits) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) + nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -214,10 +216,10 @@ encodeLoop: for index0 < s-1 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hash8(cv0, betterLongTableBits) + h0 := hashLen(cv0, betterLongTableBits, betterLongLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[hash5(cv1, betterShortTableBits)] = tableEntry{offset: off + 1, val: uint32(cv1)} + e.table[hashLen(cv1, betterShortTableBits, betterShortLen)] = tableEntry{offset: off + 1, val: uint32(cv1)} index0 += 2 } cv = load6432(src, s) @@ -275,10 +277,10 @@ encodeLoop: for index0 < s-1 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hash8(cv0, betterLongTableBits) + h0 := hashLen(cv0, betterLongTableBits, betterLongLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[hash5(cv1, betterShortTableBits)] = tableEntry{offset: off + 1, val: uint32(cv1)} + e.table[hashLen(cv1, betterShortTableBits, betterShortLen)] = tableEntry{offset: off + 1, val: uint32(cv1)} index0 += 2 } cv = load6432(src, s) @@ -353,7 +355,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hash8(cv, betterLongTableBits) + nextHashL = hashLen(cv, betterLongTableBits, betterLongLen) candidateL = e.longTable[nextHashL] coffsetL = candidateL.offset - e.cur @@ -413,8 +415,8 @@ encodeLoop: } // Try to find a better match by searching for a long match at the end of the current best match - if true && s+matched < sLimit { - nextHashL := hash8(load6432(src, s+matched), betterLongTableBits) + if s+matched < sLimit { + nextHashL := hashLen(load6432(src, s+matched), betterLongTableBits, betterLongLen) cv := load3232(src, s) candidateL := e.longTable[nextHashL] coffsetL := candidateL.offset - e.cur - matched @@ -495,10 +497,10 @@ encodeLoop: for index0 < s-1 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hash8(cv0, betterLongTableBits) + h0 := hashLen(cv0, betterLongTableBits, betterLongLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} - e.table[hash5(cv1, betterShortTableBits)] = tableEntry{offset: off + 1, val: uint32(cv1)} + e.table[hashLen(cv1, betterShortTableBits, betterShortLen)] = tableEntry{offset: off + 1, val: uint32(cv1)} index0 += 2 } @@ -516,8 +518,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hash5(cv, betterShortTableBits) - nextHashL := hash8(cv, betterLongTableBits) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) + nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -672,8 +674,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hash5(cv, betterShortTableBits) - nextHashL := hash8(cv, betterLongTableBits) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) + nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -734,11 +736,11 @@ encodeLoop: for index0 < s-1 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hash8(cv0, betterLongTableBits) + h0 := hashLen(cv0, betterLongTableBits, betterLongLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} e.markLongShardDirty(h0) - h1 := hash5(cv1, betterShortTableBits) + h1 := hashLen(cv1, betterShortTableBits, betterShortLen) e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} e.markShortShardDirty(h1) index0 += 2 @@ -798,11 +800,11 @@ encodeLoop: for index0 < s-1 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hash8(cv0, betterLongTableBits) + h0 := hashLen(cv0, betterLongTableBits, betterLongLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} e.markLongShardDirty(h0) - h1 := hash5(cv1, betterShortTableBits) + h1 := hashLen(cv1, betterShortTableBits, betterShortLen) e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} e.markShortShardDirty(h1) index0 += 2 @@ -879,7 +881,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hash8(cv, betterLongTableBits) + nextHashL = hashLen(cv, betterLongTableBits, betterLongLen) candidateL = e.longTable[nextHashL] coffsetL = candidateL.offset - e.cur @@ -940,7 +942,7 @@ encodeLoop: } // Try to find a better match by searching for a long match at the end of the current best match if s+matched < sLimit { - nextHashL := hash8(load6432(src, s+matched), betterLongTableBits) + nextHashL := hashLen(load6432(src, s+matched), betterLongTableBits, betterLongLen) cv := load3232(src, s) candidateL := e.longTable[nextHashL] coffsetL := candidateL.offset - e.cur - matched @@ -1021,11 +1023,11 @@ encodeLoop: for index0 < s-1 { cv0 := load6432(src, index0) cv1 := cv0 >> 8 - h0 := hash8(cv0, betterLongTableBits) + h0 := hashLen(cv0, betterLongTableBits, betterLongLen) off := index0 + e.cur e.longTable[h0] = prevEntry{offset: off, prev: e.longTable[h0].offset} e.markLongShardDirty(h0) - h1 := hash5(cv1, betterShortTableBits) + h1 := hashLen(cv1, betterShortTableBits, betterShortLen) e.table[h1] = tableEntry{offset: off + 1, val: uint32(cv1)} e.markShortShardDirty(h1) index0 += 2 @@ -1045,8 +1047,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hash5(cv, betterShortTableBits) - nextHashL := hash8(cv, betterLongTableBits) + nextHashS := hashLen(cv, betterShortTableBits, betterShortLen) + nextHashL := hashLen(cv, betterLongTableBits, betterLongLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -1113,10 +1115,10 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { const hashLog = betterShortTableBits cv := load6432(d.content, i-e.maxMatchOff) - nextHash := hash5(cv, hashLog) // 0 -> 4 - nextHash1 := hash5(cv>>8, hashLog) // 1 -> 5 - nextHash2 := hash5(cv>>16, hashLog) // 2 -> 6 - nextHash3 := hash5(cv>>24, hashLog) // 3 -> 7 + nextHash := hashLen(cv, hashLog, betterShortLen) // 0 -> 4 + nextHash1 := hashLen(cv>>8, hashLog, betterShortLen) // 1 -> 5 + nextHash2 := hashLen(cv>>16, hashLog, betterShortLen) // 2 -> 6 + nextHash3 := hashLen(cv>>24, hashLog, betterShortLen) // 3 -> 7 e.dictTable[nextHash] = tableEntry{ val: uint32(cv), offset: i, @@ -1145,7 +1147,7 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { } if len(d.content) >= 8 { cv := load6432(d.content, 0) - h := hash8(cv, betterLongTableBits) + h := hashLen(cv, betterLongTableBits, betterLongLen) e.dictLongTable[h] = prevEntry{ offset: e.maxMatchOff, prev: e.dictLongTable[h].offset, @@ -1155,7 +1157,7 @@ func (e *betterFastEncoderDict) Reset(d *dict, singleBlock bool) { off := 8 // First to read for i := e.maxMatchOff + 1; i < end; i++ { cv = cv>>8 | (uint64(d.content[off]) << 56) - h := hash8(cv, betterLongTableBits) + h := hashLen(cv, betterLongTableBits, betterLongLen) e.dictLongTable[h] = prevEntry{ offset: i, prev: e.dictLongTable[h].offset, diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index 96b21b90e..d6b310424 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -10,6 +10,7 @@ const ( dFastLongTableBits = 17 // Bits used in the long match table dFastLongTableSize = 1 << dFastLongTableBits // Size of the table dFastLongTableMask = dFastLongTableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. + dFastLongLen = 8 // Bytes used for table hash dLongTableShardCnt = 1 << (dFastLongTableBits - dictShardBits) // Number of shards in the table dLongTableShardSize = dFastLongTableSize / tableShardCnt // Size of an individual shard @@ -17,6 +18,8 @@ const ( dFastShortTableBits = tableBits // Bits used in the short match table dFastShortTableSize = 1 << dFastShortTableBits // Size of the table dFastShortTableMask = dFastShortTableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. + dFastShortLen = 5 // Bytes used for table hash + ) type doubleFastEncoder struct { @@ -124,8 +127,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hash5(cv, dFastShortTableBits) - nextHashL := hash8(cv, dFastLongTableBits) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -208,7 +211,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hash8(cv, dFastLongTableBits) + nextHashL = hashLen(cv, dFastLongTableBits, dFastLongLen) candidateL = e.longTable[nextHashL] coffsetL = s - (candidateL.offset - e.cur) + checkAt @@ -304,16 +307,16 @@ encodeLoop: cv1 := load6432(src, index1) te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} - e.longTable[hash8(cv0, dFastLongTableBits)] = te0 - e.longTable[hash8(cv1, dFastLongTableBits)] = te1 + e.longTable[hashLen(cv0, dFastLongTableBits, dFastLongLen)] = te0 + e.longTable[hashLen(cv1, dFastLongTableBits, dFastLongLen)] = te1 cv0 >>= 8 cv1 >>= 8 te0.offset++ te1.offset++ te0.val = uint32(cv0) te1.val = uint32(cv1) - e.table[hash5(cv0, dFastShortTableBits)] = te0 - e.table[hash5(cv1, dFastShortTableBits)] = te1 + e.table[hashLen(cv0, dFastShortTableBits, dFastShortLen)] = te0 + e.table[hashLen(cv1, dFastShortTableBits, dFastShortLen)] = te1 cv = load6432(src, s) @@ -330,8 +333,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hash5(cv, dFastShortTableBits) - nextHashL := hash8(cv, dFastLongTableBits) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -436,8 +439,8 @@ encodeLoop: var t int32 for { - nextHashS := hash5(cv, dFastShortTableBits) - nextHashL := hash8(cv, dFastLongTableBits) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -521,7 +524,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hash8(cv, dFastLongTableBits) + nextHashL = hashLen(cv, dFastLongTableBits, dFastLongLen) candidateL = e.longTable[nextHashL] coffsetL = s - (candidateL.offset - e.cur) + checkAt @@ -614,16 +617,16 @@ encodeLoop: cv1 := load6432(src, index1) te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} - e.longTable[hash8(cv0, dFastLongTableBits)] = te0 - e.longTable[hash8(cv1, dFastLongTableBits)] = te1 + e.longTable[hashLen(cv0, dFastLongTableBits, dFastLongLen)] = te0 + e.longTable[hashLen(cv1, dFastLongTableBits, dFastLongLen)] = te1 cv0 >>= 8 cv1 >>= 8 te0.offset++ te1.offset++ te0.val = uint32(cv0) te1.val = uint32(cv1) - e.table[hash5(cv0, dFastShortTableBits)] = te0 - e.table[hash5(cv1, dFastShortTableBits)] = te1 + e.table[hashLen(cv0, dFastShortTableBits, dFastShortLen)] = te0 + e.table[hashLen(cv1, dFastShortTableBits, dFastShortLen)] = te1 cv = load6432(src, s) @@ -640,8 +643,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hash5(cv1>>8, dFastShortTableBits) - nextHashL := hash8(cv, dFastLongTableBits) + nextHashS := hashLen(cv1>>8, dFastShortTableBits, dFastShortLen) + nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -782,8 +785,8 @@ encodeLoop: panic("offset0 was 0") } - nextHashS := hash5(cv, dFastShortTableBits) - nextHashL := hash8(cv, dFastLongTableBits) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) candidateL := e.longTable[nextHashL] candidateS := e.table[nextHashS] @@ -868,7 +871,7 @@ encodeLoop: // See if we can find a long match at s+1 const checkAt = 1 cv := load6432(src, s+checkAt) - nextHashL = hash8(cv, dFastLongTableBits) + nextHashL = hashLen(cv, dFastLongTableBits, dFastLongLen) candidateL = e.longTable[nextHashL] coffsetL = s - (candidateL.offset - e.cur) + checkAt @@ -965,8 +968,8 @@ encodeLoop: cv1 := load6432(src, index1) te0 := tableEntry{offset: index0 + e.cur, val: uint32(cv0)} te1 := tableEntry{offset: index1 + e.cur, val: uint32(cv1)} - longHash1 := hash8(cv0, dFastLongTableBits) - longHash2 := hash8(cv0, dFastLongTableBits) + longHash1 := hashLen(cv0, dFastLongTableBits, dFastLongLen) + longHash2 := hashLen(cv0, dFastLongTableBits, dFastLongLen) e.longTable[longHash1] = te0 e.longTable[longHash2] = te1 e.markLongShardDirty(longHash1) @@ -977,8 +980,8 @@ encodeLoop: te1.offset++ te0.val = uint32(cv0) te1.val = uint32(cv1) - hashVal1 := hash5(cv0, dFastShortTableBits) - hashVal2 := hash5(cv1, dFastShortTableBits) + hashVal1 := hashLen(cv0, dFastShortTableBits, dFastShortLen) + hashVal2 := hashLen(cv1, dFastShortTableBits, dFastShortLen) e.table[hashVal1] = te0 e.markShardDirty(hashVal1) e.table[hashVal2] = te1 @@ -999,8 +1002,8 @@ encodeLoop: } // Store this, since we have it. - nextHashS := hash5(cv, dFastShortTableBits) - nextHashL := hash8(cv, dFastLongTableBits) + nextHashS := hashLen(cv, dFastShortTableBits, dFastShortLen) + nextHashL := hashLen(cv, dFastLongTableBits, dFastLongLen) // We have at least 4 byte match. // No need to check backwards. We come straight from a match @@ -1071,14 +1074,14 @@ func (e *doubleFastEncoderDict) Reset(d *dict, singleBlock bool) { } if len(d.content) >= 8 { cv := load6432(d.content, 0) - e.dictLongTable[hash8(cv, dFastLongTableBits)] = tableEntry{ + e.dictLongTable[hashLen(cv, dFastLongTableBits, dFastLongLen)] = tableEntry{ val: uint32(cv), offset: e.maxMatchOff, } end := int32(len(d.content)) - 8 + e.maxMatchOff for i := e.maxMatchOff + 1; i < end; i++ { cv = cv>>8 | (uint64(d.content[i-e.maxMatchOff+7]) << 56) - e.dictLongTable[hash8(cv, dFastLongTableBits)] = tableEntry{ + e.dictLongTable[hashLen(cv, dFastLongTableBits, dFastLongLen)] = tableEntry{ val: uint32(cv), offset: i, } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_fast.go b/vendor/github.com/klauspost/compress/zstd/enc_fast.go index 2246d286d..f2502629b 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_fast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_fast.go @@ -11,12 +11,13 @@ import ( ) const ( - tableBits = 15 // Bits used in the table - tableSize = 1 << tableBits // Size of the table - tableShardCnt = 1 << (tableBits - dictShardBits) // Number of shards in the table - tableShardSize = tableSize / tableShardCnt // Size of an individual shard - tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. - maxMatchLength = 131074 + tableBits = 15 // Bits used in the table + tableSize = 1 << tableBits // Size of the table + tableShardCnt = 1 << (tableBits - dictShardBits) // Number of shards in the table + tableShardSize = tableSize / tableShardCnt // Size of an individual shard + tableFastHashLen = 6 + tableMask = tableSize - 1 // Mask for table indices. Redundant, but can eliminate bounds checks. + maxMatchLength = 131074 ) type tableEntry struct { @@ -122,8 +123,8 @@ encodeLoop: panic("offset0 was 0") } - nextHash := hash6(cv, hashLog) - nextHash2 := hash6(cv>>8, hashLog) + nextHash := hashLen(cv, hashLog, tableFastHashLen) + nextHash2 := hashLen(cv>>8, hashLog, tableFastHashLen) candidate := e.table[nextHash] candidate2 := e.table[nextHash2] repIndex := s - offset1 + 2 @@ -301,7 +302,7 @@ encodeLoop: } // Store this, since we have it. - nextHash := hash6(cv, hashLog) + nextHash := hashLen(cv, hashLog, tableFastHashLen) e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} seq.matchLen = uint32(l) - zstdMinMatch seq.litLen = 0 @@ -405,8 +406,8 @@ encodeLoop: // By not using them for the first 3 matches for { - nextHash := hash6(cv, hashLog) - nextHash2 := hash6(cv>>8, hashLog) + nextHash := hashLen(cv, hashLog, tableFastHashLen) + nextHash2 := hashLen(cv>>8, hashLog, tableFastHashLen) candidate := e.table[nextHash] candidate2 := e.table[nextHash2] repIndex := s - offset1 + 2 @@ -589,7 +590,7 @@ encodeLoop: } // Store this, since we have it. - nextHash := hash6(cv, hashLog) + nextHash := hashLen(cv, hashLog, tableFastHashLen) e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} seq.matchLen = uint32(l) - zstdMinMatch seq.litLen = 0 @@ -715,8 +716,8 @@ encodeLoop: panic("offset0 was 0") } - nextHash := hash6(cv, hashLog) - nextHash2 := hash6(cv>>8, hashLog) + nextHash := hashLen(cv, hashLog, tableFastHashLen) + nextHash2 := hashLen(cv>>8, hashLog, tableFastHashLen) candidate := e.table[nextHash] candidate2 := e.table[nextHash2] repIndex := s - offset1 + 2 @@ -896,7 +897,7 @@ encodeLoop: } // Store this, since we have it. - nextHash := hash6(cv, hashLog) + nextHash := hashLen(cv, hashLog, tableFastHashLen) e.table[nextHash] = tableEntry{offset: s + e.cur, val: uint32(cv)} e.markShardDirty(nextHash) seq.matchLen = uint32(l) - zstdMinMatch @@ -957,9 +958,9 @@ func (e *fastEncoderDict) Reset(d *dict, singleBlock bool) { const hashLog = tableBits cv := load6432(d.content, i-e.maxMatchOff) - nextHash := hash6(cv, hashLog) // 0 -> 5 - nextHash1 := hash6(cv>>8, hashLog) // 1 -> 6 - nextHash2 := hash6(cv>>16, hashLog) // 2 -> 7 + nextHash := hashLen(cv, hashLog, tableFastHashLen) // 0 -> 5 + nextHash1 := hashLen(cv>>8, hashLog, tableFastHashLen) // 1 -> 6 + nextHash2 := hashLen(cv>>16, hashLog, tableFastHashLen) // 2 -> 7 e.dictTable[nextHash] = tableEntry{ val: uint32(cv), offset: i, diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index ea85548fc..e6e315969 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -33,7 +33,7 @@ type encoder interface { Block() *blockEnc CRC() *xxhash.Digest AppendCRC([]byte) []byte - WindowSize(size int) int32 + WindowSize(size int64) int32 UseBlock(*blockEnc) Reset(d *dict, singleBlock bool) } @@ -48,6 +48,8 @@ type encoderState struct { err error writeErr error nWritten int64 + nInput int64 + frameContentSize int64 headerWritten bool eofWritten bool fullFrameWritten bool @@ -120,7 +122,21 @@ func (e *Encoder) Reset(w io.Writer) { s.w = w s.err = nil s.nWritten = 0 + s.nInput = 0 s.writeErr = nil + s.frameContentSize = 0 +} + +// ResetContentSize will reset and set a content size for the next stream. +// If the bytes written does not match the size given an error will be returned +// when calling Close(). +// This is removed when Reset is called. +// Sizes <= 0 results in no content size set. +func (e *Encoder) ResetContentSize(w io.Writer, size int64) { + e.Reset(w) + if size >= 0 { + e.state.frameContentSize = size + } } // Write data to the encoder. @@ -190,6 +206,7 @@ func (e *Encoder) nextBlock(final bool) error { return s.err } s.nWritten += int64(n2) + s.nInput += int64(len(s.filling)) s.current = s.current[:0] s.filling = s.filling[:0] s.headerWritten = true @@ -200,8 +217,8 @@ func (e *Encoder) nextBlock(final bool) error { var tmp [maxHeaderSize]byte fh := frameHeader{ - ContentSize: 0, - WindowSize: uint32(s.encoder.WindowSize(0)), + ContentSize: uint64(s.frameContentSize), + WindowSize: uint32(s.encoder.WindowSize(s.frameContentSize)), SingleSegment: false, Checksum: e.o.crc, DictID: e.o.dict.ID(), @@ -243,6 +260,7 @@ func (e *Encoder) nextBlock(final bool) error { // Move blocks forward. s.filling, s.current, s.previous = s.previous[:0], s.filling, s.current + s.nInput += int64(len(s.current)) s.wg.Add(1) go func(src []byte) { if debugEncoder { @@ -394,6 +412,11 @@ func (e *Encoder) Close() error { if err != nil { return err } + if s.frameContentSize > 0 { + if s.nInput != s.frameContentSize { + return fmt.Errorf("frame content size %d given, but %d bytes was written", s.frameContentSize, s.nInput) + } + } if e.state.fullFrameWritten { return s.err } @@ -470,7 +493,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } fh := frameHeader{ ContentSize: uint64(len(src)), - WindowSize: uint32(enc.WindowSize(len(src))), + WindowSize: uint32(enc.WindowSize(int64(len(src)))), SingleSegment: single, Checksum: e.o.crc, DictID: e.o.dict.ID(), diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index 16d4ab63c..7d29e1d68 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -189,7 +189,7 @@ func EncoderLevelFromZstd(level int) EncoderLevel { case level >= 6 && level < 10: return SpeedBetterCompression case level >= 10: - return SpeedBetterCompression + return SpeedBestCompression } return SpeedDefault } diff --git a/vendor/github.com/klauspost/compress/zstd/hash.go b/vendor/github.com/klauspost/compress/zstd/hash.go index 4a752067f..cf33f29a1 100644 --- a/vendor/github.com/klauspost/compress/zstd/hash.go +++ b/vendor/github.com/klauspost/compress/zstd/hash.go @@ -13,24 +13,24 @@ const ( prime8bytes = 0xcf1bbcdcb7a56463 ) -// hashLen returns a hash of the lowest l bytes of u for a size size of h bytes. -// l must be >=4 and <=8. Any other value will return hash for 4 bytes. -// h should always be <32. -// Preferably h and l should be a constant. -// FIXME: This does NOT get resolved, if 'mls' is constant, -// so this cannot be used. -func hashLen(u uint64, hashLog, mls uint8) uint32 { +// hashLen returns a hash of the lowest mls bytes of with length output bits. +// mls must be >=3 and <=8. Any other value will return hash for 4 bytes. +// length should always be < 32. +// Preferably length and mls should be a constant for inlining. +func hashLen(u uint64, length, mls uint8) uint32 { switch mls { + case 3: + return (uint32(u<<8) * prime3bytes) >> (32 - length) case 5: - return hash5(u, hashLog) + return uint32(((u << (64 - 40)) * prime5bytes) >> (64 - length)) case 6: - return hash6(u, hashLog) + return uint32(((u << (64 - 48)) * prime6bytes) >> (64 - length)) case 7: - return hash7(u, hashLog) + return uint32(((u << (64 - 56)) * prime7bytes) >> (64 - length)) case 8: - return hash8(u, hashLog) + return uint32((u * prime8bytes) >> (64 - length)) default: - return hash4x64(u, hashLog) + return (uint32(u) * prime4bytes) >> (32 - length) } } @@ -39,39 +39,3 @@ func hashLen(u uint64, hashLog, mls uint8) uint32 { func hash3(u uint32, h uint8) uint32 { return ((u << (32 - 24)) * prime3bytes) >> ((32 - h) & 31) } - -// hash4 returns the hash of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <32. -func hash4(u uint32, h uint8) uint32 { - return (u * prime4bytes) >> ((32 - h) & 31) -} - -// hash4x64 returns the hash of the lowest 4 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <32. -func hash4x64(u uint64, h uint8) uint32 { - return (uint32(u) * prime4bytes) >> ((32 - h) & 31) -} - -// hash5 returns the hash of the lowest 5 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <64. -func hash5(u uint64, h uint8) uint32 { - return uint32(((u << (64 - 40)) * prime5bytes) >> ((64 - h) & 63)) -} - -// hash6 returns the hash of the lowest 6 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <64. -func hash6(u uint64, h uint8) uint32 { - return uint32(((u << (64 - 48)) * prime6bytes) >> ((64 - h) & 63)) -} - -// hash7 returns the hash of the lowest 7 bytes of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <64. -func hash7(u uint64, h uint8) uint32 { - return uint32(((u << (64 - 56)) * prime7bytes) >> ((64 - h) & 63)) -} - -// hash8 returns the hash of u to fit in a hash table with h bits. -// Preferably h should be a constant and should always be <64. -func hash8(u uint64, h uint8) uint32 { - return uint32((u * prime8bytes) >> ((64 - h) & 63)) -} diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/rchcon.go b/vendor/github.com/opencontainers/selinux/go-selinux/rchcon.go new file mode 100644 index 000000000..897ecbac4 --- /dev/null +++ b/vendor/github.com/opencontainers/selinux/go-selinux/rchcon.go @@ -0,0 +1,22 @@ +// +build linux,go1.16 + +package selinux + +import ( + "errors" + "io/fs" + "os" + + "github.com/opencontainers/selinux/pkg/pwalkdir" +) + +func rchcon(fpath, label string) error { + return pwalkdir.Walk(fpath, func(p string, _ fs.DirEntry, _ error) error { + e := setFileLabel(p, label) + // Walk a file tree can race with removal, so ignore ENOENT. + if errors.Is(e, os.ErrNotExist) { + return nil + } + return e + }) +} diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/rchcon_go115.go b/vendor/github.com/opencontainers/selinux/go-selinux/rchcon_go115.go new file mode 100644 index 000000000..2c8b033ce --- /dev/null +++ b/vendor/github.com/opencontainers/selinux/go-selinux/rchcon_go115.go @@ -0,0 +1,21 @@ +// +build linux,!go1.16 + +package selinux + +import ( + "errors" + "os" + + "github.com/opencontainers/selinux/pkg/pwalk" +) + +func rchcon(fpath, label string) error { + return pwalk.Walk(fpath, func(p string, _ os.FileInfo, _ error) error { + e := setFileLabel(p, label) + // Walk a file tree can race with removal, so ignore ENOENT. + if errors.Is(e, os.ErrNotExist) { + return nil + } + return e + }) +} diff --git a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go index 62df82a34..a804473e4 100644 --- a/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go +++ b/vendor/github.com/opencontainers/selinux/go-selinux/selinux_linux.go @@ -18,7 +18,6 @@ import ( "sync" "github.com/bits-and-blooms/bitset" - "github.com/opencontainers/selinux/pkg/pwalk" "golang.org/x/sys/unix" ) @@ -1048,17 +1047,10 @@ func chcon(fpath string, label string, recurse bool) error { } if !recurse { - return SetFileLabel(fpath, label) + return setFileLabel(fpath, label) } - return pwalk.Walk(fpath, func(p string, info os.FileInfo, err error) error { - e := SetFileLabel(p, label) - // Walk a file tree can race with removal, so ignore ENOENT - if errors.Is(e, os.ErrNotExist) { - return nil - } - return e - }) + return rchcon(fpath, label) } // dupSecOpt takes an SELinux process label and returns security options that diff --git a/vendor/github.com/opencontainers/selinux/pkg/pwalk/README.md b/vendor/github.com/opencontainers/selinux/pkg/pwalk/README.md index 16c4dfd3e..7e78dce01 100644 --- a/vendor/github.com/opencontainers/selinux/pkg/pwalk/README.md +++ b/vendor/github.com/opencontainers/selinux/pkg/pwalk/README.md @@ -8,6 +8,12 @@ By default, it utilizes 2\*runtime.NumCPU() goroutines for callbacks. This can be changed by using WalkN function which has the additional parameter, specifying the number of goroutines (concurrency). +### pwalk vs pwalkdir + +This package is deprecated in favor of +[pwalkdir](https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalkdir), +which is faster, but requires at least Go 1.16. + ### Caveats Please note the following limitations of this code: diff --git a/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go b/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go index a8088a196..011fe862a 100644 --- a/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go +++ b/vendor/github.com/opencontainers/selinux/pkg/pwalk/pwalk.go @@ -19,7 +19,7 @@ type WalkFunc = filepath.WalkFunc // // Note that this implementation only supports primitive error handling: // -// - no errors are ever passed to WalkFn; +// - no errors are ever passed to walkFn; // // - once a walkFn returns any error, all further processing stops // and the error is returned to the caller of Walk; @@ -95,7 +95,7 @@ func WalkN(root string, walkFn WalkFunc, num int) error { return err } -// walkArgs holds the arguments that were passed to the Walk or WalkLimit +// walkArgs holds the arguments that were passed to the Walk or WalkN // functions. type walkArgs struct { path string diff --git a/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/README.md b/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/README.md new file mode 100644 index 000000000..068ac4005 --- /dev/null +++ b/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/README.md @@ -0,0 +1,54 @@ +## pwalkdir: parallel implementation of filepath.WalkDir + +This is a wrapper for [filepath.WalkDir](https://pkg.go.dev/path/filepath#WalkDir) +which may speed it up by calling multiple callback functions (WalkDirFunc) +in parallel, utilizing goroutines. + +By default, it utilizes 2\*runtime.NumCPU() goroutines for callbacks. +This can be changed by using WalkN function which has the additional +parameter, specifying the number of goroutines (concurrency). + +### pwalk vs pwalkdir + +This package is very similar to +[pwalk](https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalkdir), +but utilizes `filepath.WalkDir` (added to Go 1.16), which does not call stat(2) +on every entry and is therefore faster (up to 3x, depending on usage scenario). + +Users who are OK with requiring Go 1.16+ should switch to this +implementation. + +### Caveats + +Please note the following limitations of this code: + +* Unlike filepath.WalkDir, the order of calls is non-deterministic; + +* Only primitive error handling is supported: + + * fs.SkipDir is not supported; + + * no errors are ever passed to WalkDirFunc; + + * once any error is returned from any walkDirFunc instance, no more calls + to WalkDirFunc are made, and the error is returned to the caller of WalkDir; + + * if more than one WalkDirFunc instance will return an error, only one + of such errors will be propagated to and returned by WalkDir, others + will be silently discarded. + +### Documentation + +For the official documentation, see +https://pkg.go.dev/github.com/opencontainers/selinux/pkg/pwalkdir + +### Benchmarks + +For a WalkDirFunc that consists solely of the return statement, this +implementation is about 15% slower than the standard library's +filepath.WalkDir. + +Otherwise (if a WalkDirFunc is actually doing something) this is usually +faster, except when the WalkDirN(..., 1) is used. Run `go test -bench .` +to see how different operations can benefit from it, as well as how the +level of paralellism affects the speed. diff --git a/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/pwalkdir.go b/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/pwalkdir.go new file mode 100644 index 000000000..222820750 --- /dev/null +++ b/vendor/github.com/opencontainers/selinux/pkg/pwalkdir/pwalkdir.go @@ -0,0 +1,103 @@ +// +build go1.16 + +package pwalkdir + +import ( + "fmt" + "io/fs" + "path/filepath" + "runtime" + "sync" +) + +// Walk is a wrapper for filepath.WalkDir which can call multiple walkFn +// in parallel, allowing to handle each item concurrently. A maximum of +// twice the runtime.NumCPU() walkFn will be called at any one time. +// If you want to change the maximum, use WalkN instead. +// +// The order of calls is non-deterministic. +// +// Note that this implementation only supports primitive error handling: +// +// - no errors are ever passed to walkFn; +// +// - once a walkFn returns any error, all further processing stops +// and the error is returned to the caller of Walk; +// +// - filepath.SkipDir is not supported; +// +// - if more than one walkFn instance will return an error, only one +// of such errors will be propagated and returned by Walk, others +// will be silently discarded. +func Walk(root string, walkFn fs.WalkDirFunc) error { + return WalkN(root, walkFn, runtime.NumCPU()*2) +} + +// WalkN is a wrapper for filepath.WalkDir which can call multiple walkFn +// in parallel, allowing to handle each item concurrently. A maximum of +// num walkFn will be called at any one time. +// +// Please see Walk documentation for caveats of using this function. +func WalkN(root string, walkFn fs.WalkDirFunc, num int) error { + // make sure limit is sensible + if num < 1 { + return fmt.Errorf("walk(%q): num must be > 0", root) + } + + files := make(chan *walkArgs, 2*num) + errCh := make(chan error, 1) // Get the first error, ignore others. + + // Start walking a tree asap. + var ( + err error + wg sync.WaitGroup + ) + wg.Add(1) + go func() { + err = filepath.WalkDir(root, func(p string, entry fs.DirEntry, err error) error { + if err != nil { + close(files) + return err + } + // Add a file to the queue unless a callback sent an error. + select { + case e := <-errCh: + close(files) + return e + default: + files <- &walkArgs{path: p, entry: entry} + return nil + } + }) + if err == nil { + close(files) + } + wg.Done() + }() + + wg.Add(num) + for i := 0; i < num; i++ { + go func() { + for file := range files { + if e := walkFn(file.path, file.entry, nil); e != nil { + select { + case errCh <- e: // sent ok + default: // buffer full + } + } + } + wg.Done() + }() + } + + wg.Wait() + + return err +} + +// walkArgs holds the arguments that were passed to the Walk or WalkN +// functions. +type walkArgs struct { + path string + entry fs.DirEntry +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 189ffbf0f..64de28cc3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -198,7 +198,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.33.1 +# github.com/containers/storage v1.34.0 github.com/containers/storage github.com/containers/storage/drivers github.com/containers/storage/drivers/aufs @@ -405,7 +405,8 @@ github.com/json-iterator/go # github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a github.com/juju/ansiterm github.com/juju/ansiterm/tabwriter -# github.com/klauspost/compress v1.13.1 +# github.com/klauspost/compress v1.13.3 +github.com/klauspost/compress github.com/klauspost/compress/flate github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 @@ -522,10 +523,11 @@ github.com/opencontainers/runtime-tools/generate github.com/opencontainers/runtime-tools/generate/seccomp github.com/opencontainers/runtime-tools/specerror github.com/opencontainers/runtime-tools/validate -# github.com/opencontainers/selinux v1.8.3 +# github.com/opencontainers/selinux v1.8.4 github.com/opencontainers/selinux/go-selinux github.com/opencontainers/selinux/go-selinux/label github.com/opencontainers/selinux/pkg/pwalk +github.com/opencontainers/selinux/pkg/pwalkdir # github.com/openshift/imagebuilder v1.2.2-0.20210415181909-87f3e48c2656 github.com/openshift/imagebuilder github.com/openshift/imagebuilder/dockerfile/command |