diff options
149 files changed, 2662 insertions, 380 deletions
diff --git a/.github/workflows/multi-arch-build.yaml b/.github/workflows/multi-arch-build.yaml new file mode 100644 index 000000000..1781604fe --- /dev/null +++ b/.github/workflows/multi-arch-build.yaml @@ -0,0 +1,181 @@ +name: build multi-arch images + +on: + # Upstream podman tends to be very active, with many merges per day. + # Only run this daily via cron schedule, or manually, not by branch push. + schedule: + - cron: '0 8 * * *' + # allows to run this workflow manually from the Actions tab + workflow_dispatch: + +jobs: + multi: + name: multi-arch Podman build + env: + PODMAN_QUAY_REGISTRY: quay.io/podman + CONTAINERS_QUAY_REGISTRY: quay.io/containers + # list of architectures for build + PLATFORMS: linux/amd64,linux/s390x,linux/ppc64le,linux/arm64 + + # build several images (upstream, testing, stable) in parallel + strategy: + matrix: + # Builds are located under contrib/podmanimage/<source> directory + source: + - upstream + - testing + - stable + runs-on: ubuntu-latest + # internal registry caches build for inspection before push + services: + registry: + image: quay.io/libpod/registry:2 + ports: + - 5000:5000 + steps: + - name: Checkout + uses: actions/checkout@v2 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v1 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v1 + with: + driver-opts: network=host + install: true + + - name: Build and locally push Podman + uses: docker/build-push-action@v2 + with: + context: contrib/podmanimage/${{ matrix.source }} + file: ./contrib/podmanimage/${{ matrix.source }}/Dockerfile + platforms: ${{ env.PLATFORMS }} + push: true + tags: localhost:5000/podman/${{ matrix.source }} + + # Simple verification that container works + grab version number + - name: amd64 container sniff test + id: sniff_test + run: | + VERSION_OUTPUT="$(docker run localhost:5000/podman/${{ matrix.source }} \ + podman --storage-driver=vfs version)" + echo "$VERSION_OUTPUT" + VERSION=$(grep -Em1 '^Version: ' <<<"$VERSION_OUTPUT" | awk '{print $2}') + test -n "$VERSION" + echo "::set-output name=version::${VERSION}" + + # Generate image FQINs, labels, check whether to push + - name: Generate image information + id: image_info + run: | + if [[ "${{ matrix.source }}" == 'stable' ]]; then + # quay.io/podman/stable:vX.X.X + ALLTAGS=$(skopeo list-tags \ + docker://${{ env.PODMAN_QUAY_REGISTRY }}/stable | \ + jq -r '.Tags[]') + PUSH="false" + if fgrep -qx "$VERSION" <<<"$ALLTAGS"; then + PUSH="true" + fi + + FQIN='${{ env.PODMAN_QUAY_REGISTRY }}/stable:v${{ steps.sniff_test.outputs.version }}' # workaround vim syntax-hilighting bug: ' + # Only push if version tag does not exist + if [[ "$PUSH" == "true" ]]; then + echo "Will push $FQIN" + echo "::set-output name=podman_push::${PUSH}" + echo "::set-output name=podman_fqin::${FQIN}" + fi + + # quay.io/containers/podman:vX.X.X + unset ALLTAGS + ALLTAGS=$(skopeo list-tags \ + docker://${{ env.CONTAINERS_QUAY_REGISTRY }}/podman | \ + jq -r '.Tags[]') + PUSH="false" + if fgrep -qx "$VERSION" <<<"$ALLTAGS"; then + PUSH="true" + fi + + FQIN='${{ env.CONTAINERS_QUAY_REGISTRY}}/podman:v${{ steps.sniff_test.outputs.version }}' # workaround vim syntax-hilighting bug: ' + # Only push if version tag does not exist + if [[ "$PUSH" == "true" ]]; then + echo "Will push $FQIN" + echo "::set-output name=containers_push::${PUSH}" + echo "::set-output name=containers_fqin::$FQIN" + fi + else # upstream and testing podman image + P_FQIN='${{ env.PODMAN_QUAY_REGISTRY }}/${{ matrix.source }}:master' # workaround vim syntax-hilighting bug: ' + C_FQIN='${{ env.CONTAINERS_QUAY_REGISTRY}}/podman:master' # workaround vim syntax-hilighting bug: ' + echo "Will push $P_FQIN and $C_FQIN" + echo "::set-output name=podman_fqin::${P_FQIN}" + echo "::set-output name=containers_fqin::${C_FQIN}" + # Always push 'master' tag + echo '::set-output name=podman_push::true' + echo '::set-output name=containers_push::true' + fi + + # Hack to set $LABELS env. var. in _future_ steps. + # https://docs.github.com/en/actions/reference/workflow-commands-for-github-actions#multiline-strings + cat << EOF | tee $GITHUB_ENV + LABELS<<DELIMITER + org.opencontainers.image.source=https://github.com/${{ github.repository }}.git + org.opencontainers.image.revision=${{ github.sha }} + org.opencontainers.image.created=$(date -u --iso-8601=seconds) + DELIMITER + EOF + + # Separate steps to login and push for podman and containers quay + # repositories are required, because 2 sets of credentials are used and `docker + # login` as well as `podman login` do not support having 2 different + # credential sets for 1 registry. + # At the same time reuse of non-shell steps is not supported by Github Actions + # via anchors or composite actions + + # Push to 'podman' Quay repo for stable, testing. and upstream + - name: Login to 'podman' Quay registry + uses: docker/login-action@v1 + if: ${{ steps.image_info.outputs.podman_push == 'true' }} + with: + registry: ${{ env.PODMAN_QUAY_REGISTRY }} + # N/B: Secrets are not passed to workflows that are triggered + # by a pull request from a fork + username: ${{ secrets.PODMAN_QUAY_USERNAME }} + password: ${{ secrets.PODMAN_QUAY_PASSWORD }} + + - name: Push images to 'podman' Quay + uses: docker/build-push-action@v2 + if: ${{ steps.image_info.outputs.podman_push == 'true' }} + with: + cache-from: type=registry,ref=localhost:5000/podman/${{ matrix.source }} + cache-to: type=inline + context: contrib/podmanimage/${{ matrix.source }} + file: ./contrib/podmanimage/${{ matrix.source }}/Dockerfile + platforms: ${{ env.PLATFORMS }} + push: true + tags: ${{ steps.image_info.outputs.podman_fqin }} + labels: | + ${{ env.LABELS }} + + # Push to 'containers' Quay repo only stable podman + - name: Login to 'containers' Quay registry + if: ${{ steps.image_info.outputs.containers_push == 'true' }} + uses: docker/login-action@v1 + with: + registry: ${{ env.CONTAINERS_QUAY_REGISTRY}} + username: ${{ secrets.CONTAINERS_QUAY_USERNAME }} + password: ${{ secrets.CONTAINERS_QUAY_PASSWORD }} + + - name: Push images to 'containers' Quay + if: ${{ steps.image_info.outputs.containers_push == 'true' }} + uses: docker/build-push-action@v2 + with: + cache-from: type=registry,ref=localhost:5000/podman/${{ matrix.source }} + cache-to: type=inline + context: contrib/podmanimage/${{ matrix.source }} + file: ./contrib/podmanimage/${{ matrix.source }}/Dockerfile + platforms: ${{ env.PLATFORMS }} + push: true + tags: ${{ steps.image_info.outputs.containers_fqin }} + labels: | + ${{ env.LABELS }} @@ -149,7 +149,7 @@ err_if_empty = $(if $(strip $($(1))),$(strip $($(1))),$(error Required variable # Podman does not work w/o CGO_ENABLED, except in some very specific cases CGO_ENABLED ?= 1 -# Default to the native OS type and archetecture unless otherwise specified +# Default to the native OS type and architecture unless otherwise specified GOOS ?= $(shell $(GO) env GOOS) ifeq ($(call err_if_empty,GOOS),windows) BINSFX := .exe @@ -255,7 +255,7 @@ test/goecho/goecho: .gopathok $(wildcard test/goecho/*.go) .PHONY: codespell codespell: - codespell -S bin,vendor,.git,go.sum,changelog.txt,.cirrus.yml,"RELEASE_NOTES.md,*.xz,*.gz,*.tar,*.tgz,bin2img,*ico,*.png,*.1,*.5,copyimg,*.orig,apidoc.go" -L uint,iff,od,seeked,splitted,marge,ERRO,hist -w + codespell -S bin,vendor,.git,go.sum,changelog.txt,.cirrus.yml,"RELEASE_NOTES.md,*.xz,*.gz,*.tar,*.tgz,bin2img,*ico,*.png,*.1,*.5,copyimg,*.orig,apidoc.go" -L uint,iff,od,seeked,splitted,marge,ERRO,hist,ether -w .PHONY: validate validate: gofmt lint .gitvalidation validate.completions man-page-check swagger-check tests-included diff --git a/RELEASE_PROCESS.md b/RELEASE_PROCESS.md index 52c08c3f2..e063fa617 100644 --- a/RELEASE_PROCESS.md +++ b/RELEASE_PROCESS.md @@ -196,7 +196,7 @@ spelled with complete minutiae. 1. Merge the PR (or ask someone else to review and merge, to be safer). 1. **Note:** This is the last point where any test-failures can be addressed by code changes. After pushing the new version-tag upstream, no further - changes can be made to the code without lots of unpleasent efforts. Please + changes can be made to the code without lots of unpleasant efforts. Please seek assistance if needed, before proceeding. 1. Assuming the "Bump to ..." PR merged successfully, and you're **really** diff --git a/cmd/podman/common/completion.go b/cmd/podman/common/completion.go index 6086df297..4aca79770 100644 --- a/cmd/podman/common/completion.go +++ b/cmd/podman/common/completion.go @@ -4,6 +4,7 @@ import ( "bufio" "fmt" "os" + "reflect" "strings" "github.com/containers/common/pkg/config" @@ -891,10 +892,85 @@ func AutocompleteNetworkFlag(cmd *cobra.Command, args []string, toComplete strin return append(networks, suggestions...), dir } -// AutocompleteJSONFormat - Autocomplete format flag option. -// -> "json" -func AutocompleteJSONFormat(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - return []string{"json"}, cobra.ShellCompDirectiveNoFileComp +// AutocompleteFormat - Autocomplete json or a given struct to use for a go template. +// The input can be nil, In this case only json will be autocompleted. +// This function will only work for structs other types are not supported. +// When "{{." is typed the field and method names of the given struct will be completed. +// This also works recursive for nested structs. +func AutocompleteFormat(o interface{}) func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + // this function provides shell completion for go templates + return func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { + // autocomplete json when nothing or json is typed + if strings.HasPrefix("json", toComplete) { + return []string{"json"}, cobra.ShellCompDirectiveNoFileComp + } + // no input struct we cannot provide completion return nothing + if o == nil { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + // toComplete could look like this: {{ .Config }} {{ .Field.F + // 1. split the template variable delimiter + vars := strings.Split(toComplete, "{{") + if len(vars) == 1 { + // no variables return no completion + return nil, cobra.ShellCompDirectiveNoFileComp + } + // clean the spaces from the last var + field := strings.Split(vars[len(vars)-1], " ") + // split this into it struct field names + fields := strings.Split(field[len(field)-1], ".") + f := reflect.ValueOf(o) + for i := 1; i < len(fields); i++ { + if f.Kind() == reflect.Ptr { + f = f.Elem() + } + + // // the only supported type is struct + if f.Kind() != reflect.Struct { + return nil, cobra.ShellCompDirectiveNoFileComp + } + + // last field get all names to suggest + if i == len(fields)-1 { + suggestions := []string{} + for j := 0; j < f.NumField(); j++ { + fname := f.Type().Field(j).Name + suffix := "}}" + kind := f.Type().Field(j).Type.Kind() + if kind == reflect.Ptr { + // make sure to read the actual type when it is a pointer + kind = f.Type().Field(j).Type.Elem().Kind() + } + // when we have a nested struct do not append braces instead append a dot + if kind == reflect.Struct { + suffix = "." + } + if strings.HasPrefix(fname, fields[i]) { + // add field name with closing braces + suggestions = append(suggestions, fname+suffix) + } + } + + for j := 0; j < f.NumMethod(); j++ { + fname := f.Type().Method(j).Name + if strings.HasPrefix(fname, fields[i]) { + // add method name with closing braces + suggestions = append(suggestions, fname+"}}") + } + } + + // add the current toComplete value in front so that the shell can complete this correctly + toCompArr := strings.Split(toComplete, ".") + toCompArr[len(toCompArr)-1] = "" + toComplete = strings.Join(toCompArr, ".") + return prefixSlice(toComplete, suggestions), cobra.ShellCompDirectiveNoSpace | cobra.ShellCompDirectiveNoFileComp + } + // set the next struct field + f = f.FieldByName(fields[i]) + } + return nil, cobra.ShellCompDirectiveNoFileComp + } } // AutocompleteEventFilter - Autocomplete event filter flag options. diff --git a/cmd/podman/common/completion_test.go b/cmd/podman/common/completion_test.go new file mode 100644 index 000000000..5bd627b85 --- /dev/null +++ b/cmd/podman/common/completion_test.go @@ -0,0 +1,142 @@ +package common_test + +import ( + "testing" + + "github.com/containers/podman/v3/cmd/podman/common" + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" +) + +type Car struct { + Brand string + Stats struct { + HP *int + Displacement int + } + Extras map[string]string +} + +func (c Car) Type() string { + return "" +} + +func (c Car) Color() string { + return "" +} + +func TestAutocompleteFormat(t *testing.T) { + testStruct := struct { + Name string + Age int + Car *Car + }{} + + testStruct.Car = &Car{} + testStruct.Car.Extras = map[string]string{"test": "1"} + + tests := []struct { + name string + toComplete string + expected []string + }{ + { + "empty completion", + "", + []string{"json"}, + }, + { + "json completion", + "json", + []string{"json"}, + }, + { + "invalid completion", + "blahblah", + nil, + }, + { + "invalid completion", + "{{", + nil, + }, + { + "invalid completion", + "{{ ", + nil, + }, + { + "invalid completion", + "{{ ..", + nil, + }, + { + "fist level struct field name", + "{{.", + []string{"{{.Name}}", "{{.Age}}", "{{.Car."}, + }, + { + "fist level struct field name", + "{{ .", + []string{"{{ .Name}}", "{{ .Age}}", "{{ .Car."}, + }, + { + "fist level struct field name", + "{{ .N", + []string{"{{ .Name}}"}, + }, + { + "second level struct field name", + "{{ .Car.", + []string{"{{ .Car.Brand}}", "{{ .Car.Stats.", "{{ .Car.Extras}}", "{{ .Car.Color}}", "{{ .Car.Type}}"}, + }, + { + "second level struct field name", + "{{ .Car.B", + []string{"{{ .Car.Brand}}"}, + }, + { + "three level struct field name", + "{{ .Car.Stats.", + []string{"{{ .Car.Stats.HP}}", "{{ .Car.Stats.Displacement}}"}, + }, + { + "three level struct field name", + "{{ .Car.Stats.D", + []string{"{{ .Car.Stats.Displacement}}"}, + }, + { + "second level struct field name", + "{{ .Car.B", + []string{"{{ .Car.Brand}}"}, + }, + { + "invalid field name", + "{{ .Ca.B", + nil, + }, + { + "map key names don't work", + "{{ .Car.Extras.", + nil, + }, + { + "two variables struct field name", + "{{ .Car.Brand }} {{ .Car.", + []string{"{{ .Car.Brand }} {{ .Car.Brand}}", "{{ .Car.Brand }} {{ .Car.Stats.", "{{ .Car.Brand }} {{ .Car.Extras}}", + "{{ .Car.Brand }} {{ .Car.Color}}", "{{ .Car.Brand }} {{ .Car.Type}}"}, + }, + { + "only dot without variable", + ".", + nil, + }, + } + + for _, test := range tests { + completion, directive := common.AutocompleteFormat(testStruct)(nil, nil, test.toComplete) + // directive should always be greater than ShellCompDirectiveNoFileComp + assert.GreaterOrEqual(t, directive, cobra.ShellCompDirectiveNoFileComp, "unexpected ShellCompDirective") + assert.Equal(t, test.expected, completion, test.name) + } +} diff --git a/cmd/podman/common/create.go b/cmd/podman/common/create.go index da391d30d..d496ae308 100644 --- a/cmd/podman/common/create.go +++ b/cmd/podman/common/create.go @@ -277,7 +277,7 @@ func DefineCreateFlags(cmd *cobra.Command, cf *ContainerCLIOpts) { createFlags.StringSliceVar( &cf.GroupAdd, groupAddFlagName, []string{}, - "Add additional groups to join", + "Add additional groups to the primary container process. 'keep-groups' allows container processes to use suplementary groups.", ) _ = cmd.RegisterFlagCompletionFunc(groupAddFlagName, completion.AutocompleteNone) diff --git a/cmd/podman/common/create_opts.go b/cmd/podman/common/create_opts.go index 040dc6570..983b9e5ca 100644 --- a/cmd/podman/common/create_opts.go +++ b/cmd/podman/common/create_opts.go @@ -252,21 +252,24 @@ func ContainerCreateToContainerCLIOpts(cc handlers.CreateContainerConfig, cgroup return nil, nil, err } - netNS := specgen.Namespace{ - NSMode: nsmode.NSMode, - Value: nsmode.Value, + var netOpts map[string][]string + parts := strings.SplitN(string(cc.HostConfig.NetworkMode), ":", 2) + if len(parts) > 1 { + netOpts = make(map[string][]string) + netOpts[parts[0]] = strings.Split(parts[1], ",") } // network // Note: we cannot emulate compat exactly here. we only allow specifics of networks to be // defined when there is only one network. netInfo := entities.NetOptions{ - AddHosts: cc.HostConfig.ExtraHosts, - DNSOptions: cc.HostConfig.DNSOptions, - DNSSearch: cc.HostConfig.DNSSearch, - DNSServers: dns, - Network: netNS, - PublishPorts: specPorts, + AddHosts: cc.HostConfig.ExtraHosts, + DNSOptions: cc.HostConfig.DNSOptions, + DNSSearch: cc.HostConfig.DNSSearch, + DNSServers: dns, + Network: nsmode, + PublishPorts: specPorts, + NetworkOptions: netOpts, } // network names diff --git a/cmd/podman/containers/create.go b/cmd/podman/containers/create.go index 507e9c221..3f495e19b 100644 --- a/cmd/podman/containers/create.go +++ b/cmd/podman/containers/create.go @@ -193,6 +193,25 @@ func createInit(c *cobra.Command) error { val := c.Flag("entrypoint").Value.String() cliVals.Entrypoint = &val } + + if c.Flags().Changed("group-add") { + groups := []string{} + for _, g := range cliVals.GroupAdd { + if g == "keep-groups" { + if len(cliVals.GroupAdd) > 1 { + return errors.New("the '--group-add keep-groups' option is not allowed with any other --group-add options") + } + if registry.IsRemote() { + return errors.New("the '--group-add keep-groups' option is not supported in remote mode") + } + cliVals.Annotation = append(cliVals.Annotation, "run.oci.keep_original_groups=1") + } else { + groups = append(groups, g) + } + } + cliVals.GroupAdd = groups + } + if c.Flags().Changed("pids-limit") { val := c.Flag("pids-limit").Value.String() pidsLimit, err := strconv.ParseInt(val, 10, 32) diff --git a/cmd/podman/containers/diff.go b/cmd/podman/containers/diff.go index f6f262066..799d01127 100644 --- a/cmd/podman/containers/diff.go +++ b/cmd/podman/containers/diff.go @@ -39,7 +39,7 @@ func init() { formatFlagName := "format" flags.StringVar(&diffOpts.Format, formatFlagName, "", "Change the output format") - _ = diffCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = diffCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(nil)) validate.AddLatestFlag(diffCmd, &diffOpts.Latest) } diff --git a/cmd/podman/containers/inspect.go b/cmd/podman/containers/inspect.go index e7921fc39..eb29b7285 100644 --- a/cmd/podman/containers/inspect.go +++ b/cmd/podman/containers/inspect.go @@ -5,6 +5,7 @@ import ( "github.com/containers/podman/v3/cmd/podman/inspect" "github.com/containers/podman/v3/cmd/podman/registry" "github.com/containers/podman/v3/cmd/podman/validate" + "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/spf13/cobra" ) @@ -35,7 +36,12 @@ func init() { formatFlagName := "format" flags.StringVarP(&inspectOpts.Format, formatFlagName, "f", "json", "Format the output to a Go template or json") - _ = inspectCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = inspectCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(define.InspectContainerData{ + State: &define.InspectContainerState{}, + NetworkSettings: &define.InspectNetworkSettings{}, + Config: &define.InspectContainerConfig{}, + HostConfig: &define.InspectContainerHostConfig{}, + })) validate.AddLatestFlag(inspectCmd, &inspectOpts.Latest) } diff --git a/cmd/podman/containers/mount.go b/cmd/podman/containers/mount.go index 7853bfae6..fd5a279d2 100644 --- a/cmd/podman/containers/mount.go +++ b/cmd/podman/containers/mount.go @@ -61,7 +61,7 @@ func mountFlags(cmd *cobra.Command) { formatFlagName := "format" flags.StringVar(&mountOpts.Format, formatFlagName, "", "Print the mounted containers in specified format (json)") - _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(nil)) flags.BoolVar(&mountOpts.NoTruncate, "notruncate", false, "Do not truncate output") } diff --git a/cmd/podman/containers/ps.go b/cmd/podman/containers/ps.go index 70bfd3574..3c0162676 100644 --- a/cmd/podman/containers/ps.go +++ b/cmd/podman/containers/ps.go @@ -87,7 +87,7 @@ func listFlagSet(cmd *cobra.Command) { formatFlagName := "format" flags.StringVar(&listOpts.Format, formatFlagName, "", "Pretty-print containers to JSON or using a Go template") - _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(entities.ListContainer{})) lastFlagName := "last" flags.IntVarP(&listOpts.Last, lastFlagName, "n", -1, "Print the n last created containers (all states)") @@ -97,6 +97,7 @@ func listFlagSet(cmd *cobra.Command) { flags.BoolVar(&noTrunc, "no-trunc", false, "Display the extended information") flags.BoolVarP(&listOpts.Pod, "pod", "p", false, "Print the ID and name of the pod the containers are associated with") flags.BoolVarP(&listOpts.Quiet, "quiet", "q", false, "Print the numeric IDs of the containers only") + flags.Bool("noheading", false, "Do not print headers") flags.BoolVarP(&listOpts.Size, "size", "s", false, "Display the total file sizes") flags.BoolVar(&listOpts.Sync, "sync", false, "Sync container state with OCI runtime") @@ -242,7 +243,8 @@ func ps(cmd *cobra.Command, _ []string) error { defer w.Flush() headers := func() error { return nil } - if !(listOpts.Quiet || cmd.Flags().Changed("format")) { + noHeading, _ := cmd.Flags().GetBool("noheading") + if !(noHeading || listOpts.Quiet || cmd.Flags().Changed("format")) { headers = func() error { return tmpl.Execute(w, hdrs) } diff --git a/cmd/podman/containers/stats.go b/cmd/podman/containers/stats.go index 4c31896be..7160f1ba8 100644 --- a/cmd/podman/containers/stats.go +++ b/cmd/podman/containers/stats.go @@ -71,7 +71,7 @@ func statFlags(cmd *cobra.Command) { formatFlagName := "format" flags.StringVar(&statsOptions.Format, formatFlagName, "", "Pretty-print container statistics to JSON or using a Go template") - _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(define.ContainerStats{})) flags.BoolVar(&statsOptions.NoReset, "no-reset", false, "Disable resetting the screen between intervals") flags.BoolVar(&statsOptions.NoStream, "no-stream", false, "Disable streaming stats and only pull the first result, default setting is false") diff --git a/cmd/podman/diff.go b/cmd/podman/diff.go index 4862d31b5..ae7d6c4bc 100644 --- a/cmd/podman/diff.go +++ b/cmd/podman/diff.go @@ -43,7 +43,7 @@ func init() { formatFlagName := "format" flags.StringVar(&diffOpts.Format, formatFlagName, "", "Change the output format") - _ = diffCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = diffCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(nil)) validate.AddLatestFlag(diffCmd, &diffOpts.Latest) } diff --git a/cmd/podman/generate/systemd.go b/cmd/podman/generate/systemd.go index 693506725..72b2e6335 100644 --- a/cmd/podman/generate/systemd.go +++ b/cmd/podman/generate/systemd.go @@ -72,7 +72,7 @@ func init() { formatFlagName := "format" flags.StringVar(&format, formatFlagName, "", "Print the created units in specified format (json)") - _ = systemdCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = systemdCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(nil)) flags.SetNormalizeFunc(utils.AliasFlags) } diff --git a/cmd/podman/images/diff.go b/cmd/podman/images/diff.go index 36b0d4e7a..7f4c3e83d 100644 --- a/cmd/podman/images/diff.go +++ b/cmd/podman/images/diff.go @@ -41,7 +41,7 @@ func diffFlags(flags *pflag.FlagSet) { formatFlagName := "format" flags.StringVar(&diffOpts.Format, formatFlagName, "", "Change the output format") - _ = diffCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = diffCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(nil)) } func diff(cmd *cobra.Command, args []string) error { diff --git a/cmd/podman/images/history.go b/cmd/podman/images/history.go index eaf56651f..16be0bb19 100644 --- a/cmd/podman/images/history.go +++ b/cmd/podman/images/history.go @@ -74,7 +74,7 @@ func historyFlags(cmd *cobra.Command) { formatFlagName := "format" flags.StringVar(&opts.format, formatFlagName, "", "Change the output to JSON or a Go template") - _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(entities.ImageHistoryLayer{})) flags.BoolVarP(&opts.human, "human", "H", true, "Display sizes and dates in human readable format") flags.BoolVar(&opts.noTrunc, "no-trunc", false, "Do not truncate the output") diff --git a/cmd/podman/images/inspect.go b/cmd/podman/images/inspect.go index fb96286fa..ac3becaa6 100644 --- a/cmd/podman/images/inspect.go +++ b/cmd/podman/images/inspect.go @@ -5,6 +5,7 @@ import ( "github.com/containers/podman/v3/cmd/podman/inspect" "github.com/containers/podman/v3/cmd/podman/registry" "github.com/containers/podman/v3/pkg/domain/entities" + inspectTypes "github.com/containers/podman/v3/pkg/inspect" "github.com/spf13/cobra" ) @@ -34,7 +35,7 @@ func init() { formatFlagName := "format" flags.StringVarP(&inspectOpts.Format, formatFlagName, "f", "json", "Format the output to a Go template or json") - _ = inspectCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = inspectCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(inspectTypes.ImageData{})) } func inspectExec(cmd *cobra.Command, args []string) error { diff --git a/cmd/podman/images/list.go b/cmd/podman/images/list.go index 7a9a8a804..132af858b 100644 --- a/cmd/podman/images/list.go +++ b/cmd/podman/images/list.go @@ -83,7 +83,7 @@ func imageListFlagSet(cmd *cobra.Command) { formatFlagName := "format" flags.StringVar(&listFlag.format, formatFlagName, "", "Change the output format to JSON or a Go template") - _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(entities.ImageSummary{})) flags.BoolVar(&listFlag.digests, "digests", false, "Show digests") flags.BoolVarP(&listFlag.noHeading, "noheading", "n", false, "Do not print column headings") diff --git a/cmd/podman/images/mount.go b/cmd/podman/images/mount.go index 79c97006d..a098aac63 100644 --- a/cmd/podman/images/mount.go +++ b/cmd/podman/images/mount.go @@ -51,7 +51,7 @@ func mountFlags(cmd *cobra.Command) { formatFlagName := "format" flags.StringVar(&mountOpts.Format, formatFlagName, "", "Print the mounted images in specified format (json)") - _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(nil)) } func init() { diff --git a/cmd/podman/machine/list.go b/cmd/podman/machine/list.go index ce4129e87..af4e2c807 100644 --- a/cmd/podman/machine/list.go +++ b/cmd/podman/machine/list.go @@ -61,6 +61,7 @@ func init() { formatFlagName := "format" flags.StringVar(&listFlag.format, formatFlagName, "{{.Name}}\t{{.VMType}}\t{{.Created}}\t{{.LastUp}}\n", "Format volume output using Go template") _ = lsCmd.RegisterFlagCompletionFunc(formatFlagName, completion.AutocompleteNone) + flags.BoolVar(&listFlag.noHeading, "noheading", false, "Do not print headers") } func list(cmd *cobra.Command, args []string) error { diff --git a/cmd/podman/machine/stop.go b/cmd/podman/machine/stop.go index 4235b64f1..4307d3eeb 100644 --- a/cmd/podman/machine/stop.go +++ b/cmd/podman/machine/stop.go @@ -30,7 +30,7 @@ func init() { }) } -// TODO Name shouldnt be required, need to create a default vm +// TODO Name shouldn't be required, need to create a default vm func stop(cmd *cobra.Command, args []string) error { var ( err error diff --git a/cmd/podman/networks/inspect.go b/cmd/podman/networks/inspect.go index 953f5e6e8..a05b9026d 100644 --- a/cmd/podman/networks/inspect.go +++ b/cmd/podman/networks/inspect.go @@ -33,7 +33,7 @@ func init() { formatFlagName := "format" flags.StringVarP(&inspectOpts.Format, formatFlagName, "f", "", "Pretty-print network to JSON or using a Go template") - _ = networkinspectCommand.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = networkinspectCommand.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(nil)) } func networkInspect(_ *cobra.Command, args []string) error { diff --git a/cmd/podman/networks/list.go b/cmd/podman/networks/list.go index fcbcb6722..e1b182cbf 100644 --- a/cmd/podman/networks/list.go +++ b/cmd/podman/networks/list.go @@ -41,13 +41,14 @@ var ( func networkListFlags(flags *pflag.FlagSet) { formatFlagName := "format" flags.StringVar(&networkListOptions.Format, formatFlagName, "", "Pretty-print networks to JSON or using a Go template") - _ = networklistCommand.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = networklistCommand.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(ListPrintReports{})) flags.BoolVarP(&networkListOptions.Quiet, "quiet", "q", false, "display only names") flags.BoolVar(&noTrunc, "no-trunc", false, "Do not truncate the network ID") filterFlagName := "filter" flags.StringArrayVarP(&filters, filterFlagName, "f", nil, "Provide filter values (e.g. 'name=podman')") + flags.Bool("noheading", false, "Do not print headers") _ = networklistCommand.RegisterFlagCompletionFunc(filterFlagName, common.AutocompleteNetworkFilters) } @@ -140,7 +141,8 @@ func templateOut(responses []*entities.NetworkListReport, cmd *cobra.Command) er w := tabwriter.NewWriter(os.Stdout, 8, 2, 2, ' ', 0) defer w.Flush() - if renderHeaders { + noHeading, _ := cmd.Flags().GetBool("noheading") + if !noHeading && renderHeaders { if err := tmpl.Execute(w, headers); err != nil { return err } diff --git a/cmd/podman/pods/inspect.go b/cmd/podman/pods/inspect.go index 96e007fa3..c66b81adb 100644 --- a/cmd/podman/pods/inspect.go +++ b/cmd/podman/pods/inspect.go @@ -10,6 +10,7 @@ import ( "github.com/containers/podman/v3/cmd/podman/common" "github.com/containers/podman/v3/cmd/podman/registry" "github.com/containers/podman/v3/cmd/podman/validate" + "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -44,7 +45,7 @@ func init() { formatFlagName := "format" flags.StringVarP(&inspectOptions.Format, formatFlagName, "f", "json", "Format the output to a Go template or json") - _ = inspectCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = inspectCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(define.InspectPodData{})) validate.AddLatestFlag(inspectCmd, &inspectOptions.Latest) } diff --git a/cmd/podman/pods/ps.go b/cmd/podman/pods/ps.go index aeba80525..beaeda871 100644 --- a/cmd/podman/pods/ps.go +++ b/cmd/podman/pods/ps.go @@ -61,8 +61,9 @@ func init() { formatFlagName := "format" flags.StringVar(&psInput.Format, formatFlagName, "", "Pretty-print pods to JSON or using a Go template") - _ = psCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = psCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(ListPodReporter{})) + flags.Bool("noheading", false, "Do not print headers") flags.BoolVar(&psInput.Namespace, "namespace", false, "Display namespace information of the pod") flags.BoolVar(&psInput.Namespace, "ns", false, "Display namespace information of the pod") flags.BoolVar(&noTrunc, "no-trunc", false, "Do not truncate pod and container IDs") @@ -134,6 +135,10 @@ func pods(cmd *cobra.Command, _ []string) error { renderHeaders = parse.HasTable(psInput.Format) row = report.NormalizeFormat(psInput.Format) } + noHeading, _ := cmd.Flags().GetBool("noheading") + if noHeading { + renderHeaders = false + } format := parse.EnforceRange(row) tmpl, err := template.New("listPods").Parse(format) diff --git a/cmd/podman/pods/stats.go b/cmd/podman/pods/stats.go index e336b864e..97147275e 100644 --- a/cmd/podman/pods/stats.go +++ b/cmd/podman/pods/stats.go @@ -58,7 +58,7 @@ func init() { formatFlagName := "format" flags.StringVar(&statsOptions.Format, formatFlagName, "", "Pretty-print container statistics to JSON or using a Go template") - _ = statsCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = statsCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(entities.PodStatsReport{})) flags.BoolVar(&statsOptions.NoReset, "no-reset", false, "Disable resetting the screen when streaming") flags.BoolVar(&statsOptions.NoStream, "no-stream", false, "Disable streaming stats and only pull the first result") diff --git a/cmd/podman/secrets/inspect.go b/cmd/podman/secrets/inspect.go index 4036291ec..bcb1adb5e 100644 --- a/cmd/podman/secrets/inspect.go +++ b/cmd/podman/secrets/inspect.go @@ -40,7 +40,7 @@ func init() { flags := inspectCmd.Flags() formatFlagName := "format" flags.StringVar(&format, formatFlagName, "", "Format volume output using Go template") - _ = inspectCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = inspectCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(entities.SecretInfoReport{})) } func inspect(cmd *cobra.Command, args []string) error { diff --git a/cmd/podman/secrets/list.go b/cmd/podman/secrets/list.go index 849a8418e..ba7065d61 100644 --- a/cmd/podman/secrets/list.go +++ b/cmd/podman/secrets/list.go @@ -47,7 +47,8 @@ func init() { flags := lsCmd.Flags() formatFlagName := "format" flags.StringVar(&listFlag.format, formatFlagName, "{{.ID}}\t{{.Name}}\t{{.Driver}}\t{{.CreatedAt}}\t{{.UpdatedAt}}\t\n", "Format volume output using Go template") - _ = lsCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = lsCmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(entities.SecretInfoReport{})) + flags.BoolVar(&listFlag.noHeading, "noheading", false, "Do not print headers") } func ls(cmd *cobra.Command, args []string) error { diff --git a/cmd/podman/system/events.go b/cmd/podman/system/events.go index 0f52282a9..568610bdc 100644 --- a/cmd/podman/system/events.go +++ b/cmd/podman/system/events.go @@ -52,7 +52,7 @@ func init() { formatFlagName := "format" flags.StringVar(&eventFormat, formatFlagName, "", "format the output using a Go template") - _ = eventsCommand.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = eventsCommand.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(events.Event{})) flags.BoolVar(&eventOptions.Stream, "stream", true, "stream new events; for testing only") diff --git a/cmd/podman/system/info.go b/cmd/podman/system/info.go index 2babd49c8..afd5b3a34 100644 --- a/cmd/podman/system/info.go +++ b/cmd/podman/system/info.go @@ -10,6 +10,7 @@ import ( "github.com/containers/podman/v3/cmd/podman/common" "github.com/containers/podman/v3/cmd/podman/registry" "github.com/containers/podman/v3/cmd/podman/validate" + "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/ghodss/yaml" "github.com/spf13/cobra" @@ -68,7 +69,7 @@ func infoFlags(cmd *cobra.Command) { formatFlagName := "format" flags.StringVarP(&inFormat, formatFlagName, "f", "", "Change the output format to JSON or a Go template") - _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = cmd.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(define.Info{Host: &define.HostInfo{}, Store: &define.StoreInfo{}})) } func info(cmd *cobra.Command, args []string) error { diff --git a/cmd/podman/system/service_abi.go b/cmd/podman/system/service_abi.go index 9e8a9f9b4..364663323 100644 --- a/cmd/podman/system/service_abi.go +++ b/cmd/podman/system/service_abi.go @@ -6,11 +6,13 @@ import ( "context" "net" "os" + "path/filepath" "strings" api "github.com/containers/podman/v3/pkg/api/server" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/domain/infra" + "github.com/containers/podman/v3/pkg/util" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/pflag" @@ -24,6 +26,17 @@ func restService(opts entities.ServiceOptions, flags *pflag.FlagSet, cfg *entiti ) if opts.URI != "" { + fields := strings.Split(opts.URI, ":") + if len(fields) == 1 { + return errors.Errorf("%s is an invalid socket destination", opts.URI) + } + path := opts.URI + if fields[0] == "unix" { + if path, err = filepath.Abs(fields[1]); err != nil { + return err + } + } + util.SetSocketPath(path) if os.Getenv("LISTEN_FDS") != "" { // If it is activated by systemd, use the first LISTEN_FD (3) // instead of opening the socket file. @@ -34,10 +47,6 @@ func restService(opts entities.ServiceOptions, flags *pflag.FlagSet, cfg *entiti } listener = &l } else { - fields := strings.Split(opts.URI, ":") - if len(fields) == 1 { - return errors.Errorf("%s is an invalid socket destination", opts.URI) - } network := fields[0] address := strings.Join(fields[1:], ":") l, err := net.Listen(network, address) diff --git a/cmd/podman/system/version.go b/cmd/podman/system/version.go index dfb10c080..ad9fd2a85 100644 --- a/cmd/podman/system/version.go +++ b/cmd/podman/system/version.go @@ -38,7 +38,7 @@ func init() { formatFlagName := "format" flags.StringVarP(&versionFormat, formatFlagName, "f", "", "Change the output format to JSON or a Go template") - _ = versionCommand.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = versionCommand.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(entities.SystemVersionReport{})) } func version(cmd *cobra.Command, args []string) error { diff --git a/cmd/podman/volumes/inspect.go b/cmd/podman/volumes/inspect.go index 91269e3d1..d52eee9f3 100644 --- a/cmd/podman/volumes/inspect.go +++ b/cmd/podman/volumes/inspect.go @@ -4,6 +4,7 @@ import ( "github.com/containers/podman/v3/cmd/podman/common" "github.com/containers/podman/v3/cmd/podman/inspect" "github.com/containers/podman/v3/cmd/podman/registry" + "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -41,7 +42,7 @@ func init() { formatFlagName := "format" flags.StringVarP(&inspectOpts.Format, formatFlagName, "f", "json", "Format volume output using Go template") - _ = inspectCommand.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = inspectCommand.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(define.InspectVolumeData{})) } func volumeInspect(cmd *cobra.Command, args []string) error { diff --git a/cmd/podman/volumes/list.go b/cmd/podman/volumes/list.go index e04f452d4..f402afa94 100644 --- a/cmd/podman/volumes/list.go +++ b/cmd/podman/volumes/list.go @@ -14,6 +14,7 @@ import ( "github.com/containers/podman/v3/cmd/podman/parse" "github.com/containers/podman/v3/cmd/podman/registry" "github.com/containers/podman/v3/cmd/podman/validate" + "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/pkg/errors" "github.com/spf13/cobra" @@ -60,8 +61,9 @@ func init() { formatFlagName := "format" flags.StringVar(&cliOpts.Format, formatFlagName, "{{.Driver}}\t{{.Name}}\n", "Format volume output using Go template") - _ = lsCommand.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteJSONFormat) + _ = lsCommand.RegisterFlagCompletionFunc(formatFlagName, common.AutocompleteFormat(define.InspectVolumeData{})) + flags.Bool("noheading", false, "Do not print headers") flags.BoolVarP(&cliOpts.Quiet, "quiet", "q", false, "Print volume output in quiet mode") } @@ -94,6 +96,7 @@ func list(cmd *cobra.Command, args []string) error { } func outputTemplate(cmd *cobra.Command, responses []*entities.VolumeListReport) error { + noHeading, _ := cmd.Flags().GetBool("noheading") headers := report.Headers(entities.VolumeListReport{}, map[string]string{ "Name": "VOLUME NAME", }) @@ -111,7 +114,7 @@ func outputTemplate(cmd *cobra.Command, responses []*entities.VolumeListReport) w := tabwriter.NewWriter(os.Stdout, 12, 2, 2, ' ', 0) defer w.Flush() - if !cliOpts.Quiet && !cmd.Flag("format").Changed { + if !(noHeading || cliOpts.Quiet || cmd.Flag("format").Changed) { if err := tmpl.Execute(w, headers); err != nil { return errors.Wrapf(err, "failed to write report column headers") } diff --git a/completions/powershell/podman-remote.ps1 b/completions/powershell/podman-remote.ps1 index 9cdbabc52..875684b34 100644 --- a/completions/powershell/podman-remote.ps1 +++ b/completions/powershell/podman-remote.ps1 @@ -161,7 +161,7 @@ Register-ArgumentCompleter -CommandName 'podman-remote' -ScriptBlock { $Values | ForEach-Object { - # store temporay because switch will overwrite $_ + # store temporary because switch will overwrite $_ $comp = $_ # PowerShell supports three different completion modes @@ -216,7 +216,7 @@ Register-ArgumentCompleter -CommandName 'podman-remote' -ScriptBlock { Default { # Like MenuComplete but we don't want to add a space here because # the user need to press space anyway to get the completion. - # Description will not be shown because thats not possible with TabCompleteNext + # Description will not be shown because that's not possible with TabCompleteNext [System.Management.Automation.CompletionResult]::new($($comp.Name | __podman-remote_escapeStringWithSpecialChars), "$($comp.Name)", 'ParameterValue', "$($comp.Description)") } } diff --git a/completions/powershell/podman.ps1 b/completions/powershell/podman.ps1 index 6b6f832d2..619c5beea 100644 --- a/completions/powershell/podman.ps1 +++ b/completions/powershell/podman.ps1 @@ -161,7 +161,7 @@ Register-ArgumentCompleter -CommandName 'podman' -ScriptBlock { $Values | ForEach-Object { - # store temporay because switch will overwrite $_ + # store temporary because switch will overwrite $_ $comp = $_ # PowerShell supports three different completion modes @@ -216,7 +216,7 @@ Register-ArgumentCompleter -CommandName 'podman' -ScriptBlock { Default { # Like MenuComplete but we don't want to add a space here because # the user need to press space anyway to get the completion. - # Description will not be shown because thats not possible with TabCompleteNext + # Description will not be shown because that's not possible with TabCompleteNext [System.Management.Automation.CompletionResult]::new($($comp.Name | __podman_escapeStringWithSpecialChars), "$($comp.Name)", 'ParameterValue', "$($comp.Description)") } } diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh index b463745d1..47f3c9405 100755 --- a/contrib/cirrus/runner.sh +++ b/contrib/cirrus/runner.sh @@ -277,7 +277,7 @@ logformatter() { |& awk --file "${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk" \ |& "${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/logformatter" "$output_name" else - # Assume script is run by a human, they want output immediatly + # Assume script is run by a human, they want output immediately cat - fi } diff --git a/docs/remote-docs.sh b/docs/remote-docs.sh index 2b7d73cd3..939c7264c 100755 --- a/docs/remote-docs.sh +++ b/docs/remote-docs.sh @@ -6,7 +6,7 @@ PLATFORM=$1 ## linux, windows or darwin TARGET=${2} ## where to output files SOURCES=${@:3} ## directories to find markdown files -# Overriden for testing. Native podman-remote binary expected filepaths +# Overridden for testing. Native podman-remote binary expected filepaths if [[ -z "$PODMAN" ]]; then case $(env -i HOME=$HOME PATH=$PATH go env GOOS) in windows) diff --git a/docs/source/markdown/podman-build.1.md b/docs/source/markdown/podman-build.1.md index 876bfe412..791e2d907 100644 --- a/docs/source/markdown/podman-build.1.md +++ b/docs/source/markdown/podman-build.1.md @@ -688,7 +688,7 @@ Set the architecture variant of the image to be pulled. bind mounts `/HOST-DIR` in the host to `/CONTAINER-DIR` in the Podman container. (This option is not available with the remote Podman client) - The `OPTIONS` are a comma delimited list and can be: <sup>[[1]](#Footnote1)</sup> + The `OPTIONS` are a comma-separated list and can be: <sup>[[1]](#Footnote1)</sup> * [rw|ro] * [z|Z|O] diff --git a/docs/source/markdown/podman-create.1.md b/docs/source/markdown/podman-create.1.md index db9ff937b..229bb82f5 100644 --- a/docs/source/markdown/podman-create.1.md +++ b/docs/source/markdown/podman-create.1.md @@ -261,8 +261,8 @@ Note: if _host_device_ is a symbolic link then it will be resolved first. The container will only store the major and minor numbers of the host device. Note: if the user only has access rights via a group, accessing the device -from inside a rootless container will fail. The **crun**(1) runtime offers a -workaround for this by adding the option **\-\-annotation run.oci.keep_original_groups=1**. +from inside a rootless container will fail. Use the `--group-add keep-groups` +flag to pass the user's supplementary group access into the container. Podman may load kernel modules required for using the specified device. The devices that podman will load modules when necessary are: @@ -361,9 +361,17 @@ GID map for the user namespace. Using this flag will run the container with user The following example maps uids 0-2000 in the container to the uids 30000-31999 on the host and gids 0-2000 in the container to the gids 30000-31999 on the host. `--gidmap=0:30000:2000` -#### **\-\-group-add**=*group* +#### **\-\-group-add**=*group|keep-groups* -Add additional groups to run as +Add additional groups to assign to primary user running within the container process. + +- `keep-groups` is a special flag that tells Podman to keep the supplementary group access. + +Allows container to use the user's supplementary group access. If file systems or +devices are only accessible by the rootless user's group, this flag tells the OCI +runtime to pass the group access into the container. Currently only available +with the `crun` OCI runtime. Note: `keep-groups` is exclusive, you cannot add any other groups +with this flag. (Not available for remote commands) #### **\-\-health-cmd**=*"command"* | *'["command", "arg1", ...]'* @@ -634,7 +642,7 @@ Valid _mode_ values are: - **none**: no networking; - **container:**_id_: reuse another container's network stack; - **host**: use the Podman host network stack. Note: the host mode gives the container full access to local system services such as D-bus and is therefore considered insecure; -- _network-id_: connect to a user-defined network, multiple networks should be comma separated; +- _network-id_: connect to a user-defined network, multiple networks should be comma-separated; - **ns:**_path_: path to a network namespace to join; - **private**: create a new namespace for the container (default) - **slirp4netns[:OPTIONS,...]**: use **slirp4netns**(1) to create a user network stack. This is the default for rootless containers. It is possible to specify these additional options: @@ -861,6 +869,8 @@ Security Options - `label=filetype:TYPE` : Set the label file type for the container files - `label=disable` : Turn off label separation for the container +Note: Labeling can be disabled for all containers by setting label=false in the **containers.conf** (`/etc/containers/containers.conf` or `$HOME/.config/containers/containers.conf`) file. + - `mask=/path/1:/path/2` : The paths to mask separated by a colon. A masked path cannot be accessed inside the container. @@ -869,13 +879,13 @@ Security Options - `seccomp=unconfined` : Turn off seccomp confinement for the container - `seccomp=profile.json` : White listed syscalls seccomp Json file to be used as a seccomp filter +- `proc-opts=OPTIONS` : Comma-separated list of options to use for the /proc mount. More details for the + possible mount options are specified in the **proc(5)** man page. + - `unmask=ALL or /path/1:/path/2` : Paths to unmask separated by a colon. If set to **ALL**, it will unmask all the paths that are masked or made read only by default. The default masked paths are **/proc/acpi, /proc/kcore, /proc/keys, /proc/latency_stats, /proc/sched_debug, /proc/scsi, /proc/timer_list, /proc/timer_stats, /sys/firmware, and /sys/fs/selinux.** The default paths that are read only are **/proc/asound, /proc/bus, /proc/fs, /proc/irq, /proc/sys, /proc/sysrq-trigger, /sys/fs/cgroup**. -- `proc-opts=OPTIONS` : Comma separated list of options to use for the /proc mount. More details for the - possible mount options are specified at **proc(5)** man page. - Note: Labeling can be disabled for all containers by setting label=false in the **containers.conf** (`/etc/containers/containers.conf` or `$HOME/.config/containers/containers.conf`) file. #### **\-\-shm-size**=*size* @@ -975,11 +985,72 @@ Remote connections use local containers.conf for defaults Set the umask inside the container. Defaults to `0022`. Remote connections use local containers.conf for defaults -#### **\-\-uidmap**=*container_uid:host_uid:amount* +#### **\-\-uidmap**=*container_uid*:*from_uid*:*amount* + +Run the container in a new user namespace using the supplied mapping. This +option conflicts with the **\-\-userns** and **\-\-subuidname** options. This +option provides a way to map host UIDs to container UIDs. It can be passed +several times to map different ranges. + +The _from_uid_ value is based upon the user running the command, either rootful or rootless users. +* rootful user: *container_uid*:*host_uid*:*amount* +* rootless user: *container_uid*:*intermediate_uid*:*amount* + +When **podman create** is called by a privileged user, the option **\-\-uidmap** +works as a direct mapping between host UIDs and container UIDs. + +host UID -> container UID + +The _amount_ specifies the number of consecutive UIDs that will be mapped. +If for example _amount_ is **4** the mapping would look like: + +| host UID | container UID | +| - | - | +| _from_uid_ | _container_uid_ | +| _from_uid_ + 1 | _container_uid_ + 1 | +| _from_uid_ + 2 | _container_uid_ + 2 | +| _from_uid_ + 3 | _container_uid_ + 3 | + +When **podman create** is called by an unprivileged user (i.e. running rootless), +the value _from_uid_ is interpreted as an "intermediate UID". In the rootless +case, host UIDs are not mapped directly to container UIDs. Instead the mapping +happens over two mapping steps: -UID map for the user namespace. Using this flag will run the container with user namespace enabled. It conflicts with the `--userns` and `--subuidname` flags. +host UID -> intermediate UID -> container UID -The following example maps uids 0-2000 in the container to the uids 30000-31999 on the host and gids 0-2000 in the container to the gids 30000-31999 on the host. `--uidmap=0:30000:2000` +The **\-\-uidmap** option only influences the second mapping step. + +The first mapping step is derived by Podman from the contents of the file +_/etc/subuid_ and the UID of the user calling Podman. + +First mapping step: + +| host UID | intermediate UID | +| - | - | +| UID for the user starting Podman | 0 | +| 1st subordinate UID for the user starting Podman | 1 | +| 2nd subordinate UID for the user starting Podman | 2 | +| 3rd subordinate UID for the user starting Podman | 3 | +| nth subordinate UID for the user starting Podman | n | + +To be able to use intermediate UIDs greater than zero, the user needs to have +subordinate UIDs configured in _/etc/subuid_. See **subuid**(5). + +The second mapping step is configured with **\-\-uidmap**. + +If for example _amount_ is **5** the second mapping step would look like: + +| intermediate UID | container UID | +| - | - | +| _from_uid_ | _container_uid_ | +| _from_uid_ + 1 | _container_uid_ + 1 | +| _from_uid_ + 2 | _container_uid_ + 2 | +| _from_uid_ + 3 | _container_uid_ + 3 | +| _from_uid_ + 4 | _container_uid_ + 4 | + +Even if a user does not have any subordinate UIDs in _/etc/subuid_, +**\-\-uidmap** could still be used to map the normal UID of the user to a +container UID by running `podman create --uidmap $container_uid:0:1 --user $container_uid ...`. #### **\-\-ulimit**=*option* @@ -1032,9 +1103,9 @@ Create a bind mount. If you specify, ` -v /HOST-DIR:/CONTAINER-DIR`, Podman bind mounts `/HOST-DIR` in the host to `/CONTAINER-DIR` in the Podman container. Similarly, `-v SOURCE-VOLUME:/CONTAINER-DIR` will mount the volume in the host to the container. If no such named volume exists, Podman will -create one. The `OPTIONS` are a comma delimited list and can be: <sup>[[1]](#Footnote1)</sup> (Note when using the remote client, the volumes will be mounted from the remote server, not necessarly the client machine.) +create one. The `OPTIONS` are a comma-separated list and can be: <sup>[[1]](#Footnote1)</sup> (Note when using the remote client, the volumes will be mounted from the remote server, not necessarly the client machine.) -The _options_ is a comma delimited list and can be: +The _options_ is a comma-separated list and can be: * **rw**|**ro** * **z**|**Z** @@ -1124,7 +1195,7 @@ host into the container to allow speeding up builds. Content mounted into the container is labeled with the private label. On SELinux systems, labels in the source directory must be readable by the container label. Usually containers can read/execute `container_share_t` -and can read/write `container_file_t`. If you can not change the labels on a +and can read/write `container_file_t`. If you cannot change the labels on a source volume, SELinux container separation must be disabled for the container to work. - The source directory mounted into the container with an overlay mount @@ -1184,10 +1255,14 @@ will convert /foo into a `shared` mount point. Alternatively one can directly change propagation properties of source mount. Say `/` is source mount for `/foo`, then use `mount --make-shared /` to convert `/` into a `shared` mount. +Note: if the user only has access rights via a group, accessing the volume +from inside a rootless container will fail. Use the `--group-add keep-groups` +flag to pass the user's supplementary group access into the container. + #### **\-\-volumes-from**[=*CONTAINER*[:*OPTIONS*]] Mount volumes from the specified container(s). Used to share volumes between -containers. The *options* is a comma delimited list with the following available elements: +containers. The *options* is a comma-separated list with the following available elements: * **rw**|**ro** * **z** @@ -1292,6 +1367,12 @@ $ podman create --name container3 --requires container1,container2 -t -i fedora $ podman start --attach container3 ``` +### Configure keep supplemental groups for access to volume + +``` +$ podman create -v /var/lib/design:/var/lib/design --group-add keep-groups ubi8 +``` + ### Rootless Containers Podman runs as a non root user on most systems. This feature requires that a new enough version of shadow-utils diff --git a/docs/source/markdown/podman-machine-list.1.md b/docs/source/markdown/podman-machine-list.1.md index bd5608258..922c19fdf 100644 --- a/docs/source/markdown/podman-machine-list.1.md +++ b/docs/source/markdown/podman-machine-list.1.md @@ -35,6 +35,10 @@ Valid placeholders for the Go template are listed below: Print usage statement. +#### **\-\-noheading** + +Omit the table headings from the listing of pods. + ## EXAMPLES ``` diff --git a/docs/source/markdown/podman-network-ls.1.md b/docs/source/markdown/podman-network-ls.1.md index 12dbb01d3..464efdc21 100644 --- a/docs/source/markdown/podman-network-ls.1.md +++ b/docs/source/markdown/podman-network-ls.1.md @@ -41,6 +41,10 @@ Valid placeholders for the Go template are listed below: | .Labels | Network labels | | .Version | CNI Version of the config file | +#### **\-\-noheading** + +Omit the table headings from the listing of networks. + #### **\-\-no-trunc** Do not truncate the network ID. The network ID is not displayed by default and must be specified with **\-\-format**. diff --git a/docs/source/markdown/podman-pod-create.1.md b/docs/source/markdown/podman-pod-create.1.md index 9ecde1ca3..6f3d7f1ca 100644 --- a/docs/source/markdown/podman-pod-create.1.md +++ b/docs/source/markdown/podman-pod-create.1.md @@ -125,7 +125,7 @@ If another pod with the same name already exists, replace and remove it. The de #### **\-\-share**=*namespace* -A comma delimited list of kernel namespaces to share. If none or "" is specified, no namespaces will be shared. The namespaces to choose from are ipc, net, pid, uts. +A comma-separated list of kernel namespaces to share. If none or "" is specified, no namespaces will be shared. The namespaces to choose from are ipc, net, pid, uts. The operator can identify a pod in three ways: UUID long identifier (“f78375b1c487e03c9438c729345e54db9d20cfa2ac1fc3494b6eb60872e74778”) diff --git a/docs/source/markdown/podman-pod-ps.1.md b/docs/source/markdown/podman-pod-ps.1.md index 0be22c2b1..d4fd6d41c 100644 --- a/docs/source/markdown/podman-pod-ps.1.md +++ b/docs/source/markdown/podman-pod-ps.1.md @@ -42,6 +42,10 @@ Includes the container statuses in the container info field Show the latest pod created (all states) (This option is not available with the remote Podman client) +#### **\-\-noheading** + +Omit the table headings from the listing of pods. + #### **\-\-no-trunc** Display the extended information diff --git a/docs/source/markdown/podman-ps.1.md b/docs/source/markdown/podman-ps.1.md index b950fede4..b9d12adc6 100644 --- a/docs/source/markdown/podman-ps.1.md +++ b/docs/source/markdown/podman-ps.1.md @@ -100,6 +100,10 @@ Show the latest container created (all states) (This option is not available wit Display namespace information +#### **\-\-noheading** + +Omit the table headings from the listing of containers. + #### **\-\-no-trunc** Display the extended information diff --git a/docs/source/markdown/podman-run.1.md b/docs/source/markdown/podman-run.1.md index f84a5913c..2e6d97a05 100644 --- a/docs/source/markdown/podman-run.1.md +++ b/docs/source/markdown/podman-run.1.md @@ -299,8 +299,8 @@ Note: if _host_device_ is a symbolic link then it will be resolved first. The container will only store the major and minor numbers of the host device. Note: if the user only has access rights via a group, accessing the device -from inside a rootless container will fail. The **crun**(1) runtime offers a -workaround for this by adding the option **\-\-annotation run.oci.keep_original_groups=1**. +from inside a rootless container will fail. Use the `--group-add keep-groups` +flag to pass the user's supplementary group access into the container. Podman may load kernel modules required for using the specified device. The devices that Podman will load modules when necessary are: @@ -405,9 +405,17 @@ Meaning **groupname** is initially mapped to gid **100000** which is referenced above: The group **groupname** is mapped to group **100000** of the initial namespace then the **30000**st id of this namespace (which is gid 130000 in this namespace) is mapped to container namespace group id **0**. (groupname -> 100000 / 30000 -> 0) -#### **\-\-group-add**=*group* +#### **\-\-group-add**=*group|keep-groups* -Add additional groups to run as +Add additional groups to assign to primary user running within the container process. + +- `keep-groups` is a special flag that tells Podman to keep the supplementary group access. + +Allows container to use the user's supplementary group access. If file systems or +devices are only accessible by the rootless user's group, this flag tells the OCI +runtime to pass the group access into the container. Currently only available +with the `crun` OCI runtime. Note: `keep-groups` is exclusive, you cannot add any other groups +with this flag. (Not available for remote commands) #### **\-\-health-cmd**=*"command"* | *'["command", "arg1", ...]'* @@ -670,7 +678,7 @@ Valid _mode_ values are: - **none**: no networking; - **container:**_id_: reuse another container's network stack; - **host**: use the Podman host network stack. Note: the host mode gives the container full access to local system services such as D-bus and is therefore considered insecure; -- _network-id_: connect to a user-defined network, multiple networks should be comma separated; +- _network-id_: connect to a user-defined network, multiple networks should be comma-separated; - **ns:**_path_: path to a network namespace to join; - **private**: create a new namespace for the container (default) - **slirp4netns[:OPTIONS,...]**: use **slirp4netns**(1) to create a user network stack. This is the default for rootless containers. It is possible to specify these additional options: @@ -905,19 +913,27 @@ Security Options - **apparmor=unconfined** : Turn off apparmor confinement for the container - **apparmor**=_your-profile_ : Set the apparmor confinement profile for the container + - **label=user:**_USER_: Set the label user for the container processes - **label=role:**_ROLE_: Set the label role for the container processes - **label=type:**_TYPE_: Set the label process type for the container processes - **label=level:**_LEVEL_: Set the label level for the container processes - **label=filetype:**TYPE_: Set the label file type for the container files - **label=disable**: Turn off label separation for the container + +Note: Labeling can be disabled for all containers by setting label=false in the **containers.conf** (`/etc/containers/containers.conf` or `$HOME/.config/containers/containers.conf`) file. + - **mask**=_/path/1:/path/2_: The paths to mask separated by a colon. A masked path cannot be accessed inside the container. + - **no-new-privileges**: Disable container processes from gaining additional privileges + - **seccomp=unconfined**: Turn off seccomp confinement for the container - **seccomp**=_profile.json_: Allowed syscall list seccomp JSON file to be used as a seccomp filter -- **proc-opts**=_OPTIONS_ : Comma separated list of options to use for the /proc mount. More details - for the possible mount options are specified at **proc(5)** man page. + +- **proc-opts**=_OPTIONS_ : Comma-separated list of options to use for the /proc mount. More details + for the possible mount options are specified in the **proc(5)** man page. + - **unmask**=_ALL_ or _/path/1:/path/2_: Paths to unmask separated by a colon. If set to **ALL**, it will unmask all the paths that are masked or made read only by default. The default masked paths are **/proc/acpi, /proc/kcore, /proc/keys, /proc/latency_stats, /proc/sched_debug, /proc/scsi, /proc/timer_list, /proc/timer_stats, /sys/firmware, and /sys/fs/selinux.**. The default paths that are read only are **/proc/asound**, **/proc/bus**, **/proc/fs**, **/proc/irq**, **/proc/sys**, **/proc/sysrq-trigger**, **/sys/fs/cgroup**. @@ -1047,23 +1063,72 @@ Remote connections use local containers.conf for defaults Set the umask inside the container. Defaults to `0022`. Remote connections use local containers.conf for defaults -#### **\-\-uidmap**=*container_uid*:*host_uid*:*amount* +#### **\-\-uidmap**=*container_uid*:*from_uid*:*amount* + +Run the container in a new user namespace using the supplied mapping. This +option conflicts with the **\-\-userns** and **\-\-subuidname** options. This +option provides a way to map host UIDs to container UIDs. It can be passed +several times to map different ranges. + +The _from_uid_ value is based upon the user running the command, either rootful or rootless users. +* rootful user: *container_uid*:*host_uid*:*amount* +* rootless user: *container_uid*:*intermediate_uid*:*amount* + +When **podman run** is called by a privileged user, the option **\-\-uidmap** +works as a direct mapping between host UIDs and container UIDs. + +host UID -> container UID + +The _amount_ specifies the number of consecutive UIDs that will be mapped. +If for example _amount_ is **4** the mapping would look like: -Run the container in a new user namespace using the supplied mapping. This option conflicts -with the **\-\-userns** and **\-\-subuidname** flags. -This option can be passed several times to map different ranges. If calling **podman run** -as an unprivileged user, the user needs to have the right to use the mapping. See **subuid**(5). +| host UID | container UID | +| - | - | +| _from_uid_ | _container_uid_ | +| _from_uid_ + 1 | _container_uid_ + 1 | +| _from_uid_ + 2 | _container_uid_ + 2 | +| _from_uid_ + 3 | _container_uid_ + 3 | -The following example maps uids 0-1999 in the container to the uids 30000-31999 on the host: **\-\-uidmap=0:30000:2000**. +When **podman run** is called by an unprivileged user (i.e. running rootless), +the value _from_uid_ is interpreted as an "intermediate UID". In the rootless +case, host UIDs are not mapped directly to container UIDs. Instead the mapping +happens over two mapping steps: -**Important note:** The new user namespace mapping based on **\-\-uidmap** is based on the initial mapping made in the _/etc/subuid_ file. -Assuming there is a _/etc/subuid_ mapping **username:100000:65536**, then **username** is initially mapped to a namespace starting with -uid **100000** for **65536** ids. From here the **\-\-uidmap** mapping to the new namespace starts from **0** again, but is based on the initial mapping. -Meaning **username** is initially mapped to uid **100000** which is referenced as **0** in the following **\-\-uidmap** mapping. In terms of the example -above: The user **username** is mapped to user **100000** of the initial namespace then the -**30000**st id of this namespace (which is uid 130000 in this namespace) is mapped to container namespace user id **0**. (username -> 100000 / 30000 -> 0) +host UID -> intermediate UID -> container UID -_Note_: A minimal mapping has to have at least container uid **0** mapped to the parent user namespace. +The **\-\-uidmap** option only influences the second mapping step. + +The first mapping step is derived by Podman from the contents of the file +_/etc/subuid_ and the UID of the user calling Podman. + +First mapping step: + +| host UID | intermediate UID | +| - | - | +| UID for the user starting Podman | 0 | +| 1st subordinate UID for the user starting Podman | 1 | +| 2nd subordinate UID for the user starting Podman | 2 | +| 3rd subordinate UID for the user starting Podman | 3 | +| nth subordinate UID for the user starting Podman | n | + +To be able to use intermediate UIDs greater than zero, the user needs to have +subordinate UIDs configured in _/etc/subuid_. See **subuid**(5). + +The second mapping step is configured with **\-\-uidmap**. + +If for example _amount_ is **5** the second mapping step would look like: + +| intermediate UID | container UID | +| - | - | +| _from_uid_ | _container_uid_ | +| _from_uid_ + 1 | _container_uid_ + 1 | +| _from_uid_ + 2 | _container_uid_ + 2 | +| _from_uid_ + 3 | _container_uid_ + 3 | +| _from_uid_ + 4 | _container_uid_ + 4 | + +Even if a user does not have any subordinate UIDs in _/etc/subuid_, +**\-\-uidmap** could still be used to map the normal UID of the user to a +container UID by running `podman run --uidmap $container_uid:0:1 --user $container_uid ...`. #### **\-\-ulimit**=*option* @@ -1115,7 +1180,7 @@ container. Similarly, _SOURCE-VOLUME_:_/CONTAINER-DIR_ will mount the volume in the host to the container. If no such named volume exists, Podman will create one. (Note when using the remote client, the volumes will be mounted from the remote server, not necessarly the client machine.) -The _options_ is a comma delimited list and can be: <sup>[[1]](#Footnote1)</sup> +The _options_ is a comma-separated list and can be: <sup>[[1]](#Footnote1)</sup> * **rw**|**ro** * **z**|**Z** @@ -1203,7 +1268,7 @@ host into the container to allow speeding up builds. Content mounted into the container is labeled with the private label. On SELinux systems, labels in the source directory must be readable by the container label. Usually containers can read/execute `container_share_t` -and can read/write `container_file_t`. If you can not change the labels on a +and can read/write `container_file_t`. If you cannot change the labels on a source volume, SELinux container separation must be disabled for the container to work. - The source directory mounted into the container with an overlay mount @@ -1265,10 +1330,14 @@ will convert /foo into a shared mount point. Alternatively, one can directly change propagation properties of source mount. Say, if _/_ is source mount for _/foo_, then use **mount --make-shared /** to convert _/_ into a shared mount. +Note: if the user only has access rights via a group, accessing the volume +from inside a rootless container will fail. Use the `--group-add keep-groups` +flag to pass the user's supplementary group access into the container. + #### **\-\-volumes-from**[=*CONTAINER*[:*OPTIONS*]] Mount volumes from the specified container(s). Used to share volumes between -containers. The *options* is a comma delimited list with the following available elements: +containers. The *options* is a comma-separated list with the following available elements: * **rw**|**ro** * **z** @@ -1652,6 +1721,12 @@ $ podman create --name container2 -t -i fedora bash $ podman run --name container3 --requires container1,container2 -t -i fedora bash ``` +### Configure keep supplemental groups for access to volume + +``` +$ podman run -v /var/lib/design:/var/lib/design --group-add keep-groups ubi8 +``` + ### Rootless Containers Podman runs as a non root user on most systems. This feature requires that a new enough version of **shadow-utils** diff --git a/docs/source/markdown/podman-save.1.md b/docs/source/markdown/podman-save.1.md index e6f6e993b..0036a9379 100644 --- a/docs/source/markdown/podman-save.1.md +++ b/docs/source/markdown/podman-save.1.md @@ -27,7 +27,7 @@ Note: `:` is a restricted character and cannot be part of the file name. #### **\-\-compress** Compress tarball image layers when pushing to a directory using the 'dir' transport. (default is same compression type, compressed or uncompressed, as source) -Note: This flag can only be set when using the **dir** transport i.e --format=oci-dir or --format-docker-dir +Note: This flag can only be set when using the **dir** transport i.e --format=oci-dir or --format=docker-dir #### **\-\-output**, **-o**=*file* diff --git a/docs/source/markdown/podman-secret-create.1.md b/docs/source/markdown/podman-secret-create.1.md index ca92dd38e..f5a97a0f3 100644 --- a/docs/source/markdown/podman-secret-create.1.md +++ b/docs/source/markdown/podman-secret-create.1.md @@ -16,7 +16,7 @@ A secret is a blob of sensitive data which a container needs at runtime but should not be stored in the image or in source control, such as usernames and passwords, TLS certificates and keys, SSH keys or other important generic strings or binary content (up to 500 kb in size). -Secrets will not be commited to an image with `podman commit`, and will not be in the archive created by a `podman export` +Secrets will not be committed to an image with `podman commit`, and will not be in the archive created by a `podman export` ## OPTIONS diff --git a/docs/source/markdown/podman-secret-ls.1.md b/docs/source/markdown/podman-secret-ls.1.md index 57a606738..18119542e 100644 --- a/docs/source/markdown/podman-secret-ls.1.md +++ b/docs/source/markdown/podman-secret-ls.1.md @@ -16,6 +16,10 @@ Lists all the secrets that exist. The output can be formatted to a Go template u Format secret output using Go template. +#### **\-\-noheading** + +Omit the table headings from the listing of secrets. . + ## EXAMPLES ``` diff --git a/docs/source/markdown/podman-volume-ls.1.md b/docs/source/markdown/podman-volume-ls.1.md index 5214980a3..47e44efc1 100644 --- a/docs/source/markdown/podman-volume-ls.1.md +++ b/docs/source/markdown/podman-volume-ls.1.md @@ -26,6 +26,10 @@ Format volume output using Go template. Print usage statement. +#### **\-\-noheading** + +Omit the table headings from the listing of volumes. + #### **\-\-quiet**, **-q** Print volume output in quiet mode. Only print the volume names. @@ -12,12 +12,12 @@ require ( github.com/containernetworking/cni v0.8.1 github.com/containernetworking/plugins v0.9.1 github.com/containers/buildah v1.20.1-0.20210402144408-36a37402d0c8 - github.com/containers/common v0.36.0 + github.com/containers/common v0.37.0 github.com/containers/conmon v2.0.20+incompatible - github.com/containers/image/v5 v5.11.0 + github.com/containers/image/v5 v5.11.1 github.com/containers/ocicrypt v1.1.1 github.com/containers/psgo v1.5.2 - github.com/containers/storage v1.29.0 + github.com/containers/storage v1.30.0 github.com/coreos/go-systemd/v22 v22.3.1 github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3 github.com/cri-o/ocicni v0.2.1-0.20210301205850-541cf7c703cf @@ -195,13 +195,13 @@ github.com/containernetworking/plugins v0.9.1/go.mod h1:xP/idU2ldlzN6m4p5LmGiwRD github.com/containers/buildah v1.20.1-0.20210402144408-36a37402d0c8 h1:RlqbDlfE3+qrq4bNTZG7NVPqCDzfZrgE/yicu0VAykQ= github.com/containers/buildah v1.20.1-0.20210402144408-36a37402d0c8/go.mod h1:iowyscoAC5jwNDhs3c5CLGdBZ9FJk5UOoN2I5TdmXFs= github.com/containers/common v0.35.4/go.mod h1:rMzxgD7nMGw++cEbsp+NZv0UJO4rgXbm7F7IbJPTwIE= -github.com/containers/common v0.36.0 h1:7/0GM3oi2ROmKAg/8pDWJ8BU2BXdbmy7Gk2/SFCTV38= -github.com/containers/common v0.36.0/go.mod h1:rMzxgD7nMGw++cEbsp+NZv0UJO4rgXbm7F7IbJPTwIE= +github.com/containers/common v0.37.0 h1:RRyR8FITTJXfrF7J9KXKSplywY4zsXoA2kuQXMaUaNo= +github.com/containers/common v0.37.0/go.mod h1:dgbJcccCPTmncqxhma56+XW+6d5VzqGF6jtkMHyu3v0= github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg= github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= github.com/containers/image/v5 v5.10.5/go.mod h1:SgIbWEedCNBbn2FI5cH0/jed1Ecy2s8XK5zTxvJTzII= -github.com/containers/image/v5 v5.11.0 h1:SwxGucW1AZ8H/5KH9jW70lo9WyuOrtxafutyQ9RPPLw= -github.com/containers/image/v5 v5.11.0/go.mod h1:dCbUB4w6gmxIEOCsE0tZQppr8iBoXb4Evr74ZKlmwoI= +github.com/containers/image/v5 v5.11.1 h1:mNybUvU6zXUwcMsQaa3n+Idsru5pV+GE7k4oRuPzYi0= +github.com/containers/image/v5 v5.11.1/go.mod h1:HC9lhJ/Nz5v3w/5Co7H431kLlgzlVlOC+auD/er3OqE= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b h1:Q8ePgVfHDplZ7U33NwHZkrVELsZP5fYj9pM5WBZB2GE= github.com/containers/libtrust v0.0.0-20190913040956-14b96171aa3b/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= @@ -215,8 +215,9 @@ github.com/containers/storage v1.23.5/go.mod h1:ha26Q6ngehFNhf3AWoXldvAvwI4jFe3E github.com/containers/storage v1.24.8/go.mod h1:YC+2pY8SkfEAcZkwycxYbpK8EiRbx5soPPwz9dxe4IQ= github.com/containers/storage v1.28.0/go.mod h1:ixAwO7Bj31cigqPEG7aCz+PYmxkDxbIFdUFioYdxbzI= github.com/containers/storage v1.28.1/go.mod h1:5bwiMh2LkrN3AWIfDFMH7A/xbVNLcve+oeXYvHvW8cc= -github.com/containers/storage v1.29.0 h1:l3Vh6+IiMKLGfQZ3rDkF84m+KF1Qb0XEcilWC+pYo2o= github.com/containers/storage v1.29.0/go.mod h1:u84RU4CCufGeJBNTRNwMB+FoE+AiFeFw4SsMoqAOeCM= +github.com/containers/storage v1.30.0 h1:KS6zmoPyy0Qcx1HCCiseQ0ysSckRvtiuoVpIGh9iwQA= +github.com/containers/storage v1.30.0/go.mod h1:M/xn0pg6ReYFrLtWl5YELI/a4Xjq+Z3e5GJxQrJCcDI= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-iptables v0.4.5/go.mod h1:/mVI274lEDI2ns62jHCDnCyBF9Iwsmekav8Dbxlm1MU= @@ -381,6 +382,8 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3 h1:JjCZWpVbqXDqFVmTfYWEVTMIYrL/NPdPSCHPJ0T/raM= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -499,8 +502,9 @@ github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.11.5/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.12/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.1 h1:/+xsCsk06wE38cyiqOR/o7U2fSftcH72xD+BQXmja/g= +github.com/klauspost/compress v1.12.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= diff --git a/libpod/container_exec.go b/libpod/container_exec.go index 8d8ed14aa..c359f1e5d 100644 --- a/libpod/container_exec.go +++ b/libpod/container_exec.go @@ -773,7 +773,7 @@ func (c *Container) cleanupExecBundle(sessionID string) error { return err } - return c.ociRuntime.ExecContainerCleanup(c, sessionID) + return nil } // the path to a containers exec session bundle diff --git a/libpod/container_internal.go b/libpod/container_internal.go index a53027ab2..80c00a622 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -685,7 +685,11 @@ func (c *Container) removeIPv4Allocations() error { // This is necessary for restarting containers func (c *Container) removeConmonFiles() error { // Files are allowed to not exist, so ignore ENOENT - attachFile := filepath.Join(c.bundlePath(), "attach") + attachFile, err := c.AttachSocketPath() + if err != nil { + return errors.Wrapf(err, "failed to get attach socket path for container %s", c.ID()) + } + if err := os.Remove(attachFile); err != nil && !os.IsNotExist(err) { return errors.Wrapf(err, "error removing container %s attach file", c.ID()) } @@ -1309,7 +1313,7 @@ func (c *Container) stop(timeout uint) error { } // We have to check stopErr *after* we lock again - otherwise, we have a - // change of panicing on a double-unlock. Ref: GH Issue 9615 + // change of panicking on a double-unlock. Ref: GH Issue 9615 if stopErr != nil { return stopErr } @@ -1672,7 +1676,7 @@ func (c *Container) chownVolume(volumeName string) error { // TODO: For now, I've disabled chowning volumes owned by non-Podman // drivers. This may be safe, but it's really going to be a case-by-case - // thing, I think - safest to leave disabled now and reenable later if + // thing, I think - safest to leave disabled now and re-enable later if // there is a demand. if vol.state.NeedsChown && !vol.UsesVolumeDriver() { vol.state.NeedsChown = false diff --git a/libpod/define/errors.go b/libpod/define/errors.go index e19ac6a27..8d943099b 100644 --- a/libpod/define/errors.go +++ b/libpod/define/errors.go @@ -206,4 +206,8 @@ var ( // ErrCanceled indicates that an operation has been cancelled by a user. // Useful for potentially long running tasks. ErrCanceled = errors.New("cancelled by user") + + // ErrConmonVersionFormat is used when the expected versio-format of conmon + // has changed. + ErrConmonVersionFormat = "conmon version changed format" ) diff --git a/libpod/define/fileinfo.go b/libpod/define/fileinfo.go index 2c7b6fe99..eec99e300 100644 --- a/libpod/define/fileinfo.go +++ b/libpod/define/fileinfo.go @@ -5,7 +5,7 @@ import ( "time" ) -// FileInfo describes the attributes of a file or diretory. +// FileInfo describes the attributes of a file or directory. type FileInfo struct { Name string `json:"name"` Size int64 `json:"size"` diff --git a/libpod/image/image.go b/libpod/image/image.go index 12dc22360..3c9fb3a37 100644 --- a/libpod/image/image.go +++ b/libpod/image/image.go @@ -617,7 +617,8 @@ func (i *Image) TopLayer() string { func (i *Image) Remove(ctx context.Context, force bool) error { parent, err := i.GetParent(ctx) if err != nil { - return err + logrus.Warnf("error determining parent of image: %v, ignoring the error", err) + parent = nil } if _, err := i.imageruntime.store.DeleteImage(i.ID(), true); err != nil { return err diff --git a/libpod/image/utils.go b/libpod/image/utils.go index 0b4264112..dfe35c017 100644 --- a/libpod/image/utils.go +++ b/libpod/image/utils.go @@ -50,7 +50,7 @@ func findImageInRepotags(search imageParts, images []*Image) (*storage.Image, er // If more then one candidate and the candidates all have same name // and only one is read/write return it. - // Othewise return error with the list of candidates + // Otherwise return error with the list of candidates if len(candidates) > 1 { var ( rwImage *Image diff --git a/libpod/oci.go b/libpod/oci.go index f2053f1b5..1f2c7dd71 100644 --- a/libpod/oci.go +++ b/libpod/oci.go @@ -94,10 +94,6 @@ type OCIRuntime interface { // ExecUpdateStatus checks the status of a given exec session. // Returns true if the session is still running, or false if it exited. ExecUpdateStatus(ctr *Container, sessionID string) (bool, error) - // ExecContainerCleanup cleans up after an exec session exits. - // It removes any files left by the exec session that are no longer - // needed, including the attach socket. - ExecContainerCleanup(ctr *Container, sessionID string) error // CheckpointContainer checkpoints the given container. // Some OCI runtimes may not support this - if SupportsCheckpoint() diff --git a/libpod/oci_conmon_exec_linux.go b/libpod/oci_conmon_exec_linux.go index b43316951..76338b86c 100644 --- a/libpod/oci_conmon_exec_linux.go +++ b/libpod/oci_conmon_exec_linux.go @@ -284,17 +284,6 @@ func (r *ConmonOCIRuntime) ExecUpdateStatus(ctr *Container, sessionID string) (b return true, nil } -// ExecContainerCleanup cleans up files created when a command is run via -// ExecContainer. This includes the attach socket for the exec session. -func (r *ConmonOCIRuntime) ExecContainerCleanup(ctr *Container, sessionID string) error { - // Clean up the sockets dir. Issue #3962 - // Also ignore if it doesn't exist for some reason; hence the conditional return below - if err := os.RemoveAll(filepath.Join(r.socketsDir, sessionID)); err != nil && !os.IsNotExist(err) { - return err - } - return nil -} - // ExecAttachSocketPath is the path to a container's exec session attach socket. func (r *ConmonOCIRuntime) ExecAttachSocketPath(ctr *Container, sessionID string) (string, error) { // We don't even use container, so don't validity check it @@ -302,7 +291,7 @@ func (r *ConmonOCIRuntime) ExecAttachSocketPath(ctr *Container, sessionID string return "", errors.Wrapf(define.ErrInvalidArg, "must provide a valid session ID to get attach socket path") } - return filepath.Join(r.socketsDir, sessionID, "attach"), nil + return filepath.Join(ctr.execBundlePath(sessionID), "attach"), nil } // This contains pipes used by the exec API. diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go index dbe91c232..c1acec977 100644 --- a/libpod/oci_conmon_linux.go +++ b/libpod/oci_conmon_linux.go @@ -59,7 +59,6 @@ type ConmonOCIRuntime struct { conmonEnv []string tmpDir string exitsDir string - socketsDir string logSizeMax int64 noPivot bool reservePorts bool @@ -149,7 +148,6 @@ func newConmonOCIRuntime(name string, paths []string, conmonPath string, runtime } runtime.exitsDir = filepath.Join(runtime.tmpDir, "exits") - runtime.socketsDir = filepath.Join(runtime.tmpDir, "socket") // Create the exit files and attach sockets directories if err := os.MkdirAll(runtime.exitsDir, 0750); err != nil { @@ -158,13 +156,6 @@ func newConmonOCIRuntime(name string, paths []string, conmonPath string, runtime return nil, errors.Wrapf(err, "error creating OCI runtime exit files directory") } } - if err := os.MkdirAll(runtime.socketsDir, 0750); err != nil { - // The directory is allowed to exist - if !os.IsExist(err) { - return nil, errors.Wrap(err, "error creating OCI runtime attach sockets directory") - } - } - return runtime, nil } @@ -865,7 +856,7 @@ func (r *ConmonOCIRuntime) AttachSocketPath(ctr *Container) (string, error) { return "", errors.Wrapf(define.ErrInvalidArg, "must provide a valid container to get attach socket path") } - return filepath.Join(r.socketsDir, ctr.ID(), "attach"), nil + return filepath.Join(ctr.bundlePath(), "attach"), nil } // ExitFilePath is the path to a container's exit file. @@ -1240,7 +1231,7 @@ func (r *ConmonOCIRuntime) sharedConmonArgs(ctr *Container, cuuid, bundlePath, p "-p", pidPath, "-n", ctr.Name(), "--exit-dir", exitDir, - "--socket-dir-path", r.socketsDir, + "--full-attach", } if len(r.runtimeFlags) > 0 { rFlags := []string{} diff --git a/libpod/oci_missing.go b/libpod/oci_missing.go index eb8cdebad..10526f368 100644 --- a/libpod/oci_missing.go +++ b/libpod/oci_missing.go @@ -151,11 +151,6 @@ func (r *MissingRuntime) ExecUpdateStatus(ctr *Container, sessionID string) (boo return false, r.printError() } -// ExecContainerCleanup is not available as the runtime is missing -func (r *MissingRuntime) ExecContainerCleanup(ctr *Container, sessionID string) error { - return r.printError() -} - // CheckpointContainer is not available as the runtime is missing func (r *MissingRuntime) CheckpointContainer(ctr *Container, options ContainerCheckpointOptions) error { return r.printError() diff --git a/libpod/runtime.go b/libpod/runtime.go index 98ca2d5a4..3518ed25a 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -2,10 +2,14 @@ package libpod import ( "bufio" + "bytes" "context" "fmt" "os" + "os/exec" "path/filepath" + "regexp" + "strconv" "strings" "sync" "syscall" @@ -25,6 +29,7 @@ import ( "github.com/containers/podman/v3/pkg/rootless" "github.com/containers/podman/v3/pkg/util" "github.com/containers/storage" + "github.com/containers/storage/pkg/unshare" "github.com/cri-o/ocicni/pkg/ocicni" "github.com/docker/docker/pkg/namesgenerator" spec "github.com/opencontainers/runtime-spec/specs-go" @@ -32,6 +37,17 @@ import ( "github.com/sirupsen/logrus" ) +const ( + // conmonMinMajorVersion is the major version required for conmon. + conmonMinMajorVersion = 2 + + // conmonMinMinorVersion is the minor version required for conmon. + conmonMinMinorVersion = 0 + + // conmonMinPatchVersion is the sub-minor version required for conmon. + conmonMinPatchVersion = 24 +) + // A RuntimeOption is a functional option which alters the Runtime created by // NewRuntime type RuntimeOption func(*Runtime) error @@ -260,7 +276,7 @@ func getLockManager(runtime *Runtime) (lock.Manager, error) { // Sets up containers/storage, state store, OCI runtime func makeRuntime(ctx context.Context, runtime *Runtime) (retErr error) { // Find a working conmon binary - cPath, err := runtime.config.FindConmon() + cPath, err := findConmon(runtime.config.Engine.ConmonPath) if err != nil { return err } @@ -323,9 +339,16 @@ func makeRuntime(ctx context.Context, runtime *Runtime) (retErr error) { } logrus.Debugf("Set libpod namespace to %q", runtime.config.Engine.Namespace) + hasCapSysAdmin, err := unshare.HasCapSysAdmin() + if err != nil { + return err + } + + needsUserns := !hasCapSysAdmin + // Set up containers/storage var store storage.Store - if os.Geteuid() != 0 { + if needsUserns { logrus.Debug("Not configuring container store") } else if runtime.noStore { logrus.Debug("No store required. Not opening container store.") @@ -465,7 +488,7 @@ func makeRuntime(ctx context.Context, runtime *Runtime) (retErr error) { // If we need to refresh, then it is safe to assume there are // no containers running. Create immediately a namespace, as // we will need to access the storage. - if os.Geteuid() != 0 { + if needsUserns { aliveLock.Unlock() // Unlock to avoid deadlock as BecomeRootInUserNS will reexec. pausePid, err := util.GetRootlessPauseProcessPidPathGivenDir(runtime.config.Engine.TmpDir) if err != nil { @@ -532,6 +555,102 @@ func makeRuntime(ctx context.Context, runtime *Runtime) (retErr error) { return nil } +// findConmon iterates over conmonPaths and returns the path +// to the first conmon binary with a new enough version. If none is found, +// we try to do a path lookup of "conmon". +func findConmon(conmonPaths []string) (string, error) { + foundOutdatedConmon := false + for _, path := range conmonPaths { + stat, err := os.Stat(path) + if err != nil { + continue + } + if stat.IsDir() { + continue + } + if err := probeConmon(path); err != nil { + logrus.Warnf("Conmon at %s invalid: %v", path, err) + foundOutdatedConmon = true + continue + } + logrus.Debugf("Using conmon: %q", path) + return path, nil + } + + // Search the $PATH as last fallback + if path, err := exec.LookPath("conmon"); err == nil { + if err := probeConmon(path); err != nil { + logrus.Warnf("Conmon at %s is invalid: %v", path, err) + foundOutdatedConmon = true + } else { + logrus.Debugf("Using conmon from $PATH: %q", path) + return path, nil + } + } + + if foundOutdatedConmon { + return "", errors.Wrapf(define.ErrConmonOutdated, + "please update to v%d.%d.%d or later", + conmonMinMajorVersion, conmonMinMinorVersion, conmonMinPatchVersion) + } + + return "", errors.Wrapf(define.ErrInvalidArg, + "could not find a working conmon binary (configured options: %v)", + conmonPaths) +} + +// probeConmon calls conmon --version and verifies it is a new enough version for +// the runtime expectations the container engine currently has. +func probeConmon(conmonBinary string) error { + cmd := exec.Command(conmonBinary, "--version") + var out bytes.Buffer + cmd.Stdout = &out + err := cmd.Run() + if err != nil { + return err + } + r := regexp.MustCompile(`^conmon version (?P<Major>\d+).(?P<Minor>\d+).(?P<Patch>\d+)`) + + matches := r.FindStringSubmatch(out.String()) + if len(matches) != 4 { + return errors.Wrap(err, define.ErrConmonVersionFormat) + } + major, err := strconv.Atoi(matches[1]) + if err != nil { + return errors.Wrap(err, define.ErrConmonVersionFormat) + } + if major < conmonMinMajorVersion { + return define.ErrConmonOutdated + } + if major > conmonMinMajorVersion { + return nil + } + + minor, err := strconv.Atoi(matches[2]) + if err != nil { + return errors.Wrap(err, define.ErrConmonVersionFormat) + } + if minor < conmonMinMinorVersion { + return define.ErrConmonOutdated + } + if minor > conmonMinMinorVersion { + return nil + } + + patch, err := strconv.Atoi(matches[3]) + if err != nil { + return errors.Wrap(err, define.ErrConmonVersionFormat) + } + if patch < conmonMinPatchVersion { + return define.ErrConmonOutdated + } + if patch > conmonMinPatchVersion { + return nil + } + + return nil +} + // TmpDir gets the current Libpod temporary files directory. func (r *Runtime) TmpDir() (string, error) { if !r.valid { diff --git a/libpod/runtime_img.go b/libpod/runtime_img.go index 3588467a5..2b101c01f 100644 --- a/libpod/runtime_img.go +++ b/libpod/runtime_img.go @@ -66,7 +66,8 @@ func (r *Runtime) RemoveImage(ctx context.Context, img *image.Image, force bool) hasChildren, err := img.IsParent(ctx) if err != nil { - return nil, err + logrus.Warnf("error determining if an image is a parent: %v, ignoring the error", err) + hasChildren = false } if (len(img.Names()) > 1 && !img.InputIsID()) || hasChildren { diff --git a/libpod/shutdown/handler.go b/libpod/shutdown/handler.go index ac1d33910..848b6729a 100644 --- a/libpod/shutdown/handler.go +++ b/libpod/shutdown/handler.go @@ -18,7 +18,7 @@ var ( stopped bool sigChan chan os.Signal cancelChan chan bool - // Syncronize accesses to the map + // Synchronize accesses to the map handlerLock sync.Mutex // Definitions of all on-shutdown handlers handlers map[string]func(os.Signal) error diff --git a/pkg/api/handlers/compat/containers.go b/pkg/api/handlers/compat/containers.go index e7146a5d8..d97a4d3bd 100644 --- a/pkg/api/handlers/compat/containers.go +++ b/pkg/api/handlers/compat/containers.go @@ -26,6 +26,7 @@ import ( "github.com/docker/go-units" "github.com/gorilla/schema" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) func RemoveContainer(w http.ResponseWriter, r *http.Request) { @@ -148,14 +149,19 @@ func ListContainers(w http.ResponseWriter, r *http.Request) { containers = containers[:query.Limit] } } - var list = make([]*handlers.Container, len(containers)) - for i, ctnr := range containers { + list := make([]*handlers.Container, 0, len(containers)) + for _, ctnr := range containers { api, err := LibpodToContainer(ctnr, query.Size) if err != nil { + if errors.Cause(err) == define.ErrNoSuchCtr { + // container was removed between the initial fetch of the list and conversion + logrus.Debugf("Container %s removed between initial fetch and conversion, ignoring in output", ctnr.ID()) + continue + } utils.InternalServerError(w, err) return } - list[i] = api + list = append(list, api) } utils.WriteResponse(w, http.StatusOK, list) } diff --git a/pkg/api/handlers/libpod/info.go b/pkg/api/handlers/libpod/info.go index 546609451..8868d563d 100644 --- a/pkg/api/handlers/libpod/info.go +++ b/pkg/api/handlers/libpod/info.go @@ -5,11 +5,13 @@ import ( "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/pkg/api/handlers/utils" + "github.com/containers/podman/v3/pkg/domain/infra/abi" ) func GetInfo(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value("runtime").(*libpod.Runtime) - info, err := runtime.Info() + containerEngine := abi.ContainerEngine{Libpod: runtime} + info, err := containerEngine.Info(r.Context()) if err != nil { utils.InternalServerError(w, err) return diff --git a/pkg/api/handlers/utils/containers.go b/pkg/api/handlers/utils/containers.go index 91e02abf1..c4c9cc2ea 100644 --- a/pkg/api/handlers/utils/containers.go +++ b/pkg/api/handlers/utils/containers.go @@ -76,7 +76,7 @@ func WaitContainerDocker(w http.ResponseWriter, r *http.Request) { exitCode, err := waitDockerCondition(ctx, name, interval, condition) msg := "" if err != nil { - logrus.Errorf("error while waiting on condtion: %q", err) + logrus.Errorf("error while waiting on condition: %q", err) msg = err.Error() } responseData := handlers.ContainerWaitOKBody{ diff --git a/pkg/domain/infra/abi/images.go b/pkg/domain/infra/abi/images.go index c02eb2bfc..84c7ebecd 100644 --- a/pkg/domain/infra/abi/images.go +++ b/pkg/domain/infra/abi/images.go @@ -620,8 +620,8 @@ func (ir *ImageEngine) Remove(ctx context.Context, images []string, opts entitie for _, img := range storageImages { isParent, err := img.IsParent(ctx) if err != nil { - rmErrors = append(rmErrors, err) - continue + logrus.Warnf("%v, ignoring the error", err) + isParent = false } // Skip parent images. if isParent { diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go index 4a13a8029..6ddd4a042 100644 --- a/pkg/domain/infra/abi/play.go +++ b/pkg/domain/infra/abi/play.go @@ -428,7 +428,7 @@ func readConfigMapFromFile(r io.Reader) (v1.ConfigMap, error) { return cm, nil } -// splitMultiDocYAML reads mutiple documents in a YAML file and +// splitMultiDocYAML reads multiple documents in a YAML file and // returns them as a list. func splitMultiDocYAML(yamlContent []byte) ([][]byte, error) { var documentList [][]byte @@ -471,7 +471,7 @@ func getKubeKind(obj []byte) (string, error) { } // sortKubeKinds adds the correct creation order for the kube kinds. -// Any pod dependecy will be created first like volumes, secrets, etc. +// Any pod dependency will be created first like volumes, secrets, etc. func sortKubeKinds(documentList [][]byte) ([][]byte, error) { var sortedDocumentList [][]byte diff --git a/pkg/domain/infra/abi/system.go b/pkg/domain/infra/abi/system.go index f87f9e370..9bba0fa6c 100644 --- a/pkg/domain/infra/abi/system.go +++ b/pkg/domain/infra/abi/system.go @@ -21,6 +21,7 @@ import ( "github.com/containers/podman/v3/pkg/util" "github.com/containers/podman/v3/utils" "github.com/containers/storage" + "github.com/containers/storage/pkg/unshare" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" @@ -32,17 +33,11 @@ func (ic *ContainerEngine) Info(ctx context.Context) (*define.Info, error) { if err != nil { return nil, err } - xdg, err := util.GetRuntimeDir() + + socketPath, err := util.SocketPath() if err != nil { return nil, err } - if len(xdg) == 0 { - // If no xdg is returned, assume root socket - xdg = "/run" - } - - // Glue the socket path together - socketPath := filepath.Join(xdg, "podman", "podman.sock") rs := define.RemoteSocket{ Path: socketPath, Exists: false, @@ -64,7 +59,11 @@ func (ic *ContainerEngine) Info(ctx context.Context) (*define.Info, error) { func (ic *ContainerEngine) SetupRootless(_ context.Context, cmd *cobra.Command) error { // do it only after podman has already re-execed and running with uid==0. - if os.Geteuid() == 0 { + hasCapSysAdmin, err := unshare.HasCapSysAdmin() + if err != nil { + return err + } + if hasCapSysAdmin { ownsCgroup, err := cgroups.UserOwnsCurrentSystemdCgroup() if err != nil { logrus.Infof("Failed to detect the owner for the current cgroup: %v", err) diff --git a/pkg/machine/pull.go b/pkg/machine/pull.go index 41abe6993..d9f34057f 100644 --- a/pkg/machine/pull.go +++ b/pkg/machine/pull.go @@ -170,7 +170,7 @@ func Decompress(localPath, uncompressedPath string) error { // Will error out if file without .xz already exists // Maybe extracting then renameing is a good idea here.. -// depends on xz: not pre-installed on mac, so it becomes a brew dependecy +// depends on xz: not pre-installed on mac, so it becomes a brew dependency func decompressXZ(src string, output io.Writer) error { fmt.Println("Extracting compressed file") cmd := exec.Command("xzcat", "-k", src) diff --git a/pkg/rootless/rootless_linux.go b/pkg/rootless/rootless_linux.go index dda230dbc..fdfeed854 100644 --- a/pkg/rootless/rootless_linux.go +++ b/pkg/rootless/rootless_linux.go @@ -4,6 +4,7 @@ package rootless import ( "bufio" + "bytes" "fmt" "io" "io/ioutil" @@ -18,6 +19,7 @@ import ( "github.com/containers/podman/v3/pkg/errorhandling" "github.com/containers/storage/pkg/idtools" + "github.com/containers/storage/pkg/unshare" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" @@ -67,6 +69,15 @@ func IsRootless() bool { } } isRootless = os.Geteuid() != 0 || os.Getenv("_CONTAINERS_USERNS_CONFIGURED") != "" + if !isRootless { + hasCapSysAdmin, err := unshare.HasCapSysAdmin() + if err != nil { + logrus.Warnf("failed to read CAP_SYS_ADMIN presence for the current process") + } + if err == nil && !hasCapSysAdmin { + isRootless = true + } + } }) return isRootless } @@ -142,8 +153,12 @@ func tryMappingTool(uid bool, pid int, hostID int, mappings []idtools.IDMap) err // namespace of the specified PID without looking up its parent. Useful to join directly // the conmon process. func joinUserAndMountNS(pid uint, pausePid string) (bool, int, error) { - if os.Geteuid() == 0 || os.Getenv("_CONTAINERS_USERNS_CONFIGURED") != "" { - return false, -1, nil + hasCapSysAdmin, err := unshare.HasCapSysAdmin() + if err != nil { + return false, 0, err + } + if hasCapSysAdmin || os.Getenv("_CONTAINERS_USERNS_CONFIGURED") != "" { + return false, 0, nil } cPausePid := C.CString(pausePid) @@ -180,8 +195,11 @@ func GetConfiguredMappings() ([]idtools.IDMap, []idtools.IDMap, error) { } mappings, err := idtools.NewIDMappings(username, username) if err != nil { - logrus.Errorf( - "cannot find UID/GID for user %s: %v - check rootless mode in man pages.", username, err) + logLevel := logrus.ErrorLevel + if os.Geteuid() == 0 && GetRootlessUID() == 0 { + logLevel = logrus.DebugLevel + } + logrus.StandardLogger().Logf(logLevel, "cannot find UID/GID for user %s: %v - check rootless mode in man pages.", username, err) } else { uids = mappings.UIDs() gids = mappings.GIDs() @@ -189,8 +207,28 @@ func GetConfiguredMappings() ([]idtools.IDMap, []idtools.IDMap, error) { return uids, gids, nil } +func copyMappings(from, to string) error { + content, err := ioutil.ReadFile(from) + if err != nil { + return err + } + // Both runc and crun check whether the current process is in a user namespace + // by looking up 4294967295 in /proc/self/uid_map. If the mappings would be + // copied as they are, the check in the OCI runtimes would fail. So just split + // it in two different ranges. + if bytes.Contains(content, []byte("4294967295")) { + content = []byte("0 0 1\n1 1 4294967294\n") + } + return ioutil.WriteFile(to, content, 0600) +} + func becomeRootInUserNS(pausePid, fileToRead string, fileOutput *os.File) (_ bool, _ int, retErr error) { - if os.Geteuid() == 0 || os.Getenv("_CONTAINERS_USERNS_CONFIGURED") != "" { + hasCapSysAdmin, err := unshare.HasCapSysAdmin() + if err != nil { + return false, 0, err + } + + if hasCapSysAdmin || os.Getenv("_CONTAINERS_USERNS_CONFIGURED") != "" { if os.Getenv("_CONTAINERS_USERNS_CONFIGURED") == "init" { return false, 0, runInUser() } @@ -247,8 +285,16 @@ func becomeRootInUserNS(pausePid, fileToRead string, fileOutput *os.File) (_ boo return false, -1, err } + uidMap := fmt.Sprintf("/proc/%d/uid_map", pid) + gidMap := fmt.Sprintf("/proc/%d/gid_map", pid) + uidsMapped := false - if uids != nil { + + if err := copyMappings("/proc/self/uid_map", uidMap); err == nil { + uidsMapped = true + } + + if uids != nil && !uidsMapped { err := tryMappingTool(true, pid, os.Geteuid(), uids) // If some mappings were specified, do not ignore the error if err != nil && len(uids) > 0 { @@ -265,7 +311,6 @@ func becomeRootInUserNS(pausePid, fileToRead string, fileOutput *os.File) (_ boo } logrus.Debugf("write setgroups file exited with 0") - uidMap := fmt.Sprintf("/proc/%d/uid_map", pid) err = ioutil.WriteFile(uidMap, []byte(fmt.Sprintf("%d %d 1\n", 0, os.Geteuid())), 0666) if err != nil { return false, -1, errors.Wrapf(err, "cannot write uid_map") @@ -274,7 +319,10 @@ func becomeRootInUserNS(pausePid, fileToRead string, fileOutput *os.File) (_ boo } gidsMapped := false - if gids != nil { + if err := copyMappings("/proc/self/gid_map", gidMap); err == nil { + gidsMapped = true + } + if gids != nil && !gidsMapped { err := tryMappingTool(false, pid, os.Getegid(), gids) // If some mappings were specified, do not ignore the error if err != nil && len(gids) > 0 { @@ -283,7 +331,6 @@ func becomeRootInUserNS(pausePid, fileToRead string, fileOutput *os.File) (_ boo gidsMapped = err == nil } if !gidsMapped { - gidMap := fmt.Sprintf("/proc/%d/gid_map", pid) err = ioutil.WriteFile(gidMap, []byte(fmt.Sprintf("%d %d 1\n", 0, os.Getegid())), 0666) if err != nil { return false, -1, errors.Wrapf(err, "cannot write gid_map") diff --git a/pkg/specgen/generate/kube/kube.go b/pkg/specgen/generate/kube/kube.go index 7aeec9d41..1347ed1e0 100644 --- a/pkg/specgen/generate/kube/kube.go +++ b/pkg/specgen/generate/kube/kube.go @@ -56,7 +56,7 @@ func ToPodGen(ctx context.Context, podName string, podYAML *v1.PodTemplateSpec) } p.DNSServer = servers } - // search domans + // search domains if domains := dnsConfig.Searches; len(domains) > 0 { p.DNSSearch = domains } diff --git a/pkg/systemd/generate/common_test.go b/pkg/systemd/generate/common_test.go index 30e758127..fdcc9d21b 100644 --- a/pkg/systemd/generate/common_test.go +++ b/pkg/systemd/generate/common_test.go @@ -199,8 +199,8 @@ func TestEscapeSystemdArguments(t *testing.T) { []string{"foo", `"command with backslash \\"`}, }, { - []string{"foo", `command with two backslashs \\`}, - []string{"foo", `"command with two backslashs \\\\"`}, + []string{"foo", `command with two backslashes \\`}, + []string{"foo", `"command with two backslashes \\\\"`}, }, } diff --git a/pkg/util/filters.go b/pkg/util/filters.go index 43bf646f1..e252c1ddf 100644 --- a/pkg/util/filters.go +++ b/pkg/util/filters.go @@ -94,7 +94,7 @@ func PrepareFilters(r *http.Request) (*map[string][]string, error) { return &filterMap, nil } -// MatchLabelFilters matches labels and returs true if they are valid +// MatchLabelFilters matches labels and returns true if they are valid func MatchLabelFilters(filterValues []string, labels map[string]string) bool { outer: for _, filterValue := range filterValues { diff --git a/pkg/util/utils.go b/pkg/util/utils.go index bbaf72981..622fbde99 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -706,3 +706,26 @@ func IDtoolsToRuntimeSpec(idMaps []idtools.IDMap) (convertedIDMap []specs.LinuxI } return convertedIDMap } + +var socketPath string + +func SetSocketPath(path string) { + socketPath = path +} + +func SocketPath() (string, error) { + if socketPath != "" { + return socketPath, nil + } + xdg, err := GetRuntimeDir() + if err != nil { + return "", err + } + if len(xdg) == 0 { + // If no xdg is returned, assume root socket + xdg = "/run" + } + + // Glue the socket path together + return filepath.Join(xdg, "podman", "podman.sock"), nil +} diff --git a/test/apiv2/rest_api/test_rest_v2_0_0.py b/test/apiv2/rest_api/test_rest_v2_0_0.py index bf0ee0603..3b089e2f2 100644 --- a/test/apiv2/rest_api/test_rest_v2_0_0.py +++ b/test/apiv2/rest_api/test_rest_v2_0_0.py @@ -378,7 +378,7 @@ class TestApi(unittest.TestCase): self.assertEqual(r.status_code, 200, r.text) objs = json.loads(r.text) self.assertIn(type(objs), (list,)) - # There should be only one offical image + # There should be only one official image self.assertEqual(len(objs), 1) def do_search4(): diff --git a/test/compose/slirp4netns_opts/docker-compose.yml b/test/compose/slirp4netns_opts/docker-compose.yml new file mode 100644 index 000000000..dcdcae04c --- /dev/null +++ b/test/compose/slirp4netns_opts/docker-compose.yml @@ -0,0 +1,5 @@ +services: + alpine: + image: alpine + network_mode: "slirp4netns:allow_host_loopback=true" + command: sh -c "echo teststring | nc 10.0.2.2 5001" diff --git a/test/compose/slirp4netns_opts/setup.sh b/test/compose/slirp4netns_opts/setup.sh new file mode 100644 index 000000000..35bbf7c70 --- /dev/null +++ b/test/compose/slirp4netns_opts/setup.sh @@ -0,0 +1,8 @@ +# -*- bash -*- + +# create tempfile to store nc output +OUTFILE=$(mktemp) +# listen on a port, the container will try to connect to it +nc -l 5001 > $OUTFILE & + +nc_pid=$! diff --git a/test/compose/slirp4netns_opts/teardown.sh b/test/compose/slirp4netns_opts/teardown.sh new file mode 100644 index 000000000..656724363 --- /dev/null +++ b/test/compose/slirp4netns_opts/teardown.sh @@ -0,0 +1,4 @@ +# -*- bash -*- + +kill $nc_pid &> /dev/null +rm -f $OUTFILE diff --git a/test/compose/slirp4netns_opts/tests.sh b/test/compose/slirp4netns_opts/tests.sh new file mode 100644 index 000000000..1efce45c4 --- /dev/null +++ b/test/compose/slirp4netns_opts/tests.sh @@ -0,0 +1,6 @@ +# -*- bash -*- + +output="$(cat $OUTFILE)" +expected="teststring" + +is "$output" "$expected" "$testname : nc received teststring" diff --git a/test/e2e/info_test.go b/test/e2e/info_test.go index 3ce294b30..0b112b312 100644 --- a/test/e2e/info_test.go +++ b/test/e2e/info_test.go @@ -109,4 +109,19 @@ var _ = Describe("Podman Info", func() { Expect(err).To(BeNil()) Expect(string(out)).To(Equal(expect)) }) + + It("podman info check RemoteSocket", func() { + session := podmanTest.Podman([]string{"info", "--format", "{{.Host.RemoteSocket.Path}}"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(MatchRegexp("/run/.*podman.*sock")) + + if IsRemote() { + session = podmanTest.Podman([]string{"info", "--format", "{{.Host.RemoteSocket.Exists}}"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(ContainSubstring("true")) + } + }) + }) diff --git a/test/e2e/run_apparmor_test.go b/test/e2e/run_apparmor_test.go index 63c52451f..1f9b9bc90 100644 --- a/test/e2e/run_apparmor_test.go +++ b/test/e2e/run_apparmor_test.go @@ -14,6 +14,7 @@ import ( . "github.com/onsi/gomega" ) +// wip func skipIfAppArmorEnabled() { if apparmor.IsEnabled() { Skip("Apparmor is enabled") diff --git a/test/system/040-ps.bats b/test/system/040-ps.bats index ae27c479f..182d75547 100644 --- a/test/system/040-ps.bats +++ b/test/system/040-ps.bats @@ -5,6 +5,9 @@ load helpers @test "podman ps - basic tests" { rand_name=$(random_string 30) + run_podman ps --noheading + is "$output" "" "baseline: empty results from ps --noheading" + run_podman run -d --name $rand_name $IMAGE sleep 5 cid=$output is "$cid" "[0-9a-f]\{64\}$" @@ -30,8 +33,6 @@ load helpers "${cid:0:12} \+$IMAGE *sleep .* Exited .* $rand_name" \ "podman ps -a" - - run_podman rm $cid } diff --git a/test/system/050-stop.bats b/test/system/050-stop.bats index a9495e350..2ed791429 100644 --- a/test/system/050-stop.bats +++ b/test/system/050-stop.bats @@ -114,7 +114,7 @@ load helpers @test "podman stop - unlock while waiting for timeout" { # Test that the container state transitions to "stopping" and that other # commands can get the container's lock. To do that, run a container that - # ingores SIGTERM such that the Podman would wait 20 seconds for the stop + # ignores SIGTERM such that the Podman would wait 20 seconds for the stop # to finish. This gives us enough time to try some commands and inspect # the container's status. diff --git a/test/system/070-build.bats b/test/system/070-build.bats index d4017ae01..6ae78de2e 100644 --- a/test/system/070-build.bats +++ b/test/system/070-build.bats @@ -354,7 +354,7 @@ Cmd[1] | $s_echo WorkingDir | $workdir Labels.$label_name | $label_value " - # FIXME: 2021-02-24: Fixed in buildah #3036; reenable this once podman + # FIXME: 2021-02-24: Fixed in buildah #3036; re-enable this once podman # vendors in a newer buildah! # Labels.\"io.buildah.version\" | $buildah_version diff --git a/test/system/160-volumes.bats b/test/system/160-volumes.bats index 4952eafc2..98992f973 100644 --- a/test/system/160-volumes.bats +++ b/test/system/160-volumes.bats @@ -23,6 +23,9 @@ function teardown() { @test "podman run --volumes : basic" { skip_if_remote "volumes cannot be shared across hosts" + run_podman volume list --noheading + is "$output" "" "baseline: empty results from list --noheading" + # Create three temporary directories vol1=${PODMAN_TMPDIR}/v1_$(random_string) vol2=${PODMAN_TMPDIR}/v2_$(random_string) diff --git a/test/system/170-run-userns.bats b/test/system/170-run-userns.bats new file mode 100644 index 000000000..2dc5b078f --- /dev/null +++ b/test/system/170-run-userns.bats @@ -0,0 +1,45 @@ +#!/usr/bin/env bats -*- bats -*- +# shellcheck disable=SC2096 +# +# Tests for podman build +# + +load helpers + +@test "podman --group-add keep-groups while in a userns" { + skip_if_rootless "choot is not allowed in rootless mode" + skip_if_remote "--group-add keep-groups not supported in remote mode" + run chroot --groups 1234 / ${PODMAN} run --uidmap 0:200000:5000 --group-add keep-groups $IMAGE id + is "$output" ".*65534(nobody)" "Check group leaked into user namespace" +} + +@test "podman --group-add keep-groups while not in a userns" { + skip_if_rootless "choot is not allowed in rootless mode" + skip_if_remote "--group-add keep-groups not supported in remote mode" + run chroot --groups 1234,5678 / ${PODMAN} run --group-add keep-groups $IMAGE id + is "$output" ".*1234" "Check group leaked into container" +} + +@test "podman --group-add without keep-groups while in a userns" { + skip_if_rootless "choot is not allowed in rootless mode" + skip_if_remote "--group-add keep-groups not supported in remote mode" + run chroot --groups 1234,5678 / ${PODMAN} run --uidmap 0:200000:5000 --group-add 457 $IMAGE id + is "$output" ".*457" "Check group leaked into container" +} + +@test "podman --remote --group-add keep-groups " { + if is_remote; then + run_podman 125 run --group-add keep-groups $IMAGE id + is "$output" ".*not supported in remote mode" "Remote check --group-add keep-groups" + fi +} + +@test "podman --group-add without keep-groups " { + run_podman run --group-add 457 $IMAGE id + is "$output" ".*457" "Check group leaked into container" +} + +@test "podman --group-add keep-groups plus added groups " { + run_podman 125 run --group-add keep-groups --group-add 457 $IMAGE id + is "$output" ".*the '--group-add keep-groups' option is not allowed with any other --group-add options" "Check group leaked into container" +} diff --git a/test/system/200-pod.bats b/test/system/200-pod.bats index c65449212..054eda908 100644 --- a/test/system/200-pod.bats +++ b/test/system/200-pod.bats @@ -17,6 +17,17 @@ function teardown() { } +@test "podman pod - basic tests" { + run_podman pod list --noheading + is "$output" "" "baseline: empty results from list --noheading" + + run_podman pod ls --noheading + is "$output" "" "baseline: empty results from ls --noheading" + + run_podman pod ps --noheading + is "$output" "" "baseline: empty results from ps --noheading" +} + @test "podman pod top - containers in different PID namespaces" { # With infra=false, we don't get a /pause container (we also # don't pull k8s.gcr.io/pause ) diff --git a/test/system/330-corrupt-images.bats b/test/system/330-corrupt-images.bats new file mode 100644 index 000000000..c51cc8d46 --- /dev/null +++ b/test/system/330-corrupt-images.bats @@ -0,0 +1,134 @@ +#!/usr/bin/env bats -*- bats -*- +# +# All tests in here perform nasty manipulations on image storage. +# + +load helpers + +############################################################################### +# BEGIN setup/teardown + +# Create a scratch directory; this is what we'll use for image store and cache +if [ -z "${PODMAN_CORRUPT_TEST_WORKDIR}" ]; then + export PODMAN_CORRUPT_TEST_WORKDIR=$(mktemp -d --tmpdir=${BATS_TMPDIR:-${TMPDIR:-/tmp}} podman_corrupt_test.XXXXXX) +fi + +PODMAN_CORRUPT_TEST_IMAGE_FQIN=quay.io/libpod/alpine@sha256:634a8f35b5f16dcf4aaa0822adc0b1964bb786fca12f6831de8ddc45e5986a00 +PODMAN_CORRUPT_TEST_IMAGE_ID=961769676411f082461f9ef46626dd7a2d1e2b2a38e6a44364bcbecf51e66dd4 + +# All tests in this file (and ONLY in this file) run with a custom rootdir +function setup() { + skip_if_remote "none of these tests run under podman-remote" + _PODMAN_TEST_OPTS="--root ${PODMAN_CORRUPT_TEST_WORKDIR}/root" +} + +function teardown() { + # No other tests should ever run with this custom rootdir + unset _PODMAN_TEST_OPTS + + is_remote && return + + # Clean up + umount ${PODMAN_CORRUPT_TEST_WORKDIR}/root/overlay || true + if is_rootless; then + run_podman unshare rm -rf ${PODMAN_CORRUPT_TEST_WORKDIR}/root + else + rm -rf ${PODMAN_CORRUPT_TEST_WORKDIR}/root + fi +} + +# END setup/teardown +############################################################################### +# BEGIN primary test helper + +# This is our main action, invoked by every actual test. It: +# - creates a new empty rootdir +# - populates it with our crafted test image +# - removes [ manifest, blob ] +# - confirms that "podman images" throws an error +# - runs the specified command (rmi -a -f, prune, reset, etc) +# - confirms that it succeeds, and also emits expected warnings +function _corrupt_image_test() { + # Run this test twice: once removing manifest, once removing blob + for what_to_rm in manifest blob; do + # I have no idea, but this sometimes remains mounted + umount ${PODMAN_CORRUPT_TEST_WORKDIR}/root/overlay || true + # Start with a fresh storage root, load prefetched image into it. + /bin/rm -rf ${PODMAN_CORRUPT_TEST_WORKDIR}/root + mkdir -p ${PODMAN_CORRUPT_TEST_WORKDIR}/root + run_podman load -i ${PODMAN_CORRUPT_TEST_WORKDIR}/img.tar + # "podman load" restores it without a tag, which (a) causes rmi-by-name + # to fail, and (b) causes "podman images" to exit 0 instead of 125 + run_podman tag ${PODMAN_CORRUPT_TEST_IMAGE_ID} ${PODMAN_CORRUPT_TEST_IMAGE_FQIN} + + # shortcut variable name + local id=${PODMAN_CORRUPT_TEST_IMAGE_ID} + + case "$what_to_rm" in + manifest) rm_path=manifest ;; + blob) rm_path="=$(echo -n "sha256:$id" | base64 -w0)" ;; + *) die "Internal error: unknown action '$what_to_rm'" ;; + esac + + # Corruptify, and confirm that 'podman images' throws an error + rm -v ${PODMAN_CORRUPT_TEST_WORKDIR}/root/*-images/$id/${rm_path} + run_podman 125 images + is "$output" "Error: error retrieving label for image \"$id\": you may need to remove the image to resolve the error" + + # Run the requested command. Confirm it succeeds, with suitable warnings + run_podman $* + is "$output" ".*error determining parent of image.*ignoring the error" \ + "$* with missing $what_to_rm" + + run_podman images -a --noheading + is "$output" "" "podman images -a, after $*, is empty" + done +} + +# END primary test helper +############################################################################### +# BEGIN first "test" does a one-time pull of our desired image + +@test "podman corrupt images - initialize" { + # Pull once, save cached copy. + run_podman pull $PODMAN_CORRUPT_TEST_IMAGE_FQIN + run_podman save -o ${PODMAN_CORRUPT_TEST_WORKDIR}/img.tar \ + $PODMAN_CORRUPT_TEST_IMAGE_FQIN +} + +# END first "test" does a one-time pull of our desired image +############################################################################### +# BEGIN actual tests + +@test "podman corrupt images - rmi -f <image-id>" { + _corrupt_image_test "rmi -f ${PODMAN_CORRUPT_TEST_IMAGE_ID}" +} + +@test "podman corrupt images - rmi -f <image-name>" { + _corrupt_image_test "rmi -f ${PODMAN_CORRUPT_TEST_IMAGE_FQIN}" +} + +@test "podman corrupt images - rmi -f -a" { + _corrupt_image_test "rmi -f -a" +} + +@test "podman corrupt images - image prune" { + _corrupt_image_test "image prune -a -f" +} + +@test "podman corrupt images - system reset" { + _corrupt_image_test "system reset -f" +} + +# END actual tests +############################################################################### +# BEGIN final cleanup + +@test "podman corrupt images - cleanup" { + rm -rf ${PODMAN_CORRUPT_TEST_WORKDIR} +} + +# END final cleanup +############################################################################### + +# vim: filetype=sh diff --git a/test/system/500-networking.bats b/test/system/500-networking.bats index cda054b15..8da864798 100644 --- a/test/system/500-networking.bats +++ b/test/system/500-networking.bats @@ -5,6 +5,19 @@ load helpers +@test "podman network - basic tests" { + heading="*NETWORK*ID*NAME*VERSION*PLUGINS*" + run_podman network ls + if [[ ${output} != ${heading} ]]; then + die "network ls expected heading is not available" + fi + + run_podman network ls --noheading + if [[ ${output} = ${heading} ]]; then + die "network ls --noheading did not remove heading: $output" + fi +} + # Copied from tsweeney's https://github.com/containers/podman/issues/4827 @test "podman networking: port on localhost" { skip_if_remote "FIXME: reevaluate this one after #7360 is fixed" @@ -20,9 +33,9 @@ load helpers # Bind-mount this file with a different name to a container running httpd run_podman run -d --name myweb -p "$HOST_PORT:80" \ - -v $INDEX1:/var/www/index.txt \ - -w /var/www \ - $IMAGE /bin/busybox-extras httpd -f -p 80 + -v $INDEX1:/var/www/index.txt \ + -w /var/www \ + $IMAGE /bin/busybox-extras httpd -f -p 80 cid=$output # In that container, create a second file, using exec and redirection @@ -71,7 +84,7 @@ load helpers # We could get more parseable output by using $NCAT_REMOTE_ADDR, # but busybox nc doesn't support that. run_podman run -d --userns=keep-id -p 127.0.0.1:$myport:$myport \ - $IMAGE nc -l -n -v -p $myport + $IMAGE nc -l -n -v -p $myport cid="$output" # emit random string, and check it @@ -108,7 +121,7 @@ load helpers # (Assert that output is formatted, not a one-line blob: #8011) run_podman network inspect $mynetname if [[ "${#lines[*]}" -lt 5 ]]; then - die "Output from 'pod inspect' is only ${#lines[*]} lines; see #8011" + die "Output from 'pod inspect' is only ${#lines[*]} lines; see #8011" fi run_podman run --rm --network $mynetname $IMAGE ip a @@ -116,7 +129,7 @@ load helpers "sdfsdf" run_podman run --rm -d --network $mynetname -p 127.0.0.1:$myport:$myport \ - $IMAGE nc -l -n -v -p $myport + $IMAGE nc -l -n -v -p $myport cid="$output" # emit random string, and check it @@ -159,9 +172,9 @@ load helpers # Bind-mount this file with a different name to a container running httpd run_podman run -d --name myweb -p "$HOST_PORT:80" \ - -v $INDEX1:/var/www/index.txt \ - -w /var/www \ - $IMAGE /bin/busybox-extras httpd -f -p 80 + -v $INDEX1:/var/www/index.txt \ + -w /var/www \ + $IMAGE /bin/busybox-extras httpd -f -p 80 cid=$output run_podman inspect $cid --format "{{.NetworkSettings.IPAddress}}" @@ -179,7 +192,7 @@ load helpers # check that we cannot curl (timeout after 5 sec) run timeout 5 curl -s $SERVER/index.txt if [ "$status" -ne 124 ]; then - die "curl did not timeout, status code: $status" + die "curl did not timeout, status code: $status" fi # reload the network to recreate the iptables rules diff --git a/vendor/github.com/containers/common/pkg/auth/auth.go b/vendor/github.com/containers/common/pkg/auth/auth.go index 88028d9f8..a9ad60f43 100644 --- a/vendor/github.com/containers/common/pkg/auth/auth.go +++ b/vendor/github.com/containers/common/pkg/auth/auth.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "os" + "path/filepath" "strings" "github.com/containers/image/v5/docker" @@ -13,18 +14,20 @@ import ( "github.com/containers/image/v5/types" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "golang.org/x/crypto/ssh/terminal" + terminal "golang.org/x/term" ) // GetDefaultAuthFile returns env value REGISTRY_AUTH_FILE as default // --authfile path used in multiple --authfile flag definitions // Will fail over to DOCKER_CONFIG if REGISTRY_AUTH_FILE environment is not set func GetDefaultAuthFile() string { - authfile := os.Getenv("REGISTRY_AUTH_FILE") - if authfile == "" { - authfile = os.Getenv("DOCKER_CONFIG") + if authfile := os.Getenv("REGISTRY_AUTH_FILE"); authfile != "" { + return authfile + } + if auth_env := os.Getenv("DOCKER_CONFIG"); auth_env != "" { + return filepath.Join(auth_env, "config.json") } - return authfile + return "" } // CheckAuthFile validates filepath given by --authfile @@ -34,7 +37,7 @@ func CheckAuthFile(authfile string) error { return nil } if _, err := os.Stat(authfile); err != nil { - return errors.Wrapf(err, "error checking authfile path %s", authfile) + return errors.Wrap(err, "checking authfile") } return nil } @@ -70,11 +73,11 @@ func Login(ctx context.Context, systemContext *types.SystemContext, opts *LoginO err error ) if len(args) > 1 { - return errors.Errorf("login accepts only one registry to login to") + return errors.New("login accepts only one registry to login to") } if len(args) == 0 { if !opts.AcceptUnspecifiedRegistry { - return errors.Errorf("please provide a registry to login to") + return errors.New("please provide a registry to login to") } if server, err = defaultRegistryWhenUnspecified(systemContext); err != nil { return err @@ -85,7 +88,7 @@ func Login(ctx context.Context, systemContext *types.SystemContext, opts *LoginO } authConfig, err := config.GetCredentials(systemContext, server) if err != nil { - return errors.Wrapf(err, "error reading auth file") + return errors.Wrap(err, "reading auth file") } if opts.GetLoginSet { if authConfig.Username == "" { @@ -95,17 +98,17 @@ func Login(ctx context.Context, systemContext *types.SystemContext, opts *LoginO return nil } if authConfig.IdentityToken != "" { - return errors.Errorf("currently logged in, auth file contains an Identity token") + return errors.New("currently logged in, auth file contains an Identity token") } password := opts.Password if opts.StdinPassword { var stdinPasswordStrBuilder strings.Builder if opts.Password != "" { - return errors.Errorf("Can't specify both --password-stdin and --password") + return errors.New("Can't specify both --password-stdin and --password") } if opts.Username == "" { - return errors.Errorf("Must provide --username with --password-stdin") + return errors.New("Must provide --username with --password-stdin") } scanner := bufio.NewScanner(opts.Stdin) for scanner.Scan() { @@ -126,7 +129,7 @@ func Login(ctx context.Context, systemContext *types.SystemContext, opts *LoginO username, password, err := getUserAndPass(opts, password, authConfig.Username) if err != nil { - return errors.Wrapf(err, "error getting username and password") + return errors.Wrap(err, "getting username and password") } if err = docker.CheckAuth(ctx, systemContext, username, password, server); err == nil { @@ -143,7 +146,7 @@ func Login(ctx context.Context, systemContext *types.SystemContext, opts *LoginO logrus.Debugf("error logging into %q: %v", server, unauthorized) return errors.Errorf("error logging into %q: invalid username/password", server) } - return errors.Wrapf(err, "error authenticating creds for %q", server) + return errors.Wrapf(err, "authenticating creds for %q", server) } // getRegistryName scrubs and parses the input to get the server name @@ -172,7 +175,7 @@ func getUserAndPass(opts *LoginOptions, password, userFromAuthFile string) (user } username, err = reader.ReadString('\n') if err != nil { - return "", "", errors.Wrapf(err, "error reading username") + return "", "", errors.Wrap(err, "reading username") } // If the user just hit enter, use the displayed user from the // the authentication file. This allows to do a lazy @@ -186,7 +189,7 @@ func getUserAndPass(opts *LoginOptions, password, userFromAuthFile string) (user fmt.Fprint(opts.Stdout, "Password: ") pass, err := terminal.ReadPassword(0) if err != nil { - return "", "", errors.Wrapf(err, "error reading password") + return "", "", errors.Wrap(err, "reading password") } password = string(pass) fmt.Fprintln(opts.Stdout) @@ -206,11 +209,11 @@ func Logout(systemContext *types.SystemContext, opts *LogoutOptions, args []stri err error ) if len(args) > 1 { - return errors.Errorf("logout accepts only one registry to logout from") + return errors.New("logout accepts only one registry to logout from") } if len(args) == 0 && !opts.All { if !opts.AcceptUnspecifiedRegistry { - return errors.Errorf("please provide a registry to logout from") + return errors.New("please provide a registry to logout from") } if server, err = defaultRegistryWhenUnspecified(systemContext); err != nil { return err @@ -219,7 +222,7 @@ func Logout(systemContext *types.SystemContext, opts *LogoutOptions, args []stri } if len(args) != 0 { if opts.All { - return errors.Errorf("--all takes no arguments") + return errors.New("--all takes no arguments") } server = getRegistryName(args[0]) } @@ -240,7 +243,7 @@ func Logout(systemContext *types.SystemContext, opts *LogoutOptions, args []stri case config.ErrNotLoggedIn: authConfig, err := config.GetCredentials(systemContext, server) if err != nil { - return errors.Wrapf(err, "error reading auth file") + return errors.Wrap(err, "reading auth file") } authInvalid := docker.CheckAuth(context.Background(), systemContext, authConfig.Username, authConfig.Password, server) if authConfig.Username != "" && authConfig.Password != "" && authInvalid == nil { @@ -249,7 +252,7 @@ func Logout(systemContext *types.SystemContext, opts *LogoutOptions, args []stri } return errors.Errorf("Not logged into %s\n", server) default: - return errors.Wrapf(err, "error logging out of %q", server) + return errors.Wrapf(err, "logging out of %q", server) } } @@ -258,10 +261,10 @@ func Logout(systemContext *types.SystemContext, opts *LogoutOptions, args []stri func defaultRegistryWhenUnspecified(systemContext *types.SystemContext) (string, error) { registriesFromFile, err := sysregistriesv2.UnqualifiedSearchRegistries(systemContext) if err != nil { - return "", errors.Wrapf(err, "error getting registry from registry.conf, please specify a registry") + return "", errors.Wrap(err, "getting registry from registry.conf, please specify a registry") } if len(registriesFromFile) == 0 { - return "", errors.Errorf("no registries found in registries.conf, a registry must be provided") + return "", errors.New("no registries found in registries.conf, a registry must be provided") } return registriesFromFile[0], nil } diff --git a/vendor/github.com/containers/common/pkg/chown/chown_unix.go b/vendor/github.com/containers/common/pkg/chown/chown_unix.go index 82342f6af..921927de4 100644 --- a/vendor/github.com/containers/common/pkg/chown/chown_unix.go +++ b/vendor/github.com/containers/common/pkg/chown/chown_unix.go @@ -16,7 +16,7 @@ func ChangeHostPathOwnership(path string, recursive bool, uid, gid int) error { // Validate if host path can be chowned isDangerous, err := DangerousHostPath(path) if err != nil { - return errors.Wrapf(err, "failed to validate if host path is dangerous") + return errors.Wrap(err, "failed to validate if host path is dangerous") } if isDangerous { @@ -42,13 +42,13 @@ func ChangeHostPathOwnership(path string, recursive bool, uid, gid int) error { }) if err != nil { - return errors.Wrapf(err, "failed to chown recursively host path") + return errors.Wrap(err, "failed to chown recursively host path") } } else { // Get host path info f, err := os.Lstat(path) if err != nil { - return errors.Wrapf(err, "failed to get host path information") + return errors.Wrap(err, "failed to get host path information") } // Get current ownership @@ -57,7 +57,7 @@ func ChangeHostPathOwnership(path string, recursive bool, uid, gid int) error { if uid != currentUID || gid != currentGID { if err := os.Lchown(path, uid, gid); err != nil { - return errors.Wrapf(err, "failed to chown host path") + return errors.Wrap(err, "failed to chown host path") } } } diff --git a/vendor/github.com/containers/common/pkg/chown/chown_windows.go b/vendor/github.com/containers/common/pkg/chown/chown_windows.go index ad6039a90..0c4b8e1b5 100644 --- a/vendor/github.com/containers/common/pkg/chown/chown_windows.go +++ b/vendor/github.com/containers/common/pkg/chown/chown_windows.go @@ -7,5 +7,5 @@ import ( // ChangeHostPathOwnership changes the uid and gid ownership of a directory or file within the host. // This is used by the volume U flag to change source volumes ownership func ChangeHostPathOwnership(path string, recursive bool, uid, gid int) error { - return errors.Errorf("windows not supported") + return errors.New("windows not supported") } diff --git a/vendor/github.com/containers/common/pkg/config/config.go b/vendor/github.com/containers/common/pkg/config/config.go index 4a98c7e92..1629bea29 100644 --- a/vendor/github.com/containers/common/pkg/config/config.go +++ b/vendor/github.com/containers/common/pkg/config/config.go @@ -465,16 +465,17 @@ func NewConfig(userConfigPath string) (*Config, error) { // Now, gather the system configs and merge them as needed. configs, err := systemConfigs() if err != nil { - return nil, errors.Wrapf(err, "error finding config on system") + return nil, errors.Wrap(err, "finding config on system") } for _, path := range configs { // Merge changes in later configs with the previous configs. // Each config file that specified fields, will override the // previous fields. if err = readConfigFromFile(path, config); err != nil { - return nil, errors.Wrapf(err, "error reading system config %q", path) + return nil, errors.Wrapf(err, "reading system config %q", path) } - logrus.Debugf("Merged system config %q: %+v", path, config) + logrus.Debugf("Merged system config %q", path) + logrus.Tracef("%+v", config) } // If the caller specified a config path to use, then we read it to @@ -484,9 +485,10 @@ func NewConfig(userConfigPath string) (*Config, error) { // readConfigFromFile reads in container config in the specified // file and then merge changes with the current default. if err = readConfigFromFile(userConfigPath, config); err != nil { - return nil, errors.Wrapf(err, "error reading user config %q", userConfigPath) + return nil, errors.Wrapf(err, "reading user config %q", userConfigPath) } - logrus.Debugf("Merged user config %q: %+v", userConfigPath, config) + logrus.Debugf("Merged user config %q", userConfigPath) + logrus.Tracef("%+v", config) } config.addCAPPrefix() @@ -502,9 +504,9 @@ func NewConfig(userConfigPath string) (*Config, error) { // default config. If the path, only specifies a few fields in the Toml file // the defaults from the config parameter will be used for all other fields. func readConfigFromFile(path string, config *Config) error { - logrus.Debugf("Reading configuration file %q", path) + logrus.Tracef("Reading configuration file %q", path) if _, err := toml.DecodeFile(path, config); err != nil { - return errors.Wrapf(err, "unable to decode configuration %v", path) + return errors.Wrapf(err, "decode configuration %v", path) } return nil } @@ -517,7 +519,7 @@ func systemConfigs() ([]string, error) { path := os.Getenv("CONTAINERS_CONF") if path != "" { if _, err := os.Stat(path); err != nil { - return nil, errors.Wrapf(err, "failed to stat of %s from CONTAINERS_CONF environment variable", path) + return nil, errors.Wrap(err, "CONTAINERS_CONF file") } return append(configs, path), nil } @@ -554,7 +556,7 @@ func (c *Config) CheckCgroupsAndAdjustConfig() { hasSession = err == nil } - if !hasSession { + if !hasSession && unshare.GetRootlessUID() != 0 { logrus.Warningf("The cgroupv2 manager is set to systemd but there is no systemd user session available") logrus.Warningf("For using systemd, you may need to login using an user session") logrus.Warningf("Alternatively, you can enable lingering with: `loginctl enable-linger %d` (possibly as root)", unshare.GetRootlessUID()) @@ -579,7 +581,7 @@ func (c *Config) addCAPPrefix() { func (c *Config) Validate() error { if err := c.Containers.Validate(); err != nil { - return errors.Wrapf(err, " error validating containers config") + return errors.Wrap(err, "validating containers config") } if !c.Containers.EnableLabeling { @@ -587,11 +589,11 @@ func (c *Config) Validate() error { } if err := c.Engine.Validate(); err != nil { - return errors.Wrapf(err, "error validating engine configs") + return errors.Wrap(err, "validating engine configs") } if err := c.Network.Validate(); err != nil { - return errors.Wrapf(err, "error validating network configs") + return errors.Wrap(err, "validating network configs") } return nil @@ -606,7 +608,7 @@ func (c *EngineConfig) findRuntime() string { } } if path, err := exec.LookPath(name); err == nil { - logrus.Warningf("Found default OCIruntime %s path which is missing from [engine.runtimes] in containers.conf", path) + logrus.Debugf("Found default OCI runtime %s path via PATH environment variable", path) return name } } @@ -1001,7 +1003,7 @@ func (c *Config) Write() error { } configFile, err := os.OpenFile(path, os.O_CREATE|os.O_RDWR|os.O_TRUNC, 0600) if err != nil { - return errors.Wrapf(err, "cannot open %s", path) + return err } defer configFile.Close() enc := toml.NewEncoder(configFile) diff --git a/vendor/github.com/containers/common/pkg/config/default.go b/vendor/github.com/containers/common/pkg/config/default.go index 4c55af5c1..72744bb12 100644 --- a/vendor/github.com/containers/common/pkg/config/default.go +++ b/vendor/github.com/containers/common/pkg/config/default.go @@ -331,10 +331,10 @@ func defaultTmpDir() (string, error) { if err := os.Mkdir(libpodRuntimeDir, 0700|os.ModeSticky); err != nil { if !os.IsExist(err) { - return "", errors.Wrapf(err, "cannot mkdir %s", libpodRuntimeDir) + return "", err } else if err := os.Chmod(libpodRuntimeDir, 0700|os.ModeSticky); err != nil { // The directory already exist, just set the sticky bit - return "", errors.Wrapf(err, "could not set sticky bit on %s", libpodRuntimeDir) + return "", errors.Wrap(err, "set sticky bit on") } } return filepath.Join(libpodRuntimeDir, "tmp"), nil diff --git a/vendor/github.com/containers/common/pkg/config/util_supported.go b/vendor/github.com/containers/common/pkg/config/util_supported.go index 326e7973a..417e3a375 100644 --- a/vendor/github.com/containers/common/pkg/config/util_supported.go +++ b/vendor/github.com/containers/common/pkg/config/util_supported.go @@ -40,7 +40,7 @@ func getRuntimeDir() (string, error) { if runtimeDir == "" { tmpDir := filepath.Join("/run", "user", uid) if err := os.MkdirAll(tmpDir, 0700); err != nil { - logrus.Debugf("unable to make temp dir %s", tmpDir) + logrus.Debugf("unable to make temp dir: %v", err) } st, err := os.Stat(tmpDir) if err == nil && int(st.Sys().(*syscall.Stat_t).Uid) == os.Geteuid() && st.Mode().Perm() == 0700 { @@ -50,7 +50,7 @@ func getRuntimeDir() (string, error) { if runtimeDir == "" { tmpDir := filepath.Join(os.TempDir(), fmt.Sprintf("run-%s", uid)) if err := os.MkdirAll(tmpDir, 0700); err != nil { - logrus.Debugf("unable to make temp dir %s", tmpDir) + logrus.Debugf("unable to make temp dir %v", err) } st, err := os.Stat(tmpDir) if err == nil && int(st.Sys().(*syscall.Stat_t).Uid) == os.Geteuid() && st.Mode().Perm() == 0700 { @@ -65,7 +65,7 @@ func getRuntimeDir() (string, error) { } resolvedHome, err := filepath.EvalSymlinks(home) if err != nil { - rootlessRuntimeDirError = errors.Wrapf(err, "cannot resolve %s", home) + rootlessRuntimeDirError = errors.Wrap(err, "cannot resolve home") return } runtimeDir = filepath.Join(resolvedHome, "rundir") diff --git a/vendor/github.com/containers/common/pkg/parse/parse.go b/vendor/github.com/containers/common/pkg/parse/parse.go index 882953309..1a25957d6 100644 --- a/vendor/github.com/containers/common/pkg/parse/parse.go +++ b/vendor/github.com/containers/common/pkg/parse/parse.go @@ -138,11 +138,11 @@ func isValidDeviceMode(mode string) bool { // ValidateVolumeHostDir validates a volume mount's source directory func ValidateVolumeHostDir(hostDir string) error { if hostDir == "" { - return errors.Errorf("host directory cannot be empty") + return errors.New("host directory cannot be empty") } if filepath.IsAbs(hostDir) { if _, err := os.Stat(hostDir); err != nil { - return errors.Wrapf(err, "error checking path %q", hostDir) + return err } } // If hostDir is not an absolute path, that means the user wants to create a @@ -153,7 +153,7 @@ func ValidateVolumeHostDir(hostDir string) error { // ValidateVolumeCtrDir validates a volume mount's destination directory. func ValidateVolumeCtrDir(ctrDir string) error { if ctrDir == "" { - return errors.Errorf("container directory cannot be empty") + return errors.New("container directory cannot be empty") } if !filepath.IsAbs(ctrDir) { return errors.Errorf("invalid container path %q, must be an absolute path", ctrDir) diff --git a/vendor/github.com/containers/common/pkg/parse/parse_unix.go b/vendor/github.com/containers/common/pkg/parse/parse_unix.go index c07471c93..ce4446a1b 100644 --- a/vendor/github.com/containers/common/pkg/parse/parse_unix.go +++ b/vendor/github.com/containers/common/pkg/parse/parse_unix.go @@ -22,7 +22,7 @@ func DeviceFromPath(device string) ([]devices.Device, error) { } srcInfo, err := os.Stat(src) if err != nil { - return nil, errors.Wrapf(err, "error getting info of source device %s", src) + return nil, err } if !srcInfo.IsDir() { diff --git a/vendor/github.com/containers/common/pkg/seccomp/default_linux.go b/vendor/github.com/containers/common/pkg/seccomp/default_linux.go index 24077778e..f86f3e2ba 100644 --- a/vendor/github.com/containers/common/pkg/seccomp/default_linux.go +++ b/vendor/github.com/containers/common/pkg/seccomp/default_linux.go @@ -299,6 +299,7 @@ func DefaultProfile() *Seccomp { "sendmmsg", "sendmsg", "sendto", + "setns", "set_robust_list", "set_thread_area", "set_tid_address", diff --git a/vendor/github.com/containers/common/pkg/seccomp/seccomp.json b/vendor/github.com/containers/common/pkg/seccomp/seccomp.json index 48420905c..8d799fd02 100644 --- a/vendor/github.com/containers/common/pkg/seccomp/seccomp.json +++ b/vendor/github.com/containers/common/pkg/seccomp/seccomp.json @@ -303,6 +303,7 @@ "sendmmsg", "sendmsg", "sendto", + "setns", "set_robust_list", "set_thread_area", "set_tid_address", diff --git a/vendor/github.com/containers/common/pkg/subscriptions/subscriptions.go b/vendor/github.com/containers/common/pkg/subscriptions/subscriptions.go index 6aa66b0c8..4b7253b31 100644 --- a/vendor/github.com/containers/common/pkg/subscriptions/subscriptions.go +++ b/vendor/github.com/containers/common/pkg/subscriptions/subscriptions.go @@ -225,7 +225,7 @@ func addSubscriptionsFromMountsFile(filePath, mountLabel, containerWorkingDir st logrus.Warnf("Path %q from %q doesn't exist, skipping", hostDirOrFile, filePath) continue } - return nil, errors.Wrapf(err, "failed to stat %q", hostDirOrFile) + return nil, err } ctrDirOrFileOnHost := filepath.Join(containerWorkingDir, ctrDirOrFile) @@ -246,11 +246,11 @@ func addSubscriptionsFromMountsFile(filePath, mountLabel, containerWorkingDir st switch mode := fileInfo.Mode(); { case mode.IsDir(): if err = os.MkdirAll(ctrDirOrFileOnHost, mode.Perm()); err != nil { - return nil, errors.Wrapf(err, "making container directory %q failed", ctrDirOrFileOnHost) + return nil, errors.Wrap(err, "making container directory") } data, err := getHostSubscriptionData(hostDirOrFile, mode.Perm()) if err != nil { - return nil, errors.Wrapf(err, "getting host subscription data failed") + return nil, errors.Wrap(err, "getting host subscription data") } for _, s := range data { if err := s.saveTo(ctrDirOrFileOnHost); err != nil { @@ -260,7 +260,7 @@ func addSubscriptionsFromMountsFile(filePath, mountLabel, containerWorkingDir st case mode.IsRegular(): data, err := readFileOrDir("", hostDirOrFile, mode.Perm()) if err != nil { - return nil, errors.Wrapf(err, "error reading file %q", hostDirOrFile) + return nil, err } for _, s := range data { @@ -268,7 +268,7 @@ func addSubscriptionsFromMountsFile(filePath, mountLabel, containerWorkingDir st return nil, err } if err := ioutil.WriteFile(ctrDirOrFileOnHost, s.data, s.mode); err != nil { - return nil, errors.Wrapf(err, "error saving data to container filesystem on host %q", ctrDirOrFileOnHost) + return nil, errors.Wrap(err, "saving data to container filesystem") } } default: @@ -285,7 +285,7 @@ func addSubscriptionsFromMountsFile(filePath, mountLabel, containerWorkingDir st } } } else if err != nil { - return nil, errors.Wrapf(err, "error getting status of %q", ctrDirOrFileOnHost) + return nil, err } m := rspec.Mount{ @@ -309,10 +309,10 @@ func addFIPSModeSubscription(mounts *[]rspec.Mount, containerWorkingDir, mountPo ctrDirOnHost := filepath.Join(containerWorkingDir, subscriptionsDir) if _, err := os.Stat(ctrDirOnHost); os.IsNotExist(err) { if err = idtools.MkdirAllAs(ctrDirOnHost, 0755, uid, gid); err != nil { //nolint - return errors.Wrapf(err, "making container directory %q on host failed", ctrDirOnHost) + return err } if err = label.Relabel(ctrDirOnHost, mountLabel, false); err != nil { - return errors.Wrapf(err, "error applying correct labels on %q", ctrDirOnHost) + return errors.Wrapf(err, "applying correct labels on %q", ctrDirOnHost) } } fipsFile := filepath.Join(ctrDirOnHost, "system-fips") @@ -320,7 +320,7 @@ func addFIPSModeSubscription(mounts *[]rspec.Mount, containerWorkingDir, mountPo if _, err := os.Stat(fipsFile); os.IsNotExist(err) { file, err := os.Create(fipsFile) if err != nil { - return errors.Wrapf(err, "error creating system-fips file in container for FIPS mode") + return errors.Wrap(err, "creating system-fips file in container for FIPS mode") } defer file.Close() } @@ -342,7 +342,7 @@ func addFIPSModeSubscription(mounts *[]rspec.Mount, containerWorkingDir, mountPo if os.IsNotExist(err) { return nil } - return errors.Wrapf(err, "failed to stat FIPS Backend directory %q", ctrDirOnHost) + return errors.Wrap(err, "FIPS Backend directory") } if !mountExists(*mounts, destDir) { diff --git a/vendor/github.com/containers/common/version/version.go b/vendor/github.com/containers/common/version/version.go index 67f454c9a..d9e7ffec7 100644 --- a/vendor/github.com/containers/common/version/version.go +++ b/vendor/github.com/containers/common/version/version.go @@ -1,4 +1,4 @@ package version // Version is the version of the build. -const Version = "0.36.0" +const Version = "0.37.0" diff --git a/vendor/github.com/containers/image/v5/copy/copy.go b/vendor/github.com/containers/image/v5/copy/copy.go index 165a8be4b..fb704283b 100644 --- a/vendor/github.com/containers/image/v5/copy/copy.go +++ b/vendor/github.com/containers/image/v5/copy/copy.go @@ -43,6 +43,10 @@ type digestingReader struct { validationSucceeded bool } +// FIXME: disable early layer commits temporarily until a solid solution to +// address #1205 has been found. +const enableEarlyCommit = false + var ( // ErrDecryptParamsMissing is returned if there is missing decryption parameters ErrDecryptParamsMissing = errors.New("Necessary DecryptParameters not present") @@ -1185,7 +1189,7 @@ func (ic *imageCopier) copyLayer(ctx context.Context, srcInfo types.BlobInfo, to // layers which requires passing the index of the layer. // Hence, we need to special case and cast. dest, ok := ic.c.dest.(internalTypes.ImageDestinationWithOptions) - if ok { + if ok && enableEarlyCommit { options := internalTypes.TryReusingBlobOptions{ Cache: ic.c.blobInfoCache, CanSubstitute: ic.canSubstituteBlobs, @@ -1546,7 +1550,7 @@ func (c *copier) copyBlobFromStream(ctx context.Context, srcStream io.Reader, sr // which requires passing the index of the layer. Hence, we need to // special case and cast. dest, ok := c.dest.(internalTypes.ImageDestinationWithOptions) - if ok { + if ok && enableEarlyCommit { options := internalTypes.PutBlobOptions{ Cache: c.blobInfoCache, IsConfig: isConfig, diff --git a/vendor/github.com/containers/image/v5/oci/layout/oci_src.go b/vendor/github.com/containers/image/v5/oci/layout/oci_src.go index 6801c8432..9925aeda7 100644 --- a/vendor/github.com/containers/image/v5/oci/layout/oci_src.go +++ b/vendor/github.com/containers/image/v5/oci/layout/oci_src.go @@ -15,7 +15,6 @@ import ( "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) type ociImageSource struct { @@ -95,7 +94,6 @@ func (s *ociImageSource) GetManifest(ctx context.Context, instanceDigest *digest m, err := ioutil.ReadFile(manifestPath) if err != nil { - logrus.Errorf("Error HERE") return nil, "", err } if mimeType == "" { diff --git a/vendor/github.com/containers/image/v5/storage/storage_image.go b/vendor/github.com/containers/image/v5/storage/storage_image.go index ae020dd66..3a2c18c89 100644 --- a/vendor/github.com/containers/image/v5/storage/storage_image.go +++ b/vendor/github.com/containers/image/v5/storage/storage_image.go @@ -763,7 +763,7 @@ func (s *storageImageDestination) commitLayer(ctx context.Context, blob manifest } // Carry over the previous ID for empty non-base layers. - if blob.EmptyLayer && index > 0 { + if blob.EmptyLayer { s.indexToStorageID[index] = &lastLayer return nil } diff --git a/vendor/github.com/containers/image/v5/version/version.go b/vendor/github.com/containers/image/v5/version/version.go index 3e9f09aab..23b2e3571 100644 --- a/vendor/github.com/containers/image/v5/version/version.go +++ b/vendor/github.com/containers/image/v5/version/version.go @@ -8,7 +8,7 @@ const ( // VersionMinor is for functionality in a backwards-compatible manner VersionMinor = 11 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 0 + VersionPatch = 1 // VersionDev indicates development branch. Releases will be empty string. VersionDev = "" diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index 5e57fb895..034552a83 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.29.0 +1.30.0 diff --git a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go index f0d8b548b..3903b1ddd 100644 --- a/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go +++ b/vendor/github.com/containers/storage/drivers/btrfs/btrfs.go @@ -88,7 +88,7 @@ func Init(home string, options graphdriver.Options) (graphdriver.Driver, error) } if userDiskQuota { - if err := driver.subvolEnableQuota(); err != nil { + if err := driver.enableQuota(); err != nil { return nil, err } } @@ -159,10 +159,6 @@ func (d *Driver) Metadata(id string) (map[string]string, error) { // Cleanup unmounts the home directory. func (d *Driver) Cleanup() error { - if err := d.subvolDisableQuota(); err != nil { - return err - } - return mount.Unmount(d.home) } @@ -320,7 +316,7 @@ func (d *Driver) updateQuotaStatus() { d.once.Do(func() { if !d.quotaEnabled { // In case quotaEnabled is not set, check qgroup and update quotaEnabled as needed - if err := subvolQgroupStatus(d.home); err != nil { + if err := qgroupStatus(d.home); err != nil { // quota is still not enabled return } @@ -329,7 +325,7 @@ func (d *Driver) updateQuotaStatus() { }) } -func (d *Driver) subvolEnableQuota() error { +func (d *Driver) enableQuota() error { d.updateQuotaStatus() if d.quotaEnabled { @@ -355,32 +351,6 @@ func (d *Driver) subvolEnableQuota() error { return nil } -func (d *Driver) subvolDisableQuota() error { - d.updateQuotaStatus() - - if !d.quotaEnabled { - return nil - } - - dir, err := openDir(d.home) - if err != nil { - return err - } - defer closeDir(dir) - - var args C.struct_btrfs_ioctl_quota_ctl_args - args.cmd = C.BTRFS_QUOTA_CTL_DISABLE - _, _, errno := unix.Syscall(unix.SYS_IOCTL, getDirFd(dir), C.BTRFS_IOC_QUOTA_CTL, - uintptr(unsafe.Pointer(&args))) - if errno != 0 { - return fmt.Errorf("Failed to disable btrfs quota for %s: %v", dir, errno.Error()) - } - - d.quotaEnabled = false - - return nil -} - func (d *Driver) subvolRescanQuota() error { d.updateQuotaStatus() @@ -423,11 +393,11 @@ func subvolLimitQgroup(path string, size uint64) error { return nil } -// subvolQgroupStatus performs a BTRFS_IOC_TREE_SEARCH on the root path +// qgroupStatus performs a BTRFS_IOC_TREE_SEARCH on the root path // with search key of BTRFS_QGROUP_STATUS_KEY. // In case qgroup is enabled, the returned key type will match BTRFS_QGROUP_STATUS_KEY. // For more details please see https://github.com/kdave/btrfs-progs/blob/v4.9/qgroup.c#L1035 -func subvolQgroupStatus(path string) error { +func qgroupStatus(path string) error { dir, err := openDir(path) if err != nil { return err @@ -603,7 +573,7 @@ func (d *Driver) setStorageSize(dir string, driver *Driver) error { return fmt.Errorf("btrfs: storage size cannot be less than %s", units.HumanSize(float64(d.options.minSpace))) } - if err := d.subvolEnableQuota(); err != nil { + if err := d.enableQuota(); err != nil { return err } @@ -674,7 +644,7 @@ func (d *Driver) Get(id string, options graphdriver.MountOpts) (string, error) { if quota, err := ioutil.ReadFile(d.quotasDirID(id)); err == nil { if size, err := strconv.ParseUint(string(quota), 10, 64); err == nil && size >= d.options.minSpace { - if err := d.subvolEnableQuota(); err != nil { + if err := d.enableQuota(); err != nil { return "", err } if err := subvolLimitQgroup(dir, size); err != nil { diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index 3a455653a..6a34d06c6 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -9,7 +9,7 @@ require ( github.com/docker/go-units v0.4.0 github.com/google/go-intervals v0.0.2 github.com/hashicorp/go-multierror v1.1.1 - github.com/klauspost/compress v1.11.13 + github.com/klauspost/compress v1.12.1 github.com/klauspost/pgzip v1.2.5 github.com/mattn/go-shellwords v1.0.11 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible diff --git a/vendor/github.com/containers/storage/go.sum b/vendor/github.com/containers/storage/go.sum index 1de8a9825..28edd4a7e 100644 --- a/vendor/github.com/containers/storage/go.sum +++ b/vendor/github.com/containers/storage/go.sum @@ -267,6 +267,8 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -337,8 +339,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= -github.com/klauspost/compress v1.11.13 h1:eSvu8Tmq6j2psUJqJrLcWH6K3w5Dwc+qipbaA6eVEN4= -github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= +github.com/klauspost/compress v1.12.1 h1:/+xsCsk06wE38cyiqOR/o7U2fSftcH72xD+BQXmja/g= +github.com/klauspost/compress v1.12.1/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= diff --git a/vendor/github.com/containers/storage/pkg/unshare/unshare.go b/vendor/github.com/containers/storage/pkg/unshare/unshare.go index a9210b0bf..53cfeb0ec 100644 --- a/vendor/github.com/containers/storage/pkg/unshare/unshare.go +++ b/vendor/github.com/containers/storage/pkg/unshare/unshare.go @@ -7,12 +7,17 @@ import ( "sync" "github.com/pkg/errors" + "github.com/syndtr/gocapability/capability" ) var ( homeDirOnce sync.Once homeDirErr error homeDir string + + hasCapSysAdminOnce sync.Once + hasCapSysAdminRet bool + hasCapSysAdminErr error ) // HomeDir returns the home directory for the current user. @@ -32,3 +37,20 @@ func HomeDir() (string, error) { }) return homeDir, homeDirErr } + +// HasCapSysAdmin returns whether the current process has CAP_SYS_ADMIN. +func HasCapSysAdmin() (bool, error) { + hasCapSysAdminOnce.Do(func() { + currentCaps, err := capability.NewPid2(0) + if err != nil { + hasCapSysAdminErr = err + return + } + if err = currentCaps.Load(); err != nil { + hasCapSysAdminErr = err + return + } + hasCapSysAdminRet = currentCaps.Get(capability.EFFECTIVE, capability.CAP_SYS_ADMIN) + }) + return hasCapSysAdminRet, hasCapSysAdminErr +} diff --git a/vendor/github.com/klauspost/compress/snappy/.gitignore b/vendor/github.com/golang/snappy/.gitignore index 042091d9b..042091d9b 100644 --- a/vendor/github.com/klauspost/compress/snappy/.gitignore +++ b/vendor/github.com/golang/snappy/.gitignore diff --git a/vendor/github.com/klauspost/compress/snappy/AUTHORS b/vendor/github.com/golang/snappy/AUTHORS index bcfa19520..203e84eba 100644 --- a/vendor/github.com/klauspost/compress/snappy/AUTHORS +++ b/vendor/github.com/golang/snappy/AUTHORS @@ -8,8 +8,10 @@ # Please keep the list sorted. +Amazon.com, Inc Damian Gryski <dgryski@gmail.com> Google Inc. Jan Mercl <0xjnml@gmail.com> +Klaus Post <klauspost@gmail.com> Rodolfo Carvalho <rhcarvalho@gmail.com> Sebastien Binet <seb.binet@gmail.com> diff --git a/vendor/github.com/klauspost/compress/snappy/CONTRIBUTORS b/vendor/github.com/golang/snappy/CONTRIBUTORS index 931ae3160..d9914732b 100644 --- a/vendor/github.com/klauspost/compress/snappy/CONTRIBUTORS +++ b/vendor/github.com/golang/snappy/CONTRIBUTORS @@ -28,7 +28,9 @@ Damian Gryski <dgryski@gmail.com> Jan Mercl <0xjnml@gmail.com> +Jonathan Swinney <jswinney@amazon.com> Kai Backman <kaib@golang.org> +Klaus Post <klauspost@gmail.com> Marc-Antoine Ruel <maruel@chromium.org> Nigel Tao <nigeltao@golang.org> Rob Pike <r@golang.org> diff --git a/vendor/github.com/klauspost/compress/snappy/LICENSE b/vendor/github.com/golang/snappy/LICENSE index 6050c10f4..6050c10f4 100644 --- a/vendor/github.com/klauspost/compress/snappy/LICENSE +++ b/vendor/github.com/golang/snappy/LICENSE diff --git a/vendor/github.com/klauspost/compress/snappy/README b/vendor/github.com/golang/snappy/README index cea12879a..cea12879a 100644 --- a/vendor/github.com/klauspost/compress/snappy/README +++ b/vendor/github.com/golang/snappy/README diff --git a/vendor/github.com/klauspost/compress/snappy/decode.go b/vendor/github.com/golang/snappy/decode.go index 72efb0353..f1e04b172 100644 --- a/vendor/github.com/klauspost/compress/snappy/decode.go +++ b/vendor/github.com/golang/snappy/decode.go @@ -52,6 +52,8 @@ const ( // Otherwise, a newly allocated slice will be returned. // // The dst and src must not overlap. It is valid to pass a nil dst. +// +// Decode handles the Snappy block format, not the Snappy stream format. func Decode(dst, src []byte) ([]byte, error) { dLen, s, err := decodedLen(src) if err != nil { @@ -83,6 +85,8 @@ func NewReader(r io.Reader) *Reader { } // Reader is an io.Reader that can read Snappy-compressed bytes. +// +// Reader handles the Snappy stream format, not the Snappy block format. type Reader struct { r io.Reader err error diff --git a/vendor/github.com/klauspost/compress/snappy/decode_amd64.s b/vendor/github.com/golang/snappy/decode_amd64.s index 1c66e3723..e6179f65e 100644 --- a/vendor/github.com/klauspost/compress/snappy/decode_amd64.s +++ b/vendor/github.com/golang/snappy/decode_amd64.s @@ -184,7 +184,9 @@ tagLit60Plus: // checks. In the asm version, we code it once instead of once per switch case. ADDQ CX, SI SUBQ $58, SI - CMPQ SI, R13 + MOVQ SI, BX + SUBQ R11, BX + CMPQ BX, R12 JA errCorrupt // case x == 60: @@ -230,7 +232,9 @@ tagCopy4: ADDQ $5, SI // if uint(s) > uint(len(src)) { etc } - CMPQ SI, R13 + MOVQ SI, BX + SUBQ R11, BX + CMPQ BX, R12 JA errCorrupt // length = 1 + int(src[s-5])>>2 @@ -247,7 +251,9 @@ tagCopy2: ADDQ $3, SI // if uint(s) > uint(len(src)) { etc } - CMPQ SI, R13 + MOVQ SI, BX + SUBQ R11, BX + CMPQ BX, R12 JA errCorrupt // length = 1 + int(src[s-3])>>2 @@ -271,7 +277,9 @@ tagCopy: ADDQ $2, SI // if uint(s) > uint(len(src)) { etc } - CMPQ SI, R13 + MOVQ SI, BX + SUBQ R11, BX + CMPQ BX, R12 JA errCorrupt // offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) diff --git a/vendor/github.com/golang/snappy/decode_arm64.s b/vendor/github.com/golang/snappy/decode_arm64.s new file mode 100644 index 000000000..7a3ead17e --- /dev/null +++ b/vendor/github.com/golang/snappy/decode_arm64.s @@ -0,0 +1,494 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !appengine +// +build gc +// +build !noasm + +#include "textflag.h" + +// The asm code generally follows the pure Go code in decode_other.go, except +// where marked with a "!!!". + +// func decode(dst, src []byte) int +// +// All local variables fit into registers. The non-zero stack size is only to +// spill registers and push args when issuing a CALL. The register allocation: +// - R2 scratch +// - R3 scratch +// - R4 length or x +// - R5 offset +// - R6 &src[s] +// - R7 &dst[d] +// + R8 dst_base +// + R9 dst_len +// + R10 dst_base + dst_len +// + R11 src_base +// + R12 src_len +// + R13 src_base + src_len +// - R14 used by doCopy +// - R15 used by doCopy +// +// The registers R8-R13 (marked with a "+") are set at the start of the +// function, and after a CALL returns, and are not otherwise modified. +// +// The d variable is implicitly R7 - R8, and len(dst)-d is R10 - R7. +// The s variable is implicitly R6 - R11, and len(src)-s is R13 - R6. +TEXT ·decode(SB), NOSPLIT, $56-56 + // Initialize R6, R7 and R8-R13. + MOVD dst_base+0(FP), R8 + MOVD dst_len+8(FP), R9 + MOVD R8, R7 + MOVD R8, R10 + ADD R9, R10, R10 + MOVD src_base+24(FP), R11 + MOVD src_len+32(FP), R12 + MOVD R11, R6 + MOVD R11, R13 + ADD R12, R13, R13 + +loop: + // for s < len(src) + CMP R13, R6 + BEQ end + + // R4 = uint32(src[s]) + // + // switch src[s] & 0x03 + MOVBU (R6), R4 + MOVW R4, R3 + ANDW $3, R3 + MOVW $1, R1 + CMPW R1, R3 + BGE tagCopy + + // ---------------------------------------- + // The code below handles literal tags. + + // case tagLiteral: + // x := uint32(src[s] >> 2) + // switch + MOVW $60, R1 + LSRW $2, R4, R4 + CMPW R4, R1 + BLS tagLit60Plus + + // case x < 60: + // s++ + ADD $1, R6, R6 + +doLit: + // This is the end of the inner "switch", when we have a literal tag. + // + // We assume that R4 == x and x fits in a uint32, where x is the variable + // used in the pure Go decode_other.go code. + + // length = int(x) + 1 + // + // Unlike the pure Go code, we don't need to check if length <= 0 because + // R4 can hold 64 bits, so the increment cannot overflow. + ADD $1, R4, R4 + + // Prepare to check if copying length bytes will run past the end of dst or + // src. + // + // R2 = len(dst) - d + // R3 = len(src) - s + MOVD R10, R2 + SUB R7, R2, R2 + MOVD R13, R3 + SUB R6, R3, R3 + + // !!! Try a faster technique for short (16 or fewer bytes) copies. + // + // if length > 16 || len(dst)-d < 16 || len(src)-s < 16 { + // goto callMemmove // Fall back on calling runtime·memmove. + // } + // + // The C++ snappy code calls this TryFastAppend. It also checks len(src)-s + // against 21 instead of 16, because it cannot assume that all of its input + // is contiguous in memory and so it needs to leave enough source bytes to + // read the next tag without refilling buffers, but Go's Decode assumes + // contiguousness (the src argument is a []byte). + CMP $16, R4 + BGT callMemmove + CMP $16, R2 + BLT callMemmove + CMP $16, R3 + BLT callMemmove + + // !!! Implement the copy from src to dst as a 16-byte load and store. + // (Decode's documentation says that dst and src must not overlap.) + // + // This always copies 16 bytes, instead of only length bytes, but that's + // OK. If the input is a valid Snappy encoding then subsequent iterations + // will fix up the overrun. Otherwise, Decode returns a nil []byte (and a + // non-nil error), so the overrun will be ignored. + // + // Note that on arm64, it is legal and cheap to issue unaligned 8-byte or + // 16-byte loads and stores. This technique probably wouldn't be as + // effective on architectures that are fussier about alignment. + LDP 0(R6), (R14, R15) + STP (R14, R15), 0(R7) + + // d += length + // s += length + ADD R4, R7, R7 + ADD R4, R6, R6 + B loop + +callMemmove: + // if length > len(dst)-d || length > len(src)-s { etc } + CMP R2, R4 + BGT errCorrupt + CMP R3, R4 + BGT errCorrupt + + // copy(dst[d:], src[s:s+length]) + // + // This means calling runtime·memmove(&dst[d], &src[s], length), so we push + // R7, R6 and R4 as arguments. Coincidentally, we also need to spill those + // three registers to the stack, to save local variables across the CALL. + MOVD R7, 8(RSP) + MOVD R6, 16(RSP) + MOVD R4, 24(RSP) + MOVD R7, 32(RSP) + MOVD R6, 40(RSP) + MOVD R4, 48(RSP) + CALL runtime·memmove(SB) + + // Restore local variables: unspill registers from the stack and + // re-calculate R8-R13. + MOVD 32(RSP), R7 + MOVD 40(RSP), R6 + MOVD 48(RSP), R4 + MOVD dst_base+0(FP), R8 + MOVD dst_len+8(FP), R9 + MOVD R8, R10 + ADD R9, R10, R10 + MOVD src_base+24(FP), R11 + MOVD src_len+32(FP), R12 + MOVD R11, R13 + ADD R12, R13, R13 + + // d += length + // s += length + ADD R4, R7, R7 + ADD R4, R6, R6 + B loop + +tagLit60Plus: + // !!! This fragment does the + // + // s += x - 58; if uint(s) > uint(len(src)) { etc } + // + // checks. In the asm version, we code it once instead of once per switch case. + ADD R4, R6, R6 + SUB $58, R6, R6 + MOVD R6, R3 + SUB R11, R3, R3 + CMP R12, R3 + BGT errCorrupt + + // case x == 60: + MOVW $61, R1 + CMPW R1, R4 + BEQ tagLit61 + BGT tagLit62Plus + + // x = uint32(src[s-1]) + MOVBU -1(R6), R4 + B doLit + +tagLit61: + // case x == 61: + // x = uint32(src[s-2]) | uint32(src[s-1])<<8 + MOVHU -2(R6), R4 + B doLit + +tagLit62Plus: + CMPW $62, R4 + BHI tagLit63 + + // case x == 62: + // x = uint32(src[s-3]) | uint32(src[s-2])<<8 | uint32(src[s-1])<<16 + MOVHU -3(R6), R4 + MOVBU -1(R6), R3 + ORR R3<<16, R4 + B doLit + +tagLit63: + // case x == 63: + // x = uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24 + MOVWU -4(R6), R4 + B doLit + + // The code above handles literal tags. + // ---------------------------------------- + // The code below handles copy tags. + +tagCopy4: + // case tagCopy4: + // s += 5 + ADD $5, R6, R6 + + // if uint(s) > uint(len(src)) { etc } + MOVD R6, R3 + SUB R11, R3, R3 + CMP R12, R3 + BGT errCorrupt + + // length = 1 + int(src[s-5])>>2 + MOVD $1, R1 + ADD R4>>2, R1, R4 + + // offset = int(uint32(src[s-4]) | uint32(src[s-3])<<8 | uint32(src[s-2])<<16 | uint32(src[s-1])<<24) + MOVWU -4(R6), R5 + B doCopy + +tagCopy2: + // case tagCopy2: + // s += 3 + ADD $3, R6, R6 + + // if uint(s) > uint(len(src)) { etc } + MOVD R6, R3 + SUB R11, R3, R3 + CMP R12, R3 + BGT errCorrupt + + // length = 1 + int(src[s-3])>>2 + MOVD $1, R1 + ADD R4>>2, R1, R4 + + // offset = int(uint32(src[s-2]) | uint32(src[s-1])<<8) + MOVHU -2(R6), R5 + B doCopy + +tagCopy: + // We have a copy tag. We assume that: + // - R3 == src[s] & 0x03 + // - R4 == src[s] + CMP $2, R3 + BEQ tagCopy2 + BGT tagCopy4 + + // case tagCopy1: + // s += 2 + ADD $2, R6, R6 + + // if uint(s) > uint(len(src)) { etc } + MOVD R6, R3 + SUB R11, R3, R3 + CMP R12, R3 + BGT errCorrupt + + // offset = int(uint32(src[s-2])&0xe0<<3 | uint32(src[s-1])) + MOVD R4, R5 + AND $0xe0, R5 + MOVBU -1(R6), R3 + ORR R5<<3, R3, R5 + + // length = 4 + int(src[s-2])>>2&0x7 + MOVD $7, R1 + AND R4>>2, R1, R4 + ADD $4, R4, R4 + +doCopy: + // This is the end of the outer "switch", when we have a copy tag. + // + // We assume that: + // - R4 == length && R4 > 0 + // - R5 == offset + + // if offset <= 0 { etc } + MOVD $0, R1 + CMP R1, R5 + BLE errCorrupt + + // if d < offset { etc } + MOVD R7, R3 + SUB R8, R3, R3 + CMP R5, R3 + BLT errCorrupt + + // if length > len(dst)-d { etc } + MOVD R10, R3 + SUB R7, R3, R3 + CMP R3, R4 + BGT errCorrupt + + // forwardCopy(dst[d:d+length], dst[d-offset:]); d += length + // + // Set: + // - R14 = len(dst)-d + // - R15 = &dst[d-offset] + MOVD R10, R14 + SUB R7, R14, R14 + MOVD R7, R15 + SUB R5, R15, R15 + + // !!! Try a faster technique for short (16 or fewer bytes) forward copies. + // + // First, try using two 8-byte load/stores, similar to the doLit technique + // above. Even if dst[d:d+length] and dst[d-offset:] can overlap, this is + // still OK if offset >= 8. Note that this has to be two 8-byte load/stores + // and not one 16-byte load/store, and the first store has to be before the + // second load, due to the overlap if offset is in the range [8, 16). + // + // if length > 16 || offset < 8 || len(dst)-d < 16 { + // goto slowForwardCopy + // } + // copy 16 bytes + // d += length + CMP $16, R4 + BGT slowForwardCopy + CMP $8, R5 + BLT slowForwardCopy + CMP $16, R14 + BLT slowForwardCopy + MOVD 0(R15), R2 + MOVD R2, 0(R7) + MOVD 8(R15), R3 + MOVD R3, 8(R7) + ADD R4, R7, R7 + B loop + +slowForwardCopy: + // !!! If the forward copy is longer than 16 bytes, or if offset < 8, we + // can still try 8-byte load stores, provided we can overrun up to 10 extra + // bytes. As above, the overrun will be fixed up by subsequent iterations + // of the outermost loop. + // + // The C++ snappy code calls this technique IncrementalCopyFastPath. Its + // commentary says: + // + // ---- + // + // The main part of this loop is a simple copy of eight bytes at a time + // until we've copied (at least) the requested amount of bytes. However, + // if d and d-offset are less than eight bytes apart (indicating a + // repeating pattern of length < 8), we first need to expand the pattern in + // order to get the correct results. For instance, if the buffer looks like + // this, with the eight-byte <d-offset> and <d> patterns marked as + // intervals: + // + // abxxxxxxxxxxxx + // [------] d-offset + // [------] d + // + // a single eight-byte copy from <d-offset> to <d> will repeat the pattern + // once, after which we can move <d> two bytes without moving <d-offset>: + // + // ababxxxxxxxxxx + // [------] d-offset + // [------] d + // + // and repeat the exercise until the two no longer overlap. + // + // This allows us to do very well in the special case of one single byte + // repeated many times, without taking a big hit for more general cases. + // + // The worst case of extra writing past the end of the match occurs when + // offset == 1 and length == 1; the last copy will read from byte positions + // [0..7] and write to [4..11], whereas it was only supposed to write to + // position 1. Thus, ten excess bytes. + // + // ---- + // + // That "10 byte overrun" worst case is confirmed by Go's + // TestSlowForwardCopyOverrun, which also tests the fixUpSlowForwardCopy + // and finishSlowForwardCopy algorithm. + // + // if length > len(dst)-d-10 { + // goto verySlowForwardCopy + // } + SUB $10, R14, R14 + CMP R14, R4 + BGT verySlowForwardCopy + +makeOffsetAtLeast8: + // !!! As above, expand the pattern so that offset >= 8 and we can use + // 8-byte load/stores. + // + // for offset < 8 { + // copy 8 bytes from dst[d-offset:] to dst[d:] + // length -= offset + // d += offset + // offset += offset + // // The two previous lines together means that d-offset, and therefore + // // R15, is unchanged. + // } + CMP $8, R5 + BGE fixUpSlowForwardCopy + MOVD (R15), R3 + MOVD R3, (R7) + SUB R5, R4, R4 + ADD R5, R7, R7 + ADD R5, R5, R5 + B makeOffsetAtLeast8 + +fixUpSlowForwardCopy: + // !!! Add length (which might be negative now) to d (implied by R7 being + // &dst[d]) so that d ends up at the right place when we jump back to the + // top of the loop. Before we do that, though, we save R7 to R2 so that, if + // length is positive, copying the remaining length bytes will write to the + // right place. + MOVD R7, R2 + ADD R4, R7, R7 + +finishSlowForwardCopy: + // !!! Repeat 8-byte load/stores until length <= 0. Ending with a negative + // length means that we overrun, but as above, that will be fixed up by + // subsequent iterations of the outermost loop. + MOVD $0, R1 + CMP R1, R4 + BLE loop + MOVD (R15), R3 + MOVD R3, (R2) + ADD $8, R15, R15 + ADD $8, R2, R2 + SUB $8, R4, R4 + B finishSlowForwardCopy + +verySlowForwardCopy: + // verySlowForwardCopy is a simple implementation of forward copy. In C + // parlance, this is a do/while loop instead of a while loop, since we know + // that length > 0. In Go syntax: + // + // for { + // dst[d] = dst[d - offset] + // d++ + // length-- + // if length == 0 { + // break + // } + // } + MOVB (R15), R3 + MOVB R3, (R7) + ADD $1, R15, R15 + ADD $1, R7, R7 + SUB $1, R4, R4 + CBNZ R4, verySlowForwardCopy + B loop + + // The code above handles copy tags. + // ---------------------------------------- + +end: + // This is the end of the "for s < len(src)". + // + // if d != len(dst) { etc } + CMP R10, R7 + BNE errCorrupt + + // return 0 + MOVD $0, ret+48(FP) + RET + +errCorrupt: + // return decodeErrCodeCorrupt + MOVD $1, R2 + MOVD R2, ret+48(FP) + RET diff --git a/vendor/github.com/klauspost/compress/snappy/decode_amd64.go b/vendor/github.com/golang/snappy/decode_asm.go index fcd192b84..7082b3491 100644 --- a/vendor/github.com/klauspost/compress/snappy/decode_amd64.go +++ b/vendor/github.com/golang/snappy/decode_asm.go @@ -5,6 +5,7 @@ // +build !appengine // +build gc // +build !noasm +// +build amd64 arm64 package snappy diff --git a/vendor/github.com/klauspost/compress/snappy/decode_other.go b/vendor/github.com/golang/snappy/decode_other.go index 94a96c5d7..2f672be55 100644 --- a/vendor/github.com/klauspost/compress/snappy/decode_other.go +++ b/vendor/github.com/golang/snappy/decode_other.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !amd64 appengine !gc noasm +// +build !amd64,!arm64 appengine !gc noasm package snappy @@ -87,7 +87,7 @@ func decode(dst, src []byte) int { } // Copy from an earlier sub-slice of dst to a later sub-slice. // If no overlap, use the built-in copy: - if offset > length { + if offset >= length { copy(dst[d:d+length], dst[d-offset:]) d += length continue diff --git a/vendor/github.com/klauspost/compress/snappy/encode.go b/vendor/github.com/golang/snappy/encode.go index 8d393e904..7f2365707 100644 --- a/vendor/github.com/klauspost/compress/snappy/encode.go +++ b/vendor/github.com/golang/snappy/encode.go @@ -15,6 +15,8 @@ import ( // Otherwise, a newly allocated slice will be returned. // // The dst and src must not overlap. It is valid to pass a nil dst. +// +// Encode handles the Snappy block format, not the Snappy stream format. func Encode(dst, src []byte) []byte { if n := MaxEncodedLen(len(src)); n < 0 { panic(ErrTooLarge) @@ -139,6 +141,8 @@ func NewBufferedWriter(w io.Writer) *Writer { } // Writer is an io.Writer that can write Snappy-compressed bytes. +// +// Writer handles the Snappy stream format, not the Snappy block format. type Writer struct { w io.Writer err error diff --git a/vendor/github.com/klauspost/compress/snappy/encode_amd64.s b/vendor/github.com/golang/snappy/encode_amd64.s index adfd979fe..adfd979fe 100644 --- a/vendor/github.com/klauspost/compress/snappy/encode_amd64.s +++ b/vendor/github.com/golang/snappy/encode_amd64.s diff --git a/vendor/github.com/golang/snappy/encode_arm64.s b/vendor/github.com/golang/snappy/encode_arm64.s new file mode 100644 index 000000000..bf83667d7 --- /dev/null +++ b/vendor/github.com/golang/snappy/encode_arm64.s @@ -0,0 +1,722 @@ +// Copyright 2020 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// +build !appengine +// +build gc +// +build !noasm + +#include "textflag.h" + +// The asm code generally follows the pure Go code in encode_other.go, except +// where marked with a "!!!". + +// ---------------------------------------------------------------------------- + +// func emitLiteral(dst, lit []byte) int +// +// All local variables fit into registers. The register allocation: +// - R3 len(lit) +// - R4 n +// - R6 return value +// - R8 &dst[i] +// - R10 &lit[0] +// +// The 32 bytes of stack space is to call runtime·memmove. +// +// The unusual register allocation of local variables, such as R10 for the +// source pointer, matches the allocation used at the call site in encodeBlock, +// which makes it easier to manually inline this function. +TEXT ·emitLiteral(SB), NOSPLIT, $32-56 + MOVD dst_base+0(FP), R8 + MOVD lit_base+24(FP), R10 + MOVD lit_len+32(FP), R3 + MOVD R3, R6 + MOVW R3, R4 + SUBW $1, R4, R4 + + CMPW $60, R4 + BLT oneByte + CMPW $256, R4 + BLT twoBytes + +threeBytes: + MOVD $0xf4, R2 + MOVB R2, 0(R8) + MOVW R4, 1(R8) + ADD $3, R8, R8 + ADD $3, R6, R6 + B memmove + +twoBytes: + MOVD $0xf0, R2 + MOVB R2, 0(R8) + MOVB R4, 1(R8) + ADD $2, R8, R8 + ADD $2, R6, R6 + B memmove + +oneByte: + LSLW $2, R4, R4 + MOVB R4, 0(R8) + ADD $1, R8, R8 + ADD $1, R6, R6 + +memmove: + MOVD R6, ret+48(FP) + + // copy(dst[i:], lit) + // + // This means calling runtime·memmove(&dst[i], &lit[0], len(lit)), so we push + // R8, R10 and R3 as arguments. + MOVD R8, 8(RSP) + MOVD R10, 16(RSP) + MOVD R3, 24(RSP) + CALL runtime·memmove(SB) + RET + +// ---------------------------------------------------------------------------- + +// func emitCopy(dst []byte, offset, length int) int +// +// All local variables fit into registers. The register allocation: +// - R3 length +// - R7 &dst[0] +// - R8 &dst[i] +// - R11 offset +// +// The unusual register allocation of local variables, such as R11 for the +// offset, matches the allocation used at the call site in encodeBlock, which +// makes it easier to manually inline this function. +TEXT ·emitCopy(SB), NOSPLIT, $0-48 + MOVD dst_base+0(FP), R8 + MOVD R8, R7 + MOVD offset+24(FP), R11 + MOVD length+32(FP), R3 + +loop0: + // for length >= 68 { etc } + CMPW $68, R3 + BLT step1 + + // Emit a length 64 copy, encoded as 3 bytes. + MOVD $0xfe, R2 + MOVB R2, 0(R8) + MOVW R11, 1(R8) + ADD $3, R8, R8 + SUB $64, R3, R3 + B loop0 + +step1: + // if length > 64 { etc } + CMP $64, R3 + BLE step2 + + // Emit a length 60 copy, encoded as 3 bytes. + MOVD $0xee, R2 + MOVB R2, 0(R8) + MOVW R11, 1(R8) + ADD $3, R8, R8 + SUB $60, R3, R3 + +step2: + // if length >= 12 || offset >= 2048 { goto step3 } + CMP $12, R3 + BGE step3 + CMPW $2048, R11 + BGE step3 + + // Emit the remaining copy, encoded as 2 bytes. + MOVB R11, 1(R8) + LSRW $3, R11, R11 + AND $0xe0, R11, R11 + SUB $4, R3, R3 + LSLW $2, R3 + AND $0xff, R3, R3 + ORRW R3, R11, R11 + ORRW $1, R11, R11 + MOVB R11, 0(R8) + ADD $2, R8, R8 + + // Return the number of bytes written. + SUB R7, R8, R8 + MOVD R8, ret+40(FP) + RET + +step3: + // Emit the remaining copy, encoded as 3 bytes. + SUB $1, R3, R3 + AND $0xff, R3, R3 + LSLW $2, R3, R3 + ORRW $2, R3, R3 + MOVB R3, 0(R8) + MOVW R11, 1(R8) + ADD $3, R8, R8 + + // Return the number of bytes written. + SUB R7, R8, R8 + MOVD R8, ret+40(FP) + RET + +// ---------------------------------------------------------------------------- + +// func extendMatch(src []byte, i, j int) int +// +// All local variables fit into registers. The register allocation: +// - R6 &src[0] +// - R7 &src[j] +// - R13 &src[len(src) - 8] +// - R14 &src[len(src)] +// - R15 &src[i] +// +// The unusual register allocation of local variables, such as R15 for a source +// pointer, matches the allocation used at the call site in encodeBlock, which +// makes it easier to manually inline this function. +TEXT ·extendMatch(SB), NOSPLIT, $0-48 + MOVD src_base+0(FP), R6 + MOVD src_len+8(FP), R14 + MOVD i+24(FP), R15 + MOVD j+32(FP), R7 + ADD R6, R14, R14 + ADD R6, R15, R15 + ADD R6, R7, R7 + MOVD R14, R13 + SUB $8, R13, R13 + +cmp8: + // As long as we are 8 or more bytes before the end of src, we can load and + // compare 8 bytes at a time. If those 8 bytes are equal, repeat. + CMP R13, R7 + BHI cmp1 + MOVD (R15), R3 + MOVD (R7), R4 + CMP R4, R3 + BNE bsf + ADD $8, R15, R15 + ADD $8, R7, R7 + B cmp8 + +bsf: + // If those 8 bytes were not equal, XOR the two 8 byte values, and return + // the index of the first byte that differs. + // RBIT reverses the bit order, then CLZ counts the leading zeros, the + // combination of which finds the least significant bit which is set. + // The arm64 architecture is little-endian, and the shift by 3 converts + // a bit index to a byte index. + EOR R3, R4, R4 + RBIT R4, R4 + CLZ R4, R4 + ADD R4>>3, R7, R7 + + // Convert from &src[ret] to ret. + SUB R6, R7, R7 + MOVD R7, ret+40(FP) + RET + +cmp1: + // In src's tail, compare 1 byte at a time. + CMP R7, R14 + BLS extendMatchEnd + MOVB (R15), R3 + MOVB (R7), R4 + CMP R4, R3 + BNE extendMatchEnd + ADD $1, R15, R15 + ADD $1, R7, R7 + B cmp1 + +extendMatchEnd: + // Convert from &src[ret] to ret. + SUB R6, R7, R7 + MOVD R7, ret+40(FP) + RET + +// ---------------------------------------------------------------------------- + +// func encodeBlock(dst, src []byte) (d int) +// +// All local variables fit into registers, other than "var table". The register +// allocation: +// - R3 . . +// - R4 . . +// - R5 64 shift +// - R6 72 &src[0], tableSize +// - R7 80 &src[s] +// - R8 88 &dst[d] +// - R9 96 sLimit +// - R10 . &src[nextEmit] +// - R11 104 prevHash, currHash, nextHash, offset +// - R12 112 &src[base], skip +// - R13 . &src[nextS], &src[len(src) - 8] +// - R14 . len(src), bytesBetweenHashLookups, &src[len(src)], x +// - R15 120 candidate +// - R16 . hash constant, 0x1e35a7bd +// - R17 . &table +// - . 128 table +// +// The second column (64, 72, etc) is the stack offset to spill the registers +// when calling other functions. We could pack this slightly tighter, but it's +// simpler to have a dedicated spill map independent of the function called. +// +// "var table [maxTableSize]uint16" takes up 32768 bytes of stack space. An +// extra 64 bytes, to call other functions, and an extra 64 bytes, to spill +// local variables (registers) during calls gives 32768 + 64 + 64 = 32896. +TEXT ·encodeBlock(SB), 0, $32896-56 + MOVD dst_base+0(FP), R8 + MOVD src_base+24(FP), R7 + MOVD src_len+32(FP), R14 + + // shift, tableSize := uint32(32-8), 1<<8 + MOVD $24, R5 + MOVD $256, R6 + MOVW $0xa7bd, R16 + MOVKW $(0x1e35<<16), R16 + +calcShift: + // for ; tableSize < maxTableSize && tableSize < len(src); tableSize *= 2 { + // shift-- + // } + MOVD $16384, R2 + CMP R2, R6 + BGE varTable + CMP R14, R6 + BGE varTable + SUB $1, R5, R5 + LSL $1, R6, R6 + B calcShift + +varTable: + // var table [maxTableSize]uint16 + // + // In the asm code, unlike the Go code, we can zero-initialize only the + // first tableSize elements. Each uint16 element is 2 bytes and each + // iterations writes 64 bytes, so we can do only tableSize/32 writes + // instead of the 2048 writes that would zero-initialize all of table's + // 32768 bytes. This clear could overrun the first tableSize elements, but + // it won't overrun the allocated stack size. + ADD $128, RSP, R17 + MOVD R17, R4 + + // !!! R6 = &src[tableSize] + ADD R6<<1, R17, R6 + +memclr: + STP.P (ZR, ZR), 64(R4) + STP (ZR, ZR), -48(R4) + STP (ZR, ZR), -32(R4) + STP (ZR, ZR), -16(R4) + CMP R4, R6 + BHI memclr + + // !!! R6 = &src[0] + MOVD R7, R6 + + // sLimit := len(src) - inputMargin + MOVD R14, R9 + SUB $15, R9, R9 + + // !!! Pre-emptively spill R5, R6 and R9 to the stack. Their values don't + // change for the rest of the function. + MOVD R5, 64(RSP) + MOVD R6, 72(RSP) + MOVD R9, 96(RSP) + + // nextEmit := 0 + MOVD R6, R10 + + // s := 1 + ADD $1, R7, R7 + + // nextHash := hash(load32(src, s), shift) + MOVW 0(R7), R11 + MULW R16, R11, R11 + LSRW R5, R11, R11 + +outer: + // for { etc } + + // skip := 32 + MOVD $32, R12 + + // nextS := s + MOVD R7, R13 + + // candidate := 0 + MOVD $0, R15 + +inner0: + // for { etc } + + // s := nextS + MOVD R13, R7 + + // bytesBetweenHashLookups := skip >> 5 + MOVD R12, R14 + LSR $5, R14, R14 + + // nextS = s + bytesBetweenHashLookups + ADD R14, R13, R13 + + // skip += bytesBetweenHashLookups + ADD R14, R12, R12 + + // if nextS > sLimit { goto emitRemainder } + MOVD R13, R3 + SUB R6, R3, R3 + CMP R9, R3 + BHI emitRemainder + + // candidate = int(table[nextHash]) + MOVHU 0(R17)(R11<<1), R15 + + // table[nextHash] = uint16(s) + MOVD R7, R3 + SUB R6, R3, R3 + + MOVH R3, 0(R17)(R11<<1) + + // nextHash = hash(load32(src, nextS), shift) + MOVW 0(R13), R11 + MULW R16, R11 + LSRW R5, R11, R11 + + // if load32(src, s) != load32(src, candidate) { continue } break + MOVW 0(R7), R3 + MOVW (R6)(R15*1), R4 + CMPW R4, R3 + BNE inner0 + +fourByteMatch: + // As per the encode_other.go code: + // + // A 4-byte match has been found. We'll later see etc. + + // !!! Jump to a fast path for short (<= 16 byte) literals. See the comment + // on inputMargin in encode.go. + MOVD R7, R3 + SUB R10, R3, R3 + CMP $16, R3 + BLE emitLiteralFastPath + + // ---------------------------------------- + // Begin inline of the emitLiteral call. + // + // d += emitLiteral(dst[d:], src[nextEmit:s]) + + MOVW R3, R4 + SUBW $1, R4, R4 + + MOVW $60, R2 + CMPW R2, R4 + BLT inlineEmitLiteralOneByte + MOVW $256, R2 + CMPW R2, R4 + BLT inlineEmitLiteralTwoBytes + +inlineEmitLiteralThreeBytes: + MOVD $0xf4, R1 + MOVB R1, 0(R8) + MOVW R4, 1(R8) + ADD $3, R8, R8 + B inlineEmitLiteralMemmove + +inlineEmitLiteralTwoBytes: + MOVD $0xf0, R1 + MOVB R1, 0(R8) + MOVB R4, 1(R8) + ADD $2, R8, R8 + B inlineEmitLiteralMemmove + +inlineEmitLiteralOneByte: + LSLW $2, R4, R4 + MOVB R4, 0(R8) + ADD $1, R8, R8 + +inlineEmitLiteralMemmove: + // Spill local variables (registers) onto the stack; call; unspill. + // + // copy(dst[i:], lit) + // + // This means calling runtime·memmove(&dst[i], &lit[0], len(lit)), so we push + // R8, R10 and R3 as arguments. + MOVD R8, 8(RSP) + MOVD R10, 16(RSP) + MOVD R3, 24(RSP) + + // Finish the "d +=" part of "d += emitLiteral(etc)". + ADD R3, R8, R8 + MOVD R7, 80(RSP) + MOVD R8, 88(RSP) + MOVD R15, 120(RSP) + CALL runtime·memmove(SB) + MOVD 64(RSP), R5 + MOVD 72(RSP), R6 + MOVD 80(RSP), R7 + MOVD 88(RSP), R8 + MOVD 96(RSP), R9 + MOVD 120(RSP), R15 + ADD $128, RSP, R17 + MOVW $0xa7bd, R16 + MOVKW $(0x1e35<<16), R16 + B inner1 + +inlineEmitLiteralEnd: + // End inline of the emitLiteral call. + // ---------------------------------------- + +emitLiteralFastPath: + // !!! Emit the 1-byte encoding "uint8(len(lit)-1)<<2". + MOVB R3, R4 + SUBW $1, R4, R4 + AND $0xff, R4, R4 + LSLW $2, R4, R4 + MOVB R4, (R8) + ADD $1, R8, R8 + + // !!! Implement the copy from lit to dst as a 16-byte load and store. + // (Encode's documentation says that dst and src must not overlap.) + // + // This always copies 16 bytes, instead of only len(lit) bytes, but that's + // OK. Subsequent iterations will fix up the overrun. + // + // Note that on arm64, it is legal and cheap to issue unaligned 8-byte or + // 16-byte loads and stores. This technique probably wouldn't be as + // effective on architectures that are fussier about alignment. + LDP 0(R10), (R0, R1) + STP (R0, R1), 0(R8) + ADD R3, R8, R8 + +inner1: + // for { etc } + + // base := s + MOVD R7, R12 + + // !!! offset := base - candidate + MOVD R12, R11 + SUB R15, R11, R11 + SUB R6, R11, R11 + + // ---------------------------------------- + // Begin inline of the extendMatch call. + // + // s = extendMatch(src, candidate+4, s+4) + + // !!! R14 = &src[len(src)] + MOVD src_len+32(FP), R14 + ADD R6, R14, R14 + + // !!! R13 = &src[len(src) - 8] + MOVD R14, R13 + SUB $8, R13, R13 + + // !!! R15 = &src[candidate + 4] + ADD $4, R15, R15 + ADD R6, R15, R15 + + // !!! s += 4 + ADD $4, R7, R7 + +inlineExtendMatchCmp8: + // As long as we are 8 or more bytes before the end of src, we can load and + // compare 8 bytes at a time. If those 8 bytes are equal, repeat. + CMP R13, R7 + BHI inlineExtendMatchCmp1 + MOVD (R15), R3 + MOVD (R7), R4 + CMP R4, R3 + BNE inlineExtendMatchBSF + ADD $8, R15, R15 + ADD $8, R7, R7 + B inlineExtendMatchCmp8 + +inlineExtendMatchBSF: + // If those 8 bytes were not equal, XOR the two 8 byte values, and return + // the index of the first byte that differs. + // RBIT reverses the bit order, then CLZ counts the leading zeros, the + // combination of which finds the least significant bit which is set. + // The arm64 architecture is little-endian, and the shift by 3 converts + // a bit index to a byte index. + EOR R3, R4, R4 + RBIT R4, R4 + CLZ R4, R4 + ADD R4>>3, R7, R7 + B inlineExtendMatchEnd + +inlineExtendMatchCmp1: + // In src's tail, compare 1 byte at a time. + CMP R7, R14 + BLS inlineExtendMatchEnd + MOVB (R15), R3 + MOVB (R7), R4 + CMP R4, R3 + BNE inlineExtendMatchEnd + ADD $1, R15, R15 + ADD $1, R7, R7 + B inlineExtendMatchCmp1 + +inlineExtendMatchEnd: + // End inline of the extendMatch call. + // ---------------------------------------- + + // ---------------------------------------- + // Begin inline of the emitCopy call. + // + // d += emitCopy(dst[d:], base-candidate, s-base) + + // !!! length := s - base + MOVD R7, R3 + SUB R12, R3, R3 + +inlineEmitCopyLoop0: + // for length >= 68 { etc } + MOVW $68, R2 + CMPW R2, R3 + BLT inlineEmitCopyStep1 + + // Emit a length 64 copy, encoded as 3 bytes. + MOVD $0xfe, R1 + MOVB R1, 0(R8) + MOVW R11, 1(R8) + ADD $3, R8, R8 + SUBW $64, R3, R3 + B inlineEmitCopyLoop0 + +inlineEmitCopyStep1: + // if length > 64 { etc } + MOVW $64, R2 + CMPW R2, R3 + BLE inlineEmitCopyStep2 + + // Emit a length 60 copy, encoded as 3 bytes. + MOVD $0xee, R1 + MOVB R1, 0(R8) + MOVW R11, 1(R8) + ADD $3, R8, R8 + SUBW $60, R3, R3 + +inlineEmitCopyStep2: + // if length >= 12 || offset >= 2048 { goto inlineEmitCopyStep3 } + MOVW $12, R2 + CMPW R2, R3 + BGE inlineEmitCopyStep3 + MOVW $2048, R2 + CMPW R2, R11 + BGE inlineEmitCopyStep3 + + // Emit the remaining copy, encoded as 2 bytes. + MOVB R11, 1(R8) + LSRW $8, R11, R11 + LSLW $5, R11, R11 + SUBW $4, R3, R3 + AND $0xff, R3, R3 + LSLW $2, R3, R3 + ORRW R3, R11, R11 + ORRW $1, R11, R11 + MOVB R11, 0(R8) + ADD $2, R8, R8 + B inlineEmitCopyEnd + +inlineEmitCopyStep3: + // Emit the remaining copy, encoded as 3 bytes. + SUBW $1, R3, R3 + LSLW $2, R3, R3 + ORRW $2, R3, R3 + MOVB R3, 0(R8) + MOVW R11, 1(R8) + ADD $3, R8, R8 + +inlineEmitCopyEnd: + // End inline of the emitCopy call. + // ---------------------------------------- + + // nextEmit = s + MOVD R7, R10 + + // if s >= sLimit { goto emitRemainder } + MOVD R7, R3 + SUB R6, R3, R3 + CMP R3, R9 + BLS emitRemainder + + // As per the encode_other.go code: + // + // We could immediately etc. + + // x := load64(src, s-1) + MOVD -1(R7), R14 + + // prevHash := hash(uint32(x>>0), shift) + MOVW R14, R11 + MULW R16, R11, R11 + LSRW R5, R11, R11 + + // table[prevHash] = uint16(s-1) + MOVD R7, R3 + SUB R6, R3, R3 + SUB $1, R3, R3 + + MOVHU R3, 0(R17)(R11<<1) + + // currHash := hash(uint32(x>>8), shift) + LSR $8, R14, R14 + MOVW R14, R11 + MULW R16, R11, R11 + LSRW R5, R11, R11 + + // candidate = int(table[currHash]) + MOVHU 0(R17)(R11<<1), R15 + + // table[currHash] = uint16(s) + ADD $1, R3, R3 + MOVHU R3, 0(R17)(R11<<1) + + // if uint32(x>>8) == load32(src, candidate) { continue } + MOVW (R6)(R15*1), R4 + CMPW R4, R14 + BEQ inner1 + + // nextHash = hash(uint32(x>>16), shift) + LSR $8, R14, R14 + MOVW R14, R11 + MULW R16, R11, R11 + LSRW R5, R11, R11 + + // s++ + ADD $1, R7, R7 + + // break out of the inner1 for loop, i.e. continue the outer loop. + B outer + +emitRemainder: + // if nextEmit < len(src) { etc } + MOVD src_len+32(FP), R3 + ADD R6, R3, R3 + CMP R3, R10 + BEQ encodeBlockEnd + + // d += emitLiteral(dst[d:], src[nextEmit:]) + // + // Push args. + MOVD R8, 8(RSP) + MOVD $0, 16(RSP) // Unnecessary, as the callee ignores it, but conservative. + MOVD $0, 24(RSP) // Unnecessary, as the callee ignores it, but conservative. + MOVD R10, 32(RSP) + SUB R10, R3, R3 + MOVD R3, 40(RSP) + MOVD R3, 48(RSP) // Unnecessary, as the callee ignores it, but conservative. + + // Spill local variables (registers) onto the stack; call; unspill. + MOVD R8, 88(RSP) + CALL ·emitLiteral(SB) + MOVD 88(RSP), R8 + + // Finish the "d +=" part of "d += emitLiteral(etc)". + MOVD 56(RSP), R1 + ADD R1, R8, R8 + +encodeBlockEnd: + MOVD dst_base+0(FP), R3 + SUB R3, R8, R8 + MOVD R8, d+48(FP) + RET diff --git a/vendor/github.com/klauspost/compress/snappy/encode_amd64.go b/vendor/github.com/golang/snappy/encode_asm.go index 150d91bc8..107c1e714 100644 --- a/vendor/github.com/klauspost/compress/snappy/encode_amd64.go +++ b/vendor/github.com/golang/snappy/encode_asm.go @@ -5,6 +5,7 @@ // +build !appengine // +build gc // +build !noasm +// +build amd64 arm64 package snappy diff --git a/vendor/github.com/klauspost/compress/snappy/encode_other.go b/vendor/github.com/golang/snappy/encode_other.go index dbcae905e..296d7f0be 100644 --- a/vendor/github.com/klauspost/compress/snappy/encode_other.go +++ b/vendor/github.com/golang/snappy/encode_other.go @@ -2,7 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -// +build !amd64 appengine !gc noasm +// +build !amd64,!arm64 appengine !gc noasm package snappy diff --git a/vendor/github.com/golang/snappy/go.mod b/vendor/github.com/golang/snappy/go.mod new file mode 100644 index 000000000..f6406bb2c --- /dev/null +++ b/vendor/github.com/golang/snappy/go.mod @@ -0,0 +1 @@ +module github.com/golang/snappy diff --git a/vendor/github.com/klauspost/compress/snappy/snappy.go b/vendor/github.com/golang/snappy/snappy.go index ea58ced88..ece692ea4 100644 --- a/vendor/github.com/klauspost/compress/snappy/snappy.go +++ b/vendor/github.com/golang/snappy/snappy.go @@ -17,7 +17,7 @@ // // The canonical, C++ implementation is at https://github.com/google/snappy and // it only implements the block format. -package snappy +package snappy // import "github.com/golang/snappy" import ( "hash/crc32" @@ -94,5 +94,5 @@ var crcTable = crc32.MakeTable(crc32.Castagnoli) // https://github.com/google/snappy/blob/master/framing_format.txt func crc(b []byte) uint32 { c := crc32.Update(0, crcTable, b) - return c>>15 | c<<17 + 0xa282ead8 + return uint32(c>>15|c<<17) + 0xa282ead8 } diff --git a/vendor/github.com/klauspost/compress/flate/deflate.go b/vendor/github.com/klauspost/compress/flate/deflate.go index 3f428d8b5..40b5802de 100644 --- a/vendor/github.com/klauspost/compress/flate/deflate.go +++ b/vendor/github.com/klauspost/compress/flate/deflate.go @@ -440,8 +440,7 @@ func (d *compressor) deflateLazy() { // index and index-1 are already inserted. If there is not enough // lookahead, the last two strings are not inserted into the hash // table. - var newIndex int - newIndex = s.index + prevLength - 1 + newIndex := s.index + prevLength - 1 // Calculate missing hashes end := newIndex if end > s.maxInsertIndex { diff --git a/vendor/github.com/klauspost/compress/fse/compress.go b/vendor/github.com/klauspost/compress/fse/compress.go index 0d31f5ebc..6f341914c 100644 --- a/vendor/github.com/klauspost/compress/fse/compress.go +++ b/vendor/github.com/klauspost/compress/fse/compress.go @@ -92,7 +92,6 @@ func (c *cState) init(bw *bitWriter, ct *cTable, tableLog uint8, first symbolTra im := int32((nbBitsOut << 16) - first.deltaNbBits) lu := (im >> nbBitsOut) + first.deltaFindState c.state = c.stateTable[lu] - return } // encode the output symbol provided and write it to the bitstream. diff --git a/vendor/github.com/klauspost/compress/huff0/compress.go b/vendor/github.com/klauspost/compress/huff0/compress.go index dea115b33..0823c928c 100644 --- a/vendor/github.com/klauspost/compress/huff0/compress.go +++ b/vendor/github.com/klauspost/compress/huff0/compress.go @@ -536,7 +536,6 @@ func (s *Scratch) huffSort() { } nodes[pos&huffNodesMask] = nodeElt{count: c, symbol: byte(n)} } - return } func (s *Scratch) setMaxHeight(lastNonNull int) uint8 { diff --git a/vendor/github.com/klauspost/compress/snappy/runbench.cmd b/vendor/github.com/klauspost/compress/snappy/runbench.cmd deleted file mode 100644 index d24eb4b47..000000000 --- a/vendor/github.com/klauspost/compress/snappy/runbench.cmd +++ /dev/null @@ -1,2 +0,0 @@ -del old.txt -go test -bench=. >>old.txt && go test -bench=. >>old.txt && go test -bench=. >>old.txt && benchstat -delta-test=ttest old.txt new.txt diff --git a/vendor/github.com/klauspost/compress/zstd/blockenc.go b/vendor/github.com/klauspost/compress/zstd/blockenc.go index 9647c64e5..e1be092f3 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockenc.go +++ b/vendor/github.com/klauspost/compress/zstd/blockenc.go @@ -386,9 +386,9 @@ func (b *blockEnc) encodeLits(lits []byte, raw bool) error { b.output = bh.appendTo(b.output) b.output = append(b.output, lits[0]) return nil + case nil: default: return err - case nil: } // Compressed... // Now, allow reuse @@ -528,11 +528,6 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { if debug { println("Adding literals RLE") } - default: - if debug { - println("Adding literals ERROR:", err) - } - return err case nil: // Compressed litLen... if reUsed { @@ -563,6 +558,11 @@ func (b *blockEnc) encode(org []byte, raw, rawAllLits bool) error { if debug { println("Adding literals compressed") } + default: + if debug { + println("Adding literals ERROR:", err) + } + return err } // Sequence compression diff --git a/vendor/github.com/klauspost/compress/zstd/decodeheader.go b/vendor/github.com/klauspost/compress/zstd/decodeheader.go index 87896c5ea..69736e8d4 100644 --- a/vendor/github.com/klauspost/compress/zstd/decodeheader.go +++ b/vendor/github.com/klauspost/compress/zstd/decodeheader.go @@ -93,7 +93,7 @@ func (h *Header) Decode(in []byte) error { h.HasCheckSum = fhd&(1<<2) != 0 if fhd&(1<<3) != 0 { - return errors.New("Reserved bit set on frame header") + return errors.New("reserved bit set on frame header") } // Read Window_Descriptor @@ -174,7 +174,7 @@ func (h *Header) Decode(in []byte) error { if len(in) < 3 { return nil } - tmp, in := in[:3], in[3:] + tmp := in[:3] bh := uint32(tmp[0]) | (uint32(tmp[1]) << 8) | (uint32(tmp[2]) << 16) h.FirstBlock.Last = bh&1 != 0 blockType := blockType((bh >> 1) & 3) diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index 1d41c25d2..f593e464b 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -179,8 +179,7 @@ func (d *Decoder) Reset(r io.Reader) error { // If bytes buffer and < 1MB, do sync decoding anyway. if bb, ok := r.(byter); ok && bb.Len() < 1<<20 { - var bb2 byter - bb2 = bb + bb2 := bb if debug { println("*bytes.Buffer detected, doing sync decode, len:", bb.Len()) } @@ -237,20 +236,17 @@ func (d *Decoder) drainOutput() { println("current already flushed") return } - for { - select { - case v := <-d.current.output: - if v.d != nil { - if debug { - printf("re-adding decoder %p", v.d) - } - d.decoders <- v.d - } - if v.err == errEndOfStream { - println("current flushed") - d.current.flushed = true - return + for v := range d.current.output { + if v.d != nil { + if debug { + printf("re-adding decoder %p", v.d) } + d.decoders <- v.d + } + if v.err == errEndOfStream { + println("current flushed") + d.current.flushed = true + return } } } diff --git a/vendor/github.com/klauspost/compress/zstd/decoder_options.go b/vendor/github.com/klauspost/compress/zstd/decoder_options.go index 284d38449..c0fd058c2 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder_options.go @@ -6,7 +6,6 @@ package zstd import ( "errors" - "fmt" "runtime" ) @@ -43,7 +42,7 @@ func WithDecoderLowmem(b bool) DOption { func WithDecoderConcurrency(n int) DOption { return func(o *decoderOptions) error { if n <= 0 { - return fmt.Errorf("Concurrency must be at least 1") + return errors.New("concurrency must be at least 1") } o.concurrent = n return nil @@ -61,7 +60,7 @@ func WithDecoderMaxMemory(n uint64) DOption { return errors.New("WithDecoderMaxMemory must be at least 1") } if n > 1<<63 { - return fmt.Errorf("WithDecoderMaxmemory must be less than 1 << 63") + return errors.New("WithDecoderMaxmemory must be less than 1 << 63") } o.maxDecodedSize = n return nil diff --git a/vendor/github.com/klauspost/compress/zstd/enc_base.go b/vendor/github.com/klauspost/compress/zstd/enc_base.go index 2d4d893eb..60f298648 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_base.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_base.go @@ -150,14 +150,15 @@ func (e *fastBase) resetBase(d *dict, singleBlock bool) { } else { e.crc.Reset() } - if (!singleBlock || d.DictContentSize() > 0) && cap(e.hist) < int(e.maxMatchOff*2)+d.DictContentSize() { - l := e.maxMatchOff*2 + int32(d.DictContentSize()) - // Make it at least 1MB. - if l < 1<<20 { - l = 1 << 20 + if d != nil { + low := e.lowMem + if singleBlock { + e.lowMem = true } - e.hist = make([]byte, 0, l) + e.ensureHist(d.DictContentSize() + maxCompressedBlockSize) + e.lowMem = low } + // We offset current position so everything will be out of reach. // If above reset line, history will be purged. if e.cur < bufferReset { diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index 6f0265099..4871dd03a 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -340,13 +340,13 @@ func (e *Encoder) ReadFrom(r io.Reader) (n int64, err error) { println("ReadFrom: got EOF final block:", len(e.state.filling)) } return n, nil + case nil: default: if debug { println("ReadFrom: got error:", err) } e.state.err = err return n, err - case nil: } if len(src) > 0 { if debug { diff --git a/vendor/github.com/klauspost/compress/zstd/encoder_options.go b/vendor/github.com/klauspost/compress/zstd/encoder_options.go index 18a47eb03..16d4ab63c 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder_options.go @@ -73,7 +73,7 @@ func WithEncoderCRC(b bool) EOption { } // WithEncoderConcurrency will set the concurrency, -// meaning the maximum number of decoders to run concurrently. +// meaning the maximum number of encoders to run concurrently. // The value supplied must be at least 1. // By default this will be set to GOMAXPROCS. func WithEncoderConcurrency(n int) EOption { diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index fc4a566d3..693c5f05d 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -121,7 +121,7 @@ func (d *frameDec) reset(br byteBuffer) error { d.SingleSegment = fhd&(1<<5) != 0 if fhd&(1<<3) != 0 { - return errors.New("Reserved bit set on frame header") + return errors.New("reserved bit set on frame header") } // Read Window_Descriptor diff --git a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go index b80709d5e..c74681b99 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_encoder.go @@ -708,7 +708,6 @@ func (c *cState) init(bw *bitWriter, ct *cTable, first symbolTransform) { im := int32((nbBitsOut << 16) - first.deltaNbBits) lu := (im >> nbBitsOut) + int32(first.deltaFindState) c.state = c.stateTable[lu] - return } // encode the output symbol provided and write it to the bitstream. diff --git a/vendor/github.com/klauspost/compress/zstd/fse_predefined.go b/vendor/github.com/klauspost/compress/zstd/fse_predefined.go index 6c17dc17f..474cb77d2 100644 --- a/vendor/github.com/klauspost/compress/zstd/fse_predefined.go +++ b/vendor/github.com/klauspost/compress/zstd/fse_predefined.go @@ -59,7 +59,7 @@ func fillBase(dst []baseOffset, base uint32, bits ...uint8) { } for i, bit := range bits { if base > math.MaxInt32 { - panic(fmt.Sprintf("invalid decoding table, base overflows int32")) + panic("invalid decoding table, base overflows int32") } dst[i] = baseOffset{ diff --git a/vendor/github.com/klauspost/compress/zstd/seqenc.go b/vendor/github.com/klauspost/compress/zstd/seqenc.go index 36bcc3cc0..8014174a7 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqenc.go +++ b/vendor/github.com/klauspost/compress/zstd/seqenc.go @@ -35,7 +35,6 @@ func (s *seqCoders) setPrev(ll, ml, of *fseEncoder) { // Ensure we cannot reuse by accident prevEnc := *prev prevEnc.symbolLen = 0 - return } compareSwap(ll, &s.llEnc, &s.llPrev) compareSwap(ml, &s.mlEnc, &s.mlPrev) diff --git a/vendor/github.com/klauspost/compress/zstd/snappy.go b/vendor/github.com/klauspost/compress/zstd/snappy.go index c95fe5111..9d9d1d567 100644 --- a/vendor/github.com/klauspost/compress/zstd/snappy.go +++ b/vendor/github.com/klauspost/compress/zstd/snappy.go @@ -10,8 +10,8 @@ import ( "hash/crc32" "io" + "github.com/golang/snappy" "github.com/klauspost/compress/huff0" - "github.com/klauspost/compress/snappy" ) const ( @@ -185,7 +185,6 @@ func (r *SnappyConverter) Convert(in io.Reader, w io.Writer) (int64, error) { r.block.reset(nil) r.block.literals, err = snappy.Decode(r.block.literals[:n], r.buf[snappyChecksumSize:chunkLen]) if err != nil { - println("snappy.Decode:", err) return written, err } err = r.block.encodeLits(r.block.literals, false) diff --git a/vendor/github.com/klauspost/compress/zstd/zstd.go b/vendor/github.com/klauspost/compress/zstd/zstd.go index 9056beef2..1ba308c8b 100644 --- a/vendor/github.com/klauspost/compress/zstd/zstd.go +++ b/vendor/github.com/klauspost/compress/zstd/zstd.go @@ -5,6 +5,7 @@ package zstd import ( "bytes" + "encoding/binary" "errors" "log" "math" @@ -126,26 +127,15 @@ func matchLen(a, b []byte) int { } func load3232(b []byte, i int32) uint32 { - // Help the compiler eliminate bounds checks on the read so it can be done in a single read. - b = b[i:] - b = b[:4] - return uint32(b[0]) | uint32(b[1])<<8 | uint32(b[2])<<16 | uint32(b[3])<<24 + return binary.LittleEndian.Uint32(b[i:]) } func load6432(b []byte, i int32) uint64 { - // Help the compiler eliminate bounds checks on the read so it can be done in a single read. - b = b[i:] - b = b[:8] - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + return binary.LittleEndian.Uint64(b[i:]) } func load64(b []byte, i int) uint64 { - // Help the compiler eliminate bounds checks on the read so it can be done in a single read. - b = b[i:] - b = b[:8] - return uint64(b[0]) | uint64(b[1])<<8 | uint64(b[2])<<16 | uint64(b[3])<<24 | - uint64(b[4])<<32 | uint64(b[5])<<40 | uint64(b[6])<<48 | uint64(b[7])<<56 + return binary.LittleEndian.Uint64(b[i:]) } type byter interface { diff --git a/vendor/modules.txt b/vendor/modules.txt index e9e252675..b0658df5b 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -96,7 +96,7 @@ github.com/containers/buildah/pkg/parse github.com/containers/buildah/pkg/rusage github.com/containers/buildah/pkg/supplemented github.com/containers/buildah/util -# github.com/containers/common v0.36.0 +# github.com/containers/common v0.37.0 github.com/containers/common/pkg/apparmor github.com/containers/common/pkg/apparmor/internal/supported github.com/containers/common/pkg/auth @@ -118,7 +118,7 @@ github.com/containers/common/pkg/umask github.com/containers/common/version # github.com/containers/conmon v2.0.20+incompatible github.com/containers/conmon/runner/config -# github.com/containers/image/v5 v5.11.0 +# github.com/containers/image/v5 v5.11.1 github.com/containers/image/v5/copy github.com/containers/image/v5/directory github.com/containers/image/v5/directory/explicitfilepath @@ -191,7 +191,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.29.0 +# github.com/containers/storage v1.30.0 github.com/containers/storage github.com/containers/storage/drivers github.com/containers/storage/drivers/aufs @@ -348,6 +348,8 @@ github.com/golang/protobuf/ptypes github.com/golang/protobuf/ptypes/any github.com/golang/protobuf/ptypes/duration github.com/golang/protobuf/ptypes/timestamp +# github.com/golang/snappy v0.0.3 +github.com/golang/snappy # github.com/google/go-cmp v0.5.5 github.com/google/go-cmp/cmp github.com/google/go-cmp/cmp/internal/diff @@ -387,11 +389,10 @@ github.com/json-iterator/go # github.com/juju/ansiterm v0.0.0-20180109212912-720a0952cc2a github.com/juju/ansiterm github.com/juju/ansiterm/tabwriter -# github.com/klauspost/compress v1.11.13 +# github.com/klauspost/compress v1.12.1 github.com/klauspost/compress/flate github.com/klauspost/compress/fse github.com/klauspost/compress/huff0 -github.com/klauspost/compress/snappy github.com/klauspost/compress/zstd github.com/klauspost/compress/zstd/internal/xxhash # github.com/klauspost/pgzip v1.2.5 |