diff options
133 files changed, 3085 insertions, 1887 deletions
@@ -569,7 +569,7 @@ remoteintegration: test-binaries ginkgo-remote .PHONY: localbenchmarks localbenchmarks: test-binaries - ACK_GINKGO_RC=true $(GOBIN)/ginkgo \ + PATH=$(PATH):$(shell pwd)/hack ACK_GINKGO_RC=true $(GOBIN)/ginkgo \ -focus "Podman Benchmark Suite" \ -tags "$(BUILDTAGS) benchmarks" -noColor \ -noisySkippings=false -noisyPendings=false \ diff --git a/cmd/podman/common/completion.go b/cmd/podman/common/completion.go index 6149a4465..e07e28dab 100644 --- a/cmd/podman/common/completion.go +++ b/cmd/podman/common/completion.go @@ -992,13 +992,6 @@ func AutocompleteFormat(o interface{}) func(cmd *cobra.Command, args []string, t fields := strings.Split(field[len(field)-1], ".") f := reflect.ValueOf(o) for i := 1; i < len(fields); i++ { - val := getActualStructType(f) - if val == nil { - // no struct return nothing to complete - return nil, cobra.ShellCompDirectiveNoFileComp - } - f = *val - // last field get all names to suggest if i == len(fields)-1 { suggestions := getStructFields(f, fields[i]) @@ -1008,6 +1001,14 @@ func AutocompleteFormat(o interface{}) func(cmd *cobra.Command, args []string, t toComplete = strings.Join(toCompArr, ".") return prefixSlice(toComplete, suggestions), cobra.ShellCompDirectiveNoSpace | cobra.ShellCompDirectiveNoFileComp } + + val := getActualStructType(f) + if val == nil { + // no struct return nothing to complete + return nil, cobra.ShellCompDirectiveNoFileComp + } + f = *val + // set the next struct field f = f.FieldByName(fields[i]) } @@ -1038,12 +1039,15 @@ func getActualStructType(f reflect.Value) *reflect.Value { // getStructFields reads all struct field names and method names and returns them. func getStructFields(f reflect.Value, prefix string) []string { - suggestions := []string{} + var suggestions []string + if f.IsValid() { + suggestions = append(suggestions, getMethodNames(f, prefix)...) + } val := getActualStructType(f) if val == nil { // no struct return nothing to complete - return nil + return suggestions } f = *val @@ -1069,7 +1073,11 @@ func getStructFields(f reflect.Value, prefix string) []string { suggestions = append(suggestions, fname+suffix) } } + return suggestions +} +func getMethodNames(f reflect.Value, prefix string) []string { + suggestions := make([]string, 0, f.NumMethod()) for j := 0; j < f.NumMethod(); j++ { fname := f.Type().Method(j).Name if strings.HasPrefix(fname, prefix) { @@ -1077,7 +1085,6 @@ func getStructFields(f reflect.Value, prefix string) []string { suggestions = append(suggestions, fname+"}}") } } - return suggestions } diff --git a/cmd/podman/common/completion_test.go b/cmd/podman/common/completion_test.go index ae117a173..13f45a662 100644 --- a/cmd/podman/common/completion_test.go +++ b/cmd/podman/common/completion_test.go @@ -25,7 +25,9 @@ func (c Car) Type() string { return "" } -func (c Car) Color() string { +// Note: It is important that this function is *Car and the Type one is just Car. +// The reflect logic behaves differently for these cases so we have to test both. +func (c *Car) Color() string { return "" } @@ -94,7 +96,7 @@ func TestAutocompleteFormat(t *testing.T) { { "second level struct field name", "{{ .Car.", - []string{"{{ .Car.Brand}}", "{{ .Car.Stats.", "{{ .Car.Extras}}", "{{ .Car.Color}}", "{{ .Car.Type}}"}, + []string{"{{ .Car.Color}}", "{{ .Car.Type}}", "{{ .Car.Brand}}", "{{ .Car.Stats.", "{{ .Car.Extras}}"}, }, { "second level struct field name", @@ -104,7 +106,7 @@ func TestAutocompleteFormat(t *testing.T) { { "second level nil struct field name", "{{ .Car2.", - []string{"{{ .Car2.Brand}}", "{{ .Car2.Stats.", "{{ .Car2.Extras}}", "{{ .Car2.Color}}", "{{ .Car2.Type}}"}, + []string{"{{ .Car2.Color}}", "{{ .Car2.Type}}", "{{ .Car2.Brand}}", "{{ .Car2.Stats.", "{{ .Car2.Extras}}"}, }, { "three level struct field name", @@ -134,8 +136,8 @@ func TestAutocompleteFormat(t *testing.T) { { "two variables struct field name", "{{ .Car.Brand }} {{ .Car.", - []string{"{{ .Car.Brand }} {{ .Car.Brand}}", "{{ .Car.Brand }} {{ .Car.Stats.", "{{ .Car.Brand }} {{ .Car.Extras}}", - "{{ .Car.Brand }} {{ .Car.Color}}", "{{ .Car.Brand }} {{ .Car.Type}}"}, + []string{"{{ .Car.Brand }} {{ .Car.Color}}", "{{ .Car.Brand }} {{ .Car.Type}}", "{{ .Car.Brand }} {{ .Car.Brand}}", + "{{ .Car.Brand }} {{ .Car.Stats.", "{{ .Car.Brand }} {{ .Car.Extras}}"}, }, { "only dot without variable", diff --git a/cmd/podman/machine/machine.go b/cmd/podman/machine/machine.go index 553f1ef7a..5a8a06b9d 100644 --- a/cmd/podman/machine/machine.go +++ b/cmd/podman/machine/machine.go @@ -115,7 +115,7 @@ func resolveEventSock() ([]string, error) { return err case info.IsDir(): return nil - case info.Type() != os.ModeSocket: + case !isUnixSocket(info): return nil case !re.MatchString(info.Name()): return nil diff --git a/cmd/podman/machine/machine_unix.go b/cmd/podman/machine/machine_unix.go new file mode 100644 index 000000000..213c24f8c --- /dev/null +++ b/cmd/podman/machine/machine_unix.go @@ -0,0 +1,12 @@ +//go:build linux || ignore || aix || ignore || android || ignore || darwin || ignore || freebsd || ignore || hurd || ignore || illumos || ignore || ios || ignore || netbsd || ignore || openbsd || ignore || solaris +// +build linux ignore aix ignore android ignore darwin ignore freebsd ignore hurd ignore illumos ignore ios ignore netbsd ignore openbsd ignore solaris + +package machine + +import ( + "os" +) + +func isUnixSocket(file os.DirEntry) bool { + return file.Type()&os.ModeSocket != 0 +} diff --git a/cmd/podman/machine/machine_windows.go b/cmd/podman/machine/machine_windows.go new file mode 100644 index 000000000..ffd5d8827 --- /dev/null +++ b/cmd/podman/machine/machine_windows.go @@ -0,0 +1,11 @@ +package machine + +import ( + "os" + "strings" +) + +func isUnixSocket(file os.DirEntry) bool { + // Assume a socket on Windows, since sock mode is not supported yet https://github.com/golang/go/issues/33357 + return !file.Type().IsDir() && strings.HasSuffix(file.Name(), ".sock") +} diff --git a/cmd/podman/play/kube.go b/cmd/podman/play/kube.go index 5fe059139..f5b121009 100644 --- a/cmd/podman/play/kube.go +++ b/cmd/podman/play/kube.go @@ -139,6 +139,15 @@ func init() { flags.StringVar(&kubeOptions.ContextDir, contextDirFlagName, "", "Path to top level of context directory") _ = kubeCmd.RegisterFlagCompletionFunc(contextDirFlagName, completion.AutocompleteDefault) + // NOTE: The service-container flag is marked as hidden as it + // is purely designed for running play-kube in systemd units. + // It is not something users should need to know or care about. + // + // Having a flag rather than an env variable is cleaner. + serviceFlagName := "service-container" + flags.BoolVar(&kubeOptions.ServiceContainer, serviceFlagName, false, "Starts a service container before all pods") + _ = flags.MarkHidden("service-container") + flags.StringVar(&kubeOptions.SignaturePolicy, "signature-policy", "", "`Pathname` of signature policy file (not usually used)") _ = flags.MarkHidden("signature-policy") diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh index 1ec18c861..83a81bd0a 100755 --- a/contrib/cirrus/runner.sh +++ b/contrib/cirrus/runner.sh @@ -246,6 +246,7 @@ function _run_altbuild() { # shellcheck disable=SC2154 msg "Performing alternate build: $ALT_NAME" msg "************************************************************" + set -x cd $GOSRC case "$ALT_NAME" in *Each*) @@ -260,7 +261,7 @@ function _run_altbuild() { context_dir=$(mktemp -d --tmpdir make-size-check.XXXXXXX) savedhead=$(git rev-parse HEAD) # Push to PR base. First run of the script will write size files - pr_base=$(git merge-base --fork-point origin/$DEST_BRANCH) + pr_base=$(git merge-base origin/$DEST_BRANCH HEAD) git checkout $pr_base hack/make-and-check-size $context_dir # pop back to PR, and run incremental makes. Subsequent script diff --git a/docs/source/Tutorials.rst b/docs/source/Tutorials.rst index 34a029484..c2cbcb8a9 100644 --- a/docs/source/Tutorials.rst +++ b/docs/source/Tutorials.rst @@ -6,7 +6,8 @@ Here are a number of useful tutorials to get you up and running with Podman. If * `Basic Setup and Use of Podman <https://github.com/containers/podman/blob/main/docs/tutorials/podman_tutorial.md>`_: Learn how to setup Podman and perform some basic commands with the utility. * `Basic Setup and Use of Podman in a Rootless environment <https://github.com/containers/podman/blob/main/docs/tutorials/rootless_tutorial.md>`_: The steps required to setup rootless Podman are enumerated. -* `Podman Mac/Windows tutorial <https://github.com/containers/podman/blob/main/docs/tutorials/mac_win_client.md>`_: Special setup for running the Podman remote client on a Mac or Windows PC and connecting to Podman running on a Linux VM are documented. +* `Podman for Windows <https://github.com/containers/podman/blob/main/docs/tutorials/podman-for-windows.md>`_: A guide to installing and using Podman on Windows. +* `Podman Remote Clients on Mac/Windows <https://github.com/containers/podman/blob/main/docs/tutorials/mac_win_client.md>`_: Advanced setup for connecting to a remote Linux system using the Podman remote client on Mac and Windows. * `How to sign and distribute container images using Podman <https://github.com/containers/podman/blob/main/docs/tutorials/image_signing.md>`_: Learn how to setup and use image signing with Podman. * `Podman remote-client tutorial <https://github.com/containers/podman/blob/main/docs/tutorials/remote_client.md>`_: A brief how-to on using the Podman remote-client. * `How to use libpod for custom/derivative projects <https://github.com/containers/podman/blob/main/docs/tutorials/podman-derivative-api.md>`_: How the libpod API can be used within your own project. diff --git a/docs/source/markdown/podman-auto-update.1.md b/docs/source/markdown/podman-auto-update.1.md index 755f46d70..992c87432 100644 --- a/docs/source/markdown/podman-auto-update.1.md +++ b/docs/source/markdown/podman-auto-update.1.md @@ -51,15 +51,6 @@ The `UPDATED` field indicates the availability of a new image with "pending". Change the default output format. This can be of a supported type like 'json' or a Go template. Valid placeholders for the Go template are listed below: -#### **--rollback** - -If restarting a systemd unit after updating the image has failed, rollback to using the previous image and restart the unit another time. Default is true. - -Please note that detecting if a systemd unit has failed is best done by the container sending the READY message via SDNOTIFY. This way, restarting the unit will wait until having received the message or a timeout kicked in. Without that, restarting the systemd unit may succeed even if the container has failed shortly after. - -For a container to send the READY message via SDNOTIFY it must be created with the `--sdnotify=container` option (see podman-run(1)). The application running inside the container can then execute `systemd-notify --ready` when ready or use the sdnotify bindings of the specific programming language (e.g., sd_notify(3)). - - | **Placeholder** | **Description** | | --------------- | -------------------------------------- | | .Unit | Name of the systemd unit | @@ -70,6 +61,14 @@ For a container to send the READY message via SDNOTIFY it must be created with t | .Policy | Auto-update policy of the container | | .Updated | Update status: true,false,failed | +#### **--rollback** + +If restarting a systemd unit after updating the image has failed, rollback to using the previous image and restart the unit another time. Default is true. + +Please note that detecting if a systemd unit has failed is best done by the container sending the READY message via SDNOTIFY. This way, restarting the unit will wait until having received the message or a timeout kicked in. Without that, restarting the systemd unit may succeed even if the container has failed shortly after. + +For a container to send the READY message via SDNOTIFY it must be created with the `--sdnotify=container` option (see podman-run(1)). The application running inside the container can then execute `systemd-notify --ready` when ready or use the sdnotify bindings of the specific programming language (e.g., sd_notify(3)). + ## EXAMPLES Autoupdate with registry policy diff --git a/docs/source/markdown/podman-history.1.md b/docs/source/markdown/podman-history.1.md index 16f1e48e6..d114e0523 100644 --- a/docs/source/markdown/podman-history.1.md +++ b/docs/source/markdown/podman-history.1.md @@ -17,6 +17,12 @@ set, the time of creation and size are printed out in a human readable format. The **--quiet** flag displays the ID of the image only when set and the **--format** flag is used to print the information using the Go template provided by the user. +## OPTIONS + +#### **--format**=*format* + +Alter the output for a format like 'json' or a Go template. + Valid placeholders for the Go template are listed below: | **Placeholder** | **Description** | @@ -28,12 +34,7 @@ Valid placeholders for the Go template are listed below: | .CreatedSince | Elapsed time since the image layer was created | | .Size | Size of layer on disk | | .Comment | Comment for the layer | -## OPTIONS - -Print the numeric IDs only (default *false*). -#### **--format**=*format* - -Alter the output for a format like 'json' or a Go template. +| .Tags | Image tags | #### **--help**, **-h** @@ -49,6 +50,8 @@ Do not truncate the output (default *false*). #### **--quiet**, **-q** +Print the numeric IDs only (default *false*). + ## EXAMPLES ``` diff --git a/docs/source/markdown/podman-system-connection-list.1.md b/docs/source/markdown/podman-system-connection-list.1.md index cc457860a..23784a319 100644 --- a/docs/source/markdown/podman-system-connection-list.1.md +++ b/docs/source/markdown/podman-system-connection-list.1.md @@ -20,10 +20,10 @@ Valid placeholders for the Go template listed below: | **Placeholder** | **Description** | | --------------- | ----------------------------------------------------------------------------- | -| *.Name* | Connection Name/Identifier | -| *.Identity* | Path to file containing SSH identity | -| *.URI* | URI to podman service. Valid schemes are ssh://[user@]*host*[:port]*Unix domain socket*[?secure=True], unix://*Unix domain socket*, and tcp://localhost[:*port*] | -| *.Default* | Indicates whether connection is the default | +| .Name | Connection Name/Identifier | +| .Identity | Path to file containing SSH identity | +| .URI | URI to podman service. Valid schemes are ssh://[user@]*host*[:port]*Unix domain socket*[?secure=True], unix://*Unix domain socket*, and tcp://localhost[:*port*] | +| .Default | Indicates whether connection is the default | ## EXAMPLE ``` diff --git a/docs/tutorials/mac_win_client.md b/docs/tutorials/mac_win_client.md index 159296d5e..553a38394 100644 --- a/docs/tutorials/mac_win_client.md +++ b/docs/tutorials/mac_win_client.md @@ -1,5 +1,9 @@ # Podman Remote clients for macOS and Windows +*** +**_NOTE:_** For running Podman on Windows, refer to the [Podman for Windows](podman-for-windows.md) guide, which uses the recommended approach of a Podman-managed Linux backend. For Mac, see the [Podman installation instructions](https://podman.io/getting-started/installation). This guide covers the advanced usage of Podman with a custom Linux VM or a remote external Linux system. +*** + ## Introduction The core Podman runtime environment can only run on Linux operating systems. But other operating systems can use the “remote client” to manage their containers to a Linux backend. This remote client is nearly identical to the standard Podman program. Certain functions that do not make sense for remote clients have been removed. For example, the “--latest” switch for container commands has been removed. diff --git a/docs/tutorials/podman-for-windows.md b/docs/tutorials/podman-for-windows.md new file mode 100644 index 000000000..bb9674774 --- /dev/null +++ b/docs/tutorials/podman-for-windows.md @@ -0,0 +1,417 @@ + + +Podman for Windows +================== + +While "containers are Linux," Podman also runs on Mac and Windows, where it +provides a native CLI and embeds a guest Linux system to launch your +containers. This guest is referred to as a Podman machine and is managed with +the `podman machine` command. On Windows, each Podman machine is backed by a +virtualized Windows System for Linux (WSLv2) distribution. The podman command +can be run directly from your Windows PowerShell (or CMD) prompt, where it +remotely communicates with the podman service running in the WSL environment. +Alternatively, you can access Podman directly from the WSL instance if you +prefer a Linux prompt and Linux tooling. In addition to command-line access, +Podman also listens for Docker API clients, supporting direct usage of +Docker-based tools and programmatic access from your language of choice. + +Prerequisites +------------- + +Since Podman uses WSL, you need a recent release of Windows 10 or Windows 11. +On x64, WSL requires build 18362 or later, and 19041 or later is required for +arm64 systems. Internally, WSL uses virtualization, so your system must +support and have hardware virtualization enabled. If you are running Windows +on a VM, you must have a VM that supports nested virtualization. + +It is also recommended to install the modern "Windows Terminal," which +provides a superior user experience to the standard PowerShell and CMD +prompts, as well as a WSL prompt, should you want it. + +You can install it by searching the Windows Store or by running the following +`winget` command: + +`winget install Microsoft.WindowsTerminal` + + +Installing Podman +----------------- + +Installing the Windows Podman client begins by downloading the Podman Windows +installer. The Windows installer is built with each Podman release and can be +downloaded from the official + [Github release page](https://github.com/containers/podman/releases). The +Windows installer file is named podman-v.#.#.#.msi, where the # symbols +represent the version number of Podman. Be sure to download a 4.1 or later +release for the capabilities discussed in this guide. + + + +Once downloaded, simply run the MSI file, and relaunch a new terminal. After +this point, podman.exe will be present on your PATH, and you will be able to run +the `podman machine init` command to create your first machine. + +`PS C:\Users\User> podman machine init` + +Automatic WSL Installation +-------------------------- + +If WSL has not been installed on your system, the first machine init command +will prompt a dialog to begin an automated install. If accepted, this process +will install the necessary Windows components, restart the system, and after +login, relaunch the machine creation process in a terminal window. Be sure to +wait a minute or two for the relaunch to occur, as Windows has a delay before +executing startup items. Alternatively, you can decline automatic installation +and install WSL manually. However, this will require additional download and +setup time. + +Machine Init Process +-------------------- + +After WSL is installed, the init command will install a minimal installation +of Fedora, customizing it to run podman. + +``` +PS C:\Users\User> podman machine init +Extracting compressed file +Importing operating system into WSL (this may take 5+ minutes on a new WSL install)... +Installing packages (this will take a while)... +Complete! +Configuring system... +Generating public/private ed25519 key pair. +Your identification has been saved in podman-machine-default +Your public key has been saved in podman-machine-default.pub +The key fingerprint is: +SHA256:RGTGg2Q/LX7ijN+mzu8+BzcS3cEWP6Hir6pYllJtceA root@WINPC +Machine init complete +To start your machine run: + + podman machine start +``` + + +Starting Machine +---------------- + +After the machine init process completes, it can then be started and stopped +as desired: + +``` +PS C:\Users\User> podman machine start + +Starting machine "podman-machine-default" + +This machine is currently configured in rootless mode. If your containers +require root permissions (e.g. ports < 1024), or if you run into compatibility +issues with non-podman clients, you can switch using the following command: + + podman machine set --rootful + +API forwarding listening on: npipe:////./pipe/docker_engine + +Docker API clients default to this address. You do not need to set DOCKER_HOST. +Machine "podman-machine-default" started successfully +``` + +First Podman Command +-------------------- + +From this point on, podman commands operate similarly to how they would on +Linux. + +For a quick working example with a small image, you can run the Linux date +command on PowerShell. + +``` +PS C:\Users\User> podman run ubi8-micro date +Thu May 5 21:56:42 UTC 2022 +``` + +Port Forwarding +--------------- + +Port forwarding also works as expected; ports will be bound against localhost +(127.0.0.1). Note: When running as rootless (the default), you must use a port +greater than 1023. See the Rooftull and Rootless section for more details. + +To launch httpd, you can run: + +``` +PS C:\Users\User> podman run --rm -d -p 8080:80 --name httpd docker.io/library/httpd +f708641300564a6caf90c145e64cd852e76f77f6a41699478bb83a162dceada9 +``` + +A curl command against localhost on the PowerShell prompt will return a +successful HTTP response: + +``` +PS C:\Users\User> curl http://localhost:8080/ -UseBasicParsing + +StatusCode : 200 +StatusDescription : OK +Content : <html><body><h1>It works!</h1></body></html> +``` + +As with Linux, to stop, run: + +`podman stop httpd` + + +Using API Forwarding +-------------------- + +API forwarding allows Docker API tools and clients to use podman as if it was +Docker. Provided there is no other service listening on the Docker API pipe; +no special settings will be required. + +``` +PS C:\Users\User> .\docker.exe run -it fedora echo "Hello Podman!" +Hello Podman! +``` + +Otherwise, after starting the machine, you will be notified of an environment +variable you can set for tools to point to podman. Alternatively, you can shut +down both the conflicting service and podman, then finally run `podman machine +start` to restart, which should grab the Docker API address. + + +``` +Another process was listening on the default Docker API pipe address. +You can still connect Docker API clients by setting DOCKER HOST using the +following PowerShell command in your terminal session: + + $Env:DOCKER_HOST = 'npipe:////./pipe/podman-machine-default' + +Or in a classic CMD prompt: + + set DOCKER_HOST = 'npipe:////./pipe/podman-machine-default' + +Alternatively, terminate the other process and restart podman machine. +Machine "podman-machine-default" started successfully + +PS C:\Users\User> $Env:DOCKER_HOST = 'npipe:////./pipe/podman-machine-default' +PS C:\Users\User>.\docker.exe version --format '{{(index .Server.Components 0).Name}}' +Podman Engine +``` + +Rootfull & Rootless +------------------- + +On the embedded WSL Linux distro, podman can either be run under the root user +(rootful) or a non-privileged user (rootless). For behavioral consistency with +Podman on Linux, rootless is the default. Note: Rootfull and Rootless +containers are distinct and isolated from one another. Podman commands against +one (e.g., podman ps) will not represent results/state for the other. + +While most containers run fine in a rootless setting, you may find a case +where the container only functions with root privileges. If this is the case, +you can switch the machine to rootful by stopping it and using the set +command: + +``` +podman machine stop +podman machine set --rootful +``` + +To restore rootless execution, set rootful to false: + +``` +Podman machine stop +Podman machine set --rootful=false +``` + +Another case in which you may wish to use rootful execution is binding a port +less than 1024. However, future versions of podman will likely drop this to a +lower number to improve compatibility with defaults on system port services (such +as MySQL) + +Volume Mounting +--------------- + +New in Podman v4.1 is the ability to perform volume mounts from Windows paths into a +Linux container. This supports several notation schemes, including: + +Windows Style Paths: + +`podman run -it c:\Users\User\myfolder:/myfolder ubi8-micro ls /myfolder` + +Unixy Windows Paths: + +`podman run -it /c/Users/User/myfolder:/myfolder ubi8-micro ls /myfolder` + +Linux paths local to the WSL filesystem: + +`podman run -it /var/myfolder:/myfolder ubi-micro ls /myfolder` + +All of the above conventions work, whether running on a Windows prompt or the +WSL Linux shell. Although when using Windows paths on Linux, appropriately quote +or escape the Windows path portion of the argument. + + +Listing Podman Machine(s) +------------------------- + +To list the available podman machine instances and their current resource +usage, use the `podman machine ls` command: + +``` +PS C:\Users\User> podman machine ls + + +NAME VM TYPE CREATED LAST UP CPUS MEMORY DISK SIZE +podman-machine-default wsl 2 hours ago Currently running 4 331.1MB 768MB +``` + +Since WSL shares the same virtual machine and Linux kernel across multiple +distributions, the CPU and Memory values represent the total resources shared +across running systems. The opposite applies to the Disk value. It is +independent and represents the amount of storage for each individual +distribution. + + +Accessing the Podman Linux Environment +-------------------------------------- + +While using the podman.exe client on the Windows environment provides a +seamless native experience supporting the usage of local desktop tools and +APIs, there are a few scenarios in which you may wish to access the Linux +environment: + ++ Updating to the latest stable packages on the embedded Fedora instance ++ Using Linux development tools directly ++ Using a workflow that relies on EXT4 filesystem performance or behavior + semantics + +There are three mechanisms to access the embedded WSL distribution: +1. SSH using `podman machine ssh` +2. WSL command on the Windows PowerShell prompt +3. Windows Terminal Integration + +### Using SSH + +SSH access provides a similar experience as Podman on Mac. It immediately +drops you into the appropriate user based on your machine's rootful/rootless +configuration (root in the former, 'user' in the latter). The --username +option can be used to override with a specific user. + +An example task using SSH is updating your Linux environment to pull down the +latest OS bugfixes: + +`podman machine ssh sudo dnf upgrade -y` + +### Using the WSL Command + +The `wsl` command provides direct access to the Linux system but enters the +shell as root first. This is due to design limitations of WSL, where running +systemd (Linux's system services) requires the usage of a privileged process +namespace. + +Unless you have no other distributions of WSL installed, it's recommended to +use the `-d` option with the name of your podman machine (podman-machine-default +is the default) + +``` +PS C:\Users\User> wsl -d podman-machine-default +``` + +You will be automatically entered into a nested process namespace where +systemd is running. If you need to access the parent namespace, hit `ctrl-d` +or type exit. This also means to log out, you need to exit twice. + +``` +[root@WINPC /]# podman --version +podman version 4.1.0 +``` + + +To access commands as the non-privileged user (rootless podman), you must +first type `su user`. Alternatively, you can prefix the `wsl` command to use the +special `enterns`: + +``` +wsl -d podman-machine-default enterns su user +[user@WINPC /]$ id +uid=1000(user) gid=1000(user) groups=1000(user),10(wheel) +``` + +Likewise, running commands as root without entering a prompt should also be +prefixed with `enterns`. + +`wsl -d podman-machine-default enterns systemctl status` + +Accessing the WSL instance as a specific user using `wsl -u` or using inline +commands without `enterns` is not recommended since commands will execute +against the incorrect namespace. + +### Using Windows Terminal Integration + +Entering WSL as root is a 2-click operation. Simply click the drop-down tag, +and pick 'podman-machine-default,' where you will be entered directly as root. + + + +As before, to switch to a non-privileged user for rootless podman commands, +type `su user`. + +``` +[root@WINPC /]# su user +[user@WINPC /]$ podman info --format '{{.Store.RunRoot}}' +/run/user/1000/containers +``` + +Stopping a Podman Machine +------------------------- + +To stop a running podman machine, use the `podman machine stop` command: + +``` +PS C:\Users\User> podman machine stop +Machine "podman-machine-default" stopped successfully +``` + +Removing a Podman Machine +------------------------- + +To remove a machine, use the `podman machine rm` command: + +``` +PS C:\Users\User> podman machine rm + +The following files will be deleted: + +C:\Users\User\.ssh\podman-machine-default +C:\Users\User\.ssh\podman-machine-default.pub +C:\Users\User\.local\share\containers\podman\machine\wsl\podman-machine-default_fedora-35-x86_64.tar +C:\Users\User\.config\containers\podman\machine\wsl\podman-machine-default.json +C:\Users\User\.local\share\containers\podman\machine\wsl\wsldist\podman-machine-default + + +Are you sure you want to continue? [y/N] y +``` + + + +Troubleshooting +--------------- + +Recovering from a failed auto-installation of WSL + +If auto-install fails and retrying is unsuccessful, you can attempt to reset +your WSL system state and perform a manual WSL installation using the `wsl +--install command`. To do so, perform the following steps: + +1. Launch PowerShell as administrator + ``` + Start-Process powershell -Verb RunAs + ``` +2. Disable WSL Features + ``` + dism.exe /online /disable-feature /featurename:Microsoft-Windows-Subsystem-Linux /norestart + dism.exe /online /enable-feature /featurename:VirtualMachinePlatform /norestart + ``` +3. Reboot +4. Run manual WSL install + ``` + wsl --install + ``` +5. Continue with podman machine init diff --git a/docs/tutorials/podman-win-install.jpg b/docs/tutorials/podman-win-install.jpg Binary files differnew file mode 100644 index 000000000..cf1b3ca86 --- /dev/null +++ b/docs/tutorials/podman-win-install.jpg diff --git a/docs/tutorials/podman-wsl-term.jpg b/docs/tutorials/podman-wsl-term.jpg Binary files differnew file mode 100644 index 000000000..a01bea84e --- /dev/null +++ b/docs/tutorials/podman-wsl-term.jpg @@ -12,19 +12,19 @@ require ( github.com/containernetworking/cni v1.1.0 github.com/containernetworking/plugins v1.1.1 github.com/containers/buildah v1.26.1 - github.com/containers/common v0.48.0 + github.com/containers/common v0.48.1-0.20220512112240-7536bf6ff9b1 github.com/containers/conmon v2.0.20+incompatible - github.com/containers/image/v5 v5.21.1 + github.com/containers/image/v5 v5.21.2-0.20220511203756-fe4fd4ed8be4 github.com/containers/ocicrypt v1.1.4-0.20220428134531-566b808bdf6f github.com/containers/psgo v1.7.2 - github.com/containers/storage v1.40.2 + github.com/containers/storage v1.41.1-0.20220511210719-cacc3325a9c8 github.com/coreos/go-systemd/v22 v22.3.2 github.com/coreos/stream-metadata-go v0.0.0-20210225230131-70edb9eb47b3 github.com/cyphar/filepath-securejoin v0.2.3 github.com/davecgh/go-spew v1.1.1 github.com/digitalocean/go-qemu v0.0.0-20210326154740-ac9e0b687001 github.com/docker/distribution v2.8.1+incompatible - github.com/docker/docker v20.10.15+incompatible + github.com/docker/docker v20.10.16+incompatible github.com/docker/go-connections v0.4.1-0.20210727194412-58542c764a11 github.com/docker/go-plugins-helpers v0.0.0-20211224144127-6eecb7beb651 github.com/docker/go-units v0.4.0 @@ -47,7 +47,7 @@ require ( github.com/onsi/gomega v1.19.0 github.com/opencontainers/go-digest v1.0.0 github.com/opencontainers/image-spec v1.0.3-0.20220114050600-8b9d41f48198 - github.com/opencontainers/runc v1.1.1 + github.com/opencontainers/runc v1.1.2 github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab github.com/opencontainers/runtime-tools v0.9.1-0.20220110225228-7e2d60f1e41f github.com/opencontainers/selinux v1.10.1 @@ -339,12 +339,14 @@ github.com/containernetworking/plugins v1.1.1 h1:+AGfFigZ5TiQH00vhR8qPeSatj53eNG github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19sZPp3ry5uHSkI4LPxV8= github.com/containers/buildah v1.26.1 h1:D65Vuo+orsI14WWtJhSX6KrpgBBa7+hveVWevzG8p8E= github.com/containers/buildah v1.26.1/go.mod h1:CsWSG8OpJd8v3mlLREJzVAOBgC93DjRNALUVHoi8QsY= -github.com/containers/common v0.48.0 h1:997nnXBZ+eNpfSM7L4SxhhZubQrfEyw3jRyNMTSsNlw= github.com/containers/common v0.48.0/go.mod h1:zPLZCfLXfnd1jI0QRsD4By54fP4k1+ifQs+tulIe3o0= +github.com/containers/common v0.48.1-0.20220512112240-7536bf6ff9b1 h1:U+2rYjzRCvI3WRSFf+Rohtu7jRgk/VhJjjFHbU6j0Sk= +github.com/containers/common v0.48.1-0.20220512112240-7536bf6ff9b1/go.mod h1:h8YZVXePE7UViJQ3fPWpYAaeDNYBCzGtL5dA3N8yfT8= github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg= github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= -github.com/containers/image/v5 v5.21.1 h1:Cr3zw2f0FZs4SCkdGlc8SN/mpcmg2AKG4OUuDbeGS/Q= github.com/containers/image/v5 v5.21.1/go.mod h1:zl35egpcDQa79IEXIuoUe1bW+D1pdxRxYjNlyb3YiXw= +github.com/containers/image/v5 v5.21.2-0.20220511203756-fe4fd4ed8be4 h1:9yDGjKniCxCIVJwdiUHGTjguGJUcntDtWLUIz+LhyzY= +github.com/containers/image/v5 v5.21.2-0.20220511203756-fe4fd4ed8be4/go.mod h1:OsX9sFexyGF0FCNAjfcVFv3IwMqDyLyV/WQY/roLPcE= github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a h1:spAGlqziZjCJL25C6F1zsQY05tfCKE9F5YwtEWWe6hU= github.com/containers/libtrust v0.0.0-20200511145503-9c3a6c22cd9a/go.mod h1:9rfv8iPl1ZP7aqh9YA68wnZv2NUDbXdcdPHVz0pFbPY= github.com/containers/ocicrypt v1.0.1/go.mod h1:MeJDzk1RJHv89LjsH0Sp5KTY3ZYkjXO/C+bKAeWFIrc= @@ -359,8 +361,10 @@ github.com/containers/psgo v1.7.2/go.mod h1:SLpqxsPOHtTqRygjutCPXmeU2PoEFzV3gzJp github.com/containers/storage v1.37.0/go.mod h1:kqeJeS0b7DO2ZT1nVWs0XufrmPFbgV3c+Q/45RlH6r4= github.com/containers/storage v1.38.0/go.mod h1:lBzt28gAk5ADZuRtwdndRJyqX22vnRaXmlF+7ktfMYc= github.com/containers/storage v1.40.0/go.mod h1:zUyPC3CFIGR1OhY1CKkffxgw9+LuH76PGvVcFj38dgs= -github.com/containers/storage v1.40.2 h1:GUlHaGnrs1JOEwv6YEvkQdgYXOXZdU1Angy4wgWNgF8= github.com/containers/storage v1.40.2/go.mod h1:zUyPC3CFIGR1OhY1CKkffxgw9+LuH76PGvVcFj38dgs= +github.com/containers/storage v1.41.0/go.mod h1:Pb0l5Sm/89kolX3o2KolKQ5cCHk5vPNpJrhNaLcdS5s= +github.com/containers/storage v1.41.1-0.20220511210719-cacc3325a9c8 h1:4XdTbn3iVIr1+kN5srZND2G3/Q3hJiZSZZtKdL6r9jg= +github.com/containers/storage v1.41.1-0.20220511210719-cacc3325a9c8/go.mod h1:Pb0l5Sm/89kolX3o2KolKQ5cCHk5vPNpJrhNaLcdS5s= github.com/coreos/bbolt v1.3.2/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/etcd v3.3.13+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -427,8 +431,9 @@ github.com/docker/docker v1.4.2-0.20190924003213-a8608b5b67c7/go.mod h1:eEKB0N0r github.com/docker/docker v20.10.3-0.20220208084023-a5c757555091+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.12+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker v20.10.14+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= -github.com/docker/docker v20.10.15+incompatible h1:dk9FewY/9Xwm4ay/HViEEHSQuM/kL4F+JaG6GQdgmGo= github.com/docker/docker v20.10.15+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= +github.com/docker/docker v20.10.16+incompatible h1:2Db6ZR/+FUR3hqPMwnogOPHFn405crbpxvWzKovETOQ= +github.com/docker/docker v20.10.16+incompatible/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/docker-credential-helpers v0.6.3/go.mod h1:WRaJzqw3CTB9bk10avuGsjVBZsD05qeibJ1/TYlvc0Y= github.com/docker/docker-credential-helpers v0.6.4 h1:axCks+yV+2MR3/kZhAmy07yC56WZ2Pwu/fKWtKuZB0o= github.com/docker/docker-credential-helpers v0.6.4/go.mod h1:ofX3UI0Gz1TteYBjtgs07O36Pyasyp66D2uKT7H8W1c= @@ -831,8 +836,9 @@ github.com/klauspost/compress v1.13.5/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.14.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.2 h1:3WH+AG7s2+T8o3nrM/8u2rdqUEcQhmga7smjrT41nAw= github.com/klauspost/compress v1.15.2/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.4 h1:1kn4/7MepF/CHmYub99/nNX8az0IJjfSOU/jbnTVfqQ= +github.com/klauspost/compress v1.15.4/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -1049,8 +1055,9 @@ github.com/opencontainers/runc v1.0.0-rc93/go.mod h1:3NOsor4w32B2tC0Zbl8Knk4Wg84 github.com/opencontainers/runc v1.0.2/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runc v1.0.3/go.mod h1:aTaHFFwQXuA71CiyxOdFFIorAoemI04suvGRQFzWTD0= github.com/opencontainers/runc v1.1.0/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= -github.com/opencontainers/runc v1.1.1 h1:PJ9DSs2sVwE0iVr++pAHE6QkS9tzcVWozlPifdwMgrU= github.com/opencontainers/runc v1.1.1/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= +github.com/opencontainers/runc v1.1.2 h1:2VSZwLx5k/BfsBxMMipG/LYUnmqOD/BPkIVgQUcTlLw= +github.com/opencontainers/runc v1.1.2/go.mod h1:Tj1hFw6eFWp/o33uxGf5yF2BX5yz2Z6iptFpuvbbKqc= github.com/opencontainers/runtime-spec v0.1.2-0.20190507144316-5b71a03e2700/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.1/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-spec v1.0.2-0.20190207185410-29686dbc5559/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= diff --git a/hack/buildah-vendor-treadmill b/hack/buildah-vendor-treadmill index 0aa4245c5..d579a180a 100755 --- a/hack/buildah-vendor-treadmill +++ b/hack/buildah-vendor-treadmill @@ -14,9 +14,10 @@ use warnings; use File::Temp qw(tempfile); use JSON; use LWP::UserAgent; +use POSIX qw(strftime); (our $ME = $0) =~ s|.*/||; -our $VERSION = '0.2'; +our $VERSION = '0.3'; # For debugging, show data structures using DumpTree($var) #use Data::TreeDumper; $Data::TreeDumper::Displayaddress = 0; @@ -65,7 +66,7 @@ eval ' sub usage { print <<"END_USAGE"; -Usage: $ME [OPTIONS] [--sync | --pick ] +Usage: $ME [OPTIONS] [--sync | --pick | --reset ] $ME is (2022-04-20) **EXPERIMENTAL** @@ -82,6 +83,9 @@ Call me with one of two options: --pick Used for really-truly vendoring in a new buildah; will cherry-pick a commit on your buildah-vendor working branch + --reset Used after vendoring buildah into main, when there + really aren't any buildah patches to keep rolling. + For latest documentation and best practices, please see: $Docs_URL @@ -105,8 +109,9 @@ our $NOT = ''; # print "blahing the blah$NOT\n" if $debug sub handle_opts { use Getopt::Long; GetOptions( - 'sync' => sub { $action{sync}++ }, - 'pick' => sub { $action{pick}++ }, + 'sync' => sub { $action{sync}++ }, + 'pick' => sub { $action{pick}++ }, + 'reset' => sub { $action{reset}++ }, 'force-old-main' => \$force_old_main, 'force-testing' => \$force_testing, @@ -183,8 +188,10 @@ sub do_sync { pull_main(); git('checkout', '-q', $current_branch); - # Preserve local patches - git('format-patch', "--output=$Patch_File", 'HEAD^'); + # Preserve local patches. --always will generate empty patches (e.g., + # after a buildah vendor when everything is copacetic); --no-signature + # prevents a buildup of "-- 2.35" (git version) lines at the end. + git('format-patch', '--always', '--no-signature', "--output=$Patch_File", 'HEAD^'); progress("Treadmill patches saved to $Patch_File"); # @@ -250,20 +257,11 @@ END_FAIL_INSTRUCTIONS } # Commit everything. - git('commit', '-as', '-m', <<"END_COMMIT_MESSAGE"); -[DO NOT MERGE] vendor in buildah \@ $buildah_new - -This is a JUNK COMMIT from $ME v$VERSION. - -DO NOT MERGE. This is just a way to keep the buildah-podman -vendoring in sync. Refer to: - - $Docs_URL -END_COMMIT_MESSAGE + git_commit_buildah($buildah_new); # And, finally, this has the highest possibility of failing progress('Reapplying preserved patches'); - git('am', $Patch_File); + git('am', '--empty=keep', $Patch_File); # It worked! Clean up: remove our local die() handler and the patch file undef $SIG{__DIE__}; @@ -638,6 +636,38 @@ END_EDIT_SCRIPT # END pick and its helpers ############################################################################### +# BEGIN reset and its helpers + +sub do_reset { + my $current_branch = git_current_branch(); + + # Make sure side branch == main (i.e., there are no commits on the branch) + if (git('rev-parse', $current_branch) ne git('rev-parse', 'main')) { + die "$ME: for --reset, $current_branch must == main\n"; + } + + # Pull main, and pivot back to this branch + pull_main(); + git('checkout', '-q', $current_branch); + + git('rebase', '--empty=keep', 'main'); + git_commit_buildah('[none]'); + + my $ymd = strftime("%Y-%m-%d", localtime); + git('commit', '--allow-empty', '-s', '-m' => <<"END_COMMIT_MESSAGE"); +$Treadmill_PR_Title + +As you run --sync, please update this commit message with your +actual changes. + +Changes since $ymd: +END_COMMIT_MESSAGE + + progress("Done. You may now run --sync.\n"); +} + +# END reset and its helpers +############################################################################### # BEGIN general-purpose helpers ############## @@ -728,6 +758,24 @@ sub git_upstream { die "$ME: did not find a remote with 'github.com/containers/podman'\n"; } +######################## +# git_commit_buildah # Do the buildah commit +######################## +sub git_commit_buildah { + my $buildah_version = shift; + + # When called by --reset, this can be empty + git('commit', '-as', '--allow-empty', '-m', <<"END_COMMIT_MESSAGE"); +DO NOT MERGE: vendor in buildah \@ $buildah_version + +This is a JUNK COMMIT from $ME v$VERSION. + +DO NOT MERGE! This is just a way to keep the buildah-podman +vendoring in sync. Refer to: + + $Docs_URL +END_COMMIT_MESSAGE +} ######### # git # Run a git command diff --git a/libpod/container.go b/libpod/container.go index d7af9a100..64b4453fb 100644 --- a/libpod/container.go +++ b/libpod/container.go @@ -211,6 +211,14 @@ type ContainerState struct { // network and an interface names NetInterfaceDescriptions ContainerNetworkDescriptions `json:"networkDescriptions,omitempty"` + // Service indicates that container is the service container of a + // service. A service consists of one or more pods. The service + // container is started before all pods and is stopped when the last + // pod stops. The service container allows for tracking and managing + // the entire life cycle of service which may be started via + // `podman-play-kube`. + Service Service + // containerPlatformState holds platform-specific container state. containerPlatformState diff --git a/libpod/container_config.go b/libpod/container_config.go index 371a1dec0..3e85ad4d5 100644 --- a/libpod/container_config.go +++ b/libpod/container_config.go @@ -382,6 +382,9 @@ type ContainerMiscConfig struct { // IsInfra is a bool indicating whether this container is an infra container used for // sharing kernel namespaces in a pod IsInfra bool `json:"pause"` + // IsService is a bool indicating whether this container is a service container used for + // tracking the life cycle of K8s service. + IsService bool `json:"isService"` // SdNotifyMode tells libpod what to do with a NOTIFY_SOCKET if passed SdNotifyMode string `json:"sdnotifyMode,omitempty"` // Systemd tells libpod to setup the container in systemd mode, a value of nil denotes false diff --git a/libpod/container_inspect.go b/libpod/container_inspect.go index 76e0e9e13..5d809644d 100644 --- a/libpod/container_inspect.go +++ b/libpod/container_inspect.go @@ -171,6 +171,7 @@ func (c *Container) getContainerInspectData(size bool, driverData *define.Driver Mounts: inspectMounts, Dependencies: c.Dependencies(), IsInfra: c.IsInfra(), + IsService: c.isService(), } if c.state.ConfigPath != "" { diff --git a/libpod/container_validate.go b/libpod/container_validate.go index c6c9a4c6d..d939c94e6 100644 --- a/libpod/container_validate.go +++ b/libpod/container_validate.go @@ -1,6 +1,8 @@ package libpod import ( + "fmt" + "github.com/containers/podman/v4/libpod/define" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -27,6 +29,12 @@ func (c *Container) validate() error { return errors.Wrapf(define.ErrInvalidArg, "must set root filesystem source to either image or rootfs") } + // A container cannot be marked as an infra and service container at + // the same time. + if c.IsInfra() && c.isService() { + return fmt.Errorf("cannot be infra and service container at the same time: %w", define.ErrInvalidArg) + } + // Cannot make a network namespace if we are joining another container's // network namespace if c.config.CreateNetNS && c.config.NetNsCtr != "" { diff --git a/libpod/define/container_inspect.go b/libpod/define/container_inspect.go index 6cdffb8b7..e7b82d654 100644 --- a/libpod/define/container_inspect.go +++ b/libpod/define/container_inspect.go @@ -683,6 +683,7 @@ type InspectContainerData struct { NetworkSettings *InspectNetworkSettings `json:"NetworkSettings"` Namespace string `json:"Namespace"` IsInfra bool `json:"IsInfra"` + IsService bool `json:"IsService"` Config *InspectContainerConfig `json:"Config"` HostConfig *InspectContainerHostConfig `json:"HostConfig"` } diff --git a/libpod/info.go b/libpod/info.go index 321680a81..bc49a6cc9 100644 --- a/libpod/info.go +++ b/libpod/info.go @@ -406,26 +406,25 @@ func getCPUUtilization() (*define.CPUUsage, error) { } defer f.Close() scanner := bufio.NewScanner(f) - // Read firt line of /proc/stat + // Read first line of /proc/stat that has entries for system ("cpu" line) for scanner.Scan() { break } // column 1 is user, column 3 is system, column 4 is idle - stats := strings.Split(scanner.Text(), " ") + stats := strings.Fields(scanner.Text()) return statToPercent(stats) } func statToPercent(stats []string) (*define.CPUUsage, error) { - // There is always an extra space between cpu and the first metric - userTotal, err := strconv.ParseFloat(stats[2], 64) + userTotal, err := strconv.ParseFloat(stats[1], 64) if err != nil { return nil, errors.Wrapf(err, "unable to parse user value %q", stats[1]) } - systemTotal, err := strconv.ParseFloat(stats[4], 64) + systemTotal, err := strconv.ParseFloat(stats[3], 64) if err != nil { return nil, errors.Wrapf(err, "unable to parse system value %q", stats[3]) } - idleTotal, err := strconv.ParseFloat(stats[5], 64) + idleTotal, err := strconv.ParseFloat(stats[4], 64) if err != nil { return nil, errors.Wrapf(err, "unable to parse idle value %q", stats[4]) } diff --git a/libpod/info_test.go b/libpod/info_test.go index 909b573c0..b0e4bf8c0 100644 --- a/libpod/info_test.go +++ b/libpod/info_test.go @@ -20,7 +20,7 @@ func Test_statToPercent(t *testing.T) { }{ { name: "GoodParse", - args: args{in0: []string{"cpu", " ", "33628064", "27537", "9696996", "1314806705", "588142", "4775073", "2789228", "0", "598711", "0"}}, + args: args{in0: []string{"cpu", "33628064", "27537", "9696996", "1314806705", "588142", "4775073", "2789228", "0", "598711", "0"}}, want: &define.CPUUsage{ UserPercent: 2.48, SystemPercent: 0.71, @@ -30,19 +30,19 @@ func Test_statToPercent(t *testing.T) { }, { name: "BadUserValue", - args: args{in0: []string{"cpu", " ", "k", "27537", "9696996", "1314806705", "588142", "4775073", "2789228", "0", "598711", "0"}}, + args: args{in0: []string{"cpu", "k", "27537", "9696996", "1314806705", "588142", "4775073", "2789228", "0", "598711", "0"}}, want: nil, wantErr: assert.Error, }, { name: "BadSystemValue", - args: args{in0: []string{"cpu", " ", "33628064", "27537", "k", "1314806705", "588142", "4775073", "2789228", "0", "598711", "0"}}, + args: args{in0: []string{"cpu", "33628064", "27537", "k", "1314806705", "588142", "4775073", "2789228", "0", "598711", "0"}}, want: nil, wantErr: assert.Error, }, { name: "BadIdleValue", - args: args{in0: []string{"cpu", " ", "33628064", "27537", "9696996", "k", "588142", "4775073", "2789228", "0", "598711", "0"}}, + args: args{in0: []string{"cpu", "33628064", "27537", "9696996", "k", "588142", "4775073", "2789228", "0", "598711", "0"}}, want: nil, wantErr: assert.Error, }, diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go index c232702e9..6aa7ce6dc 100644 --- a/libpod/oci_conmon_linux.go +++ b/libpod/oci_conmon_linux.go @@ -36,7 +36,6 @@ import ( "github.com/containers/podman/v4/utils" "github.com/containers/storage/pkg/homedir" pmount "github.com/containers/storage/pkg/mount" - "github.com/coreos/go-systemd/v22/daemon" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" @@ -1279,19 +1278,6 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co // conmon not having a pid file is a valid state, so don't set it if we don't have it logrus.Infof("Got Conmon PID as %d", conmonPID) ctr.state.ConmonPID = conmonPID - - // Send the MAINPID via sdnotify if needed. - switch ctr.config.SdNotifyMode { - case define.SdNotifyModeContainer, define.SdNotifyModeIgnore: - // Nothing to do or conmon takes care of it already. - - default: - if sent, err := daemon.SdNotify(false, fmt.Sprintf("MAINPID=%d", conmonPID)); err != nil { - logrus.Errorf("Notifying systemd of Conmon PID: %v", err) - } else if sent { - logrus.Debugf("Notify MAINPID sent successfully") - } - } } runtimeRestoreDuration := func() int64 { diff --git a/libpod/options.go b/libpod/options.go index 9b83cb76a..feb89510f 100644 --- a/libpod/options.go +++ b/libpod/options.go @@ -1,6 +1,7 @@ package libpod import ( + "fmt" "net" "os" "path/filepath" @@ -1477,7 +1478,7 @@ func WithCreateCommand(cmd []string) CtrCreateOption { } } -// withIsInfra allows us to dfferentiate between infra containers and regular containers +// withIsInfra allows us to dfferentiate between infra containers and other containers // within the container config func withIsInfra() CtrCreateOption { return func(ctr *Container) error { @@ -1491,6 +1492,20 @@ func withIsInfra() CtrCreateOption { } } +// WithIsService allows us to dfferentiate between service containers and other container +// within the container config +func WithIsService() CtrCreateOption { + return func(ctr *Container) error { + if ctr.valid { + return define.ErrCtrFinalized + } + + ctr.config.IsService = true + + return nil + } +} + // WithCreateWorkingDir tells Podman to create the container's working directory // if it does not exist. func WithCreateWorkingDir() CtrCreateOption { @@ -2081,6 +2096,27 @@ func WithInfraContainer() PodCreateOption { } } +// WithServiceContainer associates the specified service container ID with the pod. +func WithServiceContainer(id string) PodCreateOption { + return func(pod *Pod) error { + if pod.valid { + return define.ErrPodFinalized + } + + ctr, err := pod.runtime.LookupContainer(id) + if err != nil { + return fmt.Errorf("looking up service container: %w", err) + } + + if err := ctr.addServicePodLocked(pod.ID()); err != nil { + return fmt.Errorf("associating service container %s with pod %s: %w", id, pod.ID(), err) + } + + pod.config.ServiceContainerID = id + return nil + } +} + // WithVolatile sets the volatile flag for the container storage. // The option can potentially cause data loss when used on a container that must survive a machine reboot. func WithVolatile() CtrCreateOption { diff --git a/libpod/pod.go b/libpod/pod.go index 2211d5be7..3c8dc43d4 100644 --- a/libpod/pod.go +++ b/libpod/pod.go @@ -64,6 +64,13 @@ type PodConfig struct { HasInfra bool `json:"hasInfra,omitempty"` + // ServiceContainerID is the main container of a service. A service + // consists of one or more pods. The service container is started + // before all pods and is stopped when the last pod stops. + // The service container allows for tracking and managing the entire + // life cycle of service which may be started via `podman-play-kube`. + ServiceContainerID string `json:"serviceContainerID,omitempty"` + // Time pod was created CreatedTime time.Time `json:"created"` diff --git a/libpod/pod_api.go b/libpod/pod_api.go index 73b28822b..eede896a9 100644 --- a/libpod/pod_api.go +++ b/libpod/pod_api.go @@ -75,6 +75,10 @@ func (p *Pod) Start(ctx context.Context) (map[string]error, error) { return nil, define.ErrPodRemoved } + if err := p.maybeStartServiceContainer(ctx); err != nil { + return nil, err + } + // Before "regular" containers start in the pod, all init containers // must have run and exited successfully. if err := p.startInitContainers(ctx); err != nil { @@ -197,6 +201,11 @@ func (p *Pod) stopWithTimeout(ctx context.Context, cleanup bool, timeout int) (m if len(ctrErrors) > 0 { return ctrErrors, errors.Wrapf(define.ErrPodPartialFail, "error stopping some containers") } + + if err := p.maybeStopServiceContainer(); err != nil { + return nil, err + } + return nil, nil } @@ -297,6 +306,10 @@ func (p *Pod) Cleanup(ctx context.Context) (map[string]error, error) { return ctrErrors, errors.Wrapf(define.ErrPodPartialFail, "error cleaning up some containers") } + if err := p.maybeStopServiceContainer(); err != nil { + return nil, err + } + return nil, nil } @@ -443,6 +456,10 @@ func (p *Pod) Restart(ctx context.Context) (map[string]error, error) { return nil, define.ErrPodRemoved } + if err := p.maybeStartServiceContainer(ctx); err != nil { + return nil, err + } + allCtrs, err := p.runtime.state.PodContainers(p) if err != nil { return nil, err @@ -530,6 +547,11 @@ func (p *Pod) Kill(ctx context.Context, signal uint) (map[string]error, error) { if len(ctrErrors) > 0 { return ctrErrors, errors.Wrapf(define.ErrPodPartialFail, "error killing some containers") } + + if err := p.maybeStopServiceContainer(); err != nil { + return nil, err + } + return nil, nil } diff --git a/libpod/runtime_img.go b/libpod/runtime_img.go index 54eadf6b8..b13482722 100644 --- a/libpod/runtime_img.go +++ b/libpod/runtime_img.go @@ -40,7 +40,7 @@ func (r *Runtime) RemoveContainersForImageCallback(ctx context.Context) libimage if ctr.config.IsInfra { pod, err := r.state.Pod(ctr.config.Pod) if err != nil { - return errors.Wrapf(err, "container %s is in pod %s, but pod cannot be retrieved", ctr.ID(), pod.ID()) + return errors.Wrapf(err, "container %s is in pod %s, but pod cannot be retrieved", ctr.ID(), ctr.config.Pod) } if err := r.removePod(ctx, pod, true, true, timeout); err != nil { return errors.Wrapf(err, "removing image %s: container %s using image could not be removed", imageID, ctr.ID()) diff --git a/libpod/runtime_pod_linux.go b/libpod/runtime_pod_linux.go index 62ec7df60..dcc3a044f 100644 --- a/libpod/runtime_pod_linux.go +++ b/libpod/runtime_pod_linux.go @@ -380,6 +380,10 @@ func (r *Runtime) removePod(ctx context.Context, p *Pod, removeCtrs, force bool, } } + if err := p.maybeRemoveServiceContainer(); err != nil { + return err + } + // Remove pod from state if err := r.state.RemovePod(p); err != nil { if removalErr != nil { diff --git a/libpod/service.go b/libpod/service.go new file mode 100644 index 000000000..ad147e87b --- /dev/null +++ b/libpod/service.go @@ -0,0 +1,213 @@ +package libpod + +import ( + "context" + "fmt" + + "github.com/containers/podman/v4/libpod/define" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// A service consists of one or more pods. The service container is started +// before all pods and is stopped when the last pod stops. The service +// container allows for tracking and managing the entire life cycle of service +// which may be started via `podman-play-kube`. +type Service struct { + // Pods running as part of the service. + Pods []string `json:"servicePods"` +} + +// Indicates whether the pod is associated with a service container. +// The pod is expected to be updated and locked. +func (p *Pod) hasServiceContainer() bool { + return p.config.ServiceContainerID != "" +} + +// Returns the pod's service container. +// The pod is expected to be updated and locked. +func (p *Pod) serviceContainer() (*Container, error) { + id := p.config.ServiceContainerID + if id == "" { + return nil, errors.Wrap(define.ErrNoSuchCtr, "pod has no service container") + } + return p.runtime.state.Container(id) +} + +// ServiceContainer returns the service container. +func (p *Pod) ServiceContainer() (*Container, error) { + p.lock.Lock() + defer p.lock.Unlock() + if err := p.updatePod(); err != nil { + return nil, err + } + return p.serviceContainer() +} + +func (c *Container) addServicePodLocked(id string) error { + c.lock.Lock() + defer c.lock.Unlock() + if err := c.syncContainer(); err != nil { + return err + } + c.state.Service.Pods = append(c.state.Service.Pods, id) + return c.save() +} + +func (c *Container) isService() bool { + return c.config.IsService +} + +// canStopServiceContainer returns true if all pods of the service are stopped. +// Note that the method acquires the container lock. +func (c *Container) canStopServiceContainerLocked() (bool, error) { + c.lock.Lock() + defer c.lock.Unlock() + if err := c.syncContainer(); err != nil { + return false, err + } + + if !c.isService() { + return false, fmt.Errorf("internal error: checking service: container %s is not a service container", c.ID()) + } + + for _, id := range c.state.Service.Pods { + pod, err := c.runtime.LookupPod(id) + if err != nil { + if errors.Is(err, define.ErrNoSuchPod) { + continue + } + return false, err + } + + status, err := pod.GetPodStatus() + if err != nil { + return false, err + } + + // We can only stop the service if all pods are done. + switch status { + case define.PodStateStopped, define.PodStateExited, define.PodStateErrored: + continue + default: + return false, nil + } + } + + return true, nil +} + +// Checks whether the service container can be stopped and does so. +func (p *Pod) maybeStopServiceContainer() error { + if !p.hasServiceContainer() { + return nil + } + + serviceCtr, err := p.serviceContainer() + if err != nil { + return fmt.Errorf("getting pod's service container: %w", err) + } + // Checking whether the service can be stopped must be done in + // the runtime's work queue to resolve ABBA dead locks in the + // pod->container->servicePods hierarchy. + p.runtime.queueWork(func() { + logrus.Debugf("Pod %s has a service %s: checking if it can be stopped", p.ID(), serviceCtr.ID()) + canStop, err := serviceCtr.canStopServiceContainerLocked() + if err != nil { + logrus.Errorf("Checking whether service of container %s can be stopped: %v", serviceCtr.ID(), err) + return + } + if !canStop { + return + } + logrus.Debugf("Stopping service container %s", serviceCtr.ID()) + if err := serviceCtr.Stop(); err != nil { + logrus.Errorf("Stopping service container %s: %v", serviceCtr.ID(), err) + } + }) + return nil +} + +// Starts the pod's service container if it's not already running. +func (p *Pod) maybeStartServiceContainer(ctx context.Context) error { + if !p.hasServiceContainer() { + return nil + } + + serviceCtr, err := p.serviceContainer() + if err != nil { + return fmt.Errorf("getting pod's service container: %w", err) + } + + serviceCtr.lock.Lock() + defer serviceCtr.lock.Unlock() + + if err := serviceCtr.syncContainer(); err != nil { + return err + } + + if serviceCtr.state.State == define.ContainerStateRunning { + return nil + } + + // Restart will reinit among other things. + return serviceCtr.restartWithTimeout(ctx, 0) +} + +// canRemoveServiceContainer returns true if all pods of the service are removed. +// Note that the method acquires the container lock. +func (c *Container) canRemoveServiceContainerLocked() (bool, error) { + c.lock.Lock() + defer c.lock.Unlock() + if err := c.syncContainer(); err != nil { + return false, err + } + + if !c.isService() { + return false, fmt.Errorf("internal error: checking service: container %s is not a service container", c.ID()) + } + + for _, id := range c.state.Service.Pods { + if _, err := c.runtime.LookupPod(id); err != nil { + if errors.Is(err, define.ErrNoSuchPod) { + continue + } + return false, err + } + return false, nil + } + + return true, nil +} + +// Checks whether the service container can be removed and does so. +func (p *Pod) maybeRemoveServiceContainer() error { + if !p.hasServiceContainer() { + return nil + } + + serviceCtr, err := p.serviceContainer() + if err != nil { + return fmt.Errorf("getting pod's service container: %w", err) + } + // Checking whether the service can be stopped must be done in + // the runtime's work queue to resolve ABBA dead locks in the + // pod->container->servicePods hierarchy. + p.runtime.queueWork(func() { + logrus.Debugf("Pod %s has a service %s: checking if it can be removed", p.ID(), serviceCtr.ID()) + canRemove, err := serviceCtr.canRemoveServiceContainerLocked() + if err != nil { + logrus.Errorf("Checking whether service of container %s can be removed: %v", serviceCtr.ID(), err) + return + } + if !canRemove { + return + } + timeout := uint(0) + logrus.Debugf("Removing service container %s", serviceCtr.ID()) + if err := p.runtime.RemoveContainer(context.Background(), serviceCtr, true, false, &timeout); err != nil { + logrus.Errorf("Removing service container %s: %v", serviceCtr.ID(), err) + } + }) + return nil +} diff --git a/pkg/domain/entities/play.go b/pkg/domain/entities/play.go index bf7c33f2b..f1ba21650 100644 --- a/pkg/domain/entities/play.go +++ b/pkg/domain/entities/play.go @@ -54,6 +54,8 @@ type PlayKubeOptions struct { LogOptions []string // Start - don't start the pod if false Start types.OptionalBool + // ServiceContainer - creates a service container that is started before and is stopped after all pods. + ServiceContainer bool // Userns - define the user namespace to use. Userns string } diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go index 019361694..420d51483 100644 --- a/pkg/domain/infra/abi/play.go +++ b/pkg/domain/infra/abi/play.go @@ -28,12 +28,54 @@ import ( "github.com/containers/podman/v4/pkg/specgenutil" "github.com/containers/podman/v4/pkg/util" "github.com/ghodss/yaml" + "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/sirupsen/logrus" yamlv2 "gopkg.in/yaml.v2" ) -func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options entities.PlayKubeOptions) (*entities.PlayKubeReport, error) { +// createServiceContainer creates a container that can later on +// be associated with the pods of a K8s yaml. It will be started along with +// the first pod. +func (ic *ContainerEngine) createServiceContainer(ctx context.Context, name string) (*libpod.Container, error) { + // Similar to infra containers, a service container is using the pause image. + image, err := generate.PullOrBuildInfraImage(ic.Libpod, "") + if err != nil { + return nil, fmt.Errorf("image for service container: %w", err) + } + + ctrOpts := entities.ContainerCreateOptions{ + // Inherited from infra containers + ImageVolume: "bind", + IsInfra: false, + MemorySwappiness: -1, + // No need to spin up slirp etc. + Net: &entities.NetOptions{Network: specgen.Namespace{NSMode: specgen.NoNetwork}}, + } + + // Create and fill out the runtime spec. + s := specgen.NewSpecGenerator(image, false) + if err := specgenutil.FillOutSpecGen(s, &ctrOpts, []string{}); err != nil { + return nil, fmt.Errorf("completing spec for service container: %w", err) + } + s.Name = name + + runtimeSpec, spec, opts, err := generate.MakeContainer(ctx, ic.Libpod, s, false, nil) + if err != nil { + return nil, fmt.Errorf("creating runtime spec for service container: %w", err) + } + opts = append(opts, libpod.WithIsService()) + + // Create a new libpod container based on the spec. + ctr, err := ic.Libpod.NewContainer(ctx, runtimeSpec, spec, false, opts...) + if err != nil { + return nil, fmt.Errorf("creating service container: %w", err) + } + + return ctr, nil +} + +func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options entities.PlayKubeOptions) (_ *entities.PlayKubeReport, finalErr error) { report := &entities.PlayKubeReport{} validKinds := 0 @@ -67,6 +109,30 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options return nil, errors.Wrap(err, "unable to read kube YAML") } + // TODO: create constants for the various "kinds" of yaml files. + var serviceContainer *libpod.Container + if options.ServiceContainer && (kind == "Pod" || kind == "Deployment") { + // The name of the service container is the first 12 + // characters of the yaml file's hash followed by the + // '-service' suffix to guarantee a predictable and + // discoverable name. + hash := digest.FromBytes(content).Encoded() + ctr, err := ic.createServiceContainer(ctx, hash[0:12]+"-service") + if err != nil { + return nil, err + } + serviceContainer = ctr + // Make sure to remove the container in case something goes wrong below. + defer func() { + if finalErr == nil { + return + } + if err := ic.Libpod.RemoveContainer(ctx, ctr, true, false, nil); err != nil { + logrus.Errorf("Cleaning up service container after failure: %v", err) + } + }() + } + switch kind { case "Pod": var podYAML v1.Pod @@ -90,7 +156,7 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options podYAML.Annotations[name] = val } - r, err := ic.playKubePod(ctx, podTemplateSpec.ObjectMeta.Name, &podTemplateSpec, options, &ipIndex, podYAML.Annotations, configMaps) + r, err := ic.playKubePod(ctx, podTemplateSpec.ObjectMeta.Name, &podTemplateSpec, options, &ipIndex, podYAML.Annotations, configMaps, serviceContainer) if err != nil { return nil, err } @@ -104,7 +170,7 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options return nil, errors.Wrap(err, "unable to read YAML as Kube Deployment") } - r, err := ic.playKubeDeployment(ctx, &deploymentYAML, options, &ipIndex, configMaps) + r, err := ic.playKubeDeployment(ctx, &deploymentYAML, options, &ipIndex, configMaps, serviceContainer) if err != nil { return nil, err } @@ -148,7 +214,7 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options return report, nil } -func (ic *ContainerEngine) playKubeDeployment(ctx context.Context, deploymentYAML *v1apps.Deployment, options entities.PlayKubeOptions, ipIndex *int, configMaps []v1.ConfigMap) (*entities.PlayKubeReport, error) { +func (ic *ContainerEngine) playKubeDeployment(ctx context.Context, deploymentYAML *v1apps.Deployment, options entities.PlayKubeOptions, ipIndex *int, configMaps []v1.ConfigMap, serviceContainer *libpod.Container) (*entities.PlayKubeReport, error) { var ( deploymentName string podSpec v1.PodTemplateSpec @@ -170,7 +236,7 @@ func (ic *ContainerEngine) playKubeDeployment(ctx context.Context, deploymentYAM // create "replicas" number of pods for i = 0; i < numReplicas; i++ { podName := fmt.Sprintf("%s-pod-%d", deploymentName, i) - podReport, err := ic.playKubePod(ctx, podName, &podSpec, options, ipIndex, deploymentYAML.Annotations, configMaps) + podReport, err := ic.playKubePod(ctx, podName, &podSpec, options, ipIndex, deploymentYAML.Annotations, configMaps, serviceContainer) if err != nil { return nil, errors.Wrapf(err, "error encountered while bringing up pod %s", podName) } @@ -179,7 +245,7 @@ func (ic *ContainerEngine) playKubeDeployment(ctx context.Context, deploymentYAM return &report, nil } -func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podYAML *v1.PodTemplateSpec, options entities.PlayKubeOptions, ipIndex *int, annotations map[string]string, configMaps []v1.ConfigMap) (*entities.PlayKubeReport, error) { +func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podYAML *v1.PodTemplateSpec, options entities.PlayKubeOptions, ipIndex *int, annotations map[string]string, configMaps []v1.ConfigMap, serviceContainer *libpod.Container) (*entities.PlayKubeReport, error) { var ( writer io.Writer playKubePod entities.PlayKubePod @@ -374,6 +440,10 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY } } + if serviceContainer != nil { + podSpec.PodSpecGen.ServiceContainerID = serviceContainer.ID() + } + // Create the Pod pod, err := generate.MakePod(&podSpec, ic.Libpod) if err != nil { diff --git a/pkg/machine/wsl/machine.go b/pkg/machine/wsl/machine.go index 57fb36fc9..0b2874baf 100644 --- a/pkg/machine/wsl/machine.go +++ b/pkg/machine/wsl/machine.go @@ -36,6 +36,7 @@ var ( const ( ErrorSuccessRebootInitiated = 1641 ErrorSuccessRebootRequired = 3010 + currentMachineVersion = 2 ) const containersConf = `[containers] @@ -168,6 +169,8 @@ type MachineVM struct { Rootful bool // SSH identity, username, etc machine.SSHConfig + // machine version + Version int } type ExitCodeError struct { @@ -241,12 +244,29 @@ func readAndMigrate(configPath string, name string) (*MachineVM, error) { return vm, err } err = json.Unmarshal(b, vm) - if err == nil && vm.Created.IsZero() { - err = vm.migrate40(configPath) + if err == nil && vm.Version < currentMachineVersion { + err = vm.migrateMachine(configPath) } + return vm, err } +func (v *MachineVM) migrateMachine(configPath string) error { + if v.Created.IsZero() { + if err := v.migrate40(configPath); err != nil { + return err + } + } + + // Update older machines to use lingering + if err := enableUserLinger(v, toDist(v.Name)); err != nil { + return err + } + + v.Version = currentMachineVersion + return v.writeConfig() +} + func (v *MachineVM) migrate40(configPath string) error { v.ConfigPath = configPath fi, err := os.Stat(configPath) @@ -255,7 +275,7 @@ func (v *MachineVM) migrate40(configPath string) error { } v.Created = fi.ModTime() v.LastUp = getLegacyLastStart(v) - return v.writeConfig() + return nil } func getLegacyLastStart(vm *MachineVM) time.Time { @@ -284,6 +304,7 @@ func (v *MachineVM) Init(opts machine.InitOptions) (bool, error) { sshDir := filepath.Join(homeDir, ".ssh") v.IdentityPath = filepath.Join(sshDir, v.Name) v.Rootful = opts.Rootful + v.Version = currentMachineVersion if err := downloadDistro(v, opts); err != nil { return false, err @@ -486,6 +507,10 @@ func configureSystem(v *MachineVM, dist string) error { return errors.Wrap(err, "could not generate linger service for guest OS") } + if err := enableUserLinger(v, dist); err != nil { + return err + } + if err := pipeCmdPassThrough("wsl", withUser(lingerSetup, user), "-d", dist, "sh"); err != nil { return errors.Wrap(err, "could not configure systemd settomgs for guest OS") } @@ -501,6 +526,15 @@ func configureSystem(v *MachineVM, dist string) error { return nil } +func enableUserLinger(v *MachineVM, dist string) error { + lingerCmd := "mkdir -p /var/lib/systemd/linger; touch /var/lib/systemd/linger/" + v.RemoteUsername + if err := runCmdPassThrough("wsl", "-d", dist, "sh", "-c", lingerCmd); err != nil { + return errors.Wrap(err, "could not enable linger for remote user on guest OS") + } + + return nil +} + func installScripts(dist string) error { if err := pipeCmdPassThrough("wsl", enterns, "-d", dist, "sh", "-c", "cat > /usr/local/bin/enterns; chmod 755 /usr/local/bin/enterns"); err != nil { diff --git a/pkg/specgen/generate/pause_image.go b/pkg/specgen/generate/pause_image.go new file mode 100644 index 000000000..ddf35f230 --- /dev/null +++ b/pkg/specgen/generate/pause_image.go @@ -0,0 +1,95 @@ +package generate + +import ( + "context" + "fmt" + "io/ioutil" + "os" + + buildahDefine "github.com/containers/buildah/define" + "github.com/containers/common/pkg/config" + "github.com/containers/podman/v4/libpod" + "github.com/containers/podman/v4/libpod/define" +) + +// PullOrBuildInfraImage pulls down the specified image or the one set in +// containers.conf. If none is set, it builds a local pause image. +func PullOrBuildInfraImage(rt *libpod.Runtime, imageName string) (string, error) { + rtConfig, err := rt.GetConfigNoCopy() + if err != nil { + return "", err + } + + if imageName == "" { + imageName = rtConfig.Engine.InfraImage + } + + if imageName != "" { + _, err := rt.LibimageRuntime().Pull(context.Background(), imageName, config.PullPolicyMissing, nil) + if err != nil { + return "", err + } + return imageName, nil + } + + name, err := buildPauseImage(rt, rtConfig) + if err != nil { + return "", fmt.Errorf("building local pause image: %w", err) + } + return name, nil +} + +func buildPauseImage(rt *libpod.Runtime, rtConfig *config.Config) (string, error) { + version, err := define.GetVersion() + if err != nil { + return "", err + } + imageName := fmt.Sprintf("localhost/podman-pause:%s-%d", version.Version, version.Built) + + // First check if the image has already been built. + if _, _, err := rt.LibimageRuntime().LookupImage(imageName, nil); err == nil { + return imageName, nil + } + + // Also look into the path as some distributions install catatonit in + // /usr/bin. + catatonitPath, err := rtConfig.FindHelperBinary("catatonit", true) + if err != nil { + return "", fmt.Errorf("finding pause binary: %w", err) + } + + buildContent := fmt.Sprintf(`FROM scratch +COPY %s /catatonit +ENTRYPOINT ["/catatonit", "-P"]`, catatonitPath) + + tmpF, err := ioutil.TempFile("", "pause.containerfile") + if err != nil { + return "", err + } + if _, err := tmpF.WriteString(buildContent); err != nil { + return "", err + } + if err := tmpF.Close(); err != nil { + return "", err + } + defer os.Remove(tmpF.Name()) + + buildOptions := buildahDefine.BuildOptions{ + CommonBuildOpts: &buildahDefine.CommonBuildOptions{}, + Output: imageName, + Quiet: true, + IgnoreFile: "/dev/null", // makes sure to not read a local .ignorefile (see #13529) + IIDFile: "/dev/null", // prevents Buildah from writing the ID on stdout + IDMappingOptions: &buildahDefine.IDMappingOptions{ + // Use the host UID/GID mappings for the build to avoid issues when + // running with a custom mapping (BZ #2083997). + HostUIDMapping: true, + HostGIDMapping: true, + }, + } + if _, _, err := rt.Build(context.Background(), buildOptions, tmpF.Name()); err != nil { + return "", err + } + + return imageName, nil +} diff --git a/pkg/specgen/generate/pod_create.go b/pkg/specgen/generate/pod_create.go index fce32d688..5b7bb2b57 100644 --- a/pkg/specgen/generate/pod_create.go +++ b/pkg/specgen/generate/pod_create.go @@ -2,13 +2,8 @@ package generate import ( "context" - "fmt" - "io/ioutil" "net" - "os" - buildahDefine "github.com/containers/buildah/define" - "github.com/containers/common/pkg/config" "github.com/containers/podman/v4/libpod" "github.com/containers/podman/v4/libpod/define" "github.com/containers/podman/v4/pkg/domain/entities" @@ -17,98 +12,18 @@ import ( "github.com/sirupsen/logrus" ) -func buildPauseImage(rt *libpod.Runtime, rtConfig *config.Config) (string, error) { - version, err := define.GetVersion() - if err != nil { - return "", err - } - imageName := fmt.Sprintf("localhost/podman-pause:%s-%d", version.Version, version.Built) - - // First check if the image has already been built. - if _, _, err := rt.LibimageRuntime().LookupImage(imageName, nil); err == nil { - return imageName, nil - } - - // Also look into the path as some distributions install catatonit in - // /usr/bin. - catatonitPath, err := rtConfig.FindHelperBinary("catatonit", true) - if err != nil { - return "", fmt.Errorf("finding pause binary: %w", err) - } - - buildContent := fmt.Sprintf(`FROM scratch -COPY %s /catatonit -ENTRYPOINT ["/catatonit", "-P"]`, catatonitPath) - - tmpF, err := ioutil.TempFile("", "pause.containerfile") - if err != nil { - return "", err - } - if _, err := tmpF.WriteString(buildContent); err != nil { - return "", err - } - if err := tmpF.Close(); err != nil { - return "", err - } - defer os.Remove(tmpF.Name()) - - buildOptions := buildahDefine.BuildOptions{ - CommonBuildOpts: &buildahDefine.CommonBuildOptions{}, - Output: imageName, - Quiet: true, - IgnoreFile: "/dev/null", // makes sure to not read a local .ignorefile (see #13529) - IIDFile: "/dev/null", // prevents Buildah from writing the ID on stdout - } - if _, _, err := rt.Build(context.Background(), buildOptions, tmpF.Name()); err != nil { - return "", err - } - - return imageName, nil -} - -func pullOrBuildInfraImage(p *entities.PodSpec, rt *libpod.Runtime) error { - if p.PodSpecGen.NoInfra { - return nil - } - - rtConfig, err := rt.GetConfigNoCopy() - if err != nil { - return err - } - - // NOTE: we need pull down the infra image if it was explicitly set by - // the user (or containers.conf) to the non-default one. - imageName := p.PodSpecGen.InfraImage - if imageName == "" { - imageName = rtConfig.Engine.InfraImage - } - - if imageName != "" { - _, err := rt.LibimageRuntime().Pull(context.Background(), imageName, config.PullPolicyMissing, nil) - if err != nil { - return err - } - } else { - name, err := buildPauseImage(rt, rtConfig) - if err != nil { - return fmt.Errorf("building local pause image: %w", err) - } - imageName = name - } - - p.PodSpecGen.InfraImage = imageName - p.PodSpecGen.InfraContainerSpec.RawImageName = imageName - - return nil -} - func MakePod(p *entities.PodSpec, rt *libpod.Runtime) (*libpod.Pod, error) { if err := p.PodSpecGen.Validate(); err != nil { return nil, err } - if err := pullOrBuildInfraImage(p, rt); err != nil { - return nil, err + if !p.PodSpecGen.NoInfra { + imageName, err := PullOrBuildInfraImage(rt, p.PodSpecGen.InfraImage) + if err != nil { + return nil, err + } + p.PodSpecGen.InfraImage = imageName + p.PodSpecGen.InfraContainerSpec.RawImageName = imageName } if !p.PodSpecGen.NoInfra && p.PodSpecGen.InfraContainerSpec != nil { @@ -180,6 +95,11 @@ func createPodOptions(p *specgen.PodSpecGenerator) ([]libpod.PodCreateOption, er options = append(options, libpod.WithPodUser()) } } + + if len(p.ServiceContainerID) > 0 { + options = append(options, libpod.WithServiceContainer(p.ServiceContainerID)) + } + if len(p.CgroupParent) > 0 { options = append(options, libpod.WithPodCgroupParent(p.CgroupParent)) } diff --git a/pkg/specgen/podspecgen.go b/pkg/specgen/podspecgen.go index 1bb64448f..603506241 100644 --- a/pkg/specgen/podspecgen.go +++ b/pkg/specgen/podspecgen.go @@ -204,6 +204,9 @@ type PodSpecGenerator struct { PodStorageConfig PodSecurityConfig InfraContainerSpec *SpecGenerator `json:"-"` + + // The ID of the pod's service container. + ServiceContainerID string `json:"serviceContainerID,omitempty"` } type PodResourceConfig struct { diff --git a/pkg/util/utils_windows.go b/pkg/util/utils_windows.go index 2732124f2..b91680f7a 100644 --- a/pkg/util/utils_windows.go +++ b/pkg/util/utils_windows.go @@ -4,6 +4,9 @@ package util import ( + "path/filepath" + + "github.com/containers/storage/pkg/homedir" "github.com/pkg/errors" ) @@ -34,7 +37,12 @@ func GetRootlessPauseProcessPidPathGivenDir(unused string) (string, error) { // GetRuntimeDir returns the runtime directory func GetRuntimeDir() (string, error) { - return "", errors.New("this function is not implemented for windows") + data, err := homedir.GetDataHome() + if err != nil { + return "", err + } + runtimeDir := filepath.Join(data, "containers", "podman") + return runtimeDir, nil } // GetRootlessConfigHomeDir returns the config home directory when running as non root diff --git a/test/e2e/build_test.go b/test/e2e/build_test.go index b5cec5fff..dcdd17143 100644 --- a/test/e2e/build_test.go +++ b/test/e2e/build_test.go @@ -786,17 +786,18 @@ RUN ls /dev/test1`, ALPINE) It("podman build use absolute path even if given relative", func() { containerFile := fmt.Sprintf(`FROM %s`, ALPINE) - err = os.Mkdir("relative", 0755) + relativeDir := filepath.Join(podmanTest.TempDir, "relativeDir") + containerFilePath := filepath.Join(relativeDir, "Containerfile") + buildRoot := filepath.Join(relativeDir, "build-root") + + err = os.Mkdir(relativeDir, 0755) Expect(err).To(BeNil()) - containerFilePath := filepath.Join("relative", "Containerfile") - err = os.Mkdir("relative/build-root", 0755) + err = os.Mkdir(buildRoot, 0755) Expect(err).To(BeNil()) err = ioutil.WriteFile(containerFilePath, []byte(containerFile), 0755) Expect(err).To(BeNil()) - build := podmanTest.Podman([]string{"build", "-f", "./relative/Containerfile", "./relative/build-root"}) + build := podmanTest.Podman([]string{"build", "-f", containerFilePath, buildRoot}) build.WaitWithDefaultTimeout() Expect(build).To(Exit(0)) - err = os.RemoveAll("relative") - Expect(err).To(BeNil()) }) }) diff --git a/test/e2e/config.go b/test/e2e/config.go index 9c810575b..2ca8e2a15 100644 --- a/test/e2e/config.go +++ b/test/e2e/config.go @@ -14,7 +14,7 @@ var ( BB = "quay.io/libpod/busybox:latest" healthcheck = "quay.io/libpod/alpine_healthcheck:latest" ImageCacheDir = "/tmp/podman/imagecachedir" - fedoraToolbox = "registry.fedoraproject.org/f32/fedora-toolbox:latest" + fedoraToolbox = "registry.fedoraproject.org/fedora-toolbox:36" volumeTest = "quay.io/libpod/volume-plugin-test-img:latest" // This image has seccomp profiles that blocks all syscalls. diff --git a/test/e2e/containers_conf_test.go b/test/e2e/containers_conf_test.go index b48e1ed62..819efa628 100644 --- a/test/e2e/containers_conf_test.go +++ b/test/e2e/containers_conf_test.go @@ -425,6 +425,7 @@ var _ = Describe("Verify podman containers.conf usage", func() { Expect(session).Should(Exit(0)) Expect(session.OutputToString()).To(Equal("/var/tmp")) + storagePath := filepath.Join(podmanTest.TempDir, "storage") configPath := filepath.Join(podmanTest.TempDir, "containers.conf") os.Setenv("CONTAINERS_CONF", configPath) @@ -441,7 +442,7 @@ var _ = Describe("Verify podman containers.conf usage", func() { Expect(session).Should(Exit(0)) Expect(session.OutputToString()).To(Equal("/foobar")) - containersConf = []byte("[engine]\nimage_copy_tmp_dir=\"storage\"") + containersConf = []byte(fmt.Sprintf("[engine]\nimage_copy_tmp_dir=%q", storagePath)) err = ioutil.WriteFile(configPath, containersConf, os.ModePerm) Expect(err).ToNot(HaveOccurred()) if IsRemote() { @@ -451,19 +452,18 @@ var _ = Describe("Verify podman containers.conf usage", func() { session = podmanTest.Podman([]string{"info", "--format", "{{.Store.ImageCopyTmpDir}}"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(session.Out.Contents()).To(ContainSubstring("containers/storage/tmp")) + Expect(session.Out.Contents()).To(ContainSubstring(storagePath)) containersConf = []byte("[engine]\nimage_copy_tmp_dir=\"storage1\"") err = ioutil.WriteFile(configPath, containersConf, os.ModePerm) Expect(err).ToNot(HaveOccurred()) - if IsRemote() { - podmanTest.RestartRemoteService() - } - session = podmanTest.Podman([]string{"info", "--format", "{{.Store.ImageCopyTmpDir}}"}) - session.WaitWithDefaultTimeout() - Expect(session).Should(Exit(0)) - Expect(session.Err.Contents()).To(ContainSubstring("invalid image_copy_tmp_dir")) + if !IsRemote() { + session = podmanTest.Podman([]string{"info", "--format", "{{.Store.ImageCopyTmpDir}}"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(125)) + Expect(session.Err.Contents()).To(ContainSubstring("invalid image_copy_tmp_dir value \"storage1\" (relative paths are not accepted)")) + } }) // FIXME not sure why this is here diff --git a/test/e2e/save_test.go b/test/e2e/save_test.go index 536eefda7..897e49ef7 100644 --- a/test/e2e/save_test.go +++ b/test/e2e/save_test.go @@ -164,12 +164,13 @@ var _ = Describe("Podman save", func() { err = cmd.Run() Expect(err).To(BeNil()) - cmd = exec.Command("cp", "/etc/containers/registries.d/default.yaml", "default.yaml") + defaultYaml := filepath.Join(podmanTest.TempDir, "default.yaml") + cmd = exec.Command("cp", "/etc/containers/registries.d/default.yaml", defaultYaml) if err = cmd.Run(); err != nil { Skip("no signature store to verify") } defer func() { - cmd = exec.Command("cp", "default.yaml", "/etc/containers/registries.d/default.yaml") + cmd = exec.Command("cp", defaultYaml, "/etc/containers/registries.d/default.yaml") err := cmd.Run() Expect(err).ToNot(HaveOccurred()) }() diff --git a/test/e2e/volume_create_test.go b/test/e2e/volume_create_test.go index 09e5da8a0..0bf5acbf1 100644 --- a/test/e2e/volume_create_test.go +++ b/test/e2e/volume_create_test.go @@ -3,6 +3,7 @@ package integration import ( "fmt" "os" + "path/filepath" . "github.com/containers/podman/v4/test/utils" . "github.com/onsi/ginkgo" @@ -90,7 +91,8 @@ var _ = Describe("Podman volume create", func() { session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - session = podmanTest.Podman([]string{"volume", "export", volName, "--output=hello.tar"}) + helloTar := filepath.Join(podmanTest.TempDir, "hello.tar") + session = podmanTest.Podman([]string{"volume", "export", volName, "--output", helloTar}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) @@ -98,7 +100,7 @@ var _ = Describe("Podman volume create", func() { session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - session = podmanTest.Podman([]string{"volume", "import", "my_vol2", "hello.tar"}) + session = podmanTest.Podman([]string{"volume", "import", "my_vol2", helloTar}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) Expect(session.OutputToString()).To(Equal(""), "output of volume import") diff --git a/test/system/170-run-userns.bats b/test/system/170-run-userns.bats index d754306b2..b80351902 100644 --- a/test/system/170-run-userns.bats +++ b/test/system/170-run-userns.bats @@ -36,6 +36,19 @@ function _require_crun() { is "$output" ".*457" "Check group leaked into container" } +@test "rootful pod with custom ID mapping" { + skip_if_rootless "does not work rootless - rootful feature" + skip_if_remote "remote --uidmap is broken (see #14233)" + random_pod_name=$(random_string 30) + run_podman pod create --uidmap 0:200000:5000 --name=$random_pod_name + run_podman pod start $random_pod_name + + # Remove the pod and the pause image + run_podman pod rm $random_pod_name + run_podman version --format "{{.Server.Version}}-{{.Server.Built}}" + run_podman rmi -f localhost/podman-pause:$output +} + @test "podman --remote --group-add keep-groups " { if is_remote; then run_podman 125 run --rm --group-add keep-groups $IMAGE id diff --git a/test/system/200-pod.bats b/test/system/200-pod.bats index 39982848f..4250f2680 100644 --- a/test/system/200-pod.bats +++ b/test/system/200-pod.bats @@ -408,19 +408,6 @@ EOF run_podman pod rm test } -# Wait for the pod (1st arg) to transition into the state (2nd arg) -function _ensure_pod_state() { - for i in {0..5}; do - run_podman pod inspect $1 --format "{{.State}}" - if [[ $output == "$2" ]]; then - break - fi - sleep 0.5 - done - - is "$output" "$2" "unexpected pod state" -} - @test "pod exit policies" { # Test setting exit policies run_podman pod create diff --git a/test/system/260-sdnotify.bats b/test/system/260-sdnotify.bats index 395e6f94f..88d84c86f 100644 --- a/test/system/260-sdnotify.bats +++ b/test/system/260-sdnotify.bats @@ -106,6 +106,9 @@ function _assert_mainpid_is_conmon() { cid="$output" wait_for_ready $cid + run_podman container inspect sdnotify_conmon_c --format "{{.State.ConmonPid}}" + mainPID="$output" + run_podman logs sdnotify_conmon_c is "$output" "READY" "\$NOTIFY_SOCKET in container" @@ -114,12 +117,8 @@ function _assert_mainpid_is_conmon() { echo "socat log:" echo "$output" - # ARGH! 'READY=1' should always be the last output line. But sometimes, - # for reasons unknown, we get an extra MAINPID=xxx after READY=1 (#8718). - # Who knows if this is a systemd bug, or conmon, or what. I don't - # even know where to begin asking. So, to eliminate the test flakes, - # we look for READY=1 _anywhere_ in the output, not just the last line. - is "$output" ".*READY=1.*" "sdnotify sent READY=1" + is "$output" "MAINPID=$mainPID +READY=1" "sdnotify sent MAINPID and READY" _assert_mainpid_is_conmon "$output" diff --git a/test/system/700-play.bats b/test/system/700-play.bats index 7988b26a4..2e5327a85 100644 --- a/test/system/700-play.bats +++ b/test/system/700-play.bats @@ -100,6 +100,61 @@ RELABEL="system_u:object_r:container_file_t:s0" run_podman pod rm -t 0 -f test_pod } +@test "podman play --service-container" { + skip_if_remote "service containers only work locally" + + TESTDIR=$PODMAN_TMPDIR/testdir + mkdir -p $TESTDIR + +yaml=" +apiVersion: v1 +kind: Pod +metadata: + labels: + app: test + name: test_pod +spec: + containers: + - command: + - top + image: $IMAGE + name: test + resources: {} +" + + echo "$yaml" > $PODMAN_TMPDIR/test.yaml + run_podman play kube --service-container=true $PODMAN_TMPDIR/test.yaml + + # Make sure that the service container exists and runs. + run_podman container inspect "352a88685060-service" --format "{{.State.Running}}" + is "$output" "true" + + # Stop the *main* container and make sure that + # 1) The pod transitions to Exited + # 2) The service container is stopped + # #) The service container is marked as an service container + run_podman stop test_pod-test + _ensure_pod_state test_pod Exited + run_podman container inspect "352a88685060-service" --format "{{.State.Running}}" + is "$output" "false" + run_podman container inspect "352a88685060-service" --format "{{.IsService}}" + is "$output" "true" + + # Restart the pod, make sure the service is running again + run_podman pod restart test_pod + run_podman container inspect "352a88685060-service" --format "{{.State.Running}}" + is "$output" "true" + + # Kill the pod and make sure the service is not running + run_podman pod kill test_pod + run_podman container inspect "352a88685060-service" --format "{{.State.Running}}" + is "$output" "false" + + # Remove the pod and make sure the service is removed along with it + run_podman pod rm test_pod + run_podman 1 container exists "352a88685060-service" +} + @test "podman play --network" { TESTDIR=$PODMAN_TMPDIR/testdir mkdir -p $TESTDIR diff --git a/test/system/helpers.bash b/test/system/helpers.bash index 138d668f4..072131202 100644 --- a/test/system/helpers.bash +++ b/test/system/helpers.bash @@ -392,6 +392,19 @@ function pause_image() { echo "localhost/podman-pause:$output" } +# Wait for the pod (1st arg) to transition into the state (2nd arg) +function _ensure_pod_state() { + for i in {0..5}; do + run_podman pod inspect $1 --format "{{.State}}" + if [[ $output == "$2" ]]; then + break + fi + sleep 0.5 + done + + is "$output" "$2" "unexpected pod state" +} + ########################### # _add_label_if_missing # make sure skip messages include rootless/remote ########################### diff --git a/vendor/github.com/containers/common/libimage/image.go b/vendor/github.com/containers/common/libimage/image.go index 661ca159b..d7c4fcd51 100644 --- a/vendor/github.com/containers/common/libimage/image.go +++ b/vendor/github.com/containers/common/libimage/image.go @@ -608,7 +608,7 @@ func (i *Image) RepoTags() ([]string, error) { // NamedTaggedRepoTags returns the repotags associated with the image as a // slice of reference.NamedTagged. func (i *Image) NamedTaggedRepoTags() ([]reference.NamedTagged, error) { - var repoTags []reference.NamedTagged + repoTags := make([]reference.NamedTagged, 0, len(i.Names())) for _, name := range i.Names() { parsed, err := reference.Parse(name) if err != nil { diff --git a/vendor/github.com/containers/common/libimage/load.go b/vendor/github.com/containers/common/libimage/load.go index 4dfac7106..c2d066645 100644 --- a/vendor/github.com/containers/common/libimage/load.go +++ b/vendor/github.com/containers/common/libimage/load.go @@ -32,8 +32,8 @@ func (r *Runtime) Load(ctx context.Context, path string, options *LoadOptions) ( options = &LoadOptions{} } - var loadErrors []error - + // we have 4 functions, so a maximum of 4 errors + loadErrors := make([]error, 0, 4) for _, f := range []func() ([]string, string, error){ // OCI func() ([]string, string, error) { @@ -88,6 +88,8 @@ func (r *Runtime) Load(ctx context.Context, path string, options *LoadOptions) ( } // Give a decent error message if nothing above worked. + // we want the colon here for the multiline error + //nolint:revive loadError := fmt.Errorf("payload does not match any of the supported image formats:") for _, err := range loadErrors { loadError = fmt.Errorf("%v\n * %v", loadError, err) diff --git a/vendor/github.com/containers/common/libimage/normalize.go b/vendor/github.com/containers/common/libimage/normalize.go index 7ceb62830..b36bbf396 100644 --- a/vendor/github.com/containers/common/libimage/normalize.go +++ b/vendor/github.com/containers/common/libimage/normalize.go @@ -115,7 +115,7 @@ type NameTagPair struct { func ToNameTagPairs(repoTags []reference.Named) ([]NameTagPair, error) { none := "<none>" - var pairs []NameTagPair + pairs := make([]NameTagPair, 0, len(repoTags)) for i, named := range repoTags { pair := NameTagPair{ Name: named.Name(), diff --git a/vendor/github.com/containers/common/libimage/pull.go b/vendor/github.com/containers/common/libimage/pull.go index ff93b6ed8..4ce8add2f 100644 --- a/vendor/github.com/containers/common/libimage/pull.go +++ b/vendor/github.com/containers/common/libimage/pull.go @@ -413,11 +413,11 @@ func (r *Runtime) imagesIDsForManifest(manifestBytes []byte, sys *types.SystemCo } imageDigest = d } - var results []string images, err := r.store.ImagesByDigest(imageDigest) if err != nil { return nil, errors.Wrapf(err, "listing images by manifest digest") } + results := make([]string, 0, len(images)) for _, image := range images { results = append(results, image.ID) } diff --git a/vendor/github.com/containers/common/libimage/runtime.go b/vendor/github.com/containers/common/libimage/runtime.go index 974b50b50..472482410 100644 --- a/vendor/github.com/containers/common/libimage/runtime.go +++ b/vendor/github.com/containers/common/libimage/runtime.go @@ -6,6 +6,7 @@ import ( "os" "strings" + "github.com/containers/common/pkg/config" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/pkg/shortnames" storageTransport "github.com/containers/image/v5/storage" @@ -22,13 +23,16 @@ import ( var json = jsoniter.ConfigCompatibleWithStandardLibrary // tmpdir returns a path to a temporary directory. -func tmpdir() string { - tmpdir := os.Getenv("TMPDIR") - if tmpdir == "" { - tmpdir = "/var/tmp" +func tmpdir() (string, error) { + var tmpdir string + defaultContainerConfig, err := config.Default() + if err == nil { + tmpdir, err = defaultContainerConfig.ImageCopyTmpDir() + if err == nil { + return tmpdir, nil + } } - - return tmpdir + return tmpdir, err } // RuntimeOptions allow for creating a customized Runtime. @@ -103,7 +107,11 @@ func RuntimeFromStore(store storage.Store, options *RuntimeOptions) (*Runtime, e systemContext = types.SystemContext{} } if systemContext.BigFilesTemporaryDir == "" { - systemContext.BigFilesTemporaryDir = tmpdir() + tmpdir, err := tmpdir() + if err != nil { + return nil, err + } + systemContext.BigFilesTemporaryDir = tmpdir } setRegistriesConfPath(&systemContext) @@ -224,16 +232,15 @@ func (r *Runtime) LookupImage(name string, options *LookupImageOptions) (*Image, } logrus.Debugf("Found image %q in local containers storage (%s)", name, storageRef.StringWithinTransport()) return r.storageToImage(img, storageRef), "", nil - } else { - // Docker compat: strip off the tag iff name is tagged and digested - // (e.g., fedora:latest@sha256...). In that case, the tag is stripped - // off and entirely ignored. The digest is the sole source of truth. - normalizedName, err := normalizeTaggedDigestedString(name) - if err != nil { - return nil, "", err - } - name = normalizedName } + // Docker compat: strip off the tag iff name is tagged and digested + // (e.g., fedora:latest@sha256...). In that case, the tag is stripped + // off and entirely ignored. The digest is the sole source of truth. + normalizedName, err := normalizeTaggedDigestedString(name) + if err != nil { + return nil, "", err + } + name = normalizedName byDigest := false originalName := name diff --git a/vendor/github.com/containers/common/libnetwork/cni/config.go b/vendor/github.com/containers/common/libnetwork/cni/config.go index c6967b600..f6954db05 100644 --- a/vendor/github.com/containers/common/libnetwork/cni/config.go +++ b/vendor/github.com/containers/common/libnetwork/cni/config.go @@ -96,7 +96,7 @@ func (n *cniNetwork) networkCreate(newNetwork *types.Network, defaultNet bool) ( newNetwork.ID = getNetworkIDFromName(newNetwork.Name) // when we do not have ipam we must disable dns - internalutil.IpamNoneDisableDns(newNetwork) + internalutil.IpamNoneDisableDNS(newNetwork) // FIXME: Should this be a hard error? if newNetwork.DNSEnabled && newNetwork.Internal && hasDNSNamePlugin(n.cniPluginDirs) { diff --git a/vendor/github.com/containers/common/libnetwork/cni/run.go b/vendor/github.com/containers/common/libnetwork/cni/run.go index c7fa86ed0..c5461d74c 100644 --- a/vendor/github.com/containers/common/libnetwork/cni/run.go +++ b/vendor/github.com/containers/common/libnetwork/cni/run.go @@ -106,7 +106,7 @@ func (n *cniNetwork) Setup(namespacePath string, options types.SetupOptions) (ma } // CNIResultToStatus convert the cni result to status block -// nolint:golint +// nolint:golint,revive func CNIResultToStatus(res cnitypes.Result) (types.StatusBlock, error) { result := types.StatusBlock{} cniResult, err := types040.GetResult(res) diff --git a/vendor/github.com/containers/common/libnetwork/internal/util/create.go b/vendor/github.com/containers/common/libnetwork/internal/util/create.go index c1a4bee75..d4d574065 100644 --- a/vendor/github.com/containers/common/libnetwork/internal/util/create.go +++ b/vendor/github.com/containers/common/libnetwork/internal/util/create.go @@ -41,7 +41,7 @@ func CommonNetworkCreate(n NetUtil, network *types.Network) error { return nil } -func IpamNoneDisableDns(network *types.Network) { +func IpamNoneDisableDNS(network *types.Network) { if network.IPAMOptions[types.Driver] == types.NoneIPAMDriver { logrus.Debugf("dns disabled for network %q because ipam driver is set to none", network.Name) network.DNSEnabled = false diff --git a/vendor/github.com/containers/common/libnetwork/netavark/config.go b/vendor/github.com/containers/common/libnetwork/netavark/config.go index 0bb0539b2..f2c72ab9e 100644 --- a/vendor/github.com/containers/common/libnetwork/netavark/config.go +++ b/vendor/github.com/containers/common/libnetwork/netavark/config.go @@ -121,7 +121,7 @@ func (n *netavarkNetwork) networkCreate(newNetwork *types.Network, defaultNet bo } // when we do not have ipam we must disable dns - internalutil.IpamNoneDisableDns(newNetwork) + internalutil.IpamNoneDisableDNS(newNetwork) // add gateway when not internal or dns enabled addGateway := !newNetwork.Internal || newNetwork.DNSEnabled diff --git a/vendor/github.com/containers/common/libnetwork/network/interface.go b/vendor/github.com/containers/common/libnetwork/network/interface.go index 893bdea2e..e70f096a4 100644 --- a/vendor/github.com/containers/common/libnetwork/network/interface.go +++ b/vendor/github.com/containers/common/libnetwork/network/interface.go @@ -46,6 +46,9 @@ const ( // 1. read ${graphroot}/defaultNetworkBackend // 2. find netavark binary (if not installed use CNI) // 3. check containers, images and CNI networks and if there are some we have an existing install and should continue to use CNI +// +// revive does not like the name because the package is already called network +//nolint:revive func NetworkBackend(store storage.Store, conf *config.Config, syslog bool) (types.NetworkBackend, types.ContainerNetwork, error) { backend := types.NetworkBackend(conf.Network.NetworkBackend) if backend == "" { diff --git a/vendor/github.com/containers/common/pkg/apparmor/apparmor_linux.go b/vendor/github.com/containers/common/pkg/apparmor/apparmor_linux.go index 35f79a1ad..1fd269255 100644 --- a/vendor/github.com/containers/common/pkg/apparmor/apparmor_linux.go +++ b/vendor/github.com/containers/common/pkg/apparmor/apparmor_linux.go @@ -251,19 +251,17 @@ func CheckProfileAndLoadDefault(name string) (string, error) { if unshare.IsRootless() { if name != "" { return "", errors.Wrapf(ErrApparmorRootless, "cannot load AppArmor profile %q", name) - } else { - logrus.Debug("Skipping loading default AppArmor profile (rootless mode)") - return "", nil } + logrus.Debug("Skipping loading default AppArmor profile (rootless mode)") + return "", nil } // Check if AppArmor is disabled and error out if a profile is to be set. if !runcaa.IsEnabled() { if name == "" { return "", nil - } else { - return "", errors.Errorf("profile %q specified but AppArmor is disabled on the host", name) } + return "", errors.Errorf("profile %q specified but AppArmor is disabled on the host", name) } if name == "" { diff --git a/vendor/github.com/containers/common/pkg/auth/auth.go b/vendor/github.com/containers/common/pkg/auth/auth.go index 6765c9e5b..188e06c12 100644 --- a/vendor/github.com/containers/common/pkg/auth/auth.go +++ b/vendor/github.com/containers/common/pkg/auth/auth.go @@ -26,8 +26,8 @@ func GetDefaultAuthFile() string { if authfile := os.Getenv("REGISTRY_AUTH_FILE"); authfile != "" { return authfile } - if auth_env := os.Getenv("DOCKER_CONFIG"); auth_env != "" { - return filepath.Join(auth_env, "config.json") + if authEnv := os.Getenv("DOCKER_CONFIG"); authEnv != "" { + return filepath.Join(authEnv, "config.json") } return "" } @@ -313,7 +313,7 @@ func Logout(systemContext *types.SystemContext, opts *LogoutOptions, args []stri fmt.Printf("Not logged into %s with current tool. Existing credentials were established via docker login. Please use docker logout instead.\n", key) return nil } - return errors.Errorf("Not logged into %s\n", key) + return errors.Errorf("not logged into %s", key) default: return errors.Wrapf(err, "logging out of %q", key) } diff --git a/vendor/github.com/containers/common/pkg/capabilities/capabilities.go b/vendor/github.com/containers/common/pkg/capabilities/capabilities.go index 10c5dd7c4..b8e3fbcb5 100644 --- a/vendor/github.com/containers/common/pkg/capabilities/capabilities.go +++ b/vendor/github.com/containers/common/pkg/capabilities/capabilities.go @@ -104,8 +104,8 @@ func AllCapabilities() []string { // NormalizeCapabilities normalizes caps by adding a "CAP_" prefix (if not yet // present). func NormalizeCapabilities(caps []string) ([]string, error) { - normalized := make([]string, len(caps)) - for i, c := range caps { + normalized := make([]string, 0, len(caps)) + for _, c := range caps { c = strings.ToUpper(c) if c == All { normalized = append(normalized, c) @@ -117,7 +117,7 @@ func NormalizeCapabilities(caps []string) ([]string, error) { if !stringInSlice(c, capabilityList) { return nil, errors.Wrapf(ErrUnknownCapability, "%q", c) } - normalized[i] = c + normalized = append(normalized, c) } sort.Strings(normalized) return normalized, nil @@ -140,8 +140,6 @@ func ValidateCapabilities(caps []string) error { // "ALL" in capAdd adds returns known capabilities // "All" in capDrop returns only the capabilities specified in capAdd func MergeCapabilities(base, adds, drops []string) ([]string, error) { - var caps []string - // Normalize the base capabilities base, err := NormalizeCapabilities(base) if err != nil { @@ -189,6 +187,7 @@ func MergeCapabilities(base, adds, drops []string) ([]string, error) { } } + caps := make([]string, 0, len(base)+len(capAdd)) // Drop any capabilities in capDrop that are in base for _, cap := range base { if stringInSlice(cap, capDrop) { diff --git a/vendor/github.com/containers/common/pkg/cgroups/cgroups_supported.go b/vendor/github.com/containers/common/pkg/cgroups/cgroups_supported.go index edb28ad18..5c6c199e0 100644 --- a/vendor/github.com/containers/common/pkg/cgroups/cgroups_supported.go +++ b/vendor/github.com/containers/common/pkg/cgroups/cgroups_supported.go @@ -9,6 +9,7 @@ import ( "io/ioutil" "os" "path/filepath" + "strconv" "strings" "sync" "syscall" @@ -96,6 +97,22 @@ func UserOwnsCurrentSystemdCgroup() (bool, error) { // It differs from os.RemoveAll as it doesn't attempt to unlink files. // On cgroupfs we are allowed only to rmdir empty directories. func rmDirRecursively(path string) error { + killProcesses := func(signal syscall.Signal) { + if signal == unix.SIGKILL { + if err := ioutil.WriteFile(filepath.Join(path, "cgroup.kill"), []byte("1"), 0600); err == nil { + return + } + } + // kill all the processes that are still part of the cgroup + if procs, err := ioutil.ReadFile(filepath.Join(path, "cgroup.procs")); err == nil { + for _, pidS := range strings.Split(string(procs), "\n") { + if pid, err := strconv.Atoi(pidS); err == nil { + _ = unix.Kill(pid, signal) + } + } + } + } + if err := os.Remove(path); err == nil || os.IsNotExist(err) { return nil } @@ -118,8 +135,16 @@ func rmDirRecursively(path string) error { return nil } if errors.Is(err, unix.EBUSY) { - // attempt up to 5 seconds if the cgroup is busy - if attempts < 500 { + // send a SIGTERM after 3 second + if attempts == 300 { + killProcesses(unix.SIGTERM) + } + // send SIGKILL after 8 seconds + if attempts == 800 { + killProcesses(unix.SIGKILL) + } + // give up after 10 seconds + if attempts < 1000 { time.Sleep(time.Millisecond * 10) attempts++ continue diff --git a/vendor/github.com/containers/common/pkg/completion/completion.go b/vendor/github.com/containers/common/pkg/completion/completion.go index c90bf540b..b5e6d6d30 100644 --- a/vendor/github.com/containers/common/pkg/completion/completion.go +++ b/vendor/github.com/containers/common/pkg/completion/completion.go @@ -51,7 +51,7 @@ func AutocompleteCapabilities(cmd *cobra.Command, args []string, toComplete stri offset = 4 } - var completions []string + completions := make([]string, 0, len(caps)) for _, cap := range caps { completions = append(completions, convertCase(cap)[offset:]) } diff --git a/vendor/github.com/containers/common/pkg/config/config.go b/vendor/github.com/containers/common/pkg/config/config.go index a86eca88e..25572968f 100644 --- a/vendor/github.com/containers/common/pkg/config/config.go +++ b/vendor/github.com/containers/common/pkg/config/config.go @@ -24,10 +24,6 @@ const ( // _configPath is the path to the containers/containers.conf // inside a given config directory. _configPath = "containers/containers.conf" - // DefaultContainersConfig holds the default containers config path - DefaultContainersConfig = "/usr/share/" + _configPath - // OverrideContainersConfig holds the default config path overridden by the root user - OverrideContainersConfig = "/etc/" + _configPath // UserOverrideContainersConfig holds the containers config path overridden by the rootless user UserOverrideContainersConfig = ".config/" + _configPath ) @@ -553,6 +549,9 @@ type SecretConfig struct { } // ConfigMapConfig represents the "configmap" TOML config table +// +// revive does not like the name because the package is already called config +//nolint:revive type ConfigMapConfig struct { // Driver specifies the configmap driver to use. // Current valid value: @@ -800,7 +799,7 @@ func (c *Config) Validate() error { func (c *EngineConfig) findRuntime() string { // Search for crun first followed by runc, kata, runsc - for _, name := range []string{"crun", "runc", "kata", "runsc"} { + for _, name := range []string{"crun", "runc", "runj", "kata", "runsc"} { for _, v := range c.OCIRuntimes[name] { if _, err := os.Stat(v); err == nil { return name @@ -1215,14 +1214,14 @@ func (c *Config) ActiveDestination() (uri, identity string, err error) { // FindHelperBinary will search the given binary name in the configured directories. // If searchPATH is set to true it will also search in $PATH. func (c *Config) FindHelperBinary(name string, searchPATH bool) (string, error) { - dir_list := c.Engine.HelperBinariesDir + dirList := c.Engine.HelperBinariesDir // If set, search this directory first. This is used in testing. if dir, found := os.LookupEnv("CONTAINERS_HELPER_BINARY_DIR"); found { - dir_list = append([]string{dir}, dir_list...) + dirList = append([]string{dir}, dirList...) } - for _, path := range dir_list { + for _, path := range dirList { fullpath := filepath.Join(path, name) if fi, err := os.Stat(fullpath); err == nil && fi.Mode().IsRegular() { return fullpath, nil diff --git a/vendor/github.com/containers/common/pkg/config/config_darwin.go b/vendor/github.com/containers/common/pkg/config/config_darwin.go index 5abb51f30..0ab9e0294 100644 --- a/vendor/github.com/containers/common/pkg/config/config_darwin.go +++ b/vendor/github.com/containers/common/pkg/config/config_darwin.go @@ -4,6 +4,14 @@ import ( "os" ) +const ( + // OverrideContainersConfig holds the default config path overridden by the root user + OverrideContainersConfig = "/etc/" + _configPath + + // DefaultContainersConfig holds the default containers config path + DefaultContainersConfig = "/usr/share/" + _configPath +) + // podman remote clients on darwin cannot use unshare.isRootless() to determine the configuration file locations. func customConfigFile() (string, error) { if path, found := os.LookupEnv("CONTAINERS_CONF"); found { diff --git a/vendor/github.com/containers/common/pkg/config/config_freebsd.go b/vendor/github.com/containers/common/pkg/config/config_freebsd.go index 85404a48d..d69812356 100644 --- a/vendor/github.com/containers/common/pkg/config/config_freebsd.go +++ b/vendor/github.com/containers/common/pkg/config/config_freebsd.go @@ -4,6 +4,14 @@ import ( "os" ) +const ( + // OverrideContainersConfig holds the default config path overridden by the root user + OverrideContainersConfig = "/usr/local/etc/" + _configPath + + // DefaultContainersConfig holds the default containers config path + DefaultContainersConfig = "/usr/local/share/" + _configPath +) + // podman remote clients on freebsd cannot use unshare.isRootless() to determine the configuration file locations. func customConfigFile() (string, error) { if path, found := os.LookupEnv("CONTAINERS_CONF"); found { diff --git a/vendor/github.com/containers/common/pkg/config/config_linux.go b/vendor/github.com/containers/common/pkg/config/config_linux.go index da0ae871a..4f0889f29 100644 --- a/vendor/github.com/containers/common/pkg/config/config_linux.go +++ b/vendor/github.com/containers/common/pkg/config/config_linux.go @@ -7,6 +7,14 @@ import ( selinux "github.com/opencontainers/selinux/go-selinux" ) +const ( + // OverrideContainersConfig holds the default config path overridden by the root user + OverrideContainersConfig = "/etc/" + _configPath + + // DefaultContainersConfig holds the default containers config path + DefaultContainersConfig = "/usr/share/" + _configPath +) + func selinuxEnabled() bool { return selinux.GetEnabled() } diff --git a/vendor/github.com/containers/common/pkg/config/config_windows.go b/vendor/github.com/containers/common/pkg/config/config_windows.go index dbe7ba00d..6c9d58485 100644 --- a/vendor/github.com/containers/common/pkg/config/config_windows.go +++ b/vendor/github.com/containers/common/pkg/config/config_windows.go @@ -2,6 +2,14 @@ package config import "os" +const ( + // OverrideContainersConfig holds the default config path overridden by the root user + OverrideContainersConfig = "/etc/" + _configPath + + // DefaultContainersConfig holds the default containers config path + DefaultContainersConfig = "/usr/share/" + _configPath +) + // podman remote clients on windows cannot use unshare.isRootless() to determine the configuration file locations. func customConfigFile() (string, error) { if path, found := os.LookupEnv("CONTAINERS_CONF"); found { diff --git a/vendor/github.com/containers/common/pkg/config/containers.conf-freebsd b/vendor/github.com/containers/common/pkg/config/containers.conf-freebsd new file mode 100644 index 000000000..50480fe73 --- /dev/null +++ b/vendor/github.com/containers/common/pkg/config/containers.conf-freebsd @@ -0,0 +1,636 @@ +# The containers configuration file specifies all of the available configuration +# command-line options/flags for container engine tools like Podman & Buildah, +# but in a TOML format that can be easily modified and versioned. + +# Please refer to containers.conf(5) for details of all configuration options. +# Not all container engines implement all of the options. +# All of the options have hard coded defaults and these options will override +# the built in defaults. Users can then override these options via the command +# line. Container engines will read containers.conf files in up to three +# locations in the following order: +# 1. /usr/local/share/containers/containers.conf +# 2. /usr/local/etc/containers/containers.conf +# 3. $HOME/.config/containers/containers.conf (Rootless containers ONLY) +# Items specified in the latter containers.conf, if they exist, override the +# previous containers.conf settings, or the default settings. + +[containers] + +# List of annotation. Specified as +# "key = value" +# If it is empty or commented out, no annotations will be added +# +#annotations = [] + +# The hosts entries from the base hosts file are added to the containers hosts +# file. This must be either an absolute path or as special values "image" which +# uses the hosts file from the container image or "none" which means +# no base hosts file is used. The default is "" which will use /etc/hosts. +# +#base_hosts_file = "" + +# List of default capabilities for containers. If it is empty or commented out, +# the default capabilities defined in the container engine will be added. +# +default_capabilities = [ + "CHOWN", + "DAC_OVERRIDE", + "FOWNER", + "FSETID", + "KILL", + "NET_BIND_SERVICE", + "SETFCAP", + "SETGID", + "SETPCAP", + "SETUID", + "SYS_CHROOT" +] + +# A list of sysctls to be set in containers by default, +# specified as "name=value", +# for example:"net.ipv4.ping_group_range=0 0". +# +default_sysctls = [ + "net.ipv4.ping_group_range=0 0", +] + +# A list of ulimits to be set in containers by default, specified as +# "<ulimit name>=<soft limit>:<hard limit>", for example: +# "nofile=1024:2048" +# See setrlimit(2) for a list of resource names. +# Any limit not specified here will be inherited from the process launching the +# container engine. +# Ulimits has limits for non privileged container engines. +# +#default_ulimits = [ +# "nofile=1280:2560", +#] + +# List of devices. Specified as +# "<device-on-host>:<device-on-container>:<permissions>", for example: +# "/dev/sdc:/dev/xvdc:rwm". +# If it is empty or commented out, only the default devices will be used +# +#devices = [] + +# List of default DNS options to be added to /etc/resolv.conf inside of the container. +# +#dns_options = [] + +# List of default DNS search domains to be added to /etc/resolv.conf inside of the container. +# +#dns_searches = [] + +# Set default DNS servers. +# This option can be used to override the DNS configuration passed to the +# container. The special value "none" can be specified to disable creation of +# /etc/resolv.conf in the container. +# The /etc/resolv.conf file in the image will be used without changes. +# +#dns_servers = [] + +# Environment variable list for the conmon process; used for passing necessary +# environment variables to conmon or the runtime. +# +#env = [ +# "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", +# "TERM=xterm", +#] + +# Pass all host environment variables into the container. +# +#env_host = false + +# Set the ip for the host.containers.internal entry in the containers /etc/hosts +# file. This can be set to "none" to disable adding this entry. By default it +# will automatically choose the host ip. +# +# NOTE: When using podman machine this entry will never be added to the containers +# hosts file instead the gvproxy dns resolver will resolve this hostname. Therefore +# it is not possible to disable the entry in this case. +# +#host_containers_internal_ip = "" + +# Default proxy environment variables passed into the container. +# The environment variables passed in include: +# http_proxy, https_proxy, ftp_proxy, no_proxy, and the upper case versions of +# these. This option is needed when host system uses a proxy but container +# should not use proxy. Proxy environment variables specified for the container +# in any other way will override the values passed from the host. +# +#http_proxy = true + +# Run an init inside the container that forwards signals and reaps processes. +# +#init = false + +# Container init binary, if init=true, this is the init binary to be used for containers. +# +#init_path = "/usr/local/libexec/podman/catatonit" + +# Default way to to create an IPC namespace (POSIX SysV IPC) for the container +# Options are: +# "host" Share host IPC Namespace with the container. +# "none" Create shareable IPC Namespace for the container without a private /dev/shm. +# "private" Create private IPC Namespace for the container, other containers are not allowed to share it. +# "shareable" Create shareable IPC Namespace for the container. +# +#ipcns = "shareable" + +# keyring tells the container engine whether to create +# a kernel keyring for use within the container. +# +#keyring = true + +# label tells the container engine whether to use container separation using +# MAC(SELinux) labeling or not. +# The label flag is ignored on label disabled systems. +# +#label = true + +# Logging driver for the container. Available options: k8s-file and journald. +# +#log_driver = "k8s-file" + +# Maximum size allowed for the container log file. Negative numbers indicate +# that no size limit is imposed. If positive, it must be >= 8192 to match or +# exceed conmon's read buffer. The file is truncated and re-opened so the +# limit is never exceeded. +# +#log_size_max = -1 + +# Specifies default format tag for container log messages. +# This is useful for creating a specific tag for container log messages. +# Containers logs default to truncated container ID as a tag. +# +#log_tag = "" + +# Default way to to create a Network namespace for the container +# Options are: +# `private` Create private Network Namespace for the container. +# `host` Share host Network Namespace with the container. +# `none` Containers do not use the network +# +#netns = "private" + +# Create /etc/hosts for the container. By default, container engine manage +# /etc/hosts, automatically adding the container's own IP address. +# +#no_hosts = false + +# Default way to to create a PID namespace for the container +# Options are: +# `private` Create private PID Namespace for the container. +# `host` Share host PID Namespace with the container. +# +#pidns = "private" + +# Maximum number of processes allowed in a container. +# +#pids_limit = 2048 + +# Copy the content from the underlying image into the newly created volume +# when the container is created instead of when it is started. If false, +# the container engine will not copy the content until the container is started. +# Setting it to true may have negative performance implications. +# +#prepare_volume_on_create = false + +# Set timezone in container. Takes IANA timezones as well as "local", +# which sets the timezone in the container to match the host machine. +# +#tz = "" + +# Set umask inside the container +# +#umask = "0022" + +# Default way to to create a User namespace for the container +# Options are: +# `auto` Create unique User Namespace for the container. +# `host` Share host User Namespace with the container. +# +#userns = "host" + +# Number of UIDs to allocate for the automatic container creation. +# UIDs are allocated from the "container" UIDs listed in +# /etc/subuid & /etc/subgid +# +#userns_size = 65536 + +# Default way to to create a UTS namespace for the container +# Options are: +# `private` Create private UTS Namespace for the container. +# `host` Share host UTS Namespace with the container. +# +#utsns = "private" + +# List of volumes. Specified as +# "<directory-on-host>:<directory-in-container>:<options>", for example: +# "/db:/var/lib/db:ro". +# If it is empty or commented out, no volumes will be added +# +#volumes = [] + +[secrets] +#driver = "file" + +[secrets.opts] +#root = "/example/directory" + +[network] + +# Network backend determines what network driver will be used to set up and tear down container networks. +# Valid values are "cni" and "netavark". +# The default value is empty which means that it will automatically choose CNI or netavark. If there are +# already containers/images or CNI networks preset it will choose CNI. +# +# Before changing this value all containers must be stopped otherwise it is likely that +# iptables rules and network interfaces might leak on the host. A reboot will fix this. +# +#network_backend = "" + +# Path to directory where CNI plugin binaries are located. +# +#cni_plugin_dirs = [ +# "/usr/local/libexec/cni", +# "/usr/libexec/cni", +# "/usr/local/lib/cni", +# "/usr/lib/cni", +# "/opt/cni/bin", +#] + +# The network name of the default network to attach pods to. +# +#default_network = "podman" + +# The default subnet for the default network given in default_network. +# If a network with that name does not exist, a new network using that name and +# this subnet will be created. +# Must be a valid IPv4 CIDR prefix. +# +#default_subnet = "10.88.0.0/16" + +# DefaultSubnetPools is a list of subnets and size which are used to +# allocate subnets automatically for podman network create. +# It will iterate through the list and will pick the first free subnet +# with the given size. This is only used for ipv4 subnets, ipv6 subnets +# are always assigned randomly. +# +#default_subnet_pools = [ +# {"base" = "10.89.0.0/16", "size" = 24}, +# {"base" = "10.90.0.0/15", "size" = 24}, +# {"base" = "10.92.0.0/14", "size" = 24}, +# {"base" = "10.96.0.0/11", "size" = 24}, +# {"base" = "10.128.0.0/9", "size" = 24}, +#] + +# Path to the directory where network configuration files are located. +# For the CNI backend the default is "/etc/cni/net.d" as root +# and "$HOME/.config/cni/net.d" as rootless. +# For the netavark backend "/etc/containers/networks" is used as root +# and "$graphroot/networks" as rootless. +# +#network_config_dir = "/usr/local/etc/cni/net.d/" + +[engine] +# Index to the active service +# +#active_service = production + +# The compression format to use when pushing an image. +# Valid options are: `gzip`, `zstd` and `zstd:chunked`. +# +#compression_format = "gzip" + +# Environment variables to pass into conmon +# +#conmon_env_vars = [ +# "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" +#] + +# Paths to look for the conmon container manager binary +# +#conmon_path = [ +# "/usr/libexec/podman/conmon", +# "/usr/local/libexec/podman/conmon", +# "/usr/local/lib/podman/conmon", +# "/usr/bin/conmon", +# "/usr/sbin/conmon", +# "/usr/local/bin/conmon", +# "/usr/local/sbin/conmon" +#] + +# Enforces using docker.io for completing short names in Podman's compatibility +# REST API. Note that this will ignore unqualified-search-registries and +# short-name aliases defined in containers-registries.conf(5). +#compat_api_enforce_docker_hub = true + +# Specify the keys sequence used to detach a container. +# Format is a single character [a-Z] or a comma separated sequence of +# `ctrl-<value>`, where `<value>` is one of: +# `a-z`, `@`, `^`, `[`, `\`, `]`, `^` or `_` +# +#detach_keys = "ctrl-p,ctrl-q" + +# Determines whether engine will reserve ports on the host when they are +# forwarded to containers. When enabled, when ports are forwarded to containers, +# ports are held open by as long as the container is running, ensuring that +# they cannot be reused by other programs on the host. However, this can cause +# significant memory usage if a container has many ports forwarded to it. +# Disabling this can save memory. +# +#enable_port_reservation = true + +# Environment variables to be used when running the container engine (e.g., Podman, Buildah). +# For example "http_proxy=internal.proxy.company.com". +# Note these environment variables will not be used within the container. +# Set the env section under [containers] table, if you want to set environment variables for the container. +# +#env = [] + +# Define where event logs will be stored, when events_logger is "file". +#events_logfile_path="" + +# Sets the maximum size for events_logfile_path. +# The size can be b (bytes), k (kilobytes), m (megabytes), or g (gigabytes). +# The format for the size is `<number><unit>`, e.g., `1b` or `3g`. +# If no unit is included then the size will be read in bytes. +# When the limit is exceeded, the logfile will be rotated and the old one will be deleted. +# If the maximum size is set to 0, then no limit will be applied, +# and the logfile will not be rotated. +#events_logfile_max_size = "1m" + +# Selects which logging mechanism to use for container engine events. +# Valid values are `journald`, `file` and `none`. +# +#events_logger = "file" + +# A is a list of directories which are used to search for helper binaries. +# +#helper_binaries_dir = [ +# "/usr/local/libexec/podman", +# "/usr/local/lib/podman", +# "/usr/libexec/podman", +# "/usr/lib/podman", +#] + +# Path to OCI hooks directories for automatically executed hooks. +# +#hooks_dir = [ +# "/usr/local/share/containers/oci/hooks.d", +#] + +# Manifest Type (oci, v2s2, or v2s1) to use when pulling, pushing, building +# container images. By default image pulled and pushed match the format of the +# source image. Building/committing defaults to OCI. +# +#image_default_format = "" + +# Default transport method for pulling and pushing for images +# +#image_default_transport = "docker://" + +# Maximum number of image layers to be copied (pulled/pushed) simultaneously. +# Not setting this field, or setting it to zero, will fall back to containers/image defaults. +# +#image_parallel_copies = 0 + +# Default command to run the infra container +# +#infra_command = "/pause" + +# Infra (pause) container image name for pod infra containers. When running a +# pod, we start a `pause` process in a container to hold open the namespaces +# associated with the pod. This container does nothing other then sleep, +# reserving the pods resources for the lifetime of the pod. By default container +# engines run a builtin container using the pause executable. If you want override +# specify an image to pull. +# +#infra_image = "" + +# Specify the locking mechanism to use; valid values are "shm" and "file". +# Change the default only if you are sure of what you are doing, in general +# "file" is useful only on platforms where cgo is not available for using the +# faster "shm" lock type. You may need to run "podman system renumber" after +# you change the lock type. +# +#lock_type** = "shm" + +# MultiImageArchive - if true, the container engine allows for storing archives +# (e.g., of the docker-archive transport) with multiple images. By default, +# Podman creates single-image archives. +# +#multi_image_archive = "false" + +# Default engine namespace +# If engine is joined to a namespace, it will see only containers and pods +# that were created in the same namespace, and will create new containers and +# pods in that namespace. +# The default namespace is "", which corresponds to no namespace. When no +# namespace is set, all containers and pods are visible. +# +#namespace = "" + +# Path to the slirp4netns binary +# +#network_cmd_path = "" + +# Default options to pass to the slirp4netns binary. +# Valid options values are: +# +# - allow_host_loopback=true|false: Allow the slirp4netns to reach the host loopback IP (`10.0.2.2`). +# Default is false. +# - mtu=MTU: Specify the MTU to use for this network. (Default is `65520`). +# - cidr=CIDR: Specify ip range to use for this network. (Default is `10.0.2.0/24`). +# - enable_ipv6=true|false: Enable IPv6. Default is true. (Required for `outbound_addr6`). +# - outbound_addr=INTERFACE: Specify the outbound interface slirp should bind to (ipv4 traffic only). +# - outbound_addr=IPv4: Specify the outbound ipv4 address slirp should bind to. +# - outbound_addr6=INTERFACE: Specify the outbound interface slirp should bind to (ipv6 traffic only). +# - outbound_addr6=IPv6: Specify the outbound ipv6 address slirp should bind to. +# - port_handler=rootlesskit: Use rootlesskit for port forwarding. Default. +# Note: Rootlesskit changes the source IP address of incoming packets to a IP address in the container +# network namespace, usually `10.0.2.100`. If your application requires the real source IP address, +# e.g. web server logs, use the slirp4netns port handler. The rootlesskit port handler is also used for +# rootless containers when connected to user-defined networks. +# - port_handler=slirp4netns: Use the slirp4netns port forwarding, it is slower than rootlesskit but +# preserves the correct source IP address. This port handler cannot be used for user-defined networks. +# +#network_cmd_options = [] + +# Whether to use chroot instead of pivot_root in the runtime +# +#no_pivot_root = false + +# Number of locks available for containers and pods. +# If this is changed, a lock renumber must be performed (e.g. with the +# 'podman system renumber' command). +# +#num_locks = 2048 + +# Whether to pull new image before running a container +# +#pull_policy = "missing" + +# Indicates whether the application should be running in remote mode. This flag modifies the +# --remote option on container engines. Setting the flag to true will default +# `podman --remote=true` for access to the remote Podman service. +# +#remote = false + +# Default OCI runtime +# +#runtime = "crun" + +# List of the OCI runtimes that support --format=json. When json is supported +# engine will use it for reporting nicer errors. +# +#runtime_supports_json = ["crun", "runc", "kata", "runsc", "krun"] + +# List of the OCI runtimes that supports running containers with KVM Separation. +# +#runtime_supports_kvm = ["kata", "krun"] + +# List of the OCI runtimes that supports running containers without cgroups. +# +#runtime_supports_nocgroups = ["crun", "krun"] + +# Default location for storing temporary container image content. Can be overridden with the TMPDIR environment +# variable. If you specify "storage", then the location of the +# container/storage tmp directory will be used. +# image_copy_tmp_dir="/var/tmp" + +# Number of seconds to wait without a connection +# before the `podman system service` times out and exits +# +#service_timeout = 5 + +# Directory for persistent engine files (database, etc) +# By default, this will be configured relative to where the containers/storage +# stores containers +# Uncomment to change location from this default +# +#static_dir = "/var/lib/containers/storage/libpod" + +# Number of seconds to wait for container to exit before sending kill signal. +# +#stop_timeout = 10 + +# Number of seconds to wait before exit command in API process is given to. +# This mimics Docker's exec cleanup behaviour, where the default is 5 minutes (value is in seconds). +# +#exit_command_delay = 300 + +# map of service destinations +# +#[service_destinations] +# [service_destinations.production] +# URI to access the Podman service +# Examples: +# rootless "unix://run/user/$UID/podman/podman.sock" (Default) +# rootful "unix://run/podman/podman.sock (Default) +# remote rootless ssh://engineering.lab.company.com/run/user/1000/podman/podman.sock +# remote rootful ssh://root@10.10.1.136:22/run/podman/podman.sock +# +# uri = "ssh://user@production.example.com/run/user/1001/podman/podman.sock" +# Path to file containing ssh identity key +# identity = "~/.ssh/id_rsa" + +# Directory for temporary files. Must be tmpfs (wiped after reboot) +# +#tmp_dir = "/run/libpod" + +# Directory for libpod named volumes. +# By default, this will be configured relative to where containers/storage +# stores containers. +# Uncomment to change location from this default. +# +#volume_path = "/var/lib/containers/storage/volumes" + +# Paths to look for a valid OCI runtime (crun, runc, kata, runsc, krun, etc) +[engine.runtimes] +#crun = [ +# "/usr/bin/crun", +# "/usr/sbin/crun", +# "/usr/local/bin/crun", +# "/usr/local/sbin/crun", +# "/sbin/crun", +# "/bin/crun", +# "/run/current-system/sw/bin/crun", +#] + +#kata = [ +# "/usr/bin/kata-runtime", +# "/usr/sbin/kata-runtime", +# "/usr/local/bin/kata-runtime", +# "/usr/local/sbin/kata-runtime", +# "/sbin/kata-runtime", +# "/bin/kata-runtime", +# "/usr/bin/kata-qemu", +# "/usr/bin/kata-fc", +#] + +#runc = [ +# "/usr/bin/runc", +# "/usr/sbin/runc", +# "/usr/local/bin/runc", +# "/usr/local/sbin/runc", +# "/sbin/runc", +# "/bin/runc", +# "/usr/lib/cri-o-runc/sbin/runc", +#] + +#runsc = [ +# "/usr/bin/runsc", +# "/usr/sbin/runsc", +# "/usr/local/bin/runsc", +# "/usr/local/sbin/runsc", +# "/bin/runsc", +# "/sbin/runsc", +# "/run/current-system/sw/bin/runsc", +#] + +#krun = [ +# "/usr/bin/krun", +# "/usr/local/bin/krun", +#] + +[engine.volume_plugins] +#testplugin = "/var/run/podman/plugins/test.sock" + +[machine] +# Number of CPU's a machine is created with. +# +#cpus=1 + +# The size of the disk in GB created when init-ing a podman-machine VM. +# +#disk_size=10 + +# The image used when creating a podman-machine VM. +# +#image = "testing" + +# Memory in MB a machine is created with. +# +#memory=2048 + +# The username to use and create on the podman machine OS for rootless +# container access. +# +#user = "core" + +# Host directories to be mounted as volumes into the VM by default. +# Environment variables like $HOME as well as complete paths are supported for +# the source and destination. An optional third field `:ro` can be used to +# tell the container engines to mount the volume readonly. +# +# volumes = [ +# "$HOME:$HOME", +#] + +# The [machine] table MUST be the last entry in this file. +# (Unless another table is added) +# TOML does not provide a way to end a table other than a further table being +# defined, so every key hereafter will be part of [machine] and not the +# main config. diff --git a/vendor/github.com/containers/common/pkg/config/default.go b/vendor/github.com/containers/common/pkg/config/default.go index 8979a406b..d988d3b1c 100644 --- a/vendor/github.com/containers/common/pkg/config/default.go +++ b/vendor/github.com/containers/common/pkg/config/default.go @@ -193,7 +193,7 @@ func DefaultConfig() (*Config, error) { ApparmorProfile: DefaultApparmorProfile, BaseHostsFile: "", CgroupNS: cgroupNS, - Cgroups: "enabled", + Cgroups: getDefaultCgroupsMode(), DefaultCapabilities: DefaultCapabilities, DefaultSysctls: []string{}, DefaultUlimits: getDefaultProcessLimits(), @@ -323,6 +323,9 @@ func defaultConfigFromMemory() (*EngineConfig, error) { "/usr/lib/cri-o-runc/sbin/runc", "/run/current-system/sw/bin/runc", }, + "runj": { + "/usr/local/bin/runj", + }, "kata": { "/usr/bin/kata-runtime", "/usr/sbin/kata-runtime", @@ -384,7 +387,7 @@ func defaultConfigFromMemory() (*EngineConfig, error) { c.SDNotify = false // TODO - ideally we should expose a `type LockType string` along with // constants. - c.LockType = "shm" + c.LockType = getDefaultLockType() c.MachineEnabled = false c.ChownCopiedFiles = true @@ -395,7 +398,7 @@ func defaultConfigFromMemory() (*EngineConfig, error) { func defaultTmpDir() (string, error) { if !unshare.IsRootless() { - return "/run/libpod", nil + return getLibpodTmpDir(), nil } runtimeDir, err := util.GetRuntimeDir() diff --git a/vendor/github.com/containers/common/pkg/config/default_darwin.go b/vendor/github.com/containers/common/pkg/config/default_darwin.go new file mode 100644 index 000000000..c502ea55e --- /dev/null +++ b/vendor/github.com/containers/common/pkg/config/default_darwin.go @@ -0,0 +1,13 @@ +package config + +func getDefaultCgroupsMode() string { + return "enabled" +} + +func getDefaultLockType() string { + return "shm" +} + +func getLibpodTmpDir() string { + return "/run/libpod" +} diff --git a/vendor/github.com/containers/common/pkg/config/default_freebsd.go b/vendor/github.com/containers/common/pkg/config/default_freebsd.go new file mode 100644 index 000000000..8b10ac1f7 --- /dev/null +++ b/vendor/github.com/containers/common/pkg/config/default_freebsd.go @@ -0,0 +1,20 @@ +package config + +func getDefaultCgroupsMode() string { + return "disabled" +} + +// In theory, FreeBSD should be able to use shm locks but in practice, +// this causes cryptic error messages from the kernel that look like: +// +// comm podman pid 90813: handling rb error 22 +// +// These seem to be related to fork/exec code paths. Fall back to +// file-based locks. +func getDefaultLockType() string { + return "file" +} + +func getLibpodTmpDir() string { + return "/var/run/libpod" +} diff --git a/vendor/github.com/containers/common/pkg/config/default_linux.go b/vendor/github.com/containers/common/pkg/config/default_linux.go index d6ea4359c..86873beb1 100644 --- a/vendor/github.com/containers/common/pkg/config/default_linux.go +++ b/vendor/github.com/containers/common/pkg/config/default_linux.go @@ -14,6 +14,10 @@ const ( oldMaxSize = uint64(1048576) ) +func getDefaultCgroupsMode() string { + return "enabled" +} + // getDefaultMachineImage returns the default machine image stream // On Linux/Mac, this returns the FCOS stream func getDefaultMachineImage() string { @@ -58,3 +62,11 @@ func getDefaultTmpDir() string { } return "/var/tmp" } + +func getDefaultLockType() string { + return "shm" +} + +func getLibpodTmpDir() string { + return "/run/libpod" +} diff --git a/vendor/github.com/containers/common/pkg/config/default_windows.go b/vendor/github.com/containers/common/pkg/config/default_windows.go index db230dfb2..1ff88fc42 100644 --- a/vendor/github.com/containers/common/pkg/config/default_windows.go +++ b/vendor/github.com/containers/common/pkg/config/default_windows.go @@ -32,3 +32,15 @@ func getDefaultTmpDir() string { } return os.Getenv("LOCALAPPDATA") + "\\Temp" } + +func getDefaultCgroupsMode() string { + return "enabled" +} + +func getDefaultLockType() string { + return "shm" +} + +func getLibpodTmpDir() string { + return "/run/libpod" +} diff --git a/vendor/github.com/containers/common/pkg/filters/filters.go b/vendor/github.com/containers/common/pkg/filters/filters.go index e26e056ad..53650efc9 100644 --- a/vendor/github.com/containers/common/pkg/filters/filters.go +++ b/vendor/github.com/containers/common/pkg/filters/filters.go @@ -36,11 +36,13 @@ func ComputeUntilTimestamp(filterValues []string) (time.Time, error) { // // Please refer to https://github.com/containers/podman/issues/6899 for some // background. +// +// revive does not like the name because the package is already called filters +//nolint:revive func FiltersFromRequest(r *http.Request) ([]string, error) { var ( compatFilters map[string]map[string]bool filters map[string][]string - libpodFilters []string raw []byte ) @@ -54,6 +56,7 @@ func FiltersFromRequest(r *http.Request) ([]string, error) { // Backwards compat with older versions of Docker. if err := json.Unmarshal(raw, &compatFilters); err == nil { + libpodFilters := make([]string, 0, len(compatFilters)) for filterKey, filterMap := range compatFilters { for filterValue, toAdd := range filterMap { if toAdd { @@ -68,6 +71,7 @@ func FiltersFromRequest(r *http.Request) ([]string, error) { return nil, err } + libpodFilters := make([]string, 0, len(filters)) for filterKey, filterSlice := range filters { f := filterKey for _, filterValue := range filterSlice { diff --git a/vendor/github.com/containers/common/pkg/machine/machine.go b/vendor/github.com/containers/common/pkg/machine/machine.go index 465eeceaf..37e89a08e 100644 --- a/vendor/github.com/containers/common/pkg/machine/machine.go +++ b/vendor/github.com/containers/common/pkg/machine/machine.go @@ -9,6 +9,8 @@ import ( "github.com/sirupsen/logrus" ) +// TODO: change name to MachineMarker since package is already called machine +//nolint:revive type MachineMarker struct { Enabled bool Type string @@ -54,6 +56,8 @@ func IsPodmanMachine() bool { return GetMachineMarker().Enabled } +// TODO: change name to HostType since package is already called machine +//nolint:revive func MachineHostType() string { return GetMachineMarker().Type } diff --git a/vendor/github.com/containers/common/pkg/parse/parse_unix.go b/vendor/github.com/containers/common/pkg/parse/parse_unix.go index d087c4a02..6fb52a014 100644 --- a/vendor/github.com/containers/common/pkg/parse/parse_unix.go +++ b/vendor/github.com/containers/common/pkg/parse/parse_unix.go @@ -1,5 +1,5 @@ -//go:build linux || darwin -// +build linux darwin +//go:build linux || darwin || freebsd +// +build linux darwin freebsd package parse @@ -13,7 +13,6 @@ import ( ) func DeviceFromPath(device string) ([]devices.Device, error) { - var devs []devices.Device src, dst, permissions, err := Device(device) if err != nil { return nil, err @@ -27,7 +26,7 @@ func DeviceFromPath(device string) ([]devices.Device, error) { } if !srcInfo.IsDir() { - + devs := make([]devices.Device, 0, 1) dev, err := devices.DeviceFromPath(src, permissions) if err != nil { return nil, errors.Wrapf(err, "%s is not a valid device", src) @@ -42,6 +41,7 @@ func DeviceFromPath(device string) ([]devices.Device, error) { if err != nil { return nil, errors.Wrapf(err, "error getting source devices from directory %s", src) } + devs := make([]devices.Device, 0, len(srcDevices)) for _, d := range srcDevices { d.Path = filepath.Join(dst, filepath.Base(d.Path)) d.Permissions = devices.Permissions(permissions) diff --git a/vendor/github.com/containers/common/pkg/retry/retry.go b/vendor/github.com/containers/common/pkg/retry/retry.go index a9573e4e8..234fd3448 100644 --- a/vendor/github.com/containers/common/pkg/retry/retry.go +++ b/vendor/github.com/containers/common/pkg/retry/retry.go @@ -17,12 +17,17 @@ import ( ) // RetryOptions defines the option to retry +// revive does not like the name because the package is already called retry +//nolint:revive type RetryOptions struct { MaxRetry int // The number of times to possibly retry Delay time.Duration // The delay to use between retries, if set } // RetryIfNecessary retries the operation in exponential backoff with the retryOptions +// +// revive does not like the name because the package is already called retry +//nolint:revive func RetryIfNecessary(ctx context.Context, operation func() error, retryOptions *RetryOptions) error { err := operation() for attempt := 0; err != nil && isRetryable(err) && attempt < retryOptions.MaxRetry; attempt++ { diff --git a/vendor/github.com/containers/common/pkg/seccomp/conversion.go b/vendor/github.com/containers/common/pkg/seccomp/conversion.go index 4c25cb1b1..cd599f0f3 100644 --- a/vendor/github.com/containers/common/pkg/seccomp/conversion.go +++ b/vendor/github.com/containers/common/pkg/seccomp/conversion.go @@ -71,11 +71,12 @@ var ( // https://github.com/opencontainers/runtime-spec/pull/1064 // specs.ActKillProcess ActKillProcess, // specs.ActKillThread ActKillThread, - specs.ActErrno: ActErrno, - specs.ActTrap: ActTrap, - specs.ActAllow: ActAllow, - specs.ActTrace: ActTrace, - specs.ActLog: ActLog, + specs.ActErrno: ActErrno, + specs.ActTrap: ActTrap, + specs.ActAllow: ActAllow, + specs.ActTrace: ActTrace, + specs.ActLog: ActLog, + specs.ActNotify: ActNotify, } specOperatorToSeccompOperatorMap = map[specs.LinuxSeccompOperator]Operator{ specs.OpNotEqual: OpNotEqual, diff --git a/vendor/github.com/containers/common/pkg/seccomp/filter.go b/vendor/github.com/containers/common/pkg/seccomp/filter.go index 5c278574c..609036c82 100644 --- a/vendor/github.com/containers/common/pkg/seccomp/filter.go +++ b/vendor/github.com/containers/common/pkg/seccomp/filter.go @@ -130,7 +130,7 @@ func matchSyscall(filter *libseccomp.ScmpFilter, call *Syscall) error { return errors.Wrapf(err, "create seccomp syscall condition for syscall %s", call.Name) } - argCounts[cond.Index] += 1 + argCounts[cond.Index]++ conditions = append(conditions, newCond) } diff --git a/vendor/github.com/containers/common/pkg/seccomp/types.go b/vendor/github.com/containers/common/pkg/seccomp/types.go index 784b1ba8d..56fd22a38 100644 --- a/vendor/github.com/containers/common/pkg/seccomp/types.go +++ b/vendor/github.com/containers/common/pkg/seccomp/types.go @@ -75,6 +75,7 @@ const ( ActTrace Action = "SCMP_ACT_TRACE" ActAllow Action = "SCMP_ACT_ALLOW" ActLog Action = "SCMP_ACT_LOG" + ActNotify Action = "SCMP_ACT_NOTIFY" ) // Operator used to match syscall arguments in Seccomp diff --git a/vendor/github.com/containers/common/pkg/secrets/filedriver/filedriver.go b/vendor/github.com/containers/common/pkg/secrets/filedriver/filedriver.go index e0c275851..6b92714d0 100644 --- a/vendor/github.com/containers/common/pkg/secrets/filedriver/filedriver.go +++ b/vendor/github.com/containers/common/pkg/secrets/filedriver/filedriver.go @@ -55,7 +55,7 @@ func (d *Driver) List() ([]string, error) { if err != nil { return nil, err } - var allID []string + allID := make([]string, 0, len(secretData)) for k := range secretData { allID = append(allID, k) } @@ -134,9 +134,8 @@ func (d *Driver) getAllData() (map[string][]byte, error) { if os.IsNotExist(err) { // the file will be created later on a store() return make(map[string][]byte), nil - } else { - return nil, err } + return nil, err } file, err := os.Open(d.secretsDataFilePath) diff --git a/vendor/github.com/containers/common/pkg/secrets/secrets.go b/vendor/github.com/containers/common/pkg/secrets/secrets.go index 4a04e2b2f..e45995b2e 100644 --- a/vendor/github.com/containers/common/pkg/secrets/secrets.go +++ b/vendor/github.com/containers/common/pkg/secrets/secrets.go @@ -53,6 +53,9 @@ var secretsFile = "secrets.json" var secretNameRegexp = regexp.MustCompile(`^[a-zA-Z0-9][a-zA-Z0-9_.-]*$`) // SecretsManager holds information on handling secrets +// +// revive does not like the name because the package is already called secrets +//nolint:revive type SecretsManager struct { // secretsPath is the path to the db file where secrets are stored secretsDBPath string @@ -82,6 +85,9 @@ type Secret struct { // The driver stores the actual bytes of secret data, as opposed to // the secret metadata. // Currently only the unencrypted filedriver is implemented. +// +// revive does not like the name because the package is already called secrets +//nolint:revive type SecretsDriver interface { // List lists all secret ids in the secrets data store List() ([]string, error) @@ -234,7 +240,7 @@ func (s *SecretsManager) List() ([]Secret, error) { if err != nil { return nil, err } - var ls []Secret + ls := make([]Secret, 0, len(secrets)) for _, v := range secrets { ls = append(ls, v) } @@ -276,9 +282,8 @@ func getDriver(name string, opts map[string]string) (SecretsDriver, error) { case "file": if path, ok := opts["path"]; ok { return filedriver.NewDriver(path) - } else { - return nil, errors.Wrap(errInvalidDriverOpt, "need path for filedriver") } + return nil, errors.Wrap(errInvalidDriverOpt, "need path for filedriver") case "pass": return passdriver.NewDriver(opts) case "shell": diff --git a/vendor/github.com/containers/common/pkg/secrets/secretsdb.go b/vendor/github.com/containers/common/pkg/secrets/secretsdb.go index 4d2ca0fca..8d21aa754 100644 --- a/vendor/github.com/containers/common/pkg/secrets/secretsdb.go +++ b/vendor/github.com/containers/common/pkg/secrets/secretsdb.go @@ -31,9 +31,8 @@ func (s *SecretsManager) loadDB() error { // the db cache will show no entries anyway. // The file will be created later on a store() return nil - } else { - return err } + return err } // We check if the file has been modified after the last time it was loaded into the cache. diff --git a/vendor/github.com/containers/common/pkg/subscriptions/subscriptions.go b/vendor/github.com/containers/common/pkg/subscriptions/subscriptions.go index 4410292a7..b111ae7ff 100644 --- a/vendor/github.com/containers/common/pkg/subscriptions/subscriptions.go +++ b/vendor/github.com/containers/common/pkg/subscriptions/subscriptions.go @@ -212,8 +212,8 @@ func rchown(chowndir string, uid, gid int) error { // addSubscriptionsFromMountsFile copies the contents of host directory to container directory // and returns a list of mounts func addSubscriptionsFromMountsFile(filePath, mountLabel, containerRunDir string, uid, gid int) ([]rspec.Mount, error) { - var mounts []rspec.Mount defaultMountsPaths := getMounts(filePath) + mounts := make([]rspec.Mount, 0, len(defaultMountsPaths)) for _, path := range defaultMountsPaths { hostDirOrFile, ctrDirOrFile, err := getMountsMap(path) if err != nil { diff --git a/vendor/github.com/containers/common/pkg/sysinfo/nummem_linux.go b/vendor/github.com/containers/common/pkg/sysinfo/nummem_linux.go index 859791e36..018c488be 100644 --- a/vendor/github.com/containers/common/pkg/sysinfo/nummem_linux.go +++ b/vendor/github.com/containers/common/pkg/sysinfo/nummem_linux.go @@ -12,6 +12,8 @@ import ( // NUMANodeCount queries the system for the count of Memory Nodes available // for use to this process. func NUMANodeCount() int { + // this is the correct flag name (not defined in the unix package) + //nolint:revive MPOL_F_MEMS_ALLOWED := (1 << 2) var mask [1024 / 64]uintptr _, _, err := unix.RawSyscall6(unix.SYS_GET_MEMPOLICY, 0, uintptr(unsafe.Pointer(&mask[0])), uintptr(len(mask)*8), 0, uintptr(MPOL_F_MEMS_ALLOWED), 0) diff --git a/vendor/github.com/containers/common/version/version.go b/vendor/github.com/containers/common/version/version.go index 61fce9d22..f8a24f823 100644 --- a/vendor/github.com/containers/common/version/version.go +++ b/vendor/github.com/containers/common/version/version.go @@ -1,4 +1,4 @@ package version // Version is the version of the build. -const Version = "0.48.0" +const Version = "0.49.0-dev" diff --git a/vendor/github.com/containers/image/v5/docker/docker_client.go b/vendor/github.com/containers/image/v5/docker/docker_client.go index f537de72d..daac45f87 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_client.go +++ b/vendor/github.com/containers/image/v5/docker/docker_client.go @@ -61,8 +61,8 @@ type certPath struct { var ( homeCertDir = filepath.FromSlash(".config/containers/certs.d") perHostCertDirs = []certPath{ - {path: "/etc/containers/certs.d", absolute: true}, - {path: "/etc/docker/certs.d", absolute: true}, + {path: etcDir + "/containers/certs.d", absolute: true}, + {path: etcDir + "/docker/certs.d", absolute: true}, } defaultUserAgent = "containers/" + version.Version + " (github.com/containers/image)" diff --git a/vendor/github.com/containers/image/v5/docker/docker_image_src.go b/vendor/github.com/containers/image/v5/docker/docker_image_src.go index 085a3afcc..27fb838f5 100644 --- a/vendor/github.com/containers/image/v5/docker/docker_image_src.go +++ b/vendor/github.com/containers/image/v5/docker/docker_image_src.go @@ -9,6 +9,7 @@ import ( "net/http" "net/url" "os" + "regexp" "strconv" "strings" "sync" @@ -343,12 +344,16 @@ func handle206Response(streams chan io.ReadCloser, errs chan error, body io.Read buffered := makeBufferedNetworkReader(body, 64, 16384) defer buffered.Close() mr := multipart.NewReader(buffered, boundary) + parts := 0 for { p, err := mr.NextPart() if err != nil { if err != io.EOF { errs <- err } + if parts != len(chunks) { + errs <- errors.Errorf("invalid number of chunks returned by the server") + } return } s := signalCloseReader{ @@ -359,7 +364,32 @@ func handle206Response(streams chan io.ReadCloser, errs chan error, body io.Read // NextPart() cannot be called while the current part // is being read, so wait until it is closed <-s.closed + parts++ + } +} + +var multipartByteRangesRe = regexp.MustCompile("multipart/byteranges; boundary=([A-Za-z-0-9:]+)") + +func parseMediaType(contentType string) (string, map[string]string, error) { + mediaType, params, err := mime.ParseMediaType(contentType) + if err != nil { + if err == mime.ErrInvalidMediaParameter { + // CloudFront returns an invalid MIME type, that contains an unquoted ":" in the boundary + // param, let's handle it here. + matches := multipartByteRangesRe.FindStringSubmatch(contentType) + if len(matches) == 2 { + mediaType = "multipart/byteranges" + params = map[string]string{ + "boundary": matches[1], + } + err = nil + } + } + if err != nil { + return "", nil, err + } } + return mediaType, params, err } // GetBlobAt returns a sequential channel of readers that contain data for the requested @@ -397,7 +427,7 @@ func (s *dockerImageSource) GetBlobAt(ctx context.Context, info types.BlobInfo, go splitHTTP200ResponseToPartial(streams, errs, res.Body, chunks) return streams, errs, nil case http.StatusPartialContent: - mediaType, params, err := mime.ParseMediaType(res.Header.Get("Content-Type")) + mediaType, params, err := parseMediaType(res.Header.Get("Content-Type")) if err != nil { return nil, nil, err } diff --git a/vendor/github.com/containers/image/v5/docker/lookaside.go b/vendor/github.com/containers/image/v5/docker/lookaside.go index d0a3f1be0..3294d7def 100644 --- a/vendor/github.com/containers/image/v5/docker/lookaside.go +++ b/vendor/github.com/containers/image/v5/docker/lookaside.go @@ -25,7 +25,7 @@ var systemRegistriesDirPath = builtinRegistriesDirPath // builtinRegistriesDirPath is the path to registries.d. // DO NOT change this, instead see systemRegistriesDirPath above. -const builtinRegistriesDirPath = "/etc/containers/registries.d" +const builtinRegistriesDirPath = etcDir + "/containers/registries.d" // userRegistriesDirPath is the path to the per user registries.d. var userRegistriesDir = filepath.FromSlash(".config/containers/registries.d") diff --git a/vendor/github.com/containers/image/v5/docker/paths_common.go b/vendor/github.com/containers/image/v5/docker/paths_common.go new file mode 100644 index 000000000..862e88039 --- /dev/null +++ b/vendor/github.com/containers/image/v5/docker/paths_common.go @@ -0,0 +1,6 @@ +//go:build !freebsd +// +build !freebsd + +package docker + +const etcDir = "/etc" diff --git a/vendor/github.com/containers/image/v5/docker/paths_freebsd.go b/vendor/github.com/containers/image/v5/docker/paths_freebsd.go new file mode 100644 index 000000000..2bf27ac06 --- /dev/null +++ b/vendor/github.com/containers/image/v5/docker/paths_freebsd.go @@ -0,0 +1,6 @@ +//go:build freebsd +// +build freebsd + +package docker + +const etcDir = "/usr/local/etc" diff --git a/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/paths_common.go b/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/paths_common.go new file mode 100644 index 000000000..07fe50294 --- /dev/null +++ b/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/paths_common.go @@ -0,0 +1,12 @@ +//go:build !freebsd +// +build !freebsd + +package sysregistriesv2 + +// builtinRegistriesConfPath is the path to the registry configuration file. +// DO NOT change this, instead see systemRegistriesConfPath above. +const builtinRegistriesConfPath = "/etc/containers/registries.conf" + +// builtinRegistriesConfDirPath is the path to the registry configuration directory. +// DO NOT change this, instead see systemRegistriesConfDirectoryPath above. +const builtinRegistriesConfDirPath = "/etc/containers/registries.conf.d" diff --git a/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/paths_freebsd.go b/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/paths_freebsd.go new file mode 100644 index 000000000..741b99f8f --- /dev/null +++ b/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/paths_freebsd.go @@ -0,0 +1,12 @@ +//go:build freebsd +// +build freebsd + +package sysregistriesv2 + +// builtinRegistriesConfPath is the path to the registry configuration file. +// DO NOT change this, instead see systemRegistriesConfPath above. +const builtinRegistriesConfPath = "/usr/local/etc/containers/registries.conf" + +// builtinRegistriesConfDirPath is the path to the registry configuration directory. +// DO NOT change this, instead see systemRegistriesConfDirectoryPath above. +const builtinRegistriesConfDirPath = "/usr/local/etc/containers/registries.conf.d" diff --git a/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go b/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go index c1753c845..002b28d75 100644 --- a/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go +++ b/vendor/github.com/containers/image/v5/pkg/sysregistriesv2/system_registries_v2.go @@ -25,20 +25,12 @@ import ( // -ldflags '-X github.com/containers/image/v5/sysregistries.systemRegistriesConfPath=$your_path' var systemRegistriesConfPath = builtinRegistriesConfPath -// builtinRegistriesConfPath is the path to the registry configuration file. -// DO NOT change this, instead see systemRegistriesConfPath above. -const builtinRegistriesConfPath = "/etc/containers/registries.conf" - // systemRegistriesConfDirPath is the path to the system-wide registry // configuration directory and is used to add/subtract potential registries for // obtaining images. You can override this at build time with // -ldflags '-X github.com/containers/image/v5/sysregistries.systemRegistriesConfDirectoryPath=$your_path' var systemRegistriesConfDirPath = builtinRegistriesConfDirPath -// builtinRegistriesConfDirPath is the path to the registry configuration directory. -// DO NOT change this, instead see systemRegistriesConfDirectoryPath above. -const builtinRegistriesConfDirPath = "/etc/containers/registries.conf.d" - // AuthenticationFileHelper is a special key for credential helpers indicating // the usage of consulting containers-auth.json files instead of a credential // helper. diff --git a/vendor/github.com/containers/image/v5/signature/policy_config.go b/vendor/github.com/containers/image/v5/signature/policy_config.go index bb91cae8c..2ed0f882b 100644 --- a/vendor/github.com/containers/image/v5/signature/policy_config.go +++ b/vendor/github.com/containers/image/v5/signature/policy_config.go @@ -32,10 +32,6 @@ import ( // -ldflags '-X github.com/containers/image/v5/signature.systemDefaultPolicyPath=$your_path' var systemDefaultPolicyPath = builtinDefaultPolicyPath -// builtinDefaultPolicyPath is the policy path used for DefaultPolicy(). -// DO NOT change this, instead see systemDefaultPolicyPath above. -const builtinDefaultPolicyPath = "/etc/containers/policy.json" - // userPolicyFile is the path to the per user policy path. var userPolicyFile = filepath.FromSlash(".config/containers/policy.json") diff --git a/vendor/github.com/containers/image/v5/signature/policy_paths_common.go b/vendor/github.com/containers/image/v5/signature/policy_paths_common.go new file mode 100644 index 000000000..290fc2459 --- /dev/null +++ b/vendor/github.com/containers/image/v5/signature/policy_paths_common.go @@ -0,0 +1,8 @@ +//go:build !freebsd +// +build !freebsd + +package signature + +// builtinDefaultPolicyPath is the policy path used for DefaultPolicy(). +// DO NOT change this, instead see systemDefaultPolicyPath above. +const builtinDefaultPolicyPath = "/etc/containers/policy.json" diff --git a/vendor/github.com/containers/image/v5/signature/policy_paths_freebsd.go b/vendor/github.com/containers/image/v5/signature/policy_paths_freebsd.go new file mode 100644 index 000000000..702b7171f --- /dev/null +++ b/vendor/github.com/containers/image/v5/signature/policy_paths_freebsd.go @@ -0,0 +1,8 @@ +//go:build freebsd +// +build freebsd + +package signature + +// builtinDefaultPolicyPath is the policy path used for DefaultPolicy(). +// DO NOT change this, instead see systemDefaultPolicyPath above. +const builtinDefaultPolicyPath = "/usr/local/etc/containers/policy.json" diff --git a/vendor/github.com/containers/image/v5/version/version.go b/vendor/github.com/containers/image/v5/version/version.go index 6295b5493..f15c5ac20 100644 --- a/vendor/github.com/containers/image/v5/version/version.go +++ b/vendor/github.com/containers/image/v5/version/version.go @@ -8,10 +8,10 @@ const ( // VersionMinor is for functionality in a backwards-compatible manner VersionMinor = 21 // VersionPatch is for backwards-compatible bug fixes - VersionPatch = 1 + VersionPatch = 2 // VersionDev indicates development branch. Releases will be empty string. - VersionDev = "" + VersionDev = "-dev" ) // Version is the specification version that the package types support. diff --git a/vendor/github.com/containers/storage/Makefile b/vendor/github.com/containers/storage/Makefile index 2c1e4a185..244576d54 100644 --- a/vendor/github.com/containers/storage/Makefile +++ b/vendor/github.com/containers/storage/Makefile @@ -59,8 +59,8 @@ binary local-binary: containers-storage local-gccgo: ## build using gccgo on the host GCCGO=$(PWD)/hack/gccgo-wrapper.sh $(GO) build $(MOD_VENDOR) -compiler gccgo $(BUILDFLAGS) -o containers-storage.gccgo ./cmd/containers-storage -local-cross: ## cross build the binaries for arm, darwin, and\nfreebsd - @for target in linux/amd64 linux/386 linux/arm linux/arm64 linux/ppc64 linux/ppc64le darwin/amd64 windows/amd64 ; do \ +local-cross: ## cross build the binaries for arm, darwin, and freebsd + @for target in linux/amd64 linux/386 linux/arm linux/arm64 linux/ppc64 linux/ppc64le darwin/amd64 windows/amd64 freebsd/amd64 freebsd/arm64 ; do \ os=`echo $${target} | cut -f1 -d/` ; \ arch=`echo $${target} | cut -f2 -d/` ; \ suffix=$${os}.$${arch} ; \ diff --git a/vendor/github.com/containers/storage/VERSION b/vendor/github.com/containers/storage/VERSION index 148dabb45..ce045fe2d 100644 --- a/vendor/github.com/containers/storage/VERSION +++ b/vendor/github.com/containers/storage/VERSION @@ -1 +1 @@ -1.40.2 +1.41.1-dev diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index 09d24ae8b..8600ee685 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -207,14 +207,18 @@ func checkSupportVolatile(home, runhome string) (bool, error) { // checkAndRecordIDMappedSupport checks and stores if the kernel supports mounting overlay on top of a // idmapped lower layer. func checkAndRecordIDMappedSupport(home, runhome string) (bool, error) { + if os.Geteuid() != 0 { + return false, nil + } + feature := "idmapped-lower-dir" overlayCacheResult, overlayCacheText, err := cachedFeatureCheck(runhome, feature) if err == nil { if overlayCacheResult { - logrus.Debugf("Cached value indicated that overlay is supported") + logrus.Debugf("Cached value indicated that idmapped mounts for overlay are supported") return true, nil } - logrus.Debugf("Cached value indicated that overlay is not supported") + logrus.Debugf("Cached value indicated that idmapped mounts for overlay are not supported") return false, errors.New(overlayCacheText) } supportsIDMappedMounts, err := supportsIdmappedLowerLayers(home) diff --git a/vendor/github.com/containers/storage/go.mod b/vendor/github.com/containers/storage/go.mod index 5d8c59960..1b9f25bcb 100644 --- a/vendor/github.com/containers/storage/go.mod +++ b/vendor/github.com/containers/storage/go.mod @@ -12,7 +12,7 @@ require ( github.com/google/go-intervals v0.0.2 github.com/hashicorp/go-multierror v1.1.1 github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.15.2 + github.com/klauspost/compress v1.15.4 github.com/klauspost/pgzip v1.2.5 github.com/mattn/go-shellwords v1.0.12 github.com/mistifyio/go-zfs v2.1.2-0.20190413222219-f784269be439+incompatible diff --git a/vendor/github.com/containers/storage/go.sum b/vendor/github.com/containers/storage/go.sum index 97a0d167d..6587fddb3 100644 --- a/vendor/github.com/containers/storage/go.sum +++ b/vendor/github.com/containers/storage/go.sum @@ -425,8 +425,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.11.3/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.11.13/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.2 h1:3WH+AG7s2+T8o3nrM/8u2rdqUEcQhmga7smjrT41nAw= -github.com/klauspost/compress v1.15.2/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= +github.com/klauspost/compress v1.15.4 h1:1kn4/7MepF/CHmYub99/nNX8az0IJjfSOU/jbnTVfqQ= +github.com/klauspost/compress v1.15.4/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= github.com/klauspost/pgzip v1.2.5 h1:qnWYvvKqedOF2ulHpMG72XQol4ILEJ8k2wwRl/Km8oE= github.com/klauspost/pgzip v1.2.5/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go index e874eb74e..482e03663 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go @@ -5,9 +5,7 @@ import ( "fmt" "io" "io/ioutil" - "net" "os" - "os/user" "path/filepath" "sync" @@ -17,13 +15,6 @@ import ( "github.com/pkg/errors" ) -func init() { - // initialize nss libraries in Glibc so that the dynamic libraries are loaded in the host - // environment not in the chroot from untrusted files. - _, _ = user.Lookup("storage") - _, _ = net.LookupHost("localhost") -} - // NewArchiver returns a new Archiver which uses chrootarchive.Untar func NewArchiver(idMappings *idtools.IDMappings) *archive.Archiver { archiver := archive.NewArchiver(idMappings) diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_linux.go b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_linux.go index 76c94c6c1..58729ec8c 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_linux.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/chroot_linux.go @@ -3,7 +3,9 @@ package chrootarchive import ( "fmt" "io/ioutil" + "net" "os" + "os/user" "path/filepath" "github.com/containers/storage/pkg/mount" @@ -23,6 +25,11 @@ func chroot(path string) (err error) { return err } + // initialize nss libraries in Glibc so that the dynamic libraries are loaded in the host + // environment not in the chroot from untrusted files. + _, _ = user.Lookup("storage") + _, _ = net.LookupHost("localhost") + // if the process doesn't have CAP_SYS_ADMIN, but does have CAP_SYS_CHROOT, we need to use the actual chroot if !caps.Get(capability.EFFECTIVE, capability.CAP_SYS_ADMIN) && caps.Get(capability.EFFECTIVE, capability.CAP_SYS_CHROOT) { return realChroot(path) diff --git a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go index 7de20feaa..9434499d2 100644 --- a/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go +++ b/vendor/github.com/containers/storage/pkg/chunked/storage_linux.go @@ -918,6 +918,9 @@ func (c *chunkedDiffer) storeMissingFiles(streams chan io.ReadCloser, errs chan case p := <-streams: part = p case err := <-errs: + if err == nil { + return errors.New("not enough data returned from the server") + } return err } if part == nil { @@ -1081,12 +1084,18 @@ func mergeMissingChunks(missingParts []missingPart, target int) []missingPart { func (c *chunkedDiffer) retrieveMissingFiles(dest string, dirfd int, missingParts []missingPart, options *archive.TarOptions) error { var chunksToRequest []ImageSourceChunk - for _, c := range missingParts { - if c.OriginFile == nil && !c.Hole { - chunksToRequest = append(chunksToRequest, *c.SourceChunk) + + calculateChunksToRequest := func() { + chunksToRequest = []ImageSourceChunk{} + for _, c := range missingParts { + if c.OriginFile == nil && !c.Hole { + chunksToRequest = append(chunksToRequest, *c.SourceChunk) + } } } + calculateChunksToRequest() + // There are some missing files. Prepare a multirange request for the missing chunks. var streams chan io.ReadCloser var err error @@ -1106,6 +1115,7 @@ func (c *chunkedDiffer) retrieveMissingFiles(dest string, dirfd int, missingPart // Merge more chunks to request missingParts = mergeMissingChunks(missingParts, requested/2) + calculateChunksToRequest() continue } return err @@ -1575,6 +1585,8 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions) (gra wg.Wait() for _, res := range copyResults[:filesToWaitFor] { + r := &mergedEntries[res.index] + if res.err != nil { return output, res.err } @@ -1584,8 +1596,6 @@ func (c *chunkedDiffer) ApplyDiff(dest string, options *archive.TarOptions) (gra continue } - r := &mergedEntries[res.index] - missingPartsSize += r.Size remainingSize := r.Size diff --git a/vendor/github.com/containers/storage/pkg/mount/mounter_freebsd.go b/vendor/github.com/containers/storage/pkg/mount/mounter_freebsd.go index 72ceec3dd..2404e331d 100644 --- a/vendor/github.com/containers/storage/pkg/mount/mounter_freebsd.go +++ b/vendor/github.com/containers/storage/pkg/mount/mounter_freebsd.go @@ -1,3 +1,6 @@ +//go:build freebsd && cgo +// +build freebsd,cgo + package mount /* diff --git a/vendor/github.com/containers/storage/pkg/mount/mounter_unsupported.go b/vendor/github.com/containers/storage/pkg/mount/mounter_unsupported.go index 9d20cfbf8..74fe66609 100644 --- a/vendor/github.com/containers/storage/pkg/mount/mounter_unsupported.go +++ b/vendor/github.com/containers/storage/pkg/mount/mounter_unsupported.go @@ -1,4 +1,6 @@ -// +build !linux,!freebsd +//go:build !linux && !(freebsd && cgo) +// +build !linux +// +build !freebsd !cgo package mount diff --git a/vendor/github.com/containers/storage/storage.conf-freebsd b/vendor/github.com/containers/storage/storage.conf-freebsd index cc655c62e..34d80152c 100644 --- a/vendor/github.com/containers/storage/storage.conf-freebsd +++ b/vendor/github.com/containers/storage/storage.conf-freebsd @@ -5,8 +5,8 @@ # files. # # Note: The storage.conf file overrides other storage.conf files based on this precedence: -# /usr/containers/storage.conf -# /etc/containers/storage.conf +# /usr/local/share/containers/storage.conf +# /usr/local/etc/containers/storage.conf # $HOME/.config/containers/storage.conf # $XDG_CONFIG_HOME/containers/storage.conf (If XDG_CONFIG_HOME is set) # See man 5 containers-storage.conf for more information diff --git a/vendor/github.com/containers/storage/store.go b/vendor/github.com/containers/storage/store.go index 30d3e8715..45912d0ca 100644 --- a/vendor/github.com/containers/storage/store.go +++ b/vendor/github.com/containers/storage/store.go @@ -1195,6 +1195,11 @@ func (s *store) imageTopLayerForMapping(image *Image, ristore ROImageStore, crea if layer == nil { layer = cLayer parentLayer = cParentLayer + if store != rlstore { + // The layer is in another store, so we cannot + // create a mapped version of it to the image. + createMappedLayer = false + } } } } diff --git a/vendor/github.com/containers/storage/types/options.go b/vendor/github.com/containers/storage/types/options.go index a71c6d2ef..d318421a4 100644 --- a/vendor/github.com/containers/storage/types/options.go +++ b/vendor/github.com/containers/storage/types/options.go @@ -26,22 +26,6 @@ type TomlConfig struct { } const ( - // these are default path for run and graph root for rootful users - // for rootless path is constructed via getRootlessStorageOpts - defaultRunRoot string = "/run/containers/storage" - defaultGraphRoot string = "/var/lib/containers/storage" -) - -// defaultConfigFile path to the system wide storage.conf file -var ( - defaultConfigFile = "/usr/share/containers/storage.conf" - defaultOverrideConfigFile = "/etc/containers/storage.conf" - defaultConfigFileSet = false - // DefaultStoreOptions is a reasonable default set of options. - defaultStoreOptions StoreOptions -) - -const ( overlayDriver = "overlay" overlay2 = "overlay2" ) diff --git a/vendor/github.com/containers/storage/types/options_darwin.go b/vendor/github.com/containers/storage/types/options_darwin.go new file mode 100644 index 000000000..d5ad50bc0 --- /dev/null +++ b/vendor/github.com/containers/storage/types/options_darwin.go @@ -0,0 +1,17 @@ +package types + +const ( + // these are default path for run and graph root for rootful users + // for rootless path is constructed via getRootlessStorageOpts + defaultRunRoot string = "/run/containers/storage" + defaultGraphRoot string = "/var/lib/containers/storage" +) + +// defaultConfigFile path to the system wide storage.conf file +var ( + defaultConfigFile = "/usr/share/containers/storage.conf" + defaultOverrideConfigFile = "/etc/containers/storage.conf" + defaultConfigFileSet = false + // DefaultStoreOptions is a reasonable default set of options. + defaultStoreOptions StoreOptions +) diff --git a/vendor/github.com/containers/storage/types/options_freebsd.go b/vendor/github.com/containers/storage/types/options_freebsd.go new file mode 100644 index 000000000..d5976b6d5 --- /dev/null +++ b/vendor/github.com/containers/storage/types/options_freebsd.go @@ -0,0 +1,17 @@ +package types + +const ( + // these are default path for run and graph root for rootful users + // for rootless path is constructed via getRootlessStorageOpts + defaultRunRoot string = "/var/run/containers/storage" + defaultGraphRoot string = "/var/db/containers/storage" +) + +// defaultConfigFile path to the system wide storage.conf file +var ( + defaultConfigFile = "/usr/local/share/containers/storage.conf" + defaultOverrideConfigFile = "/usr/local/etc/containers/storage.conf" + defaultConfigFileSet = false + // DefaultStoreOptions is a reasonable default set of options. + defaultStoreOptions StoreOptions +) diff --git a/vendor/github.com/containers/storage/types/options_linux.go b/vendor/github.com/containers/storage/types/options_linux.go new file mode 100644 index 000000000..d5ad50bc0 --- /dev/null +++ b/vendor/github.com/containers/storage/types/options_linux.go @@ -0,0 +1,17 @@ +package types + +const ( + // these are default path for run and graph root for rootful users + // for rootless path is constructed via getRootlessStorageOpts + defaultRunRoot string = "/run/containers/storage" + defaultGraphRoot string = "/var/lib/containers/storage" +) + +// defaultConfigFile path to the system wide storage.conf file +var ( + defaultConfigFile = "/usr/share/containers/storage.conf" + defaultOverrideConfigFile = "/etc/containers/storage.conf" + defaultConfigFileSet = false + // DefaultStoreOptions is a reasonable default set of options. + defaultStoreOptions StoreOptions +) diff --git a/vendor/github.com/containers/storage/types/options_windows.go b/vendor/github.com/containers/storage/types/options_windows.go new file mode 100644 index 000000000..d5ad50bc0 --- /dev/null +++ b/vendor/github.com/containers/storage/types/options_windows.go @@ -0,0 +1,17 @@ +package types + +const ( + // these are default path for run and graph root for rootful users + // for rootless path is constructed via getRootlessStorageOpts + defaultRunRoot string = "/run/containers/storage" + defaultGraphRoot string = "/var/lib/containers/storage" +) + +// defaultConfigFile path to the system wide storage.conf file +var ( + defaultConfigFile = "/usr/share/containers/storage.conf" + defaultOverrideConfigFile = "/etc/containers/storage.conf" + defaultConfigFileSet = false + // DefaultStoreOptions is a reasonable default set of options. + defaultStoreOptions StoreOptions +) diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index 5b7cf781a..c3ec9d8a7 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -17,6 +17,16 @@ This package provides various compression algorithms. # changelog
+* May 5, 2022 (v1.15.3)
+ * zstd: Allow to ignore checksum checking by @WojciechMula [#572](https://github.com/klauspost/compress/pull/572)
+ * s2: Fix incorrect seek for io.SeekEnd in [#575](https://github.com/klauspost/compress/pull/575)
+
+* Apr 26, 2022 (v1.15.2)
+ * zstd: Add x86-64 assembly for decompression on streams and blocks. Contributed by [@WojciechMula](https://github.com/WojciechMula). Typically 2x faster. [#528](https://github.com/klauspost/compress/pull/528) [#531](https://github.com/klauspost/compress/pull/531) [#545](https://github.com/klauspost/compress/pull/545) [#537](https://github.com/klauspost/compress/pull/537)
+ * zstd: Add options to ZipDecompressor and fixes [#539](https://github.com/klauspost/compress/pull/539)
+ * s2: Use sorted search for index [#555](https://github.com/klauspost/compress/pull/555)
+ * Minimum version is Go 1.16, added CI test on 1.18.
+
* Mar 11, 2022 (v1.15.1)
* huff0: Add x86 assembly of Decode4X by @WojciechMula in [#512](https://github.com/klauspost/compress/pull/512)
* zstd: Reuse zip decoders in [#514](https://github.com/klauspost/compress/pull/514)
diff --git a/vendor/github.com/klauspost/compress/flate/inflate_gen.go b/vendor/github.com/klauspost/compress/flate/inflate_gen.go index 8d632cea0..61342b6b8 100644 --- a/vendor/github.com/klauspost/compress/flate/inflate_gen.go +++ b/vendor/github.com/klauspost/compress/flate/inflate_gen.go @@ -24,7 +24,7 @@ func (f *decompressor) huffmanBytesBuffer() { // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, // but is smart enough to keep local variables in registers, so use nb and b, // inline call to moreBits and reassign b,nb back to f on return. - fnb, fb := f.nb, f.b + fnb, fb, dict := f.nb, f.b, &f.dict switch f.stepState { case stateInit: @@ -82,9 +82,9 @@ readLiteral: var length int switch { case v < 256: - f.dict.writeByte(byte(v)) - if f.dict.availWrite() == 0 { - f.toRead = f.dict.readFlush() + dict.writeByte(byte(v)) + if dict.availWrite() == 0 { + f.toRead = dict.readFlush() f.step = (*decompressor).huffmanBytesBuffer f.stepState = stateInit f.b, f.nb = fb, fnb @@ -227,10 +227,10 @@ readLiteral: } // No check on length; encoding can be prescient. - if dist > uint32(f.dict.histSize()) { + if dist > uint32(dict.histSize()) { f.b, f.nb = fb, fnb if debugDecode { - fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize()) + fmt.Println("dist > dict.histSize():", dist, dict.histSize()) } f.err = CorruptInputError(f.roffset) return @@ -243,14 +243,14 @@ readLiteral: copyHistory: // Perform a backwards copy according to RFC section 3.2.3. { - cnt := f.dict.tryWriteCopy(f.copyDist, f.copyLen) + cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) if cnt == 0 { - cnt = f.dict.writeCopy(f.copyDist, f.copyLen) + cnt = dict.writeCopy(f.copyDist, f.copyLen) } f.copyLen -= cnt - if f.dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = f.dict.readFlush() + if dict.availWrite() == 0 || f.copyLen > 0 { + f.toRead = dict.readFlush() f.step = (*decompressor).huffmanBytesBuffer // We need to continue this work f.stepState = stateDict f.b, f.nb = fb, fnb @@ -275,7 +275,7 @@ func (f *decompressor) huffmanBytesReader() { // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, // but is smart enough to keep local variables in registers, so use nb and b, // inline call to moreBits and reassign b,nb back to f on return. - fnb, fb := f.nb, f.b + fnb, fb, dict := f.nb, f.b, &f.dict switch f.stepState { case stateInit: @@ -333,9 +333,9 @@ readLiteral: var length int switch { case v < 256: - f.dict.writeByte(byte(v)) - if f.dict.availWrite() == 0 { - f.toRead = f.dict.readFlush() + dict.writeByte(byte(v)) + if dict.availWrite() == 0 { + f.toRead = dict.readFlush() f.step = (*decompressor).huffmanBytesReader f.stepState = stateInit f.b, f.nb = fb, fnb @@ -478,10 +478,10 @@ readLiteral: } // No check on length; encoding can be prescient. - if dist > uint32(f.dict.histSize()) { + if dist > uint32(dict.histSize()) { f.b, f.nb = fb, fnb if debugDecode { - fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize()) + fmt.Println("dist > dict.histSize():", dist, dict.histSize()) } f.err = CorruptInputError(f.roffset) return @@ -494,14 +494,14 @@ readLiteral: copyHistory: // Perform a backwards copy according to RFC section 3.2.3. { - cnt := f.dict.tryWriteCopy(f.copyDist, f.copyLen) + cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) if cnt == 0 { - cnt = f.dict.writeCopy(f.copyDist, f.copyLen) + cnt = dict.writeCopy(f.copyDist, f.copyLen) } f.copyLen -= cnt - if f.dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = f.dict.readFlush() + if dict.availWrite() == 0 || f.copyLen > 0 { + f.toRead = dict.readFlush() f.step = (*decompressor).huffmanBytesReader // We need to continue this work f.stepState = stateDict f.b, f.nb = fb, fnb @@ -526,7 +526,7 @@ func (f *decompressor) huffmanBufioReader() { // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, // but is smart enough to keep local variables in registers, so use nb and b, // inline call to moreBits and reassign b,nb back to f on return. - fnb, fb := f.nb, f.b + fnb, fb, dict := f.nb, f.b, &f.dict switch f.stepState { case stateInit: @@ -584,9 +584,9 @@ readLiteral: var length int switch { case v < 256: - f.dict.writeByte(byte(v)) - if f.dict.availWrite() == 0 { - f.toRead = f.dict.readFlush() + dict.writeByte(byte(v)) + if dict.availWrite() == 0 { + f.toRead = dict.readFlush() f.step = (*decompressor).huffmanBufioReader f.stepState = stateInit f.b, f.nb = fb, fnb @@ -729,10 +729,10 @@ readLiteral: } // No check on length; encoding can be prescient. - if dist > uint32(f.dict.histSize()) { + if dist > uint32(dict.histSize()) { f.b, f.nb = fb, fnb if debugDecode { - fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize()) + fmt.Println("dist > dict.histSize():", dist, dict.histSize()) } f.err = CorruptInputError(f.roffset) return @@ -745,14 +745,14 @@ readLiteral: copyHistory: // Perform a backwards copy according to RFC section 3.2.3. { - cnt := f.dict.tryWriteCopy(f.copyDist, f.copyLen) + cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) if cnt == 0 { - cnt = f.dict.writeCopy(f.copyDist, f.copyLen) + cnt = dict.writeCopy(f.copyDist, f.copyLen) } f.copyLen -= cnt - if f.dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = f.dict.readFlush() + if dict.availWrite() == 0 || f.copyLen > 0 { + f.toRead = dict.readFlush() f.step = (*decompressor).huffmanBufioReader // We need to continue this work f.stepState = stateDict f.b, f.nb = fb, fnb @@ -777,7 +777,7 @@ func (f *decompressor) huffmanStringsReader() { // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, // but is smart enough to keep local variables in registers, so use nb and b, // inline call to moreBits and reassign b,nb back to f on return. - fnb, fb := f.nb, f.b + fnb, fb, dict := f.nb, f.b, &f.dict switch f.stepState { case stateInit: @@ -835,9 +835,9 @@ readLiteral: var length int switch { case v < 256: - f.dict.writeByte(byte(v)) - if f.dict.availWrite() == 0 { - f.toRead = f.dict.readFlush() + dict.writeByte(byte(v)) + if dict.availWrite() == 0 { + f.toRead = dict.readFlush() f.step = (*decompressor).huffmanStringsReader f.stepState = stateInit f.b, f.nb = fb, fnb @@ -980,10 +980,10 @@ readLiteral: } // No check on length; encoding can be prescient. - if dist > uint32(f.dict.histSize()) { + if dist > uint32(dict.histSize()) { f.b, f.nb = fb, fnb if debugDecode { - fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize()) + fmt.Println("dist > dict.histSize():", dist, dict.histSize()) } f.err = CorruptInputError(f.roffset) return @@ -996,14 +996,14 @@ readLiteral: copyHistory: // Perform a backwards copy according to RFC section 3.2.3. { - cnt := f.dict.tryWriteCopy(f.copyDist, f.copyLen) + cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) if cnt == 0 { - cnt = f.dict.writeCopy(f.copyDist, f.copyLen) + cnt = dict.writeCopy(f.copyDist, f.copyLen) } f.copyLen -= cnt - if f.dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = f.dict.readFlush() + if dict.availWrite() == 0 || f.copyLen > 0 { + f.toRead = dict.readFlush() f.step = (*decompressor).huffmanStringsReader // We need to continue this work f.stepState = stateDict f.b, f.nb = fb, fnb @@ -1028,7 +1028,7 @@ func (f *decompressor) huffmanGenericReader() { // Optimization. Compiler isn't smart enough to keep f.b,f.nb in registers, // but is smart enough to keep local variables in registers, so use nb and b, // inline call to moreBits and reassign b,nb back to f on return. - fnb, fb := f.nb, f.b + fnb, fb, dict := f.nb, f.b, &f.dict switch f.stepState { case stateInit: @@ -1086,9 +1086,9 @@ readLiteral: var length int switch { case v < 256: - f.dict.writeByte(byte(v)) - if f.dict.availWrite() == 0 { - f.toRead = f.dict.readFlush() + dict.writeByte(byte(v)) + if dict.availWrite() == 0 { + f.toRead = dict.readFlush() f.step = (*decompressor).huffmanGenericReader f.stepState = stateInit f.b, f.nb = fb, fnb @@ -1231,10 +1231,10 @@ readLiteral: } // No check on length; encoding can be prescient. - if dist > uint32(f.dict.histSize()) { + if dist > uint32(dict.histSize()) { f.b, f.nb = fb, fnb if debugDecode { - fmt.Println("dist > f.dict.histSize():", dist, f.dict.histSize()) + fmt.Println("dist > dict.histSize():", dist, dict.histSize()) } f.err = CorruptInputError(f.roffset) return @@ -1247,14 +1247,14 @@ readLiteral: copyHistory: // Perform a backwards copy according to RFC section 3.2.3. { - cnt := f.dict.tryWriteCopy(f.copyDist, f.copyLen) + cnt := dict.tryWriteCopy(f.copyDist, f.copyLen) if cnt == 0 { - cnt = f.dict.writeCopy(f.copyDist, f.copyLen) + cnt = dict.writeCopy(f.copyDist, f.copyLen) } f.copyLen -= cnt - if f.dict.availWrite() == 0 || f.copyLen > 0 { - f.toRead = f.dict.readFlush() + if dict.availWrite() == 0 || f.copyLen > 0 { + f.toRead = dict.readFlush() f.step = (*decompressor).huffmanGenericReader // We need to continue this work f.stepState = stateDict f.b, f.nb = fb, fnb diff --git a/vendor/github.com/klauspost/compress/huff0/autogen.go b/vendor/github.com/klauspost/compress/huff0/autogen.go deleted file mode 100644 index ff2c69d60..000000000 --- a/vendor/github.com/klauspost/compress/huff0/autogen.go +++ /dev/null @@ -1,5 +0,0 @@ -package huff0 - -//go:generate go run generate.go -//go:generate asmfmt -w decompress_amd64.s -//go:generate asmfmt -w decompress_8b_amd64.s diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_8b_amd64.s b/vendor/github.com/klauspost/compress/huff0/decompress_8b_amd64.s deleted file mode 100644 index 0d6cb1a96..000000000 --- a/vendor/github.com/klauspost/compress/huff0/decompress_8b_amd64.s +++ /dev/null @@ -1,488 +0,0 @@ -// +build !appengine -// +build gc -// +build !noasm - -#include "textflag.h" -#include "funcdata.h" -#include "go_asm.h" - -#define bufoff 256 // see decompress.go, we're using [4][256]byte table - -// func decompress4x_main_loop_x86(pbr0, pbr1, pbr2, pbr3 *bitReaderShifted, -// peekBits uint8, buf *byte, tbl *dEntrySingle) (int, bool) -TEXT ·decompress4x_8b_loop_x86(SB), NOSPLIT, $8 -#define off R8 -#define buffer DI -#define table SI - -#define br_bits_read R9 -#define br_value R10 -#define br_offset R11 -#define peek_bits R12 -#define exhausted DX - -#define br0 R13 -#define br1 R14 -#define br2 R15 -#define br3 BP - - MOVQ BP, 0(SP) - - XORQ exhausted, exhausted // exhausted = false - XORQ off, off // off = 0 - - MOVBQZX peekBits+32(FP), peek_bits - MOVQ buf+40(FP), buffer - MOVQ tbl+48(FP), table - - MOVQ pbr0+0(FP), br0 - MOVQ pbr1+8(FP), br1 - MOVQ pbr2+16(FP), br2 - MOVQ pbr3+24(FP), br3 - -main_loop: - - // const stream = 0 - // br0.fillFast() - MOVBQZX bitReaderShifted_bitsRead(br0), br_bits_read - MOVQ bitReaderShifted_value(br0), br_value - MOVQ bitReaderShifted_off(br0), br_offset - - // if b.bitsRead >= 32 { - CMPQ br_bits_read, $32 - JB skip_fill0 - - SUBQ $32, br_bits_read // b.bitsRead -= 32 - SUBQ $4, br_offset // b.off -= 4 - - // v := b.in[b.off-4 : b.off] - // v = v[:4] - // low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - MOVQ bitReaderShifted_in(br0), AX - MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVQ br_bits_read, CX - SHLQ CL, AX - ORQ AX, br_value - - // exhausted = exhausted || (br0.off < 4) - CMPQ br_offset, $4 - SETLT DL - ORB DL, DH - - // } -skip_fill0: - - // val0 := br0.peekTopBits(peekBits) - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v0 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 - - // br0.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // val1 := br0.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v1 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 - - // br0.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - MOVBQZX AL, CX - SHLQ CX, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // these two writes get coalesced - // buf[stream][off] = uint8(v0.entry >> 8) - // buf[stream][off+1] = uint8(v1.entry >> 8) - MOVW BX, 0(buffer)(off*1) - - // SECOND PART: - // val2 := br0.peekTopBits(peekBits) - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v2 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 - - // br0.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // val3 := br0.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v3 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 - - // br0.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - MOVBQZX AL, CX - SHLQ CX, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // these two writes get coalesced - // buf[stream][off+2] = uint8(v2.entry >> 8) - // buf[stream][off+3] = uint8(v3.entry >> 8) - MOVW BX, 0+2(buffer)(off*1) - - // update the bitrader reader structure - MOVB br_bits_read, bitReaderShifted_bitsRead(br0) - MOVQ br_value, bitReaderShifted_value(br0) - MOVQ br_offset, bitReaderShifted_off(br0) - - // const stream = 1 - // br1.fillFast() - MOVBQZX bitReaderShifted_bitsRead(br1), br_bits_read - MOVQ bitReaderShifted_value(br1), br_value - MOVQ bitReaderShifted_off(br1), br_offset - - // if b.bitsRead >= 32 { - CMPQ br_bits_read, $32 - JB skip_fill1 - - SUBQ $32, br_bits_read // b.bitsRead -= 32 - SUBQ $4, br_offset // b.off -= 4 - - // v := b.in[b.off-4 : b.off] - // v = v[:4] - // low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - MOVQ bitReaderShifted_in(br1), AX - MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVQ br_bits_read, CX - SHLQ CL, AX - ORQ AX, br_value - - // exhausted = exhausted || (br1.off < 4) - CMPQ br_offset, $4 - SETLT DL - ORB DL, DH - - // } -skip_fill1: - - // val0 := br1.peekTopBits(peekBits) - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v0 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 - - // br1.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // val1 := br1.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v1 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 - - // br1.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - MOVBQZX AL, CX - SHLQ CX, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // these two writes get coalesced - // buf[stream][off] = uint8(v0.entry >> 8) - // buf[stream][off+1] = uint8(v1.entry >> 8) - MOVW BX, 256(buffer)(off*1) - - // SECOND PART: - // val2 := br1.peekTopBits(peekBits) - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v2 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 - - // br1.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // val3 := br1.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v3 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 - - // br1.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - MOVBQZX AL, CX - SHLQ CX, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // these two writes get coalesced - // buf[stream][off+2] = uint8(v2.entry >> 8) - // buf[stream][off+3] = uint8(v3.entry >> 8) - MOVW BX, 256+2(buffer)(off*1) - - // update the bitrader reader structure - MOVB br_bits_read, bitReaderShifted_bitsRead(br1) - MOVQ br_value, bitReaderShifted_value(br1) - MOVQ br_offset, bitReaderShifted_off(br1) - - // const stream = 2 - // br2.fillFast() - MOVBQZX bitReaderShifted_bitsRead(br2), br_bits_read - MOVQ bitReaderShifted_value(br2), br_value - MOVQ bitReaderShifted_off(br2), br_offset - - // if b.bitsRead >= 32 { - CMPQ br_bits_read, $32 - JB skip_fill2 - - SUBQ $32, br_bits_read // b.bitsRead -= 32 - SUBQ $4, br_offset // b.off -= 4 - - // v := b.in[b.off-4 : b.off] - // v = v[:4] - // low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - MOVQ bitReaderShifted_in(br2), AX - MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVQ br_bits_read, CX - SHLQ CL, AX - ORQ AX, br_value - - // exhausted = exhausted || (br2.off < 4) - CMPQ br_offset, $4 - SETLT DL - ORB DL, DH - - // } -skip_fill2: - - // val0 := br2.peekTopBits(peekBits) - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v0 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 - - // br2.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // val1 := br2.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v1 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 - - // br2.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - MOVBQZX AL, CX - SHLQ CX, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // these two writes get coalesced - // buf[stream][off] = uint8(v0.entry >> 8) - // buf[stream][off+1] = uint8(v1.entry >> 8) - MOVW BX, 512(buffer)(off*1) - - // SECOND PART: - // val2 := br2.peekTopBits(peekBits) - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v2 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 - - // br2.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // val3 := br2.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v3 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 - - // br2.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - MOVBQZX AL, CX - SHLQ CX, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // these two writes get coalesced - // buf[stream][off+2] = uint8(v2.entry >> 8) - // buf[stream][off+3] = uint8(v3.entry >> 8) - MOVW BX, 512+2(buffer)(off*1) - - // update the bitrader reader structure - MOVB br_bits_read, bitReaderShifted_bitsRead(br2) - MOVQ br_value, bitReaderShifted_value(br2) - MOVQ br_offset, bitReaderShifted_off(br2) - - // const stream = 3 - // br3.fillFast() - MOVBQZX bitReaderShifted_bitsRead(br3), br_bits_read - MOVQ bitReaderShifted_value(br3), br_value - MOVQ bitReaderShifted_off(br3), br_offset - - // if b.bitsRead >= 32 { - CMPQ br_bits_read, $32 - JB skip_fill3 - - SUBQ $32, br_bits_read // b.bitsRead -= 32 - SUBQ $4, br_offset // b.off -= 4 - - // v := b.in[b.off-4 : b.off] - // v = v[:4] - // low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - MOVQ bitReaderShifted_in(br3), AX - MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVQ br_bits_read, CX - SHLQ CL, AX - ORQ AX, br_value - - // exhausted = exhausted || (br3.off < 4) - CMPQ br_offset, $4 - SETLT DL - ORB DL, DH - - // } -skip_fill3: - - // val0 := br3.peekTopBits(peekBits) - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v0 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 - - // br3.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // val1 := br3.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v1 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 - - // br3.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - MOVBQZX AL, CX - SHLQ CX, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // these two writes get coalesced - // buf[stream][off] = uint8(v0.entry >> 8) - // buf[stream][off+1] = uint8(v1.entry >> 8) - MOVW BX, 768(buffer)(off*1) - - // SECOND PART: - // val2 := br3.peekTopBits(peekBits) - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v2 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 - - // br3.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // val3 := br3.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v3 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 - - // br3.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - MOVBQZX AL, CX - SHLQ CX, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // these two writes get coalesced - // buf[stream][off+2] = uint8(v2.entry >> 8) - // buf[stream][off+3] = uint8(v3.entry >> 8) - MOVW BX, 768+2(buffer)(off*1) - - // update the bitrader reader structure - MOVB br_bits_read, bitReaderShifted_bitsRead(br3) - MOVQ br_value, bitReaderShifted_value(br3) - MOVQ br_offset, bitReaderShifted_off(br3) - - ADDQ $4, off // off += 2 - - TESTB DH, DH // any br[i].ofs < 4? - JNZ end - - CMPQ off, $bufoff - JL main_loop - -end: - MOVQ 0(SP), BP - - MOVB off, ret+56(FP) - RET - -#undef off -#undef buffer -#undef table - -#undef br_bits_read -#undef br_value -#undef br_offset -#undef peek_bits -#undef exhausted - -#undef br0 -#undef br1 -#undef br2 -#undef br3 diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_8b_amd64.s.in b/vendor/github.com/klauspost/compress/huff0/decompress_8b_amd64.s.in deleted file mode 100644 index 6d477a2c1..000000000 --- a/vendor/github.com/klauspost/compress/huff0/decompress_8b_amd64.s.in +++ /dev/null @@ -1,197 +0,0 @@ -// +build !appengine -// +build gc -// +build !noasm - -#include "textflag.h" -#include "funcdata.h" -#include "go_asm.h" - - -#define bufoff 256 // see decompress.go, we're using [4][256]byte table - -//func decompress4x_main_loop_x86(pbr0, pbr1, pbr2, pbr3 *bitReaderShifted, -// peekBits uint8, buf *byte, tbl *dEntrySingle) (int, bool) -TEXT ·decompress4x_8b_loop_x86(SB), NOSPLIT, $8 -#define off R8 -#define buffer DI -#define table SI - -#define br_bits_read R9 -#define br_value R10 -#define br_offset R11 -#define peek_bits R12 -#define exhausted DX - -#define br0 R13 -#define br1 R14 -#define br2 R15 -#define br3 BP - - MOVQ BP, 0(SP) - - XORQ exhausted, exhausted // exhausted = false - XORQ off, off // off = 0 - - MOVBQZX peekBits+32(FP), peek_bits - MOVQ buf+40(FP), buffer - MOVQ tbl+48(FP), table - - MOVQ pbr0+0(FP), br0 - MOVQ pbr1+8(FP), br1 - MOVQ pbr2+16(FP), br2 - MOVQ pbr3+24(FP), br3 - -main_loop: -{{ define "decode_2_values_x86" }} - // const stream = {{ var "id" }} - // br{{ var "id"}}.fillFast() - MOVBQZX bitReaderShifted_bitsRead(br{{ var "id" }}), br_bits_read - MOVQ bitReaderShifted_value(br{{ var "id" }}), br_value - MOVQ bitReaderShifted_off(br{{ var "id" }}), br_offset - - // if b.bitsRead >= 32 { - CMPQ br_bits_read, $32 - JB skip_fill{{ var "id" }} - - SUBQ $32, br_bits_read // b.bitsRead -= 32 - SUBQ $4, br_offset // b.off -= 4 - - // v := b.in[b.off-4 : b.off] - // v = v[:4] - // low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - MOVQ bitReaderShifted_in(br{{ var "id" }}), AX - MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) - - // b.value |= uint64(low) << (b.bitsRead & 63) - MOVQ br_bits_read, CX - SHLQ CL, AX - ORQ AX, br_value - - // exhausted = exhausted || (br{{ var "id"}}.off < 4) - CMPQ br_offset, $4 - SETLT DL - ORB DL, DH - // } -skip_fill{{ var "id" }}: - - // val0 := br{{ var "id"}}.peekTopBits(peekBits) - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v0 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 - - // br{{ var "id"}}.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // val1 := br{{ var "id"}}.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v1 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 - - // br{{ var "id"}}.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - MOVBQZX AL, CX - SHLQ CX, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - - // these two writes get coalesced - // buf[stream][off] = uint8(v0.entry >> 8) - // buf[stream][off+1] = uint8(v1.entry >> 8) - MOVW BX, {{ var "bufofs" }}(buffer)(off*1) - - // SECOND PART: - // val2 := br{{ var "id"}}.peekTopBits(peekBits) - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v2 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 - - // br{{ var "id"}}.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - // val3 := br{{ var "id"}}.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - - // v3 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 - - // br{{ var "id"}}.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - MOVBQZX AL, CX - SHLQ CX, br_value // value <<= n - ADDQ CX, br_bits_read // bits_read += n - - - // these two writes get coalesced - // buf[stream][off+2] = uint8(v2.entry >> 8) - // buf[stream][off+3] = uint8(v3.entry >> 8) - MOVW BX, {{ var "bufofs" }}+2(buffer)(off*1) - - // update the bitrader reader structure - MOVB br_bits_read, bitReaderShifted_bitsRead(br{{ var "id" }}) - MOVQ br_value, bitReaderShifted_value(br{{ var "id" }}) - MOVQ br_offset, bitReaderShifted_off(br{{ var "id" }}) -{{ end }} - - {{ set "id" "0" }} - {{ set "ofs" "0" }} - {{ set "bufofs" "0" }} {{/* id * bufoff */}} - {{ template "decode_2_values_x86" . }} - - {{ set "id" "1" }} - {{ set "ofs" "8" }} - {{ set "bufofs" "256" }} - {{ template "decode_2_values_x86" . }} - - {{ set "id" "2" }} - {{ set "ofs" "16" }} - {{ set "bufofs" "512" }} - {{ template "decode_2_values_x86" . }} - - {{ set "id" "3" }} - {{ set "ofs" "24" }} - {{ set "bufofs" "768" }} - {{ template "decode_2_values_x86" . }} - - ADDQ $4, off // off += 2 - - TESTB DH, DH // any br[i].ofs < 4? - JNZ end - - CMPQ off, $bufoff - JL main_loop -end: - MOVQ 0(SP), BP - - MOVB off, ret+56(FP) - RET -#undef off -#undef buffer -#undef table - -#undef br_bits_read -#undef br_value -#undef br_offset -#undef peek_bits -#undef exhausted - -#undef br0 -#undef br1 -#undef br2 -#undef br3 diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go index ce8e93bcd..3415e5da2 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.go @@ -13,19 +13,30 @@ import ( // decompress4x_main_loop_x86 is an x86 assembler implementation // of Decompress4X when tablelog > 8. //go:noescape -func decompress4x_main_loop_x86(pbr0, pbr1, pbr2, pbr3 *bitReaderShifted, - peekBits uint8, buf *byte, tbl *dEntrySingle) uint8 +func decompress4x_main_loop_amd64(ctx *decompress4xContext) // decompress4x_8b_loop_x86 is an x86 assembler implementation // of Decompress4X when tablelog <= 8 which decodes 4 entries // per loop. //go:noescape -func decompress4x_8b_loop_x86(pbr0, pbr1, pbr2, pbr3 *bitReaderShifted, - peekBits uint8, buf *byte, tbl *dEntrySingle) uint8 +func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) // fallback8BitSize is the size where using Go version is faster. const fallback8BitSize = 800 +type decompress4xContext struct { + pbr0 *bitReaderShifted + pbr1 *bitReaderShifted + pbr2 *bitReaderShifted + pbr3 *bitReaderShifted + peekBits uint8 + out *byte + dstEvery int + tbl *dEntrySingle + decoded int + limit *byte +} + // Decompress4X will decompress a 4X encoded stream. // The length of the supplied input must match the end of a block exactly. // The *capacity* of the dst slice must match the destination size of @@ -42,6 +53,7 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { if cap(dst) < fallback8BitSize && use8BitTables { return d.decompress4X8bit(dst, src) } + var br [4]bitReaderShifted // Decode "jump table" start := 6 @@ -71,70 +83,28 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { const tlMask = tlSize - 1 single := d.dt.single[:tlSize] - // Use temp table to avoid bound checks/append penalty. - buf := d.buffer() - var off uint8 var decoded int - const debug = false - - // see: bitReaderShifted.peekBitsFast() - peekBits := uint8((64 - d.actualTableLog) & 63) - - // Decode 2 values from each decoder/loop. - const bufoff = 256 - for { - if br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4 { - break + if len(out) > 4*4 && !(br[0].off < 4 || br[1].off < 4 || br[2].off < 4 || br[3].off < 4) { + ctx := decompress4xContext{ + pbr0: &br[0], + pbr1: &br[1], + pbr2: &br[2], + pbr3: &br[3], + peekBits: uint8((64 - d.actualTableLog) & 63), // see: bitReaderShifted.peekBitsFast() + out: &out[0], + dstEvery: dstEvery, + tbl: &single[0], + limit: &out[dstEvery-4], // Always stop decoding when first buffer gets here to avoid writing OOB on last. } - if use8BitTables { - off = decompress4x_8b_loop_x86(&br[0], &br[1], &br[2], &br[3], peekBits, &buf[0][0], &single[0]) + decompress4x_8b_main_loop_amd64(&ctx) } else { - off = decompress4x_main_loop_x86(&br[0], &br[1], &br[2], &br[3], peekBits, &buf[0][0], &single[0]) - } - if debug { - fmt.Print("DEBUG: ") - fmt.Printf("off=%d,", off) - for i := 0; i < 4; i++ { - fmt.Printf(" br[%d]={bitsRead=%d, value=%x, off=%d}", - i, br[i].bitsRead, br[i].value, br[i].off) - } - fmt.Println("") - } - - if off != 0 { - break + decompress4x_main_loop_amd64(&ctx) } - if bufoff > dstEvery { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 1") - } - copy(out, buf[0][:]) - copy(out[dstEvery:], buf[1][:]) - copy(out[dstEvery*2:], buf[2][:]) - copy(out[dstEvery*3:], buf[3][:]) - out = out[bufoff:] - decoded += bufoff * 4 - // There must at least be 3 buffers left. - if len(out) < dstEvery*3 { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 2") - } - } - if off > 0 { - ioff := int(off) - if len(out) < dstEvery*3+ioff { - d.bufs.Put(buf) - return nil, errors.New("corruption detected: stream overrun 3") - } - copy(out, buf[0][:off]) - copy(out[dstEvery:], buf[1][:off]) - copy(out[dstEvery*2:], buf[2][:off]) - copy(out[dstEvery*3:], buf[3][:off]) - decoded += int(off) * 4 - out = out[off:] + decoded = ctx.decoded + out = out[decoded/4:] } // Decode remaining. @@ -150,7 +120,6 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { for bitsLeft > 0 { br.fill() if offset >= endsAt { - d.bufs.Put(buf) return nil, errors.New("corruption detected: stream overrun 4") } @@ -164,7 +133,6 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { offset++ } if offset != endsAt { - d.bufs.Put(buf) return nil, fmt.Errorf("corruption detected: short output block %d, end %d != %d", i, offset, endsAt) } decoded += offset - dstEvery*i @@ -173,7 +141,6 @@ func (d *Decoder) Decompress4X(dst, src []byte) ([]byte, error) { return nil, err } } - d.bufs.Put(buf) if dstSize != decoded { return nil, errors.New("corruption detected: short output block") } diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s index 2edad3ea5..06287f568 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s +++ b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s @@ -1,506 +1,662 @@ -// +build !appengine -// +build gc -// +build !noasm - -#include "textflag.h" -#include "funcdata.h" -#include "go_asm.h" - -#ifdef GOAMD64_v4 -#ifndef GOAMD64_v3 -#define GOAMD64_v3 -#endif -#endif - -#define bufoff 256 // see decompress.go, we're using [4][256]byte table - -// func decompress4x_main_loop_x86(pbr0, pbr1, pbr2, pbr3 *bitReaderShifted, -// peekBits uint8, buf *byte, tbl *dEntrySingle) (int, bool) -TEXT ·decompress4x_main_loop_x86(SB), NOSPLIT, $8 -#define off R8 -#define buffer DI -#define table SI - -#define br_bits_read R9 -#define br_value R10 -#define br_offset R11 -#define peek_bits R12 -#define exhausted DX - -#define br0 R13 -#define br1 R14 -#define br2 R15 -#define br3 BP - - MOVQ BP, 0(SP) - - XORQ exhausted, exhausted // exhausted = false - XORQ off, off // off = 0 - - MOVBQZX peekBits+32(FP), peek_bits - MOVQ buf+40(FP), buffer - MOVQ tbl+48(FP), table - - MOVQ pbr0+0(FP), br0 - MOVQ pbr1+8(FP), br1 - MOVQ pbr2+16(FP), br2 - MOVQ pbr3+24(FP), br3 - +// Code generated by command: go run gen.go -out ../decompress_amd64.s -pkg=huff0. DO NOT EDIT. + +//go:build amd64 && !appengine && !noasm && gc +// +build amd64,!appengine,!noasm,gc + +// func decompress4x_main_loop_amd64(ctx *decompress4xContext) +TEXT ·decompress4x_main_loop_amd64(SB), $8-8 + XORQ DX, DX + + // Preload values + MOVQ ctx+0(FP), AX + MOVBQZX 32(AX), SI + MOVQ 40(AX), DI + MOVQ DI, BX + MOVQ 72(AX), CX + MOVQ CX, (SP) + MOVQ 48(AX), R8 + MOVQ 56(AX), R9 + MOVQ (AX), R10 + MOVQ 8(AX), R11 + MOVQ 16(AX), R12 + MOVQ 24(AX), R13 + + // Main loop main_loop: - - // const stream = 0 - // br0.fillFast() - MOVBQZX bitReaderShifted_bitsRead(br0), br_bits_read - MOVQ bitReaderShifted_value(br0), br_value - MOVQ bitReaderShifted_off(br0), br_offset - - // We must have at least 2 * max tablelog left - CMPQ br_bits_read, $64-22 - JBE skip_fill0 - - SUBQ $32, br_bits_read // b.bitsRead -= 32 - SUBQ $4, br_offset // b.off -= 4 - - // v := b.in[b.off-4 : b.off] - // v = v[:4] - // low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - MOVQ bitReaderShifted_in(br0), AX + MOVQ BX, DI + CMPQ DI, (SP) + SETGE DL + + // br0.fillFast32() + MOVQ 32(R10), R14 + MOVBQZX 40(R10), R15 + CMPQ R15, $0x20 + JBE skip_fill0 + MOVQ 24(R10), AX + SUBQ $0x20, R15 + SUBQ $0x04, AX + MOVQ (R10), BP // b.value |= uint64(low) << (b.bitsRead & 63) -#ifdef GOAMD64_v3 - SHLXQ br_bits_read, 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) << (b.bitsRead & 63) - -#else - MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) - MOVQ br_bits_read, CX - SHLQ CL, AX - -#endif - - ORQ AX, br_value + MOVL (AX)(BP*1), BP + MOVQ R15, CX + SHLQ CL, BP + MOVQ AX, 24(R10) + ORQ BP, R14 // exhausted = exhausted || (br0.off < 4) - CMPQ br_offset, $4 - SETLT DL - ORB DL, DH + CMPQ AX, $0x04 + SETLT AL + ORB AL, DL - // } skip_fill0: - // val0 := br0.peekTopBits(peekBits) -#ifdef GOAMD64_v3 - SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask - -#else - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - -#endif + MOVQ R14, BP + MOVQ SI, CX + SHRQ CL, BP // v0 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 - - // br0.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) - -#ifdef GOAMD64_v3 - MOVBQZX AL, CX - SHLXQ AX, br_value, br_value // value <<= n + MOVW (R9)(BP*2), CX -#else - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n + // br0.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R14 + ADDB CL, R15 -#endif - - ADDQ CX, br_bits_read // bits_read += n - -#ifdef GOAMD64_v3 - SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask - -#else // val1 := br0.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - -#endif + MOVQ SI, CX + MOVQ R14, BP + SHRQ CL, BP // v1 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 + MOVW (R9)(BP*2), CX // br0.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - -#ifdef GOAMD64_v3 - MOVBQZX AL, CX - SHLXQ AX, br_value, br_value // value <<= n - -#else - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - -#endif - - ADDQ CX, br_bits_read // bits_read += n + MOVB CH, AH + SHLQ CL, R14 + ADDB CL, R15 // these two writes get coalesced - // buf[stream][off] = uint8(v0.entry >> 8) - // buf[stream][off+1] = uint8(v1.entry >> 8) - MOVW BX, 0(buffer)(off*1) + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + MOVW AX, (DI) // update the bitrader reader structure - MOVB br_bits_read, bitReaderShifted_bitsRead(br0) - MOVQ br_value, bitReaderShifted_value(br0) - MOVQ br_offset, bitReaderShifted_off(br0) - - // const stream = 1 - // br1.fillFast() - MOVBQZX bitReaderShifted_bitsRead(br1), br_bits_read - MOVQ bitReaderShifted_value(br1), br_value - MOVQ bitReaderShifted_off(br1), br_offset - - // We must have at least 2 * max tablelog left - CMPQ br_bits_read, $64-22 - JBE skip_fill1 - - SUBQ $32, br_bits_read // b.bitsRead -= 32 - SUBQ $4, br_offset // b.off -= 4 - - // v := b.in[b.off-4 : b.off] - // v = v[:4] - // low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - MOVQ bitReaderShifted_in(br1), AX + MOVQ R14, 32(R10) + MOVB R15, 40(R10) + ADDQ R8, DI + + // br1.fillFast32() + MOVQ 32(R11), R14 + MOVBQZX 40(R11), R15 + CMPQ R15, $0x20 + JBE skip_fill1 + MOVQ 24(R11), AX + SUBQ $0x20, R15 + SUBQ $0x04, AX + MOVQ (R11), BP // b.value |= uint64(low) << (b.bitsRead & 63) -#ifdef GOAMD64_v3 - SHLXQ br_bits_read, 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) << (b.bitsRead & 63) - -#else - MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) - MOVQ br_bits_read, CX - SHLQ CL, AX - -#endif - - ORQ AX, br_value + MOVL (AX)(BP*1), BP + MOVQ R15, CX + SHLQ CL, BP + MOVQ AX, 24(R11) + ORQ BP, R14 // exhausted = exhausted || (br1.off < 4) - CMPQ br_offset, $4 - SETLT DL - ORB DL, DH + CMPQ AX, $0x04 + SETLT AL + ORB AL, DL - // } skip_fill1: - // val0 := br1.peekTopBits(peekBits) -#ifdef GOAMD64_v3 - SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask - -#else - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - -#endif + MOVQ R14, BP + MOVQ SI, CX + SHRQ CL, BP // v0 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 - - // br1.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) - -#ifdef GOAMD64_v3 - MOVBQZX AL, CX - SHLXQ AX, br_value, br_value // value <<= n + MOVW (R9)(BP*2), CX -#else - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n + // br1.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R14 + ADDB CL, R15 -#endif - - ADDQ CX, br_bits_read // bits_read += n - -#ifdef GOAMD64_v3 - SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask - -#else // val1 := br1.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - -#endif + MOVQ SI, CX + MOVQ R14, BP + SHRQ CL, BP // v1 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 + MOVW (R9)(BP*2), CX // br1.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - -#ifdef GOAMD64_v3 - MOVBQZX AL, CX - SHLXQ AX, br_value, br_value // value <<= n - -#else - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - -#endif - - ADDQ CX, br_bits_read // bits_read += n + MOVB CH, AH + SHLQ CL, R14 + ADDB CL, R15 // these two writes get coalesced - // buf[stream][off] = uint8(v0.entry >> 8) - // buf[stream][off+1] = uint8(v1.entry >> 8) - MOVW BX, 256(buffer)(off*1) + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + MOVW AX, (DI) // update the bitrader reader structure - MOVB br_bits_read, bitReaderShifted_bitsRead(br1) - MOVQ br_value, bitReaderShifted_value(br1) - MOVQ br_offset, bitReaderShifted_off(br1) - - // const stream = 2 - // br2.fillFast() - MOVBQZX bitReaderShifted_bitsRead(br2), br_bits_read - MOVQ bitReaderShifted_value(br2), br_value - MOVQ bitReaderShifted_off(br2), br_offset - - // We must have at least 2 * max tablelog left - CMPQ br_bits_read, $64-22 - JBE skip_fill2 - - SUBQ $32, br_bits_read // b.bitsRead -= 32 - SUBQ $4, br_offset // b.off -= 4 - - // v := b.in[b.off-4 : b.off] - // v = v[:4] - // low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - MOVQ bitReaderShifted_in(br2), AX + MOVQ R14, 32(R11) + MOVB R15, 40(R11) + ADDQ R8, DI + + // br2.fillFast32() + MOVQ 32(R12), R14 + MOVBQZX 40(R12), R15 + CMPQ R15, $0x20 + JBE skip_fill2 + MOVQ 24(R12), AX + SUBQ $0x20, R15 + SUBQ $0x04, AX + MOVQ (R12), BP // b.value |= uint64(low) << (b.bitsRead & 63) -#ifdef GOAMD64_v3 - SHLXQ br_bits_read, 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) << (b.bitsRead & 63) - -#else - MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) - MOVQ br_bits_read, CX - SHLQ CL, AX - -#endif - - ORQ AX, br_value + MOVL (AX)(BP*1), BP + MOVQ R15, CX + SHLQ CL, BP + MOVQ AX, 24(R12) + ORQ BP, R14 // exhausted = exhausted || (br2.off < 4) - CMPQ br_offset, $4 - SETLT DL - ORB DL, DH + CMPQ AX, $0x04 + SETLT AL + ORB AL, DL - // } skip_fill2: - // val0 := br2.peekTopBits(peekBits) -#ifdef GOAMD64_v3 - SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask - -#else - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - -#endif + MOVQ R14, BP + MOVQ SI, CX + SHRQ CL, BP // v0 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 + MOVW (R9)(BP*2), CX - // br2.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) + // br2.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R14 + ADDB CL, R15 -#ifdef GOAMD64_v3 - MOVBQZX AL, CX - SHLXQ AX, br_value, br_value // value <<= n - -#else - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - -#endif - - ADDQ CX, br_bits_read // bits_read += n - -#ifdef GOAMD64_v3 - SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask - -#else // val1 := br2.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - -#endif + MOVQ SI, CX + MOVQ R14, BP + SHRQ CL, BP // v1 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 + MOVW (R9)(BP*2), CX // br2.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - -#ifdef GOAMD64_v3 - MOVBQZX AL, CX - SHLXQ AX, br_value, br_value // value <<= n - -#else - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n - -#endif - - ADDQ CX, br_bits_read // bits_read += n + MOVB CH, AH + SHLQ CL, R14 + ADDB CL, R15 // these two writes get coalesced - // buf[stream][off] = uint8(v0.entry >> 8) - // buf[stream][off+1] = uint8(v1.entry >> 8) - MOVW BX, 512(buffer)(off*1) + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + MOVW AX, (DI) // update the bitrader reader structure - MOVB br_bits_read, bitReaderShifted_bitsRead(br2) - MOVQ br_value, bitReaderShifted_value(br2) - MOVQ br_offset, bitReaderShifted_off(br2) - - // const stream = 3 - // br3.fillFast() - MOVBQZX bitReaderShifted_bitsRead(br3), br_bits_read - MOVQ bitReaderShifted_value(br3), br_value - MOVQ bitReaderShifted_off(br3), br_offset - - // We must have at least 2 * max tablelog left - CMPQ br_bits_read, $64-22 - JBE skip_fill3 - - SUBQ $32, br_bits_read // b.bitsRead -= 32 - SUBQ $4, br_offset // b.off -= 4 - - // v := b.in[b.off-4 : b.off] - // v = v[:4] - // low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - MOVQ bitReaderShifted_in(br3), AX + MOVQ R14, 32(R12) + MOVB R15, 40(R12) + ADDQ R8, DI + + // br3.fillFast32() + MOVQ 32(R13), R14 + MOVBQZX 40(R13), R15 + CMPQ R15, $0x20 + JBE skip_fill3 + MOVQ 24(R13), AX + SUBQ $0x20, R15 + SUBQ $0x04, AX + MOVQ (R13), BP // b.value |= uint64(low) << (b.bitsRead & 63) -#ifdef GOAMD64_v3 - SHLXQ br_bits_read, 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) << (b.bitsRead & 63) - -#else - MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) - MOVQ br_bits_read, CX - SHLQ CL, AX - -#endif - - ORQ AX, br_value + MOVL (AX)(BP*1), BP + MOVQ R15, CX + SHLQ CL, BP + MOVQ AX, 24(R13) + ORQ BP, R14 // exhausted = exhausted || (br3.off < 4) - CMPQ br_offset, $4 - SETLT DL - ORB DL, DH + CMPQ AX, $0x04 + SETLT AL + ORB AL, DL - // } skip_fill3: - // val0 := br3.peekTopBits(peekBits) -#ifdef GOAMD64_v3 - SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask - -#else - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask - -#endif + MOVQ R14, BP + MOVQ SI, CX + SHRQ CL, BP // v0 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 + MOVW (R9)(BP*2), CX - // br3.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) + // br3.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R14 + ADDB CL, R15 -#ifdef GOAMD64_v3 - MOVBQZX AL, CX - SHLXQ AX, br_value, br_value // value <<= n - -#else - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n + // val1 := br3.peekTopBits(peekBits) + MOVQ SI, CX + MOVQ R14, BP + SHRQ CL, BP -#endif + // v1 := table[val1&mask] + MOVW (R9)(BP*2), CX - ADDQ CX, br_bits_read // bits_read += n + // br3.advance(uint8(v1.entry)) + MOVB CH, AH + SHLQ CL, R14 + ADDB CL, R15 -#ifdef GOAMD64_v3 - SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask + // these two writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + MOVW AX, (DI) -#else - // val1 := br3.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask + // update the bitrader reader structure + MOVQ R14, 32(R13) + MOVB R15, 40(R13) + ADDQ $0x02, BX + TESTB DL, DL + JZ main_loop + MOVQ ctx+0(FP), AX + MOVQ 40(AX), CX + MOVQ BX, DX + SUBQ CX, DX + SHLQ $0x02, DX + MOVQ DX, 64(AX) + RET -#endif +// func decompress4x_8b_main_loop_amd64(ctx *decompress4xContext) +TEXT ·decompress4x_8b_main_loop_amd64(SB), $16-8 + XORQ DX, DX + + // Preload values + MOVQ ctx+0(FP), CX + MOVBQZX 32(CX), BX + MOVQ 40(CX), SI + MOVQ SI, (SP) + MOVQ 72(CX), DX + MOVQ DX, 8(SP) + MOVQ 48(CX), DI + MOVQ 56(CX), R8 + MOVQ (CX), R9 + MOVQ 8(CX), R10 + MOVQ 16(CX), R11 + MOVQ 24(CX), R12 + + // Main loop +main_loop: + MOVQ (SP), SI + CMPQ SI, 8(SP) + SETGE DL + + // br1000.fillFast32() + MOVQ 32(R9), R13 + MOVBQZX 40(R9), R14 + CMPQ R14, $0x20 + JBE skip_fill1000 + MOVQ 24(R9), R15 + SUBQ $0x20, R14 + SUBQ $0x04, R15 + MOVQ (R9), BP - // v1 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 + // b.value |= uint64(low) << (b.bitsRead & 63) + MOVL (R15)(BP*1), BP + MOVQ R14, CX + SHLQ CL, BP + MOVQ R15, 24(R9) + ORQ BP, R13 + + // exhausted = exhausted || (br1000.off < 4) + CMPQ R15, $0x04 + SETLT AL + ORB AL, DL + +skip_fill1000: + // val0 := br0.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 - // br3.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) + // v0 := table[val0&mask] + MOVW (R8)(R15*2), CX -#ifdef GOAMD64_v3 - MOVBQZX AL, CX - SHLXQ AX, br_value, br_value // value <<= n + // br0.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R13 + ADDB CL, R14 -#else - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n + // val1 := br0.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 + + // v1 := table[val0&mask] + MOVW (R8)(R15*2), CX + + // br0.advance(uint8(v1.entry) + MOVB CH, AH + SHLQ CL, R13 + ADDB CL, R14 + BSWAPL AX + + // val2 := br0.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 + + // v2 := table[val0&mask] + MOVW (R8)(R15*2), CX + + // br0.advance(uint8(v2.entry) + MOVB CH, AH + SHLQ CL, R13 + ADDB CL, R14 + + // val3 := br0.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 + + // v3 := table[val0&mask] + MOVW (R8)(R15*2), CX + + // br0.advance(uint8(v3.entry) + MOVB CH, AL + SHLQ CL, R13 + ADDB CL, R14 + BSWAPL AX + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + MOVL AX, (SI) + + // update the bitreader reader structure + MOVQ R13, 32(R9) + MOVB R14, 40(R9) + ADDQ DI, SI + + // br1001.fillFast32() + MOVQ 32(R10), R13 + MOVBQZX 40(R10), R14 + CMPQ R14, $0x20 + JBE skip_fill1001 + MOVQ 24(R10), R15 + SUBQ $0x20, R14 + SUBQ $0x04, R15 + MOVQ (R10), BP -#endif + // b.value |= uint64(low) << (b.bitsRead & 63) + MOVL (R15)(BP*1), BP + MOVQ R14, CX + SHLQ CL, BP + MOVQ R15, 24(R10) + ORQ BP, R13 + + // exhausted = exhausted || (br1001.off < 4) + CMPQ R15, $0x04 + SETLT AL + ORB AL, DL + +skip_fill1001: + // val0 := br1.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 - ADDQ CX, br_bits_read // bits_read += n + // v0 := table[val0&mask] + MOVW (R8)(R15*2), CX - // these two writes get coalesced - // buf[stream][off] = uint8(v0.entry >> 8) - // buf[stream][off+1] = uint8(v1.entry >> 8) - MOVW BX, 768(buffer)(off*1) + // br1.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R13 + ADDB CL, R14 - // update the bitrader reader structure - MOVB br_bits_read, bitReaderShifted_bitsRead(br3) - MOVQ br_value, bitReaderShifted_value(br3) - MOVQ br_offset, bitReaderShifted_off(br3) + // val1 := br1.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 + + // v1 := table[val0&mask] + MOVW (R8)(R15*2), CX + + // br1.advance(uint8(v1.entry) + MOVB CH, AH + SHLQ CL, R13 + ADDB CL, R14 + BSWAPL AX + + // val2 := br1.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 + + // v2 := table[val0&mask] + MOVW (R8)(R15*2), CX + + // br1.advance(uint8(v2.entry) + MOVB CH, AH + SHLQ CL, R13 + ADDB CL, R14 + + // val3 := br1.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 + + // v3 := table[val0&mask] + MOVW (R8)(R15*2), CX + + // br1.advance(uint8(v3.entry) + MOVB CH, AL + SHLQ CL, R13 + ADDB CL, R14 + BSWAPL AX + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + MOVL AX, (SI) + + // update the bitreader reader structure + MOVQ R13, 32(R10) + MOVB R14, 40(R10) + ADDQ DI, SI + + // br1002.fillFast32() + MOVQ 32(R11), R13 + MOVBQZX 40(R11), R14 + CMPQ R14, $0x20 + JBE skip_fill1002 + MOVQ 24(R11), R15 + SUBQ $0x20, R14 + SUBQ $0x04, R15 + MOVQ (R11), BP - ADDQ $2, off // off += 2 + // b.value |= uint64(low) << (b.bitsRead & 63) + MOVL (R15)(BP*1), BP + MOVQ R14, CX + SHLQ CL, BP + MOVQ R15, 24(R11) + ORQ BP, R13 + + // exhausted = exhausted || (br1002.off < 4) + CMPQ R15, $0x04 + SETLT AL + ORB AL, DL + +skip_fill1002: + // val0 := br2.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 - TESTB DH, DH // any br[i].ofs < 4? - JNZ end + // v0 := table[val0&mask] + MOVW (R8)(R15*2), CX - CMPQ off, $bufoff - JL main_loop + // br2.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R13 + ADDB CL, R14 -end: - MOVQ 0(SP), BP + // val1 := br2.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 + + // v1 := table[val0&mask] + MOVW (R8)(R15*2), CX + + // br2.advance(uint8(v1.entry) + MOVB CH, AH + SHLQ CL, R13 + ADDB CL, R14 + BSWAPL AX + + // val2 := br2.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 + + // v2 := table[val0&mask] + MOVW (R8)(R15*2), CX + + // br2.advance(uint8(v2.entry) + MOVB CH, AH + SHLQ CL, R13 + ADDB CL, R14 + + // val3 := br2.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 + + // v3 := table[val0&mask] + MOVW (R8)(R15*2), CX + + // br2.advance(uint8(v3.entry) + MOVB CH, AL + SHLQ CL, R13 + ADDB CL, R14 + BSWAPL AX + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + MOVL AX, (SI) + + // update the bitreader reader structure + MOVQ R13, 32(R11) + MOVB R14, 40(R11) + ADDQ DI, SI + + // br1003.fillFast32() + MOVQ 32(R12), R13 + MOVBQZX 40(R12), R14 + CMPQ R14, $0x20 + JBE skip_fill1003 + MOVQ 24(R12), R15 + SUBQ $0x20, R14 + SUBQ $0x04, R15 + MOVQ (R12), BP - MOVB off, ret+56(FP) - RET + // b.value |= uint64(low) << (b.bitsRead & 63) + MOVL (R15)(BP*1), BP + MOVQ R14, CX + SHLQ CL, BP + MOVQ R15, 24(R12) + ORQ BP, R13 + + // exhausted = exhausted || (br1003.off < 4) + CMPQ R15, $0x04 + SETLT AL + ORB AL, DL + +skip_fill1003: + // val0 := br3.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 -#undef off -#undef buffer -#undef table + // v0 := table[val0&mask] + MOVW (R8)(R15*2), CX -#undef br_bits_read -#undef br_value -#undef br_offset -#undef peek_bits -#undef exhausted + // br3.advance(uint8(v0.entry) + MOVB CH, AL + SHLQ CL, R13 + ADDB CL, R14 -#undef br0 -#undef br1 -#undef br2 -#undef br3 + // val1 := br3.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 + + // v1 := table[val0&mask] + MOVW (R8)(R15*2), CX + + // br3.advance(uint8(v1.entry) + MOVB CH, AH + SHLQ CL, R13 + ADDB CL, R14 + BSWAPL AX + + // val2 := br3.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 + + // v2 := table[val0&mask] + MOVW (R8)(R15*2), CX + + // br3.advance(uint8(v2.entry) + MOVB CH, AH + SHLQ CL, R13 + ADDB CL, R14 + + // val3 := br3.peekTopBits(peekBits) + MOVQ R13, R15 + MOVQ BX, CX + SHRQ CL, R15 + + // v3 := table[val0&mask] + MOVW (R8)(R15*2), CX + + // br3.advance(uint8(v3.entry) + MOVB CH, AL + SHLQ CL, R13 + ADDB CL, R14 + BSWAPL AX + + // these four writes get coalesced + // out[id * dstEvery + 0] = uint8(v0.entry >> 8) + // out[id * dstEvery + 1] = uint8(v1.entry >> 8) + // out[id * dstEvery + 3] = uint8(v2.entry >> 8) + // out[id * dstEvery + 4] = uint8(v3.entry >> 8) + MOVL AX, (SI) + + // update the bitreader reader structure + MOVQ R13, 32(R12) + MOVB R14, 40(R12) + ADDQ $0x04, (SP) + TESTB DL, DL + JZ main_loop + MOVQ ctx+0(FP), AX + MOVQ 40(AX), CX + MOVQ (SP), DX + SUBQ CX, DX + SHLQ $0x02, DX + MOVQ DX, 64(AX) + RET diff --git a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s.in b/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s.in deleted file mode 100644 index 330d86ae1..000000000 --- a/vendor/github.com/klauspost/compress/huff0/decompress_amd64.s.in +++ /dev/null @@ -1,195 +0,0 @@ -// +build !appengine -// +build gc -// +build !noasm - -#include "textflag.h" -#include "funcdata.h" -#include "go_asm.h" - -#ifdef GOAMD64_v4 -#ifndef GOAMD64_v3 -#define GOAMD64_v3 -#endif -#endif - -#define bufoff 256 // see decompress.go, we're using [4][256]byte table - -//func decompress4x_main_loop_x86(pbr0, pbr1, pbr2, pbr3 *bitReaderShifted, -// peekBits uint8, buf *byte, tbl *dEntrySingle) (int, bool) -TEXT ·decompress4x_main_loop_x86(SB), NOSPLIT, $8 -#define off R8 -#define buffer DI -#define table SI - -#define br_bits_read R9 -#define br_value R10 -#define br_offset R11 -#define peek_bits R12 -#define exhausted DX - -#define br0 R13 -#define br1 R14 -#define br2 R15 -#define br3 BP - - MOVQ BP, 0(SP) - - XORQ exhausted, exhausted // exhausted = false - XORQ off, off // off = 0 - - MOVBQZX peekBits+32(FP), peek_bits - MOVQ buf+40(FP), buffer - MOVQ tbl+48(FP), table - - MOVQ pbr0+0(FP), br0 - MOVQ pbr1+8(FP), br1 - MOVQ pbr2+16(FP), br2 - MOVQ pbr3+24(FP), br3 - -main_loop: -{{ define "decode_2_values_x86" }} - // const stream = {{ var "id" }} - // br{{ var "id"}}.fillFast() - MOVBQZX bitReaderShifted_bitsRead(br{{ var "id" }}), br_bits_read - MOVQ bitReaderShifted_value(br{{ var "id" }}), br_value - MOVQ bitReaderShifted_off(br{{ var "id" }}), br_offset - - // We must have at least 2 * max tablelog left - CMPQ br_bits_read, $64-22 - JBE skip_fill{{ var "id" }} - - SUBQ $32, br_bits_read // b.bitsRead -= 32 - SUBQ $4, br_offset // b.off -= 4 - - // v := b.in[b.off-4 : b.off] - // v = v[:4] - // low := (uint32(v[0])) | (uint32(v[1]) << 8) | (uint32(v[2]) << 16) | (uint32(v[3]) << 24) - MOVQ bitReaderShifted_in(br{{ var "id" }}), AX - - // b.value |= uint64(low) << (b.bitsRead & 63) -#ifdef GOAMD64_v3 - SHLXQ br_bits_read, 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) << (b.bitsRead & 63) -#else - MOVL 0(br_offset)(AX*1), AX // AX = uint32(b.in[b.off:b.off+4]) - MOVQ br_bits_read, CX - SHLQ CL, AX -#endif - - ORQ AX, br_value - - // exhausted = exhausted || (br{{ var "id"}}.off < 4) - CMPQ br_offset, $4 - SETLT DL - ORB DL, DH - // } -skip_fill{{ var "id" }}: - - // val0 := br{{ var "id"}}.peekTopBits(peekBits) -#ifdef GOAMD64_v3 - SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask -#else - MOVQ br_value, AX - MOVQ peek_bits, CX - SHRQ CL, AX // AX = (value >> peek_bits) & mask -#endif - - // v0 := table[val0&mask] - MOVW 0(table)(AX*2), AX // AX - v0 - - // br{{ var "id"}}.advance(uint8(v0.entry)) - MOVB AH, BL // BL = uint8(v0.entry >> 8) - -#ifdef GOAMD64_v3 - MOVBQZX AL, CX - SHLXQ AX, br_value, br_value // value <<= n -#else - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n -#endif - - ADDQ CX, br_bits_read // bits_read += n - - -#ifdef GOAMD64_v3 - SHRXQ peek_bits, br_value, AX // AX = (value >> peek_bits) & mask -#else - // val1 := br{{ var "id"}}.peekTopBits(peekBits) - MOVQ peek_bits, CX - MOVQ br_value, AX - SHRQ CL, AX // AX = (value >> peek_bits) & mask -#endif - - // v1 := table[val1&mask] - MOVW 0(table)(AX*2), AX // AX - v1 - - // br{{ var "id"}}.advance(uint8(v1.entry)) - MOVB AH, BH // BH = uint8(v1.entry >> 8) - -#ifdef GOAMD64_v3 - MOVBQZX AL, CX - SHLXQ AX, br_value, br_value // value <<= n -#else - MOVBQZX AL, CX - SHLQ CL, br_value // value <<= n -#endif - - ADDQ CX, br_bits_read // bits_read += n - - - // these two writes get coalesced - // buf[stream][off] = uint8(v0.entry >> 8) - // buf[stream][off+1] = uint8(v1.entry >> 8) - MOVW BX, {{ var "bufofs" }}(buffer)(off*1) - - // update the bitrader reader structure - MOVB br_bits_read, bitReaderShifted_bitsRead(br{{ var "id" }}) - MOVQ br_value, bitReaderShifted_value(br{{ var "id" }}) - MOVQ br_offset, bitReaderShifted_off(br{{ var "id" }}) -{{ end }} - - {{ set "id" "0" }} - {{ set "ofs" "0" }} - {{ set "bufofs" "0" }} {{/* id * bufoff */}} - {{ template "decode_2_values_x86" . }} - - {{ set "id" "1" }} - {{ set "ofs" "8" }} - {{ set "bufofs" "256" }} - {{ template "decode_2_values_x86" . }} - - {{ set "id" "2" }} - {{ set "ofs" "16" }} - {{ set "bufofs" "512" }} - {{ template "decode_2_values_x86" . }} - - {{ set "id" "3" }} - {{ set "ofs" "24" }} - {{ set "bufofs" "768" }} - {{ template "decode_2_values_x86" . }} - - ADDQ $2, off // off += 2 - - TESTB DH, DH // any br[i].ofs < 4? - JNZ end - - CMPQ off, $bufoff - JL main_loop -end: - MOVQ 0(SP), BP - - MOVB off, ret+56(FP) - RET -#undef off -#undef buffer -#undef table - -#undef br_bits_read -#undef br_value -#undef br_offset -#undef peek_bits -#undef exhausted - -#undef br0 -#undef br1 -#undef br2 -#undef br3 diff --git a/vendor/github.com/klauspost/compress/zstd/decoder.go b/vendor/github.com/klauspost/compress/zstd/decoder.go index c65ea9795..36119f385 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder.go @@ -439,7 +439,7 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) { println("got", len(d.current.b), "bytes, error:", d.current.err, "data crc:", tmp) } - if len(next.b) > 0 { + if !d.o.ignoreChecksum && len(next.b) > 0 { n, err := d.current.crc.Write(next.b) if err == nil { if n != len(next.b) { @@ -451,7 +451,7 @@ func (d *Decoder) nextBlock(blocking bool) (ok bool) { got := d.current.crc.Sum64() var tmp [4]byte binary.LittleEndian.PutUint32(tmp[:], uint32(got)) - if !bytes.Equal(tmp[:], next.d.checkCRC) && !ignoreCRC { + if !d.o.ignoreChecksum && !bytes.Equal(tmp[:], next.d.checkCRC) { if debugDecoder { println("CRC Check Failed:", tmp[:], " (got) !=", next.d.checkCRC, "(on stream)") } @@ -535,9 +535,15 @@ func (d *Decoder) nextBlockSync() (ok bool) { // Update/Check CRC if d.frame.HasCheckSum { - d.frame.crc.Write(d.current.b) + if !d.o.ignoreChecksum { + d.frame.crc.Write(d.current.b) + } if d.current.d.Last { - d.current.err = d.frame.checkCRC() + if !d.o.ignoreChecksum { + d.current.err = d.frame.checkCRC() + } else { + d.current.err = d.frame.consumeCRC() + } if d.current.err != nil { println("CRC error:", d.current.err) return false diff --git a/vendor/github.com/klauspost/compress/zstd/decoder_options.go b/vendor/github.com/klauspost/compress/zstd/decoder_options.go index fc52ebc40..c70e6fa0f 100644 --- a/vendor/github.com/klauspost/compress/zstd/decoder_options.go +++ b/vendor/github.com/klauspost/compress/zstd/decoder_options.go @@ -19,6 +19,7 @@ type decoderOptions struct { maxDecodedSize uint64 maxWindowSize uint64 dicts []dict + ignoreChecksum bool } func (o *decoderOptions) setDefault() { @@ -112,3 +113,11 @@ func WithDecoderMaxWindow(size uint64) DOption { return nil } } + +// IgnoreChecksum allows to forcibly ignore checksum checking. +func IgnoreChecksum(b bool) DOption { + return func(o *decoderOptions) error { + o.ignoreChecksum = b + return nil + } +} diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index 509d5cece..3ff109cce 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -290,13 +290,6 @@ func (d *frameDec) checkCRC() error { if !d.HasCheckSum { return nil } - var tmp [4]byte - got := d.crc.Sum64() - // Flip to match file order. - tmp[0] = byte(got >> 0) - tmp[1] = byte(got >> 8) - tmp[2] = byte(got >> 16) - tmp[3] = byte(got >> 24) // We can overwrite upper tmp now want, err := d.rawInput.readSmall(4) @@ -305,7 +298,19 @@ func (d *frameDec) checkCRC() error { return err } - if !bytes.Equal(tmp[:], want) && !ignoreCRC { + if d.o.ignoreChecksum { + return nil + } + + var tmp [4]byte + got := d.crc.Sum64() + // Flip to match file order. + tmp[0] = byte(got >> 0) + tmp[1] = byte(got >> 8) + tmp[2] = byte(got >> 16) + tmp[3] = byte(got >> 24) + + if !bytes.Equal(tmp[:], want) { if debugDecoder { println("CRC Check Failed:", tmp[:], "!=", want) } @@ -317,6 +322,19 @@ func (d *frameDec) checkCRC() error { return nil } +// consumeCRC reads the checksum data if the frame has one. +func (d *frameDec) consumeCRC() error { + if d.HasCheckSum { + _, err := d.rawInput.readSmall(4) + if err != nil { + println("CRC missing?", err) + return err + } + } + + return nil +} + // runDecoder will create a sync decoder that will decode a block of data. func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { saved := d.history.b @@ -373,13 +391,17 @@ func (d *frameDec) runDecoder(dst []byte, dec *blockDec) ([]byte, error) { if d.FrameContentSize != fcsUnknown && uint64(len(d.history.b)-crcStart) != d.FrameContentSize { err = ErrFrameSizeMismatch } else if d.HasCheckSum { - var n int - n, err = d.crc.Write(dst[crcStart:]) - if err == nil { - if n != len(dst)-crcStart { - err = io.ErrShortWrite - } else { - err = d.checkCRC() + if d.o.ignoreChecksum { + err = d.consumeCRC() + } else { + var n int + n, err = d.crc.Write(dst[crcStart:]) + if err == nil { + if n != len(dst)-crcStart { + err = io.ErrShortWrite + } else { + err = d.checkCRC() + } } } } diff --git a/vendor/github.com/klauspost/compress/zstd/fuzz.go b/vendor/github.com/klauspost/compress/zstd/fuzz.go deleted file mode 100644 index 7f2210e05..000000000 --- a/vendor/github.com/klauspost/compress/zstd/fuzz.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build ignorecrc -// +build ignorecrc - -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -// ignoreCRC can be used for fuzz testing to ignore CRC values... -const ignoreCRC = true diff --git a/vendor/github.com/klauspost/compress/zstd/fuzz_none.go b/vendor/github.com/klauspost/compress/zstd/fuzz_none.go deleted file mode 100644 index 6811c68a8..000000000 --- a/vendor/github.com/klauspost/compress/zstd/fuzz_none.go +++ /dev/null @@ -1,11 +0,0 @@ -//go:build !ignorecrc -// +build !ignorecrc - -// Copyright 2019+ Klaus Post. All rights reserved. -// License information can be found in the LICENSE file. -// Based on work by Yann Collet, released under BSD License. - -package zstd - -// ignoreCRC can be used for fuzz testing to ignore CRC values... -const ignoreCRC = false diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 01cc23fa8..2585b2e98 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -1326,30 +1326,30 @@ copy_match: JA copy_overlapping_match // Copy non-overlapping match - XORQ R12, R12 + ADDQ R13, DI + MOVQ BX, R12 + ADDQ R13, BX copy_2: - MOVUPS (R11)(R12*1), X0 - MOVUPS X0, (BX)(R12*1) + MOVUPS (R11), X0 + MOVUPS X0, (R12) + ADDQ $0x10, R11 ADDQ $0x10, R12 - CMPQ R12, R13 - JB copy_2 - ADDQ R13, BX - ADDQ R13, DI + SUBQ $0x10, R13 + JHI copy_2 JMP handle_loop // Copy overlapping match copy_overlapping_match: - XORQ R12, R12 + ADDQ R13, DI copy_slow_3: - MOVB (R11)(R12*1), R14 - MOVB R14, (BX)(R12*1) - INCQ R12 - CMPQ R12, R13 - JB copy_slow_3 - ADDQ R13, BX - ADDQ R13, DI + MOVB (R11), R12 + MOVB R12, (BX) + INCQ R11 + INCQ BX + DECQ R13 + JNZ copy_slow_3 handle_loop: ADDQ $0x18, AX @@ -1826,30 +1826,30 @@ copy_match: JA copy_overlapping_match // Copy non-overlapping match - XORQ CX, CX + ADDQ R13, R12 + MOVQ R10, CX + ADDQ R13, R10 copy_2: - MOVUPS (AX)(CX*1), X0 - MOVUPS X0, (R10)(CX*1) + MOVUPS (AX), X0 + MOVUPS X0, (CX) + ADDQ $0x10, AX ADDQ $0x10, CX - CMPQ CX, R13 - JB copy_2 - ADDQ R13, R10 - ADDQ R13, R12 + SUBQ $0x10, R13 + JHI copy_2 JMP handle_loop // Copy overlapping match copy_overlapping_match: - XORQ CX, CX + ADDQ R13, R12 copy_slow_3: - MOVB (AX)(CX*1), R14 - MOVB R14, (R10)(CX*1) - INCQ CX - CMPQ CX, R13 - JB copy_slow_3 - ADDQ R13, R10 - ADDQ R13, R12 + MOVB (AX), CL + MOVB CL, (R10) + INCQ AX + INCQ R10 + DECQ R13 + JNZ copy_slow_3 handle_loop: MOVQ ctx+16(FP), AX @@ -2333,30 +2333,30 @@ copy_match: JA copy_overlapping_match // Copy non-overlapping match - XORQ R12, R12 + ADDQ R13, R11 + MOVQ R9, R12 + ADDQ R13, R9 copy_2: - MOVUPS (CX)(R12*1), X0 - MOVUPS X0, (R9)(R12*1) + MOVUPS (CX), X0 + MOVUPS X0, (R12) + ADDQ $0x10, CX ADDQ $0x10, R12 - CMPQ R12, R13 - JB copy_2 - ADDQ R13, R9 - ADDQ R13, R11 + SUBQ $0x10, R13 + JHI copy_2 JMP handle_loop // Copy overlapping match copy_overlapping_match: - XORQ R12, R12 + ADDQ R13, R11 copy_slow_3: - MOVB (CX)(R12*1), R14 - MOVB R14, (R9)(R12*1) - INCQ R12 - CMPQ R12, R13 - JB copy_slow_3 - ADDQ R13, R9 - ADDQ R13, R11 + MOVB (CX), R12 + MOVB R12, (R9) + INCQ CX + INCQ R9 + DECQ R13 + JNZ copy_slow_3 handle_loop: MOVQ ctx+16(FP), CX @@ -2862,6 +2862,7 @@ copy_match: JA copy_overlapping_match // Copy non-overlapping match + ADDQ R13, R12 XORQ CX, CX TESTQ $0x00000001, R13 JZ copy_2_word @@ -2900,21 +2901,19 @@ copy_2_test: CMPQ CX, R13 JB copy_2 ADDQ R13, R10 - ADDQ R13, R12 JMP handle_loop // Copy overlapping match copy_overlapping_match: - XORQ CX, CX + ADDQ R13, R12 copy_slow_3: - MOVB (AX)(CX*1), R14 - MOVB R14, (R10)(CX*1) - INCQ CX - CMPQ CX, R13 - JB copy_slow_3 - ADDQ R13, R10 - ADDQ R13, R12 + MOVB (AX), CL + MOVB CL, (R10) + INCQ AX + INCQ R10 + DECQ R13 + JNZ copy_slow_3 handle_loop: MOVQ ctx+16(FP), AX @@ -3398,6 +3397,7 @@ copy_match: JA copy_overlapping_match // Copy non-overlapping match + ADDQ R13, R11 XORQ R12, R12 TESTQ $0x00000001, R13 JZ copy_2_word @@ -3436,21 +3436,19 @@ copy_2_test: CMPQ R12, R13 JB copy_2 ADDQ R13, R9 - ADDQ R13, R11 JMP handle_loop // Copy overlapping match copy_overlapping_match: - XORQ R12, R12 + ADDQ R13, R11 copy_slow_3: - MOVB (CX)(R12*1), R14 - MOVB R14, (R9)(R12*1) - INCQ R12 - CMPQ R12, R13 - JB copy_slow_3 - ADDQ R13, R9 - ADDQ R13, R11 + MOVB (CX), R12 + MOVB R12, (R9) + INCQ CX + INCQ R9 + DECQ R13 + JNZ copy_slow_3 handle_loop: MOVQ ctx+16(FP), CX diff --git a/vendor/modules.txt b/vendor/modules.txt index 51e0d5f4a..c73f11330 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -109,7 +109,7 @@ github.com/containers/buildah/pkg/rusage github.com/containers/buildah/pkg/sshagent github.com/containers/buildah/pkg/util github.com/containers/buildah/util -# github.com/containers/common v0.48.0 +# github.com/containers/common v0.48.1-0.20220512112240-7536bf6ff9b1 ## explicit github.com/containers/common/libimage github.com/containers/common/libimage/manifests @@ -155,7 +155,7 @@ github.com/containers/common/version # github.com/containers/conmon v2.0.20+incompatible ## explicit github.com/containers/conmon/runner/config -# github.com/containers/image/v5 v5.21.1 +# github.com/containers/image/v5 v5.21.2-0.20220511203756-fe4fd4ed8be4 ## explicit github.com/containers/image/v5/copy github.com/containers/image/v5/directory @@ -235,7 +235,7 @@ github.com/containers/psgo/internal/dev github.com/containers/psgo/internal/host github.com/containers/psgo/internal/proc github.com/containers/psgo/internal/process -# github.com/containers/storage v1.40.2 +# github.com/containers/storage v1.41.1-0.20220511210719-cacc3325a9c8 ## explicit github.com/containers/storage github.com/containers/storage/drivers @@ -328,7 +328,7 @@ github.com/docker/distribution/registry/client/auth/challenge github.com/docker/distribution/registry/client/transport github.com/docker/distribution/registry/storage/cache github.com/docker/distribution/registry/storage/cache/memory -# github.com/docker/docker v20.10.15+incompatible +# github.com/docker/docker v20.10.16+incompatible ## explicit github.com/docker/docker/api github.com/docker/docker/api/types @@ -450,7 +450,7 @@ github.com/jinzhu/copier # github.com/json-iterator/go v1.1.12 ## explicit github.com/json-iterator/go -# github.com/klauspost/compress v1.15.2 +# github.com/klauspost/compress v1.15.4 github.com/klauspost/compress github.com/klauspost/compress/flate github.com/klauspost/compress/fse @@ -553,7 +553,7 @@ github.com/opencontainers/go-digest ## explicit github.com/opencontainers/image-spec/specs-go github.com/opencontainers/image-spec/specs-go/v1 -# github.com/opencontainers/runc v1.1.1 +# github.com/opencontainers/runc v1.1.2 ## explicit github.com/opencontainers/runc/libcontainer/apparmor github.com/opencontainers/runc/libcontainer/cgroups |
