diff options
57 files changed, 1489 insertions, 587 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index e3ddc4933..7da409ffb 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -33,7 +33,7 @@ env: UBUNTU_NAME: "ubuntu-2204" # Image identifiers - IMAGE_SUFFIX: "c5495735033528320" + IMAGE_SUFFIX: "c5823947156488192" # EC2 images FEDORA_AMI: "fedora-aws-${IMAGE_SUFFIX}" FEDORA_AARCH64_AMI: "fedora-podman-aws-arm64-${IMAGE_SUFFIX}" @@ -986,8 +986,12 @@ meta_task: ${FEDORA_CACHE_IMAGE_NAME} ${UBUNTU_CACHE_IMAGE_NAME} build-push-${IMAGE_SUFFIX} + EC2IMGNAMES: >- + ${FEDORA_AARCH64_AMI} + ${FEDORA_AMI} BUILDID: "${CIRRUS_BUILD_ID}" REPOREF: "${CIRRUS_REPO_NAME}" + AWSINI: ENCRYPTED[21b2db557171b11eb5abdbccae593f48c9caeba86dfcc4d4ff109edee9b4656ab6720a110dadfcd51e88cc59a71cc7af] GCPJSON: ENCRYPTED[3a198350077849c8df14b723c0f4c9fece9ebe6408d35982e7adf2105a33f8e0e166ed3ed614875a0887e1af2b8775f4] GCPNAME: ENCRYPTED[2f9738ef295a706f66a13891b40e8eaa92a89e0e87faf8bed66c41eca72bf76cfd190a6f2d0e8444c631fdf15ed32ef6] GCPPROJECT: libpod-218412 @@ -267,7 +267,7 @@ test/version/version: version/version.go .PHONY: codespell codespell: - codespell -S bin,vendor,.git,go.sum,.cirrus.yml,"RELEASE_NOTES.md,*.xz,*.gz,*.ps1,*.tar,swagger.yaml,*.tgz,bin2img,*ico,*.png,*.1,*.5,copyimg,*.orig,apidoc.go" -L pullrequest,uint,iff,od,seeked,splitted,marge,erro,hist,ether -w + codespell -S bin,vendor,.git,go.sum,.cirrus.yml,"RELEASE_NOTES.md,*.xz,*.gz,*.ps1,*.tar,swagger.yaml,*.tgz,bin2img,*ico,*.png,*.1,*.5,copyimg,*.orig,apidoc.go" -L clos,ans,pullrequest,uint,iff,od,seeked,splitted,marge,erro,hist,ether -w .PHONY: validate validate: lint .gitvalidation validate.completions man-page-check swagger-check tests-included tests-expect-exit pr-removes-fixed-skips diff --git a/cmd/podman/images/trust_set.go b/cmd/podman/images/trust_set.go index 832e9f724..e7339f0b1 100644 --- a/cmd/podman/images/trust_set.go +++ b/cmd/podman/images/trust_set.go @@ -53,7 +53,7 @@ File(s) must exist before using this command`) } func setTrust(cmd *cobra.Command, args []string) error { - validTrustTypes := []string{"accept", "insecureAcceptAnything", "reject", "signedBy"} + validTrustTypes := []string{"accept", "insecureAcceptAnything", "reject", "signedBy", "sigstoreSigned"} valid, err := isValidImageURI(args[0]) if err != nil || !valid { @@ -61,7 +61,7 @@ func setTrust(cmd *cobra.Command, args []string) error { } if !util.StringInSlice(setOptions.Type, validTrustTypes) { - return fmt.Errorf("invalid choice: %s (choose from 'accept', 'reject', 'signedBy')", setOptions.Type) + return fmt.Errorf("invalid choice: %s (choose from 'accept', 'reject', 'signedBy', 'sigstoreSigned')", setOptions.Type) } return registry.ImageEngine().SetTrust(registry.Context(), args, setOptions) } diff --git a/cmd/podman/syslog_linux.go b/cmd/podman/syslog_common.go index ac7bbfe0f..e035e6365 100644 --- a/cmd/podman/syslog_linux.go +++ b/cmd/podman/syslog_common.go @@ -1,3 +1,6 @@ +//go:build linux || freebsd +// +build linux freebsd + package main import ( diff --git a/cmd/podman/syslog_unsupported.go b/cmd/podman/syslog_unsupported.go index 42a7851ab..365e5b2b4 100644 --- a/cmd/podman/syslog_unsupported.go +++ b/cmd/podman/syslog_unsupported.go @@ -1,5 +1,5 @@ -//go:build !linux -// +build !linux +//go:build !linux && !freebsd +// +build !linux,!freebsd package main diff --git a/cmd/podman/system/service_abi.go b/cmd/podman/system/service_abi.go index 8d0240a8d..68ac8902b 100644 --- a/cmd/podman/system/service_abi.go +++ b/cmd/podman/system/service_abi.go @@ -105,7 +105,9 @@ func restService(flags *pflag.FlagSet, cfg *entities.PodmanConfig, opts entities } if err := utils.MaybeMoveToSubCgroup(); err != nil { - return err + // it is a best effort operation, so just print the + // error for debugging purposes. + logrus.Debugf("Could not move to subcgroup: %v", err) } servicereaper.Start() diff --git a/contrib/pkginstaller/Makefile b/contrib/pkginstaller/Makefile index c84a08482..b7636fe14 100644 --- a/contrib/pkginstaller/Makefile +++ b/contrib/pkginstaller/Makefile @@ -1,7 +1,6 @@ SHELL := bash ARCH ?= aarch64 -PODMAN_VERSION ?= 4.1.0 GVPROXY_VERSION ?= 0.4.0 QEMU_VERSION ?= 7.0.0-2 GVPROXY_RELEASE_URL ?= https://github.com/containers/gvisor-tap-vsock/releases/download/v$(GVPROXY_VERSION)/gvproxy-darwin @@ -13,6 +12,9 @@ PKG_NAME := podman-installer-macos-$(ARCH).pkg default: pkginstaller +podman_version: + make -C ../../ test/version/version + $(TMP_DOWNLOAD)/gvproxy: mkdir -p $(TMP_DOWNLOAD) cd $(TMP_DOWNLOAD) && curl -sLo gvproxy $(GVPROXY_RELEASE_URL) @@ -21,7 +23,7 @@ $(TMP_DOWNLOAD)/podman-machine-qemu-$(ARCH)-$(QEMU_VERSION).tar.xz: mkdir -p $(TMP_DOWNLOAD) cd $(TMP_DOWNLOAD) && curl -sLO $(QEMU_RELEASE_URL) -packagedir: package_root Distribution welcome.html +packagedir: podman_version package_root Distribution welcome.html mkdir -p $(PACKAGE_DIR) cp -r Resources $(PACKAGE_DIR)/ cp welcome.html $(PACKAGE_DIR)/Resources/ @@ -30,7 +32,7 @@ packagedir: package_root Distribution welcome.html cp -r $(PACKAGE_ROOT) $(PACKAGE_DIR)/ cp package.sh $(PACKAGE_DIR)/ cd $(PACKAGE_DIR) && pkgbuild --analyze --root ./root component.plist - echo -n $(PODMAN_VERSION) > $(PACKAGE_DIR)/VERSION + ../../test/version/version > $(PACKAGE_DIR)/VERSION echo -n $(ARCH) > $(PACKAGE_DIR)/ARCH cp ../../LICENSE $(PACKAGE_DIR)/Resources/LICENSE.txt cp hvf.entitlements $(PACKAGE_DIR)/ @@ -41,8 +43,8 @@ package_root: clean-pkgroot $(TMP_DOWNLOAD)/podman-machine-qemu-$(ARCH)-$(QEMU_V cp $(TMP_DOWNLOAD)/gvproxy $(PACKAGE_ROOT)/podman/bin/ chmod a+x $(PACKAGE_ROOT)/podman/bin/* -%: %.in - @sed -e 's/__VERSION__/'$(PODMAN_VERSION)'/g' $< >$@ +%: %.in podman_version + @sed -e 's/__VERSION__/'$(shell ../../test/version/version)'/g' $< >$@ pkginstaller: packagedir cd $(PACKAGE_DIR) && ./package.sh .. @@ -55,7 +57,7 @@ notarize: _notarize .PHONY: clean clean-pkgroot clean: - rm -rf $(TMP_DOWNLOAD) $(PACKAGE_ROOT) $(PACKAGE_DIR) Distribution welcome.html + rm -rf $(TMP_DOWNLOAD) $(PACKAGE_ROOT) $(PACKAGE_DIR) Distribution welcome.html ../../test/version/version clean-pkgroot: rm -rf $(PACKAGE_ROOT) $(PACKAGE_DIR) Distribution welcome.html diff --git a/docs/source/markdown/options/systemd.md b/docs/source/markdown/options/systemd.md new file mode 100644 index 000000000..a341edbc2 --- /dev/null +++ b/docs/source/markdown/options/systemd.md @@ -0,0 +1,29 @@ +#### **--systemd**=*true* | *false* | *always* + +Run container in systemd mode. The default is **true**. + +The value *always* enforces the systemd mode is enforced without +looking at the executable name. Otherwise, if set to true and the +command you are running inside the container is **systemd**, **/usr/sbin/init**, +**/sbin/init** or **/usr/local/sbin/init**. + +Running the container in systemd mode causes the following changes: + +* Podman mounts tmpfs file systems on the following directories + * _/run_ + * _/run/lock_ + * _/tmp_ + * _/sys/fs/cgroup/systemd_ + * _/var/lib/journal_ +* Podman sets the default stop signal to **SIGRTMIN+3**. +* Podman sets **container_uuid** environment variable in the container to the +first 32 characters of the container id. + +This allows systemd to run in a confined container without any modifications. + +Note that on **SELinux** systems, systemd attempts to write to the cgroup +file system. Containers writing to the cgroup file system are denied by default. +The **container_manage_cgroup** boolean must be enabled for this to be allowed on an SELinux separated system. +``` +setsebool -P container_manage_cgroup true +``` diff --git a/docs/source/markdown/podman-create.1.md.in b/docs/source/markdown/podman-create.1.md.in index 4dbc75551..1ff7429c7 100644 --- a/docs/source/markdown/podman-create.1.md.in +++ b/docs/source/markdown/podman-create.1.md.in @@ -632,34 +632,7 @@ Network Namespace - current sysctls allowed: Note: if you use the --network=host option these sysctls will not be allowed. -#### **--systemd**=*true* | *false* | *always* - -Run container in systemd mode. The default is *true*. - -The value *always* enforces the systemd mode is enforced without -looking at the executable name. Otherwise, if set to true and the -command you are running inside the container is **systemd**, **/usr/sbin/init**, -**/sbin/init** or **/usr/local/sbin/init**. - -Running the container in systemd mode causes the following changes: - -* Podman mounts tmpfs file systems on the following directories - * _/run_ - * _/run/lock_ - * _/tmp_ - * _/sys/fs/cgroup/systemd_ - * _/var/lib/journal_ -* Podman sets the default stop signal to **SIGRTMIN+3**. -* Podman sets **container_uuid** environment variable in the container to the -first 32 characters of the container id. - -This allows systemd to run in a confined container without any modifications. - -Note: On `SELinux` systems, systemd attempts to write to the cgroup -file system. Containers writing to the cgroup file system are denied by default. -The `container_manage_cgroup` boolean must be enabled for this to be allowed on an SELinux separated system. - -`setsebool -P container_manage_cgroup true` +@@option systemd @@option timeout diff --git a/docs/source/markdown/podman-generate-systemd.1.md b/docs/source/markdown/podman-generate-systemd.1.md index fc2ce171e..88dff2a45 100644 --- a/docs/source/markdown/podman-generate-systemd.1.md +++ b/docs/source/markdown/podman-generate-systemd.1.md @@ -26,7 +26,7 @@ therefore the overridden default value._ A Kubernetes YAML can be executed in systemd via the `podman-kube@.service` systemd template. The template's argument is the path to the YAML file. Given a `workload.yaml` file in the home directory, it can be executed as follows: ``` -$ escaped=$(systemd-escape ~/sysadmin.yaml) +$ escaped=$(systemd-escape ~/workload.yaml) $ systemctl --user start podman-kube@$escaped.service $ systemctl --user is-active podman-kube@$escaped.service active diff --git a/docs/source/markdown/podman-image-trust.1.md b/docs/source/markdown/podman-image-trust.1.md index 4e80bdcf5..2a7da82cc 100644 --- a/docs/source/markdown/podman-image-trust.1.md +++ b/docs/source/markdown/podman-image-trust.1.md @@ -32,7 +32,8 @@ Trust **type** provides a way to: Allowlist ("accept") or Denylist ("reject") registries or -Require signature (“signedBy”). +Require a simple signing signature (“signedBy”), +Require a sigstore signature ("sigstoreSigned"). Trust may be updated using the command **podman image trust set** for an existing trust scope. @@ -45,12 +46,14 @@ Trust may be updated using the command **podman image trust set** for an existin #### **--pubkeysfile**, **-f**=*KEY1* A path to an exported public key on the local system. Key paths will be referenced in policy.json. Any path to a file may be used but locating the file in **/etc/pki/containers** is recommended. Options may be used multiple times to - require an image be signed by multiple keys. The **--pubkeysfile** option is required for the **signedBy** type. + require an image be signed by multiple keys. The **--pubkeysfile** option is required for the **signedBy** and **sigstoreSigned** types. #### **--type**, **-t**=*value* The trust type for this policy entry. Accepted values: - **signedBy** (default): Require signatures with corresponding list of + **signedBy** (default): Require simple signing signatures with corresponding list of + public keys + **sigstoreSigned**: Require sigstore signatures with corresponding list of public keys **accept**: do not require any signatures for this registry scope diff --git a/docs/source/markdown/podman-pod-restart.1.md b/docs/source/markdown/podman-pod-restart.1.md index 677eca3a3..51f13dbf8 100644 --- a/docs/source/markdown/podman-pod-restart.1.md +++ b/docs/source/markdown/podman-pod-restart.1.md @@ -24,17 +24,27 @@ Instead of providing the pod name or ID, restart the last created pod. (This opt ## EXAMPLE +Restart pod with a given name ``` podman pod restart mywebserverpod cc8f0bea67b1a1a11aec1ecd38102a1be4b145577f21fc843c7c83b77fc28907 +``` +Restart multiple pods with given IDs +``` podman pod restart 490eb 3557fb 490eb241aaf704d4dd2629904410fe4aa31965d9310a735f8755267f4ded1de5 3557fbea6ad61569de0506fe037479bd9896603c31d3069a6677f23833916fab +``` +Restart the last created pod +``` podman pod restart --latest 3557fbea6ad61569de0506fe037479bd9896603c31d3069a6677f23833916fab +``` +Restart all pods +``` podman pod restart --all 19456b4cd557eaf9629825113a552681a6013f8c8cad258e36ab825ef536e818 3557fbea6ad61569de0506fe037479bd9896603c31d3069a6677f23833916fab @@ -42,7 +52,6 @@ podman pod restart --all 70c358daecf71ef9be8f62404f926080ca0133277ef7ce4f6aa2d5af6bb2d3e9 cc8f0bea67b1a1a11aec1ecd38102a1be4b145577f21fc843c7c83b77fc28907 ``` - ## SEE ALSO **[podman(1)](podman.1.md)**, **[podman-pod(1)](podman-pod.1.md)**, **[podman-restart(1)](podman-restart.1.md)** diff --git a/docs/source/markdown/podman-pod-stop.1.md.in b/docs/source/markdown/podman-pod-stop.1.md.in index 3655c3938..abcc69e9e 100644 --- a/docs/source/markdown/podman-pod-stop.1.md.in +++ b/docs/source/markdown/podman-pod-stop.1.md.in @@ -29,20 +29,20 @@ Seconds to wait before forcibly stopping the containers in the pod. ## EXAMPLE -Stop a pod called *mywebserverpod* +Stop pod with a given name ``` $ podman pod stop mywebserverpod cc8f0bea67b1a1a11aec1ecd38102a1be4b145577f21fc843c7c83b77fc28907 ``` -Stop two pods by their short IDs. +Stop multiple pods with given IDs. ``` $ podman pod stop 490eb 3557fb 490eb241aaf704d4dd2629904410fe4aa31965d9310a735f8755267f4ded1de5 3557fbea6ad61569de0506fe037479bd9896603c31d3069a6677f23833916fab ``` -Stop the most recent pod +Stop the last created pod ``` $ podman pod stop --latest 3557fbea6ad61569de0506fe037479bd9896603c31d3069a6677f23833916fab @@ -65,7 +65,7 @@ $ podman pod stop --pod-id-file file1 --pod-id-file file2 cc8f0bea67b1a1a11aec1ecd38102a1be4b145577f21fc843c7c83b77fc28907 ``` -Stop all pods with a timeout of 1 second. +Stop all pods with a timeout of 1 second ``` $ podman pod stop -a -t 1 3557fbea6ad61569de0506fe037479bd9896603c31d3069a6677f23833916fab diff --git a/docs/source/markdown/podman-rename.1.md b/docs/source/markdown/podman-rename.1.md index 4017db505..0a807e6de 100644 --- a/docs/source/markdown/podman-rename.1.md +++ b/docs/source/markdown/podman-rename.1.md @@ -19,18 +19,18 @@ At present, only containers are supported; pods and volumes cannot be renamed. ## EXAMPLES +Rename container with a given name ``` -# Rename a container by name $ podman rename oldContainer aNewName ``` +Rename container with a given ID ``` -# Rename a container by ID $ podman rename 717716c00a6b testcontainer ``` +Create an alias for container with a given ID ``` -# Use the container rename alias $ podman container rename 6e7514b47180 databaseCtr ``` diff --git a/docs/source/markdown/podman-rm.1.md.in b/docs/source/markdown/podman-rm.1.md.in index c0fa94d82..9eb44dcc1 100644 --- a/docs/source/markdown/podman-rm.1.md.in +++ b/docs/source/markdown/podman-rm.1.md.in @@ -73,37 +73,37 @@ Remove anonymous volumes associated with the container. This does not include na created with **podman volume create**, or the **--volume** option of **podman run** and **podman create**. ## EXAMPLE -Remove a container by its name *mywebserver* +Remove container with a given name ``` $ podman rm mywebserver ``` -Remove a *mywebserver* container and all of the containers that depend on it +Remove container with a given name and all of the containers that depend on it ``` $ podman rm --depend mywebserver ``` -Remove several containers by name and container id. +Remove multiple containers with given names or IDs ``` $ podman rm mywebserver myflaskserver 860a4b23 ``` -Remove several containers reading their IDs from files. +Remove multiple containers with IDs read from files ``` $ podman rm --cidfile ./cidfile-1 --cidfile /home/user/cidfile-2 ``` -Forcibly remove a container by container ID. +Forcibly remove container with a given ID ``` $ podman rm -f 860a4b23 ``` -Remove all containers regardless of its run state. +Remove all containers regardless of the run state ``` $ podman rm -f -a ``` -Forcibly remove the latest container created. +Forcibly remove the last created container ``` $ podman rm -f --latest ``` diff --git a/docs/source/markdown/podman-run.1.md.in b/docs/source/markdown/podman-run.1.md.in index c7985d7e1..f172ffc9e 100644 --- a/docs/source/markdown/podman-run.1.md.in +++ b/docs/source/markdown/podman-run.1.md.in @@ -682,35 +682,7 @@ For the network namespace, the following sysctls are allowed: Note: if you use the **--network=host** option, these sysctls will not be allowed. -#### **--systemd**=*true* | *false* | *always* - -Run container in systemd mode. The default is **true**. - -The value *always* enforces the systemd mode is enforced without -looking at the executable name. Otherwise, if set to true and the -command you are running inside the container is **systemd**, **/usr/sbin/init**, -**/sbin/init** or **/usr/local/sbin/init**. - -Running the container in systemd mode causes the following changes: - -* Podman mounts tmpfs file systems on the following directories - * _/run_ - * _/run/lock_ - * _/tmp_ - * _/sys/fs/cgroup/systemd_ - * _/var/lib/journal_ -* Podman sets the default stop signal to **SIGRTMIN+3**. -* Podman sets **container_uuid** environment variable in the container to the -first 32 characters of the container id. - -This allows systemd to run in a confined container without any modifications. - -Note that on **SELinux** systems, systemd attempts to write to the cgroup -file system. Containers writing to the cgroup file system are denied by default. -The **container_manage_cgroup** boolean must be enabled for this to be allowed on an SELinux separated system. -``` -setsebool -P container_manage_cgroup true -``` +@@option systemd @@option timeout @@ -58,7 +58,7 @@ require ( github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/ulikunitz/xz v0.5.10 - github.com/vbauerster/mpb/v7 v7.4.2 + github.com/vbauerster/mpb/v7 v7.5.2 github.com/vishvananda/netlink v1.1.1-0.20220115184804-dd687eb2f2d4 go.etcd.io/bbolt v1.3.6 golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f @@ -1593,8 +1593,9 @@ github.com/valyala/quicktemplate v1.7.0/go.mod h1:sqKJnoaOF88V07vkO+9FL8fb9uZg/V github.com/valyala/tcplisten v1.0.0/go.mod h1:T0xQ8SeCZGxckz9qRXTfG43PvQ/mcWh7FwZEA7Ioqkc= github.com/vbatts/tar-split v0.11.2 h1:Via6XqJr0hceW4wff3QRzD5gAk/tatMw/4ZA7cTlIME= github.com/vbatts/tar-split v0.11.2/go.mod h1:vV3ZuO2yWSVsz+pfFzDG/upWH1JhjOiEaWq6kXyQ3VI= -github.com/vbauerster/mpb/v7 v7.4.2 h1:n917F4d8EWdUKc9c81wFkksyG6P6Mg7IETfKCE1Xqng= github.com/vbauerster/mpb/v7 v7.4.2/go.mod h1:UmOiIUI8aPqWXIps0ciik3RKMdzx7+ooQpq+fBcXwBA= +github.com/vbauerster/mpb/v7 v7.5.2 h1:Ph3JvpBcoIwzIG1QwbUq97KQifrTRbKcMXN9rN5BYAs= +github.com/vbauerster/mpb/v7 v7.5.2/go.mod h1:UmOiIUI8aPqWXIps0ciik3RKMdzx7+ooQpq+fBcXwBA= github.com/viki-org/dnscache v0.0.0-20130720023526-c70c1f23c5d8/go.mod h1:dniwbG03GafCjFohMDmz6Zc6oCuiqgH6tGNyXTkHzXE= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= diff --git a/libpod/boltdb_state.go b/libpod/boltdb_state.go index 81f11410b..e5a7e20fc 100644 --- a/libpod/boltdb_state.go +++ b/libpod/boltdb_state.go @@ -1278,7 +1278,7 @@ func (s *BoltState) NetworkConnect(ctr *Container, network string, opts types.Pe } netConnected := ctrNetworksBkt.Get([]byte(network)) if netConnected != nil { - return fmt.Errorf("container %s is already connected to network %q: %w", ctr.ID(), network, define.ErrNetworkExists) + return fmt.Errorf("container %s is already connected to network %q: %w", ctr.ID(), network, define.ErrNetworkConnected) } // Add the network diff --git a/libpod/container_internal_unsupported.go b/libpod/container_internal_unsupported.go index de92ff260..074aeee47 100644 --- a/libpod/container_internal_unsupported.go +++ b/libpod/container_internal_unsupported.go @@ -69,21 +69,21 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti // getHostsEntries returns the container ip host entries for the correct netmode func (c *Container) getHostsEntries() (etchosts.HostEntries, error) { - return nil, errors.New("unspported (*Container) getHostsEntries") + return nil, errors.New("unsupported (*Container) getHostsEntries") } // Fix ownership and permissions of the specified volume if necessary. func (c *Container) fixVolumePermissions(v *ContainerNamedVolume) error { - return errors.New("unspported (*Container) fixVolumePermissions") + return errors.New("unsupported (*Container) fixVolumePermissions") } func (c *Container) expectPodCgroup() (bool, error) { - return false, errors.New("unspported (*Container) expectPodCgroup") + return false, errors.New("unsupported (*Container) expectPodCgroup") } // Get cgroup path in a format suitable for the OCI spec func (c *Container) getOCICgroupPath() (string, error) { - return "", errors.New("unspported (*Container) getOCICgroupPath") + return "", errors.New("unsupported (*Container) getOCICgroupPath") } func getLocalhostHostEntry(c *Container) etchosts.HostEntries { diff --git a/libpod/define/errors.go b/libpod/define/errors.go index fd27e89de..be471c27e 100644 --- a/libpod/define/errors.go +++ b/libpod/define/errors.go @@ -179,6 +179,9 @@ var ( // ErrNetworkInUse indicates the requested operation failed because the network was in use ErrNetworkInUse = errors.New("network is being used") + // ErrNetworkConnected indicates that the required operation failed because the container is already a network endpoint + ErrNetworkConnected = errors.New("network is already connected") + // ErrStoreNotInitialized indicates that the container storage was never // initialized. ErrStoreNotInitialized = errors.New("the container storage was never initialized") diff --git a/libpod/define/exec_codes.go b/libpod/define/exec_codes.go index 3f2da4910..a84730e72 100644 --- a/libpod/define/exec_codes.go +++ b/libpod/define/exec_codes.go @@ -11,8 +11,8 @@ const ( // ExecErrorCodeGeneric is the default error code to return from an exec session if libpod failed // prior to calling the runtime ExecErrorCodeGeneric = 125 - // ExecErrorCodeCannotInvoke is the error code to return when the runtime fails to invoke a command - // an example of this can be found by trying to execute a directory: + // ExecErrorCodeCannotInvoke is the error code to return when the runtime fails to invoke a command. + // An example of this can be found by trying to execute a directory: // `podman exec -l /etc` ExecErrorCodeCannotInvoke = 126 // ExecErrorCodeNotFound is the error code to return when a command cannot be found diff --git a/libpod/kube.go b/libpod/kube.go index 8c09a6bb5..a0fb52973 100644 --- a/libpod/kube.go +++ b/libpod/kube.go @@ -267,6 +267,8 @@ func GenerateKubeServiceFromV1Pod(pod *v1.Pod, servicePorts []v1.ServicePort) (Y } service.Spec = serviceSpec service.ObjectMeta = pod.ObjectMeta + // Reset the annotations for the service as the pod annotations are not needed for the service + service.ObjectMeta.Annotations = nil tm := v12.TypeMeta{ Kind: "Service", APIVersion: pod.TypeMeta.APIVersion, diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go index c05796768..c10c3c0b2 100644 --- a/libpod/networking_linux.go +++ b/libpod/networking_linux.go @@ -1357,6 +1357,11 @@ func (c *Container) NetworkConnect(nameOrID, netName string, netOpts types.PerNe } if err := c.runtime.state.NetworkConnect(c, netName, netOpts); err != nil { + // Docker compat: treat requests to attach already attached networks as a no-op, ignoring opts + if errors.Is(err, define.ErrNetworkConnected) && c.ensureState(define.ContainerStateConfigured) { + return nil + } + return err } c.newNetworkEvent(events.NetworkConnect, netName) diff --git a/libpod/oci_conmon_common.go b/libpod/oci_conmon_common.go index c3725cdb4..b96f92d3a 100644 --- a/libpod/oci_conmon_common.go +++ b/libpod/oci_conmon_common.go @@ -277,15 +277,6 @@ func (r *ConmonOCIRuntime) UpdateContainerStatus(ctr *Container) error { ctr.ID(), state.Status, define.ErrInternal) } - // Only grab exit status if we were not already stopped - // If we were, it should already be in the database - if ctr.state.State == define.ContainerStateStopped && oldState != define.ContainerStateStopped { - if _, err := ctr.Wait(context.Background()); err != nil { - logrus.Errorf("Waiting for container %s to exit: %v", ctr.ID(), err) - } - return nil - } - // Handle ContainerStateStopping - keep it unless the container // transitioned to no longer running. if oldState == define.ContainerStateStopping && (ctr.state.State == define.ContainerStatePaused || ctr.state.State == define.ContainerStateRunning) { diff --git a/pkg/api/handlers/compat/events.go b/pkg/api/handlers/compat/events.go index 18fb35966..105404a0d 100644 --- a/pkg/api/handlers/compat/events.go +++ b/pkg/api/handlers/compat/events.go @@ -89,6 +89,12 @@ func GetEvents(w http.ResponseWriter, r *http.Request) { } e := entities.ConvertToEntitiesEvent(*evt) + // Some events differ between Libpod and Docker endpoints. + // Handle these differences for Docker-compat. + if !utils.IsLibpodRequest(r) && e.Type == "image" && e.Status == "remove" { + e.Status = "delete" + e.Action = "delete" + } if !utils.IsLibpodRequest(r) && e.Status == "died" { e.Status = "die" e.Action = "die" diff --git a/pkg/api/handlers/compat/images_build.go b/pkg/api/handlers/compat/images_build.go index 020991cc7..7ba1029a7 100644 --- a/pkg/api/handlers/compat/images_build.go +++ b/pkg/api/handlers/compat/images_build.go @@ -101,6 +101,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { ForceRm bool `schema:"forcerm"` From string `schema:"from"` HTTPProxy bool `schema:"httpproxy"` + IDMappingOptions string `schema:"idmappingoptions"` IdentityLabel bool `schema:"identitylabel"` Ignore bool `schema:"ignore"` Isolation string `schema:"isolation"` @@ -389,6 +390,14 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { } } + var idMappingOptions buildahDefine.IDMappingOptions + if _, found := r.URL.Query()["idmappingoptions"]; found { + if err := json.Unmarshal([]byte(query.IDMappingOptions), &idMappingOptions); err != nil { + utils.BadRequest(w, "idmappingoptions", query.IDMappingOptions, err) + return + } + } + var cacheFrom reference.Named if _, found := r.URL.Query()["cachefrom"]; found { cacheFrom, err = parse.RepoNameToNamedReference(query.CacheFrom) @@ -644,6 +653,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { Excludes: excludes, ForceRmIntermediateCtrs: query.ForceRm, From: fromImage, + IDMappingOptions: &idMappingOptions, IgnoreUnrecognizedInstructions: query.Ignore, Isolation: isolation, Jobs: &jobs, diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go index 2615bc516..8348ac54b 100644 --- a/pkg/bindings/images/build.go +++ b/pkg/bindings/images/build.go @@ -88,6 +88,13 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO } params.Set("additionalbuildcontexts", string(additionalBuildContextMap)) } + if options.IDMappingOptions != nil { + idmappingsOptions, err := jsoniter.Marshal(options.IDMappingOptions) + if err != nil { + return nil, err + } + params.Set("idmappingoptions", string(idmappingsOptions)) + } if buildArgs := options.Args; len(buildArgs) > 0 { bArgs, err := jsoniter.MarshalToString(buildArgs) if err != nil { diff --git a/pkg/domain/infra/abi/terminal/sigproxy_linux.go b/pkg/domain/infra/abi/terminal/sigproxy_commn.go index 16d345f06..3a0132ef3 100644 --- a/pkg/domain/infra/abi/terminal/sigproxy_linux.go +++ b/pkg/domain/infra/abi/terminal/sigproxy_commn.go @@ -1,3 +1,6 @@ +//go:build linux || freebsd +// +build linux freebsd + package terminal import ( diff --git a/pkg/domain/infra/abi/terminal/terminal_linux.go b/pkg/domain/infra/abi/terminal/terminal_common.go index 222590871..afae2c085 100644 --- a/pkg/domain/infra/abi/terminal/terminal_linux.go +++ b/pkg/domain/infra/abi/terminal/terminal_common.go @@ -1,3 +1,6 @@ +//go:build linux || freebsd +// +build linux freebsd + package terminal import ( diff --git a/pkg/domain/infra/abi/terminal/terminal_unsupported.go b/pkg/domain/infra/abi/terminal/terminal_unsupported.go index 8fe325736..21ed6c8d4 100644 --- a/pkg/domain/infra/abi/terminal/terminal_unsupported.go +++ b/pkg/domain/infra/abi/terminal/terminal_unsupported.go @@ -1,5 +1,5 @@ -//go:build !linux -// +build !linux +//go:build !linux && !freebsd +// +build !linux,!freebsd package terminal diff --git a/pkg/domain/infra/abi/trust.go b/pkg/domain/infra/abi/trust.go index 0e3d8fad9..c58ddff06 100644 --- a/pkg/domain/infra/abi/trust.go +++ b/pkg/domain/infra/abi/trust.go @@ -2,16 +2,11 @@ package abi import ( "context" - "encoding/json" - "errors" "fmt" "io/ioutil" - "os" - "strings" "github.com/containers/podman/v4/pkg/domain/entities" "github.com/containers/podman/v4/pkg/trust" - "github.com/sirupsen/logrus" ) func (ir *ImageEngine) ShowTrust(ctx context.Context, args []string, options entities.ShowTrustOptions) (*entities.ShowTrustReport, error) { @@ -34,11 +29,7 @@ func (ir *ImageEngine) ShowTrust(ctx context.Context, args []string, options ent if len(options.RegistryPath) > 0 { report.SystemRegistriesDirPath = options.RegistryPath } - policyContentStruct, err := trust.GetPolicy(policyPath) - if err != nil { - return nil, fmt.Errorf("could not read trust policies: %w", err) - } - report.Policies, err = getPolicyShowOutput(policyContentStruct, report.SystemRegistriesDirPath) + report.Policies, err = trust.PolicyDescription(policyPath, report.SystemRegistriesDirPath) if err != nil { return nil, fmt.Errorf("could not show trust policies: %w", err) } @@ -46,133 +37,19 @@ func (ir *ImageEngine) ShowTrust(ctx context.Context, args []string, options ent } func (ir *ImageEngine) SetTrust(ctx context.Context, args []string, options entities.SetTrustOptions) error { - var ( - policyContentStruct trust.PolicyContent - newReposContent []trust.RepoContent - ) - trustType := options.Type - if trustType == "accept" { - trustType = "insecureAcceptAnything" - } - - pubkeysfile := options.PubKeysFile - if len(pubkeysfile) == 0 && trustType == "signedBy" { - return errors.New("at least one public key must be defined for type 'signedBy'") + if len(args) != 1 { + return fmt.Errorf("SetTrust called with unexpected %d args", len(args)) } + scope := args[0] policyPath := trust.DefaultPolicyPath(ir.Libpod.SystemContext()) if len(options.PolicyPath) > 0 { policyPath = options.PolicyPath } - _, err := os.Stat(policyPath) - if !os.IsNotExist(err) { - policyContent, err := ioutil.ReadFile(policyPath) - if err != nil { - return err - } - if err := json.Unmarshal(policyContent, &policyContentStruct); err != nil { - return errors.New("could not read trust policies") - } - } - if len(pubkeysfile) != 0 { - for _, filepath := range pubkeysfile { - newReposContent = append(newReposContent, trust.RepoContent{Type: trustType, KeyType: "GPGKeys", KeyPath: filepath}) - } - } else { - newReposContent = append(newReposContent, trust.RepoContent{Type: trustType}) - } - if args[0] == "default" { - policyContentStruct.Default = newReposContent - } else { - if len(policyContentStruct.Default) == 0 { - return errors.New("default trust policy must be set") - } - registryExists := false - for transport, transportval := range policyContentStruct.Transports { - _, registryExists = transportval[args[0]] - if registryExists { - policyContentStruct.Transports[transport][args[0]] = newReposContent - break - } - } - if !registryExists { - if policyContentStruct.Transports == nil { - policyContentStruct.Transports = make(map[string]trust.RepoMap) - } - if policyContentStruct.Transports["docker"] == nil { - policyContentStruct.Transports["docker"] = make(map[string][]trust.RepoContent) - } - policyContentStruct.Transports["docker"][args[0]] = append(policyContentStruct.Transports["docker"][args[0]], newReposContent...) - } - } - - data, err := json.MarshalIndent(policyContentStruct, "", " ") - if err != nil { - return fmt.Errorf("error setting trust policy: %w", err) - } - return ioutil.WriteFile(policyPath, data, 0644) -} - -func getPolicyShowOutput(policyContentStruct trust.PolicyContent, systemRegistriesDirPath string) ([]*trust.Policy, error) { - var output []*trust.Policy - - registryConfigs, err := trust.LoadAndMergeConfig(systemRegistriesDirPath) - if err != nil { - return nil, err - } - - if len(policyContentStruct.Default) > 0 { - defaultPolicyStruct := trust.Policy{ - Transport: "all", - Name: "* (default)", - RepoName: "default", - Type: trustTypeDescription(policyContentStruct.Default[0].Type), - } - output = append(output, &defaultPolicyStruct) - } - for transport, transval := range policyContentStruct.Transports { - if transport == "docker" { - transport = "repository" - } - for repo, repoval := range transval { - tempTrustShowOutput := trust.Policy{ - Name: repo, - RepoName: repo, - Transport: transport, - Type: trustTypeDescription(repoval[0].Type), - } - // TODO - keyarr is not used and I don't know its intent; commenting out for now for someone to fix later - // keyarr := []string{} - uids := []string{} - for _, repoele := range repoval { - if len(repoele.KeyPath) > 0 { - // keyarr = append(keyarr, repoele.KeyPath) - uids = append(uids, trust.GetGPGIdFromKeyPath(repoele.KeyPath)...) - } - if len(repoele.KeyData) > 0 { - // keyarr = append(keyarr, string(repoele.KeyData)) - uids = append(uids, trust.GetGPGIdFromKeyData(repoele.KeyData)...) - } - } - tempTrustShowOutput.GPGId = strings.Join(uids, ", ") - - registryNamespace := trust.HaveMatchRegistry(repo, registryConfigs) - if registryNamespace != nil { - tempTrustShowOutput.SignatureStore = registryNamespace.SigStore - } - output = append(output, &tempTrustShowOutput) - } - } - return output, nil -} - -var typeDescription = map[string]string{"insecureAcceptAnything": "accept", "signedBy": "signed", "reject": "reject"} - -func trustTypeDescription(trustType string) string { - trustDescription, exist := typeDescription[trustType] - if !exist { - logrus.Warnf("Invalid trust type %s", trustType) - } - return trustDescription + return trust.AddPolicyEntries(policyPath, trust.AddPolicyEntriesInput{ + Scope: scope, + Type: options.Type, + PubKeyFiles: options.PubKeysFile, + }) } diff --git a/pkg/machine/config.go b/pkg/machine/config.go index 5162006db..54aa9e1b7 100644 --- a/pkg/machine/config.go +++ b/pkg/machine/config.go @@ -175,7 +175,7 @@ func (rc RemoteConnectionType) MakeSSHURL(host, path, port, userName string) url return uri } -// GetCacheDir returns the dir where VM images are downladed into when pulled +// GetCacheDir returns the dir where VM images are downloaded into when pulled func GetCacheDir(vmType string) (string, error) { dataDir, err := GetDataDir(vmType) if err != nil { diff --git a/pkg/systemd/notifyproxy/notifyproxy_test.go b/pkg/systemd/notifyproxy/notifyproxy_test.go index edad95659..ce63fc9cd 100644 --- a/pkg/systemd/notifyproxy/notifyproxy_test.go +++ b/pkg/systemd/notifyproxy/notifyproxy_test.go @@ -37,7 +37,7 @@ func TestWaitAndClose(t *testing.T) { time.Sleep(250 * time.Millisecond) select { case err := <-ch: - t.Fatalf("Should stil be waiting but received %v", err) + t.Fatalf("Should still be waiting but received %v", err) default: } diff --git a/pkg/trust/config.go b/pkg/trust/config.go deleted file mode 100644 index 6186d4cbd..000000000 --- a/pkg/trust/config.go +++ /dev/null @@ -1,12 +0,0 @@ -package trust - -// Policy describes a basic trust policy configuration -type Policy struct { - Transport string `json:"transport"` - Name string `json:"name,omitempty"` - RepoName string `json:"repo_name,omitempty"` - Keys []string `json:"keys,omitempty"` - SignatureStore string `json:"sigstore,omitempty"` - Type string `json:"type"` - GPGId string `json:"gpg_id,omitempty"` -} diff --git a/pkg/trust/policy.go b/pkg/trust/policy.go new file mode 100644 index 000000000..326fe17af --- /dev/null +++ b/pkg/trust/policy.go @@ -0,0 +1,248 @@ +package trust + +import ( + "bufio" + "bytes" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "strings" + + "github.com/containers/image/v5/types" + "github.com/sirupsen/logrus" +) + +// policyContent is the overall structure of a policy.json file (= c/image/v5/signature.Policy) +type policyContent struct { + Default []repoContent `json:"default"` + Transports transportsContent `json:"transports,omitempty"` +} + +// transportsContent contains policies for individual transports (= c/image/v5/signature.Policy.Transports) +type transportsContent map[string]repoMap + +// repoMap maps a scope name to requirements that apply to that scope (= c/image/v5/signature.PolicyTransportScopes) +type repoMap map[string][]repoContent + +// repoContent is a single policy requirement (one of possibly several for a scope), representing all of the individual alternatives in a single merged struct +// (= c/image/v5/signature.{PolicyRequirement,pr*}) +type repoContent struct { + Type string `json:"type"` + KeyType string `json:"keyType,omitempty"` + KeyPath string `json:"keyPath,omitempty"` + KeyPaths []string `json:"keyPaths,omitempty"` + KeyData string `json:"keyData,omitempty"` + SignedIdentity json.RawMessage `json:"signedIdentity,omitempty"` +} + +// genericPolicyContent is the overall structure of a policy.json file (= c/image/v5/signature.Policy), using generic data for individual requirements. +type genericPolicyContent struct { + Default json.RawMessage `json:"default"` + Transports genericTransportsContent `json:"transports,omitempty"` +} + +// genericTransportsContent contains policies for individual transports (= c/image/v5/signature.Policy.Transports), using generic data for individual requirements. +type genericTransportsContent map[string]genericRepoMap + +// genericRepoMap maps a scope name to requirements that apply to that scope (= c/image/v5/signature.PolicyTransportScopes) +type genericRepoMap map[string]json.RawMessage + +// DefaultPolicyPath returns a path to the default policy of the system. +func DefaultPolicyPath(sys *types.SystemContext) string { + systemDefaultPolicyPath := "/etc/containers/policy.json" + if sys != nil { + if sys.SignaturePolicyPath != "" { + return sys.SignaturePolicyPath + } + if sys.RootForImplicitAbsolutePaths != "" { + return filepath.Join(sys.RootForImplicitAbsolutePaths, systemDefaultPolicyPath) + } + } + return systemDefaultPolicyPath +} + +// gpgIDReader returns GPG key IDs of keys stored at the provided path. +// It exists only for tests, production code should always use getGPGIdFromKeyPath. +type gpgIDReader func(string) []string + +// createTmpFile creates a temp file under dir and writes the content into it +func createTmpFile(dir, pattern string, content []byte) (string, error) { + tmpfile, err := ioutil.TempFile(dir, pattern) + if err != nil { + return "", err + } + defer tmpfile.Close() + + if _, err := tmpfile.Write(content); err != nil { + return "", err + } + return tmpfile.Name(), nil +} + +// getGPGIdFromKeyPath returns GPG key IDs of keys stored at the provided path. +func getGPGIdFromKeyPath(path string) []string { + cmd := exec.Command("gpg2", "--with-colons", path) + results, err := cmd.Output() + if err != nil { + logrus.Errorf("Getting key identity: %s", err) + return nil + } + return parseUids(results) +} + +// getGPGIdFromKeyData returns GPG key IDs of keys in the provided keyring. +func getGPGIdFromKeyData(idReader gpgIDReader, key string) []string { + decodeKey, err := base64.StdEncoding.DecodeString(key) + if err != nil { + logrus.Errorf("%s, error decoding key data", err) + return nil + } + tmpfileName, err := createTmpFile("", "", decodeKey) + if err != nil { + logrus.Errorf("Creating key date temp file %s", err) + } + defer os.Remove(tmpfileName) + return idReader(tmpfileName) +} + +func parseUids(colonDelimitKeys []byte) []string { + var parseduids []string + scanner := bufio.NewScanner(bytes.NewReader(colonDelimitKeys)) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "uid:") || strings.HasPrefix(line, "pub:") { + uid := strings.Split(line, ":")[9] + if uid == "" { + continue + } + parseduid := uid + if strings.Contains(uid, "<") && strings.Contains(uid, ">") { + parseduid = strings.SplitN(strings.SplitAfterN(uid, "<", 2)[1], ">", 2)[0] + } + parseduids = append(parseduids, parseduid) + } + } + return parseduids +} + +// getPolicy parses policy.json into policyContent. +func getPolicy(policyPath string) (policyContent, error) { + var policyContentStruct policyContent + policyContent, err := ioutil.ReadFile(policyPath) + if err != nil { + return policyContentStruct, fmt.Errorf("unable to read policy file: %w", err) + } + if err := json.Unmarshal(policyContent, &policyContentStruct); err != nil { + return policyContentStruct, fmt.Errorf("could not parse trust policies from %s: %w", policyPath, err) + } + return policyContentStruct, nil +} + +var typeDescription = map[string]string{"insecureAcceptAnything": "accept", "signedBy": "signed", "sigstoreSigned": "sigstoreSigned", "reject": "reject"} + +func trustTypeDescription(trustType string) string { + trustDescription, exist := typeDescription[trustType] + if !exist { + logrus.Warnf("Invalid trust type %s", trustType) + } + return trustDescription +} + +// AddPolicyEntriesInput collects some parameters to AddPolicyEntries, +// primarily so that the callers use named values instead of just strings in a sequence. +type AddPolicyEntriesInput struct { + Scope string // "default" or a docker/atomic scope name + Type string + PubKeyFiles []string // For signature enforcement types, paths to public keys files (where the image needs to be signed by at least one key from _each_ of the files). File format depends on Type. +} + +// AddPolicyEntries adds one or more policy entries necessary to implement AddPolicyEntriesInput. +func AddPolicyEntries(policyPath string, input AddPolicyEntriesInput) error { + var ( + policyContentStruct genericPolicyContent + newReposContent []repoContent + ) + trustType := input.Type + if trustType == "accept" { + trustType = "insecureAcceptAnything" + } + pubkeysfile := input.PubKeyFiles + + // The error messages in validation failures use input.Type instead of trustType to match the user’s input. + switch trustType { + case "insecureAcceptAnything", "reject": + if len(pubkeysfile) != 0 { + return fmt.Errorf("%d public keys unexpectedly provided for trust type %v", len(pubkeysfile), input.Type) + } + newReposContent = append(newReposContent, repoContent{Type: trustType}) + + case "signedBy": + if len(pubkeysfile) == 0 { + return errors.New("at least one public key must be defined for type 'signedBy'") + } + for _, filepath := range pubkeysfile { + newReposContent = append(newReposContent, repoContent{Type: trustType, KeyType: "GPGKeys", KeyPath: filepath}) + } + + case "sigstoreSigned": + if len(pubkeysfile) == 0 { + return errors.New("at least one public key must be defined for type 'sigstoreSigned'") + } + for _, filepath := range pubkeysfile { + newReposContent = append(newReposContent, repoContent{Type: trustType, KeyPath: filepath}) + } + + default: + return fmt.Errorf("unknown trust type %q", input.Type) + } + newReposJSON, err := json.Marshal(newReposContent) + if err != nil { + return err + } + + _, err = os.Stat(policyPath) + if !os.IsNotExist(err) { + policyContent, err := ioutil.ReadFile(policyPath) + if err != nil { + return err + } + if err := json.Unmarshal(policyContent, &policyContentStruct); err != nil { + return errors.New("could not read trust policies") + } + } + if input.Scope == "default" { + policyContentStruct.Default = json.RawMessage(newReposJSON) + } else { + if len(policyContentStruct.Default) == 0 { + return errors.New("default trust policy must be set") + } + registryExists := false + for transport, transportval := range policyContentStruct.Transports { + _, registryExists = transportval[input.Scope] + if registryExists { + policyContentStruct.Transports[transport][input.Scope] = json.RawMessage(newReposJSON) + break + } + } + if !registryExists { + if policyContentStruct.Transports == nil { + policyContentStruct.Transports = make(map[string]genericRepoMap) + } + if policyContentStruct.Transports["docker"] == nil { + policyContentStruct.Transports["docker"] = make(map[string]json.RawMessage) + } + policyContentStruct.Transports["docker"][input.Scope] = json.RawMessage(newReposJSON) + } + } + + data, err := json.MarshalIndent(policyContentStruct, "", " ") + if err != nil { + return fmt.Errorf("error setting trust policy: %w", err) + } + return ioutil.WriteFile(policyPath, data, 0644) +} diff --git a/pkg/trust/policy_test.go b/pkg/trust/policy_test.go new file mode 100644 index 000000000..3952b72c3 --- /dev/null +++ b/pkg/trust/policy_test.go @@ -0,0 +1,196 @@ +package trust + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/containers/image/v5/signature" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAddPolicyEntries(t *testing.T) { + tempDir := t.TempDir() + policyPath := filepath.Join(tempDir, "policy.json") + + minimalPolicy := &signature.Policy{ + Default: []signature.PolicyRequirement{ + signature.NewPRInsecureAcceptAnything(), + }, + } + minimalPolicyJSON, err := json.Marshal(minimalPolicy) + require.NoError(t, err) + err = os.WriteFile(policyPath, minimalPolicyJSON, 0600) + require.NoError(t, err) + + // Invalid input: + for _, invalid := range []AddPolicyEntriesInput{ + { + Scope: "default", + Type: "accept", + PubKeyFiles: []string{"/does-not-make-sense"}, + }, + { + Scope: "default", + Type: "insecureAcceptAnything", + PubKeyFiles: []string{"/does-not-make-sense"}, + }, + { + Scope: "default", + Type: "reject", + PubKeyFiles: []string{"/does-not-make-sense"}, + }, + { + Scope: "default", + Type: "signedBy", + PubKeyFiles: []string{}, // A key is missing + }, + { + Scope: "default", + Type: "sigstoreSigned", + PubKeyFiles: []string{}, // A key is missing + }, + { + Scope: "default", + Type: "this-is-unknown", + PubKeyFiles: []string{}, + }, + } { + err := AddPolicyEntries(policyPath, invalid) + assert.Error(t, err, "%#v", invalid) + } + + err = AddPolicyEntries(policyPath, AddPolicyEntriesInput{ + Scope: "default", + Type: "reject", + }) + assert.NoError(t, err) + err = AddPolicyEntries(policyPath, AddPolicyEntriesInput{ + Scope: "quay.io/accepted", + Type: "accept", + }) + assert.NoError(t, err) + err = AddPolicyEntries(policyPath, AddPolicyEntriesInput{ + Scope: "quay.io/multi-signed", + Type: "signedBy", + PubKeyFiles: []string{"/1.pub", "/2.pub"}, + }) + assert.NoError(t, err) + err = AddPolicyEntries(policyPath, AddPolicyEntriesInput{ + Scope: "quay.io/sigstore-signed", + Type: "sigstoreSigned", + PubKeyFiles: []string{"/1.pub", "/2.pub"}, + }) + assert.NoError(t, err) + + // Test that the outcome is consumable, and compare it with the expected values. + parsedPolicy, err := signature.NewPolicyFromFile(policyPath) + require.NoError(t, err) + assert.Equal(t, &signature.Policy{ + Default: signature.PolicyRequirements{ + signature.NewPRReject(), + }, + Transports: map[string]signature.PolicyTransportScopes{ + "docker": { + "quay.io/accepted": { + signature.NewPRInsecureAcceptAnything(), + }, + "quay.io/multi-signed": { + xNewPRSignedByKeyPath(t, "/1.pub", signature.NewPRMMatchRepoDigestOrExact()), + xNewPRSignedByKeyPath(t, "/2.pub", signature.NewPRMMatchRepoDigestOrExact()), + }, + "quay.io/sigstore-signed": { + xNewPRSigstoreSignedKeyPath(t, "/1.pub", signature.NewPRMMatchRepoDigestOrExact()), + xNewPRSigstoreSignedKeyPath(t, "/2.pub", signature.NewPRMMatchRepoDigestOrExact()), + }, + }, + }, + }, parsedPolicy) + + // Test that completely unknown JSON is preserved + jsonWithUnknownData := `{ + "default": [ + { + "type": "this is unknown", + "unknown field": "should be preserved" + } + ], + "transports": + { + "docker-daemon": + { + "": [{ + "type":"this is unknown 2", + "unknown field 2": "should be preserved 2" + }] + } + } +}` + err = os.WriteFile(policyPath, []byte(jsonWithUnknownData), 0600) + require.NoError(t, err) + err = AddPolicyEntries(policyPath, AddPolicyEntriesInput{ + Scope: "quay.io/innocuous", + Type: "signedBy", + PubKeyFiles: []string{"/1.pub"}, + }) + require.NoError(t, err) + updatedJSONWithUnknownData, err := os.ReadFile(policyPath) + require.NoError(t, err) + // Decode updatedJSONWithUnknownData so that this test does not depend on details of the encoding. + // To reduce noise in the constants below: + type a = []interface{} + type m = map[string]interface{} + var parsedUpdatedJSON m + err = json.Unmarshal(updatedJSONWithUnknownData, &parsedUpdatedJSON) + require.NoError(t, err) + assert.Equal(t, m{ + "default": a{ + m{ + "type": "this is unknown", + "unknown field": "should be preserved", + }, + }, + "transports": m{ + "docker-daemon": m{ + "": a{ + m{ + "type": "this is unknown 2", + "unknown field 2": "should be preserved 2", + }, + }, + }, + "docker": m{ + "quay.io/innocuous": a{ + m{ + "type": "signedBy", + "keyType": "GPGKeys", + "keyPath": "/1.pub", + }, + }, + }, + }, + }, parsedUpdatedJSON) +} + +// xNewPRSignedByKeyPath is a wrapper for NewPRSignedByKeyPath which must not fail. +func xNewPRSignedByKeyPath(t *testing.T, keyPath string, signedIdentity signature.PolicyReferenceMatch) signature.PolicyRequirement { + pr, err := signature.NewPRSignedByKeyPath(signature.SBKeyTypeGPGKeys, keyPath, signedIdentity) + require.NoError(t, err) + return pr +} + +// xNewPRSignedByKeyPaths is a wrapper for NewPRSignedByKeyPaths which must not fail. +func xNewPRSignedByKeyPaths(t *testing.T, keyPaths []string, signedIdentity signature.PolicyReferenceMatch) signature.PolicyRequirement { + pr, err := signature.NewPRSignedByKeyPaths(signature.SBKeyTypeGPGKeys, keyPaths, signedIdentity) + require.NoError(t, err) + return pr +} + +// xNewPRSigstoreSignedKeyPath is a wrapper for NewPRSigstoreSignedKeyPath which must not fail. +func xNewPRSigstoreSignedKeyPath(t *testing.T, keyPath string, signedIdentity signature.PolicyReferenceMatch) signature.PolicyRequirement { + pr, err := signature.NewPRSigstoreSignedKeyPath(keyPath, signedIdentity) + require.NoError(t, err) + return pr +} diff --git a/pkg/trust/registries.go b/pkg/trust/registries.go new file mode 100644 index 000000000..0adc38232 --- /dev/null +++ b/pkg/trust/registries.go @@ -0,0 +1,126 @@ +package trust + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strings" + + "github.com/containers/image/v5/types" + "github.com/docker/docker/pkg/homedir" + "github.com/ghodss/yaml" +) + +// registryConfiguration is one of the files in registriesDirPath configuring lookaside locations, or the result of merging them all. +// NOTE: Keep this in sync with docs/registries.d.md! +type registryConfiguration struct { + DefaultDocker *registryNamespace `json:"default-docker"` + // The key is a namespace, using fully-expanded Docker reference format or parent namespaces (per dockerReference.PolicyConfiguration*), + Docker map[string]registryNamespace `json:"docker"` +} + +// registryNamespace defines lookaside locations for a single namespace. +type registryNamespace struct { + Lookaside string `json:"lookaside"` // For reading, and if LookasideStaging is not present, for writing. + LookasideStaging string `json:"lookaside-staging"` // For writing only. + SigStore string `json:"sigstore"` // For reading, and if SigStoreStaging is not present, for writing. + SigStoreStaging string `json:"sigstore-staging"` // For writing only. +} + +// systemRegistriesDirPath is the path to registries.d. +const systemRegistriesDirPath = "/etc/containers/registries.d" + +// userRegistriesDir is the path to the per user registries.d. +var userRegistriesDir = filepath.FromSlash(".config/containers/registries.d") + +// RegistriesDirPath returns a path to registries.d +func RegistriesDirPath(sys *types.SystemContext) string { + if sys != nil && sys.RegistriesDirPath != "" { + return sys.RegistriesDirPath + } + userRegistriesDirPath := filepath.Join(homedir.Get(), userRegistriesDir) + if _, err := os.Stat(userRegistriesDirPath); err == nil { + return userRegistriesDirPath + } + if sys != nil && sys.RootForImplicitAbsolutePaths != "" { + return filepath.Join(sys.RootForImplicitAbsolutePaths, systemRegistriesDirPath) + } + + return systemRegistriesDirPath +} + +// loadAndMergeConfig loads registries.d configuration files in dirPath +func loadAndMergeConfig(dirPath string) (*registryConfiguration, error) { + mergedConfig := registryConfiguration{Docker: map[string]registryNamespace{}} + dockerDefaultMergedFrom := "" + nsMergedFrom := map[string]string{} + + dir, err := os.Open(dirPath) + if err != nil { + if os.IsNotExist(err) { + return &mergedConfig, nil + } + return nil, err + } + configNames, err := dir.Readdirnames(0) + if err != nil { + return nil, err + } + for _, configName := range configNames { + if !strings.HasSuffix(configName, ".yaml") { + continue + } + configPath := filepath.Join(dirPath, configName) + configBytes, err := ioutil.ReadFile(configPath) + if err != nil { + return nil, err + } + var config registryConfiguration + err = yaml.Unmarshal(configBytes, &config) + if err != nil { + return nil, fmt.Errorf("error parsing %s: %w", configPath, err) + } + if config.DefaultDocker != nil { + if mergedConfig.DefaultDocker != nil { + return nil, fmt.Errorf(`error parsing signature storage configuration: "default-docker" defined both in "%s" and "%s"`, + dockerDefaultMergedFrom, configPath) + } + mergedConfig.DefaultDocker = config.DefaultDocker + dockerDefaultMergedFrom = configPath + } + for nsName, nsConfig := range config.Docker { // includes config.Docker == nil + if _, ok := mergedConfig.Docker[nsName]; ok { + return nil, fmt.Errorf(`error parsing signature storage configuration: "docker" namespace "%s" defined both in "%s" and "%s"`, + nsName, nsMergedFrom[nsName], configPath) + } + mergedConfig.Docker[nsName] = nsConfig + nsMergedFrom[nsName] = configPath + } + } + return &mergedConfig, nil +} + +// registriesDConfigurationForScope returns registries.d configuration for the provided scope. +// scope can be "" to return only the global default configuration entry. +func registriesDConfigurationForScope(registryConfigs *registryConfiguration, scope string) *registryNamespace { + searchScope := scope + if searchScope != "" { + if !strings.Contains(searchScope, "/") { + val, exists := registryConfigs.Docker[searchScope] + if exists { + return &val + } + } + for range strings.Split(scope, "/") { + val, exists := registryConfigs.Docker[searchScope] + if exists { + return &val + } + if strings.Contains(searchScope, "/") { + searchScope = searchScope[:strings.LastIndex(searchScope, "/")] + } + } + } + return registryConfigs.DefaultDocker +} diff --git a/pkg/trust/testdata/default.yaml b/pkg/trust/testdata/default.yaml new file mode 100644 index 000000000..31bcd35ef --- /dev/null +++ b/pkg/trust/testdata/default.yaml @@ -0,0 +1,25 @@ +# This is a default registries.d configuration file. You may +# add to this file or create additional files in registries.d/. +# +# lookaside: indicates a location that is read and write +# lookaside-staging: indicates a location that is only for write +# +# lookaside and lookaside-staging take a value of the following: +# lookaside: {schema}://location +# +# For reading signatures, schema may be http, https, or file. +# For writing signatures, schema may only be file. + +# This is the default signature write location for docker registries. +default-docker: +# lookaside: file:///var/lib/containers/sigstore + lookaside-staging: file:///var/lib/containers/sigstore + +# The 'docker' indicator here is the start of the configuration +# for docker registries. +# +# docker: +# +# privateregistry.com: +# lookaside: http://privateregistry.com/sigstore/ +# lookaside-staging: /mnt/nfs/privateregistry/sigstore diff --git a/pkg/trust/testdata/quay.io.yaml b/pkg/trust/testdata/quay.io.yaml new file mode 100644 index 000000000..80071596d --- /dev/null +++ b/pkg/trust/testdata/quay.io.yaml @@ -0,0 +1,3 @@ +docker: + quay.io/multi-signed: + lookaside: https://quay.example.com/sigstore diff --git a/pkg/trust/testdata/redhat.yaml b/pkg/trust/testdata/redhat.yaml new file mode 100644 index 000000000..8e40a4174 --- /dev/null +++ b/pkg/trust/testdata/redhat.yaml @@ -0,0 +1,5 @@ +docker: + registry.redhat.io: + sigstore: https://registry.redhat.io/containers/sigstore + registry.access.redhat.com: + sigstore: https://registry.redhat.io/containers/sigstore diff --git a/pkg/trust/trust.go b/pkg/trust/trust.go index 663a1b5e2..07d144bc1 100644 --- a/pkg/trust/trust.go +++ b/pkg/trust/trust.go @@ -1,243 +1,127 @@ package trust import ( - "bufio" - "bytes" - "encoding/base64" - "encoding/json" "fmt" - "io/ioutil" - "os" - "os/exec" - "path/filepath" + "sort" "strings" - - "github.com/containers/image/v5/types" - "github.com/docker/docker/pkg/homedir" - "github.com/ghodss/yaml" - "github.com/sirupsen/logrus" ) -// PolicyContent struct for policy.json file -type PolicyContent struct { - Default []RepoContent `json:"default"` - Transports TransportsContent `json:"transports,omitempty"` -} - -// RepoContent struct used under each repo -type RepoContent struct { - Type string `json:"type"` - KeyType string `json:"keyType,omitempty"` - KeyPath string `json:"keyPath,omitempty"` - KeyData string `json:"keyData,omitempty"` - SignedIdentity json.RawMessage `json:"signedIdentity,omitempty"` -} - -// RepoMap map repo name to policycontent for each repo -type RepoMap map[string][]RepoContent - -// TransportsContent struct for content under "transports" -type TransportsContent map[string]RepoMap - -// RegistryConfiguration is one of the files in registriesDirPath configuring lookaside locations, or the result of merging them all. -// NOTE: Keep this in sync with docs/registries.d.md! -type RegistryConfiguration struct { - DefaultDocker *RegistryNamespace `json:"default-docker"` - // The key is a namespace, using fully-expanded Docker reference format or parent namespaces (per dockerReference.PolicyConfiguration*), - Docker map[string]RegistryNamespace `json:"docker"` +// Policy describes a basic trust policy configuration +type Policy struct { + Transport string `json:"transport"` + Name string `json:"name,omitempty"` + RepoName string `json:"repo_name,omitempty"` + Keys []string `json:"keys,omitempty"` + SignatureStore string `json:"sigstore,omitempty"` + Type string `json:"type"` + GPGId string `json:"gpg_id,omitempty"` } -// RegistryNamespace defines lookaside locations for a single namespace. -type RegistryNamespace struct { - SigStore string `json:"sigstore"` // For reading, and if SigStoreStaging is not present, for writing. - SigStoreStaging string `json:"sigstore-staging"` // For writing only. +// PolicyDescription returns an user-focused description of the policy in policyPath and registries.d data from registriesDirPath. +func PolicyDescription(policyPath, registriesDirPath string) ([]*Policy, error) { + return policyDescriptionWithGPGIDReader(policyPath, registriesDirPath, getGPGIdFromKeyPath) } -// ShowOutput keep the fields for image trust show command -type ShowOutput struct { - Repo string - Trusttype string - GPGid string - Sigstore string -} - -// systemRegistriesDirPath is the path to registries.d. -const systemRegistriesDirPath = "/etc/containers/registries.d" - -// userRegistriesDir is the path to the per user registries.d. -var userRegistriesDir = filepath.FromSlash(".config/containers/registries.d") - -// DefaultPolicyPath returns a path to the default policy of the system. -func DefaultPolicyPath(sys *types.SystemContext) string { - systemDefaultPolicyPath := "/etc/containers/policy.json" - if sys != nil { - if sys.SignaturePolicyPath != "" { - return sys.SignaturePolicyPath - } - if sys.RootForImplicitAbsolutePaths != "" { - return filepath.Join(sys.RootForImplicitAbsolutePaths, systemDefaultPolicyPath) - } - } - return systemDefaultPolicyPath -} - -// RegistriesDirPath returns a path to registries.d -func RegistriesDirPath(sys *types.SystemContext) string { - if sys != nil && sys.RegistriesDirPath != "" { - return sys.RegistriesDirPath - } - userRegistriesDirPath := filepath.Join(homedir.Get(), userRegistriesDir) - if _, err := os.Stat(userRegistriesDirPath); err == nil { - return userRegistriesDirPath +// policyDescriptionWithGPGIDReader is PolicyDescription with a gpgIDReader parameter. It exists only to make testing easier. +func policyDescriptionWithGPGIDReader(policyPath, registriesDirPath string, idReader gpgIDReader) ([]*Policy, error) { + policyContentStruct, err := getPolicy(policyPath) + if err != nil { + return nil, fmt.Errorf("could not read trust policies: %w", err) } - if sys != nil && sys.RootForImplicitAbsolutePaths != "" { - return filepath.Join(sys.RootForImplicitAbsolutePaths, systemRegistriesDirPath) + res, err := getPolicyShowOutput(policyContentStruct, registriesDirPath, idReader) + if err != nil { + return nil, fmt.Errorf("could not show trust policies: %w", err) } - - return systemRegistriesDirPath + return res, nil } -// LoadAndMergeConfig loads configuration files in dirPath -func LoadAndMergeConfig(dirPath string) (*RegistryConfiguration, error) { - mergedConfig := RegistryConfiguration{Docker: map[string]RegistryNamespace{}} - dockerDefaultMergedFrom := "" - nsMergedFrom := map[string]string{} +func getPolicyShowOutput(policyContentStruct policyContent, systemRegistriesDirPath string, idReader gpgIDReader) ([]*Policy, error) { + var output []*Policy - dir, err := os.Open(dirPath) + registryConfigs, err := loadAndMergeConfig(systemRegistriesDirPath) if err != nil { - if os.IsNotExist(err) { - return &mergedConfig, nil - } return nil, err } - configNames, err := dir.Readdirnames(0) - if err != nil { - return nil, err - } - for _, configName := range configNames { - if !strings.HasSuffix(configName, ".yaml") { - continue - } - configPath := filepath.Join(dirPath, configName) - configBytes, err := ioutil.ReadFile(configPath) - if err != nil { - return nil, err + + if len(policyContentStruct.Default) > 0 { + template := Policy{ + Transport: "all", + Name: "* (default)", + RepoName: "default", } - var config RegistryConfiguration - err = yaml.Unmarshal(configBytes, &config) - if err != nil { - return nil, fmt.Errorf("error parsing %s: %w", configPath, err) + output = append(output, descriptionsOfPolicyRequirements(policyContentStruct.Default, template, registryConfigs, "", idReader)...) + } + // FIXME: This should use x/exp/maps.Keys after we update to Go 1.18. + transports := []string{} + for t := range policyContentStruct.Transports { + transports = append(transports, t) + } + sort.Strings(transports) + for _, transport := range transports { + transval := policyContentStruct.Transports[transport] + if transport == "docker" { + transport = "repository" } - if config.DefaultDocker != nil { - if mergedConfig.DefaultDocker != nil { - return nil, fmt.Errorf(`error parsing signature storage configuration: "default-docker" defined both in "%s" and "%s"`, - dockerDefaultMergedFrom, configPath) - } - mergedConfig.DefaultDocker = config.DefaultDocker - dockerDefaultMergedFrom = configPath + + // FIXME: This should use x/exp/maps.Keys after we update to Go 1.18. + scopes := []string{} + for s := range transval { + scopes = append(scopes, s) } - for nsName, nsConfig := range config.Docker { // includes config.Docker == nil - if _, ok := mergedConfig.Docker[nsName]; ok { - return nil, fmt.Errorf(`error parsing signature storage configuration: "docker" namespace "%s" defined both in "%s" and "%s"`, - nsName, nsMergedFrom[nsName], configPath) + sort.Strings(scopes) + for _, repo := range scopes { + repoval := transval[repo] + template := Policy{ + Transport: transport, + Name: repo, + RepoName: repo, } - mergedConfig.Docker[nsName] = nsConfig - nsMergedFrom[nsName] = configPath + output = append(output, descriptionsOfPolicyRequirements(repoval, template, registryConfigs, repo, idReader)...) } } - return &mergedConfig, nil + return output, nil } -// HaveMatchRegistry checks if trust settings for the registry have been configured in yaml file -func HaveMatchRegistry(key string, registryConfigs *RegistryConfiguration) *RegistryNamespace { - searchKey := key - if !strings.Contains(searchKey, "/") { - val, exists := registryConfigs.Docker[searchKey] - if exists { - return &val - } - } - for range strings.Split(key, "/") { - val, exists := registryConfigs.Docker[searchKey] - if exists { - return &val - } - if strings.Contains(searchKey, "/") { - searchKey = searchKey[:strings.LastIndex(searchKey, "/")] +// descriptionsOfPolicyRequirements turns reqs into user-readable policy entries, with Transport/Name/Reponame coming from template, potentially looking up scope (which may be "") in registryConfigs. +func descriptionsOfPolicyRequirements(reqs []repoContent, template Policy, registryConfigs *registryConfiguration, scope string, idReader gpgIDReader) []*Policy { + res := []*Policy{} + + var lookasidePath string + registryNamespace := registriesDConfigurationForScope(registryConfigs, scope) + if registryNamespace != nil { + if registryNamespace.Lookaside != "" { + lookasidePath = registryNamespace.Lookaside + } else { // incl. registryNamespace.SigStore == "" + lookasidePath = registryNamespace.SigStore } } - return registryConfigs.DefaultDocker -} - -// CreateTmpFile creates a temp file under dir and writes the content into it -func CreateTmpFile(dir, pattern string, content []byte) (string, error) { - tmpfile, err := ioutil.TempFile(dir, pattern) - if err != nil { - return "", err - } - defer tmpfile.Close() - - if _, err := tmpfile.Write(content); err != nil { - return "", err - } - return tmpfile.Name(), nil -} - -// GetGPGIdFromKeyPath return user keyring from key path -func GetGPGIdFromKeyPath(path string) []string { - cmd := exec.Command("gpg2", "--with-colons", path) - results, err := cmd.Output() - if err != nil { - logrus.Errorf("Getting key identity: %s", err) - return nil - } - return parseUids(results) -} -// GetGPGIdFromKeyData return user keyring from keydata -func GetGPGIdFromKeyData(key string) []string { - decodeKey, err := base64.StdEncoding.DecodeString(key) - if err != nil { - logrus.Errorf("%s, error decoding key data", err) - return nil - } - tmpfileName, err := CreateTmpFile("", "", decodeKey) - if err != nil { - logrus.Errorf("Creating key date temp file %s", err) - } - defer os.Remove(tmpfileName) - return GetGPGIdFromKeyPath(tmpfileName) -} + for _, repoele := range reqs { + entry := template + entry.Type = trustTypeDescription(repoele.Type) -func parseUids(colonDelimitKeys []byte) []string { - var parseduids []string - scanner := bufio.NewScanner(bytes.NewReader(colonDelimitKeys)) - for scanner.Scan() { - line := scanner.Text() - if strings.HasPrefix(line, "uid:") || strings.HasPrefix(line, "pub:") { - uid := strings.Split(line, ":")[9] - if uid == "" { - continue + var gpgIDString string + switch repoele.Type { + case "signedBy": + uids := []string{} + if len(repoele.KeyPath) > 0 { + uids = append(uids, idReader(repoele.KeyPath)...) } - parseduid := uid - if strings.Contains(uid, "<") && strings.Contains(uid, ">") { - parseduid = strings.SplitN(strings.SplitAfterN(uid, "<", 2)[1], ">", 2)[0] + for _, path := range repoele.KeyPaths { + uids = append(uids, idReader(path)...) } - parseduids = append(parseduids, parseduid) + if len(repoele.KeyData) > 0 { + uids = append(uids, getGPGIdFromKeyData(idReader, repoele.KeyData)...) + } + gpgIDString = strings.Join(uids, ", ") + + case "sigstoreSigned": + gpgIDString = "N/A" // We could potentially return key fingerprints here, but they would not be _GPG_ fingerprints. } + entry.GPGId = gpgIDString + entry.SignatureStore = lookasidePath // We do this even for sigstoreSigned and things like type: reject, to show that the sigstore is being read. + res = append(res, &entry) } - return parseduids -} -// GetPolicy parse policy.json into PolicyContent struct -func GetPolicy(policyPath string) (PolicyContent, error) { - var policyContentStruct PolicyContent - policyContent, err := ioutil.ReadFile(policyPath) - if err != nil { - return policyContentStruct, fmt.Errorf("unable to read policy file: %w", err) - } - if err := json.Unmarshal(policyContent, &policyContentStruct); err != nil { - return policyContentStruct, fmt.Errorf("could not parse trust policies from %s: %w", policyPath, err) - } - return policyContentStruct, nil + return res } diff --git a/pkg/trust/trust_test.go b/pkg/trust/trust_test.go new file mode 100644 index 000000000..22b780fc9 --- /dev/null +++ b/pkg/trust/trust_test.go @@ -0,0 +1,376 @@ +package trust + +import ( + "encoding/json" + "os" + "path/filepath" + "strings" + "testing" + + "github.com/containers/image/v5/signature" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPolicyDescription(t *testing.T) { + tempDir := t.TempDir() + policyPath := filepath.Join(tempDir, "policy.json") + + // Override getGPGIdFromKeyPath because we don't want to bother with (and spend the unit-test time on) generating valid GPG keys, and running the real GPG binary. + // Instead of reading the files at all, just expect file names like /id1,id2,...,idN.pub + idReader := func(keyPath string) []string { + require.True(t, strings.HasPrefix(keyPath, "/")) + require.True(t, strings.HasSuffix(keyPath, ".pub")) + return strings.Split(keyPath[1:len(keyPath)-4], ",") + } + + for _, c := range []struct { + policy *signature.Policy + expected []*Policy + }{ + { + &signature.Policy{ + Default: signature.PolicyRequirements{ + signature.NewPRReject(), + }, + Transports: map[string]signature.PolicyTransportScopes{ + "docker": { + "quay.io/accepted": { + signature.NewPRInsecureAcceptAnything(), + }, + "registry.redhat.io": { + xNewPRSignedByKeyPath(t, "/redhat.pub", signature.NewPRMMatchRepoDigestOrExact()), + }, + "registry.access.redhat.com": { + xNewPRSignedByKeyPaths(t, []string{"/redhat.pub", "/redhat-beta.pub"}, signature.NewPRMMatchRepoDigestOrExact()), + }, + "quay.io/multi-signed": { + xNewPRSignedByKeyPath(t, "/1.pub", signature.NewPRMMatchRepoDigestOrExact()), + xNewPRSignedByKeyPath(t, "/2,3.pub", signature.NewPRMMatchRepoDigestOrExact()), + }, + "quay.io/sigstore-signed": { + xNewPRSigstoreSignedKeyPath(t, "/1.pub", signature.NewPRMMatchRepoDigestOrExact()), + xNewPRSigstoreSignedKeyPath(t, "/2.pub", signature.NewPRMMatchRepoDigestOrExact()), + }, + }, + }, + }, + []*Policy{ + { + Transport: "all", + Name: "* (default)", + RepoName: "default", + Type: "reject", + }, + { + Transport: "repository", + Name: "quay.io/accepted", + RepoName: "quay.io/accepted", + Type: "accept", + }, + { + Transport: "repository", + Name: "quay.io/multi-signed", + RepoName: "quay.io/multi-signed", + Type: "signed", + SignatureStore: "https://quay.example.com/sigstore", + GPGId: "1", + }, + { + Transport: "repository", + Name: "quay.io/multi-signed", + RepoName: "quay.io/multi-signed", + Type: "signed", + SignatureStore: "https://quay.example.com/sigstore", + GPGId: "2, 3", + }, + { + Transport: "repository", + Name: "quay.io/sigstore-signed", + RepoName: "quay.io/sigstore-signed", + Type: "sigstoreSigned", + SignatureStore: "", + GPGId: "N/A", + }, + { + Transport: "repository", + Name: "quay.io/sigstore-signed", + RepoName: "quay.io/sigstore-signed", + Type: "sigstoreSigned", + SignatureStore: "", + GPGId: "N/A", + }, + { + Transport: "repository", + Name: "registry.access.redhat.com", + RepoName: "registry.access.redhat.com", + Type: "signed", + SignatureStore: "https://registry.redhat.io/containers/sigstore", + GPGId: "redhat, redhat-beta", + }, { + Transport: "repository", + Name: "registry.redhat.io", + RepoName: "registry.redhat.io", + Type: "signed", + SignatureStore: "https://registry.redhat.io/containers/sigstore", + GPGId: "redhat", + }, + }, + }, + { + &signature.Policy{ + Default: signature.PolicyRequirements{ + xNewPRSignedByKeyPath(t, "/1.pub", signature.NewPRMMatchRepoDigestOrExact()), + xNewPRSignedByKeyPath(t, "/2,3.pub", signature.NewPRMMatchRepoDigestOrExact()), + }, + }, + []*Policy{ + { + Transport: "all", + Name: "* (default)", + RepoName: "default", + Type: "signed", + SignatureStore: "", + GPGId: "1", + }, + { + Transport: "all", + Name: "* (default)", + RepoName: "default", + Type: "signed", + SignatureStore: "", + GPGId: "2, 3", + }, + }, + }, + } { + policyJSON, err := json.Marshal(c.policy) + require.NoError(t, err) + err = os.WriteFile(policyPath, policyJSON, 0600) + require.NoError(t, err) + + res, err := policyDescriptionWithGPGIDReader(policyPath, "./testdata", idReader) + require.NoError(t, err) + assert.Equal(t, c.expected, res) + } +} + +func TestDescriptionsOfPolicyRequirements(t *testing.T) { + // Override getGPGIdFromKeyPath because we don't want to bother with (and spend the unit-test time on) generating valid GPG keys, and running the real GPG binary. + // Instead of reading the files at all, just expect file names like /id1,id2,...,idN.pub + idReader := func(keyPath string) []string { + require.True(t, strings.HasPrefix(keyPath, "/")) + require.True(t, strings.HasSuffix(keyPath, ".pub")) + return strings.Split(keyPath[1:len(keyPath)-4], ",") + } + + template := Policy{ + Transport: "transport", + Name: "name", + RepoName: "repoName", + } + registryConfigs, err := loadAndMergeConfig("./testdata") + require.NoError(t, err) + + for _, c := range []struct { + scope string + reqs signature.PolicyRequirements + expected []*Policy + }{ + { + "", + signature.PolicyRequirements{ + signature.NewPRReject(), + }, + []*Policy{ + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "reject", + }, + }, + }, + { + "quay.io/accepted", + signature.PolicyRequirements{ + signature.NewPRInsecureAcceptAnything(), + }, + []*Policy{ + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "accept", + }, + }, + }, + { + "registry.redhat.io", + signature.PolicyRequirements{ + xNewPRSignedByKeyPath(t, "/redhat.pub", signature.NewPRMMatchRepoDigestOrExact()), + }, + []*Policy{ + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "signed", + SignatureStore: "https://registry.redhat.io/containers/sigstore", + GPGId: "redhat", + }, + }, + }, + { + "registry.access.redhat.com", + signature.PolicyRequirements{ + xNewPRSignedByKeyPaths(t, []string{"/redhat.pub", "/redhat-beta.pub"}, signature.NewPRMMatchRepoDigestOrExact()), + }, + []*Policy{ + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "signed", + SignatureStore: "https://registry.redhat.io/containers/sigstore", + GPGId: "redhat, redhat-beta", + }, + }, + }, + { + "quay.io/multi-signed", + signature.PolicyRequirements{ + xNewPRSignedByKeyPath(t, "/1.pub", signature.NewPRMMatchRepoDigestOrExact()), + xNewPRSignedByKeyPath(t, "/2,3.pub", signature.NewPRMMatchRepoDigestOrExact()), + }, + []*Policy{ + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "signed", + SignatureStore: "https://quay.example.com/sigstore", + GPGId: "1", + }, + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "signed", + SignatureStore: "https://quay.example.com/sigstore", + GPGId: "2, 3", + }, + }, + }, { + "quay.io/sigstore-signed", + signature.PolicyRequirements{ + xNewPRSigstoreSignedKeyPath(t, "/1.pub", signature.NewPRMMatchRepoDigestOrExact()), + xNewPRSigstoreSignedKeyPath(t, "/2.pub", signature.NewPRMMatchRepoDigestOrExact()), + }, + []*Policy{ + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "sigstoreSigned", + SignatureStore: "", + GPGId: "N/A", + }, + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "sigstoreSigned", + SignatureStore: "", + GPGId: "N/A", + }, + }, + }, + { // Multiple kinds of requirements are represented individually. + "registry.redhat.io", + signature.PolicyRequirements{ + signature.NewPRReject(), + signature.NewPRInsecureAcceptAnything(), + xNewPRSignedByKeyPath(t, "/redhat.pub", signature.NewPRMMatchRepoDigestOrExact()), + xNewPRSignedByKeyPaths(t, []string{"/redhat.pub", "/redhat-beta.pub"}, signature.NewPRMMatchRepoDigestOrExact()), + xNewPRSignedByKeyPath(t, "/1.pub", signature.NewPRMMatchRepoDigestOrExact()), + xNewPRSignedByKeyPath(t, "/2,3.pub", signature.NewPRMMatchRepoDigestOrExact()), + xNewPRSigstoreSignedKeyPath(t, "/1.pub", signature.NewPRMMatchRepoDigestOrExact()), + xNewPRSigstoreSignedKeyPath(t, "/2.pub", signature.NewPRMMatchRepoDigestOrExact()), + }, + []*Policy{ + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + SignatureStore: "https://registry.redhat.io/containers/sigstore", + Type: "reject", + }, + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + SignatureStore: "https://registry.redhat.io/containers/sigstore", + Type: "accept", + }, + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "signed", + SignatureStore: "https://registry.redhat.io/containers/sigstore", + GPGId: "redhat", + }, + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "signed", + SignatureStore: "https://registry.redhat.io/containers/sigstore", + GPGId: "redhat, redhat-beta", + }, + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "signed", + SignatureStore: "https://registry.redhat.io/containers/sigstore", + GPGId: "1", + }, + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "signed", + SignatureStore: "https://registry.redhat.io/containers/sigstore", + GPGId: "2, 3", + }, + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "sigstoreSigned", + SignatureStore: "https://registry.redhat.io/containers/sigstore", + GPGId: "N/A", + }, + { + Transport: "transport", + Name: "name", + RepoName: "repoName", + Type: "sigstoreSigned", + SignatureStore: "https://registry.redhat.io/containers/sigstore", + GPGId: "N/A", + }, + }, + }, + } { + reqsJSON, err := json.Marshal(c.reqs) + require.NoError(t, err) + var parsedRegs []repoContent + err = json.Unmarshal(reqsJSON, &parsedRegs) + require.NoError(t, err) + + res := descriptionsOfPolicyRequirements(parsedRegs, template, registryConfigs, c.scope, idReader) + assert.Equal(t, c.expected, res) + } +} diff --git a/test/apiv2/10-images.at b/test/apiv2/10-images.at index 4fd954e37..86ee2a1f5 100644 --- a/test/apiv2/10-images.at +++ b/test/apiv2/10-images.at @@ -239,4 +239,23 @@ fi cleanBuildTest +# compat API vs libpod API event differences: +# on image removal, libpod produces 'remove' events. +# compat produces 'delete' events. +podman image build -t test:test -<<EOF +from $IMAGE +EOF + +START=$(date +%s) + +t DELETE libpod/images/test:test 200 +# HACK HACK HACK There is a race around events being added to the journal +# This sleep seems to avoid the race. +# If it fails and begins to flake, investigate a retry loop. +sleep 1 +t GET "libpod/events?stream=false&since=$START" 200 \ + 'select(.status | contains("remove")).Action=remove' +t GET "events?stream=false&since=$START" 200 \ + 'select(.status | contains("delete")).Action=delete' + # vim: filetype=sh diff --git a/test/e2e/build/Containerfile.userns-auto b/test/e2e/build/Containerfile.userns-auto new file mode 100644 index 000000000..921610982 --- /dev/null +++ b/test/e2e/build/Containerfile.userns-auto @@ -0,0 +1,2 @@ +FROM alpine +RUN cat /proc/self/uid_map diff --git a/test/e2e/network_connect_disconnect_test.go b/test/e2e/network_connect_disconnect_test.go index ece1b519d..30a5c6482 100644 --- a/test/e2e/network_connect_disconnect_test.go +++ b/test/e2e/network_connect_disconnect_test.go @@ -157,7 +157,7 @@ var _ = Describe("Podman network connect and disconnect", func() { Expect(con.ErrorToString()).To(ContainSubstring(`"slirp4netns" is not supported: invalid network mode`)) }) - It("podman connect on a container that already is connected to the network should error", func() { + It("podman connect on a container that already is connected to the network should error after init", func() { netName := "aliasTest" + stringid.GenerateNonCryptoID() session := podmanTest.Podman([]string{"network", "create", netName}) session.WaitWithDefaultTimeout() @@ -177,7 +177,15 @@ var _ = Describe("Podman network connect and disconnect", func() { con := podmanTest.Podman([]string{"network", "connect", netName, "test"}) con.WaitWithDefaultTimeout() - Expect(con).Should(ExitWithError()) + Expect(con).Should(Exit(0)) + + init := podmanTest.Podman([]string{"init", "test"}) + init.WaitWithDefaultTimeout() + Expect(init).Should(Exit(0)) + + con2 := podmanTest.Podman([]string{"network", "connect", netName, "test"}) + con2.WaitWithDefaultTimeout() + Expect(con2).Should(ExitWithError()) }) It("podman network connect", func() { diff --git a/test/e2e/restart_test.go b/test/e2e/restart_test.go index dd0070f54..9df884292 100644 --- a/test/e2e/restart_test.go +++ b/test/e2e/restart_test.go @@ -228,7 +228,7 @@ var _ = Describe("Podman restart", func() { Expect(beforeRestart.OutputToString()).To(Equal(afterRestart.OutputToString())) }) - It("podman restart all stoped containers with --all", func() { + It("podman restart all stopped containers with --all", func() { session := podmanTest.RunTopContainer("") session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) diff --git a/test/e2e/run_userns_test.go b/test/e2e/run_userns_test.go index f247b2dac..62e512d3a 100644 --- a/test/e2e/run_userns_test.go +++ b/test/e2e/run_userns_test.go @@ -8,6 +8,7 @@ import ( "strings" . "github.com/containers/podman/v4/test/utils" + "github.com/containers/storage" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gexec" @@ -42,6 +43,33 @@ var _ = Describe("Podman UserNS support", func() { }) + // Note: Lot of tests for build with --userns=auto are already there in buildah + // but they are skipped in podman CI because bud tests are executed in rootfull + // environment ( where mappings for the `containers` user is not present in /etc/subuid ) + // causing them to skip hence this is a redundant test for sanity to make sure + // we don't break this feature for podman-remote. + It("podman build with --userns=auto", func() { + u, err := user.Current() + Expect(err).To(BeNil()) + name := u.Name + if name == "root" { + name = "containers" + } + content, err := ioutil.ReadFile("/etc/subuid") + if err != nil { + Skip("cannot read /etc/subuid") + } + if !strings.Contains(string(content), name) { + Skip("cannot find mappings for the current user") + } + session := podmanTest.Podman([]string{"build", "-f", "build/Containerfile.userns-auto", "-t", "test", "--userns=auto"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + // `1024` is the default size or length of the range of user IDs + // that is mapped between the two user namespaces by --userns=auto. + Expect(session.OutputToString()).To(ContainSubstring(fmt.Sprintf("%d", storage.AutoUserNsMinSize))) + }) + It("podman uidmapping and gidmapping", func() { session := podmanTest.Podman([]string{"run", "--uidmap=0:100:5000", "--gidmap=0:200:5000", "alpine", "echo", "hello"}) session.WaitWithDefaultTimeout() @@ -157,6 +185,8 @@ var _ = Describe("Podman UserNS support", func() { session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) l := session.OutputToString() + // `1024` is the default size or length of the range of user IDs + // that is mapped between the two user namespaces by --userns=auto. Expect(l).To(ContainSubstring("1024")) m[l] = l } diff --git a/vendor/github.com/vbauerster/mpb/v7/bar.go b/vendor/github.com/vbauerster/mpb/v7/bar.go index 4991f4f15..7db860e30 100644 --- a/vendor/github.com/vbauerster/mpb/v7/bar.go +++ b/vendor/github.com/vbauerster/mpb/v7/bar.go @@ -29,7 +29,7 @@ type Bar struct { recoveredPanic interface{} } -type extenderFunc func(in io.Reader, reqWidth int, st decor.Statistics) (out io.Reader, lines int) +type extenderFunc func(rows []io.Reader, width int, stat decor.Statistics) []io.Reader // bState is actual bar's state. type bState struct { @@ -57,14 +57,15 @@ type bState struct { extender extenderFunc debugOut io.Writer - afterBar *Bar // key for (*pState).queueBars - sync bool + wait struct { + bar *Bar // key for (*pState).queueBars + sync bool + } } type renderFrame struct { - reader io.Reader - lines int - shutdown bool + rows []io.Reader + shutdown int } func newBar(container *Progress, bs *bState) *Bar { @@ -339,8 +340,8 @@ func (b *Bar) Wait() { func (b *Bar) serve(ctx context.Context, bs *bState) { defer b.container.bwg.Done() - if bs.afterBar != nil && bs.sync { - bs.afterBar.Wait() + if bs.wait.bar != nil && bs.wait.sync { + bs.wait.bar.Wait() } for { select { @@ -359,48 +360,58 @@ func (b *Bar) serve(ctx context.Context, bs *bState) { func (b *Bar) render(tw int) { select { case b.operateState <- func(s *bState) { - var reader io.Reader - var lines int + var rows []io.Reader stat := newStatistics(tw, s) defer func() { // recovering if user defined decorator panics for example if p := recover(); p != nil { if s.debugOut != nil { - fmt.Fprintln(s.debugOut, p) - _, _ = s.debugOut.Write(debug.Stack()) + for _, fn := range []func() (int, error){ + func() (int, error) { + return fmt.Fprintln(s.debugOut, p) + }, + func() (int, error) { + return s.debugOut.Write(debug.Stack()) + }, + } { + if _, err := fn(); err != nil { + panic(err) + } + } } s.aborted = !s.completed s.extender = makePanicExtender(p) - reader, lines = s.extender(nil, s.reqWidth, stat) b.recoveredPanic = p } - frame := renderFrame{ - reader: reader, - lines: lines + 1, - shutdown: s.completed || s.aborted, + if fn := s.extender; fn != nil { + rows = fn(rows, s.reqWidth, stat) + } + frame := &renderFrame{ + rows: rows, } - if frame.shutdown { + if s.completed || s.aborted { b.cancel() + frame.shutdown++ } - b.frameCh <- &frame + b.frameCh <- frame }() if b.recoveredPanic == nil { - reader = s.draw(stat) + rows = append(rows, s.draw(stat)) } - reader, lines = s.extender(reader, s.reqWidth, stat) }: case <-b.done: - var reader io.Reader - var lines int - stat, s := newStatistics(tw, b.bs), b.bs + var rows []io.Reader + s, stat := b.bs, newStatistics(tw, b.bs) if b.recoveredPanic == nil { - reader = s.draw(stat) + rows = append(rows, s.draw(stat)) + } + if fn := s.extender; fn != nil { + rows = fn(rows, s.reqWidth, stat) } - reader, lines = s.extender(reader, s.reqWidth, stat) - b.frameCh <- &renderFrame{ - reader: reader, - lines: lines + 1, + frame := &renderFrame{ + rows: rows, } + b.frameCh <- frame } } @@ -446,7 +457,7 @@ func (b *Bar) wSyncTable() [][]chan int { func (s *bState) draw(stat decor.Statistics) io.Reader { bufP, bufB, bufA := s.buffers[0], s.buffers[1], s.buffers[2] - nlr := strings.NewReader("\n") + nlr := bytes.NewReader([]byte("\n")) tw := stat.AvailableWidth for _, d := range s.pDecorators { str := d.Decor(stat) @@ -596,11 +607,11 @@ func extractBaseDecorator(d decor.Decorator) decor.Decorator { func makePanicExtender(p interface{}) extenderFunc { pstr := fmt.Sprint(p) - return func(_ io.Reader, _ int, st decor.Statistics) (io.Reader, int) { - mr := io.MultiReader( - strings.NewReader(runewidth.Truncate(pstr, st.AvailableWidth, "…")), - strings.NewReader("\n"), + return func(rows []io.Reader, _ int, stat decor.Statistics) []io.Reader { + r := io.MultiReader( + strings.NewReader(runewidth.Truncate(pstr, stat.AvailableWidth, "…")), + bytes.NewReader([]byte("\n")), ) - return mr, 0 + return append(rows, r) } } diff --git a/vendor/github.com/vbauerster/mpb/v7/bar_option.go b/vendor/github.com/vbauerster/mpb/v7/bar_option.go index 8599f0a57..3506ed2f1 100644 --- a/vendor/github.com/vbauerster/mpb/v7/bar_option.go +++ b/vendor/github.com/vbauerster/mpb/v7/bar_option.go @@ -60,6 +60,7 @@ func BarWidth(width int) BarOption { } // BarQueueAfter puts this (being constructed) bar into the queue. +// BarPriority will be inherited from the argument bar. // When argument bar completes or aborts queued bar replaces its place. // If sync is true queued bar is suspended until argument bar completes // or aborts. @@ -68,8 +69,8 @@ func BarQueueAfter(bar *Bar, sync bool) BarOption { return nil } return func(s *bState) { - s.afterBar = bar - s.sync = sync + s.wait.bar = bar + s.wait.sync = sync } } @@ -111,29 +112,61 @@ func BarFillerMiddleware(middle func(BarFiller) BarFiller) BarOption { } // BarPriority sets bar's priority. Zero is highest priority, i.e. bar -// will be on top. If `BarReplaceOnComplete` option is supplied, this -// option is ignored. +// will be on top. This option isn't effective with `BarQueueAfter` option. func BarPriority(priority int) BarOption { return func(s *bState) { s.priority = priority } } -// BarExtender provides a way to extend bar to the next new line. +// BarExtender extends bar with arbitrary lines. Provided BarFiller will be +// called at each render/flush cycle. Any lines written to the underlying +// io.Writer will be printed after the bar itself. func BarExtender(filler BarFiller) BarOption { + return barExtender(filler, false) +} + +// BarExtenderRev extends bar with arbitrary lines in reverse order. Provided +// BarFiller will be called at each render/flush cycle. Any lines written +// to the underlying io.Writer will be printed before the bar itself. +func BarExtenderRev(filler BarFiller) BarOption { + return barExtender(filler, true) +} + +func barExtender(filler BarFiller, rev bool) BarOption { if filler == nil { return nil } return func(s *bState) { - s.extender = makeExtenderFunc(filler) + s.extender = makeExtenderFunc(filler, rev) } } -func makeExtenderFunc(filler BarFiller) extenderFunc { +func makeExtenderFunc(filler BarFiller, rev bool) extenderFunc { buf := new(bytes.Buffer) - return func(r io.Reader, reqWidth int, st decor.Statistics) (io.Reader, int) { - filler.Fill(buf, reqWidth, st) - return io.MultiReader(r, buf), bytes.Count(buf.Bytes(), []byte("\n")) + base := func(rows []io.Reader, width int, stat decor.Statistics) []io.Reader { + buf.Reset() + filler.Fill(buf, width, stat) + for { + b, err := buf.ReadBytes('\n') + if err != nil { + break + } + rows = append(rows, bytes.NewReader(b)) + } + return rows + } + + if !rev { + return base + } else { + return func(rows []io.Reader, width int, stat decor.Statistics) []io.Reader { + rows = base(rows, width, stat) + for left, right := 0, len(rows)-1; left < right; left, right = left+1, right-1 { + rows[left], rows[right] = rows[right], rows[left] + } + return rows + } } } diff --git a/vendor/github.com/vbauerster/mpb/v7/container_option.go b/vendor/github.com/vbauerster/mpb/v7/container_option.go index e523a1759..bfaa3286a 100644 --- a/vendor/github.com/vbauerster/mpb/v7/container_option.go +++ b/vendor/github.com/vbauerster/mpb/v7/container_option.go @@ -31,7 +31,7 @@ func WithWidth(width int) ContainerOption { } } -// WithRefreshRate overrides default 120ms refresh rate. +// WithRefreshRate overrides default 150ms refresh rate. func WithRefreshRate(d time.Duration) ContainerOption { return func(s *pState) { s.rr = d diff --git a/vendor/github.com/vbauerster/mpb/v7/cwriter/writer.go b/vendor/github.com/vbauerster/mpb/v7/cwriter/writer.go index fac15b3bc..19fd90e94 100644 --- a/vendor/github.com/vbauerster/mpb/v7/cwriter/writer.go +++ b/vendor/github.com/vbauerster/mpb/v7/cwriter/writer.go @@ -20,19 +20,30 @@ const ( // Writer is a buffered the writer that updates the terminal. The // contents of writer will be flushed when Flush is called. type Writer struct { - out io.Writer - buf bytes.Buffer - lines int - fd int - isTerminal bool + out io.Writer + buf bytes.Buffer + lines int // how much lines to clear before flushing new ones + fd int + terminal bool + termSize func(int) (int, int, error) } // New returns a new Writer with defaults. func New(out io.Writer) *Writer { - w := &Writer{out: out} + w := &Writer{ + out: out, + termSize: func(_ int) (int, int, error) { + return -1, -1, ErrNotTTY + }, + } if f, ok := out.(*os.File); ok { w.fd = int(f.Fd()) - w.isTerminal = IsTerminal(w.fd) + if IsTerminal(w.fd) { + w.terminal = true + w.termSize = func(fd int) (int, int, error) { + return GetSize(fd) + } + } } return w } @@ -67,13 +78,9 @@ func (w *Writer) ReadFrom(r io.Reader) (n int64, err error) { return w.buf.ReadFrom(r) } -// GetWidth returns width of underlying terminal. -func (w *Writer) GetWidth() (int, error) { - if !w.isTerminal { - return -1, ErrNotTTY - } - tw, _, err := GetSize(w.fd) - return tw, err +// GetTermSize returns WxH of underlying terminal. +func (w *Writer) GetTermSize() (width, height int, err error) { + return w.termSize(w.fd) } func (w *Writer) ansiCuuAndEd() error { diff --git a/vendor/github.com/vbauerster/mpb/v7/cwriter/writer_windows.go b/vendor/github.com/vbauerster/mpb/v7/cwriter/writer_windows.go index 8f99dbe32..2c4c3707b 100644 --- a/vendor/github.com/vbauerster/mpb/v7/cwriter/writer_windows.go +++ b/vendor/github.com/vbauerster/mpb/v7/cwriter/writer_windows.go @@ -16,7 +16,7 @@ var ( ) func (w *Writer) clearLines() error { - if !w.isTerminal { + if !w.terminal { // hope it's cygwin or similar return w.ansiCuuAndEd() } diff --git a/vendor/github.com/vbauerster/mpb/v7/decor/on_condition.go b/vendor/github.com/vbauerster/mpb/v7/decor/on_condition.go index a9db0653a..74a3d9667 100644 --- a/vendor/github.com/vbauerster/mpb/v7/decor/on_condition.go +++ b/vendor/github.com/vbauerster/mpb/v7/decor/on_condition.go @@ -1,27 +1,55 @@ package decor -// OnPredicate returns decorator if predicate evaluates to true. +// OnCondition applies decorator only if a condition is true. // // `decorator` Decorator // -// `predicate` func() bool +// `cond` bool // -func OnPredicate(decorator Decorator, predicate func() bool) Decorator { - if predicate() { - return decorator - } - return nil +func OnCondition(decorator Decorator, cond bool) Decorator { + return Conditional(cond, decorator, nil) } -// OnCondition returns decorator if condition is true. +// OnPredicate applies decorator only if a predicate evaluates to true. // // `decorator` Decorator // +// `predicate` func() bool +// +func OnPredicate(decorator Decorator, predicate func() bool) Decorator { + return Predicative(predicate, decorator, nil) +} + +// Conditional returns decorator `a` if condition is true, otherwise +// decorator `b`. +// // `cond` bool // -func OnCondition(decorator Decorator, cond bool) Decorator { +// `a` Decorator +// +// `b` Decorator +// +func Conditional(cond bool, a, b Decorator) Decorator { if cond { - return decorator + return a + } else { + return b + } +} + +// Predicative returns decorator `a` if predicate evaluates to true, +// otherwise decorator `b`. +// +// `predicate` func() bool +// +// `a` Decorator +// +// `b` Decorator +// +func Predicative(predicate func() bool, a, b Decorator) Decorator { + if predicate() { + return a + } else { + return b } - return nil } diff --git a/vendor/github.com/vbauerster/mpb/v7/priority_queue.go b/vendor/github.com/vbauerster/mpb/v7/priority_queue.go index 29d9bd5a8..152482e9a 100644 --- a/vendor/github.com/vbauerster/mpb/v7/priority_queue.go +++ b/vendor/github.com/vbauerster/mpb/v7/priority_queue.go @@ -6,7 +6,8 @@ type priorityQueue []*Bar func (pq priorityQueue) Len() int { return len(pq) } func (pq priorityQueue) Less(i, j int) bool { - return pq[i].priority < pq[j].priority + // less priority pops first + return pq[i].priority > pq[j].priority } func (pq priorityQueue) Swap(i, j int) { diff --git a/vendor/github.com/vbauerster/mpb/v7/progress.go b/vendor/github.com/vbauerster/mpb/v7/progress.go index 1d9a53e5c..ea5a0c15e 100644 --- a/vendor/github.com/vbauerster/mpb/v7/progress.go +++ b/vendor/github.com/vbauerster/mpb/v7/progress.go @@ -12,7 +12,6 @@ import ( "time" "github.com/vbauerster/mpb/v7/cwriter" - "github.com/vbauerster/mpb/v7/decor" ) const ( @@ -41,6 +40,7 @@ type pState struct { // following are provided/overrided by user idCount int reqWidth int + popPriority int popCompleted bool outputDiscarded bool rr time.Duration @@ -64,10 +64,11 @@ func New(options ...ContainerOption) *Progress { // method has been called. func NewWithContext(ctx context.Context, options ...ContainerOption) *Progress { s := &pState{ - bHeap: priorityQueue{}, - rr: prr, - queueBars: make(map[*Bar]*Bar), - output: os.Stdout, + bHeap: priorityQueue{}, + rr: prr, + queueBars: make(map[*Bar]*Bar), + output: os.Stdout, + popPriority: math.MinInt32, } for _, opt := range options { @@ -118,8 +119,8 @@ func (p *Progress) Add(total int64, filler BarFiller, options ...BarOption) *Bar case p.operateState <- func(ps *pState) { bs := ps.makeBarState(total, filler, options...) bar := newBar(p, bs) - if bs.afterBar != nil { - ps.queueBars[bs.afterBar] = bar + if bs.wait.bar != nil { + ps.queueBars[bs.wait.bar] = bar } else { heap.Push(&ps.bHeap, bar) ps.heapUpdated = true @@ -204,33 +205,27 @@ func (p *Progress) serve(s *pState, cw *cwriter.Writer) { p.refreshCh = s.newTicker(p.done) + render := func(debugOut io.Writer) { + err := s.render(cw) + for err != nil { + if debugOut != nil { + _, err = fmt.Fprintln(debugOut, err) + } else { + panic(err) + } + debugOut = nil + } + } + for { select { case op := <-p.operateState: op(s) case <-p.refreshCh: - if err := s.render(cw); err != nil { - if s.debugOut != nil { - _, e := fmt.Fprintln(s.debugOut, err) - if e != nil { - panic(err) - } - } else { - panic(err) - } - } + render(s.debugOut) case <-s.shutdownNotifier: for s.heapUpdated { - if err := s.render(cw); err != nil { - if s.debugOut != nil { - _, e := fmt.Fprintln(s.debugOut, err) - if e != nil { - panic(err) - } - } else { - panic(err) - } - } + render(s.debugOut) } return } @@ -245,42 +240,52 @@ func (s *pState) render(cw *cwriter.Writer) error { syncWidth(s.pMatrix) syncWidth(s.aMatrix) - tw, err := cw.GetWidth() + width, height, err := cw.GetTermSize() if err != nil { - tw = s.reqWidth + width = s.reqWidth + height = s.bHeap.Len() } for i := 0; i < s.bHeap.Len(); i++ { bar := s.bHeap[i] - go bar.render(tw) + go bar.render(width) } - return s.flush(cw) + return s.flush(cw, height) } -func (s *pState) flush(cw *cwriter.Writer) error { - var lines int +func (s *pState) flush(cw *cwriter.Writer, height int) error { + var popCount int + rows := make([]io.Reader, 0, height) pool := make([]*Bar, 0, s.bHeap.Len()) for s.bHeap.Len() > 0 { + var frameRowsUsed int b := heap.Pop(&s.bHeap).(*Bar) frame := <-b.frameCh - lines += frame.lines - _, err := cw.ReadFrom(frame.reader) - if err != nil { - return err + for i := len(frame.rows) - 1; i >= 0; i-- { + if len(rows) == height { + break + } + rows = append(rows, frame.rows[i]) + frameRowsUsed++ } - if frame.shutdown { + if frame.shutdown != 0 { b.Wait() // waiting for b.done, so it's safe to read b.bs - var toDrop bool + drop := b.bs.dropOnComplete if qb, ok := s.queueBars[b]; ok { delete(s.queueBars, b) qb.priority = b.priority pool = append(pool, qb) - toDrop = true + drop = true } else if s.popCompleted && !b.bs.noPop { - lines -= frame.lines - toDrop = true + if frame.shutdown > 1 { + popCount += frameRowsUsed + drop = true + } else { + s.popPriority++ + b.priority = s.popPriority + } } - if toDrop || b.bs.dropOnComplete { + if drop { s.heapUpdated = true continue } @@ -292,7 +297,14 @@ func (s *pState) flush(cw *cwriter.Writer) error { heap.Push(&s.bHeap, b) } - return cw.Flush(lines) + for i := len(rows) - 1; i >= 0; i-- { + _, err := cw.ReadFrom(rows[i]) + if err != nil { + return err + } + } + + return cw.Flush(len(rows) - popCount) } func (s *pState) newTicker(done <-chan struct{}) chan time.Time { @@ -358,7 +370,6 @@ func (s *pState) makeBarState(total int64, filler BarFiller, options ...BarOptio reqWidth: s.reqWidth, total: total, filler: filler, - extender: func(r io.Reader, _ int, _ decor.Statistics) (io.Reader, int) { return r, 0 }, debugOut: s.debugOut, } @@ -377,10 +388,6 @@ func (s *pState) makeBarState(total int64, filler BarFiller, options ...BarOptio bs.middleware = nil } - if s.popCompleted && !bs.noPop { - bs.priority = -(math.MaxInt32 - s.idCount) - } - for i := 0; i < len(bs.buffers); i++ { bs.buffers[i] = bytes.NewBuffer(make([]byte, 0, 512)) } diff --git a/vendor/modules.txt b/vendor/modules.txt index 20f781318..db7e9b5bb 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -720,7 +720,7 @@ github.com/ulikunitz/xz/lzma github.com/vbatts/tar-split/archive/tar github.com/vbatts/tar-split/tar/asm github.com/vbatts/tar-split/tar/storage -# github.com/vbauerster/mpb/v7 v7.4.2 +# github.com/vbauerster/mpb/v7 v7.5.2 ## explicit github.com/vbauerster/mpb/v7 github.com/vbauerster/mpb/v7/cwriter |