diff options
165 files changed, 6076 insertions, 1079 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 304efb2f4..09f13a7d0 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -5,48 +5,118 @@ # and storage. gcp_credentials: ENCRYPTED[885c6e4297dd8d6f67593c42b810353af0c505a7a670e2c6fd830c56e86bbb2debcc3c18f942d0d46ab36b63521061d4] -# Default VM to use for testing, unless values overriden by specific tasks (below) -gce_instance: - image_project: "libpod-218412" - zone: "us-central1-a" # Required by Cirrus for the time being - cpu: 2 - memory: "4Gb" - disk: 40 - -# Main collection of env. varss to set for all scripts. All others -# are cooked in by $SCRIPT_BASE/setup_environment.sh +# Default timeout for each task +timeout_in: 120m + +# Main collection of env. vars to set for all tasks and scripts. env: - FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554" - CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9" - CRIO_COMMIT: "7a283c391abb7bd25086a8ff91dbb36ebdd24466" - CRIU_COMMIT: "c74b83cd49c00589c0c0468ba5fe685b67fdbd0a" - RUNC_COMMIT: "869add33186caff4a22e3e11a7472a2d48d77889" + #### + #### Global variables used for all tasks + #### # File to update in home-dir with task-specific env. var values ENVLIB: ".bash_profile" # Overrides default location (/tmp/cirrus) for repo clone - CIRRUS_WORKING_DIR: "/go/src/github.com/containers/libpod" + CIRRUS_WORKING_DIR: "/var/tmp/go/src/github.com/containers/libpod" # Required so $ENVLIB gets loaded CIRRUS_SHELL: "/bin/bash" # Save a little typing (path relative to $CIRRUS_WORKING_DIR) SCRIPT_BASE: "./contrib/cirrus" PACKER_BASE: "./contrib/cirrus/packer" + + #### + #### Variables for composing new cache-images (used in PR testing) from + #### base-images (pre-existing in GCE) + #### + # Git commits to use while building dependencies into cache-images + FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554" + CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9" + CRIO_COMMIT: "7a283c391abb7bd25086a8ff91dbb36ebdd24466" + CRIU_COMMIT: "c74b83cd49c00589c0c0468ba5fe685b67fdbd0a" + RUNC_COMMIT: "25f3f893c86d07426df93b7aa172f33fdf093fbd" + # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) + PACKER_BUILDS: "ubuntu-18,fedora-29" # TODO: fah-29,rhel-7,centos-7 + # Version of packer to use + PACKER_VER: "1.3.1" + # Google-maintained base-image names + UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20181203a" + CENTOS_BASE_IMAGE: "centos-7-v20181113" + # Manually produced base-image names (see $SCRIPT_BASE/README.md) + FEDORA_BASE_IMAGE: "fedora-cloud-base-29-1-2-1541789245" + FAH_BASE_IMAGE: "fedora-atomichost-29-20181025-1-1541787861" + # RHEL image must be imported, google bills extra for their native image. + RHEL_BASE_IMAGE: "rhel-guest-image-7-6-210-x86-64-qcow2-1541783972" + + #### + #### Credentials and other secret-sauces, decrypted at runtime when authorized. + #### + # Freenode IRC credentials for posting status messages IRCID: ENCRYPTED[e87bba62a8e924dc70bdb2b66b16f6ab4a60d2870e6e5534ae9e2b0076f483c71c84091c655ca239101e6816c5ec0883] + # Command to register a RHEL VM to install/update packages + RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187] + # Needed to build GCE images, within a GCE VM + SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a] + # User ID for cirrus to ssh into VMs + GCE_SSH_USERNAME: ENCRYPTED[a7706b9e4b8bbb47f76358df7407f4fffa2e8552531190cc0b3315180c4b50588f560c4f85731e99cb5f43a396778277] + # Name where this repositories cloud resources are located + GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] + + # Space separated list of environment variables to unset before testing + UNSET_ENV_VARS: >- + GCP_PROJECT_ID GCE_SSH_USERNAME SERVICE_ACCOUNT RHSM_COMMAND BUILT_IMAGE_SUFFIX + IRCID RHEL_BASE_IMAGE FAH_BASE_IMAGE FEDORA_BASE_IMAGE CENTOS_BASE_IMAGE + UBUNTU_BASE_IMAGE PACKER_VER PACKER_BUILDS RUNC_COMMIT CRIU_COMMIT + CRIO_COMMIT CNI_COMMIT FEDORA_CNI_COMMIT PACKER_BASE SCRIPT_BASE + CIRRUS_SHELL CIRRUS_WORKING_DIR ENVLIB BUILT_IMAGE_SUFFIX CIRRUS_CI + CI_NODE_INDEX CI_NODE_TOTAL CIRRUS_BASE_BRANCH CIRRUS_BASE_SHA + CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO CIRRUS_CLONE_DEPTH + CIRRUS_COMMIT_MESSAGE CIRRUS_CHANGE_MESSAGE CIRRUS_REPO_CLONE_HOST + CIRRUS_DEFAULT_BRANCH CIRRUS_PR CIRRUS_TAG CIRRUS_OS CIRRUS_TASK_NAME + CIRRUS_TASK_ID CIRRUS_REPO_NAME CIRRUS_REPO_OWNER CIRRUS_REPO_FULL_NAME + CIRRUS_REPO_CLONE_URL CIRRUS_SHELL CIRRUS_USER_COLLABORATOR CIRRUS_USER_PERMISSION + CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST PACKER_BUILDS BUILT_IMAGE_SUFFIX + XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID + +# Every *_task runs in parallel in separate VMsd. The name prefix only for reference +# in WebUI, and will be followed by matrix details. This task gates all others with +# quick format, lint, and unit tests on the standard platform. +gating_task: + + env: + CIRRUS_WORKING_DIR: "/usr/src/libpod" + + # Runs within Cirrus's "community cluster" + container: + image: "quay.io/libpod/gate:latest" + cpu: 4 + memory: 12 + + gate_script: + - '/usr/local/bin/entrypoint.sh validate' + - '/usr/local/bin/entrypoint.sh lint' + -# Every *_task runs in parallel in separate VMs. The name prefix only for reference -# in WebUI, and will be followed by matrix details. This task does all the -# per-pr unit/integration testing. -full_vm_testing_task: +# This task does the unit and integration testing for every platform +testing_task: + + depends_on: + - "gating" gce_instance: - # Generate multiple 'test' tasks, covering all possible - # 'matrix' combinations. All run in parallel. + image_project: "libpod-218412" + zone: "us-central1-a" # Required by Cirrus for the time being + cpu: 2 + memory: "4Gb" + disk: 40 + # Generate multiple parallel tasks, covering all possible + # 'matrix' combinations. matrix: - # Images are generated separetly, from build_images_task (below) + # Images are generated separately, from build_images_task (below) image_name: "ubuntu-18-libpod-0c954a67" - # TODO: Make these work (also build_images_task below) - #image_name: "rhel-server-ec2-7-5-165-1-libpod-fce09afe" - #image_name: "centos-7-v20180911-libpod-fce09afe" - #image_name: "fedora-cloud-base-28-1-1-7-libpod-fce09afe" + image_name: "fedora-29-libpod-0c954a67" + # TODO: tests fail + # image_name: "rhel-7-something-something" + # image_name: "centos-7-something-something" + # image_name: "fah-29-libpod-5070733157859328" timeout_in: 120m @@ -55,11 +125,9 @@ full_vm_testing_task: setup_environment_script: $SCRIPT_BASE/setup_environment.sh # ...or lists of strings - verify_source_script: - - whoami # root! - - $SCRIPT_BASE/verify_source.sh - - unit_test_script: $SCRIPT_BASE/unit_test.sh + unit_test_script: + - go version + - $SCRIPT_BASE/unit_test.sh integration_test_script: $SCRIPT_BASE/integration_test.sh @@ -68,8 +136,7 @@ full_vm_testing_task: # Because system tests are stored within the repository, it is sometimes # necessary to execute them within a PR to validate changes. - -optional_system_testing_task: +optional_testing_task: # Only run system tests in PRs (not on merge) if magic string is present # in the PR description. Post-merge system testing is assumed to happen @@ -79,12 +146,13 @@ optional_system_testing_task: $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' gce_instance: + image_project: "libpod-218412" matrix: - image_name: "ubuntu-1804-bionic-v20180911-libpod-e8d18305" + image_name: "ubuntu-18-libpod-0c954a67" + image_name: "fedora-29-libpod-0c954a67" # TODO: Make these work (also build_images_task below) #image_name: "rhel-server-ec2-7-5-165-1-libpod-fce09afe" #image_name: "centos-7-v20180911-libpod-fce09afe" - #image_name: "fedora-cloud-base-28-1-1-7-libpod-fce09afe" timeout_in: 60m @@ -93,13 +161,11 @@ optional_system_testing_task: success_script: $SCRIPT_BASE/success.sh -# This task builds new cache-images for future PR testing. These images save -# time installing/setting up the environment while an engineer is waiting. -# The 'active' cache-images for full_vm_testing are selected by the -# 'image_name' keys. Updating those items requires manually modification, -# but this could be automated (see comment at end of build_vm_images_task). - -build_vm_images_task: +# Build new cache-images for future PR testing, but only after a PR merge. +# The cache-images save install/setup time needed test every PR. The 'active' images +# are selected by the 'image_name' items tasks above. Currently this requires +# manually updating the names, but this could be automated (see comment below). +cache_images_task: # Only produce new cache-images after a PR merge, and if a magic string # is present in the most recent commit-message. only_if: >- @@ -108,45 +174,27 @@ build_vm_images_task: # Require tests to pass first. depends_on: - - full_vm_testing # i.e. 'full_vm_testing_task' - - env: - # CSV of packer builder names to enable (see $PACKER_BASE/libpod_images.json) - PACKER_BUILDS: "ubuntu-18" - # TODO: Make these work (also full_vm_testing_task above) - # PACKER_BUILDS: "rhel-7,centos-7,fedora-28,ubuntu-18" - CENTOS_BASE_IMAGE: "centos-7-v20180911" - RHEL_BASE_IMAGE: "rhel-server-ec2-7-5-165-1" - FEDORA_BASE_IMAGE: "fedora-cloud-base-28-1-1-7" - UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20180911" - # low-level base VM image name inputs to packer - - # Command to register a RHEL VM - RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187] - # Additional environment variables needed to build GCE images, within a GCE VM - SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a] - GCE_SSH_USERNAME: ENCRYPTED[a7706b9e4b8bbb47f76358df7407f4fffa2e8552531190cc0b3315180c4b50588f560c4f85731e99cb5f43a396778277] - GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] - # Version of packer to use - PACKER_VER: "1.3.1" + - "gating" + - "testing" # VMs created by packer are not cleaned up by cirrus auto_cancellation: $CI != "true" gce_instance: - image_name: "image-builder-image" # Simply CentOS 7 + packer dependencies + image_project: "libpod-218412" + zone: "us-central1-a" # Required by Cirrus for the time being + cpu: 4 + memory: "4Gb" + disk: 20 + image_name: "image-builder-image-1541772081" # Simply CentOS 7 + packer dependencies # Additional permissions for building GCE images, within a GCE VM scopes: - compute - devstorage.full_control - # Doesn't need many local resources to run - cpu: 2 - memory: "2Gb" - disk: 20 environment_script: $SCRIPT_BASE/setup_environment.sh build_vm_images_script: $SCRIPT_BASE/build_vm_images.sh - # TODO,Continuous Delivery: Automaticly open a libpod PR after using 'sed' to replace + # TODO,Continuous Delivery: Automatically open a libpod PR after using 'sed' to replace # the image_names with the new (just build) images. That will # cause a new round of testing to happen (via the PR) using # the new images. When all is good, the PR may be manually @@ -122,6 +122,7 @@ packages: - python3-varlink - python3-dateutil - python3-psutil + - https://kojipkgs.fedoraproject.org//packages/runc/1.0.0/54.dev.git00dc700.fc28/x86_64/runc-1.0.0-54.dev.git00dc700.fc28.x86_64.rpm tests: - sed 's/^expand-check.*/expand-check=0/g' -i /etc/selinux/semanage.conf @@ -9,6 +9,14 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [func Commit(name: string, image_name: string, changes: []string, author: string, message: string, pause: bool, manifestType: string) string](#Commit) +[func ContainerCheckpoint(name: string, keep: bool, leaveRunning: bool, tcpEstablished: bool) string](#ContainerCheckpoint) + +[func ContainerExists(name: string) int](#ContainerExists) + +[func ContainerRestore(name: string, keep: bool, tcpEstablished: bool) string](#ContainerRestore) + +[func ContainerRunlabel(runlabel: Runlabel) ](#ContainerRunlabel) + [func CreateContainer(create: Create) string](#CreateContainer) [func CreateImage() NotImplemented](#CreateImage) @@ -43,6 +51,8 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [func HistoryImage(name: string) ImageHistory](#HistoryImage) +[func ImageExists(name: string) int](#ImageExists) + [func ImportImage(source: string, reference: string, message: string, changes: []string) string](#ImportImage) [func InspectContainer(name: string) string](#InspectContainer) @@ -57,6 +67,10 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [func ListContainerChanges(name: string) ContainerChanges](#ListContainerChanges) +[func ListContainerMounts() []string](#ListContainerMounts) + +[func ListContainerPorts(name: string) NotImplemented](#ListContainerPorts) + [func ListContainerProcesses(name: string, opts: []string) []string](#ListContainerProcesses) [func ListContainers() ListContainerData](#ListContainers) @@ -65,6 +79,8 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [func ListPods() ListPodData](#ListPods) +[func MountContainer(name: string) string](#MountContainer) + [func PauseContainer(name: string) string](#PauseContainer) [func PausePod(name: string) string](#PausePod) @@ -103,6 +119,8 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [func TopPod() NotImplemented](#TopPod) +[func UnmountContainer(name: string, force: bool) ](#UnmountContainer) + [func UnpauseContainer(name: string) string](#UnpauseContainer) [func UnpausePod(name: string) string](#UnpausePod) @@ -165,6 +183,8 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in [type PodmanInfo](#PodmanInfo) +[type Runlabel](#Runlabel) + [type Sockets](#Sockets) [type StringResponse](#StringResponse) @@ -211,6 +231,31 @@ attributes: _CMD, ENTRYPOINT, ENV, EXPOSE, LABEL, ONBUILD, STOPSIGNAL, USER, VOL container while it is being committed, pass a _true_ bool for the pause argument. If the container cannot be found by the ID or name provided, a (ContainerNotFound)[#ContainerNotFound] error will be returned; otherwise, the resulting image's ID will be returned as a string. +### <a name="ContainerCheckpoint"></a>func ContainerCheckpoint +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method ContainerCheckpoint(name: [string](https://godoc.org/builtin#string), keep: [bool](https://godoc.org/builtin#bool), leaveRunning: [bool](https://godoc.org/builtin#bool), tcpEstablished: [bool](https://godoc.org/builtin#bool)) [string](https://godoc.org/builtin#string)</div> +ContainerCheckPoint performs a checkpopint on a container by its name or full/partial container +ID. On successful checkpoint, the id of the checkpointed container is returned. +### <a name="ContainerExists"></a>func ContainerExists +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method ContainerExists(name: [string](https://godoc.org/builtin#string)) [int](https://godoc.org/builtin#int)</div> +ContainerExists takes a full or partial container ID or name and returns an int as to +whether the container exists in local storage. A result of 0 means the container does +exists; whereas a result of 1 means it could not be found. +### <a name="ContainerRestore"></a>func ContainerRestore +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method ContainerRestore(name: [string](https://godoc.org/builtin#string), keep: [bool](https://godoc.org/builtin#bool), tcpEstablished: [bool](https://godoc.org/builtin#bool)) [string](https://godoc.org/builtin#string)</div> +ContainerRestore restores a container that has been checkpointed. The container to be restored can +be identified by its name or full/partial container ID. A successful restore will result in the return +of the container's ID. +### <a name="ContainerRunlabel"></a>func ContainerRunlabel +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method ContainerRunlabel(runlabel: [Runlabel](#Runlabel)) </div> +ContainerRunlabel runs executes a command as described by a given container image label. ### <a name="CreateContainer"></a>func CreateContainer <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -403,6 +448,13 @@ method HistoryImage(name: [string](https://godoc.org/builtin#string)) [ImageHist HistoryImage takes the name or ID of an image and returns information about its history and layers. The returned history is in the form of an array of ImageHistory structures. If the image cannot be found, an [ImageNotFound](#ImageNotFound) error is returned. +### <a name="ImageExists"></a>func ImageExists +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method ImageExists(name: [string](https://godoc.org/builtin#string)) [int](https://godoc.org/builtin#int)</div> +ImageExists talks a full or partial image ID or name and returns an int as to whether +the image exists in local storage. An int result of 0 means the image does exist in +local storage; whereas 1 indicates the image does not exists in local storage. ### <a name="ImportImage"></a>func ImportImage <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -453,6 +505,17 @@ See also [StopPod](StopPod). method ListContainerChanges(name: [string](https://godoc.org/builtin#string)) [ContainerChanges](#ContainerChanges)</div> ListContainerChanges takes a name or ID of a container and returns changes between the container and its base image. It returns a struct of changed, deleted, and added path names. +### <a name="ListContainerMounts"></a>func ListContainerMounts +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method ListContainerMounts() [[]string](#[]string)</div> +ListContainerMounts gathers all the mounted container mount points and returns them as an array +of strings +### <a name="ListContainerPorts"></a>func ListContainerPorts +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method ListContainerPorts(name: [string](https://godoc.org/builtin#string)) [NotImplemented](#NotImplemented)</div> +This function is not implemented yet. ### <a name="ListContainerProcesses"></a>func ListContainerProcesses <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -491,6 +554,12 @@ an image currently in storage. See also [InspectImage](InspectImage). method ListPods() [ListPodData](#ListPodData)</div> ListPods returns a list of pods in no particular order. They are returned as an array of ListPodData structs. See also [GetPod](#GetPod). +### <a name="MountContainer"></a>func MountContainer +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method MountContainer(name: [string](https://godoc.org/builtin#string)) [string](https://godoc.org/builtin#string)</div> +MountContainer mounts a container by name or full/partial ID. Upon a successful mount, the destination +mount is returned as a string. ### <a name="PauseContainer"></a>func PauseContainer <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -540,7 +609,8 @@ $ varlink call -m unix:/run/podman/io.podman/io.podman.PullImage '{"name": "regi method PushImage(name: [string](https://godoc.org/builtin#string), tag: [string](https://godoc.org/builtin#string), tlsverify: [bool](https://godoc.org/builtin#bool)) [string](https://godoc.org/builtin#string)</div> PushImage takes three input arguments: the name or ID of an image, the fully-qualified destination name of the image, -and a boolean as to whether tls-verify should be used. It will return an [ImageNotFound](#ImageNotFound) error if +and a boolean as to whether tls-verify should be used (with false disabling TLS, not affecting the default behavior). +It will return an [ImageNotFound](#ImageNotFound) error if the image cannot be found in local storage; otherwise the ID of the image will be returned on success. ### <a name="RemoveContainer"></a>func RemoveContainer <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -696,6 +766,11 @@ be found, an [ImageNotFound](#ImageNotFound) error will be returned; otherwise, method TopPod() [NotImplemented](#NotImplemented)</div> This method has not been implemented yet. +### <a name="UnmountContainer"></a>func UnmountContainer +<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> + +method UnmountContainer(name: [string](https://godoc.org/builtin#string), force: [bool](https://godoc.org/builtin#bool)) </div> +UnmountContainer umounts a container by its name or full/partial container ID. ### <a name="UnpauseContainer"></a>func UnpauseContainer <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> @@ -1293,6 +1368,33 @@ insecure_registries [[]string](#[]string) store [InfoStore](#InfoStore) podman [InfoPodmanBinary](#InfoPodmanBinary) +### <a name="Runlabel"></a>type Runlabel + +Runlabel describes the required input for container runlabel + +image [string](https://godoc.org/builtin#string) + +authfile [string](https://godoc.org/builtin#string) + +certDir [string](https://godoc.org/builtin#string) + +creds [string](https://godoc.org/builtin#string) + +display [bool](https://godoc.org/builtin#bool) + +name [string](https://godoc.org/builtin#string) + +pull [bool](https://godoc.org/builtin#bool) + +signaturePolicyPath [string](https://godoc.org/builtin#string) + +tlsVerify [bool](https://godoc.org/builtin#bool) + +label [string](https://godoc.org/builtin#string) + +extraArgs [[]string](#[]string) + +opts [map[string]](#map[string]) ### <a name="Sockets"></a>type Sockets Sockets describes sockets location for a container @@ -1336,7 +1438,7 @@ ImageNotFound means the image could not be found by the provided name or ID in l NoContainerRunning means none of the containers requested are running in a command that requires a running container. ### <a name="NoContainersInPod"></a>type NoContainersInPod -NoContainersInPod means a pod has no containers on which to perform operation. It contains +NoContainersInPod means a pod has no containers on which to perform the operation. It contains the pod ID. ### <a name="PodContainerError"></a>type PodContainerError diff --git a/Dockerfile b/Dockerfile index e1d7b4649..08af0f851 100644 --- a/Dockerfile +++ b/Dockerfile @@ -52,7 +52,7 @@ ADD . /go/src/github.com/containers/libpod RUN set -x && cd /go/src/github.com/containers/libpod && make install.libseccomp.sudo # Install runc -ENV RUNC_COMMIT 869add33186caff4a22e3e11a7472a2d48d77889 +ENV RUNC_COMMIT 96ec2177ae841256168fcf76954f7177af9446eb RUN set -x \ && export GOPATH="$(mktemp -d)" \ && git clone https://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" \ @@ -1,6 +1,6 @@ GO ?= go DESTDIR ?= / -EPOCH_TEST_COMMIT ?= 1b52843cfd2ae254a6e52c74e564730f1c875c4c +EPOCH_TEST_COMMIT ?= c264da80aab889c787ca4f9b72a9739b4f6f187e HEAD ?= HEAD CHANGELOG_BASE ?= HEAD~ CHANGELOG_TARGET ?= HEAD @@ -172,6 +172,7 @@ testunit: libpodimage localunit: test/goecho/goecho varlink_generate $(GO) test -tags "$(BUILDTAGS)" -cover $(PACKAGES) + $(MAKE) -C contrib/cirrus/packer test ginkgo: ginkgo -v -tags "$(BUILDTAGS)" -cover -flakeAttempts 3 -progress -trace -noColor test/e2e/. @@ -1,7 +1,7 @@  # libpod - library for running OCI-based containers in Pods -### Latest Version: 0.11.1.1 +### Latest Version: 0.12.1 ### Status: Active Development ### Continuous Integration: [](https://cirrus-ci.com/github/containers/libpod) diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 05854c8d7..f88b0b911 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -1,5 +1,43 @@ # Release Notes +## 0.12.1 +### Features +- Rootless Podman now creates the storage.conf, libpod.conf, and mounts.conf configuration files automatically in `~/.config/containers/` for ease of reconfiguration +- The `podman pod create` command can expose ports in the pod's network namespace, allowing public services to be created in pods +- The `podman container checkpoint` command can now keep containers running after they are checkpointed with the `--leave-running` flag +- The `podman container checkpoint` and `podman container restore` commands now support the `--tcp-established` flag to checkpoint and restore containers with active TCP connections +- The `podman version` command now has a `--format` flag to produce machine-readable output +- Added the `podman container exists`, `podman pod exists`, and `podman image exists` commands to easily check for a container/pod/image, respectively, by name or ID +- The `podman ps --pod` flag now has a short alias, `-p` +- The `podman rmi` and `podman rm` commands now have a `--prune` flag to prune unused images and containers, respectively +- The `podman ps` command now has a `--sync` flag to force a sync of Podman's state against the OCI runtime, resolving some state desync errors +- Added the `podman volume` set of commands for creating and managing local-only named volumes + +### Bugfixes +- Fixed a breaking change in rootless Podman where a change in default paths caused Podman to be unable to function on systems upgraded from 0.10.x or earlier +- Fixed a bug where `podman exec` without `-t` would still use a terminal if the container was created with `-t` +- Fixed a bug where container root propogation was not being properly adjusted if volumes with root propogation set were mounted into the container +- Fixed a bug where `podman exec` could hold the container lock longer than necessary waiting for an exited container +- Fixed a bug where rootless containers using `slirp4netns` for networking were reporting using `bridge` networking in `podman inspect` +- Fixed a bug where `podman container restore -a` was attempting to restore all containers, including created and running ones. It will now only attempt to restore stopped and exited containers +- Fixed a bug where rootless Podman detached containers were not being properly cleaned up +- Fixed a bug where privileged containers were being mounted with incorrect (too restrictive) mount options such as `nodev` +- Fixed a bug where `podman stop` would throw an error attempting to stop a container that had already stopped +- Fixed a bug where `NOTIFY_SOCKET` was not properly being passed into Podman containers +- Fixed a bug where `/dev/shm` was not properly mounted in rootless containers +- Fixed a bug where rootless Podman would set up the CNI plugins for networking (despite not using them in rootless mode), potentially causing `inotify` related errors +- Fixed a bug where Podman would error on numeric GIDs that do not exist in the container's `/etc/group` +- Fixed a bug where containers in pods or created with `--net=container` were not mounting `/etc/resolv.conf` and `/etc/hosts` + +### Misc +- `podman build` now defaults the `--force-rm` flag to `true` +- Improved `podman runlabel` support for labels featuring arguments with whitespace +- Containers without a network namespace will now use the host's `resolv.conf` +- The `slirp4netns` network mode can now be used with containers running as root. It may be useful for container-in-container scenarios where the outer container does not have host networking set +- Podman now uses `inotify` to wait for container exit files to be created, instead of polling. If `inotify` cannot be used, Podman will fall back to polling to check if the file has been created +- The `podman logs` command now uses improved short-options handling, allowing its flags to be combined if desired (for example, `podman logs -lf` instead of `podman logs -l -f`) +- Hardcoded OCI hooks directories used by Podman are now deprecated; they should instead be coded into the `libpod.conf` configuration file. They can be specified as an array via `hooks_dir` + ## 0.11.1.1 ### Bugfixes - Fixed a bug where Podman was not correctly adding firewall rules for containers, preventing them from accessing the network diff --git a/changelog.txt b/changelog.txt index 7b0f0f3af..6373f219c 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,138 @@ +- Changelog for v0.12.1 (2018-12-06) + * Update release notes for 0.12.1 + * bind mount /etc/resolv.conf|hosts in pods + * Remove --sync flag from `podman rm` + * Add locking to Sync() on containers + * Add --sync flag to podman ps + * Add --sync option to podman rm + * Tests for podman volume commands + * Add "podman volume" command + * tutorial: add ostree dependency + * Pick registry to login from full image name as well + * Add ability to prune containers and images + * Invert tlsverify default in API + * set .54 version for f28 due to memory error + * Vendor in latest containers/storage + * pkg/lookup: Return ID-only pointers on ErrNo*Entries + * test for rmi with children + * libpod/container_internal_linux: Allow gids that aren't in the group file + * Don't initialize CNI when running as rootless + * correct algorithm for deleting all images + * Use runtime lockDir in BoltDB state + * test: update runc again + * vendor: update containers/storage + * create pod on the fly + * libpod/container_internal: Deprecate implicit hook directories + * Revert changes to GetDefaultStoreOptions + * Fix libpod static dir selection when graphroot changed + * podman pod exists + * Adding more varlink endpoints + * Ensure directory where we will make database exists + * Fix typo + * rootless: raise error if newuidmap/newgidmap are not installed + * Add better descriptions for validation errors in DB + * Fix gofmt and lint + * Make locks dir in unit tests + * Do not initialize locks dir in BoltDB + * Move rootless storage config into libpod + * Set default paths from DB if not explicitly overridden + * Add a struct indicating if some Runtime fields were set + * Make DB config validation an explicit step + * Move DB configuration up in runtime setup + * Add ability to retrieve runtime configuration from DB + * Add short-option handling to logs + * tests: always install runc on Ubuntu + * cirrus: update ubuntu image + * cirrus: make apt noninteractive + * Dockerfile, .cirrus.yml: update runc commit + * rootless: propagate XDG_RUNTIME_DIR to the OCI runtime + * Update ubuntu VM image w/ newer runc + * add pod short option to ps + * Add create test with --mount flag + * Only include container SizeRootFs when requested + * /dev/shm should be mounted even in rootless mode. + * disable checkpoint tests on f29 + * test, rootless: specify USER env variable + * Revert "downgrade runc due a rootless bug" + * Fix completions to work with podman run command + * hide kube command for now + * pypod create/run: ignore args for container command + * Add support for --all in pypodman ps command + * Fixes #1867 + * tests: fix NOTIFY_SOCKET test + * Fix golang formatting issues + * oci: propagate NOTIFY_SOCKET on runtime start + * test: fix test for NOTIFY_SOCKET + * Add test to ensure stopping a stopped container works + * Stopping a stopped container is not an error for Podman + * Disable mount options when running --privileged + * Vendor in latest containers/storage + * util: use fsnotify to wait for file + * vendor: update selinux + * rootless: store only subset of storage.conf + * rootless: fix cleanup + * network: allow slirp4netns mode also for root containers + * Added more checkpoint/restore test cases + * Fix podman container restore -a + * Update bash completion for checkpoint/restore + * Add '--tcp-established' to checkpoint/restore man page + * Added tcp-established to checkpoint/restore + * Remove unused CRIU_COMMIT variable + * Point CRIU_COMMIT to CRIU release 3.11 + * Updated CRIO_COMMIT to pull in new conmon for CRIU + * Use also a struct to pass options to Restore() + * _split_token(): handle None + * Use host's resolv.conf if no network namespace enabled + * rootless: add new netmode "slirp4netns" + * tests: change return type for PodmanAsUser to PodmanTestIntegration + * test: cleanup CNI network used by the tests + * exec: don't wait for pidfile when the runtime exited + * Remove mount options relatime from podman run --mount with shared + * Update test case name to podman run with --mount flag + * Add some tests for --ip flag with run and create command + * Add history and namespaceoptions to image inspect + * add podman container|image exists + * set root propagation based on volume properties + * Actually set version for podman module / pypodman + * implement --format for version command + * podman_tutorial.md typos: arguement -> argument; missing 'a' + * Load NAT modules to fix tests involving CRIU + * Vendor in latest containers/buildah + * Update checkpoint/restore man pages + * Added option to keep containers running after checkpointing + * Use a struct to pass options to Checkpoint() + * exec: always make explicit the tty value + * Allow users to expose ports from the pod to the host + * Improve speed of containers.list() + * output libpod container to kubernetes yaml + * rootless: create empty mounts.conf if it doesn't exist + * registries: check user registries file only in rootless mode + * rootless: create storage.conf when it doesn't exist + * rootless: create libpod.conf when it doesn't exist + * Don't use $HOST and $USER variables for remote + * Implement pypodman start command + * runlabel: use shlex for splitting commands + * Add a rule to compile system test in Makefile + * Fix no-new-privileges test + * The system test write with ginkgo + * Separate common used test functions and structs to test/utils + * Add version command to pypodman + * Bump gitvalidation epoch + * Bump to v0.11.2-dev + * Cirrus: Add documentation for system-testing + * Cirrus: Simplify optional system-test script + * Cirrus: Reveal magic, parallel system-testing + * libpod should know if the network is disabled + * Lint: Silence few given goconst lint warnings + * Lint: Extract constant unknownPackage + * Lint: Tests: add missing assertions + * Lint: Do not ignore errors from docker run command when selinux enabled + * Lint: InspectImage varlink api should return errors that occurred + * Lint: Exclude autogenerated files from lint test + * Lint: Update metalinter dependency + * Set --force-rm for podman build to true by default + * Vendor in latest containers/storage + - Changelog for v0.11.1.1 (2018-11-15) * Vendor in containers/storage * Add release notes for 0.11.1.1 diff --git a/cmd/podman/container.go b/cmd/podman/container.go index b6262f890..b0232c874 100644 --- a/cmd/podman/container.go +++ b/cmd/podman/container.go @@ -22,7 +22,7 @@ var ( mountCommand, pauseCommand, portCommand, - // pruneCommand, + pruneContainersCommand, refreshCommand, restartCommand, restoreCommand, diff --git a/cmd/podman/containers_prune.go b/cmd/podman/containers_prune.go new file mode 100644 index 000000000..92604e82f --- /dev/null +++ b/cmd/podman/containers_prune.go @@ -0,0 +1,74 @@ +package main + +import ( + "github.com/containers/libpod/cmd/podman/libpodruntime" + "github.com/containers/libpod/cmd/podman/shared" + "github.com/containers/libpod/libpod" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var ( + pruneContainersDescription = ` + podman container prune + + Removes all exited containers +` + + pruneContainersCommand = cli.Command{ + Name: "prune", + Usage: "Remove all stopped containers", + Description: pruneContainersDescription, + Action: pruneContainersCmd, + OnUsageError: usageErrorHandler, + } +) + +func pruneContainersCmd(c *cli.Context) error { + var ( + deleteFuncs []shared.ParallelWorkerInput + ) + + ctx := getContext() + runtime, err := libpodruntime.GetRuntime(c) + if err != nil { + return errors.Wrapf(err, "could not get runtime") + } + defer runtime.Shutdown(false) + + filter := func(c *libpod.Container) bool { + state, _ := c.State() + if state == libpod.ContainerStateStopped || (state == libpod.ContainerStateExited && err == nil && c.PodID() == "") { + return true + } + return false + } + delContainers, err := runtime.GetContainers(filter) + if err != nil { + return err + } + if len(delContainers) < 1 { + return nil + } + for _, container := range delContainers { + con := container + f := func() error { + return runtime.RemoveContainer(ctx, con, c.Bool("force")) + } + + deleteFuncs = append(deleteFuncs, shared.ParallelWorkerInput{ + ContainerID: con.ID(), + ParallelFunc: f, + }) + } + maxWorkers := shared.Parallelize("rm") + if c.GlobalIsSet("max-workers") { + maxWorkers = c.GlobalInt("max-workers") + } + logrus.Debugf("Setting maximum workers to %d", maxWorkers) + + // Run the parallel funcs + deleteErrors, errCount := shared.ParallelExecuteWorkerPool(maxWorkers, deleteFuncs) + return printParallelOutput(deleteErrors, errCount) +} diff --git a/cmd/podman/create.go b/cmd/podman/create.go index bcf830c7c..6c6bcfb41 100644 --- a/cmd/podman/create.go +++ b/cmd/podman/create.go @@ -11,6 +11,7 @@ import ( "syscall" "github.com/containers/libpod/cmd/podman/libpodruntime" + "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod/image" ann "github.com/containers/libpod/pkg/annotations" @@ -66,7 +67,7 @@ func createCmd(c *cli.Context) error { rootless.SetSkipStorageSetup(true) } - runtime, err := libpodruntime.GetContainerRuntime(c) + runtime, err := libpodruntime.GetRuntime(c) if err != nil { return errors.Wrapf(err, "error creating libpod runtime") } @@ -128,7 +129,7 @@ func createContainer(c *cli.Context, runtime *libpod.Runtime) (*libpod.Container var data *inspect.ImageData = nil if rootfs == "" && !rootless.SkipStorageSetup() { - newImage, err := runtime.ImageRuntime().New(ctx, c.Args()[0], rtc.SignaturePolicyPath, "", os.Stderr, nil, image.SigningOptions{}, false, false) + newImage, err := runtime.ImageRuntime().New(ctx, c.Args()[0], rtc.SignaturePolicyPath, "", os.Stderr, nil, image.SigningOptions{}, false) if err != nil { return nil, nil, err } @@ -375,8 +376,8 @@ func configureEntrypoint(c *cli.Context, data *inspect.ImageData) []string { return entrypoint } -func configurePod(c *cli.Context, runtime *libpod.Runtime, namespaces map[string]string) (map[string]string, error) { - pod, err := runtime.LookupPod(c.String("pod")) +func configurePod(c *cli.Context, runtime *libpod.Runtime, namespaces map[string]string, podName string) (map[string]string, error) { + pod, err := runtime.LookupPod(podName) if err != nil { return namespaces, err } @@ -409,6 +410,7 @@ func parseCreateOpts(ctx context.Context, c *cli.Context, runtime *libpod.Runtim inputCommand, command []string memoryLimit, memoryReservation, memorySwap, memoryKernel int64 blkioWeight uint16 + namespaces map[string]string ) idmappings, err := util.ParseIDMapping(c.StringSlice("uidmap"), c.StringSlice("gidmap"), c.String("subuidname"), c.String("subgidname")) if err != nil { @@ -492,12 +494,21 @@ func parseCreateOpts(ctx context.Context, c *cli.Context, runtime *libpod.Runtim return nil, errors.Errorf("--cpu-quota and --cpus cannot be set together") } + // EXPOSED PORTS + var portBindings map[nat.Port][]nat.PortBinding + if data != nil { + portBindings, err = cc.ExposedPorts(c.StringSlice("expose"), c.StringSlice("publish"), c.Bool("publish-all"), data.ContainerConfig.ExposedPorts) + if err != nil { + return nil, err + } + } + // Kernel Namespaces // TODO Fix handling of namespace from pod // Instead of integrating here, should be done in libpod // However, that also involves setting up security opts // when the pod's namespace is integrated - namespaces := map[string]string{ + namespaces = map[string]string{ "pid": c.String("pid"), "net": c.String("net"), "ipc": c.String("ipc"), @@ -505,8 +516,41 @@ func parseCreateOpts(ctx context.Context, c *cli.Context, runtime *libpod.Runtim "uts": c.String("uts"), } + originalPodName := c.String("pod") + podName := strings.Replace(originalPodName, "new:", "", 1) + // after we strip out :new, make sure there is something left for a pod name + if len(podName) < 1 && c.IsSet("pod") { + return nil, errors.Errorf("new pod name must be at least one character") + } if c.IsSet("pod") { - namespaces, err = configurePod(c, runtime, namespaces) + if strings.HasPrefix(originalPodName, "new:") { + // pod does not exist; lets make it + var podOptions []libpod.PodCreateOption + podOptions = append(podOptions, libpod.WithPodName(podName), libpod.WithInfraContainer(), libpod.WithPodCgroups()) + if len(portBindings) > 0 { + ociPortBindings, err := cc.NatToOCIPortBindings(portBindings) + if err != nil { + return nil, err + } + podOptions = append(podOptions, libpod.WithInfraContainerPorts(ociPortBindings)) + } + + podNsOptions, err := shared.GetNamespaceOptions(strings.Split(DefaultKernelNamespaces, ",")) + if err != nil { + return nil, err + } + podOptions = append(podOptions, podNsOptions...) + // make pod + pod, err := runtime.NewPod(ctx, podOptions...) + if err != nil { + return nil, err + } + logrus.Debugf("pod %s created by new container request", pod.ID()) + + // The container now cannot have port bindings; so we reset the map + portBindings = make(map[nat.Port][]nat.PortBinding) + } + namespaces, err = configurePod(c, runtime, namespaces, podName) if err != nil { return nil, err } @@ -535,7 +579,7 @@ func parseCreateOpts(ctx context.Context, c *cli.Context, runtime *libpod.Runtim // Make sure if network is set to container namespace, port binding is not also being asked for netMode := ns.NetworkMode(namespaces["net"]) if netMode.IsContainer() { - if len(c.StringSlice("publish")) > 0 || c.Bool("publish-all") { + if len(portBindings) > 0 { return nil, errors.Errorf("cannot set port bindings on an existing container network namespace") } } @@ -644,15 +688,6 @@ func parseCreateOpts(ctx context.Context, c *cli.Context, runtime *libpod.Runtim return nil, errors.Errorf("No command specified on command line or as CMD or ENTRYPOINT in this image") } - // EXPOSED PORTS - var portBindings map[nat.Port][]nat.PortBinding - if data != nil { - portBindings, err = cc.ExposedPorts(c.StringSlice("expose"), c.StringSlice("publish"), c.Bool("publish-all"), data.ContainerConfig.ExposedPorts) - if err != nil { - return nil, err - } - } - // SHM Size shmSize, err := units.FromHumanSize(c.String("shm-size")) if err != nil { @@ -746,7 +781,7 @@ func parseCreateOpts(ctx context.Context, c *cli.Context, runtime *libpod.Runtim NetMode: netMode, UtsMode: utsMode, PidMode: pidMode, - Pod: c.String("pod"), + Pod: podName, Privileged: c.Bool("privileged"), Publish: c.StringSlice("publish"), PublishAll: c.Bool("publish-all"), diff --git a/cmd/podman/create_cli.go b/cmd/podman/create_cli.go index 218e9b806..b3a30d185 100644 --- a/cmd/podman/create_cli.go +++ b/cmd/podman/create_cli.go @@ -201,12 +201,13 @@ func parseVolumesFrom(volumesFrom []string) error { } func validateVolumeHostDir(hostDir string) error { - if !filepath.IsAbs(hostDir) { - return errors.Errorf("invalid host path, must be an absolute path %q", hostDir) - } - if _, err := os.Stat(hostDir); err != nil { - return errors.Wrapf(err, "error checking path %q", hostDir) + if filepath.IsAbs(hostDir) { + if _, err := os.Stat(hostDir); err != nil { + return errors.Wrapf(err, "error checking path %q", hostDir) + } } + // If hostDir is not an absolute path, that means the user wants to create a + // named volume. This will be done later on in the code. return nil } diff --git a/cmd/podman/image.go b/cmd/podman/image.go index 418b442e3..95af36df5 100644 --- a/cmd/podman/image.go +++ b/cmd/podman/image.go @@ -13,7 +13,7 @@ var ( inspectCommand, loadCommand, lsImagesCommand, - // pruneCommand, + pruneImagesCommand, pullCommand, pushCommand, rmImageCommand, diff --git a/cmd/podman/images_prune.go b/cmd/podman/images_prune.go new file mode 100644 index 000000000..cb72a498f --- /dev/null +++ b/cmd/podman/images_prune.go @@ -0,0 +1,34 @@ +package main + +import ( + "github.com/containers/libpod/cmd/podman/libpodruntime" + "github.com/containers/libpod/cmd/podman/shared" + "github.com/pkg/errors" + "github.com/urfave/cli" +) + +var ( + pruneImagesDescription = ` + podman image prune + + Removes all unnamed images from local storage +` + + pruneImagesCommand = cli.Command{ + Name: "prune", + Usage: "Remove unused images", + Description: pruneImagesDescription, + Action: pruneImagesCmd, + OnUsageError: usageErrorHandler, + } +) + +func pruneImagesCmd(c *cli.Context) error { + runtime, err := libpodruntime.GetRuntime(c) + if err != nil { + return errors.Wrapf(err, "could not get runtime") + } + defer runtime.Shutdown(false) + + return shared.Prune(runtime.ImageRuntime()) +} diff --git a/cmd/podman/libpodruntime/runtime.go b/cmd/podman/libpodruntime/runtime.go index a4b3581be..d7a0dd931 100644 --- a/cmd/podman/libpodruntime/runtime.go +++ b/cmd/podman/libpodruntime/runtime.go @@ -11,31 +11,22 @@ import ( // GetRuntime generates a new libpod runtime configured by command line options func GetRuntime(c *cli.Context) (*libpod.Runtime, error) { - storageOpts, err := util.GetDefaultStoreOptions() - if err != nil { - return nil, err - } - return GetRuntimeWithStorageOpts(c, &storageOpts) -} + storageOpts := new(storage.StoreOptions) + options := []libpod.RuntimeOption{} -// GetContainerRuntime generates a new libpod runtime configured by command line options for containers -func GetContainerRuntime(c *cli.Context) (*libpod.Runtime, error) { - mappings, err := util.ParseIDMapping(c.StringSlice("uidmap"), c.StringSlice("gidmap"), c.String("subuidmap"), c.String("subgidmap")) + _, volumePath, err := util.GetDefaultStoreOptions() if err != nil { return nil, err } - storageOpts, err := util.GetDefaultStoreOptions() - if err != nil { - return nil, err - } - storageOpts.UIDMap = mappings.UIDMap - storageOpts.GIDMap = mappings.GIDMap - return GetRuntimeWithStorageOpts(c, &storageOpts) -} -// GetRuntime generates a new libpod runtime configured by command line options -func GetRuntimeWithStorageOpts(c *cli.Context, storageOpts *storage.StoreOptions) (*libpod.Runtime, error) { - options := []libpod.RuntimeOption{} + if c.IsSet("uidmap") || c.IsSet("gidmap") || c.IsSet("subuidmap") || c.IsSet("subgidmap") { + mappings, err := util.ParseIDMapping(c.StringSlice("uidmap"), c.StringSlice("gidmap"), c.String("subuidmap"), c.String("subgidmap")) + if err != nil { + return nil, err + } + storageOpts.UIDMap = mappings.UIDMap + storageOpts.GIDMap = mappings.GIDMap + } if c.GlobalIsSet("root") { storageOpts.GraphRoot = c.GlobalString("root") @@ -90,8 +81,8 @@ func GetRuntimeWithStorageOpts(c *cli.Context, storageOpts *storage.StoreOptions if c.GlobalIsSet("default-mounts-file") { options = append(options, libpod.WithDefaultMountsFile(c.GlobalString("default-mounts-file"))) } - if c.GlobalIsSet("hooks-dir-path") { - options = append(options, libpod.WithHooksDir(c.GlobalString("hooks-dir-path"))) + if c.GlobalIsSet("hooks-dir") { + options = append(options, libpod.WithHooksDir(c.GlobalStringSlice("hooks-dir")...)) } // TODO flag to set CNI plugins dir? @@ -104,6 +95,7 @@ func GetRuntimeWithStorageOpts(c *cli.Context, storageOpts *storage.StoreOptions if c.IsSet("infra-command") { options = append(options, libpod.WithDefaultInfraCommand(c.String("infra-command"))) } + options = append(options, libpod.WithVolumePath(volumePath)) if c.IsSet("config") { return libpod.NewRuntimeFromConfig(c.String("config"), options...) } diff --git a/cmd/podman/login.go b/cmd/podman/login.go index aa26d1466..cfdd8005b 100644 --- a/cmd/podman/login.go +++ b/cmd/podman/login.go @@ -2,13 +2,13 @@ package main import ( "bufio" - "context" "fmt" "os" "strings" "github.com/containers/image/docker" "github.com/containers/image/pkg/docker/config" + "github.com/containers/image/types" "github.com/containers/libpod/libpod/common" "github.com/pkg/errors" "github.com/urfave/cli" @@ -60,27 +60,50 @@ func loginCmd(c *cli.Context) error { if len(args) == 0 { return errors.Errorf("registry must be given") } - server := scrubServer(args[0]) + server := registryFromFullName(scrubServer(args[0])) authfile := getAuthFile(c.String("authfile")) sc := common.GetSystemContext("", authfile, false) // username of user logged in to server (if one exists) - userFromAuthFile, err := config.GetUserLoggedIn(sc, server) + userFromAuthFile, passFromAuthFile, err := config.GetAuthentication(sc, server) if err != nil { return errors.Wrapf(err, "error getting logged-in user") } - username, password, err := getUserAndPass(c.String("username"), c.String("password"), userFromAuthFile) + + ctx := getContext() + + var ( + username string + password string + ) + + if userFromAuthFile != "" { + username = userFromAuthFile + password = passFromAuthFile + fmt.Println("Authenticating with existing credentials...") + if err := docker.CheckAuth(ctx, sc, username, password, server); err == nil { + fmt.Println("Existing credentials are valid. Already logged in to", server) + return nil + } + fmt.Println("Existing credentials are invalid, please enter valid username and password") + } + + username, password, err = getUserAndPass(c.String("username"), c.String("password"), userFromAuthFile) if err != nil { return errors.Wrapf(err, "error getting username and password") } - sc.DockerInsecureSkipTLSVerify = !c.BoolT("tls-verify") + + if c.IsSet("tls-verify") { + sc.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!c.BoolT("tls-verify")) + } if c.String("cert-dir") != "" { sc.DockerCertPath = c.String("cert-dir") } - if err = docker.CheckAuth(context.TODO(), sc, username, password, server); err == nil { - if err := config.SetAuthentication(sc, server, username, password); err != nil { + if err = docker.CheckAuth(ctx, sc, username, password, server); err == nil { + // Write the new credentials to the authfile + if err = config.SetAuthentication(sc, server, username, password); err != nil { return err } } @@ -126,3 +149,14 @@ func getUserAndPass(username, password, userFromAuthFile string) (string, string } return strings.TrimSpace(username), password, err } + +// registryFromFullName gets the registry from the input. If the input is of the form +// quay.io/myuser/myimage, it will parse it and just return quay.io +// It also returns true if a full image name was given +func registryFromFullName(input string) string { + split := strings.Split(input, "/") + if len(split) > 1 { + return split[0] + } + return split[0] +} diff --git a/cmd/podman/main.go b/cmd/podman/main.go index 6be192593..280448dc8 100644 --- a/cmd/podman/main.go +++ b/cmd/podman/main.go @@ -2,13 +2,13 @@ package main import ( "fmt" + "log/syslog" "os" "os/exec" "runtime/pprof" "syscall" "github.com/containers/libpod/libpod" - "github.com/containers/libpod/pkg/hooks" _ "github.com/containers/libpod/pkg/hooks/0.1.0" "github.com/containers/libpod/pkg/rootless" "github.com/containers/libpod/version" @@ -17,7 +17,6 @@ import ( "github.com/sirupsen/logrus" lsyslog "github.com/sirupsen/logrus/hooks/syslog" "github.com/urfave/cli" - "log/syslog" ) // This is populated by the Makefile from the VERSION file @@ -103,6 +102,7 @@ func main() { umountCommand, unpauseCommand, versionCommand, + volumeCommand, waitCommand, } @@ -206,11 +206,9 @@ func main() { Usage: "path to default mounts file", Hidden: true, }, - cli.StringFlag{ - Name: "hooks-dir-path", - Usage: "set the OCI hooks directory path", - Value: hooks.DefaultDir, - Hidden: true, + cli.StringSliceFlag{ + Name: "hooks-dir", + Usage: "set the OCI hooks directory path (may be set multiple times)", }, cli.IntFlag{ Name: "max-workers", diff --git a/cmd/podman/ps.go b/cmd/podman/ps.go index 0b03388a2..7a4a80769 100644 --- a/cmd/podman/ps.go +++ b/cmd/podman/ps.go @@ -200,6 +200,10 @@ var ( Usage: "Sort output by command, created, id, image, names, runningfor, size, or status", Value: "created", }, + cli.BoolFlag{ + Name: "sync", + Usage: "Sync container state with OCI runtime", + }, } psDescription = "Prints out information about the containers" psCommand = cli.Command{ @@ -260,6 +264,7 @@ func psCmd(c *cli.Context) error { Size: c.Bool("size"), Namespace: c.Bool("namespace"), Sort: c.String("sort"), + Sync: c.Bool("sync"), } filters := c.StringSlice("filter") diff --git a/cmd/podman/pull.go b/cmd/podman/pull.go index 8fb3971bd..47130805e 100644 --- a/cmd/podman/pull.go +++ b/cmd/podman/pull.go @@ -64,7 +64,6 @@ specified, the image with the 'latest' tag (if it exists) is pulled // pullCmd gets the data from the command line and calls pullImage // to copy an image from a registry to a local machine func pullCmd(c *cli.Context) error { - forceSecure := false runtime, err := libpodruntime.GetRuntime(c) if err != nil { return errors.Wrapf(err, "could not get runtime") @@ -104,12 +103,11 @@ func pullCmd(c *cli.Context) error { } dockerRegistryOptions := image2.DockerRegistryOptions{ - DockerRegistryCreds: registryCreds, - DockerCertPath: c.String("cert-dir"), - DockerInsecureSkipTLSVerify: !c.BoolT("tls-verify"), + DockerRegistryCreds: registryCreds, + DockerCertPath: c.String("cert-dir"), } if c.IsSet("tls-verify") { - forceSecure = c.Bool("tls-verify") + dockerRegistryOptions.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!c.BoolT("tls-verify")) } // Possible for docker-archive to have multiple tags, so use LoadFromArchiveReference instead @@ -125,7 +123,7 @@ func pullCmd(c *cli.Context) error { imgID = newImage[0].ID() } else { authfile := getAuthFile(c.String("authfile")) - newImage, err := runtime.ImageRuntime().New(getContext(), image, c.String("signature-policy"), authfile, writer, &dockerRegistryOptions, image2.SigningOptions{}, true, forceSecure) + newImage, err := runtime.ImageRuntime().New(getContext(), image, c.String("signature-policy"), authfile, writer, &dockerRegistryOptions, image2.SigningOptions{}, true) if err != nil { return errors.Wrapf(err, "error pulling image %q", image) } diff --git a/cmd/podman/push.go b/cmd/podman/push.go index 331f92cd2..82589f3f1 100644 --- a/cmd/podman/push.go +++ b/cmd/podman/push.go @@ -81,7 +81,6 @@ func pushCmd(c *cli.Context) error { var ( registryCreds *types.DockerAuthConfig destName string - forceSecure bool ) args := c.Args() @@ -108,7 +107,6 @@ func pushCmd(c *cli.Context) error { } certPath := c.String("cert-dir") - skipVerify := !c.BoolT("tls-verify") removeSignatures := c.Bool("remove-signatures") signBy := c.String("sign-by") @@ -145,14 +143,12 @@ func pushCmd(c *cli.Context) error { } } - if c.IsSet("tls-verify") { - forceSecure = c.Bool("tls-verify") - } - dockerRegistryOptions := image.DockerRegistryOptions{ - DockerRegistryCreds: registryCreds, - DockerCertPath: certPath, - DockerInsecureSkipTLSVerify: skipVerify, + DockerRegistryCreds: registryCreds, + DockerCertPath: certPath, + } + if c.IsSet("tls-verify") { + dockerRegistryOptions.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!c.BoolT("tls-verify")) } so := image.SigningOptions{ @@ -167,5 +163,5 @@ func pushCmd(c *cli.Context) error { authfile := getAuthFile(c.String("authfile")) - return newImage.PushImageToHeuristicDestination(getContext(), destName, manifestType, authfile, c.String("signature-policy"), writer, c.Bool("compress"), so, &dockerRegistryOptions, forceSecure, nil) + return newImage.PushImageToHeuristicDestination(getContext(), destName, manifestType, authfile, c.String("signature-policy"), writer, c.Bool("compress"), so, &dockerRegistryOptions, nil) } diff --git a/cmd/podman/rmi.go b/cmd/podman/rmi.go index c0a0d69df..0f4f8765b 100644 --- a/cmd/podman/rmi.go +++ b/cmd/podman/rmi.go @@ -91,8 +91,23 @@ func rmiCmd(c *cli.Context) error { if err != nil { return errors.Wrapf(err, "unable to query local images") } - for _, i := range imagesToDelete { - removeImage(i) + lastNumberofImages := 0 + for len(imagesToDelete) > 0 { + if lastNumberofImages == len(imagesToDelete) { + return errors.New("unable to delete all images; re-run the rmi command again.") + } + for _, i := range imagesToDelete { + isParent, err := i.IsParent() + if err != nil { + return err + } + if isParent { + continue + } + removeImage(i) + } + lastNumberofImages = len(imagesToDelete) + imagesToDelete, err = runtime.ImageRuntime().GetImages() } } else { // Create image.image objects for deletion from user input. diff --git a/cmd/podman/run.go b/cmd/podman/run.go index af6ced45d..a4b5c918e 100644 --- a/cmd/podman/run.go +++ b/cmd/podman/run.go @@ -44,7 +44,7 @@ func runCmd(c *cli.Context) error { rootless.SetSkipStorageSetup(true) } - runtime, err := libpodruntime.GetContainerRuntime(c) + runtime, err := libpodruntime.GetRuntime(c) if err != nil { return errors.Wrapf(err, "error creating libpod runtime") } diff --git a/cmd/podman/runlabel.go b/cmd/podman/runlabel.go index e1dee1fb2..48a296260 100644 --- a/cmd/podman/runlabel.go +++ b/cmd/podman/runlabel.go @@ -10,7 +10,6 @@ import ( "github.com/containers/libpod/cmd/podman/libpodruntime" "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/libpod/image" - "github.com/containers/libpod/pkg/util" "github.com/containers/libpod/utils" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -94,7 +93,7 @@ func runlabelCmd(c *cli.Context) error { imageName string stdErr, stdOut io.Writer stdIn io.Reader - newImage *image.Image + extraArgs []string ) // Evil images could trick into recursively executing the runlabel @@ -124,6 +123,9 @@ func runlabelCmd(c *cli.Context) error { return errors.Errorf("the display and quiet flags cannot be used together.") } + if len(args) > 2 { + extraArgs = args[2:] + } pull := c.Bool("pull") label := args[0] @@ -151,75 +153,26 @@ func runlabelCmd(c *cli.Context) error { stdIn = nil } - if pull { - var registryCreds *types.DockerAuthConfig - if c.IsSet("creds") { - creds, err := util.ParseRegistryCreds(c.String("creds")) - if err != nil { - return err - } - registryCreds = creds - } - dockerRegistryOptions := image.DockerRegistryOptions{ - DockerRegistryCreds: registryCreds, - DockerCertPath: c.String("cert-dir"), - DockerInsecureSkipTLSVerify: !c.BoolT("tls-verify"), - } - authfile := getAuthFile(c.String("authfile")) - - newImage, err = runtime.ImageRuntime().New(ctx, runlabelImage, c.String("signature-policy"), authfile, stdOut, &dockerRegistryOptions, image.SigningOptions{}, false, false) - } else { - newImage, err = runtime.ImageRuntime().NewFromLocal(runlabelImage) + dockerRegistryOptions := image.DockerRegistryOptions{ + DockerCertPath: c.String("cert-dir"), } - if err != nil { - return errors.Wrapf(err, "unable to find image") + if c.IsSet("tls-verify") { + dockerRegistryOptions.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!c.BoolT("tls-verify")) } - if len(newImage.Names()) < 1 { - imageName = newImage.ID() - } else { - imageName = newImage.Names()[0] - } - - runLabel, err := newImage.GetLabel(ctx, label) + authfile := getAuthFile(c.String("authfile")) + runLabel, imageName, err := shared.GetRunlabel(label, runlabelImage, ctx, runtime, pull, c.String("creds"), dockerRegistryOptions, authfile, c.String("signature-policy"), stdOut) if err != nil { return err } - - // If no label to execute, we return if runLabel == "" { return nil } - // The user provided extra arguments that need to be tacked onto the label's command - if len(args) > 2 { - runLabel = fmt.Sprintf("%s %s", runLabel, strings.Join(args[2:], " ")) - } - - cmd, err := shared.GenerateCommand(runLabel, imageName, c.String("name")) + cmd, env, err := shared.GenerateRunlabelCommand(runLabel, imageName, c.String("name"), opts, extraArgs) if err != nil { - return errors.Wrapf(err, "unable to generate command") - } - env := shared.GenerateRunEnvironment(c.String("name"), imageName, opts) - env = append(env, "PODMAN_RUNLABEL_NESTED=1") - - envmap := envSliceToMap(env) - - envmapper := func(k string) string { - switch k { - case "OPT1": - return envmap["OPT1"] - case "OPT2": - return envmap["OPT2"] - case "OPT3": - return envmap["OPT3"] - } - return "" + return err } - - newS := os.Expand(strings.Join(cmd, " "), envmapper) - cmd = strings.Split(newS, " ") - if !c.Bool("quiet") { fmt.Printf("Command: %s\n", strings.Join(cmd, " ")) if c.Bool("display") { @@ -228,12 +181,3 @@ func runlabelCmd(c *cli.Context) error { } return utils.ExecCmdWithStdStreams(stdIn, stdOut, stdErr, env, cmd[0], cmd[1:]...) } - -func envSliceToMap(env []string) map[string]string { - m := make(map[string]string) - for _, i := range env { - split := strings.Split(i, "=") - m[split[0]] = strings.Join(split[1:], " ") - } - return m -} diff --git a/cmd/podman/save.go b/cmd/podman/save.go index 7edc42e0d..139f3918a 100644 --- a/cmd/podman/save.go +++ b/cmd/podman/save.go @@ -146,7 +146,7 @@ func saveCmd(c *cli.Context) error { return err } } - if err := newImage.PushImageToReference(getContext(), destRef, manifestType, "", "", writer, c.Bool("compress"), libpodImage.SigningOptions{}, &libpodImage.DockerRegistryOptions{}, false, additionaltags); err != nil { + if err := newImage.PushImageToReference(getContext(), destRef, manifestType, "", "", writer, c.Bool("compress"), libpodImage.SigningOptions{}, &libpodImage.DockerRegistryOptions{}, additionaltags); err != nil { if err2 := os.Remove(output); err2 != nil { logrus.Errorf("error deleting %q: %v", output, err) } diff --git a/cmd/podman/search.go b/cmd/podman/search.go index fa11dad32..442ebb57f 100644 --- a/cmd/podman/search.go +++ b/cmd/podman/search.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/containers/image/docker" + "github.com/containers/image/types" "github.com/containers/libpod/cmd/podman/formats" "github.com/containers/libpod/libpod/common" sysreg "github.com/containers/libpod/pkg/registries" @@ -72,11 +73,12 @@ type searchParams struct { } type searchOpts struct { - filter []string - limit int - noTrunc bool - format string - authfile string + filter []string + limit int + noTrunc bool + format string + authfile string + insecureSkipTLSVerify types.OptionalBool } type searchFilterParams struct { @@ -116,7 +118,10 @@ func searchCmd(c *cli.Context) error { filter: c.StringSlice("filter"), authfile: getAuthFile(c.String("authfile")), } - regAndSkipTLS, err := getRegistriesAndSkipTLS(c, registry) + if c.IsSet("tls-verify") { + opts.insecureSkipTLSVerify = types.NewOptionalBool(!c.BoolT("tls-verify")) + } + registries, err := getRegistries(registry) if err != nil { return err } @@ -126,7 +131,7 @@ func searchCmd(c *cli.Context) error { return err } - return generateSearchOutput(term, regAndSkipTLS, opts, *filter) + return generateSearchOutput(term, registries, opts, *filter) } func genSearchFormat(format string) string { @@ -157,16 +162,8 @@ func (s *searchParams) headerMap() map[string]string { return values } -// A function for finding which registries can skip TLS -func getRegistriesAndSkipTLS(c *cli.Context, registry string) (map[string]bool, error) { - // Variables for setting up Registry and TLSVerify - tlsVerify := c.BoolT("tls-verify") - forceSecure := false - - if c.IsSet("tls-verify") { - forceSecure = c.BoolT("tls-verify") - } - +// getRegistries returns the list of registries to search, depending on an optional registry specification +func getRegistries(registry string) ([]string, error) { var registries []string if registry != "" { registries = append(registries, registry) @@ -177,35 +174,10 @@ func getRegistriesAndSkipTLS(c *cli.Context, registry string) (map[string]bool, return nil, errors.Wrapf(err, "error getting registries to search") } } - regAndSkipTLS := make(map[string]bool) - // If tls-verify is set to false, allow insecure always. - if !tlsVerify { - for _, reg := range registries { - regAndSkipTLS[reg] = true - } - } else { - // initially set all registries to verify with TLS - for _, reg := range registries { - regAndSkipTLS[reg] = false - } - // if the user didn't allow nor disallow insecure registries, check to see if the registry is insecure - if !forceSecure { - insecureRegistries, err := sysreg.GetInsecureRegistries() - if err != nil { - return nil, errors.Wrapf(err, "error getting insecure registries to search") - } - for _, reg := range insecureRegistries { - // if there are any insecure registries in registries, allow for HTTP - if _, ok := regAndSkipTLS[reg]; ok { - regAndSkipTLS[reg] = true - } - } - } - } - return regAndSkipTLS, nil + return registries, nil } -func getSearchOutput(term string, regAndSkipTLS map[string]bool, opts searchOpts, filter searchFilterParams) ([]searchParams, error) { +func getSearchOutput(term string, registries []string, opts searchOpts, filter searchFilterParams) ([]searchParams, error) { // Max number of queries by default is 25 limit := maxQueries if opts.limit != 0 { @@ -213,10 +185,10 @@ func getSearchOutput(term string, regAndSkipTLS map[string]bool, opts searchOpts } sc := common.GetSystemContext("", opts.authfile, false) + sc.DockerInsecureSkipTLSVerify = opts.insecureSkipTLSVerify + sc.SystemRegistriesConfPath = sysreg.SystemRegistriesConfPath() // FIXME: Set this more globally. Probably no reason not to have it in every types.SystemContext, and to compute the value just once in one place. var paramsArr []searchParams - for reg, skipTLS := range regAndSkipTLS { - // set the SkipTLSVerify bool depending on the registry being searched through - sc.DockerInsecureSkipTLSVerify = skipTLS + for _, reg := range registries { results, err := docker.SearchRegistry(context.TODO(), sc, reg, term, limit) if err != nil { logrus.Errorf("error searching registry %q: %v", reg, err) @@ -276,8 +248,8 @@ func getSearchOutput(term string, regAndSkipTLS map[string]bool, opts searchOpts return paramsArr, nil } -func generateSearchOutput(term string, regAndSkipTLS map[string]bool, opts searchOpts, filter searchFilterParams) error { - searchOutput, err := getSearchOutput(term, regAndSkipTLS, opts, filter) +func generateSearchOutput(term string, registries []string, opts searchOpts, filter searchFilterParams) error { + searchOutput, err := getSearchOutput(term, registries, opts, filter) if err != nil { return err } diff --git a/cmd/podman/shared/container.go b/cmd/podman/shared/container.go index 4404268d4..6236d19b4 100644 --- a/cmd/podman/shared/container.go +++ b/cmd/podman/shared/container.go @@ -1,10 +1,10 @@ package shared import ( + "context" "encoding/json" "fmt" - "github.com/cri-o/ocicni/pkg/ocicni" - "github.com/docker/go-units" + "io" "os" "path/filepath" "regexp" @@ -13,9 +13,14 @@ import ( "sync" "time" + "github.com/containers/image/types" "github.com/containers/libpod/libpod" + "github.com/containers/libpod/libpod/image" "github.com/containers/libpod/pkg/inspect" cc "github.com/containers/libpod/pkg/spec" + "github.com/containers/libpod/pkg/util" + "github.com/cri-o/ocicni/pkg/ocicni" + "github.com/docker/go-units" "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -40,6 +45,7 @@ type PsOptions struct { Sort string Label string Namespace bool + Sync bool } // BatchContainerStruct is the return obkect from BatchContainer and contains @@ -121,6 +127,12 @@ func NewBatchContainer(ctr *libpod.Container, opts PsOptions) (PsContainerOutput pso PsContainerOutput ) batchErr := ctr.Batch(func(c *libpod.Container) error { + if opts.Sync { + if err := c.Sync(); err != nil { + return err + } + } + conState, err = c.State() if err != nil { return errors.Wrapf(err, "unable to obtain container state") @@ -589,3 +601,79 @@ func portsToString(ports []ocicni.PortMapping) string { } return strings.Join(portDisplay, ", ") } + +// GetRunlabel is a helper function for runlabel; it gets the image if needed and begins the +// contruction of the runlabel output and environment variables +func GetRunlabel(label string, runlabelImage string, ctx context.Context, runtime *libpod.Runtime, pull bool, inputCreds string, dockerRegistryOptions image.DockerRegistryOptions, authfile string, signaturePolicyPath string, output io.Writer) (string, string, error) { + var ( + newImage *image.Image + err error + imageName string + ) + if pull { + var registryCreds *types.DockerAuthConfig + if inputCreds != "" { + creds, err := util.ParseRegistryCreds(inputCreds) + if err != nil { + return "", "", err + } + registryCreds = creds + } + dockerRegistryOptions.DockerRegistryCreds = registryCreds + newImage, err = runtime.ImageRuntime().New(ctx, runlabelImage, signaturePolicyPath, authfile, output, &dockerRegistryOptions, image.SigningOptions{}, false) + } else { + newImage, err = runtime.ImageRuntime().NewFromLocal(runlabelImage) + } + if err != nil { + return "", "", errors.Wrapf(err, "unable to find image") + } + + if len(newImage.Names()) < 1 { + imageName = newImage.ID() + } else { + imageName = newImage.Names()[0] + } + + runLabel, err := newImage.GetLabel(ctx, label) + return runLabel, imageName, err +} + +// GenerateRunlabelCommand generates the command that will eventually be execucted by podman +func GenerateRunlabelCommand(runLabel, imageName, name string, opts map[string]string, extraArgs []string) ([]string, []string, error) { + // The user provided extra arguments that need to be tacked onto the label's command + if len(extraArgs) > 0 { + runLabel = fmt.Sprintf("%s %s", runLabel, strings.Join(extraArgs, " ")) + } + cmd, err := GenerateCommand(runLabel, imageName, name) + if err != nil { + return nil, nil, errors.Wrapf(err, "unable to generate command") + } + env := GenerateRunEnvironment(name, imageName, opts) + env = append(env, "PODMAN_RUNLABEL_NESTED=1") + + envmap := envSliceToMap(env) + + envmapper := func(k string) string { + switch k { + case "OPT1": + return envmap["OPT1"] + case "OPT2": + return envmap["OPT2"] + case "OPT3": + return envmap["OPT3"] + } + return "" + } + newS := os.Expand(strings.Join(cmd, " "), envmapper) + cmd = strings.Split(newS, " ") + return cmd, env, nil +} + +func envSliceToMap(env []string) map[string]string { + m := make(map[string]string) + for _, i := range env { + split := strings.Split(i, "=") + m[split[0]] = strings.Join(split[1:], " ") + } + return m +} diff --git a/cmd/podman/shared/prune.go b/cmd/podman/shared/prune.go new file mode 100644 index 000000000..90cfe4475 --- /dev/null +++ b/cmd/podman/shared/prune.go @@ -0,0 +1,24 @@ +package shared + +import ( + "fmt" + "github.com/pkg/errors" + + "github.com/containers/libpod/libpod/image" +) + +// Prune removes all unnamed and unused images from the local store +func Prune(ir *image.Runtime) error { + pruneImages, err := ir.GetPruneImages() + if err != nil { + return err + } + + for _, i := range pruneImages { + if err := i.Remove(true); err != nil { + return errors.Wrapf(err, "failed to remove %s", i.ID()) + } + fmt.Println(i.ID()) + } + return nil +} diff --git a/cmd/podman/utils.go b/cmd/podman/utils.go index 5735156c2..a59535b43 100644 --- a/cmd/podman/utils.go +++ b/cmd/podman/utils.go @@ -3,6 +3,9 @@ package main import ( "context" "fmt" + "os" + gosignal "os/signal" + "github.com/containers/libpod/libpod" "github.com/docker/docker/pkg/signal" "github.com/docker/docker/pkg/term" @@ -11,8 +14,6 @@ import ( "github.com/urfave/cli" "golang.org/x/crypto/ssh/terminal" "k8s.io/client-go/tools/remotecommand" - "os" - gosignal "os/signal" ) type RawTtyFormatter struct { @@ -208,6 +209,35 @@ func getPodsFromContext(c *cli.Context, r *libpod.Runtime) ([]*libpod.Pod, error return pods, lastError } +func getVolumesFromContext(c *cli.Context, r *libpod.Runtime) ([]*libpod.Volume, error) { + args := c.Args() + var ( + vols []*libpod.Volume + lastError error + err error + ) + + if c.Bool("all") { + vols, err = r.Volumes() + if err != nil { + return nil, errors.Wrapf(err, "unable to get all volumes") + } + } + + for _, i := range args { + vol, err := r.GetVolume(i) + if err != nil { + if lastError != nil { + logrus.Errorf("%q", lastError) + } + lastError = errors.Wrapf(err, "unable to find volume %s", i) + continue + } + vols = append(vols, vol) + } + return vols, lastError +} + //printParallelOutput takes the map of parallel worker results and outputs them // to stdout func printParallelOutput(m map[string]error, errCount int) error { diff --git a/cmd/podman/varlink/io.podman.varlink b/cmd/podman/varlink/io.podman.varlink index 4a4a1854c..486f4e60c 100644 --- a/cmd/podman/varlink/io.podman.varlink +++ b/cmd/podman/varlink/io.podman.varlink @@ -371,6 +371,22 @@ type PodContainerErrorData ( reason: string ) +# Runlabel describes the required input for container runlabel +type Runlabel( + image: string, + authfile: string, + certDir: string, + creds: string, + display: bool, + name: string, + pull: bool, + signaturePolicyPath: string, + tlsVerify: bool, + label: string, + extraArgs: []string, + opts: [string]string +) + # Ping provides a response for developers to ensure their varlink setup is working. # #### Example # ~~~ @@ -594,7 +610,8 @@ method InspectImage(name: string) -> (image: string) method HistoryImage(name: string) -> (history: []ImageHistory) # PushImage takes three input arguments: the name or ID of an image, the fully-qualified destination name of the image, -# and a boolean as to whether tls-verify should be used. It will return an [ImageNotFound](#ImageNotFound) error if +# and a boolean as to whether tls-verify should be used (with false disabling TLS, not affecting the default behavior). +# It will return an [ImageNotFound](#ImageNotFound) error if # the image cannot be found in local storage; otherwise the ID of the image will be returned on success. method PushImage(name: string, tag: string, tlsverify: bool) -> (image: string) @@ -804,6 +821,42 @@ method TopPod() -> (notimplemented: NotImplemented) # ~~~ method GetPodStats(name: string) -> (pod: string, containers: []ContainerStats) +# ImageExists talks a full or partial image ID or name and returns an int as to whether +# the image exists in local storage. An int result of 0 means the image does exist in +# local storage; whereas 1 indicates the image does not exists in local storage. +method ImageExists(name: string) -> (exists: int) + +# ContainerExists takes a full or partial container ID or name and returns an int as to +# whether the container exists in local storage. A result of 0 means the container does +# exists; whereas a result of 1 means it could not be found. +method ContainerExists(name: string) -> (exists: int) + +# ContainerCheckPoint performs a checkpopint on a container by its name or full/partial container +# ID. On successful checkpoint, the id of the checkpointed container is returned. +method ContainerCheckpoint(name: string, keep: bool, leaveRunning: bool, tcpEstablished: bool) -> (id: string) + +# ContainerRestore restores a container that has been checkpointed. The container to be restored can +# be identified by its name or full/partial container ID. A successful restore will result in the return +# of the container's ID. +method ContainerRestore(name: string, keep: bool, tcpEstablished: bool) -> (id: string) + +# ContainerRunlabel runs executes a command as described by a given container image label. +method ContainerRunlabel(runlabel: Runlabel) -> () + +# ListContainerMounts gathers all the mounted container mount points and returns them as an array +# of strings +method ListContainerMounts() -> (mounts: []string) + +# MountContainer mounts a container by name or full/partial ID. Upon a successful mount, the destination +# mount is returned as a string. +method MountContainer(name: string) -> (path: string) + +# UnmountContainer umounts a container by its name or full/partial container ID. +method UnmountContainer(name: string, force: bool) -> () + +# This function is not implemented yet. +method ListContainerPorts(name: string) -> (notimplemented: NotImplemented) + # ImageNotFound means the image could not be found by the provided name or ID in local storage. error ImageNotFound (name: string) diff --git a/cmd/podman/volume.go b/cmd/podman/volume.go new file mode 100644 index 000000000..913592e74 --- /dev/null +++ b/cmd/podman/volume.go @@ -0,0 +1,26 @@ +package main + +import ( + "github.com/urfave/cli" +) + +var ( + volumeDescription = `Manage volumes. + +Volumes are created in and can be shared between containers.` + + volumeSubCommands = []cli.Command{ + volumeCreateCommand, + volumeLsCommand, + volumeRmCommand, + volumeInspectCommand, + volumePruneCommand, + } + volumeCommand = cli.Command{ + Name: "volume", + Usage: "Manage volumes", + Description: volumeDescription, + UseShortOptionHandling: true, + Subcommands: volumeSubCommands, + } +) diff --git a/cmd/podman/volume_create.go b/cmd/podman/volume_create.go new file mode 100644 index 000000000..0b5f8d1e3 --- /dev/null +++ b/cmd/podman/volume_create.go @@ -0,0 +1,97 @@ +package main + +import ( + "fmt" + + "github.com/containers/libpod/cmd/podman/libpodruntime" + "github.com/containers/libpod/libpod" + "github.com/pkg/errors" + "github.com/urfave/cli" +) + +var volumeCreateDescription = ` +podman volume create + +Creates a new volume. If using the default driver, "local", the volume will +be created at.` + +var volumeCreateFlags = []cli.Flag{ + cli.StringFlag{ + Name: "driver", + Usage: "Specify volume driver name (default local)", + }, + cli.StringSliceFlag{ + Name: "label, l", + Usage: "Set metadata for a volume (default [])", + }, + cli.StringSliceFlag{ + Name: "opt, o", + Usage: "Set driver specific options (default [])", + }, +} + +var volumeCreateCommand = cli.Command{ + Name: "create", + Usage: "Create a new volume", + Description: volumeCreateDescription, + Flags: volumeCreateFlags, + Action: volumeCreateCmd, + SkipArgReorder: true, + ArgsUsage: "[VOLUME-NAME]", + UseShortOptionHandling: true, +} + +func volumeCreateCmd(c *cli.Context) error { + var ( + options []libpod.VolumeCreateOption + err error + volName string + ) + + if err = validateFlags(c, volumeCreateFlags); err != nil { + return err + } + + runtime, err := libpodruntime.GetRuntime(c) + if err != nil { + return errors.Wrapf(err, "error creating libpod runtime") + } + defer runtime.Shutdown(false) + + if len(c.Args()) > 1 { + return errors.Errorf("too many arguments, create takes at most 1 argument") + } + + if len(c.Args()) > 0 { + volName = c.Args()[0] + options = append(options, libpod.WithVolumeName(volName)) + } + + if c.IsSet("driver") { + options = append(options, libpod.WithVolumeDriver(c.String("driver"))) + } + + labels, err := getAllLabels([]string{}, c.StringSlice("label")) + if err != nil { + return errors.Wrapf(err, "unable to process labels") + } + if len(labels) != 0 { + options = append(options, libpod.WithVolumeLabels(labels)) + } + + opts, err := getAllLabels([]string{}, c.StringSlice("opt")) + if err != nil { + return errors.Wrapf(err, "unable to process options") + } + if len(options) != 0 { + options = append(options, libpod.WithVolumeOptions(opts)) + } + + vol, err := runtime.NewVolume(getContext(), options...) + if err != nil { + return err + } + fmt.Printf("%s\n", vol.Name()) + + return nil +} diff --git a/cmd/podman/volume_inspect.go b/cmd/podman/volume_inspect.go new file mode 100644 index 000000000..152f1d098 --- /dev/null +++ b/cmd/podman/volume_inspect.go @@ -0,0 +1,63 @@ +package main + +import ( + "github.com/containers/libpod/cmd/podman/libpodruntime" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var volumeInspectDescription = ` +podman volume inspect + +Display detailed information on one or more volumes. Can change the format +from JSON to a Go template. +` + +var volumeInspectFlags = []cli.Flag{ + cli.BoolFlag{ + Name: "all, a", + Usage: "Inspect all volumes", + }, + cli.StringFlag{ + Name: "format, f", + Usage: "Format volume output using Go template", + Value: "json", + }, +} + +var volumeInspectCommand = cli.Command{ + Name: "inspect", + Usage: "Display detailed information on one or more volumes", + Description: volumeInspectDescription, + Flags: volumeInspectFlags, + Action: volumeInspectCmd, + SkipArgReorder: true, + ArgsUsage: "[VOLUME-NAME ...]", + UseShortOptionHandling: true, +} + +func volumeInspectCmd(c *cli.Context) error { + var err error + + if err = validateFlags(c, volumeInspectFlags); err != nil { + return err + } + + runtime, err := libpodruntime.GetRuntime(c) + if err != nil { + return errors.Wrapf(err, "error creating libpod runtime") + } + defer runtime.Shutdown(false) + + opts := volumeLsOptions{ + Format: c.String("format"), + } + + vols, lastError := getVolumesFromContext(c, runtime) + if lastError != nil { + logrus.Errorf("%q", lastError) + } + + return generateVolLsOutput(vols, opts, runtime) +} diff --git a/cmd/podman/volume_ls.go b/cmd/podman/volume_ls.go new file mode 100644 index 000000000..0f94549ee --- /dev/null +++ b/cmd/podman/volume_ls.go @@ -0,0 +1,308 @@ +package main + +import ( + "reflect" + "strings" + + "github.com/containers/libpod/cmd/podman/formats" + "github.com/containers/libpod/cmd/podman/libpodruntime" + "github.com/containers/libpod/libpod" + "github.com/pkg/errors" + "github.com/urfave/cli" +) + +// volumeOptions is the "ls" command options +type volumeLsOptions struct { + Format string + Quiet bool +} + +// volumeLsTemplateParams is the template parameters to list the volumes +type volumeLsTemplateParams struct { + Name string + Labels string + MountPoint string + Driver string + Options string + Scope string +} + +// volumeLsJSONParams is the JSON parameters to list the volumes +type volumeLsJSONParams struct { + Name string `json:"name"` + Labels map[string]string `json:"labels"` + MountPoint string `json:"mountPoint"` + Driver string `json:"driver"` + Options map[string]string `json:"options"` + Scope string `json:"scope"` +} + +var volumeLsDescription = ` +podman volume ls + +List all available volumes. The output of the volumes can be filtered +and the output format can be changed to JSON or a user specified Go template. +` + +var volumeLsFlags = []cli.Flag{ + cli.StringFlag{ + Name: "filter, f", + Usage: "Filter volume output", + }, + cli.StringFlag{ + Name: "format", + Usage: "Format volume output using Go template", + Value: "table {{.Driver}}\t{{.Name}}", + }, + cli.BoolFlag{ + Name: "quiet, q", + Usage: "Print volume output in quiet mode", + }, +} + +var volumeLsCommand = cli.Command{ + Name: "ls", + Aliases: []string{"list"}, + Usage: "List volumes", + Description: volumeLsDescription, + Flags: volumeLsFlags, + Action: volumeLsCmd, + SkipArgReorder: true, + UseShortOptionHandling: true, +} + +func volumeLsCmd(c *cli.Context) error { + if err := validateFlags(c, volumeLsFlags); err != nil { + return err + } + + runtime, err := libpodruntime.GetRuntime(c) + if err != nil { + return errors.Wrapf(err, "error creating libpod runtime") + } + defer runtime.Shutdown(false) + + if len(c.Args()) > 0 { + return errors.Errorf("too many arguments, ls takes no arguments") + } + + opts := volumeLsOptions{ + Quiet: c.Bool("quiet"), + } + opts.Format = genVolLsFormat(c) + + // Get the filter functions based on any filters set + var filterFuncs []libpod.VolumeFilter + if c.String("filter") != "" { + filters := strings.Split(c.String("filter"), ",") + for _, f := range filters { + filterSplit := strings.Split(f, "=") + if len(filterSplit) < 2 { + return errors.Errorf("filter input must be in the form of filter=value: %s is invalid", f) + } + generatedFunc, err := generateVolumeFilterFuncs(filterSplit[0], filterSplit[1], runtime) + if err != nil { + return errors.Wrapf(err, "invalid filter") + } + filterFuncs = append(filterFuncs, generatedFunc) + } + } + + volumes, err := runtime.GetAllVolumes() + if err != nil { + return err + } + + // Get the volumes that match the filter + volsFiltered := make([]*libpod.Volume, 0, len(volumes)) + for _, vol := range volumes { + include := true + for _, filter := range filterFuncs { + include = include && filter(vol) + } + + if include { + volsFiltered = append(volsFiltered, vol) + } + } + return generateVolLsOutput(volsFiltered, opts, runtime) +} + +// generate the template based on conditions given +func genVolLsFormat(c *cli.Context) string { + var format string + if c.String("format") != "" { + // "\t" from the command line is not being recognized as a tab + // replacing the string "\t" to a tab character if the user passes in "\t" + format = strings.Replace(c.String("format"), `\t`, "\t", -1) + } + if c.Bool("quiet") { + format = "{{.Name}}" + } + return format +} + +// Convert output to genericParams for printing +func volLsToGeneric(templParams []volumeLsTemplateParams, JSONParams []volumeLsJSONParams) (genericParams []interface{}) { + if len(templParams) > 0 { + for _, v := range templParams { + genericParams = append(genericParams, interface{}(v)) + } + return + } + for _, v := range JSONParams { + genericParams = append(genericParams, interface{}(v)) + } + return +} + +// generate the accurate header based on template given +func (vol *volumeLsTemplateParams) volHeaderMap() map[string]string { + v := reflect.Indirect(reflect.ValueOf(vol)) + values := make(map[string]string) + + for i := 0; i < v.NumField(); i++ { + key := v.Type().Field(i).Name + value := key + if value == "Name" { + value = "Volume" + value + } + values[key] = strings.ToUpper(splitCamelCase(value)) + } + return values +} + +// getVolTemplateOutput returns all the volumes in the volumeLsTemplateParams format +func getVolTemplateOutput(lsParams []volumeLsJSONParams, opts volumeLsOptions) ([]volumeLsTemplateParams, error) { + var lsOutput []volumeLsTemplateParams + + for _, lsParam := range lsParams { + var ( + labels string + options string + ) + + for k, v := range lsParam.Labels { + label := k + if v != "" { + label += "=" + v + } + labels += label + } + for k, v := range lsParam.Options { + option := k + if v != "" { + option += "=" + v + } + options += option + } + params := volumeLsTemplateParams{ + Name: lsParam.Name, + Driver: lsParam.Driver, + MountPoint: lsParam.MountPoint, + Scope: lsParam.Scope, + Labels: labels, + Options: options, + } + + lsOutput = append(lsOutput, params) + } + return lsOutput, nil +} + +// getVolJSONParams returns the volumes in JSON format +func getVolJSONParams(volumes []*libpod.Volume, opts volumeLsOptions, runtime *libpod.Runtime) ([]volumeLsJSONParams, error) { + var lsOutput []volumeLsJSONParams + + for _, volume := range volumes { + params := volumeLsJSONParams{ + Name: volume.Name(), + Labels: volume.Labels(), + MountPoint: volume.MountPoint(), + Driver: volume.Driver(), + Options: volume.Options(), + Scope: volume.Scope(), + } + + lsOutput = append(lsOutput, params) + } + return lsOutput, nil +} + +// generateVolLsOutput generates the output based on the format, JSON or Go Template, and prints it out +func generateVolLsOutput(volumes []*libpod.Volume, opts volumeLsOptions, runtime *libpod.Runtime) error { + if len(volumes) == 0 && opts.Format != formats.JSONString { + return nil + } + lsOutput, err := getVolJSONParams(volumes, opts, runtime) + if err != nil { + return err + } + var out formats.Writer + + switch opts.Format { + case formats.JSONString: + if err != nil { + return errors.Wrapf(err, "unable to create JSON for volume output") + } + out = formats.JSONStructArray{Output: volLsToGeneric([]volumeLsTemplateParams{}, lsOutput)} + default: + lsOutput, err := getVolTemplateOutput(lsOutput, opts) + if err != nil { + return errors.Wrapf(err, "unable to create volume output") + } + out = formats.StdoutTemplateArray{Output: volLsToGeneric(lsOutput, []volumeLsJSONParams{}), Template: opts.Format, Fields: lsOutput[0].volHeaderMap()} + } + return formats.Writer(out).Out() +} + +// generateVolumeFilterFuncs returns the true if the volume matches the filter set, otherwise it returns false. +func generateVolumeFilterFuncs(filter, filterValue string, runtime *libpod.Runtime) (func(volume *libpod.Volume) bool, error) { + switch filter { + case "name": + return func(v *libpod.Volume) bool { + return strings.Contains(v.Name(), filterValue) + }, nil + case "driver": + return func(v *libpod.Volume) bool { + return v.Driver() == filterValue + }, nil + case "scope": + return func(v *libpod.Volume) bool { + return v.Scope() == filterValue + }, nil + case "label": + filterArray := strings.SplitN(filterValue, "=", 2) + filterKey := filterArray[0] + if len(filterArray) > 1 { + filterValue = filterArray[1] + } else { + filterValue = "" + } + return func(v *libpod.Volume) bool { + for labelKey, labelValue := range v.Labels() { + if labelKey == filterKey && ("" == filterValue || labelValue == filterValue) { + return true + } + } + return false + }, nil + case "opt": + filterArray := strings.SplitN(filterValue, "=", 2) + filterKey := filterArray[0] + if len(filterArray) > 1 { + filterValue = filterArray[1] + } else { + filterValue = "" + } + return func(v *libpod.Volume) bool { + for labelKey, labelValue := range v.Options() { + if labelKey == filterKey && ("" == filterValue || labelValue == filterValue) { + return true + } + } + return false + }, nil + } + return nil, errors.Errorf("%s is an invalid filter", filter) +} diff --git a/cmd/podman/volume_prune.go b/cmd/podman/volume_prune.go new file mode 100644 index 000000000..652c50f42 --- /dev/null +++ b/cmd/podman/volume_prune.go @@ -0,0 +1,86 @@ +package main + +import ( + "bufio" + "fmt" + "os" + "strings" + + "github.com/containers/libpod/cmd/podman/libpodruntime" + "github.com/containers/libpod/libpod" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var volumePruneDescription = ` +podman volume prune + +Remove all unused volumes. Will prompt for confirmation if not +using force. +` + +var volumePruneFlags = []cli.Flag{ + cli.BoolFlag{ + Name: "force, f", + Usage: "Do not prompt for confirmation", + }, +} + +var volumePruneCommand = cli.Command{ + Name: "prune", + Usage: "Remove all unused volumes", + Description: volumePruneDescription, + Flags: volumePruneFlags, + Action: volumePruneCmd, + SkipArgReorder: true, + UseShortOptionHandling: true, +} + +func volumePruneCmd(c *cli.Context) error { + var lastError error + + if err := validateFlags(c, volumePruneFlags); err != nil { + return err + } + + runtime, err := libpodruntime.GetRuntime(c) + if err != nil { + return errors.Wrapf(err, "error creating libpod runtime") + } + defer runtime.Shutdown(false) + + ctx := getContext() + + // Prompt for confirmation if --force is not set + if !c.Bool("force") { + reader := bufio.NewReader(os.Stdin) + fmt.Println("WARNING! This will remove all volumes not used by at least one container.") + fmt.Print("Are you sure you want to continue? [y/N] ") + ans, err := reader.ReadString('\n') + if err != nil { + return errors.Wrapf(err, "error reading input") + } + if strings.ToLower(ans)[0] != 'y' { + return nil + } + } + + volumes, err := runtime.GetAllVolumes() + if err != nil { + return err + } + + for _, vol := range volumes { + err = runtime.RemoveVolume(ctx, vol, false, true) + if err == nil { + fmt.Println(vol.Name()) + } else if err != libpod.ErrVolumeBeingUsed { + if lastError != nil { + logrus.Errorf("%q", lastError) + } + lastError = errors.Wrapf(err, "failed to remove volume %q", vol.Name()) + } + } + return lastError +} diff --git a/cmd/podman/volume_rm.go b/cmd/podman/volume_rm.go new file mode 100644 index 000000000..3fb623624 --- /dev/null +++ b/cmd/podman/volume_rm.go @@ -0,0 +1,71 @@ +package main + +import ( + "fmt" + + "github.com/containers/libpod/cmd/podman/libpodruntime" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var volumeRmDescription = ` +podman volume rm + +Remove one or more existing volumes. Will only remove volumes that are +not being used by any containers. To remove the volumes anyways, use the +--force flag. +` + +var volumeRmFlags = []cli.Flag{ + cli.BoolFlag{ + Name: "all, a", + Usage: "Remove all volumes", + }, + cli.BoolFlag{ + Name: "force, f", + Usage: "Remove a volume by force, even if it is being used by a container", + }, +} + +var volumeRmCommand = cli.Command{ + Name: "rm", + Aliases: []string{"remove"}, + Usage: "Remove one or more volumes", + Description: volumeRmDescription, + Flags: volumeRmFlags, + Action: volumeRmCmd, + ArgsUsage: "[VOLUME-NAME ...]", + SkipArgReorder: true, + UseShortOptionHandling: true, +} + +func volumeRmCmd(c *cli.Context) error { + var err error + + if err = validateFlags(c, volumeRmFlags); err != nil { + return err + } + + runtime, err := libpodruntime.GetRuntime(c) + if err != nil { + return errors.Wrapf(err, "error creating libpod runtime") + } + defer runtime.Shutdown(false) + + ctx := getContext() + + vols, lastError := getVolumesFromContext(c, runtime) + for _, vol := range vols { + err = runtime.RemoveVolume(ctx, vol, c.Bool("force"), false) + if err != nil { + if lastError != nil { + logrus.Errorf("%q", lastError) + } + lastError = errors.Wrapf(err, "failed to remove volume %q", vol.Name()) + } else { + fmt.Println(vol.Name()) + } + } + return lastError +} diff --git a/commands.md b/commands.md index f1212a3e5..43796722f 100644 --- a/commands.md +++ b/commands.md @@ -63,4 +63,9 @@ | [podman-unpause(1)](/docs/podman-unpause.1.md) | Unpause one or more running containers |[](https://asciinema.org/a/141292)| | [podman-varlink(1)](/docs/podman-varlink.1.md) | Run the varlink backend || | [podman-version(1)](/docs/podman-version.1.md) | Display the version information |[](https://asciinema.org/a/mfrn61pjZT9Fc8L4NbfdSqfgu)| +| [podman-volume-create(1)](/docs/podman-volume-create.1.md) | Create a volume || +| [podman-volume-inspect(1)](/docs/podman-volume-inspect.1.md) | Get detailed information on one or more volumes || +| [podman-volume-ls(1)](/docs/podman-volume-ls.1.md) | List all the available volumes || +| [podman-volume-rm(1)](/docs/podman-volume-rm.1.md) | Remove one or more volumes || +| [podman-volume-prune(1)](/docs/podman-volume-prune.1.md) | Remove all unused volumes || | [podman-wait(1)](/docs/podman-wait.1.md) | Wait on one or more containers to stop and print their exit codes |[](https://asciinema.org/a/QNPGKdjWuPgI96GcfkycQtah0)| diff --git a/completions/bash/podman b/completions/bash/podman index 4314cef1a..eab82ec1f 100644 --- a/completions/bash/podman +++ b/completions/bash/podman @@ -689,6 +689,23 @@ __podman_images() { __podman_q images $images_args | awk "$awk_script" | grep -v '<none>$' } +# __podman_complete_volumes applies completion of volumes based on the current +# value of `$cur` or the value of the optional first option `--cur`, if given. +__podman_complete_volumes() { + local current="$cur" + if [ "$1" = "--cur" ] ; then + current="$2" + shift 2 + fi + COMPREPLY=( $(compgen -W "$(__podman_volume "$@")" -- "$current") ) +} + +__podman_complete_volume_names() { + local names=( $(__podman_q volume ls --quiet) ) + COMPREPLY=( $(compgen -W "${names[*]}" -- "$cur") ) +} + + _podman_attach() { local options_with_args=" --detach-keys @@ -890,6 +907,7 @@ _podman_container() { create diff exec + exists export inspect kill @@ -898,6 +916,7 @@ _podman_container() { mount pause port + prune refresh restart restore @@ -1229,11 +1248,13 @@ _podman_image() { " subcommands=" build + exists history import inspect load ls + prune pull push rm @@ -2056,6 +2077,7 @@ _podman_ps() { --quiet -q --size -s --namespace --ns + --sync " _complete_ "$options_with_args" "$boolean_options" } @@ -2254,6 +2276,26 @@ _podman_container_runlabel() { esac } +_podman_images_prune() { + local options_with_args=" + " + + local boolean_options=" + -h + --help + " +} + +_podman_container_prune() { + local options_with_args=" + " + + local boolean_options=" + -h + --help + " +} + _podman_container_exists() { local options_with_args=" " @@ -2538,6 +2580,128 @@ _podman_pod() { esac } +_podman_volume_create() { + local options_with_args=" + --driver + --label + -l + --opt + -o + " + + local boolean_options=" + --help + -h + " + + _complete_ "$options_with_args" "$boolean_options" +} + +_podman_volume_ls() { + local options_with_args=" + --filter + --format + -f + " + + local boolean_options=" + --help + -h + --quiet + -q + " + + _complete_ "$options_with_args" "$boolean_options" +} + +_podman_volume_inspect() { + local options_with_args=" + --format + -f + " + + local boolean_options=" + --all + -a + --help + -h + " + + _complete_ "$options_with_args" "$boolean_options" + case "$cur" in + -*) + COMPREPLY=($(compgen -W "$boolean_options $options_with_args" -- "$cur")) + ;; + *) + __podman_complete_volume_names + ;; + esac +} + +_podman_volume_rm() { + local options_with_args="" + + local boolean_options=" + --all + -a + --force + -f + --help + -h + " + + _complete_ "$options_with_args" "$boolean_options" + case "$cur" in + -*) + COMPREPLY=($(compgen -W "$boolean_options $options_with_args" -- "$cur")) + ;; + *) + __podman_complete_volume_names + ;; + esac +} + +_podman_volume_prune() { + local options_with_args="" + + local boolean_options=" + --force + -f + --help + -h + " + + _complete_ "$options_with_args" "$boolean_options" +} + +_podman_volume() { + local boolean_options=" + --help + -h + " + subcommands=" + create + inspect + ls + rm + prune + " + local aliases=" + list + remove + " + __podman_subcommands "$subcommands $aliases" && return + + case "$cur" in + -*) + COMPREPLY=( $( compgen -W "--help" -- "$cur" ) ) + ;; + *) + COMPREPLY=( $( compgen -W "$subcommands" -- "$cur" ) ) + ;; + esac +} + _podman_podman() { local options_with_args=" --config -c @@ -2599,6 +2763,7 @@ _podman_podman() { unpause varlink version + volume wait " diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index fa233a2cb..c5c976358 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -5,6 +5,7 @@ Similar to other integrated github CI/CD services, Cirrus utilizes a simple YAML-based configuration/description file: ``.cirrus.yml``. Ref: https://cirrus-ci.org/ + ## Workflow All tasks execute in parallel, unless there are conditions or dependencies @@ -12,24 +13,34 @@ which alter this behavior. Within each task, each script executes in sequence, so long as any previous script exited successfully. The overall state of each task (pass or fail) is set based on the exit status of the last script to execute. -### ``full_vm_testing`` Task -1. Unconditionally, spin up one VM per ``matrix: image_name`` item defined - in ``.cirrus.yml``. Once accessible, ``ssh`` into each VM and run the following - scripts. +### ``gating`` Task + +***N/B: Steps below are performed by automation*** + +1. Launch a purpose-built container in Cirrus's community cluster. + For container image details, please see + [the contributors guide](https://github.com/containers/libpod/blob/master/CONTRIBUTING.md#go-format-and-lint). + +3. ``validate``: Perform standard `make validate` source verification, + Should run for less than a minute or two. + +4. ``lint``: Execute regular `make lint` to check for any code cruft. + Should also run for less than a few minutes. + -2. ``setup_environment.sh``: Configure root's ``.bash_profile`` - for all subsequent scripts (each run in a new shell). Any - distribution-specific environment variables are also defined - here. For example, setting tags/flags to use compiling. +### ``testing`` Task -3. ``verify_source.sh``: Perform per-distribution source - verification, lint-checking, etc. This acts as a minimal - gate, blocking extended use of VMs when a PR's code or commits - would otherwise not be accepted. Should run for less than a minute. +***N/B: Steps below are performed by automation*** -4. ``unit_test.sh``: Execute unit-testing, as defined by the ``Makefile``. - This should execute within 10-minutes, but often much faster. +1. After `gating` passes, spin up one VM per + `matrix: image_name` item. Once accessible, ``ssh`` + into each VM as the `root` user. + +2. ``setup_environment.sh``: Configure root's `.bash_profile` + for all subsequent scripts (each run in a new shell). Any + distribution-specific environment variables are also defined + here. For example, setting tags/flags to use compiling. 5. ``integration_test.sh``: Execute integration-testing. This is much more involved, and relies on access to external @@ -37,9 +48,12 @@ task (pass or fail) is set based on the exit status of the last script to execut Total execution time is capped at 2-hours (includes all the above) but this script normally completes in less than an hour. -### ``optional_system_testing`` Task -1. Optionally executes in parallel with ``full_vm_testing``. Requires +### ``optional_testing`` Task + +***N/B: Steps below are performed by automation*** + +1. Optionally executes in parallel with ``testing``. Requires **prior** to job-start, the magic string ``***CIRRUS: SYSTEM TEST***`` is found in the pull-request *description*. The *description* is the first text-box under the main *summary* line in the github WebUI. @@ -49,16 +63,17 @@ task (pass or fail) is set based on the exit status of the last script to execut 3. ``system_test.sh``: Build both dependencies and libpod, install them, then execute `make localsystem` from the repository root. -### ``build_vm_images`` Task -1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), Cirrus - checks the last commit message. If it contains the magic string - ``***CIRRUS: REBUILD IMAGES***``, then this task continues. +### ``cache_images`` Task -2. Execute run another round of the ``full_vm_testing`` task (above). - After the tests pass (post-merge), spin up a special VM - (from the `image-builder-image`) capable of communicating with the - GCE API. Once accessible, ``ssh`` into the VM and run the following scripts. +***N/B: Steps below are performed by automation*** + +1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), run another + round of the ``gating`` and ``testing`` tasks (above). + +2. Assuming tests pass, if the commit message contains the magic string + ``***CIRRUS: REBUILD IMAGES***``, then this task continues. Otherwise + simply mark the master branch as 'passed'. 3. ``setup_environment.sh``: Same as for other tasks. @@ -70,13 +85,108 @@ task (pass or fail) is set based on the exit status of the last script to execut 1. On a base-image VM, as root, copy the current state of the repository into ``/tmp/libpod``. 2. Execute distribution-specific scripts to prepare the image for - use by the ``full_vm_testing`` task (above). These scripts all - end with the suffix `_setup.sh` within the `$PACKER_BASE` directory. + use by the ``integration_testing`` task (above). For example, + ``fedora_setup.sh``. 3. If successful, shut down each VM and create a new GCE Image - named after the base image and the commit sha of the merge. + named with the base image, and the commit sha of the merge. ***Note:*** The ``.cirrus.yml`` file must be manually updated with the new images names, then the change sent in via a secondary pull-request. This -ensures that all the ``full_vm_testing`` tasks can pass with the new images, +ensures that all the ``integration_testing`` tasks can pass with the new images, before subjecting all future PRs to them. A workflow to automate this process is described in comments at the end of the ``.cirrus.yml`` file. + +### Base-images + +Base-images are VM disk-images specially prepared for executing as GCE VMs. +In particular, they run services on startup similar in purpose/function +as the standard 'cloud-init' services. + +* The google services are required for full support of ssh-key management + and GCE OAuth capabilities. Google provides native images in GCE + with services pre-installed, for many platforms. For example, + RHEL, CentOS, and Ubuntu. + +* Google does ***not*** provide any images for Fedora or Fedora Atomic + Host (as of 11/2018), nor do they provide a base-image prepared to + run packer for creating other images in the ``build_vm_images`` Task + (above). + +* Base images do not need to be produced often, but doing so completely + manually would be time-consuming and error-prone. Therefor a special + semi-automatic *Makefile* target is provided to assist with producing + all the base-images: ``libpod_base_images`` + +To produce new base-images, including an `image-builder-image` (used by +the ``cache_images`` Task) some input parameters are required: + + * ``GCP_PROJECT_ID``: The complete GCP project ID string e.g. foobar-12345 + identifying where the images will be stored. + + * ``GOOGLE_APPLICATION_CREDENTIALS``: A *JSON* file containing + credentials for a GCE service account. This can be [a service + account](https://cloud.google.com/docs/authentication/production#obtaining_and_providing_service_account_credentials_manually) + or [end-user + credentials](https://cloud.google.com/docs/authentication/end-user#creating_your_client_credentials] + + * ``RHEL_IMAGE_FILE`` and ``RHEL_CSUM_FILE`` complete paths + to a `rhel-server-ec2-*.raw.xz` and it's cooresponding + checksum file. These must be supplied manually because + they're not available directly via URL like other images. + + * ``RHSM_COMMAND`` contains the complete string needed to register + the VM for installing package dependencies. The VM will be de-registered + upon completion. + + * Optionally, CSV's may be specified to ``PACKER_BUILDS`` + to limit the base-images produced. For example, + ``PACKER_BUILDS=fedora,image-builder-image``. + +The following process should be performed on a bare-metal CentOS 7 machine +with network access to GCE. Software dependencies can be obtained from +the ``packer/image-builder-image_base_setup.sh`` script. + +Alternatively, an existing image-builder-image may be used from within GCE. +However it must be created with elevated cloud privileges. For example, + +``` +$ alias pgcloud='sudo podman run -it --rm -e AS_ID=$UID + -e AS_USER=$USER -v /home/$USER:/home/$USER:z cevich/gcloud_centos:latest' + +$ URL=https://www.googleapis.com/auth +$ SCOPES=$URL/userinfo.email,$URL/compute,$URL/devstorage.full_control + +$ pgcloud compute instances create $USER-making-images \ + --image-family image-builder-image \ + --boot-disk-size "200GB" \ + --min-cpu-platform "Intel Haswell" \ + --machine-type n1-standard-2 \ + --scopes $SCOPES + +$ pgcloud compute ssh centos@$USER-making-images +... +``` + +When ready, change to the ``packer`` sub-directory, and run: + +``` +$ make libpod_base_images GCP_PROJECT_ID=<VALUE> \ + GOOGLE_APPLICATION_CREDENTIALS=<VALUE> \ + RHEL_IMAGE_FILE=<VALUE> \ + RHEL_CSUM_FILE=<VALUE> \ + PACKER_BUILDS=<OPTIONAL> +``` + +Assuming this is successful (hence the semi-automatic part), packer will +produce a ``packer-manifest.json`` output file. This contains the base-image +names suitable for updating in ``.cirrus.yml``, `env` keys ``*_BASE_IMAGE``. + +On failure, it should be possible to determine the problem from the packer +output. The only exception is for the Fedora and FAH builds, which utilize +local qemu-kvm virtualisation. To observe the serial-port output from those +builds, set the ``TTYDEV`` parameter to your current device. For example: + +``` +$ make libpod_base_images ... TTYDEV=$(tty) + ... +``` diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index c8ff55445..ecdf1d877 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -8,12 +8,13 @@ CNI_COMMIT $CNI_COMMIT CRIO_COMMIT $CRIO_COMMIT RUNC_COMMIT $RUNC_COMMIT PACKER_BUILDS $PACKER_BUILDS +BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX CENTOS_BASE_IMAGE $CENTOS_BASE_IMAGE UBUNTU_BASE_IMAGE $UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE $FEDORA_BASE_IMAGE +FAH_BASE_IMAGE $FAH_BASE_IMAGE RHEL_BASE_IMAGE $RHEL_BASE_IMAGE RHSM_COMMAND $RHSM_COMMAND -BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX SERVICE_ACCOUNT $SERVICE_ACCOUNT GCE_SSH_USERNAME $GCE_SSH_USERNAME GCP_PROJECT_ID $GCP_PROJECT_ID @@ -28,28 +29,24 @@ show_env_vars # Assume basic dependencies are all met, but there could be a newer version # of the packer binary PACKER_FILENAME="packer_${PACKER_VER}_linux_amd64.zip" -mkdir -p "$HOME/packer" -cd "$HOME/packer" -# image_builder_image has packer pre-installed, check if same version requested -if ! [[ -r "$PACKER_FILENAME" ]] +if [[ -d "$HOME/packer" ]] then - curl -L -O https://releases.hashicorp.com/packer/$PACKER_VER/$PACKER_FILENAME - curl -L https://releases.hashicorp.com/packer/${PACKER_VER}/packer_${PACKER_VER}_SHA256SUMS | \ - grep 'linux_amd64' > ./sha256sums - sha256sum --check ./sha256sums - unzip -o $PACKER_FILENAME - ./packer --help &> /dev/null # verify exit(0) + cd "$HOME/packer" + # image_builder_image has packer pre-installed, check if same version requested + if [[ -r "$PACKER_FILENAME" ]] + then + cp $PACKER_FILENAME "$GOSRC/$PACKER_BASE/" + cp packer "$GOSRC/$PACKER_BASE/" + fi fi set -x -cd "$GOSRC" -# N/B: /usr/sbin/packer is a DIFFERENT tool, and will exit 0 given the args below :( -TEMPLATE="./$PACKER_BASE/libpod_images.json" - -$HOME/packer/packer inspect "$TEMPLATE" - -#$HOME/packer/packer build -machine-readable "-only=$PACKER_BUILDS" "$TEMPLATE" | tee /tmp/packer_log.csv -$HOME/packer/packer build "-only=$PACKER_BUILDS" "$TEMPLATE" - -# TODO: Report back to PR names of built images +cd "$GOSRC/$PACKER_BASE" +make libpod_images \ + PACKER_BUILDS=$PACKER_BUILDS \ + PACKER_VER=$PACKER_VER \ + GOSRC=$GOSRC \ + SCRIPT_BASE=$SCRIPT_BASE \ + PACKER_BASE=$PACKER_BASE \ + BUILT_IMAGE_SUFFIX=$BUILT_IMAGE_SUFFIX diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index 226053724..a50bd448f 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -9,7 +9,7 @@ OS_RELEASE_ID $OS_RELEASE_ID OS_RELEASE_VER $OS_RELEASE_VER " -show_env_vars +clean_env set -x cd "$GOSRC" @@ -19,10 +19,13 @@ case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in make test-binaries "BUILDTAGS=$BUILDTAGS" SKIP_USERNS=1 make localintegration "BUILDTAGS=$BUILDTAGS" ;; - fedora-28) ;& # Continue to the next item + fedora-29) ;& # Continue to the next item + fedora-28) ;& centos-7) ;& rhel-7) - stub 'integration testing not working on $OS_RELEASE_ID' + make install PREFIX=/usr ETCDIR=/etc + make test-binaries + make localintegration ;; *) bad_os_id_ver ;; esac diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 04314e5fe..985264f22 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -4,8 +4,8 @@ # to be sourced by other scripts, not called directly. # Under some contexts these values are not set, make sure they are. -USER="$(whoami)" -HOME="$(getent passwd $USER | cut -d : -f 6)" +export USER="$(whoami)" +export HOME="$(getent passwd $USER | cut -d : -f 6)" if ! [[ "$PATH" =~ "/usr/local/bin" ]] then export PATH="$PATH:/usr/local/bin" @@ -73,6 +73,18 @@ PACKER_BUILDS $PACKER_BUILDS do [[ -z "$NAME" ]] || echo "export $NAME=\"$VALUE\"" done + echo "" + echo "##### $(go version) #####" + echo "" +} + +# Unset environment variables not needed for testing purposes +clean_env() { + req_env_var " + UNSET_ENV_VARS $UNSET_ENV_VARS + " + echo "Unsetting $(echo $UNSET_ENV_VARS | wc -w) environment variables" + unset -v UNSET_ENV_VARS $UNSET_ENV_VARS || true # don't fail on read-only } # Return a GCE image-name compatible string representation of distribution name @@ -269,21 +281,29 @@ install_varlink(){ } _finalize(){ + set +e # Don't fail at the very end + set +e # make errors non-fatal echo "Removing leftover giblets from cloud-init" cd / sudo rm -rf /var/lib/cloud/instance? sudo rm -rf /root/.ssh/* sudo rm -rf /home/* + sudo rm -rf /tmp/* + sudo rm -rf /tmp/.??* + sync + sudo fstrim -av } rh_finalize(){ + set +e # Don't fail at the very end # Allow root ssh-logins if [[ -r /etc/cloud/cloud.cfg ]] then sudo sed -re 's/^disable_root:.*/disable_root: 0/g' -i /etc/cloud/cloud.cfg fi echo "Resetting to fresh-state for usage as cloud-image." - sudo $(type -P dnf || type -P yum) clean all + PKG=$(type -P dnf || type -P yum || echo "") + [[ -z "$PKG" ]] || sudo $PKG clean all # not on atomic sudo rm -rf /var/cache/{yum,dnf} sudo rm -f /etc/udev/rules.d/*-persistent-*.rules sudo touch /.unconfigured # force firstboot to run @@ -291,7 +311,35 @@ rh_finalize(){ } ubuntu_finalize(){ + set +e # Don't fail at the very end echo "Resetting to fresh-state for usage as cloud-image." sudo rm -rf /var/cache/apt _finalize } + +rhel_exit_handler() { + set +ex + req_env_var " + GOPATH $GOPATH + RHSMCMD $RHSMCMD + " + cd / + sudo rm -rf "$RHSMCMD" + sudo rm -rf "$GOPATH" + sudo subscription-manager remove --all + sudo subscription-manager unregister + sudo subscription-manager clean +} + +rhsm_enable() { + req_env_var " + RHSM_COMMAND $RHSM_COMMAND + " + export GOPATH="$(mktemp -d)" + export RHSMCMD="$(mktemp)" + trap "rhel_exit_handler" EXIT + # Avoid logging sensitive details + echo "$RHSM_COMMAND" > "$RHSMCMD" + ooe.sh sudo bash "$RHSMCMD" + sudo rm -rf "$RHSMCMD" +} diff --git a/contrib/cirrus/packer/.gitignore b/contrib/cirrus/packer/.gitignore new file mode 100644 index 000000000..8f7bdeaf7 --- /dev/null +++ b/contrib/cirrus/packer/.gitignore @@ -0,0 +1,7 @@ +*json +packer +packer*zip +packer_cache +cidata* +meta-data +user-data diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile new file mode 100644 index 000000000..9bf27373e --- /dev/null +++ b/contrib/cirrus/packer/Makefile @@ -0,0 +1,108 @@ + +# N/B: PACKER_BUILDS variable is required. Should contain CSV of +# builder name(s) from applicable YAML file, +# e.g for names see libpod_images.yml + +PACKER_VER ?= 1.3.1 +PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_amd64.zip + +# Only needed for libpod_base_images target +TIMESTAMP := $(shell date +%s) +GOSRC ?= $(shell realpath "./../../../") +PACKER_BASE ?= contrib/cirrus/packer +SCRIPT_BASE ?= contrib/cirrus + +# For debugging nested-virt, use +#TTYDEV := $(shell tty) +TTYDEV := /dev/null + +.PHONY: all +all: libpod_images + +%.json: %.yml + @python3 -c 'import json,yaml; json.dump( yaml.load(open("$<").read()), open("$@","w"), indent=2);' + +${PACKER_DIST_FILENAME}: + @curl -L --silent --show-error \ + -O https://releases.hashicorp.com/packer/${PACKER_VER}/${PACKER_DIST_FILENAME} + +packer: ${PACKER_DIST_FILENAME} + @curl -L --silent --show-error \ + https://releases.hashicorp.com/packer/${PACKER_VER}/packer_${PACKER_VER}_SHA256SUMS \ + | grep 'linux_amd64' > /tmp/packer_sha256sums + @sha256sum --check /tmp/packer_sha256sums + @unzip -o ${PACKER_DIST_FILENAME} + @touch --reference=Makefile ${PACKER_DIST_FILENAME} + +.PHONY: test +test: libpod_base_images.json libpod_images.json packer + ./packer inspect libpod_base_images.json > /dev/null + ./packer inspect libpod_images.json > /dev/null + @echo "All good" + +.PHONY: libpod_images +libpod_images: libpod_images.json packer +ifndef PACKER_BUILDS + $(error PACKER_BUILDS is undefined, expected builder-names CSV) +endif + ./packer build -only=${PACKER_BUILDS} \ + -var GOSRC=$(GOSRC) \ + -var PACKER_BASE=$(PACKER_BASE) \ + -var SCRIPT_BASE=$(SCRIPT_BASE) \ + libpod_images.json + @echo "" + @echo "Finished. The images mentioned above, and in packer-manifest.json" + @echo "can be used in .cirrus.yml as values for the 'image_name' keys" + @echo "" + +cidata.ssh: + ssh-keygen -f $@ -P "" -q + +cidata.ssh.pub: cidata.ssh + touch $@ + +meta-data: + echo "local-hostname: localhost.localdomain" > $@ + +user-data: cidata.ssh.pub + bash make-user-data.sh + +cidata.iso: user-data meta-data + genisoimage -output cidata.iso -volid cidata -input-charset utf-8 -joliet -rock user-data meta-data + +# This is intended to be run by a human, with admin access to the libpod GCE project. +.PHONY: libpod_base_images +libpod_base_images: libpod_base_images.json cidata.iso cidata.ssh packer +ifndef GCP_PROJECT_ID + $(error GCP_PROJECT_ID is undefined, expected complete GCP project ID string e.g. foobar-12345) +endif +ifndef GOOGLE_APPLICATION_CREDENTIALS + $(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json) +endif +ifndef RHEL_IMAGE_FILE + $(error RHEL_IMAGE_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz file) +endif +ifndef RHEL_CSUM_FILE + $(error RHEL_CSUM_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz.SHA256SUM file) +endif +ifndef RHSM_COMMAND + $(error RHSM_COMMAND is undefined, expected string required for temporarily registering VM) +endif + PACKER_CACHE_DIR=/tmp ./packer build \ + -var TIMESTAMP=$(TIMESTAMP) \ + -var TTYDEV=$(TTYDEV) \ + -var GCP_PROJECT_ID=$(GCP_PROJECT_ID) \ + -var GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \ + -var GOSRC=$(GOSRC) \ + -var PACKER_BASE=$(PACKER_BASE) \ + -var SCRIPT_BASE=$(SCRIPT_BASE) \ + -var RHEL_BASE_IMAGE_NAME=$(shell basename $(RHEL_IMAGE_FILE) | tr -d '[[:space:]]' | sed -r -e 's/\.x86_64\.raw\.xz//' | tr '[[:upper:]]' '[[:lower:]]' | tr '[[:punct:]]' '-') \ + -var RHEL_IMAGE_FILE=$(RHEL_IMAGE_FILE) \ + -var RHEL_CSUM_FILE=$(RHEL_CSUM_FILE) \ + -var 'RHSM_COMMAND=$(RHSM_COMMAND)' \ + -only $(PACKER_BUILDS) \ + libpod_base_images.json + @echo "" + @echo "Finished. The images mentioned above, and in packer-manifest.json" + @echo "can be used in .cirrus.yml as values for the *_BASE_IMAGE keys." + @echo "" diff --git a/contrib/cirrus/packer/README.md b/contrib/cirrus/packer/README.md index 8ff6947e9..9a07ed960 100644 --- a/contrib/cirrus/packer/README.md +++ b/contrib/cirrus/packer/README.md @@ -1,2 +1,3 @@ These are definitions and scripts consumed by packer to produce the -various distribution images used for CI testing. +various distribution images used for CI testing. For more details +see the [Cirrus CI documentation](../README.md) diff --git a/contrib/cirrus/packer/fah_base-setup.sh b/contrib/cirrus/packer/fah_base-setup.sh new file mode 100644 index 000000000..606c4f336 --- /dev/null +++ b/contrib/cirrus/packer/fah_base-setup.sh @@ -0,0 +1,45 @@ + +# N/B: This script is not intended to be run by humans. It is used to configure the +# FAH base image for importing, so that it will boot in GCE. + +set -e + +# Load in library (copied by packer, before this script was run) +source $GOSRC/$SCRIPT_BASE/lib.sh + +install_ooe + +if [[ "$1" == "pre" ]] +then + echo "Upgrading Atomic Host" + setenforce 0 + ooe.sh atomic host upgrade + + echo "Configuring Repositories" + ooe.sh sudo tee /etc/yum.repos.d/ngompa-gce-oslogin.repo <<EOF +[ngompa-gce-oslogin] +name=Copr repo for gce-oslogin owned by ngompa +baseurl=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/fedora-\$releasever-\$basearch/ +type=rpm-md +skip_if_unavailable=True +gpgcheck=1 +gpgkey=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/pubkey.gpg +repo_gpgcheck=0 +enabled=1 +enabled_metadata=1 +EOF + echo "Installing necessary packages and google services" + # Google services are enabled by default, upon install. + ooe.sh rpm-ostree install rng-tools google-compute-engine google-compute-engine-oslogin + echo "Rebooting..." + systemctl reboot # Required for upgrade + package installs to be active +elif [[ "$1" == "post" ]] +then + echo "Enabling necessary services" + systemctl enable rngd # Must reboot before enabling + rh_finalize + echo "SUCCESS!" +else + echo "Expected to be called with 'pre' or 'post'" + exit 6 +fi diff --git a/contrib/cirrus/packer/fah_setup.sh b/contrib/cirrus/packer/fah_setup.sh new file mode 100644 index 000000000..2e053b396 --- /dev/null +++ b/contrib/cirrus/packer/fah_setup.sh @@ -0,0 +1,23 @@ +#!/bin/bash + +# This script is called by packer on the subject fah VM, to setup the podman +# build/test environment. It's not intended to be used outside of this context. + +set -e + +# Load in library (copied by packer, before this script was run) +source /tmp/libpod/$SCRIPT_BASE/lib.sh + +req_env_var " +SCRIPT_BASE $SCRIPT_BASE +" + +install_ooe + +ooe.sh sudo atomic host upgrade + +ooe.sh sudo rpm-ostree uninstall cloud-init + +rh_finalize + +echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh new file mode 100644 index 000000000..c0a1e422c --- /dev/null +++ b/contrib/cirrus/packer/fedora_base-setup.sh @@ -0,0 +1,27 @@ +#!/bin/bash + +# N/B: This script is not intended to be run by humans. It is used to configure the +# fedora base image for importing, so that it will boot in GCE + +set -e + +# Load in library (copied by packer, before this script was run) +source $GOSRC/$SCRIPT_BASE/lib.sh + +[[ "$1" == "post" ]] || exit 0 # nothing to do + +install_ooe + +echo "Updating packages" +ooe.sh dnf -y update + +echo "Installing necessary packages and google services" +ooe.sh dnf -y copr enable ngompa/gce-oslogin +ooe.sh dnf -y install rng-tools google-compute-engine google-compute-engine-oslogin + +echo "Enabling services" +ooe.sh systemctl enable rngd + +rh_finalize + +echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index f9fea04a7..4e4391e59 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -10,6 +10,7 @@ source /tmp/libpod/$SCRIPT_BASE/lib.sh req_env_var " SCRIPT_BASE $SCRIPT_BASE +FEDORA_CNI_COMMIT $FEDORA_CNI_COMMIT CNI_COMMIT $CNI_COMMIT CRIO_COMMIT $CRIO_COMMIT CRIU_COMMIT $CRIU_COMMIT @@ -65,11 +66,13 @@ ooe.sh sudo dnf install -y \ runc \ skopeo-containers \ slirp4netns \ + unzip \ which \ xz install_varlink +CNI_COMMIT=$FEDORA_CNI_COMMIT install_cni_plugins install_buildah diff --git a/contrib/cirrus/packer/image-builder-image_base-setup.sh b/contrib/cirrus/packer/image-builder-image_base-setup.sh new file mode 100644 index 000000000..b8e2824a7 --- /dev/null +++ b/contrib/cirrus/packer/image-builder-image_base-setup.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# This script is called by packer on a vanilla CentOS VM, to setup the image +# used for building images FROM base images. It's not intended to be used +# outside of this context. + +set -e + +[[ "$1" == "post" ]] || exit 0 # pre stage not needed + +# Load in library (copied by packer, before this script was run) +source $GOSRC/$SCRIPT_BASE/lib.sh + +req_env_var " + TIMESTAMP $TIMESTAMP + GOSRC $GOSRC + SCRIPT_BASE $SCRIPT_BASE + PACKER_BASE $PACKER_BASE +" + +install_ooe + +echo "Updating packages" +ooe.sh sudo yum -y update + +echo "Configuring repositories" +ooe.sh sudo yum -y install centos-release-scl epel-release + +echo "Installing packages" +ooe.sh sudo yum -y install \ + genisoimage \ + golang \ + google-cloud-sdk \ + libvirt \ + libvirt-admin \ + libvirt-client \ + libvirt-daemon \ + make \ + python34 \ + python34 \ + python34-PyYAML \ + python34-PyYAML \ + qemu-img \ + qemu-kvm \ + qemu-kvm-tools \ + qemu-user \ + rsync \ + unzip \ + util-linux \ + vim + +sudo ln -s /usr/libexec/qemu-kvm /usr/bin/ + +sudo tee /etc/modprobe.d/kvm-nested.conf <<EOF +options kvm-intel nested=1 +options kvm-intel enable_shadow_vmcs=1 +options kvm-intel enable_apicv=1 +options kvm-intel ept=1 +EOF + +echo "Installing packer" +sudo mkdir -p /root/$(basename $PACKER_BASE) +sudo cp $GOSRC/$PACKER_BASE/*packer* /root/$(basename $PACKER_BASE) +sudo mkdir -p /root/$(basename $SCRIPT_BASE) +sudo cp $GOSRC/$SCRIPT_BASE/*.sh /root/$(basename $SCRIPT_BASE) + +install_scl_git + +echo "Cleaning up" +cd / +rm -rf $GOSRC + +rh_finalize + +echo "SUCCESS!" diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml new file mode 100644 index 000000000..4ae44e0d9 --- /dev/null +++ b/contrib/cirrus/packer/libpod_base_images.yml @@ -0,0 +1,179 @@ +--- + +variables: + # Complete local path to this repository (Required) + GOSRC: + # Relative path to this (packer) subdirectory (Required) + PACKER_BASE: + # Relative path to cirrus scripts subdirectory (Required) + SCRIPT_BASE: + # Unique ID for naming new base-images (required) + TIMESTAMP: + # Required for output from qemu builders + TTYDEV: + # RHEL images require click-through agreements to obtain (required) + RHEL_BASE_IMAGE_NAME: + RHEL_IMAGE_FILE: + RHEL_CSUM_FILE: + # RHEL requires a subscription to install/update packages + RHSM_COMMAND: + + # Fedora images are obtainable by direct download + FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2" + FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM" + FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2' # Name to use in GCE + FAH_IMAGE_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1.x86_64.qcow2" + FAH_CSUM_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1-x86_64-CHECKSUM" + FAH_BASE_IMAGE_NAME: 'fedora-atomichost-29-20181025-1' # Name to use in GCE + + # The name of the image in GCE used for packer build libpod_images.yml + IBI_BASE_NAME: 'image-builder-image' + CIDATA_ISO: 'cidata.iso' # produced by Makefile + + # Path to json file (required, likely ~/.config/gcloud/legacy_credentials/*/adc.json) + GOOGLE_APPLICATION_CREDENTIALS: + # The complete project ID (required, not the short name) + GCP_PROJECT_ID: + # Pre-existing storage bucket w/ lifecycle-enabled + XFERBUCKET: "packer-import" # pre-created, globally unique, lifecycle-enabled + +# Don't leak sensitive values in error messages / output +sensitive-variables: + - 'GOOGLE_APPLICATION_CREDENTIALS' + - 'GCP_PROJECT_ID' + - 'RHSM_COMMAND' + +# What images to produce in which cloud +builders: + - name: '{{user `IBI_BASE_NAME`}}' + type: 'googlecompute' + image_name: '{{user `IBI_BASE_NAME`}}-{{user `TIMESTAMP`}}' + image_family: '{{user `IBI_BASE_NAME`}}' + source_image_project_id: 'centos-cloud' + source_image_family: 'centos-7' + project_id: '{{user `GCP_PROJECT_ID`}}' + account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' + communicator: 'ssh' + ssh_username: 'centos' + ssh_pty: 'true' + # The only supported zone in Cirrus-CI, as of addition of this comment + zone: 'us-central1-a' + # Enable nested virtualization in case it's ever needed + image_licenses: + - 'https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx' + min_cpu_platform: "Intel Broadwell" # nested-virt requirement + + - &nested_virt + name: 'fedora' + type: 'qemu' + accelerator: "kvm" + iso_url: '{{user `FEDORA_IMAGE_URL`}}' + disk_image: true + format: "raw" + disk_size: 5120 + iso_checksum_url: '{{user `FEDORA_CSUM_URL`}}' + iso_checksum_type: "sha256" + output_directory: '/tmp/{{build_name}}' + vm_name: "disk.raw" # actually qcow2, name required for post-processing + boot_wait: '5s' + shutdown_command: 'shutdown -h now' + headless: true + qemu_binary: "/usr/libexec/qemu-kvm" + qemuargs: # List-of-list format required to override packer-generated args + - - "-m" + - "1024" + - - "-cpu" + - "host" + - - "-device" + - "virtio-rng-pci" + - - "-chardev" + - "tty,id=pts,path={{user `TTYDEV`}}" + - - "-device" + - "isa-serial,chardev=pts" + - - "-cdrom" + - "{{user `CIDATA_ISO`}}" + - - "-netdev" + - "user,id=net0,hostfwd=tcp::{{ .SSHHostPort }}-:22" + - - "-device" + - "virtio-net,netdev=net0" + communicator: 'ssh' + ssh_private_key_file: 'cidata.ssh' + ssh_username: 'root' + + - <<: *nested_virt + name: 'fah' + iso_url: '{{user `FAH_IMAGE_URL`}}' + iso_checksum_url: '{{user `FAH_CSUM_URL`}}' + disk_size: 10240 + + - <<: *nested_virt + name: 'rhel' + iso_url: 'file://{{user `RHEL_IMAGE_FILE`}}' + iso_checksum_url: 'file://{{user `RHEL_CSUM_FILE`}}' + disk_size: 10240 + +provisioners: + - type: 'shell' + inline: + - 'mkdir -p /tmp/libpod/{{user `SCRIPT_BASE`}}' + - 'mkdir -p /tmp/libpod/{{user `PACKER_BASE`}}' + + - type: 'file' + source: '{{user `GOSRC`}}/.cirrus.yml' + destination: '/tmp/libpod/.cirrus.yml' + + - type: 'file' + source: '{{user `GOSRC`}}/{{user `SCRIPT_BASE`}}/' + destination: '/tmp/libpod/{{user `SCRIPT_BASE`}}/' + + - type: 'file' + source: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/' + destination: '/tmp/libpod/{{user `PACKER_BASE`}}/' + + - &shell_script + type: 'shell' + inline: + - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh' + - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh pre' + expect_disconnect: true # Allow this to reboot the VM + environment_vars: + - 'TIMESTAMP={{user `TIMESTAMP`}}' + - 'GOSRC=/tmp/libpod' + - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' + - 'PACKER_BASE={{user `PACKER_BASE`}}' + - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}' + + - <<: *shell_script + inline: ['{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'] + expect_disconnect: false + pause_before: '10s' + inline: + - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh post' + +post-processors: + - - type: "compress" + only: ['fedora', 'fah', 'rhel'] + output: '/tmp/{{build_name}}/disk.raw.tar.gz' + format: '.tar.gz' + compression_level: 9 + - &gcp_import + only: ['fedora'] + type: "googlecompute-import" + project_id: '{{user `GCP_PROJECT_ID`}}' + account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' + bucket: '{{user `XFERBUCKET`}}' + gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}-{{uuid}}.tar.gz' + image_name: "{{user `FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" + image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}' + image_family: '{{user `FEDORA_BASE_IMAGE_NAME`}}' + - <<: *gcp_import + only: ['fah'] + image_name: "{{user `FAH_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" + image_description: 'Based on {{user `FAH_IMAGE_URL`}}' + image_family: '{{user `FAH_BASE_IMAGE_NAME`}}' + - <<: *gcp_import + only: ['rhel'] + image_name: "{{user `RHEL_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" + image_description: 'Based on {{user `RHEL_IMAGE_FILE`}}' + image_family: '{{user `RHEL_BASE_IMAGE_NAME`}}' + - type: 'manifest' diff --git a/contrib/cirrus/packer/libpod_images.json b/contrib/cirrus/packer/libpod_images.json deleted file mode 100644 index 9dac3e8ea..000000000 --- a/contrib/cirrus/packer/libpod_images.json +++ /dev/null @@ -1,130 +0,0 @@ -{ - "variables": { - "FEDORA_CNI_COMMIT": "{{env `FEDORA_CNI_COMMIT`}}", - "CNI_COMMIT": "{{env `CNI_COMMIT`}}", - "CRIO_COMMIT": "{{env `CRIO_COMMIT`}}", - "CRIU_COMMIT": "{{env `CRIU_COMMIT`}}", - "RUNC_COMMIT": "{{env `RUNC_COMMIT`}}", - - "CENTOS_BASE_IMAGE": "{{env `CENTOS_BASE_IMAGE`}}" , - "UBUNTU_BASE_IMAGE": "{{env `UBUNTU_BASE_IMAGE`}}", - "FEDORA_BASE_IMAGE": "{{env `FEDORA_BASE_IMAGE`}}", - "RHEL_BASE_IMAGE": "{{env `RHEL_BASE_IMAGE`}}", - - "GOSRC": "{{env `GOSRC`}}", - "PACKER_BASE": "{{env `PACKER_BASE`}}", - "SCRIPT_BASE": "{{env `SCRIPT_BASE`}}", - - "SERVICE_ACCOUNT": "{{env `SERVICE_ACCOUNT`}}", - "GCP_PROJECT_ID": "{{env `GCP_PROJECT_ID`}}", - "BUILT_IMAGE_SUFFIX": "{{env `BUILT_IMAGE_SUFFIX`}}", - "GCE_SSH_USERNAME": "{{env `GCE_SSH_USERNAME`}}", - "RHSM_COMMAND": "{{env `RHSM_COMMAND`}}" - }, - "sensitive-variables": [ - "GCP_PROJECT_ID", "SERVICE_ACCOUNT", "GCE_SSH_USERNAME", "RHSM_COMMAND" - ], - "builders": [ - { - "name": "rhel-7", - "type": "googlecompute", - "project_id": "{{user `GCP_PROJECT_ID`}}", - "zone": "us-central1-a", - "source_image": "{{user `RHEL_BASE_IMAGE`}}", - "image_name": "{{user `RHEL_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", - "image_family": "{{user `RHEL_BASE_IMAGE`}}-libpod", - "service_account_email": "{{user `SERVICE_ACCOUNT`}}", - "communicator": "ssh", - "ssh_username": "ec2-user", - "ssh_pty": "true" - },{ - "name": "centos-7", - "type": "googlecompute", - "project_id": "{{user `GCP_PROJECT_ID`}}", - "zone": "us-central1-a", - "source_image": "{{user `CENTOS_BASE_IMAGE`}}", - "image_name": "{{user `CENTOS_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", - "image_family": "{{user `CENTOS_BASE_IMAGE`}}-libpod", - "service_account_email": "{{user `SERVICE_ACCOUNT`}}", - "communicator": "ssh", - "ssh_username": "{{user `GCE_SSH_USERNAME`}}", - "ssh_pty": "true" - },{ - "name": "fedora-28", - "type": "googlecompute", - "project_id": "{{user `GCP_PROJECT_ID`}}", - "zone": "us-central1-a", - "source_image": "{{user `FEDORA_BASE_IMAGE`}}", - "image_name": "{{user `FEDORA_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", - "image_family": "{{user `FEDORA_BASE_IMAGE`}}-libpod", - "service_account_email": "{{user `SERVICE_ACCOUNT`}}", - "communicator": "ssh", - "ssh_username": "fedora", - "ssh_pty": "true" - },{ - "name": "ubuntu-18", - "type": "googlecompute", - "project_id": "{{user `GCP_PROJECT_ID`}}", - "zone": "us-central1-a", - "source_image": "{{user `UBUNTU_BASE_IMAGE`}}", - "image_name": "{{user `UBUNTU_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}", - "image_family": "{{user `UBUNTU_BASE_IMAGE`}}-libpod", - "service_account_email": "{{user `SERVICE_ACCOUNT`}}", - "communicator": "ssh", - "ssh_username": "{{user `GCE_SSH_USERNAME`}}", - "ssh_pty": "true" - } - ], - "provisioners": [ - { - "type": "file", - "source": "{{user `GOSRC`}}", - "destination": "/tmp/libpod" - },{ - "type": "shell", - "only": ["rhel-7"], - "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/rhel_setup.sh", - "environment_vars": [ - "SCRIPT_BASE={{user `SCRIPT_BASE`}}", - "CNI_COMMIT={{user `CNI_COMMIT`}}", - "CRIO_COMMIT={{user `CRIO_COMMIT`}}", - "CRIU_COMMIT={{user `CRIU_COMMIT`}}", - "RUNC_COMMIT={{user `RUNC_COMMIT`}}", - "RHSM_COMMAND={{user `RHSM_COMMAND`}}" - ] - },{ - "type": "shell", - "only": ["centos-7"], - "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/centos_setup.sh", - "environment_vars": [ - "SCRIPT_BASE={{user `SCRIPT_BASE`}}", - "CNI_COMMIT={{user `CNI_COMMIT`}}", - "CRIO_COMMIT={{user `CRIO_COMMIT`}}", - "CRIU_COMMIT={{user `CRIU_COMMIT`}}", - "RUNC_COMMIT={{user `RUNC_COMMIT`}}" - ] - },{ - "type": "shell", - "only": ["fedora-28"], - "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/fedora_setup.sh", - "environment_vars": [ - "SCRIPT_BASE={{user `SCRIPT_BASE`}}", - "CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}", - "CRIO_COMMIT={{user `CRIO_COMMIT`}}", - "CRIU_COMMIT={{user `CRIU_COMMIT`}}", - "RUNC_COMMIT={{user `RUNC_COMMIT`}}" - ] - },{ - "type": "shell", - "only": ["ubuntu-18"], - "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/ubuntu_setup.sh", - "environment_vars": [ - "SCRIPT_BASE={{user `SCRIPT_BASE`}}", - "CNI_COMMIT={{user `CNI_COMMIT`}}", - "CRIO_COMMIT={{user `CRIO_COMMIT`}}", - "CRIU_COMMIT={{user `CRIU_COMMIT`}}", - "RUNC_COMMIT={{user `RUNC_COMMIT`}}" - ] - } - ] -} diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml new file mode 100644 index 000000000..7b95b08cc --- /dev/null +++ b/contrib/cirrus/packer/libpod_images.yml @@ -0,0 +1,91 @@ +--- + +# All of these are required +variables: + # Names of GCE Base images to start from, in .cirrus.yml + RHEL_BASE_IMAGE: '{{env `RHEL_BASE_IMAGE`}}' + CENTOS_BASE_IMAGE: '{{env `CENTOS_BASE_IMAGE`}}' + UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}' + FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}' + FAH_BASE_IMAGE: '{{env `FAH_BASE_IMAGE`}}' + + # libpod dependencies to build and install into images + FEDORA_CNI_COMMIT: "{{env `FEDORA_CNI_COMMIT`}}" + CNI_COMMIT: "{{env `CNI_COMMIT`}}" + CRIO_COMMIT: "{{env `CRIO_COMMIT`}}" + CRIU_COMMIT: "{{env `CRIU_COMMIT`}}" + RUNC_COMMIT: "{{env `RUNC_COMMIT`}}" + + BUILT_IMAGE_SUFFIX: '{{env `BUILT_IMAGE_SUFFIX`}}' + GOSRC: '{{env `GOSRC`}}' + PACKER_BASE: '{{env `PACKER_BASE`}}' + SCRIPT_BASE: '{{env `SCRIPT_BASE`}}' + + # Protected credentials, decrypted by Cirrus at runtime + GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}' + GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}' + RHSM_COMMAND: '{{env `RHSM_COMMAND`}}' + SERVICE_ACCOUNT: '{{env `SERVICE_ACCOUNT`}}' + GOOGLE_APPLICATION_CREDENTIALS: '{{env `GOOGLE_APPLICATION_CREDENTIALS`}}' + +# Don't leak sensitive values in error messages / output +sensitive-variables: + - 'GCE_SSH_USERNAME' + - 'GCP_PROJECT_ID' + - 'RHSM_COMMAND' + - 'SERVICE_ACCOUNT' + +# What images to produce in which cloud +builders: + # v----- is a YAML anchor, allows referencing this object by name (below) + - &gce_hosted_image + name: 'ubuntu-18' + type: 'googlecompute' + image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}' + image_family: '{{build_name}}-libpod' + source_image: '{{user `UBUNTU_BASE_IMAGE`}}' + disk_size: 20 + project_id: '{{user `GCP_PROJECT_ID`}}' + service_account_email: '{{user `SERVICE_ACCOUNT`}}' + communicator: 'ssh' + ssh_username: '{{user `GCE_SSH_USERNAME`}}' + ssh_pty: 'true' + # The only supported zone in Cirrus-CI, as of addition of this comment + zone: 'us-central1-a' + + # v----- is a YAML alias, allows partial re-use of the anchor object + - <<: *gce_hosted_image + name: 'rhel-7' + source_image: '{{user `RHEL_BASE_IMAGE`}}' + + - <<: *gce_hosted_image + name: 'centos-7' + source_image: '{{user `CENTOS_BASE_IMAGE`}}' + + - <<: *gce_hosted_image + name: 'fedora-29' + source_image: '{{user `FEDORA_BASE_IMAGE`}}' + + - <<: *gce_hosted_image + name: 'fah-29' + source_image: '{{user `FAH_BASE_IMAGE`}}' + +# The brains of the operation, making actual modifications to the base-image. +provisioners: + - type: 'file' + source: '{{user `GOSRC`}}' + destination: '/tmp/libpod' + + - type: 'shell' + script: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{split build_name "-" 0}}_setup.sh' + environment_vars: + - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' + - 'CNI_COMMIT={{user `CNI_COMMIT`}}' + - 'FEDORA_CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}' + - 'CRIO_COMMIT={{user `CRIO_COMMIT`}}' + - 'CRIU_COMMIT={{user `CRIU_COMMIT`}}' + - 'RUNC_COMMIT={{user `RUNC_COMMIT`}}' + - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}' + +post-processors: + - - type: 'manifest' diff --git a/contrib/cirrus/packer/make-user-data.sh b/contrib/cirrus/packer/make-user-data.sh new file mode 100644 index 000000000..7f7fa1c1a --- /dev/null +++ b/contrib/cirrus/packer/make-user-data.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +# This script is utilized by Makefile, it's not intended to be run by humans + +cat <<EOF > user-data +#cloud-config +timezone: US/Eastern +growpart: + mode: auto +disable_root: false +ssh_pwauth: True +ssh_import_id: [root] +ssh_authorized_keys: + - $(cat cidata.ssh.pub) +users: + - name: root + primary-group: root + homedir: /root + system: true +EOF diff --git a/contrib/cirrus/packer/rhel_base-setup.sh b/contrib/cirrus/packer/rhel_base-setup.sh new file mode 100644 index 000000000..8b2073d4f --- /dev/null +++ b/contrib/cirrus/packer/rhel_base-setup.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +# N/B: This script is not intended to be run by humans. It is used to configure the +# rhel base image for importing, so that it will boot in GCE + +set -e + +[[ "$1" == "post" ]] || exit 0 # pre stage is not needed + +# Load in library (copied by packer, before this script was run) +source $GOSRC/$SCRIPT_BASE/lib.sh + +req_env_var " + RHSM_COMMAND $RHSM_COMMAND +" + +install_ooe + +echo "Setting up repos" +# Frequently needed +ooe.sh sudo yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm + +# Required for google to manage ssh keys +ooe.sh sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOM +[google-cloud-compute] +name=google-cloud-compute +baseurl=https://packages.cloud.google.com/yum/repos/google-cloud-compute-el7-x86_64 +enabled=1 +gpgcheck=1 +repo_gpgcheck=1 +gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg + https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg +EOM + +rhsm_enable + +echo "Installing/removing packages" +ooe.sh sudo yum -y install google-compute-engine google-compute-engine-oslogin +ooe.sh sudo yum -y erase "cloud-init" "rh-amazon-rhui-client*" || true +ooe.sh sudo systemctl enable \ + google-accounts-daemon \ + google-clock-skew-daemon \ + google-instance-setup \ + google-network-daemon \ + google-shutdown-scripts \ + google-startup-scripts + +rhel_exit_handler # release subscription! + +rh_finalize + +echo "SUCCESS!" diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh index d296713fc..7f0d4e589 100644 --- a/contrib/cirrus/packer/rhel_setup.sh +++ b/contrib/cirrus/packer/rhel_setup.sh @@ -18,24 +18,7 @@ RHSM_COMMAND $RHSM_COMMAND install_ooe -export GOPATH="$(mktemp -d)" -export RHSMCMD="$(mktemp)" - -exit_handler() { - set +ex - cd / - sudo rm -rf "$RHSMCMD" - sudo rm -rf "$GOPATH" - sudo subscription-manager remove --all - sudo subscription-manager unregister - sudo subscription-manager clean -} -trap "exit_handler" EXIT - -# Avoid logging sensitive details -echo "$RHSM_COMMAND" > "$RHSMCMD" -ooe.sh sudo bash "$RHSMCMD" -sudo rm -rf "$RHSMCMD" +rhsm_enable ooe.sh sudo yum -y erase "rh-amazon-rhui-client*" ooe.sh sudo subscription-manager repos "--disable=*" @@ -47,21 +30,6 @@ ooe.sh sudo subscription-manager repos \ ooe.sh sudo yum -y update -# Frequently needed -ooe.sh sudo yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - -# Required for google to manage ssh keys -sudo tee -a /etc/yum.repos.d/google-cloud-sdk.repo << EOM -[google-cloud-compute] -name=google-cloud-compute -baseurl=https://packages.cloud.google.com/yum/repos/google-cloud-compute-el7-x86_64 -enabled=1 -gpgcheck=1 -repo_gpgcheck=1 -gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg - https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg -EOM - ooe.sh sudo yum -y install \ atomic-registries \ btrfs-progs-devel \ @@ -74,8 +42,6 @@ ooe.sh sudo yum -y install \ golang \ golang-github-cpuguy83-go-md2man \ golang-github-cpuguy83-go-md2man \ - google-compute-engine \ - google-compute-engine-oslogin \ gpgme-devel \ iptables \ libassuan-devel \ @@ -118,7 +84,7 @@ install_criu install_packer_copied_files -exit_handler # release subscription! +rhel_exit_handler # release subscription! rh_finalize diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index ef209a4a4..6e3613462 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -21,6 +21,7 @@ install_ooe export GOPATH="$(mktemp -d)" trap "sudo rm -rf $GOPATH" EXIT +# Avoid getting stuck waiting for user input export DEBIAN_FRONTEND=noninteractive # Try twice as workaround for minor networking problems @@ -56,6 +57,8 @@ ooe.sh sudo -E apt-get -qq install --no-install-recommends \ libostree-dev \ libprotobuf-c0-dev \ libprotobuf-dev \ + libseccomp-dev \ + libseccomp2 \ libtool \ libudev-dev \ lsof \ diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 2563b5f43..5ba842cf1 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -16,12 +16,11 @@ CIRRUS_BUILD_ID $CIRRUS_BUILD_ID" cd "$CIRRUS_WORKING_DIR" # for clarity of initial conditions # Verify basic dependencies -for depbin in go rsync unzip sha256sum curl make +for depbin in go rsync unzip sha256sum curl make python3 git do if ! type -P "$depbin" &> /dev/null then - echo "ERROR: $depbin binary not found in $PATH" - exit 2 + echo "***** WARNING: $depbin binary not found in $PATH *****" fi done @@ -35,14 +34,15 @@ then # N/B: Single-quote items evaluated every time, double-quotes only once (right now). for envstr in \ "$MARK" \ + "export EPOCH_TEST_COMMIT=\"$CIRRUS_BASE_SHA\"" \ "export HEAD=\"$CIRRUS_CHANGE_IN_REPO\"" \ "export TRAVIS=\"1\"" \ "export GOSRC=\"$CIRRUS_WORKING_DIR\"" \ "export OS_RELEASE_ID=\"$(os_release_id)\"" \ "export OS_RELEASE_VER=\"$(os_release_ver)\"" \ - "export OS_REL_VER=\"${OS_RELEASE_ID}-${OS_RELEASE_VER}\"" \ + "export OS_REL_VER=\"$(os_release_id)-$(os_release_ver)\"" \ "export BUILT_IMAGE_SUFFIX=\"-$CIRRUS_REPO_NAME-${CIRRUS_CHANGE_IN_REPO:0:8}\"" \ - "export GOPATH=\"/go\"" \ + "export GOPATH=\"/var/tmp/go\"" \ 'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \ 'export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"' do @@ -57,7 +57,8 @@ then install_runc_from_git envstr='export BUILDTAGS="seccomp $($GOSRC/hack/btrfs_tag.sh) $($GOSRC/hack/btrfs_installed_tag.sh) $($GOSRC/hack/ostree_tag.sh) varlink exclude_graphdriver_devicemapper"' ;; - fedora-28) ;& # Continue to the next item + fedora-29) ;& # Continue to the next item + fedora-28) ;& centos-7) ;& rhel-7) envstr='unset BUILDTAGS' # Use default from Makefile diff --git a/contrib/cirrus/system_test.sh b/contrib/cirrus/system_test.sh index 7c727d336..66974f8c6 100755 --- a/contrib/cirrus/system_test.sh +++ b/contrib/cirrus/system_test.sh @@ -9,7 +9,7 @@ OS_RELEASE_ID $OS_RELEASE_ID OS_RELEASE_VER $OS_RELEASE_VER " -show_env_vars +clean_env set -x cd "$GOSRC" diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh index cacc23045..e5b167e79 100755 --- a/contrib/cirrus/unit_test.sh +++ b/contrib/cirrus/unit_test.sh @@ -9,22 +9,22 @@ OS_RELEASE_ID $OS_RELEASE_ID OS_RELEASE_VER $OS_RELEASE_VER " -show_env_vars +clean_env set -x cd "$GOSRC" case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in ubuntu-18) + make install.tools "BUILDTAGS=$BUILDTAGS" make localunit "BUILDTAGS=$BUILDTAGS" make "BUILDTAGS=$BUILDTAGS" ;; - fedora-28) + fedora-29) ;& # Continue to the next item + centos-7) ;& + rhel-7) + make install.tools make localunit make ;; - centos-7) ;& # Continue to the next item - rhel-7) - stub 'unit testing not working on $OS_RELEASE_ID' - ;; *) bad_os_id_ver ;; esac diff --git a/contrib/cirrus/verify_source.sh b/contrib/cirrus/verify_source.sh deleted file mode 100755 index 860bafc00..000000000 --- a/contrib/cirrus/verify_source.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/bin/bash - -set -e -source $(dirname $0)/lib.sh - -req_env_var " -OS_RELEASE_ID $OS_RELEASE_ID -OS_RELEASE_VER $OS_RELEASE_VER -" - -show_env_vars - -set -x -cd "$GOSRC" - -case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in - ubuntu-18) - make install.tools "BUILDTAGS=$BUILDTAGS" - make validate "BUILDTAGS=$BUILDTAGS" - # make lint "BUILDTAGS=$BUILDTAGS" - ;; - fedora-28) ;& - centos-7) ;& - rhel-7) - make install.tools - make validate - # make lint - ;; - *) bad_os_id_ver ;; -esac diff --git a/contrib/python/podman/podman/libs/images.py b/contrib/python/podman/podman/libs/images.py index 6dfc4a656..ae1b86390 100644 --- a/contrib/python/podman/podman/libs/images.py +++ b/contrib/python/podman/podman/libs/images.py @@ -75,7 +75,7 @@ class Image(collections.UserDict): obj = json.loads(results['image'], object_hook=fold_keys()) return collections.namedtuple('ImageInspect', obj.keys())(**obj) - def push(self, target, tlsverify=False): + def push(self, target, tlsverify=True): """Copy image to target, return id on success.""" with self._client() as podman: results = podman.PushImage(self._id, target, tlsverify) diff --git a/contrib/python/podman/test/test_images.py b/contrib/python/podman/test/test_images.py index f97e13b4c..45f0a2964 100644 --- a/contrib/python/podman/test/test_images.py +++ b/contrib/python/podman/test/test_images.py @@ -102,7 +102,7 @@ class TestImages(PodmanTestCase): def test_push(self): path = '{}/alpine_push'.format(self.tmpdir) target = 'dir:{}'.format(path) - self.alpine_image.push(target) + self.alpine_image.push(target, tlsverify=False) self.assertTrue(os.path.isfile(os.path.join(path, 'manifest.json'))) self.assertTrue(os.path.isfile(os.path.join(path, 'version'))) diff --git a/contrib/python/pypodman/pypodman/lib/actions/inspect_action.py b/contrib/python/pypodman/pypodman/lib/actions/inspect_action.py index 514b4702a..a581e7e4e 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/inspect_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/inspect_action.py @@ -23,9 +23,9 @@ class Inspect(AbstractActionBase): help='Type of object to inspect', ) parser.add_argument( - 'size', + '--size', action='store_true', - default=True, + default=False, help='Display the total file size if the type is a container.' ' Always True.') parser.add_argument( @@ -59,7 +59,7 @@ class Inspect(AbstractActionBase): def inspect(self): """Inspect provided podman objects.""" - output = {} + output = [] try: for ident in self._args.objects: obj = None @@ -78,7 +78,13 @@ class Inspect(AbstractActionBase): msg = 'Object "{}" not found'.format(ident) print(msg, file=sys.stderr, flush=True) else: - output.update(obj._asdict()) + fields = obj._asdict() + if not self._args.size: + try: + del fields['sizerootfs'] + except KeyError: + pass + output.append(fields) except podman.ErrorOccurred as e: sys.stdout.flush() print( diff --git a/contrib/spec/podman.spec.in b/contrib/spec/podman.spec.in index 3192cbfed..20e2a84ea 100644 --- a/contrib/spec/podman.spec.in +++ b/contrib/spec/podman.spec.in @@ -39,7 +39,7 @@ %global shortcommit_conmon %(c=%{commit_conmon}; echo ${c:0:7}) Name: podman -Version: 0.11.2 +Version: 0.12.2 Release: #COMMITDATE#.git%{shortcommit0}%{?dist} Summary: Manage Pods, Containers and Container Images License: ASL 2.0 diff --git a/docs/libpod.conf.5.md b/docs/libpod.conf.5.md index 198e927ee..d63baeb88 100644 --- a/docs/libpod.conf.5.md +++ b/docs/libpod.conf.5.md @@ -24,6 +24,18 @@ libpod to manage containers. **cgroup_manager**="" Specify the CGroup Manager to use; valid values are "systemd" and "cgroupfs" +**hooks_dir**=["*path*", ...] + + Each `*.json` file in the path configures a hook for Podman containers. For more details on the syntax of the JSON files and the semantics of hook injection, see `oci-hooks(5)`. Podman and libpod currently support both the 1.0.0 and 0.1.0 hook schemas, although the 0.1.0 schema is deprecated. + + Paths listed later in the array higher precedence (`oci-hooks(5)` discusses directory precedence). + + For the annotation conditions, libpod uses any annotations set in the generated OCI configuration. + + For the bind-mount conditions, only mounts explicitly requested by the caller via `--volume` are considered. Bind mounts that libpod inserts by default (e.g. `/dev/shm`) are not considered. + + If `hooks_dir` is unset for root callers, Podman and libpod will currently default to `/usr/share/containers/oci/hooks.d` and `/etc/containers/oci/hooks.d` in order of increasing precedence. Using these defaults is deprecated, and callers should migrate to explicitly setting `hooks_dir`. + **static_dir**="" Directory for persistent libpod files (database, etc) By default this will be configured relative to where containers/storage diff --git a/docs/podman-container-prune.1.md b/docs/podman-container-prune.1.md new file mode 100644 index 000000000..1f3ef1d41 --- /dev/null +++ b/docs/podman-container-prune.1.md @@ -0,0 +1,31 @@ +% PODMAN(1) Podman Man Pages +% Brent Baude +% December 2018 +# NAME +podman-container-prune - Remove all stopped containers + +# SYNOPSIS +**podman container prune** +[**-h**|**--help**] + +# DESCRIPTION +**podman container prune** removes all stopped containers from local storage. + +## Examples ## + +Remove all stopped containers from local storage +``` +$ sudo podman container prune +878392adf2e6c5c9bb1fc19b69d37d2e98c8abf9d539c0bce4b15b46bbcce471 +37664467fbe3618bf9479c34393ac29c02696675addf1750f9e346581636cde7 +ed0c6468b8e1cb641b4621d1fe30cb477e1fefc5c0bceb66feaf2f7cb50e5962 +6ac6c8f0067b7a4682e6b8e18902665b57d1a0e07e885d9abcd382232a543ccd +fff1c5b6c3631746055ec40598ce8ecaa4b82aef122f9e3a85b03b55c0d06c23 +602d343cd47e7cb3dfc808282a9900a3e4555747787ec6723bb68cedab8384d5 +``` + +## SEE ALSO +podman(1), podman-ps + +# HISTORY +December 2018, Originally compiled by Brent Baude (bbaude at redhat dot com) diff --git a/docs/podman-container-runlabel.1.md b/docs/podman-container-runlabel.1.md index 73b7d7e15..6f7b4dae8 100644 --- a/docs/podman-container-runlabel.1.md +++ b/docs/podman-container-runlabel.1.md @@ -95,8 +95,8 @@ option be used, as the default behavior of using the system-wide default policy **--tls-verify** Require HTTPS and verify certificates when contacting registries (default: true). If explicitly set to true, -then tls verification will be used, If set to false then tls verification will not be used. If not specified -tls verification will be used unless the target registry is listed as an insecure registry in registries.conf +then TLS verification will be used. If set to false, then TLS verification will not be used. If not specified, +TLS verification will be used unless the target registry is listed as an insecure registry in registries.conf ## Examples ## diff --git a/docs/podman-container.1.md b/docs/podman-container.1.md index aa5dfa82c..3675d9719 100644 --- a/docs/podman-container.1.md +++ b/docs/podman-container.1.md @@ -29,6 +29,7 @@ The container command allows you to manage containers | mount | [podman-mount(1)](podman-mount.1.md) | Mount a working container's root filesystem. | | pause | [podman-pause(1)](podman-pause.1.md) | Pause one or more containers. | | port | [podman-port(1)](podman-port.1.md) | List port mappings for the container. | +| prune | [podman-container-prune(1)](podman-container-prune.1.md) | Remove all stopped containers from local storage | | refresh | [podman-refresh(1)](podman-container-refresh.1.md) | Refresh the state of all containers | | restart | [podman-restart(1)](podman-restart.1.md) | Restart one or more containers. | | restore | [podman-container-restore(1)](podman-container-restore.1.md) | Restores one or more containers from a checkpoint. | diff --git a/docs/podman-create.1.md b/docs/podman-create.1.md index 6fbdd0d03..f1409a554 100644 --- a/docs/podman-create.1.md +++ b/docs/podman-create.1.md @@ -455,7 +455,8 @@ Tune the container's pids limit. Set `-1` to have unlimited pids for the contain **--pod**="" -Run container in an existing pod +Run container in an existing pod. If you want podman to make the pod for you, preference the pod name with `new:`. +To make a pod with more granular options, use the `podman pod create` command before creating a container. **--privileged**=*true*|*false* diff --git a/docs/podman-image-prune.1.md b/docs/podman-image-prune.1.md new file mode 100644 index 000000000..db76b26e0 --- /dev/null +++ b/docs/podman-image-prune.1.md @@ -0,0 +1,32 @@ +% PODMAN(1) Podman Man Pages +% Brent Baude +% December 2018 +# NAME +podman-image-prune - Remove all unused images + +# SYNOPSIS +**podman image prune** +[**-h**|**--help**] + +# DESCRIPTION +**podman image prune** removes all unused images from local storage. An unused image +is defined as an image that does not have any containers based on it. + +## Examples ## + +Remove all unused images from local storage +``` +$ sudo podman image prune +f3e20dc537fb04cb51672a5cb6fdf2292e61d411315549391a0d1f64e4e3097e +324a7a3b2e0135f4226ffdd473e4099fd9e477a74230cdc35de69e84c0f9d907 +6125002719feb1ddf3030acab1df6156da7ce0e78e571e9b6e9c250424d6220c +91e732da5657264c6f4641b8d0c4001c218ae6c1adb9dcef33ad00cafd37d8b6 +e4e5109420323221f170627c138817770fb64832da7d8fe2babd863148287fca +77a57fa8285e9656dbb7b23d9efa837a106957409ddd702f995605af27a45ebe +``` + +## SEE ALSO +podman(1), podman-images + +# HISTORY +December 2018, Originally compiled by Brent Baude (bbaude at redhat dot com) diff --git a/docs/podman-image.1.md b/docs/podman-image.1.md index 446f8667d..8b812af11 100644 --- a/docs/podman-image.1.md +++ b/docs/podman-image.1.md @@ -21,6 +21,7 @@ The image command allows you to manage images | load | [podman-load(1)](podman-load.1.md) | Load an image from the docker archive. | | ls | [podman-images(1)](podman-images.1.md) | Prints out information about images. | | pull | [podman-pull(1)](podman-pull.1.md) | Pull an image from a registry. | +| prune| [podman-container-prune(1)](podman-container-prune.1.md) | Removed all unused images from the local store | | push | [podman-push(1)](podman-push.1.md) | Push an image from local storage to elsewhere. | | rm | [podman-rm(1)](podman-rmi.1.md) | Removes one or more locally stored images. | | save | [podman-save(1)](podman-save.1.md) | Save an image to docker-archive or oci. | diff --git a/docs/podman-login.1.md b/docs/podman-login.1.md index a3ee2929c..7c033d7c5 100644 --- a/docs/podman-login.1.md +++ b/docs/podman-login.1.md @@ -43,7 +43,9 @@ Default certificates directory is _/etc/containers/certs.d_. **--tls-verify** -Require HTTPS and verify certificates when contacting registries (default: true) +Require HTTPS and verify certificates when contacting registries (default: true). If explicitly set to true, +then TLS verification will be used. If set to false, then TLS verification will not be used. If not specified, +TLS verification will be used unless the target registry is listed as an insecure registry in registries.conf. **--help**, **-h** diff --git a/docs/podman-ps.1.md b/docs/podman-ps.1.md index 7333a1095..8b86703d8 100644 --- a/docs/podman-ps.1.md +++ b/docs/podman-ps.1.md @@ -103,6 +103,13 @@ Valid filters are listed below: Print usage statement +**--sync** + +Force a sync of container state with the OCI runtime. +In some cases, a container's state in the runtime can become out of sync with Podman's state. +This will update Podman's state based on what the OCI runtime reports. +Forcibly syncing is much slower, but can resolve inconsistent state issues. + ## EXAMPLES ``` diff --git a/docs/podman-pull.1.md b/docs/podman-pull.1.md index 86c6823af..2196e251e 100644 --- a/docs/podman-pull.1.md +++ b/docs/podman-pull.1.md @@ -77,8 +77,8 @@ option be used, as the default behavior of using the system-wide default policy **--tls-verify** Require HTTPS and verify certificates when contacting registries (default: true). If explicitly set to true, -then tls verification will be used, If set to false then tls verification will not be used. If not specified -tls verification will be used unless the target registry is listed as an insecure registry in registries.conf. +then TLS verification will be used. If set to false, then TLS verification will not be used. If not specified, +TLS verification will be used unless the target registry is listed as an insecure registry in registries.conf. **--help**, **-h** diff --git a/docs/podman-push.1.md b/docs/podman-push.1.md index 537988ea0..3ce156010 100644 --- a/docs/podman-push.1.md +++ b/docs/podman-push.1.md @@ -93,7 +93,9 @@ Add a signature at the destination using the specified key **--tls-verify** -Require HTTPS and verify certificates when contacting registries (default: true) +Require HTTPS and verify certificates when contacting registries (default: true). If explicitly set to true, +then TLS verification will be used. If set to false, then TLS verification will not be used. If not specified, +TLS verification will be used unless the target registry is listed as an insecure registry in registries.conf. ## EXAMPLE diff --git a/docs/podman-rmi.1.md b/docs/podman-rmi.1.md index f035897ee..9c080c9f1 100644 --- a/docs/podman-rmi.1.md +++ b/docs/podman-rmi.1.md @@ -19,15 +19,25 @@ Remove all images in the local storage. This option will cause podman to remove all containers that are using the image before removing the image from the system. -## EXAMPLE - -podman rmi imageID +Remove an image by its short ID +``` +podman rmi c0ed59d05ff7 +``` +Remove an image and its associated containers. +``` podman rmi --force imageID +```` -podman rmi imageID1 imageID2 imageID3 +Remove multiple images by their shortened IDs. +``` +podman rmi c4dfb1609ee2 93fd78260bd1 c0ed59d05ff7 +``` +Remove all images and containers. +``` podman rmi -a -f +``` ## SEE ALSO podman(1) diff --git a/docs/podman-run.1.md b/docs/podman-run.1.md index a6761a393..5917f6f7a 100644 --- a/docs/podman-run.1.md +++ b/docs/podman-run.1.md @@ -439,7 +439,8 @@ Tune the container's pids limit. Set `-1` to have unlimited pids for the contain **--pod**="" -Run container in an existing pod +Run container in an existing pod. If you want podman to make the pod for you, preference the pod name with `new:`. +To make a pod with more granular options, use the `podman pod create` command before creating a container. **--privileged**=*true*|*false* diff --git a/docs/podman-search.1.md b/docs/podman-search.1.md index ea1228f94..61f50f1dc 100644 --- a/docs/podman-search.1.md +++ b/docs/podman-search.1.md @@ -72,8 +72,8 @@ Do not truncate the output **--tls-verify** Require HTTPS and verify certificates when contacting registries (default: true). If explicitly set to true, -then tls verification will be used. If set to false then tls verification will not be used if needed. If not specified -default registries will be searched through (in /etc/containers/registries.conf), and tls will be skipped if a default +then TLS verification will be used. If set to false, then TLS verification will not be used if needed. If not specified, +default registries will be searched through (in /etc/containers/registries.conf), and TLS will be skipped if a default registry is listed in the insecure registries. **--help**, **-h** diff --git a/docs/podman-volume-create.1.md b/docs/podman-volume-create.1.md new file mode 100644 index 000000000..795d7b449 --- /dev/null +++ b/docs/podman-volume-create.1.md @@ -0,0 +1,48 @@ +% podman-volume-create(1) + +## NAME +podman\-volume\-create - Create a new volume + +## SYNOPSIS +**podman volume create** [*options*] + +## DESCRIPTION + +Creates an empty volume and prepares it to be used by containers. The volume +can be created with a specific name, if a name is not given a random name is +generated. You can add metadata to the volume by using the **--label** flag and +driver options can be set using the **--opt** flag. + +## OPTIONS + +**--driver**="" + +Specify the volume driver name (default local). + +**--help** + +Print usage statement + +**-l**, **--label**=[] + +Set metadata for a volume (e.g., --label mykey=value). + +**-o**, **--opt**=[] + +Set driver specific options. + +## EXAMPLES + +``` +$ podman volume create myvol + +$ podman volume create + +$ podman volume create --label foo=bar myvol +``` + +## SEE ALSO +podman-volume(1) + +## HISTORY +November 2018, Originally compiled by Urvashi Mohnani <umohnani@redhat.com> diff --git a/docs/podman-volume-inspect.1.md b/docs/podman-volume-inspect.1.md new file mode 100644 index 000000000..6d5b184ee --- /dev/null +++ b/docs/podman-volume-inspect.1.md @@ -0,0 +1,45 @@ +% podman-volume-inspect(1) + +## NAME +podman\-volume\-inspect - Inspect one or more volumes + +## SYNOPSIS +**podman volume inspect** [*options*] + +## DESCRIPTION + +Display detailed information on one or more volumes. The output can be formated using +the **--format** flag and a Go template. To get detailed information about all the +existing volumes, use the **--all** flag. + + +## OPTIONS + +**-a**, **--all**="" + +Inspect all volumes. + +**--format**="" + +Format volume output using Go template + +**--help** + +Print usage statement + + +## EXAMPLES + +``` +$ podman volume inspect myvol + +$ podman volume inspect --all + +$ podman volume inspect --format "{{.Driver}} {{.Scope}}" myvol +``` + +## SEE ALSO +podman-volume(1) + +## HISTORY +November 2018, Originally compiled by Urvashi Mohnani <umohnani@redhat.com> diff --git a/docs/podman-volume-ls.1.md b/docs/podman-volume-ls.1.md new file mode 100644 index 000000000..c061e27fe --- /dev/null +++ b/docs/podman-volume-ls.1.md @@ -0,0 +1,49 @@ +% podman-volume-ls(1) + +## NAME +podman\-volume\-ls - List volumes + +## SYNOPSIS +**podman volume ls** [*options*] + +## DESCRIPTION + +Lists all the volumes that exist. The output can be filtered using the **--filter** +flag and can be formatted to either JSON or a Go template using the **--format** +flag. Use the **--quiet** flag to print only the volume names. + +## OPTIONS + +**--filter**="" + +Filter volume output. + +**--format**="" + +Format volume output using Go template. + +**--help** + +Print usage statement. + +**-q**, **--quiet**=[] + +Print volume output in quiet mode. Only print the volume names. + +## EXAMPLES + +``` +$ podman volume ls + +$ podman volume ls --format json + +$ podman volume ls --format "{{.Driver}} {{.Scope}}" + +$ podman volume ls --filter name=foo,label=blue +``` + +## SEE ALSO +podman-volume(1) + +## HISTORY +November 2018, Originally compiled by Urvashi Mohnani <umohnani@redhat.com> diff --git a/docs/podman-volume-prune.1.md b/docs/podman-volume-prune.1.md new file mode 100644 index 000000000..a06bb2fa4 --- /dev/null +++ b/docs/podman-volume-prune.1.md @@ -0,0 +1,38 @@ +% podman-volume-prune(1) + +## NAME +podman\-volume\-prune - Remove all unused volumes + +## SYNOPSIS +**podman volume rm** [*options*] + +## DESCRIPTION + +Removes all unused volumes. You will be prompted to confirm the removal of all the +unused volumes. To bypass the confirmation, use the **--force** flag. + + +## OPTIONS + +**-f**, **--force**="" + +Do not prompt for confirmation. + +**--help** + +Print usage statement + + +## EXAMPLES + +``` +$ podman volume prune + +$ podman volume prune --force +``` + +## SEE ALSO +podman-volume(1) + +## HISTORY +November 2018, Originally compiled by Urvashi Mohnani <umohnani@redhat.com> diff --git a/docs/podman-volume-rm.1.md b/docs/podman-volume-rm.1.md new file mode 100644 index 000000000..c23d7675c --- /dev/null +++ b/docs/podman-volume-rm.1.md @@ -0,0 +1,45 @@ +% podman-volume-rm(1) + +## NAME +podman\-volume\-rm - Remove one or more volumes + +## SYNOPSIS +**podman volume rm** [*options*] + +## DESCRIPTION + +Removes one ore more volumes. Only volumes that are not being used will be removed. +If a volume is being used by a container, an error will be returned unless the **--force** +flag is being used. To remove all the volumes, use the **--all** flag. + + +## OPTIONS + +**-a**, **--all**="" + +Remove all volumes. + +**-f**, **--force**="" + +Remove a volume by force, even if it is being used by a container + +**--help** + +Print usage statement + + +## EXAMPLES + +``` +$ podman volume rm myvol1 myvol2 + +$ podman volume rm --all + +$ podman volume rm --force myvol +``` + +## SEE ALSO +podman-volume(1) + +## HISTORY +November 2018, Originally compiled by Urvashi Mohnani <umohnani@redhat.com> diff --git a/docs/podman-volume.1.md b/docs/podman-volume.1.md new file mode 100644 index 000000000..ac32abbd6 --- /dev/null +++ b/docs/podman-volume.1.md @@ -0,0 +1,23 @@ +% podman-volume(1) + +## NAME +podman\-volume - Simple management tool for volumes. + +## SYNOPSIS +**podman volume** *subcommand* + +## DESCRIPTION +podman volume is a set of subcommands that manage volumes. + +## SUBCOMMANDS + +| Subcommand | Description | +| ------------------------------------------------- | ------------------------------------------------------------------------------ | +| [podman-volume-create(1)](podman-volume-create.1.md) | Create a new volume. | +| [podman-volume-inspect(1)](podman-volume-inspect.1.md) | Get detailed information on one or more volumes. | +| [podman-volume-ls(1)](podman-volume-ls.1.md) | List all the available volumes. | +| [podman-volume-rm(1)](podman-volume-rm.1.md) | Remove one or more volumes. | +| [podman-volume-prune(1)](podman-volume-prune.1.md) | Remove all unused volumes. | + +## HISTORY +November 2018, Originally compiled by Urvashi Mohnani <umohnani@redhat.com> diff --git a/docs/podman.1.md b/docs/podman.1.md index b7433d850..bde349e6f 100644 --- a/docs/podman.1.md +++ b/docs/podman.1.md @@ -31,6 +31,18 @@ CGroup manager to use for container cgroups. Supported values are cgroupfs or sy Path to where the cpu performance results should be written +**--hooks-dir**=**path** + +Each `*.json` file in the path configures a hook for Podman containers. For more details on the syntax of the JSON files and the semantics of hook injection, see `oci-hooks(5)`. Podman and libpod currently support both the 1.0.0 and 0.1.0 hook schemas, although the 0.1.0 schema is deprecated. + +This option may be set multiple times; paths from later options have higher precedence (`oci-hooks(5)` discusses directory precedence). + +For the annotation conditions, libpod uses any annotations set in the generated OCI configuration. + +For the bind-mount conditions, only mounts explicitly requested by the caller via `--volume` are considered. Bind mounts that libpod inserts by default (e.g. `/dev/shm`) are not considered. + +If `--hooks-dir` is unset for root callers, Podman and libpod will currently default to `/usr/share/containers/oci/hooks.d` and `/etc/containers/oci/hooks.d` in order of increasing precedence. Using these defaults is deprecated, and callers should migrate to explicitly setting `--hooks-dir`. + **--log-level** Log messages above specified level: debug, info, warn, error (default), fatal or panic @@ -161,18 +173,6 @@ the exit codes follow the `chroot` standard, see below: The mounts.conf file specifies volume mount directories that are automatically mounted inside containers when executing the `podman run` or `podman start` commands. When Podman runs in rootless mode, the file `$HOME/.config/containers/mounts.conf` is also used. Please refer to containers-mounts.conf(5) for further details. -**OCI hooks JSON** (`/etc/containers/oci/hooks.d/*.json`, `/usr/share/containers/oci/hooks.d/*.json`) - - Each `*.json` file in `/etc/containers/oci/hooks.d` and `/usr/share/containers/oci/hooks.d` configures a hook for Podman containers, with `/etc/containers/oci/hooks.d` having higher precedence. For more details on the syntax of the JSON files and the semantics of hook injection, see `oci-hooks(5)`. - - Podman and libpod currently support both the 1.0.0 and 0.1.0 hook schemas, although the 0.1.0 schema is deprecated. - - For the annotation conditions, libpod uses any annotations set in the generated OCI configuration. - - For the bind-mount conditions, only mounts explicitly requested by the caller via `--volume` are considered. Bind mounts that libpod inserts by default (e.g. `/dev/shm`) are not considered. - - Hooks are not used when running in rootless mode. - **policy.json** (`/etc/containers/policy.json`) Signature verification policy files are used to specify policy, e.g. trusted keys, applicable when deciding whether to accept an image, or individual signatures of that image, as valid. diff --git a/docs/tutorials/podman_tutorial.md b/docs/tutorials/podman_tutorial.md index ce94d7d15..659973b28 100644 --- a/docs/tutorials/podman_tutorial.md +++ b/docs/tutorials/podman_tutorial.md @@ -24,7 +24,7 @@ acquire the source, and build it. sudo dnf install -y git runc libassuan-devel golang golang-github-cpuguy83-go-md2man glibc-static \ gpgme-devel glib2-devel device-mapper-devel libseccomp-devel \ atomic-registries iptables skopeo-containers containernetworking-cni \ - conmon + conmon ostree-devel ``` ### Building and installing podman @@ -54,7 +54,7 @@ tutorial. For this tutorial, the Ubuntu **artful-server-cloudimg** image was use #### Installing base packages ```console sudo apt-get update -sudo apt-get install libdevmapper-dev libglib2.0-dev libgpgme11-dev golang libseccomp-dev \ +sudo apt-get install libdevmapper-dev libglib2.0-dev libgpgme11-dev golang libseccomp-dev libostree-dev \ go-md2man libprotobuf-dev libprotobuf-c0-dev libseccomp-dev python3-setuptools ``` #### Building and installing conmon diff --git a/libpod/boltdb_state.go b/libpod/boltdb_state.go index 42f029379..b154d8bda 100644 --- a/libpod/boltdb_state.go +++ b/libpod/boltdb_state.go @@ -3,7 +3,6 @@ package libpod import ( "bytes" "encoding/json" - "os" "strings" "sync" @@ -19,7 +18,6 @@ type BoltState struct { dbLock sync.Mutex namespace string namespaceBytes []byte - lockDir string runtime *Runtime } @@ -52,25 +50,15 @@ type BoltState struct { // containers/storage do not occur. // NewBoltState creates a new bolt-backed state database -func NewBoltState(path, lockDir string, runtime *Runtime) (State, error) { +func NewBoltState(path string, runtime *Runtime) (State, error) { state := new(BoltState) state.dbPath = path - state.lockDir = lockDir state.runtime = runtime state.namespace = "" state.namespaceBytes = nil logrus.Debugf("Initializing boltdb state at %s", path) - // Make the directory that will hold container lockfiles - if err := os.MkdirAll(lockDir, 0750); err != nil { - // The directory is allowed to exist - if !os.IsExist(err) { - return nil, errors.Wrapf(err, "error creating lockfiles dir %s", lockDir) - } - } - state.lockDir = lockDir - db, err := bolt.Open(path, 0600, nil) if err != nil { return nil, errors.Wrapf(err, "error opening database %s", path) @@ -106,6 +94,12 @@ func NewBoltState(path, lockDir string, runtime *Runtime) (State, error) { if _, err := tx.CreateBucketIfNotExists(allPodsBkt); err != nil { return errors.Wrapf(err, "error creating all pods bucket") } + if _, err := tx.CreateBucketIfNotExists(volBkt); err != nil { + return errors.Wrapf(err, "error creating volume bucket") + } + if _, err := tx.CreateBucketIfNotExists(allVolsBkt); err != nil { + return errors.Wrapf(err, "error creating all volumes bucket") + } if _, err := tx.CreateBucketIfNotExists(runtimeConfigBkt); err != nil { return errors.Wrapf(err, "error creating runtime-config bucket") } @@ -115,11 +109,6 @@ func NewBoltState(path, lockDir string, runtime *Runtime) (State, error) { return nil, errors.Wrapf(err, "error creating initial database layout") } - // Check runtime configuration - if err := checkRuntimeConfig(db, runtime); err != nil { - return nil, err - } - state.valid = true return state, nil @@ -240,6 +229,72 @@ func (s *BoltState) Refresh() error { return err } +// GetDBConfig retrieves runtime configuration fields that were created when +// the database was first initialized +func (s *BoltState) GetDBConfig() (*DBConfig, error) { + if !s.valid { + return nil, ErrDBClosed + } + + cfg := new(DBConfig) + + db, err := s.getDBCon() + if err != nil { + return nil, err + } + defer s.closeDBCon(db) + + err = db.View(func(tx *bolt.Tx) error { + configBucket, err := getRuntimeConfigBucket(tx) + if err != nil { + return nil + } + + // Some of these may be nil + // When we convert to string, Go will coerce them to "" + // That's probably fine - we could raise an error if the key is + // missing, but just not including it is also OK. + libpodRoot := configBucket.Get(staticDirKey) + libpodTmp := configBucket.Get(tmpDirKey) + storageRoot := configBucket.Get(graphRootKey) + storageTmp := configBucket.Get(runRootKey) + graphDriver := configBucket.Get(graphDriverKey) + + cfg.LibpodRoot = string(libpodRoot) + cfg.LibpodTmp = string(libpodTmp) + cfg.StorageRoot = string(storageRoot) + cfg.StorageTmp = string(storageTmp) + cfg.GraphDriver = string(graphDriver) + + return nil + }) + if err != nil { + return nil, err + } + + return cfg, nil +} + +// ValidateDBConfig validates paths in the given runtime against the database +func (s *BoltState) ValidateDBConfig(runtime *Runtime) error { + if !s.valid { + return ErrDBClosed + } + + db, err := s.getDBCon() + if err != nil { + return err + } + defer s.closeDBCon(db) + + // Check runtime configuration + if err := checkRuntimeConfig(db, runtime); err != nil { + return err + } + + return nil +} + // SetNamespace sets the namespace that will be used for container and pod // retrieval func (s *BoltState) SetNamespace(ns string) error { @@ -1101,6 +1156,378 @@ func (s *BoltState) PodContainers(pod *Pod) ([]*Container, error) { return ctrs, nil } +// AddVolume adds the given volume to the state. It also adds ctrDepID to +// the sub bucket holding the container dependencies that this volume has +func (s *BoltState) AddVolume(volume *Volume) error { + if !s.valid { + return ErrDBClosed + } + + if !volume.valid { + return ErrVolumeRemoved + } + + volName := []byte(volume.Name()) + + volConfigJSON, err := json.Marshal(volume.config) + if err != nil { + return errors.Wrapf(err, "error marshalling volume %s config to JSON", volume.Name()) + } + + db, err := s.getDBCon() + if err != nil { + return err + } + defer s.closeDBCon(db) + + err = db.Update(func(tx *bolt.Tx) error { + volBkt, err := getVolBucket(tx) + if err != nil { + return err + } + + allVolsBkt, err := getAllVolsBucket(tx) + if err != nil { + return err + } + + // Check if we already have a volume with the given name + volExists := allVolsBkt.Get(volName) + if volExists != nil { + return errors.Wrapf(ErrVolumeExists, "name %s is in use", volume.Name()) + } + + // We are good to add the volume + // Make a bucket for it + newVol, err := volBkt.CreateBucket(volName) + if err != nil { + return errors.Wrapf(err, "error creating bucket for volume %s", volume.Name()) + } + + // Make a subbucket for the containers using the volume. Dependent container IDs will be addedremoved to + // this bucket in addcontainer/removeContainer + if _, err := newVol.CreateBucket(volDependenciesBkt); err != nil { + return errors.Wrapf(err, "error creating bucket for containers using volume %s", volume.Name()) + } + + if err := newVol.Put(configKey, volConfigJSON); err != nil { + return errors.Wrapf(err, "error storing volume %s configuration in DB", volume.Name()) + } + + if err := allVolsBkt.Put(volName, volName); err != nil { + return errors.Wrapf(err, "error storing volume %s in all volumes bucket in DB", volume.Name()) + } + + return nil + }) + return err +} + +// RemoveVolCtrDep updates the container dependencies sub bucket of the given volume. +// It deletes it from the bucket when found. +// This is important when force removing a volume and we want to get rid of the dependencies. +func (s *BoltState) RemoveVolCtrDep(volume *Volume, ctrID string) error { + if ctrID == "" { + return nil + } + + if !s.valid { + return ErrDBBadConfig + } + + if !volume.valid { + return ErrVolumeRemoved + } + + volName := []byte(volume.Name()) + + db, err := s.getDBCon() + if err != nil { + return err + } + defer s.closeDBCon(db) + + err = db.Update(func(tx *bolt.Tx) error { + volBkt, err := getVolBucket(tx) + if err != nil { + return err + } + + volDB := volBkt.Bucket(volName) + if volDB == nil { + volume.valid = false + return errors.Wrapf(ErrNoSuchVolume, "no volume with name %s found in database", volume.Name()) + } + + // Make a subbucket for the containers using the volume + ctrDepsBkt := volDB.Bucket(volDependenciesBkt) + depCtrID := []byte(ctrID) + if depExists := ctrDepsBkt.Get(depCtrID); depExists != nil { + if err := ctrDepsBkt.Delete(depCtrID); err != nil { + return errors.Wrapf(err, "error deleting container dependencies %q for volume %s in ctrDependencies bucket in DB", ctrID, volume.Name()) + } + } + + return nil + }) + return err +} + +// RemoveVolume removes the given volume from the state +func (s *BoltState) RemoveVolume(volume *Volume) error { + if !s.valid { + return ErrDBClosed + } + + if !volume.valid { + return ErrVolumeRemoved + } + + volName := []byte(volume.Name()) + + db, err := s.getDBCon() + if err != nil { + return err + } + defer s.closeDBCon(db) + + err = db.Update(func(tx *bolt.Tx) error { + volBkt, err := getVolBucket(tx) + if err != nil { + return err + } + + allVolsBkt, err := getAllVolsBucket(tx) + if err != nil { + return err + } + + // Check if the volume exists + volDB := volBkt.Bucket(volName) + if volDB == nil { + volume.valid = false + return errors.Wrapf(ErrNoSuchVolume, "volume %s does not exist in DB", volume.Name()) + } + + // Check if volume is not being used by any container + // This should never be nil + // But if it is, we can assume that no containers are using + // the volume. + volCtrsBkt := volDB.Bucket(volDependenciesBkt) + if volCtrsBkt != nil { + var deps []string + err = volCtrsBkt.ForEach(func(id, value []byte) error { + deps = append(deps, string(id)) + return nil + }) + if err != nil { + return errors.Wrapf(err, "error getting list of dependencies from dependencies bucket for volumes %q", volume.Name()) + } + if len(deps) > 0 { + return errors.Wrapf(ErrVolumeBeingUsed, "volume %s is being used by container(s) %s", volume.Name(), strings.Join(deps, ",")) + } + } + + // volume is ready for removal + // Let's kick it out + if err := allVolsBkt.Delete(volName); err != nil { + return errors.Wrapf(err, "error removing volume %s from all volumes bucket in DB", volume.Name()) + } + if err := volBkt.DeleteBucket(volName); err != nil { + return errors.Wrapf(err, "error removing volume %s from DB", volume.Name()) + } + + return nil + }) + return err +} + +// AllVolumes returns all volumes present in the state +func (s *BoltState) AllVolumes() ([]*Volume, error) { + if !s.valid { + return nil, ErrDBClosed + } + + volumes := []*Volume{} + + db, err := s.getDBCon() + if err != nil { + return nil, err + } + defer s.closeDBCon(db) + + err = db.View(func(tx *bolt.Tx) error { + allVolsBucket, err := getAllVolsBucket(tx) + if err != nil { + return err + } + + volBucket, err := getVolBucket(tx) + if err != nil { + return err + } + err = allVolsBucket.ForEach(func(id, name []byte) error { + volExists := volBucket.Bucket(id) + // This check can be removed if performance becomes an + // issue, but much less helpful errors will be produced + if volExists == nil { + return errors.Wrapf(ErrInternal, "inconsistency in state - volume %s is in all volumes bucket but volume not found", string(id)) + } + + volume := new(Volume) + volume.config = new(VolumeConfig) + + if err := s.getVolumeFromDB(id, volume, volBucket); err != nil { + if errors.Cause(err) != ErrNSMismatch { + logrus.Errorf("Error retrieving volume %s from the database: %v", string(id), err) + } + } else { + volumes = append(volumes, volume) + } + + return nil + }) + return err + }) + if err != nil { + return nil, err + } + + return volumes, nil +} + +// Volume retrieves a volume from full name +func (s *BoltState) Volume(name string) (*Volume, error) { + if name == "" { + return nil, ErrEmptyID + } + + if !s.valid { + return nil, ErrDBClosed + } + + volName := []byte(name) + + volume := new(Volume) + volume.config = new(VolumeConfig) + + db, err := s.getDBCon() + if err != nil { + return nil, err + } + defer s.closeDBCon(db) + + err = db.View(func(tx *bolt.Tx) error { + volBkt, err := getVolBucket(tx) + if err != nil { + return err + } + + return s.getVolumeFromDB(volName, volume, volBkt) + }) + if err != nil { + return nil, err + } + + return volume, nil +} + +// HasVolume returns true if the given volume exists in the state, otherwise it returns false +func (s *BoltState) HasVolume(name string) (bool, error) { + if name == "" { + return false, ErrEmptyID + } + + if !s.valid { + return false, ErrDBClosed + } + + volName := []byte(name) + + exists := false + + db, err := s.getDBCon() + if err != nil { + return false, err + } + defer s.closeDBCon(db) + + err = db.View(func(tx *bolt.Tx) error { + volBkt, err := getVolBucket(tx) + if err != nil { + return err + } + + volDB := volBkt.Bucket(volName) + if volDB != nil { + exists = true + } + + return nil + }) + if err != nil { + return false, err + } + + return exists, nil +} + +// VolumeInUse checks if any container is using the volume +// It returns a slice of the IDs of the containers using the given +// volume. If the slice is empty, no containers use the given volume +func (s *BoltState) VolumeInUse(volume *Volume) ([]string, error) { + if !s.valid { + return nil, ErrDBClosed + } + + if !volume.valid { + return nil, ErrVolumeRemoved + } + + depCtrs := []string{} + + db, err := s.getDBCon() + if err != nil { + return nil, err + } + defer s.closeDBCon(db) + + err = db.View(func(tx *bolt.Tx) error { + volBucket, err := getVolBucket(tx) + if err != nil { + return err + } + + volDB := volBucket.Bucket([]byte(volume.Name())) + if volDB == nil { + volume.valid = false + return errors.Wrapf(ErrNoSuchVolume, "no volume with name %s found in DB", volume.Name()) + } + + dependsBkt := volDB.Bucket(volDependenciesBkt) + if dependsBkt == nil { + return errors.Wrapf(ErrInternal, "volume %s has no dependencies bucket", volume.Name()) + } + + // Iterate through and add dependencies + err = dependsBkt.ForEach(func(id, value []byte) error { + depCtrs = append(depCtrs, string(id)) + + return nil + }) + if err != nil { + return err + } + + return nil + }) + if err != nil { + return nil, err + } + + return depCtrs, nil +} + // AddPod adds the given pod to the state. func (s *BoltState) AddPod(pod *Pod) error { if !s.valid { diff --git a/libpod/boltdb_state_internal.go b/libpod/boltdb_state_internal.go index cc7d106cc..0970f4d41 100644 --- a/libpod/boltdb_state_internal.go +++ b/libpod/boltdb_state_internal.go @@ -21,15 +21,25 @@ const ( allCtrsName = "all-ctrs" podName = "pod" allPodsName = "allPods" + volName = "vol" + allVolsName = "allVolumes" runtimeConfigName = "runtime-config" - configName = "config" - stateName = "state" - dependenciesName = "dependencies" - netNSName = "netns" - containersName = "containers" - podIDName = "pod-id" - namespaceName = "namespace" + configName = "config" + stateName = "state" + dependenciesName = "dependencies" + volCtrDependencies = "vol-dependencies" + netNSName = "netns" + containersName = "containers" + podIDName = "pod-id" + namespaceName = "namespace" + + staticDirName = "static-dir" + tmpDirName = "tmp-dir" + runRootName = "run-root" + graphRootName = "graph-root" + graphDriverName = "graph-driver-name" + osName = "os" ) var ( @@ -40,30 +50,31 @@ var ( allCtrsBkt = []byte(allCtrsName) podBkt = []byte(podName) allPodsBkt = []byte(allPodsName) + volBkt = []byte(volName) + allVolsBkt = []byte(allVolsName) runtimeConfigBkt = []byte(runtimeConfigName) - configKey = []byte(configName) - stateKey = []byte(stateName) - dependenciesBkt = []byte(dependenciesName) - netNSKey = []byte(netNSName) - containersBkt = []byte(containersName) - podIDKey = []byte(podIDName) - namespaceKey = []byte(namespaceName) + configKey = []byte(configName) + stateKey = []byte(stateName) + dependenciesBkt = []byte(dependenciesName) + volDependenciesBkt = []byte(volCtrDependencies) + netNSKey = []byte(netNSName) + containersBkt = []byte(containersName) + podIDKey = []byte(podIDName) + namespaceKey = []byte(namespaceName) + + staticDirKey = []byte(staticDirName) + tmpDirKey = []byte(tmpDirName) + runRootKey = []byte(runRootName) + graphRootKey = []byte(graphRootName) + graphDriverKey = []byte(graphDriverName) + osKey = []byte(osName) ) // Check if the configuration of the database is compatible with the // configuration of the runtime opening it // If there is no runtime configuration loaded, load our own func checkRuntimeConfig(db *bolt.DB, rt *Runtime) error { - var ( - staticDir = []byte("static-dir") - tmpDir = []byte("tmp-dir") - runRoot = []byte("run-root") - graphRoot = []byte("graph-root") - graphDriverName = []byte("graph-driver-name") - osKey = []byte("os") - ) - err := db.Update(func(tx *bolt.Tx) error { configBkt, err := getRuntimeConfigBucket(tx) if err != nil { @@ -74,31 +85,31 @@ func checkRuntimeConfig(db *bolt.DB, rt *Runtime) error { return err } - if err := validateDBAgainstConfig(configBkt, "static dir", - rt.config.StaticDir, staticDir, ""); err != nil { + if err := validateDBAgainstConfig(configBkt, "libpod root directory (staticdir)", + rt.config.StaticDir, staticDirKey, ""); err != nil { return err } - if err := validateDBAgainstConfig(configBkt, "tmp dir", - rt.config.TmpDir, tmpDir, ""); err != nil { + if err := validateDBAgainstConfig(configBkt, "libpod temporary files directory (tmpdir)", + rt.config.TmpDir, tmpDirKey, ""); err != nil { return err } - if err := validateDBAgainstConfig(configBkt, "run root", - rt.config.StorageConfig.RunRoot, runRoot, + if err := validateDBAgainstConfig(configBkt, "storage temporary directory (runroot)", + rt.config.StorageConfig.RunRoot, runRootKey, storage.DefaultStoreOptions.RunRoot); err != nil { return err } - if err := validateDBAgainstConfig(configBkt, "graph root", - rt.config.StorageConfig.GraphRoot, graphRoot, + if err := validateDBAgainstConfig(configBkt, "storage graph root directory (graphroot)", + rt.config.StorageConfig.GraphRoot, graphRootKey, storage.DefaultStoreOptions.GraphRoot); err != nil { return err } - return validateDBAgainstConfig(configBkt, "graph driver name", + return validateDBAgainstConfig(configBkt, "storage graph driver", rt.config.StorageConfig.GraphDriverName, - graphDriverName, + graphDriverKey, storage.DefaultStoreOptions.GraphDriverName) }) @@ -229,6 +240,22 @@ func getAllPodsBucket(tx *bolt.Tx) (*bolt.Bucket, error) { return bkt, nil } +func getVolBucket(tx *bolt.Tx) (*bolt.Bucket, error) { + bkt := tx.Bucket(volBkt) + if bkt == nil { + return nil, errors.Wrapf(ErrDBBadConfig, "volumes bucket not found in DB") + } + return bkt, nil +} + +func getAllVolsBucket(tx *bolt.Tx) (*bolt.Bucket, error) { + bkt := tx.Bucket(allVolsBkt) + if bkt == nil { + return nil, errors.Wrapf(ErrDBBadConfig, "all volumes bucket not found in DB") + } + return bkt, nil +} + func getRuntimeConfigBucket(tx *bolt.Tx) (*bolt.Bucket, error) { bkt := tx.Bucket(runtimeConfigBkt) if bkt == nil { @@ -261,7 +288,7 @@ func (s *BoltState) getContainerFromDB(id []byte, ctr *Container, ctrsBkt *bolt. } // Get the lock - lockPath := filepath.Join(s.lockDir, string(id)) + lockPath := filepath.Join(s.runtime.lockDir, string(id)) lock, err := storage.GetLockfile(lockPath) if err != nil { return errors.Wrapf(err, "error retrieving lockfile for container %s", string(id)) @@ -297,7 +324,7 @@ func (s *BoltState) getPodFromDB(id []byte, pod *Pod, podBkt *bolt.Bucket) error } // Get the lock - lockPath := filepath.Join(s.lockDir, string(id)) + lockPath := filepath.Join(s.runtime.lockDir, string(id)) lock, err := storage.GetLockfile(lockPath) if err != nil { return errors.Wrapf(err, "error retrieving lockfile for pod %s", string(id)) @@ -310,6 +337,35 @@ func (s *BoltState) getPodFromDB(id []byte, pod *Pod, podBkt *bolt.Bucket) error return nil } +func (s *BoltState) getVolumeFromDB(name []byte, volume *Volume, volBkt *bolt.Bucket) error { + volDB := volBkt.Bucket(name) + if volDB == nil { + return errors.Wrapf(ErrNoSuchVolume, "volume with name %s not found", string(name)) + } + + volConfigBytes := volDB.Get(configKey) + if volConfigBytes == nil { + return errors.Wrapf(ErrInternal, "volume %s is missing configuration key in DB", string(name)) + } + + if err := json.Unmarshal(volConfigBytes, volume.config); err != nil { + return errors.Wrapf(err, "error unmarshalling volume %s config from DB", string(name)) + } + + // Get the lock + lockPath := filepath.Join(s.runtime.lockDir, string(name)) + lock, err := storage.GetLockfile(lockPath) + if err != nil { + return errors.Wrapf(err, "error retrieving lockfile for volume %s", string(name)) + } + volume.lock = lock + + volume.runtime = s.runtime + volume.valid = true + + return nil +} + // Add a container to the DB // If pod is not nil, the container is added to the pod as well func (s *BoltState) addContainer(ctr *Container, pod *Pod) error { @@ -371,6 +427,11 @@ func (s *BoltState) addContainer(ctr *Container, pod *Pod) error { return err } + volBkt, err := getVolBucket(tx) + if err != nil { + return err + } + // If a pod was given, check if it exists var podDB *bolt.Bucket var podCtrs *bolt.Bucket @@ -503,6 +564,25 @@ func (s *BoltState) addContainer(ctr *Container, pod *Pod) error { } } + // Add container to volume dependencies bucket if container is using a named volume + for _, vol := range ctr.config.Spec.Mounts { + if strings.Contains(vol.Source, ctr.runtime.config.VolumePath) { + volName := strings.Split(vol.Source[len(ctr.runtime.config.VolumePath)+1:], "/")[0] + + volDB := volBkt.Bucket([]byte(volName)) + if volDB == nil { + return errors.Wrapf(ErrNoSuchVolume, "no volume with name %s found in database", volName) + } + + ctrDepsBkt := volDB.Bucket(volDependenciesBkt) + if depExists := ctrDepsBkt.Get(ctrID); depExists == nil { + if err := ctrDepsBkt.Put(ctrID, ctrID); err != nil { + return errors.Wrapf(err, "error storing container dependencies %q for volume %s in ctrDependencies bucket in DB", ctr.ID(), volName) + } + } + } + } + return nil }) return err @@ -540,6 +620,11 @@ func (s *BoltState) removeContainer(ctr *Container, pod *Pod, tx *bolt.Tx) error return err } + volBkt, err := getVolBucket(tx) + if err != nil { + return err + } + // Does the pod exist? var podDB *bolt.Bucket if pod != nil { @@ -658,5 +743,25 @@ func (s *BoltState) removeContainer(ctr *Container, pod *Pod, tx *bolt.Tx) error } } + // Remove container from volume dependencies bucket if container is using a named volume + for _, vol := range ctr.config.Spec.Mounts { + if strings.Contains(vol.Source, ctr.runtime.config.VolumePath) { + volName := strings.Split(vol.Source[len(ctr.runtime.config.VolumePath)+1:], "/")[0] + + volDB := volBkt.Bucket([]byte(volName)) + if volDB == nil { + // Let's assume the volume was already deleted and continue to remove the container + continue + } + + ctrDepsBkt := volDB.Bucket(volDependenciesBkt) + if depExists := ctrDepsBkt.Get(ctrID); depExists != nil { + if err := ctrDepsBkt.Delete(ctrID); err != nil { + return errors.Wrapf(err, "error deleting container dependencies %q for volume %s in ctrDependencies bucket in DB", ctr.ID(), volName) + } + } + } + } + return nil } diff --git a/libpod/common/common.go b/libpod/common/common.go index 932f1f6da..5d10bee36 100644 --- a/libpod/common/common.go +++ b/libpod/common/common.go @@ -1,32 +1,9 @@ package common import ( - "io" - - cp "github.com/containers/image/copy" "github.com/containers/image/types" ) -// GetCopyOptions constructs a new containers/image/copy.Options{} struct from the given parameters -func GetCopyOptions(reportWriter io.Writer, signaturePolicyPath string, srcDockerRegistry, destDockerRegistry *DockerRegistryOptions, signing SigningOptions, authFile, manifestType string, forceCompress bool) *cp.Options { - if srcDockerRegistry == nil { - srcDockerRegistry = &DockerRegistryOptions{} - } - if destDockerRegistry == nil { - destDockerRegistry = &DockerRegistryOptions{} - } - srcContext := srcDockerRegistry.GetSystemContext(signaturePolicyPath, authFile, forceCompress) - destContext := destDockerRegistry.GetSystemContext(signaturePolicyPath, authFile, forceCompress) - return &cp.Options{ - RemoveSignatures: signing.RemoveSignatures, - SignBy: signing.SignBy, - ReportWriter: reportWriter, - SourceCtx: srcContext, - DestinationCtx: destContext, - ForceManifestMIMEType: manifestType, - } -} - // GetSystemContext Constructs a new containers/image/types.SystemContext{} struct from the given signaturePolicy path func GetSystemContext(signaturePolicyPath, authFilePath string, forceCompress bool) *types.SystemContext { sc := &types.SystemContext{} diff --git a/libpod/common/docker_registry_options.go b/libpod/common/docker_registry_options.go deleted file mode 100644 index f79ae0c54..000000000 --- a/libpod/common/docker_registry_options.go +++ /dev/null @@ -1,35 +0,0 @@ -package common - -import "github.com/containers/image/types" - -// DockerRegistryOptions encapsulates settings that affect how we connect or -// authenticate to a remote registry. -type DockerRegistryOptions struct { - // DockerRegistryCreds is the user name and password to supply in case - // we need to pull an image from a registry, and it requires us to - // authenticate. - DockerRegistryCreds *types.DockerAuthConfig - // DockerCertPath is the location of a directory containing CA - // certificates which will be used to verify the registry's certificate - // (all files with names ending in ".crt"), and possibly client - // certificates and private keys (pairs of files with the same name, - // except for ".cert" and ".key" suffixes). - DockerCertPath string - // DockerInsecureSkipTLSVerify turns off verification of TLS - // certificates and allows connecting to registries without encryption. - DockerInsecureSkipTLSVerify bool -} - -// GetSystemContext constructs a new system context from the given signaturePolicy path and the -// values in the DockerRegistryOptions -func (o DockerRegistryOptions) GetSystemContext(signaturePolicyPath, authFile string, forceCompress bool) *types.SystemContext { - sc := &types.SystemContext{ - SignaturePolicyPath: signaturePolicyPath, - DockerAuthConfig: o.DockerRegistryCreds, - DockerCertPath: o.DockerCertPath, - DockerInsecureSkipTLSVerify: o.DockerInsecureSkipTLSVerify, - AuthFilePath: authFile, - DirForceCompress: forceCompress, - } - return sc -} diff --git a/libpod/common/output_interfaces.go b/libpod/common/output_interfaces.go deleted file mode 100644 index 805d0c79a..000000000 --- a/libpod/common/output_interfaces.go +++ /dev/null @@ -1 +0,0 @@ -package common diff --git a/libpod/common_test.go b/libpod/common_test.go index b7fee2764..81c8f1920 100644 --- a/libpod/common_test.go +++ b/libpod/common_test.go @@ -74,6 +74,11 @@ func getTestContainer(id, name, locksDir string) (*Container, error) { "/test/file.test": "/test2/file2.test", }, }, + runtime: &Runtime{ + config: &RuntimeConfig{ + VolumePath: "/does/not/exist/tmp/volumes", + }, + }, valid: true, } diff --git a/libpod/container.go b/libpod/container.go index b6155a809..b5346e581 100644 --- a/libpod/container.go +++ b/libpod/container.go @@ -1001,13 +1001,28 @@ func (c *Container) IsReadOnly() bool { } // NetworkDisabled returns whether the container is running with a disabled network -func (c *Container) NetworkDisabled() bool { +func (c *Container) NetworkDisabled() (bool, error) { + if c.config.NetNsCtr != "" { + container, err := c.runtime.LookupContainer(c.config.NetNsCtr) + if err != nil { + return false, err + } + return networkDisabled(container) + } + return networkDisabled(c) + +} + +func networkDisabled(c *Container) (bool, error) { + if c.config.CreateNetNS { + return false, nil + } if !c.config.PostConfigureNetNS { for _, ns := range c.config.Spec.Linux.Namespaces { if ns.Type == spec.NetworkNamespace { - return ns.Path == "" + return ns.Path == "", nil } } } - return false + return false, nil } diff --git a/libpod/container_api.go b/libpod/container_api.go index bc92cae69..09bc46905 100644 --- a/libpod/container_api.go +++ b/libpod/container_api.go @@ -675,22 +675,27 @@ func (c *Container) Batch(batchFunc func(*Container) error) error { return err } -// Sync updates the current state of the container, checking whether its state -// has changed -// Sync can only be used inside Batch() - otherwise, it will be done -// automatically. -// When called outside Batch(), Sync() is a no-op +// Sync updates the status of a container by querying the OCI runtime. +// If the container has not been created inside the OCI runtime, nothing will be +// done. +// Most of the time, Podman does not explicitly query the OCI runtime for +// container status, and instead relies upon exit files created by conmon. +// This can cause a disconnect between running state and what Podman sees in +// cases where Conmon was killed unexpected, or runc was upgraded. +// Running a manual Sync() ensures that container state will be correct in +// such situations. func (c *Container) Sync() error { if !c.batched { - return nil + c.lock.Lock() + defer c.lock.Unlock() } // If runtime knows about the container, update its status in runtime // And then save back to disk if (c.state.State != ContainerStateUnknown) && - (c.state.State != ContainerStateConfigured) { + (c.state.State != ContainerStateConfigured) && + (c.state.State != ContainerStateExited) { oldState := c.state.State - // TODO: optionally replace this with a stat for the exit file if err := c.runtime.ociRuntime.updateContainerStatus(c, true); err != nil { return err } diff --git a/libpod/container_inspect.go b/libpod/container_inspect.go index 9b07198bc..06a0c9f32 100644 --- a/libpod/container_inspect.go +++ b/libpod/container_inspect.go @@ -1,8 +1,11 @@ package libpod import ( + "strings" + "github.com/containers/libpod/pkg/inspect" "github.com/cri-o/ocicni/pkg/ocicni" + specs "github.com/opencontainers/runtime-spec/specs-go" "github.com/sirupsen/logrus" ) @@ -48,6 +51,17 @@ func (c *Container) getContainerInspectData(size bool, driverData *inspect.Data) hostnamePath = getPath } + var mounts []specs.Mount + for i, mnt := range spec.Mounts { + mounts = append(mounts, mnt) + // We only want to show the name of the named volume in the inspect + // output, so split the path and get the name out of it. + if strings.Contains(mnt.Source, c.runtime.config.VolumePath) { + split := strings.Split(mnt.Source[len(c.runtime.config.VolumePath)+1:], "/") + mounts[i].Source = split[0] + } + } + data := &inspect.ContainerInspectData{ ID: config.ID, Created: config.CreatedTime, @@ -85,7 +99,7 @@ func (c *Container) getContainerInspectData(size bool, driverData *inspect.Data) AppArmorProfile: spec.Process.ApparmorProfile, ExecIDs: execIDs, GraphDriver: driverData, - Mounts: spec.Mounts, + Mounts: mounts, Dependencies: c.Dependencies(), NetworkSettings: &inspect.NetworkSettings{ Bridge: "", // TODO diff --git a/libpod/container_internal.go b/libpod/container_internal.go index e31a8099c..0148e3e7c 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -601,7 +601,11 @@ func (c *Container) checkDependenciesRunningLocked(depCtrs map[string]*Container } func (c *Container) completeNetworkSetup() error { - if !c.config.PostConfigureNetNS || c.NetworkDisabled() { + netDisabled, err := c.NetworkDisabled() + if err != nil { + return err + } + if !c.config.PostConfigureNetNS || netDisabled { return nil } if err := c.syncContainer(); err != nil { @@ -1168,10 +1172,6 @@ func (c *Container) saveSpec(spec *spec.Spec) error { } func (c *Container) setupOCIHooks(ctx context.Context, config *spec.Spec) (extensionStageHooks map[string][]spec.Hook, err error) { - if len(c.runtime.config.HooksDir) == 0 { - return nil, nil - } - var locale string var ok bool for _, envVar := range []string{ @@ -1199,25 +1199,39 @@ func (c *Container) setupOCIHooks(ctx context.Context, config *spec.Spec) (exten } } - allHooks := make(map[string][]spec.Hook) - for _, hDir := range c.runtime.config.HooksDir { - manager, err := hooks.New(ctx, []string{hDir}, []string{"poststop"}, lang) - if err != nil { - if c.runtime.config.HooksDirNotExistFatal || !os.IsNotExist(err) { - return nil, err - } - logrus.Warnf("failed to load hooks: %q", err) + if c.runtime.config.HooksDir == nil { + if rootless.IsRootless() { return nil, nil } - hooks, err := manager.Hooks(config, c.Spec().Annotations, len(c.config.UserVolumes) > 0) - if err != nil { - return nil, err - } - for i, hook := range hooks { - allHooks[i] = hook + allHooks := make(map[string][]spec.Hook) + for _, hDir := range []string{hooks.DefaultDir, hooks.OverrideDir} { + manager, err := hooks.New(ctx, []string{hDir}, []string{"poststop"}, lang) + if err != nil { + if os.IsNotExist(err) { + continue + } + return nil, err + } + hooks, err := manager.Hooks(config, c.Spec().Annotations, len(c.config.UserVolumes) > 0) + if err != nil { + return nil, err + } + if len(hooks) > 0 || config.Hooks != nil { + logrus.Warnf("implicit hook directories are deprecated; set --hooks-dir=%q explicitly to continue to load hooks from this directory", hDir) + } + for i, hook := range hooks { + allHooks[i] = hook + } } + return allHooks, nil + } + + manager, err := hooks.New(ctx, c.runtime.config.HooksDir, []string{"poststop"}, lang) + if err != nil { + return nil, err } - return allHooks, nil + + return manager.Hooks(config, c.Spec().Annotations, len(c.config.UserVolumes) > 0) } // mount mounts the container's root filesystem diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index 8861d7728..f9b0592f9 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -136,7 +136,14 @@ func (c *Container) prepare() (err error) { // cleanupNetwork unmounts and cleans up the container's network func (c *Container) cleanupNetwork() error { - if c.NetworkDisabled() { + if c.config.NetNsCtr != "" { + return nil + } + netDisabled, err := c.NetworkDisabled() + if err != nil { + return err + } + if netDisabled { return nil } if c.state.NetNS == nil { @@ -180,7 +187,6 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { if err := c.makeBindMounts(); err != nil { return nil, err } - // Check if the spec file mounts contain the label Relabel flags z or Z. // If they do, relabel the source directory and then remove the option. for _, m := range g.Mounts() { @@ -224,10 +230,8 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { } } - if !rootless.IsRootless() { - if c.state.ExtensionStageHooks, err = c.setupOCIHooks(ctx, g.Config); err != nil { - return nil, errors.Wrapf(err, "error setting up OCI Hooks") - } + if c.state.ExtensionStageHooks, err = c.setupOCIHooks(ctx, g.Config); err != nil { + return nil, errors.Wrapf(err, "error setting up OCI Hooks") } // Bind builtin image volumes @@ -238,9 +242,6 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { } if c.config.User != "" { - if !c.state.Mounted { - return nil, errors.Wrapf(ErrCtrStateInvalid, "container %s must be mounted in order to translate User field", c.ID()) - } // User and Group must go together g.SetProcessUID(uint32(execUser.Uid)) g.SetProcessGID(uint32(execUser.Gid)) @@ -248,9 +249,6 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { // Add addition groups if c.config.GroupAdd is not empty if len(c.config.Groups) > 0 { - if !c.state.Mounted { - return nil, errors.Wrapf(ErrCtrStateInvalid, "container %s must be mounted in order to add additional groups", c.ID()) - } gids, _ := lookup.GetContainerGroups(c.config.Groups, c.state.Mountpoint, nil) for _, gid := range gids { g.AddProcessAdditionalGid(gid) @@ -641,8 +639,12 @@ func (c *Container) makeBindMounts() error { if c.state.BindMounts == nil { c.state.BindMounts = make(map[string]string) } + netDisabled, err := c.NetworkDisabled() + if err != nil { + return err + } - if !c.NetworkDisabled() { + if !netDisabled { // Make /etc/resolv.conf if _, ok := c.state.BindMounts["/etc/resolv.conf"]; ok { // If it already exists, delete so we can recreate @@ -802,7 +804,6 @@ func (c *Container) generateHosts() (string, error) { func (c *Container) generatePasswd() (string, error) { var ( groupspec string - group *user.Group gid int ) if c.config.User == "" { @@ -827,17 +828,16 @@ func (c *Container) generatePasswd() (string, error) { return "", nil } if groupspec != "" { - if !c.state.Mounted { - return "", errors.Wrapf(ErrCtrStateInvalid, "container %s must be mounted in order to translate group field for passwd record", c.ID()) - } - group, err = lookup.GetGroup(c.state.Mountpoint, groupspec) - if err != nil { - if err == user.ErrNoGroupEntries { + ugid, err := strconv.ParseUint(groupspec, 10, 32) + if err == nil { + gid = int(ugid) + } else { + group, err := lookup.GetGroup(c.state.Mountpoint, groupspec) + if err != nil { return "", errors.Wrapf(err, "unable to get gid %s from group file", groupspec) } - return "", err + gid = group.Gid } - gid = group.Gid } originPasswdFile := filepath.Join(c.state.Mountpoint, "/etc/passwd") orig, err := ioutil.ReadFile(originPasswdFile) diff --git a/libpod/container_internal_unsupported.go b/libpod/container_internal_unsupported.go index eed0449a9..4af0cd56c 100644 --- a/libpod/container_internal_unsupported.go +++ b/libpod/container_internal_unsupported.go @@ -28,10 +28,10 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { return nil, ErrNotImplemented } -func (c *Container) checkpoint(ctx context.Context, keep bool) error { +func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointOptions) error { return ErrNotImplemented } -func (c *Container) restore(ctx context.Context, keep bool) error { +func (c *Container) restore(ctx context.Context, options ContainerCheckpointOptions) error { return ErrNotImplemented } diff --git a/libpod/errors.go b/libpod/errors.go index 75b4928da..d6614141c 100644 --- a/libpod/errors.go +++ b/libpod/errors.go @@ -11,18 +11,24 @@ var ( ErrNoSuchPod = errors.New("no such pod") // ErrNoSuchImage indicates the requested image does not exist ErrNoSuchImage = errors.New("no such image") + // ErrNoSuchVolume indicates the requested volume does not exist + ErrNoSuchVolume = errors.New("no such volume") // ErrCtrExists indicates a container with the same name or ID already // exists ErrCtrExists = errors.New("container already exists") // ErrPodExists indicates a pod with the same name or ID already exists ErrPodExists = errors.New("pod already exists") - // ErrImageExists indicated an image with the same ID already exists + // ErrImageExists indicates an image with the same ID already exists ErrImageExists = errors.New("image already exists") + // ErrVolumeExists indicates a volume with the same name already exists + ErrVolumeExists = errors.New("volume already exists") // ErrCtrStateInvalid indicates a container is in an improper state for // the requested operation ErrCtrStateInvalid = errors.New("container state improper") + // ErrVolumeBeingUsed indicates that a volume is being used by at least one container + ErrVolumeBeingUsed = errors.New("volume is being used") // ErrRuntimeFinalized indicates that the runtime has already been // created and cannot be modified @@ -33,6 +39,9 @@ var ( // ErrPodFinalized indicates that the pod has already been created and // cannot be modified ErrPodFinalized = errors.New("pod has been finalized") + // ErrVolumeFinalized indicates that the volume has already been created and + // cannot be modified + ErrVolumeFinalized = errors.New("volume has been finalized") // ErrInvalidArg indicates that an invalid argument was passed ErrInvalidArg = errors.New("invalid argument") @@ -55,6 +64,9 @@ var ( // ErrPodRemoved indicates that the pod has already been removed and no // further operations can be performed on it ErrPodRemoved = errors.New("pod has already been removed") + // ErrVolumeRemoved indicates that the volume has already been removed and + // no further operations can be performed on it + ErrVolumeRemoved = errors.New("volume has already been removed") // ErrDBClosed indicates that the connection to the state database has // already been closed diff --git a/libpod/image/docker_registry_options.go b/libpod/image/docker_registry_options.go index 97a151396..c191a3ca2 100644 --- a/libpod/image/docker_registry_options.go +++ b/libpod/image/docker_registry_options.go @@ -19,8 +19,9 @@ type DockerRegistryOptions struct { // except for ".cert" and ".key" suffixes). DockerCertPath string // DockerInsecureSkipTLSVerify turns off verification of TLS - // certificates and allows connecting to registries without encryption. - DockerInsecureSkipTLSVerify bool + // certificates and allows connecting to registries without encryption + // - or forces it on even if registries.conf has the registry configured as insecure. + DockerInsecureSkipTLSVerify types.OptionalBool } // GetSystemContext constructs a new system context from a parent context. the values in the DockerRegistryOptions, and other parameters. diff --git a/libpod/image/image.go b/libpod/image/image.go index 434f9031e..476d28226 100644 --- a/libpod/image/image.go +++ b/libpod/image/image.go @@ -125,7 +125,7 @@ func (ir *Runtime) NewFromLocal(name string) (*Image, error) { // New creates a new image object where the image could be local // or remote -func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile string, writer io.Writer, dockeroptions *DockerRegistryOptions, signingoptions SigningOptions, forcePull, forceSecure bool) (*Image, error) { +func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile string, writer io.Writer, dockeroptions *DockerRegistryOptions, signingoptions SigningOptions, forcePull bool) (*Image, error) { // We don't know if the image is local or not ... check local first newImage := Image{ InputName: name, @@ -145,7 +145,7 @@ func (ir *Runtime) New(ctx context.Context, name, signaturePolicyPath, authfile if signaturePolicyPath == "" { signaturePolicyPath = ir.SignaturePolicyPath } - imageName, err := ir.pullImageFromHeuristicSource(ctx, name, writer, authfile, signaturePolicyPath, signingoptions, dockeroptions, forceSecure) + imageName, err := ir.pullImageFromHeuristicSource(ctx, name, writer, authfile, signaturePolicyPath, signingoptions, dockeroptions) if err != nil { return nil, errors.Wrapf(err, "unable to pull %s", name) } @@ -167,7 +167,7 @@ func (ir *Runtime) LoadFromArchiveReference(ctx context.Context, srcRef types.Im if signaturePolicyPath == "" { signaturePolicyPath = ir.SignaturePolicyPath } - imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}, false) + imageNames, err := ir.pullImageFromReference(ctx, srcRef, writer, "", signaturePolicyPath, SigningOptions{}, &DockerRegistryOptions{}) if err != nil { return nil, errors.Wrapf(err, "unable to pull %s", transports.ImageName(srcRef)) } @@ -498,7 +498,7 @@ func (i *Image) UntagImage(tag string) error { // PushImageToHeuristicDestination pushes the given image to "destination", which is heuristically parsed. // Use PushImageToReference if the destination is known precisely. -func (i *Image) PushImageToHeuristicDestination(ctx context.Context, destination, manifestMIMEType, authFile, signaturePolicyPath string, writer io.Writer, forceCompress bool, signingOptions SigningOptions, dockerRegistryOptions *DockerRegistryOptions, forceSecure bool, additionalDockerArchiveTags []reference.NamedTagged) error { +func (i *Image) PushImageToHeuristicDestination(ctx context.Context, destination, manifestMIMEType, authFile, signaturePolicyPath string, writer io.Writer, forceCompress bool, signingOptions SigningOptions, dockerRegistryOptions *DockerRegistryOptions, additionalDockerArchiveTags []reference.NamedTagged) error { if destination == "" { return errors.Wrapf(syscall.EINVAL, "destination image name must be specified") } @@ -516,11 +516,11 @@ func (i *Image) PushImageToHeuristicDestination(ctx context.Context, destination return err } } - return i.PushImageToReference(ctx, dest, manifestMIMEType, authFile, signaturePolicyPath, writer, forceCompress, signingOptions, dockerRegistryOptions, forceSecure, additionalDockerArchiveTags) + return i.PushImageToReference(ctx, dest, manifestMIMEType, authFile, signaturePolicyPath, writer, forceCompress, signingOptions, dockerRegistryOptions, additionalDockerArchiveTags) } // PushImageToReference pushes the given image to a location described by the given path -func (i *Image) PushImageToReference(ctx context.Context, dest types.ImageReference, manifestMIMEType, authFile, signaturePolicyPath string, writer io.Writer, forceCompress bool, signingOptions SigningOptions, dockerRegistryOptions *DockerRegistryOptions, forceSecure bool, additionalDockerArchiveTags []reference.NamedTagged) error { +func (i *Image) PushImageToReference(ctx context.Context, dest types.ImageReference, manifestMIMEType, authFile, signaturePolicyPath string, writer io.Writer, forceCompress bool, signingOptions SigningOptions, dockerRegistryOptions *DockerRegistryOptions, additionalDockerArchiveTags []reference.NamedTagged) error { sc := GetSystemContext(signaturePolicyPath, authFile, forceCompress) policyContext, err := getPolicyContext(sc) @@ -534,23 +534,8 @@ func (i *Image) PushImageToReference(ctx context.Context, dest types.ImageRefere if err != nil { return errors.Wrapf(err, "error getting source imageReference for %q", i.InputName) } - insecureRegistries, err := registries.GetInsecureRegistries() - if err != nil { - return err - } copyOptions := getCopyOptions(sc, writer, nil, dockerRegistryOptions, signingOptions, manifestMIMEType, additionalDockerArchiveTags) - if dest.Transport().Name() == DockerTransport { - imgRef := dest.DockerReference() - if imgRef == nil { // This should never happen; such references can’t be created. - return fmt.Errorf("internal error: DockerTransport reference %s does not have a DockerReference", transports.ImageName(dest)) - } - registry := reference.Domain(imgRef) - - if util.StringInSlice(registry, insecureRegistries) && !forceSecure { - copyOptions.DestinationCtx.DockerInsecureSkipTLSVerify = true - logrus.Info(fmt.Sprintf("%s is an insecure registry; pushing with tls-verify=false", registry)) - } - } + copyOptions.DestinationCtx.SystemRegistriesConfPath = registries.SystemRegistriesConfPath() // FIXME: Set this more globally. Probably no reason not to have it in every types.SystemContext, and to compute the value just once in one place. // Copy the image to the remote destination _, err = cp.Image(ctx, policyContext, dest, src, copyOptions) if err != nil { diff --git a/libpod/image/image_test.go b/libpod/image/image_test.go index f187631b4..91bb2411b 100644 --- a/libpod/image/image_test.go +++ b/libpod/image/image_test.go @@ -86,9 +86,9 @@ func TestImage_NewFromLocal(t *testing.T) { // Need images to be present for this test ir, err := NewImageRuntimeFromOptions(so) assert.NoError(t, err) - bb, err := ir.New(context.Background(), "docker.io/library/busybox:latest", "", "", writer, nil, SigningOptions{}, false, false) + bb, err := ir.New(context.Background(), "docker.io/library/busybox:latest", "", "", writer, nil, SigningOptions{}, false) assert.NoError(t, err) - bbglibc, err := ir.New(context.Background(), "docker.io/library/busybox:glibc", "", "", writer, nil, SigningOptions{}, false, false) + bbglibc, err := ir.New(context.Background(), "docker.io/library/busybox:glibc", "", "", writer, nil, SigningOptions{}, false) assert.NoError(t, err) tm, err := makeLocalMatrix(bb, bbglibc) @@ -135,7 +135,7 @@ func TestImage_New(t *testing.T) { // Iterate over the names and delete the image // after the pull for _, img := range names { - newImage, err := ir.New(context.Background(), img, "", "", writer, nil, SigningOptions{}, false, false) + newImage, err := ir.New(context.Background(), img, "", "", writer, nil, SigningOptions{}, false) assert.NoError(t, err) assert.NotEqual(t, newImage.ID(), "") err = newImage.Remove(false) @@ -163,7 +163,7 @@ func TestImage_MatchRepoTag(t *testing.T) { } ir, err := NewImageRuntimeFromOptions(so) assert.NoError(t, err) - newImage, err := ir.New(context.Background(), "busybox", "", "", os.Stdout, nil, SigningOptions{}, false, false) + newImage, err := ir.New(context.Background(), "busybox", "", "", os.Stdout, nil, SigningOptions{}, false) assert.NoError(t, err) err = newImage.TagImage("foo:latest") assert.NoError(t, err) diff --git a/libpod/image/prune.go b/libpod/image/prune.go new file mode 100644 index 000000000..6a1f160d5 --- /dev/null +++ b/libpod/image/prune.go @@ -0,0 +1,26 @@ +package image + +// GetPruneImages returns a slice of images that have no names/unused +func (ir *Runtime) GetPruneImages() ([]*Image, error) { + var ( + unamedImages []*Image + ) + allImages, err := ir.GetImages() + if err != nil { + return nil, err + } + for _, i := range allImages { + if len(i.Names()) == 0 { + unamedImages = append(unamedImages, i) + continue + } + containers, err := i.Containers() + if err != nil { + return nil, err + } + if len(containers) < 1 { + unamedImages = append(unamedImages, i) + } + } + return unamedImages, nil +} diff --git a/libpod/image/pull.go b/libpod/image/pull.go index bfa04d069..09935fe7c 100644 --- a/libpod/image/pull.go +++ b/libpod/image/pull.go @@ -10,7 +10,6 @@ import ( "github.com/containers/image/directory" "github.com/containers/image/docker" dockerarchive "github.com/containers/image/docker/archive" - "github.com/containers/image/docker/reference" "github.com/containers/image/docker/tarfile" ociarchive "github.com/containers/image/oci/archive" "github.com/containers/image/pkg/sysregistries" @@ -19,7 +18,6 @@ import ( "github.com/containers/image/transports/alltransports" "github.com/containers/image/types" "github.com/containers/libpod/pkg/registries" - "github.com/containers/libpod/pkg/util" multierror "github.com/hashicorp/go-multierror" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -193,7 +191,7 @@ func (ir *Runtime) pullGoalFromImageReference(ctx context.Context, srcRef types. // pullImageFromHeuristicSource pulls an image based on inputName, which is heuristically parsed and may involve configured registries. // Use pullImageFromReference if the source is known precisely. -func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName string, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, forceSecure bool) ([]string, error) { +func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName string, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions) ([]string, error) { var goal *pullGoal sc := GetSystemContext(signaturePolicyPath, authfile, false) srcRef, err := alltransports.ParseImageName(inputName) @@ -209,48 +207,33 @@ func (ir *Runtime) pullImageFromHeuristicSource(ctx context.Context, inputName s return nil, errors.Wrapf(err, "error determining pull goal for image %q", inputName) } } - return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, forceSecure) + return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions) } // pullImageFromReference pulls an image from a types.imageReference. -func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.ImageReference, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, forceSecure bool) ([]string, error) { +func (ir *Runtime) pullImageFromReference(ctx context.Context, srcRef types.ImageReference, writer io.Writer, authfile, signaturePolicyPath string, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions) ([]string, error) { sc := GetSystemContext(signaturePolicyPath, authfile, false) goal, err := ir.pullGoalFromImageReference(ctx, srcRef, transports.ImageName(srcRef), sc) if err != nil { return nil, errors.Wrapf(err, "error determining pull goal for image %q", transports.ImageName(srcRef)) } - return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions, forceSecure) + return ir.doPullImage(ctx, sc, *goal, writer, signingOptions, dockerOptions) } // doPullImage is an internal helper interpreting pullGoal. Almost everyone should call one of the callers of doPullImage instead. -func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goal pullGoal, writer io.Writer, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions, forceSecure bool) ([]string, error) { +func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goal pullGoal, writer io.Writer, signingOptions SigningOptions, dockerOptions *DockerRegistryOptions) ([]string, error) { policyContext, err := getPolicyContext(sc) if err != nil { return nil, err } defer policyContext.Destroy() - insecureRegistries, err := registries.GetInsecureRegistries() - if err != nil { - return nil, err - } + systemRegistriesConfPath := registries.SystemRegistriesConfPath() var images []string var pullErrors *multierror.Error for _, imageInfo := range goal.refPairs { copyOptions := getCopyOptions(sc, writer, dockerOptions, nil, signingOptions, "", nil) - if imageInfo.srcRef.Transport().Name() == DockerTransport { - imgRef := imageInfo.srcRef.DockerReference() - if imgRef == nil { // This should never happen; such references can’t be created. - return nil, fmt.Errorf("internal error: DockerTransport reference %s does not have a DockerReference", - transports.ImageName(imageInfo.srcRef)) - } - registry := reference.Domain(imgRef) - - if util.StringInSlice(registry, insecureRegistries) && !forceSecure { - copyOptions.SourceCtx.DockerInsecureSkipTLSVerify = true - logrus.Info(fmt.Sprintf("%s is an insecure registry; pulling with tls-verify=false", registry)) - } - } + copyOptions.SourceCtx.SystemRegistriesConfPath = systemRegistriesConfPath // FIXME: Set this more globally. Probably no reason not to have it in every types.SystemContext, and to compute the value just once in one place. // Print the following statement only when pulling from a docker or atomic registry if writer != nil && (imageInfo.srcRef.Transport().Name() == DockerTransport || imageInfo.srcRef.Transport().Name() == AtomicTransport) { io.WriteString(writer, fmt.Sprintf("Trying to pull %s...", imageInfo.image)) @@ -271,7 +254,7 @@ func (ir *Runtime) doPullImage(ctx context.Context, sc *types.SystemContext, goa } // If no image was found, we should handle. Lets be nicer to the user and see if we can figure out why. if len(images) == 0 { - registryPath := sysregistries.RegistriesConfPath(&types.SystemContext{}) + registryPath := sysregistries.RegistriesConfPath(&types.SystemContext{SystemRegistriesConfPath: systemRegistriesConfPath}) if goal.usedSearchRegistries && len(goal.searchedRegistries) == 0 { return nil, errors.Errorf("image name provided is a short name and no search registries are defined in %s.", registryPath) } diff --git a/libpod/in_memory_state.go b/libpod/in_memory_state.go index 78e765ccd..314799309 100644 --- a/libpod/in_memory_state.go +++ b/libpod/in_memory_state.go @@ -18,8 +18,10 @@ type InMemoryState struct { pods map[string]*Pod // Maps container ID to container struct. containers map[string]*Container + volumes map[string]*Volume // Maps container ID to a list of IDs of dependencies. - ctrDepends map[string][]string + ctrDepends map[string][]string + volumeDepends map[string][]string // Maps pod ID to a map of container ID to container struct. podContainers map[string]map[string]*Container // Global name registry - ensures name uniqueness and performs lookups. @@ -46,8 +48,10 @@ func NewInMemoryState() (State, error) { state.pods = make(map[string]*Pod) state.containers = make(map[string]*Container) + state.volumes = make(map[string]*Volume) state.ctrDepends = make(map[string][]string) + state.volumeDepends = make(map[string][]string) state.podContainers = make(map[string]map[string]*Container) @@ -73,6 +77,18 @@ func (s *InMemoryState) Refresh() error { return nil } +// GetDBConfig is not implemented for in-memory state. +// As we do not store a config, return an empty one. +func (s *InMemoryState) GetDBConfig() (*DBConfig, error) { + return &DBConfig{}, nil +} + +// ValidateDBConfig is not implemented for the in-memory state. +// Since we do nothing just return no error. +func (s *InMemoryState) ValidateDBConfig(runtime *Runtime) error { + return nil +} + // SetNamespace sets the namespace for container and pod retrieval. func (s *InMemoryState) SetNamespace(ns string) error { s.namespace = ns @@ -232,6 +248,14 @@ func (s *InMemoryState) AddContainer(ctr *Container) error { s.addCtrToDependsMap(ctr.ID(), depCtr) } + // Add container to volume dependencies + for _, vol := range ctr.config.Spec.Mounts { + if strings.Contains(vol.Source, ctr.runtime.config.VolumePath) { + volName := strings.Split(vol.Source[len(ctr.runtime.config.VolumePath)+1:], "/")[0] + s.addCtrToVolDependsMap(ctr.ID(), volName) + } + } + return nil } @@ -282,6 +306,14 @@ func (s *InMemoryState) RemoveContainer(ctr *Container) error { s.removeCtrFromDependsMap(ctr.ID(), depCtr) } + // Remove container from volume dependencies + for _, vol := range ctr.config.Spec.Mounts { + if strings.Contains(vol.Source, ctr.runtime.config.VolumePath) { + volName := strings.Split(vol.Source[len(ctr.runtime.config.VolumePath)+1:], "/")[0] + s.removeCtrFromVolDependsMap(ctr.ID(), volName) + } + } + return nil } @@ -346,6 +378,114 @@ func (s *InMemoryState) ContainerInUse(ctr *Container) ([]string, error) { return arr, nil } +// Volume retrieves a volume from its full name +func (s *InMemoryState) Volume(name string) (*Volume, error) { + if name == "" { + return nil, ErrEmptyID + } + + vol, ok := s.volumes[name] + if !ok { + return nil, errors.Wrapf(ErrNoSuchCtr, "no volume with name %s found", name) + } + + return vol, nil +} + +// HasVolume checks if a volume with the given name is present in the state +func (s *InMemoryState) HasVolume(name string) (bool, error) { + if name == "" { + return false, ErrEmptyID + } + + _, ok := s.volumes[name] + if !ok { + return false, nil + } + + return true, nil +} + +// AddVolume adds a volume to the state +func (s *InMemoryState) AddVolume(volume *Volume) error { + if !volume.valid { + return errors.Wrapf(ErrVolumeRemoved, "volume with name %s is not valid", volume.Name()) + } + + if _, ok := s.volumes[volume.Name()]; ok { + return errors.Wrapf(ErrVolumeExists, "volume with name %s already exists in state", volume.Name()) + } + + s.volumes[volume.Name()] = volume + + return nil +} + +// RemoveVolume removes a volume from the state +func (s *InMemoryState) RemoveVolume(volume *Volume) error { + // Ensure we don't remove a volume which containers depend on + deps, ok := s.volumeDepends[volume.Name()] + if ok && len(deps) != 0 { + depsStr := strings.Join(deps, ", ") + return errors.Wrapf(ErrVolumeExists, "the following containers depend on volume %s: %s", volume.Name(), depsStr) + } + + if _, ok := s.volumes[volume.Name()]; !ok { + volume.valid = false + return errors.Wrapf(ErrVolumeRemoved, "no volume exists in state with name %s", volume.Name()) + } + + delete(s.volumes, volume.Name()) + + return nil +} + +// RemoveVolCtrDep updates the container dependencies of the volume +func (s *InMemoryState) RemoveVolCtrDep(volume *Volume, ctrID string) error { + if !volume.valid { + return errors.Wrapf(ErrVolumeRemoved, "volume with name %s is not valid", volume.Name()) + } + + if _, ok := s.volumes[volume.Name()]; !ok { + return errors.Wrapf(ErrNoSuchVolume, "volume with name %s doesn't exists in state", volume.Name()) + } + + // Remove container that is using this volume + s.removeCtrFromVolDependsMap(ctrID, volume.Name()) + + return nil +} + +// VolumeInUse checks if the given volume is being used by at least one container +func (s *InMemoryState) VolumeInUse(volume *Volume) ([]string, error) { + if !volume.valid { + return nil, ErrVolumeRemoved + } + + // If the volume does not exist, return error + if _, ok := s.volumes[volume.Name()]; !ok { + volume.valid = false + return nil, errors.Wrapf(ErrNoSuchVolume, "volume with name %s not found in state", volume.Name()) + } + + arr, ok := s.volumeDepends[volume.Name()] + if !ok { + return []string{}, nil + } + + return arr, nil +} + +// AllVolumes returns all volumes that exist in the state +func (s *InMemoryState) AllVolumes() ([]*Volume, error) { + allVols := make([]*Volume, 0, len(s.volumes)) + for _, v := range s.volumes { + allVols = append(allVols, v) + } + + return allVols, nil +} + // AllContainers retrieves all containers from the state func (s *InMemoryState) AllContainers() ([]*Container, error) { ctrs := make([]*Container, 0, len(s.containers)) @@ -933,6 +1073,44 @@ func (s *InMemoryState) removeCtrFromDependsMap(ctrID, dependsID string) { } } +// Add a container to the dependency mappings for the volume +func (s *InMemoryState) addCtrToVolDependsMap(depCtrID, volName string) { + if volName != "" { + arr, ok := s.volumeDepends[volName] + if !ok { + // Do not have a mapping for that volume yet + s.volumeDepends[volName] = []string{depCtrID} + } else { + // Have a mapping for the volume + arr = append(arr, depCtrID) + s.volumeDepends[volName] = arr + } + } +} + +// Remove a container from the dependency mappings for the volume +func (s *InMemoryState) removeCtrFromVolDependsMap(depCtrID, volName string) { + if volName != "" { + arr, ok := s.volumeDepends[volName] + if !ok { + // Internal state seems inconsistent + // But the dependency is definitely gone + // So just return + return + } + + newArr := make([]string, 0, len(arr)) + + for _, id := range arr { + if id != depCtrID { + newArr = append(newArr, id) + } + } + + s.volumeDepends[volName] = newArr + } +} + // Check if we can access a pod or container, or if that is blocked by // namespaces. func (s *InMemoryState) checkNSMatch(id, ns string) error { diff --git a/libpod/options.go b/libpod/options.go index 7f4e3ac6b..352e6a506 100644 --- a/libpod/options.go +++ b/libpod/options.go @@ -29,19 +29,40 @@ func WithStorageConfig(config storage.StoreOptions) RuntimeOption { return ErrRuntimeFinalized } - rt.config.StorageConfig.RunRoot = config.RunRoot - rt.config.StorageConfig.GraphRoot = config.GraphRoot - rt.config.StorageConfig.GraphDriverName = config.GraphDriverName - rt.config.StaticDir = filepath.Join(config.GraphRoot, "libpod") + if config.RunRoot != "" { + rt.config.StorageConfig.RunRoot = config.RunRoot + rt.configuredFrom.storageRunRootSet = true + } + + if config.GraphRoot != "" { + rt.config.StorageConfig.GraphRoot = config.GraphRoot + rt.configuredFrom.storageGraphRootSet = true + + // Also set libpod static dir, so we are a subdirectory + // of the c/storage store by default + rt.config.StaticDir = filepath.Join(config.GraphRoot, "libpod") + rt.configuredFrom.libpodStaticDirSet = true + } - rt.config.StorageConfig.GraphDriverOptions = make([]string, len(config.GraphDriverOptions)) - copy(rt.config.StorageConfig.GraphDriverOptions, config.GraphDriverOptions) + if config.GraphDriverName != "" { + rt.config.StorageConfig.GraphDriverName = config.GraphDriverName + rt.configuredFrom.storageGraphDriverSet = true + } + + if config.GraphDriverOptions != nil { + rt.config.StorageConfig.GraphDriverOptions = make([]string, len(config.GraphDriverOptions)) + copy(rt.config.StorageConfig.GraphDriverOptions, config.GraphDriverOptions) + } - rt.config.StorageConfig.UIDMap = make([]idtools.IDMap, len(config.UIDMap)) - copy(rt.config.StorageConfig.UIDMap, config.UIDMap) + if config.UIDMap != nil { + rt.config.StorageConfig.UIDMap = make([]idtools.IDMap, len(config.UIDMap)) + copy(rt.config.StorageConfig.UIDMap, config.UIDMap) + } - rt.config.StorageConfig.GIDMap = make([]idtools.IDMap, len(config.GIDMap)) - copy(rt.config.StorageConfig.GIDMap, config.GIDMap) + if config.GIDMap != nil { + rt.config.StorageConfig.GIDMap = make([]idtools.IDMap, len(config.GIDMap)) + copy(rt.config.StorageConfig.GIDMap, config.GIDMap) + } return nil } @@ -174,26 +195,26 @@ func WithStaticDir(dir string) RuntimeOption { } rt.config.StaticDir = dir + rt.configuredFrom.libpodStaticDirSet = true return nil } } -// WithHooksDir sets the directory to look for OCI runtime hooks config. -// Note we are not saving this in database, since this is really just for used -// for testing. -func WithHooksDir(hooksDir string) RuntimeOption { +// WithHooksDir sets the directories to look for OCI runtime hook configuration. +func WithHooksDir(hooksDirs ...string) RuntimeOption { return func(rt *Runtime) error { if rt.valid { return ErrRuntimeFinalized } - if hooksDir == "" { - return errors.Wrap(ErrInvalidArg, "empty-string hook directories are not supported") + for _, hooksDir := range hooksDirs { + if hooksDir == "" { + return errors.Wrap(ErrInvalidArg, "empty-string hook directories are not supported") + } } - rt.config.HooksDir = []string{hooksDir} - rt.config.HooksDirNotExistFatal = true + rt.config.HooksDir = hooksDirs return nil } } @@ -226,6 +247,7 @@ func WithTmpDir(dir string) RuntimeOption { } rt.config.TmpDir = dir + rt.configuredFrom.libpodTmpDirSet = true return nil } @@ -305,6 +327,22 @@ func WithNamespace(ns string) RuntimeOption { } } +// WithVolumePath sets the path under which all named volumes +// should be created. +// The path changes based on whethe rthe user is running as root +// or not. +func WithVolumePath(volPath string) RuntimeOption { + return func(rt *Runtime) error { + if rt.valid { + return ErrRuntimeFinalized + } + + rt.config.VolumePath = volPath + + return nil + } +} + // WithDefaultInfraImage sets the infra image for libpod. // An infra image is used for inter-container kernel // namespace sharing within a pod. Typically, an infra @@ -1103,6 +1141,70 @@ func withIsInfra() CtrCreateOption { } } +// Volume Creation Options + +// WithVolumeName sets the name of the volume. +func WithVolumeName(name string) VolumeCreateOption { + return func(volume *Volume) error { + if volume.valid { + return ErrVolumeFinalized + } + + // Check the name against a regex + if !nameRegex.MatchString(name) { + return errors.Wrapf(ErrInvalidArg, "name must match regex [a-zA-Z0-9_-]+") + } + volume.config.Name = name + + return nil + } +} + +// WithVolumeLabels sets the labels of the volume. +func WithVolumeLabels(labels map[string]string) VolumeCreateOption { + return func(volume *Volume) error { + if volume.valid { + return ErrVolumeFinalized + } + + volume.config.Labels = make(map[string]string) + for key, value := range labels { + volume.config.Labels[key] = value + } + + return nil + } +} + +// WithVolumeDriver sets the driver of the volume. +func WithVolumeDriver(driver string) VolumeCreateOption { + return func(volume *Volume) error { + if volume.valid { + return ErrVolumeFinalized + } + + volume.config.Driver = driver + + return nil + } +} + +// WithVolumeOptions sets the options of the volume. +func WithVolumeOptions(options map[string]string) VolumeCreateOption { + return func(volume *Volume) error { + if volume.valid { + return ErrVolumeFinalized + } + + volume.config.Options = make(map[string]string) + for key, value := range options { + volume.config.Options[key] = value + } + + return nil + } +} + // Pod Creation Options // WithPodName sets the name of the pod. diff --git a/libpod/runtime.go b/libpod/runtime.go index 9feae03fc..82473aae9 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -12,7 +12,6 @@ import ( "github.com/containers/image/types" "github.com/containers/libpod/libpod/image" "github.com/containers/libpod/pkg/firewall" - "github.com/containers/libpod/pkg/hooks" sysreg "github.com/containers/libpod/pkg/registries" "github.com/containers/libpod/pkg/rootless" "github.com/containers/libpod/pkg/util" @@ -84,6 +83,7 @@ type Runtime struct { lock sync.RWMutex imageRuntime *image.Runtime firewallBackend firewall.FirewallBackend + configuredFrom *runtimeConfiguredFrom } // RuntimeConfig contains configuration options used to set up the runtime @@ -92,6 +92,7 @@ type RuntimeConfig struct { // Not included in on-disk config, use the dedicated containers/storage // configuration file instead StorageConfig storage.StoreOptions `toml:"-"` + VolumePath string `toml:"volume_path"` // ImageDefaultTransport is the default transport method used to fetch // images ImageDefaultTransport string `toml:"image_default_transport"` @@ -141,11 +142,11 @@ type RuntimeConfig struct { // CNIDefaultNetwork is the network name of the default CNI network // to attach pods to CNIDefaultNetwork string `toml:"cni_default_network,omitempty"` - // HooksDir Path to the directory containing hooks configuration files + // HooksDir holds paths to the directories containing hooks + // configuration files. When the same filename is present in in + // multiple directories, the file in the directory listed last in + // this slice takes precedence. HooksDir []string `toml:"hooks_dir"` - // HooksDirNotExistFatal switches between fatal errors and non-fatal - // warnings if the configured HooksDir does not exist. - HooksDirNotExistFatal bool `toml:"hooks_dir_not_exist_fatal"` // DefaultMountsFile is the path to the default mounts file for testing // purposes only DefaultMountsFile string `toml:"-"` @@ -175,6 +176,20 @@ type RuntimeConfig struct { EnableLabeling bool `toml:"label"` } +// runtimeConfiguredFrom is a struct used during early runtime init to help +// assemble the full RuntimeConfig struct from defaults. +// It indicated whether several fields in the runtime configuration were set +// explicitly. +// If they were not, we may override them with information from the database, +// if it exists and differs from what is present in the system already. +type runtimeConfiguredFrom struct { + storageGraphDriverSet bool + storageGraphRootSet bool + storageRunRootSet bool + libpodStaticDirSet bool + libpodTmpDirSet bool +} + var ( defaultRuntimeConfig = RuntimeConfig{ // Leave this empty so containers/storage will use its defaults @@ -203,7 +218,6 @@ var ( "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", }, CgroupManager: SystemdCgroupsManager, - HooksDir: []string{hooks.DefaultDir, hooks.OverrideDir}, StaticDir: filepath.Join(storage.DefaultStoreOptions.GraphRoot, "libpod"), TmpDir: "", MaxLogSize: -1, @@ -253,6 +267,7 @@ func SetXdgRuntimeDir(val string) error { func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) { runtime = new(Runtime) runtime.config = new(RuntimeConfig) + runtime.configuredFrom = new(runtimeConfiguredFrom) // Copy the default configuration tmpDir, err := getDefaultTmpDir() @@ -262,6 +277,17 @@ func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) { deepcopier.Copy(defaultRuntimeConfig).To(runtime.config) runtime.config.TmpDir = tmpDir + if rootless.IsRootless() { + // If we're rootless, override the default storage config + storageConf, volumePath, err := util.GetDefaultStoreOptions() + if err != nil { + return nil, errors.Wrapf(err, "error retrieving rootless storage config") + } + runtime.config.StorageConfig = storageConf + runtime.config.StaticDir = filepath.Join(storageConf.GraphRoot, "libpod") + runtime.config.VolumePath = volumePath + } + configPath := ConfigPath foundConfig := true rootlessConfigPath := "" @@ -307,6 +333,25 @@ func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) { if err != nil { return nil, errors.Wrapf(err, "error reading configuration file %s", configPath) } + + // This is ugly, but we need to decode twice. + // Once to check if libpod static and tmp dirs were explicitly + // set (not enough to check if they're not the default value, + // might have been explicitly configured to the default). + // A second time to actually get a usable config. + tmpConfig := new(RuntimeConfig) + if _, err := toml.Decode(string(contents), tmpConfig); err != nil { + return nil, errors.Wrapf(err, "error decoding configuration file %s", + configPath) + } + + if tmpConfig.StaticDir != "" { + runtime.configuredFrom.libpodStaticDirSet = true + } + if tmpConfig.TmpDir != "" { + runtime.configuredFrom.libpodTmpDirSet = true + } + if _, err := toml.Decode(string(contents), runtime.config); err != nil { return nil, errors.Wrapf(err, "error decoding configuration file %s", configPath) } @@ -348,6 +393,7 @@ func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) { func NewRuntimeFromConfig(configPath string, options ...RuntimeOption) (runtime *Runtime, err error) { runtime = new(Runtime) runtime.config = new(RuntimeConfig) + runtime.configuredFrom = new(runtimeConfiguredFrom) // Set two fields not in the TOML config runtime.config.StateType = defaultRuntimeConfig.StateType @@ -426,6 +472,77 @@ func makeRuntime(runtime *Runtime) (err error) { runtime.config.ConmonPath) } + // Make the static files directory if it does not exist + if err := os.MkdirAll(runtime.config.StaticDir, 0700); err != nil { + // The directory is allowed to exist + if !os.IsExist(err) { + return errors.Wrapf(err, "error creating runtime static files directory %s", + runtime.config.StaticDir) + } + } + + // Set up the state + switch runtime.config.StateType { + case InMemoryStateStore: + state, err := NewInMemoryState() + if err != nil { + return err + } + runtime.state = state + case SQLiteStateStore: + return errors.Wrapf(ErrInvalidArg, "SQLite state is currently disabled") + case BoltDBStateStore: + dbPath := filepath.Join(runtime.config.StaticDir, "bolt_state.db") + + state, err := NewBoltState(dbPath, runtime) + if err != nil { + return err + } + runtime.state = state + default: + return errors.Wrapf(ErrInvalidArg, "unrecognized state type passed") + } + + // Grab config from the database so we can reset some defaults + dbConfig, err := runtime.state.GetDBConfig() + if err != nil { + return errors.Wrapf(err, "error retrieving runtime configuration from database") + } + + // Reset defaults if they were not explicitly set + if !runtime.configuredFrom.storageGraphDriverSet && dbConfig.GraphDriver != "" { + runtime.config.StorageConfig.GraphDriverName = dbConfig.GraphDriver + } + if !runtime.configuredFrom.storageGraphRootSet && dbConfig.StorageRoot != "" { + runtime.config.StorageConfig.GraphRoot = dbConfig.StorageRoot + } + if !runtime.configuredFrom.storageRunRootSet && dbConfig.StorageTmp != "" { + runtime.config.StorageConfig.RunRoot = dbConfig.StorageTmp + } + if !runtime.configuredFrom.libpodStaticDirSet && dbConfig.LibpodRoot != "" { + runtime.config.StaticDir = dbConfig.LibpodRoot + } + if !runtime.configuredFrom.libpodTmpDirSet && dbConfig.LibpodTmp != "" { + runtime.config.TmpDir = dbConfig.LibpodTmp + } + + logrus.Debugf("Using graph driver %s", runtime.config.StorageConfig.GraphDriverName) + logrus.Debugf("Using graph root %s", runtime.config.StorageConfig.GraphRoot) + logrus.Debugf("Using run root %s", runtime.config.StorageConfig.RunRoot) + logrus.Debugf("Using static dir %s", runtime.config.StaticDir) + logrus.Debugf("Using tmp dir %s", runtime.config.TmpDir) + + // Validate our config against the database, now that we've set our + // final storage configuration + if err := runtime.state.ValidateDBConfig(runtime); err != nil { + return err + } + + if err := runtime.state.SetNamespace(runtime.config.Namespace); err != nil { + return errors.Wrapf(err, "error setting libpod namespace in state") + } + logrus.Debugf("Set libpod namespace to %q", runtime.config.Namespace) + // Set up containers/storage var store storage.Store if rootless.SkipStorageSetup() { @@ -493,15 +610,6 @@ func makeRuntime(runtime *Runtime) (err error) { } runtime.ociRuntime = ociRuntime - // Make the static files directory if it does not exist - if err := os.MkdirAll(runtime.config.StaticDir, 0755); err != nil { - // The directory is allowed to exist - if !os.IsExist(err) { - return errors.Wrapf(err, "error creating runtime static files directory %s", - runtime.config.StaticDir) - } - } - // Make a directory to hold container lockfiles lockDir := filepath.Join(runtime.config.TmpDir, "lock") if err := os.MkdirAll(lockDir, 0755); err != nil { @@ -523,11 +631,13 @@ func makeRuntime(runtime *Runtime) (err error) { } // Set up the CNI net plugin - netPlugin, err := ocicni.InitCNI(runtime.config.CNIDefaultNetwork, runtime.config.CNIConfigDir, runtime.config.CNIPluginDir...) - if err != nil { - return errors.Wrapf(err, "error configuring CNI network plugin") + if !rootless.IsRootless() { + netPlugin, err := ocicni.InitCNI(runtime.config.CNIDefaultNetwork, runtime.config.CNIConfigDir, runtime.config.CNIPluginDir...) + if err != nil { + return errors.Wrapf(err, "error configuring CNI network plugin") + } + runtime.netPlugin = netPlugin } - runtime.netPlugin = netPlugin // Set up a firewall backend backendType := "" @@ -540,33 +650,6 @@ func makeRuntime(runtime *Runtime) (err error) { } runtime.firewallBackend = fwBackend - // Set up the state - switch runtime.config.StateType { - case InMemoryStateStore: - state, err := NewInMemoryState() - if err != nil { - return err - } - runtime.state = state - case SQLiteStateStore: - return errors.Wrapf(ErrInvalidArg, "SQLite state is currently disabled") - case BoltDBStateStore: - dbPath := filepath.Join(runtime.config.StaticDir, "bolt_state.db") - - state, err := NewBoltState(dbPath, runtime.lockDir, runtime) - if err != nil { - return err - } - runtime.state = state - default: - return errors.Wrapf(ErrInvalidArg, "unrecognized state type passed") - } - - if err := runtime.state.SetNamespace(runtime.config.Namespace); err != nil { - return errors.Wrapf(err, "error setting libpod namespace in state") - } - logrus.Debugf("Set libpod namespace to %q", runtime.config.Namespace) - // We now need to see if the system has restarted // We check for the presence of a file in our tmp directory to verify this // This check must be locked to prevent races diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 09d0ec042..ba8eaacbe 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -154,6 +154,24 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. } }() + // Go through the volume mounts and check for named volumes + // If the named volme already exists continue, otherwise create + // the storage for the named volume. + for i, vol := range ctr.config.Spec.Mounts { + if vol.Source[0] != '/' && isNamedVolume(vol.Source) { + volInfo, err := r.state.Volume(vol.Source) + if err != nil { + newVol, err := r.newVolume(ctx, WithVolumeName(vol.Source)) + if err != nil { + logrus.Errorf("error creating named volume %q: %v", vol.Source, err) + } + ctr.config.Spec.Mounts[i].Source = newVol.MountPoint() + continue + } + ctr.config.Spec.Mounts[i].Source = volInfo.MountPoint() + } + } + if ctr.config.LogPath == "" { ctr.config.LogPath = filepath.Join(ctr.config.StaticDir, "ctr.log") } @@ -170,6 +188,7 @@ func (r *Runtime) newContainer(ctx context.Context, rSpec *spec.Spec, options .. } ctr.config.Mounts = append(ctr.config.Mounts, ctr.config.ShmDir) } + // Add the container to the state // TODO: May be worth looking into recovering from name/ID collisions here if ctr.config.Pod != "" { @@ -474,3 +493,11 @@ func (r *Runtime) GetLatestContainer() (*Container, error) { } return ctrs[lastCreatedIndex], nil } + +// Check if volName is a named volume and not one of the default mounts we add to containers +func isNamedVolume(volName string) bool { + if volName != "proc" && volName != "tmpfs" && volName != "devpts" && volName != "shm" && volName != "mqueue" && volName != "sysfs" && volName != "cgroup" { + return true + } + return false +} diff --git a/libpod/runtime_img.go b/libpod/runtime_img.go index be8711734..66844bb31 100644 --- a/libpod/runtime_img.go +++ b/libpod/runtime_img.go @@ -3,50 +3,15 @@ package libpod import ( "context" "fmt" - "io" "github.com/containers/buildah/imagebuildah" - "github.com/containers/libpod/libpod/common" "github.com/containers/libpod/libpod/image" "github.com/containers/storage" - "github.com/containers/storage/pkg/archive" - ociv1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" ) // Runtime API -// CopyOptions contains the options given when pushing or pulling images -type CopyOptions struct { - // Compression specifies the type of compression which is applied to - // layer blobs. The default is to not use compression, but - // archive.Gzip is recommended. - Compression archive.Compression - // DockerRegistryOptions encapsulates settings that affect how we - // connect or authenticate to a remote registry to which we want to - // push the image. - common.DockerRegistryOptions - // SigningOptions encapsulates settings that control whether or not we - // strip or add signatures to the image when pushing (uploading) the - // image to a registry. - common.SigningOptions - - // SigningPolicyPath this points to a alternative signature policy file, used mainly for testing - SignaturePolicyPath string - // AuthFile is the path of the cached credentials file defined by the user - AuthFile string - // Writer is the reportWriter for the output - Writer io.Writer - // Reference is the name for the image created when a tar archive is imported - Reference string - // ImageConfig is the Image spec for the image created when a tar archive is imported - ImageConfig ociv1.Image - // ManifestMIMEType is the manifest type of the image when saving to a directory - ManifestMIMEType string - // ForceCompress compresses the image layers when saving to a directory using the dir transport if true - ForceCompress bool -} - // RemoveImage deletes an image from local storage // Images being used by running containers can only be removed if force=true func (r *Runtime) RemoveImage(ctx context.Context, img *image.Image, force bool) (string, error) { diff --git a/libpod/runtime_pod_infra_linux.go b/libpod/runtime_pod_infra_linux.go index 8a5dbef56..5e1051150 100644 --- a/libpod/runtime_pod_infra_linux.go +++ b/libpod/runtime_pod_infra_linux.go @@ -67,7 +67,7 @@ func (r *Runtime) createInfraContainer(ctx context.Context, p *Pod) (*Container, return nil, ErrRuntimeStopped } - newImage, err := r.ImageRuntime().New(ctx, r.config.InfraImage, "", "", nil, nil, image.SigningOptions{}, false, false) + newImage, err := r.ImageRuntime().New(ctx, r.config.InfraImage, "", "", nil, nil, image.SigningOptions{}, false) if err != nil { return nil, err } diff --git a/libpod/runtime_volume.go b/libpod/runtime_volume.go new file mode 100644 index 000000000..3921758ee --- /dev/null +++ b/libpod/runtime_volume.go @@ -0,0 +1,107 @@ +package libpod + +import ( + "context" +) + +// Contains the public Runtime API for volumes + +// A VolumeCreateOption is a functional option which alters the Volume created by +// NewVolume +type VolumeCreateOption func(*Volume) error + +// VolumeFilter is a function to determine whether a volume is included in command +// output. Volumes to be outputted are tested using the function. a true return will +// include the volume, a false return will exclude it. +type VolumeFilter func(*Volume) bool + +// RemoveVolume removes a volumes +func (r *Runtime) RemoveVolume(ctx context.Context, v *Volume, force, prune bool) error { + r.lock.Lock() + defer r.lock.Unlock() + + if !r.valid { + return ErrRuntimeStopped + } + + if !v.valid { + if ok, _ := r.state.HasVolume(v.Name()); !ok { + // Volume probably already removed + // Or was never in the runtime to begin with + return nil + } + } + + v.lock.Lock() + defer v.lock.Unlock() + + return r.removeVolume(ctx, v, force, prune) +} + +// GetVolume retrieves a volume by its name +func (r *Runtime) GetVolume(name string) (*Volume, error) { + r.lock.RLock() + defer r.lock.RUnlock() + + if !r.valid { + return nil, ErrRuntimeStopped + } + + return r.state.Volume(name) +} + +// HasVolume checks to see if a volume with the given name exists +func (r *Runtime) HasVolume(name string) (bool, error) { + r.lock.RLock() + defer r.lock.RUnlock() + + if !r.valid { + return false, ErrRuntimeStopped + } + + return r.state.HasVolume(name) +} + +// Volumes retrieves all volumes +// Filters can be provided which will determine which volumes are included in the +// output. Multiple filters are handled by ANDing their output, so only volumes +// matching all filters are returned +func (r *Runtime) Volumes(filters ...VolumeFilter) ([]*Volume, error) { + r.lock.RLock() + defer r.lock.RUnlock() + + if !r.valid { + return nil, ErrRuntimeStopped + } + + vols, err := r.state.AllVolumes() + if err != nil { + return nil, err + } + + volsFiltered := make([]*Volume, 0, len(vols)) + for _, vol := range vols { + include := true + for _, filter := range filters { + include = include && filter(vol) + } + + if include { + volsFiltered = append(volsFiltered, vol) + } + } + + return volsFiltered, nil +} + +// GetAllVolumes retrieves all the volumes +func (r *Runtime) GetAllVolumes() ([]*Volume, error) { + r.lock.RLock() + defer r.lock.RUnlock() + + if !r.valid { + return nil, ErrRuntimeStopped + } + + return r.state.AllVolumes() +} diff --git a/libpod/runtime_volume_linux.go b/libpod/runtime_volume_linux.go new file mode 100644 index 000000000..5cc0938f0 --- /dev/null +++ b/libpod/runtime_volume_linux.go @@ -0,0 +1,132 @@ +// +build linux + +package libpod + +import ( + "context" + "os" + "path/filepath" + "strings" + + "github.com/containers/storage" + "github.com/containers/storage/pkg/stringid" + "github.com/opencontainers/selinux/go-selinux/label" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +// NewVolume creates a new empty volume +func (r *Runtime) NewVolume(ctx context.Context, options ...VolumeCreateOption) (*Volume, error) { + r.lock.Lock() + defer r.lock.Unlock() + + if !r.valid { + return nil, ErrRuntimeStopped + } + return r.newVolume(ctx, options...) +} + +// newVolume creates a new empty volume +func (r *Runtime) newVolume(ctx context.Context, options ...VolumeCreateOption) (*Volume, error) { + volume, err := newVolume(r) + if err != nil { + return nil, errors.Wrapf(err, "error creating volume") + } + + for _, option := range options { + if err := option(volume); err != nil { + return nil, errors.Wrapf(err, "error running volume create option") + } + } + + if volume.config.Name == "" { + volume.config.Name = stringid.GenerateNonCryptoID() + } + // TODO: support for other volume drivers + if volume.config.Driver == "" { + volume.config.Driver = "local" + } + // TODO: determine when the scope is global and set it to that + if volume.config.Scope == "" { + volume.config.Scope = "local" + } + + // Create the mountpoint of this volume + fullVolPath := filepath.Join(r.config.VolumePath, volume.config.Name, "_data") + if err := os.MkdirAll(fullVolPath, 0755); err != nil { + return nil, errors.Wrapf(err, "error creating volume directory %q", fullVolPath) + } + _, mountLabel, err := label.InitLabels([]string{}) + if err != nil { + return nil, errors.Wrapf(err, "error getting default mountlabels") + } + if err := label.ReleaseLabel(mountLabel); err != nil { + return nil, errors.Wrapf(err, "error releasing label %q", mountLabel) + } + if err := label.Relabel(fullVolPath, mountLabel, true); err != nil { + return nil, errors.Wrapf(err, "error setting selinux label to %q", fullVolPath) + } + volume.config.MountPoint = fullVolPath + + // Path our lock file will reside at + lockPath := filepath.Join(r.lockDir, volume.config.Name) + // Grab a lockfile at the given path + lock, err := storage.GetLockfile(lockPath) + if err != nil { + return nil, errors.Wrapf(err, "error creating lockfile for new volume") + } + volume.lock = lock + + volume.valid = true + + // Add the volume to state + if err := r.state.AddVolume(volume); err != nil { + return nil, errors.Wrapf(err, "error adding volume to state") + } + + return volume, nil +} + +// removeVolume removes the specified volume from state as well tears down its mountpoint and storage +func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force, prune bool) error { + if !v.valid { + return ErrNoSuchVolume + } + + deps, err := r.state.VolumeInUse(v) + if err != nil { + return err + } + if len(deps) != 0 { + if prune { + return ErrVolumeBeingUsed + } + depsStr := strings.Join(deps, ", ") + if !force { + return errors.Wrapf(ErrVolumeBeingUsed, "volume %s is being used by the following container(s): %s", v.Name(), depsStr) + } + // If using force, log the warning that the volume is being used by at least one container + logrus.Warnf("volume %s is being used by the following container(s): %s", v.Name(), depsStr) + // Remove the container dependencies so we can go ahead and delete the volume + for _, dep := range deps { + if err := r.state.RemoveVolCtrDep(v, dep); err != nil { + return errors.Wrapf(err, "unable to remove container dependency %q from volume %q while trying to delete volume by force", dep, v.Name()) + } + } + } + + // Delete the mountpoint path of the volume, that is delete the volume from /var/lib/containers/storage/volumes + if err := v.teardownStorage(); err != nil { + return errors.Wrapf(err, "error cleaning up volume storage for %q", v.Name()) + } + + // Remove the volume from the state + if err := r.state.RemoveVolume(v); err != nil { + return errors.Wrapf(err, "error removing volume %s", v.Name()) + } + + // Set volume as invalid so it can no longer be used + v.valid = false + + return nil +} diff --git a/libpod/state.go b/libpod/state.go index 273e81318..88d89f673 100644 --- a/libpod/state.go +++ b/libpod/state.go @@ -1,5 +1,15 @@ package libpod +// DBConfig is a set of Libpod runtime configuration settings that are saved +// in a State when it is first created, and can subsequently be retrieved. +type DBConfig struct { + LibpodRoot string + LibpodTmp string + StorageRoot string + StorageTmp string + GraphDriver string +} + // State is a storage backend for libpod's current state. // A State is only initialized once per instance of libpod. // As such, initialization methods for State implementations may safely assume @@ -21,6 +31,22 @@ type State interface { // Refresh clears container and pod states after a reboot Refresh() error + // GetDBConfig retrieves several paths configured within the database + // when it was created - namely, Libpod root and tmp dirs, c/storage + // root and tmp dirs, and c/storage graph driver. + // This is not implemented by the in-memory state, as it has no need to + // validate runtime configuration. + GetDBConfig() (*DBConfig, error) + + // ValidateDBConfig validates the config in the given Runtime struct + // against paths stored in the configured database. + // Libpod root and tmp dirs and c/storage root and tmp dirs and graph + // driver are validated. + // This is not implemented by the in-memory state, as it has no need to + // validate runtime configuration that may change over multiple runs of + // the program. + ValidateDBConfig(runtime *Runtime) error + // SetNamespace() sets the namespace for the store, and will determine // what containers are retrieved with container and pod retrieval calls. // A namespace of "", the empty string, acts as no namespace, and @@ -127,4 +153,27 @@ type State interface { // If a namespace has been set, only pods in that namespace will be // returned. AllPods() ([]*Pod, error) + + // Volume accepts full name of volume + // If the volume doesn't exist, an error will be returned + Volume(volName string) (*Volume, error) + // HasVolume returns true if volName exists in the state, + // otherwise it returns false + HasVolume(volName string) (bool, error) + // VolumeInUse goes through the container dependencies of a volume + // and checks if the volume is being used by any container. If it is + // a slice of container IDs using the volume is returned + VolumeInUse(volume *Volume) ([]string, error) + // AddVolume adds the specified volume to state. The volume's name + // must be unique within the list of existing volumes + AddVolume(volume *Volume) error + // RemoveVolCtrDep updates the list of container dependencies that the + // volume has. It either deletes the dependent container ID from + // the sub-bucket + RemoveVolCtrDep(volume *Volume, ctrID string) error + // RemoveVolume removes the specified volume. + // Only volumes that have no container dependencies can be removed + RemoveVolume(volume *Volume) error + // AllVolumes returns all the volumes available in the state + AllVolumes() ([]*Volume, error) } diff --git a/libpod/state_test.go b/libpod/state_test.go index 04572fb29..d93a371f3 100644 --- a/libpod/state_test.go +++ b/libpod/state_test.go @@ -45,11 +45,16 @@ func getEmptyBoltState() (s State, p string, p2 string, err error) { dbPath := filepath.Join(tmpDir, "db.sql") lockDir := filepath.Join(tmpDir, "locks") + if err := os.Mkdir(lockDir, 0755); err != nil { + return nil, "", "", err + } + runtime := new(Runtime) runtime.config = new(RuntimeConfig) runtime.config.StorageConfig = storage.StoreOptions{} + runtime.lockDir = lockDir - state, err := NewBoltState(dbPath, lockDir, runtime) + state, err := NewBoltState(dbPath, runtime) if err != nil { return nil, "", "", err } diff --git a/libpod/testdata/config.toml b/libpod/testdata/config.toml index e19d36017..1d78f2083 100644 --- a/libpod/testdata/config.toml +++ b/libpod/testdata/config.toml @@ -14,7 +14,7 @@ seccomp_profile = "/etc/crio/seccomp.json" apparmor_profile = "crio-default" cgroup_manager = "cgroupfs" - hooks_dir_path = "/usr/share/containers/oci/hooks.d" + hooks_dir = ["/usr/share/containers/oci/hooks.d"] pids_limit = 2048 container_exits_dir = "/var/run/podman/exits" [crio.image] diff --git a/libpod/util.go b/libpod/util.go index aa3494529..b7578135a 100644 --- a/libpod/util.go +++ b/libpod/util.go @@ -9,10 +9,8 @@ import ( "strings" "time" - "github.com/containerd/cgroups" "github.com/containers/image/signature" "github.com/containers/image/types" - "github.com/containers/libpod/pkg/util" "github.com/fsnotify/fsnotify" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -189,26 +187,3 @@ func validPodNSOption(p *Pod, ctrPod string) error { } return nil } - -// GetV1CGroups gets the V1 cgroup subsystems and then "filters" -// out any subsystems that are provided by the caller. Passing nil -// for excludes will return the subsystems unfiltered. -//func GetV1CGroups(excludes []string) ([]cgroups.Subsystem, error) { -func GetV1CGroups(excludes []string) cgroups.Hierarchy { - return func() ([]cgroups.Subsystem, error) { - var filtered []cgroups.Subsystem - - subSystem, err := cgroups.V1() - if err != nil { - return nil, err - } - for _, s := range subSystem { - // If the name of the subsystem is not in the list of excludes, then - // add it as a keeper. - if !util.StringInSlice(string(s.Name()), excludes) { - filtered = append(filtered, s) - } - } - return filtered, nil - } -} diff --git a/libpod/util_linux.go b/libpod/util_linux.go index 0cd486379..30e2538c3 100644 --- a/libpod/util_linux.go +++ b/libpod/util_linux.go @@ -7,6 +7,7 @@ import ( "strings" "github.com/containerd/cgroups" + "github.com/containers/libpod/pkg/util" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -67,3 +68,26 @@ func assembleSystemdCgroupName(baseSlice, newSlice string) (string, error) { return final, nil } + +// GetV1CGroups gets the V1 cgroup subsystems and then "filters" +// out any subsystems that are provided by the caller. Passing nil +// for excludes will return the subsystems unfiltered. +//func GetV1CGroups(excludes []string) ([]cgroups.Subsystem, error) { +func GetV1CGroups(excludes []string) cgroups.Hierarchy { + return func() ([]cgroups.Subsystem, error) { + var filtered []cgroups.Subsystem + + subSystem, err := cgroups.V1() + if err != nil { + return nil, err + } + for _, s := range subSystem { + // If the name of the subsystem is not in the list of excludes, then + // add it as a keeper. + if !util.StringInSlice(string(s.Name()), excludes) { + filtered = append(filtered, s) + } + } + return filtered, nil + } +} diff --git a/libpod/volume.go b/libpod/volume.go new file mode 100644 index 000000000..b732e8aa7 --- /dev/null +++ b/libpod/volume.go @@ -0,0 +1,63 @@ +package libpod + +import "github.com/containers/storage" + +// Volume is the type used to create named volumes +// TODO: all volumes should be created using this and the Volume API +type Volume struct { + config *VolumeConfig + + valid bool + runtime *Runtime + lock storage.Locker +} + +// VolumeConfig holds the volume's config information +//easyjson:json +type VolumeConfig struct { + Name string `json:"name"` + Labels map[string]string `json:"labels"` + MountPoint string `json:"mountPoint"` + Driver string `json:"driver"` + Options map[string]string `json:"options"` + Scope string `json:"scope"` +} + +// Name retrieves the volume's name +func (v *Volume) Name() string { + return v.config.Name +} + +// Labels returns the volume's labels +func (v *Volume) Labels() map[string]string { + labels := make(map[string]string) + for key, value := range v.config.Labels { + labels[key] = value + } + return labels +} + +// MountPoint returns the volume's mountpoint on the host +func (v *Volume) MountPoint() string { + return v.config.MountPoint +} + +// Driver returns the volume's driver +func (v *Volume) Driver() string { + return v.config.Driver +} + +// Options return the volume's options +func (v *Volume) Options() map[string]string { + options := make(map[string]string) + for key, value := range v.config.Options { + options[key] = value + } + + return options +} + +// Scope returns the scope of the volume +func (v *Volume) Scope() string { + return v.config.Scope +} diff --git a/libpod/volume_internal.go b/libpod/volume_internal.go new file mode 100644 index 000000000..800e6d106 --- /dev/null +++ b/libpod/volume_internal.go @@ -0,0 +1,29 @@ +package libpod + +import ( + "os" + "path/filepath" +) + +// VolumePath is the path under which all volumes that are created using the +// local driver will be created +// const VolumePath = "/var/lib/containers/storage/volumes" + +// Creates a new volume +func newVolume(runtime *Runtime) (*Volume, error) { + volume := new(Volume) + volume.config = new(VolumeConfig) + volume.runtime = runtime + volume.config.Labels = make(map[string]string) + volume.config.Options = make(map[string]string) + + return volume, nil +} + +// teardownStorage deletes the volume from volumePath +func (v *Volume) teardownStorage() error { + if !v.valid { + return ErrNoSuchVolume + } + return os.RemoveAll(filepath.Join(v.runtime.config.VolumePath, v.Name())) +} diff --git a/pkg/lookup/lookup.go b/pkg/lookup/lookup.go index a9d975b4b..70b97144f 100644 --- a/pkg/lookup/lookup.go +++ b/pkg/lookup/lookup.go @@ -99,9 +99,11 @@ func GetContainerGroups(groups []string, containerMount string, override *Overri return uintgids, nil } -// GetUser takes a containermount path and user name or id and returns +// GetUser takes a containermount path and user name or ID and returns // a matching User structure from /etc/passwd. If it cannot locate a user // with the provided information, an ErrNoPasswdEntries is returned. +// When the provided user name was an ID, a User structure with Uid +// set is returned along with ErrNoPasswdEntries. func GetUser(containerMount, userIDorName string) (*user.User, error) { var inputIsName bool uid, err := strconv.Atoi(userIDorName) @@ -124,12 +126,17 @@ func GetUser(containerMount, userIDorName string) (*user.User, error) { if len(users) > 0 { return &users[0], nil } + if !inputIsName { + return &user.User{Uid: uid}, user.ErrNoPasswdEntries + } return nil, user.ErrNoPasswdEntries } -// GetGroup takes ac ontainermount path and a group name or id and returns -// a match Group struct from /etc/group. if it cannot locate a group, -// an ErrNoGroupEntries error is returned. +// GetGroup takes a containermount path and a group name or ID and returns +// a match Group struct from /etc/group. If it cannot locate a group, +// an ErrNoGroupEntries error is returned. When the provided group name +// was an ID, a Group structure with Gid set is returned along with +// ErrNoGroupEntries. func GetGroup(containerMount, groupIDorName string) (*user.Group, error) { var inputIsName bool gid, err := strconv.Atoi(groupIDorName) @@ -154,5 +161,8 @@ func GetGroup(containerMount, groupIDorName string) (*user.Group, error) { if len(groups) > 0 { return &groups[0], nil } + if !inputIsName { + return &user.Group{Gid: gid}, user.ErrNoGroupEntries + } return nil, user.ErrNoGroupEntries } diff --git a/pkg/registries/registries.go b/pkg/registries/registries.go index c26f15cb6..cbb8b730c 100644 --- a/pkg/registries/registries.go +++ b/pkg/registries/registries.go @@ -13,21 +13,28 @@ import ( // userRegistriesFile is the path to the per user registry configuration file. var userRegistriesFile = filepath.Join(os.Getenv("HOME"), ".config/containers/registries.conf") -// GetRegistries obtains the list of registries defined in the global registries file. -func GetRegistries() ([]string, error) { - registryConfigPath := "" +// SystemRegistriesConfPath returns an appropriate value for types.SystemContext.SystemRegistriesConfPath +// (possibly "", which is not an error), taking into account rootless mode and environment variable overrides. +// +// FIXME: This should be centralized in a global SystemContext initializer inherited throughout the code, +// not haphazardly called throughout the way it is being called now. +func SystemRegistriesConfPath() string { + if envOverride := os.Getenv("REGISTRIES_CONFIG_PATH"); len(envOverride) > 0 { + return envOverride + } if rootless.IsRootless() { if _, err := os.Stat(userRegistriesFile); err == nil { - registryConfigPath = userRegistriesFile + return userRegistriesFile } } - envOverride := os.Getenv("REGISTRIES_CONFIG_PATH") - if len(envOverride) > 0 { - registryConfigPath = envOverride - } - searchRegistries, err := sysregistries.GetRegistries(&types.SystemContext{SystemRegistriesConfPath: registryConfigPath}) + return "" +} + +// GetRegistries obtains the list of registries defined in the global registries file. +func GetRegistries() ([]string, error) { + searchRegistries, err := sysregistries.GetRegistries(&types.SystemContext{SystemRegistriesConfPath: SystemRegistriesConfPath()}) if err != nil { return nil, errors.Wrapf(err, "unable to parse the registries.conf file") } @@ -36,19 +43,7 @@ func GetRegistries() ([]string, error) { // GetInsecureRegistries obtains the list of insecure registries from the global registration file. func GetInsecureRegistries() ([]string, error) { - registryConfigPath := "" - - if rootless.IsRootless() { - if _, err := os.Stat(userRegistriesFile); err == nil { - registryConfigPath = userRegistriesFile - } - } - - envOverride := os.Getenv("REGISTRIES_CONFIG_PATH") - if len(envOverride) > 0 { - registryConfigPath = envOverride - } - registries, err := sysregistries.GetInsecureRegistries(&types.SystemContext{SystemRegistriesConfPath: registryConfigPath}) + registries, err := sysregistries.GetInsecureRegistries(&types.SystemContext{SystemRegistriesConfPath: SystemRegistriesConfPath()}) if err != nil { return nil, errors.Wrapf(err, "unable to parse the registries.conf file") } diff --git a/pkg/spec/createconfig.go b/pkg/spec/createconfig.go index a0fd40318..25f8cd7a1 100644 --- a/pkg/spec/createconfig.go +++ b/pkg/spec/createconfig.go @@ -496,8 +496,13 @@ func (c *CreateConfig) GetContainerCreateOptions(runtime *libpod.Runtime) ([]lib // CreatePortBindings iterates ports mappings and exposed ports into a format CNI understands func (c *CreateConfig) CreatePortBindings() ([]ocicni.PortMapping, error) { + return NatToOCIPortBindings(c.PortBindings) +} + +// NatToOCIPortBindings iterates a nat.portmap slice and creates []ocicni portmapping slice +func NatToOCIPortBindings(ports nat.PortMap) ([]ocicni.PortMapping, error) { var portBindings []ocicni.PortMapping - for containerPb, hostPb := range c.PortBindings { + for containerPb, hostPb := range ports { var pm ocicni.PortMapping pm.ContainerPort = int32(containerPb.Int()) for _, i := range hostPb { diff --git a/pkg/util/utils.go b/pkg/util/utils.go index de29bc5d8..f567f2675 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -250,30 +250,40 @@ func GetRootlessRuntimeDir() (string, error) { return runtimeDir, nil } -// GetRootlessStorageOpts returns the storage ops for containers running as non root -func GetRootlessStorageOpts() (storage.StoreOptions, error) { - var opts storage.StoreOptions - +// GetRootlessDirInfo returns the parent path of where the storage for containers and +// volumes will be in rootless mode +func GetRootlessDirInfo() (string, string, error) { rootlessRuntime, err := GetRootlessRuntimeDir() if err != nil { - return opts, err + return "", "", err } - opts.RunRoot = rootlessRuntime dataDir := os.Getenv("XDG_DATA_HOME") if dataDir == "" { home := os.Getenv("HOME") if home == "" { - return opts, fmt.Errorf("neither XDG_DATA_HOME nor HOME was set non-empty") + return "", "", fmt.Errorf("neither XDG_DATA_HOME nor HOME was set non-empty") } // runc doesn't like symlinks in the rootfs path, and at least // on CoreOS /home is a symlink to /var/home, so resolve any symlink. resolvedHome, err := filepath.EvalSymlinks(home) if err != nil { - return opts, errors.Wrapf(err, "cannot resolve %s", home) + return "", "", errors.Wrapf(err, "cannot resolve %s", home) } dataDir = filepath.Join(resolvedHome, ".local", "share") } + return dataDir, rootlessRuntime, nil +} + +// GetRootlessStorageOpts returns the storage opts for containers running as non root +func GetRootlessStorageOpts() (storage.StoreOptions, error) { + var opts storage.StoreOptions + + dataDir, rootlessRuntime, err := GetRootlessDirInfo() + if err != nil { + return opts, err + } + opts.RunRoot = rootlessRuntime opts.GraphRoot = filepath.Join(dataDir, "containers", "storage") if path, err := exec.LookPath("fuse-overlayfs"); err == nil { opts.GraphDriverName = "overlay" @@ -284,6 +294,15 @@ func GetRootlessStorageOpts() (storage.StoreOptions, error) { return opts, nil } +// GetRootlessVolumeInfo returns where all the name volumes will be created in rootless mode +func GetRootlessVolumeInfo() (string, error) { + dataDir, _, err := GetRootlessDirInfo() + if err != nil { + return "", err + } + return filepath.Join(dataDir, "containers", "storage", "volumes"), nil +} + type tomlOptionsConfig struct { MountProgram string `toml:"mount_program"` } @@ -313,14 +332,21 @@ func getTomlStorage(storeOptions *storage.StoreOptions) *tomlConfig { return config } -// GetDefaultStoreOptions returns the storage ops for containers -func GetDefaultStoreOptions() (storage.StoreOptions, error) { +// GetDefaultStoreOptions returns the storage ops for containers and the volume path +// for the volume API +// It also returns the path where all named volumes will be created using the volume API +func GetDefaultStoreOptions() (storage.StoreOptions, string, error) { storageOpts := storage.DefaultStoreOptions + volumePath := "/var/lib/containers/storage" if rootless.IsRootless() { var err error storageOpts, err = GetRootlessStorageOpts() if err != nil { - return storageOpts, err + return storageOpts, volumePath, err + } + volumePath, err = GetRootlessVolumeInfo() + if err != nil { + return storageOpts, volumePath, err } storageConf := filepath.Join(os.Getenv("HOME"), ".config/containers/storage.conf") @@ -330,7 +356,7 @@ func GetDefaultStoreOptions() (storage.StoreOptions, error) { os.MkdirAll(filepath.Dir(storageConf), 0755) file, err := os.OpenFile(storageConf, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666) if err != nil { - return storageOpts, errors.Wrapf(err, "cannot open %s", storageConf) + return storageOpts, volumePath, errors.Wrapf(err, "cannot open %s", storageConf) } tomlConfiguration := getTomlStorage(&storageOpts) @@ -341,5 +367,5 @@ func GetDefaultStoreOptions() (storage.StoreOptions, error) { } } } - return storageOpts, nil + return storageOpts, volumePath, nil } diff --git a/pkg/varlinkapi/containers.go b/pkg/varlinkapi/containers.go index f517e9b6e..07d981786 100644 --- a/pkg/varlinkapi/containers.go +++ b/pkg/varlinkapi/containers.go @@ -278,6 +278,18 @@ func (i *LibpodAPI) RestartContainer(call iopodman.VarlinkCall, name string, tim return call.ReplyRestartContainer(ctr.ID()) } +// ContainerExists looks in local storage for the existence of a container +func (i *LibpodAPI) ContainerExists(call iopodman.VarlinkCall, name string) error { + _, err := i.Runtime.LookupContainer(name) + if errors.Cause(err) == libpod.ErrNoSuchCtr { + return call.ReplyContainerExists(1) + } + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + return call.ReplyContainerExists(0) +} + // KillContainer kills a running container. If you want to use the default SIGTERM signal, just send a -1 // for the signal arg. func (i *LibpodAPI) KillContainer(call iopodman.VarlinkCall, name string, signal int64) error { @@ -413,3 +425,40 @@ func (i *LibpodAPI) GetAttachSockets(call iopodman.VarlinkCall, name string) err } return call.ReplyGetAttachSockets(s) } + +// ContainerCheckpoint ... +func (i *LibpodAPI) ContainerCheckpoint(call iopodman.VarlinkCall, name string, keep, leaveRunning, tcpEstablished bool) error { + ctx := getContext() + ctr, err := i.Runtime.LookupContainer(name) + if err != nil { + return call.ReplyContainerNotFound(name) + } + + options := libpod.ContainerCheckpointOptions{ + Keep: keep, + TCPEstablished: tcpEstablished, + KeepRunning: leaveRunning, + } + if err := ctr.Checkpoint(ctx, options); err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + return call.ReplyContainerCheckpoint(ctr.ID()) +} + +// ContainerRestore ... +func (i *LibpodAPI) ContainerRestore(call iopodman.VarlinkCall, name string, keep, tcpEstablished bool) error { + ctx := getContext() + ctr, err := i.Runtime.LookupContainer(name) + if err != nil { + return call.ReplyContainerNotFound(name) + } + + options := libpod.ContainerCheckpointOptions{ + Keep: keep, + TCPEstablished: tcpEstablished, + } + if err := ctr.Restore(ctx, options); err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + return call.ReplyContainerRestore(ctr.ID()) +} diff --git a/pkg/varlinkapi/containers_create.go b/pkg/varlinkapi/containers_create.go index f9a2db9c8..bb6273fd1 100644 --- a/pkg/varlinkapi/containers_create.go +++ b/pkg/varlinkapi/containers_create.go @@ -25,7 +25,7 @@ func (i *LibpodAPI) CreateContainer(call iopodman.VarlinkCall, config iopodman.C rtc := i.Runtime.GetConfig() ctx := getContext() - newImage, err := i.Runtime.ImageRuntime().New(ctx, config.Image, rtc.SignaturePolicyPath, "", os.Stderr, nil, image.SigningOptions{}, false, false) + newImage, err := i.Runtime.ImageRuntime().New(ctx, config.Image, rtc.SignaturePolicyPath, "", os.Stderr, nil, image.SigningOptions{}, false) if err != nil { return call.ReplyErrorOccurred(err.Error()) } diff --git a/pkg/varlinkapi/images.go b/pkg/varlinkapi/images.go index 42e285b53..cb3b1c73b 100644 --- a/pkg/varlinkapi/images.go +++ b/pkg/varlinkapi/images.go @@ -5,6 +5,7 @@ import ( "encoding/json" "fmt" "io" + "os" "path/filepath" "strings" "time" @@ -14,11 +15,13 @@ import ( "github.com/containers/image/docker" "github.com/containers/image/manifest" "github.com/containers/image/types" + "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/cmd/podman/varlink" "github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod/image" sysreg "github.com/containers/libpod/pkg/registries" "github.com/containers/libpod/pkg/util" + "github.com/containers/libpod/utils" "github.com/docker/go-units" "github.com/opencontainers/image-spec/specs-go/v1" "github.com/opencontainers/runtime-spec/specs-go" @@ -319,13 +322,14 @@ func (i *LibpodAPI) PushImage(call iopodman.VarlinkCall, name, tag string, tlsVe destname = tag } - dockerRegistryOptions := image.DockerRegistryOptions{ - DockerInsecureSkipTLSVerify: !tlsVerify, + dockerRegistryOptions := image.DockerRegistryOptions{} + if !tlsVerify { + dockerRegistryOptions.DockerInsecureSkipTLSVerify = types.OptionalBoolTrue } so := image.SigningOptions{} - if err := newImage.PushImageToHeuristicDestination(getContext(), destname, "", "", "", nil, false, so, &dockerRegistryOptions, false, nil); err != nil { + if err := newImage.PushImageToHeuristicDestination(getContext(), destname, "", "", "", nil, false, so, &dockerRegistryOptions, nil); err != nil { return call.ReplyErrorOccurred(err.Error()) } return call.ReplyPushImage(newImage.ID()) @@ -485,7 +489,7 @@ func (i *LibpodAPI) ExportImage(call iopodman.VarlinkCall, name, destination str return err } - if err := newImage.PushImageToHeuristicDestination(getContext(), destination, "", "", "", nil, compress, image.SigningOptions{}, &image.DockerRegistryOptions{}, false, additionalTags); err != nil { + if err := newImage.PushImageToHeuristicDestination(getContext(), destination, "", "", "", nil, compress, image.SigningOptions{}, &image.DockerRegistryOptions{}, additionalTags); err != nil { return call.ReplyErrorOccurred(err.Error()) } return call.ReplyExportImage(newImage.ID()) @@ -494,9 +498,53 @@ func (i *LibpodAPI) ExportImage(call iopodman.VarlinkCall, name, destination str // PullImage pulls an image from a registry to the image store. // TODO This implementation is incomplete func (i *LibpodAPI) PullImage(call iopodman.VarlinkCall, name string) error { - newImage, err := i.Runtime.ImageRuntime().New(getContext(), name, "", "", nil, &image.DockerRegistryOptions{}, image.SigningOptions{}, true, false) + newImage, err := i.Runtime.ImageRuntime().New(getContext(), name, "", "", nil, &image.DockerRegistryOptions{}, image.SigningOptions{}, true) if err != nil { return call.ReplyErrorOccurred(fmt.Sprintf("unable to pull %s: %s", name, err.Error())) } return call.ReplyPullImage(newImage.ID()) } + +// ImageExists returns bool as to whether the input image exists in local storage +func (i *LibpodAPI) ImageExists(call iopodman.VarlinkCall, name string) error { + _, err := i.Runtime.ImageRuntime().NewFromLocal(name) + if errors.Cause(err) == libpod.ErrNoSuchImage { + return call.ReplyImageExists(1) + } + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + return call.ReplyImageExists(0) +} + +// ContainerRunlabel ... +func (i *LibpodAPI) ContainerRunlabel(call iopodman.VarlinkCall, input iopodman.Runlabel) error { + ctx := getContext() + dockerRegistryOptions := image.DockerRegistryOptions{ + DockerCertPath: input.CertDir, + } + if !input.TlsVerify { + dockerRegistryOptions.DockerInsecureSkipTLSVerify = types.OptionalBoolTrue + } + + stdErr := os.Stderr + stdOut := os.Stdout + stdIn := os.Stdin + + runLabel, imageName, err := shared.GetRunlabel(input.Label, input.Image, ctx, i.Runtime, input.Pull, input.Creds, dockerRegistryOptions, input.Authfile, input.SignaturePolicyPath, nil) + if err != nil { + return err + } + if runLabel == "" { + return nil + } + + cmd, env, err := shared.GenerateRunlabelCommand(runLabel, imageName, input.Name, input.Opts, input.ExtraArgs) + if err != nil { + return err + } + if err := utils.ExecCmdWithStdStreams(stdIn, stdOut, stdErr, env, cmd[0], cmd[1:]...); err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + return call.ReplyContainerRunlabel() +} diff --git a/pkg/varlinkapi/mount.go b/pkg/varlinkapi/mount.go new file mode 100644 index 000000000..84e6b2709 --- /dev/null +++ b/pkg/varlinkapi/mount.go @@ -0,0 +1,49 @@ +package varlinkapi + +import ( + "github.com/containers/libpod/cmd/podman/varlink" +) + +// ListContainerMounts ... +func (i *LibpodAPI) ListContainerMounts(call iopodman.VarlinkCall) error { + var mounts []string + allContainers, err := i.Runtime.GetAllContainers() + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + for _, container := range allContainers { + mounted, mountPoint, err := container.Mounted() + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + if mounted { + mounts = append(mounts, mountPoint) + } + } + return call.ReplyListContainerMounts(mounts) +} + +// MountContainer ... +func (i *LibpodAPI) MountContainer(call iopodman.VarlinkCall, name string) error { + container, err := i.Runtime.LookupContainer(name) + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + path, err := container.Mount() + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + return call.ReplyMountContainer(path) +} + +// UnmountContainer ... +func (i *LibpodAPI) UnmountContainer(call iopodman.VarlinkCall, name string, force bool) error { + container, err := i.Runtime.LookupContainer(name) + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + if err := container.Unmount(force); err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + return call.ReplyUnmountContainer() +} diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go index 366a58f02..684a7cd88 100644 --- a/test/e2e/create_test.go +++ b/test/e2e/create_test.go @@ -189,4 +189,15 @@ var _ = Describe("Podman create", func() { Expect(session.ExitCode()).To(Equal(0)) Expect(session.OutputToString()).To(ContainSubstring("/create/test rw,nosuid,nodev,noexec,relatime - tmpfs")) }) + + It("podman create --pod automatically", func() { + session := podmanTest.Podman([]string{"create", "--pod", "new:foobar", ALPINE, "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + check := podmanTest.Podman([]string{"pod", "ps", "--no-trunc"}) + check.WaitWithDefaultTimeout() + match, _ := check.GrepString("foobar") + Expect(match).To(BeTrue()) + }) }) diff --git a/test/e2e/libpod_suite_test.go b/test/e2e/libpod_suite_test.go index f4f154ef2..268db5aa1 100644 --- a/test/e2e/libpod_suite_test.go +++ b/test/e2e/libpod_suite_test.go @@ -227,6 +227,17 @@ func (p *PodmanTestIntegration) CleanupPod() { } } +// CleanupVolume cleans up the temporary store +func (p *PodmanTestIntegration) CleanupVolume() { + // Remove all containers + session := p.Podman([]string{"volume", "rm", "-fa"}) + session.Wait(90) + // Nuke tempdir + if err := os.RemoveAll(p.TempDir); err != nil { + fmt.Printf("%q\n", err) + } +} + // PullImages pulls multiple images func (p *PodmanTestIntegration) PullImages(images []string) error { for _, i := range images { diff --git a/test/e2e/prune_test.go b/test/e2e/prune_test.go new file mode 100644 index 000000000..6679a676c --- /dev/null +++ b/test/e2e/prune_test.go @@ -0,0 +1,88 @@ +package integration + +import ( + "fmt" + "os" + + . "github.com/containers/libpod/test/utils" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var pruneImage = ` +FROM alpine:latest +LABEL RUN podman --version +RUN apk update +RUN apk add bash` + +var _ = Describe("Podman rm", func() { + var ( + tempdir string + err error + podmanTest *PodmanTestIntegration + ) + + BeforeEach(func() { + tempdir, err = CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest = PodmanTestCreate(tempdir) + podmanTest.RestoreAllArtifacts() + }) + + AfterEach(func() { + podmanTest.Cleanup() + f := CurrentGinkgoTestDescription() + timedResult := fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds()) + GinkgoWriter.Write([]byte(timedResult)) + }) + + It("podman container prune containers", func() { + top := podmanTest.RunTopContainer("") + top.WaitWithDefaultTimeout() + Expect(top.ExitCode()).To(Equal(0)) + + session := podmanTest.Podman([]string{"run", ALPINE, "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + prune := podmanTest.Podman([]string{"container", "prune"}) + prune.WaitWithDefaultTimeout() + Expect(prune.ExitCode()).To(Equal(0)) + + Expect(podmanTest.NumberOfContainers()).To(Equal(1)) + }) + + It("podman image prune none images", func() { + podmanTest.BuildImage(pruneImage, "alpine_bash:latest", "true") + + none := podmanTest.Podman([]string{"images", "-a"}) + none.WaitWithDefaultTimeout() + Expect(none.ExitCode()).To(Equal(0)) + hasNone, _ := none.GrepString("<none>") + Expect(hasNone).To(BeTrue()) + + prune := podmanTest.Podman([]string{"image", "prune"}) + prune.WaitWithDefaultTimeout() + Expect(prune.ExitCode()).To(Equal(0)) + + after := podmanTest.Podman([]string{"images", "-a"}) + after.WaitWithDefaultTimeout() + Expect(none.ExitCode()).To(Equal(0)) + hasNoneAfter, _ := after.GrepString("<none>") + Expect(hasNoneAfter).To(BeFalse()) + }) + + It("podman image prune unused images", func() { + prune := podmanTest.Podman([]string{"image", "prune"}) + prune.WaitWithDefaultTimeout() + Expect(prune.ExitCode()).To(Equal(0)) + + images := podmanTest.Podman([]string{"images", "-a"}) + images.WaitWithDefaultTimeout() + // all images are unused, so they all should be deleted! + Expect(len(images.OutputToStringArray())).To(Equal(0)) + }) + +}) diff --git a/test/e2e/rmi_test.go b/test/e2e/rmi_test.go index c2eb8b7d7..22bfbbe8c 100644 --- a/test/e2e/rmi_test.go +++ b/test/e2e/rmi_test.go @@ -250,4 +250,25 @@ var _ = Describe("Podman rmi", func() { session2.WaitWithDefaultTimeout() Expect(session2.ExitCode()).To(Equal(0)) }) + + It("podman rmi -a with parent|child images", func() { + dockerfile := `FROM docker.io/library/alpine:latest AS base +RUN touch /1 +ENV LOCAL=/1 +RUN find $LOCAL +FROM base +RUN find $LOCAL + +` + podmanTest.BuildImage(dockerfile, "test", "true") + session := podmanTest.Podman([]string{"rmi", "-a"}) + session.WaitWithDefaultTimeout() + fmt.Println(session.OutputToString()) + Expect(session.ExitCode()).To(Equal(0)) + + images := podmanTest.Podman([]string{"images", "--all"}) + images.WaitWithDefaultTimeout() + Expect(images.ExitCode()).To(Equal(0)) + Expect(len(images.OutputToStringArray())).To(Equal(0)) + }) }) diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go index 38504828b..4a9bd4e46 100644 --- a/test/e2e/run_test.go +++ b/test/e2e/run_test.go @@ -336,7 +336,7 @@ var _ = Describe("Podman run", func() { hooksDir := tempdir + "/hooks" os.Mkdir(hooksDir, 0755) fileutils.CopyFile("hooks/hooks.json", hooksDir) - os.Setenv("HOOK_OPTION", fmt.Sprintf("--hooks-dir-path=%s", hooksDir)) + os.Setenv("HOOK_OPTION", fmt.Sprintf("--hooks-dir=%s", hooksDir)) os.Remove(hcheck) session := podmanTest.Podman([]string{"run", ALPINE, "ls"}) session.Wait(10) @@ -628,4 +628,15 @@ USER mail` isSharedOnly := !strings.Contains(shared[0], "shared,") Expect(isSharedOnly).Should(BeTrue()) }) + + It("podman run --pod automatically", func() { + session := podmanTest.Podman([]string{"run", "--pod", "new:foobar", ALPINE, "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + check := podmanTest.Podman([]string{"pod", "ps", "--no-trunc"}) + check.WaitWithDefaultTimeout() + match, _ := check.GrepString("foobar") + Expect(match).To(BeTrue()) + }) }) diff --git a/test/e2e/volume_create_test.go b/test/e2e/volume_create_test.go new file mode 100644 index 000000000..50ee63f2a --- /dev/null +++ b/test/e2e/volume_create_test.go @@ -0,0 +1,60 @@ +package integration + +import ( + "fmt" + "os" + + . "github.com/containers/libpod/test/utils" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Podman volume create", func() { + var ( + tempdir string + err error + podmanTest *PodmanTestIntegration + ) + + BeforeEach(func() { + tempdir, err = CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest = PodmanTestCreate(tempdir) + podmanTest.RestoreAllArtifacts() + }) + + AfterEach(func() { + podmanTest.CleanupVolume() + f := CurrentGinkgoTestDescription() + timedResult := fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds()) + GinkgoWriter.Write([]byte(timedResult)) + }) + + It("podman create volume", func() { + session := podmanTest.Podman([]string{"volume", "create"}) + session.WaitWithDefaultTimeout() + volName := session.OutputToString() + Expect(session.ExitCode()).To(Equal(0)) + + check := podmanTest.Podman([]string{"volume", "ls", "-q"}) + check.WaitWithDefaultTimeout() + match, _ := check.GrepString(volName) + Expect(match).To(BeTrue()) + Expect(len(check.OutputToStringArray())).To(Equal(1)) + }) + + It("podman create volume with name", func() { + session := podmanTest.Podman([]string{"volume", "create", "myvol"}) + session.WaitWithDefaultTimeout() + volName := session.OutputToString() + Expect(session.ExitCode()).To(Equal(0)) + + check := podmanTest.Podman([]string{"volume", "ls", "-q"}) + check.WaitWithDefaultTimeout() + match, _ := check.GrepString(volName) + Expect(match).To(BeTrue()) + Expect(len(check.OutputToStringArray())).To(Equal(1)) + }) +}) diff --git a/test/e2e/volume_inspect_test.go b/test/e2e/volume_inspect_test.go new file mode 100644 index 000000000..d0d5a601e --- /dev/null +++ b/test/e2e/volume_inspect_test.go @@ -0,0 +1,77 @@ +package integration + +import ( + "fmt" + "os" + + . "github.com/containers/libpod/test/utils" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Podman volume inspect", func() { + var ( + tempdir string + err error + podmanTest *PodmanTestIntegration + ) + + BeforeEach(func() { + tempdir, err = CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest = PodmanTestCreate(tempdir) + podmanTest.RestoreAllArtifacts() + }) + + AfterEach(func() { + podmanTest.CleanupVolume() + f := CurrentGinkgoTestDescription() + timedResult := fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds()) + GinkgoWriter.Write([]byte(timedResult)) + }) + + It("podman inspect volume", func() { + session := podmanTest.Podman([]string{"volume", "create", "myvol"}) + session.WaitWithDefaultTimeout() + volName := session.OutputToString() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "inspect", volName}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.IsJSONOutputValid()).To(BeTrue()) + }) + + It("podman inspect volume with Go format", func() { + session := podmanTest.Podman([]string{"volume", "create", "myvol"}) + session.WaitWithDefaultTimeout() + volName := session.OutputToString() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "inspect", "--format", "{{.Name}}", volName}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.OutputToString()).To(Equal(volName)) + }) + + It("podman inspect volume with --all flag", func() { + session := podmanTest.Podman([]string{"volume", "create", "myvol1"}) + session.WaitWithDefaultTimeout() + volName1 := session.OutputToString() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "create", "myvol2"}) + session.WaitWithDefaultTimeout() + volName2 := session.OutputToString() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "inspect", "--format", "{{.Name}}", "--all"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(2)) + Expect(session.OutputToStringArray()[0]).To(Equal(volName1)) + Expect(session.OutputToStringArray()[1]).To(Equal(volName2)) + }) +}) diff --git a/test/e2e/volume_ls_test.go b/test/e2e/volume_ls_test.go new file mode 100644 index 000000000..119d29d9b --- /dev/null +++ b/test/e2e/volume_ls_test.go @@ -0,0 +1,84 @@ +package integration + +import ( + "fmt" + "os" + + . "github.com/containers/libpod/test/utils" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Podman volume ls", func() { + var ( + tempdir string + err error + podmanTest *PodmanTestIntegration + ) + + BeforeEach(func() { + tempdir, err = CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest = PodmanTestCreate(tempdir) + podmanTest.RestoreAllArtifacts() + }) + + AfterEach(func() { + podmanTest.CleanupVolume() + f := CurrentGinkgoTestDescription() + timedResult := fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds()) + GinkgoWriter.Write([]byte(timedResult)) + }) + + It("podman ls volume", func() { + session := podmanTest.Podman([]string{"volume", "create", "myvol"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(2)) + }) + + It("podman ls volume with JSON format", func() { + session := podmanTest.Podman([]string{"volume", "create", "myvol"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls", "--format", "json"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(session.IsJSONOutputValid()).To(BeTrue()) + }) + + It("podman ls volume with Go template", func() { + session := podmanTest.Podman([]string{"volume", "create", "myvol"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls", "--format", "table {{.Name}} {{.Driver}} {{.Scope}}"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(2)) + }) + + It("podman ls volume with --filter flag", func() { + session := podmanTest.Podman([]string{"volume", "create", "--label", "foo=bar", "myvol"}) + volName := session.OutputToString() + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "create"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls", "--filter", "label=foo"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(2)) + Expect(session.OutputToStringArray()[1]).To(ContainSubstring(volName)) + }) +}) diff --git a/test/e2e/volume_prune_test.go b/test/e2e/volume_prune_test.go new file mode 100644 index 000000000..8c0a10e77 --- /dev/null +++ b/test/e2e/volume_prune_test.go @@ -0,0 +1,64 @@ +package integration + +import ( + "fmt" + "os" + + . "github.com/containers/libpod/test/utils" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Podman volume prune", func() { + var ( + tempdir string + err error + podmanTest *PodmanTestIntegration + ) + + BeforeEach(func() { + tempdir, err = CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest = PodmanTestCreate(tempdir) + podmanTest.RestoreAllArtifacts() + }) + + AfterEach(func() { + podmanTest.CleanupVolume() + f := CurrentGinkgoTestDescription() + timedResult := fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds()) + GinkgoWriter.Write([]byte(timedResult)) + }) + + It("podman prune volume", func() { + session := podmanTest.Podman([]string{"volume", "create"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "create"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"create", "-v", "myvol:/myvol", ALPINE, "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(4)) + + session = podmanTest.Podman([]string{"volume", "prune", "--force"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(2)) + + podmanTest.Cleanup() + }) +}) diff --git a/test/e2e/volume_rm_test.go b/test/e2e/volume_rm_test.go new file mode 100644 index 000000000..cebb09467 --- /dev/null +++ b/test/e2e/volume_rm_test.go @@ -0,0 +1,91 @@ +package integration + +import ( + "fmt" + "os" + + . "github.com/containers/libpod/test/utils" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var _ = Describe("Podman volume rm", func() { + var ( + tempdir string + err error + podmanTest *PodmanTestIntegration + ) + + BeforeEach(func() { + tempdir, err = CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest = PodmanTestCreate(tempdir) + podmanTest.RestoreAllArtifacts() + }) + + AfterEach(func() { + podmanTest.CleanupVolume() + f := CurrentGinkgoTestDescription() + timedResult := fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds()) + GinkgoWriter.Write([]byte(timedResult)) + }) + + It("podman rm volume", func() { + session := podmanTest.Podman([]string{"volume", "create", "myvol"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "rm", "myvol"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(0)) + }) + + It("podman rm with --force flag", func() { + session := podmanTest.Podman([]string{"create", "-v", "myvol:/myvol", ALPINE, "ls"}) + cid := session.OutputToString() + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "rm", "myvol"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Not(Equal(0))) + Expect(session.ErrorToString()).To(ContainSubstring(cid)) + + session = podmanTest.Podman([]string{"volume", "rm", "-f", "myvol"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(0)) + + podmanTest.Cleanup() + }) + + It("podman rm with --all flag", func() { + session := podmanTest.Podman([]string{"volume", "create", "myvol"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "create"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "rm", "-a"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(0)) + }) +}) diff --git a/vendor.conf b/vendor.conf index be306a181..ac8a38355 100644 --- a/vendor.conf +++ b/vendor.conf @@ -11,8 +11,8 @@ github.com/containerd/cgroups 58556f5ad8448d99a6f7bea69ea4bdb7747cfeb0 github.com/containerd/continuity master github.com/containernetworking/cni v0.7.0-alpha1 github.com/containernetworking/plugins 1562a1e60ed101aacc5e08ed9dbeba8e9f3d4ec1 -github.com/containers/image bd10b1b53b2976f215b3f2f848fb8e7cad779aeb -github.com/containers/storage 9d3838cd434d32042f9bdf1f9f4a2bf2d6ea8dc5 +github.com/containers/image 63a1cbdc5e6537056695cf0d627c0a33b334df53 +github.com/containers/storage db40f96d853dfced60c563e61fb66ba231ce7c8d github.com/containers/psgo 5dde6da0bc8831b35243a847625bcf18183bd1ee github.com/coreos/go-systemd v14 github.com/cri-o/ocicni 2d2983e40c242322a56c22a903785e7f83eb378c @@ -92,7 +92,7 @@ k8s.io/kube-openapi 275e2ce91dec4c05a4094a7b1daee5560b555ac9 https://github.com/ k8s.io/utils 258e2a2fa64568210fbd6267cf1d8fd87c3cb86e https://github.com/kubernetes/utils github.com/mrunalp/fileutils master github.com/varlink/go master -github.com/containers/buildah 2ac987a52ff8412fb8f2908a191009751a6a1c62 +github.com/containers/buildah 9c65e5699cfa486531b3f123d9ce74873f0e18aa github.com/Nvveen/Gotty master github.com/fsouza/go-dockerclient master github.com/openshift/imagebuilder master diff --git a/vendor/github.com/containers/buildah/README.md b/vendor/github.com/containers/buildah/README.md index 2b539bba8..12eafdf88 100644 --- a/vendor/github.com/containers/buildah/README.md +++ b/vendor/github.com/containers/buildah/README.md @@ -105,6 +105,7 @@ $ sudo ./lighttpd.sh | [buildah-copy(1)](/docs/buildah-copy.md) | Copies the contents of a file, URL, or directory into a container's working directory. | | [buildah-from(1)](/docs/buildah-from.md) | Creates a new working container, either from scratch or using a specified image as a starting point. | | [buildah-images(1)](/docs/buildah-images.md) | List images in local storage. | +| [buildah-info(1)](/docs/buildah-info.md) | Display Buildah system information. | | [buildah-inspect(1)](/docs/buildah-inspect.md) | Inspects the configuration of a container or image. | | [buildah-mount(1)](/docs/buildah-mount.md) | Mount the working container's root filesystem. | | [buildah-pull(1)](/docs/buildah-pull.md) | Pull an image from the specified location. | diff --git a/vendor/github.com/containers/buildah/buildah.go b/vendor/github.com/containers/buildah/buildah.go index 1a642ed3d..91ce2b09d 100644 --- a/vendor/github.com/containers/buildah/buildah.go +++ b/vendor/github.com/containers/buildah/buildah.go @@ -25,7 +25,7 @@ const ( Package = "buildah" // Version for the Package. Bump version in contrib/rpm/buildah.spec // too. - Version = "1.5-dev" + Version = "1.6-dev" // The value we use to identify what type of information, currently a // serialized Builder structure, we are using as per-container state. // This should only be changed when we make incompatible changes to diff --git a/vendor/github.com/containers/buildah/common.go b/vendor/github.com/containers/buildah/common.go index be59215df..dfdc33a22 100644 --- a/vendor/github.com/containers/buildah/common.go +++ b/vendor/github.com/containers/buildah/common.go @@ -38,7 +38,6 @@ func getCopyOptions(reportWriter io.Writer, sourceReference types.ImageReference if err != nil { logrus.Debugf("error determining if registry for %q is insecure: %v", transports.ImageName(sourceReference), err) } else if sourceInsecure { - sourceCtx.DockerInsecureSkipTLSVerify = true sourceCtx.OCIInsecureSkipTLSVerify = true } @@ -56,7 +55,6 @@ func getCopyOptions(reportWriter io.Writer, sourceReference types.ImageReference if err != nil { logrus.Debugf("error determining if registry for %q is insecure: %v", transports.ImageName(destinationReference), err) } else if destinationInsecure { - destinationCtx.DockerInsecureSkipTLSVerify = true destinationCtx.OCIInsecureSkipTLSVerify = true } diff --git a/vendor/github.com/containers/buildah/config.go b/vendor/github.com/containers/buildah/config.go index 89224b674..3609694f6 100644 --- a/vendor/github.com/containers/buildah/config.go +++ b/vendor/github.com/containers/buildah/config.go @@ -543,3 +543,37 @@ func (b *Builder) SetStopSignal(stopSignal string) { b.OCIv1.Config.StopSignal = stopSignal b.Docker.Config.StopSignal = stopSignal } + +// Healthcheck returns information that recommends how a container engine +// should check if a running container is "healthy". +func (b *Builder) Healthcheck() *docker.HealthConfig { + if b.Docker.Config.Healthcheck == nil { + return nil + } + return &docker.HealthConfig{ + Test: copyStringSlice(b.Docker.Config.Healthcheck.Test), + Interval: b.Docker.Config.Healthcheck.Interval, + Timeout: b.Docker.Config.Healthcheck.Timeout, + StartPeriod: b.Docker.Config.Healthcheck.StartPeriod, + Retries: b.Docker.Config.Healthcheck.Retries, + } +} + +// SetHealthcheck sets recommended commands to run in order to verify that a +// running container based on this image is "healthy", along with information +// specifying how often that test should be run, and how many times the test +// should fail before the container should be considered unhealthy. +// Note: this setting is not present in the OCIv1 image format, so it is +// discarded when writing images using OCIv1 formats. +func (b *Builder) SetHealthcheck(config *docker.HealthConfig) { + b.Docker.Config.Healthcheck = nil + if config != nil { + b.Docker.Config.Healthcheck = &docker.HealthConfig{ + Test: copyStringSlice(config.Test), + Interval: config.Interval, + Timeout: config.Timeout, + StartPeriod: config.StartPeriod, + Retries: config.Retries, + } + } +} diff --git a/vendor/github.com/containers/buildah/docker/types.go b/vendor/github.com/containers/buildah/docker/types.go index 759fc1246..6847d36fd 100644 --- a/vendor/github.com/containers/buildah/docker/types.go +++ b/vendor/github.com/containers/buildah/docker/types.go @@ -60,8 +60,9 @@ type HealthConfig struct { Test []string `json:",omitempty"` // Zero means to inherit. Durations are expressed as integer nanoseconds. - Interval time.Duration `json:",omitempty"` // Interval is the time to wait between checks. - Timeout time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung. + Interval time.Duration `json:",omitempty"` // Interval is the time to wait between checks. + Timeout time.Duration `json:",omitempty"` // Timeout is the time to wait before considering the check to have hung. + StartPeriod time.Duration `json:",omitempty"` // Time to wait after the container starts before running the first check. // Retries is the number of consecutive failures needed to consider a container as unhealthy. // Zero means inherit. diff --git a/vendor/github.com/containers/buildah/imagebuildah/build.go b/vendor/github.com/containers/buildah/imagebuildah/build.go index 701241683..e6ee6a071 100644 --- a/vendor/github.com/containers/buildah/imagebuildah/build.go +++ b/vendor/github.com/containers/buildah/imagebuildah/build.go @@ -15,6 +15,7 @@ import ( "time" "github.com/containers/buildah" + buildahdocker "github.com/containers/buildah/docker" "github.com/containers/buildah/util" cp "github.com/containers/image/copy" "github.com/containers/image/docker/reference" @@ -225,6 +226,18 @@ type Executor struct { copyFrom string // Used to keep track of the --from flag from COPY and ADD } +// builtinAllowedBuildArgs is list of built-in allowed build args +var builtinAllowedBuildArgs = map[string]bool{ + "HTTP_PROXY": true, + "http_proxy": true, + "HTTPS_PROXY": true, + "https_proxy": true, + "FTP_PROXY": true, + "ftp_proxy": true, + "NO_PROXY": true, + "no_proxy": true, +} + // withName creates a new child executor that will be used whenever a COPY statement uses --from=NAME. func (b *Executor) withName(name string, index int) *Executor { if b.named == nil { @@ -793,12 +806,28 @@ func (b *Executor) Execute(ctx context.Context, stage imagebuilder.Stage) error commitName := b.output b.containerIDs = nil + var leftoverArgs []string + for arg := range b.builder.Args { + if !builtinAllowedBuildArgs[arg] { + leftoverArgs = append(leftoverArgs, arg) + } + } for i, node := range node.Children { step := ib.Step() if err := step.Resolve(node); err != nil { return errors.Wrapf(err, "error resolving step %+v", *node) } logrus.Debugf("Parsed Step: %+v", *step) + if step.Command == "arg" { + for index, arg := range leftoverArgs { + for _, Arg := range step.Args { + list := strings.SplitN(Arg, "=", 2) + if arg == list[0] { + leftoverArgs = append(leftoverArgs[:index], leftoverArgs[index+1:]...) + } + } + } + } if !b.quiet { b.log("%s", step.Original) } @@ -895,6 +924,9 @@ func (b *Executor) Execute(ctx context.Context, stage imagebuilder.Stage) error } } } + if len(leftoverArgs) > 0 { + fmt.Fprintf(b.out, "[Warning] One or more build-args %v were not consumed\n", leftoverArgs) + } return nil } @@ -1139,6 +1171,17 @@ func (b *Executor) Commit(ctx context.Context, ib *imagebuilder.Builder, created b.builder.SetEntrypoint(config.Entrypoint) b.builder.SetShell(config.Shell) b.builder.SetStopSignal(config.StopSignal) + if config.Healthcheck != nil { + b.builder.SetHealthcheck(&buildahdocker.HealthConfig{ + Test: append([]string{}, config.Healthcheck.Test...), + Interval: config.Healthcheck.Interval, + Timeout: config.Healthcheck.Timeout, + StartPeriod: config.Healthcheck.StartPeriod, + Retries: config.Healthcheck.Retries, + }) + } else { + b.builder.SetHealthcheck(nil) + } b.builder.ClearLabels() for k, v := range config.Labels { b.builder.SetLabel(k, v) diff --git a/vendor/github.com/containers/buildah/info.go b/vendor/github.com/containers/buildah/info.go new file mode 100644 index 000000000..8cd5e4438 --- /dev/null +++ b/vendor/github.com/containers/buildah/info.go @@ -0,0 +1,207 @@ +package buildah + +import ( + "bufio" + "bytes" + "fmt" + "io/ioutil" + "os" + "runtime" + "strconv" + "strings" + "time" + + "github.com/containers/libpod/pkg/rootless" + "github.com/containers/storage" + "github.com/containers/storage/pkg/system" + "github.com/sirupsen/logrus" +) + +// InfoData holds the info type, i.e store, host etc and the data for each type +type InfoData struct { + Type string + Data map[string]interface{} +} + +// Info returns the store and host information +func Info(store storage.Store) ([]InfoData, error) { + info := []InfoData{} + // get host information + hostInfo, err := hostInfo() + if err != nil { + logrus.Error(err, "error getting host info") + } + info = append(info, InfoData{Type: "host", Data: hostInfo}) + + // get store information + storeInfo, err := storeInfo(store) + if err != nil { + logrus.Error(err, "error getting store info") + } + info = append(info, InfoData{Type: "store", Data: storeInfo}) + return info, nil +} + +func hostInfo() (map[string]interface{}, error) { + info := map[string]interface{}{} + info["os"] = runtime.GOOS + info["arch"] = runtime.GOARCH + info["cpus"] = runtime.NumCPU() + info["rootless"] = rootless.IsRootless() + mi, err := system.ReadMemInfo() + if err != nil { + logrus.Error(err, "err reading memory info") + info["MemTotal"] = "" + info["MenFree"] = "" + info["SwapTotal"] = "" + info["SwapFree"] = "" + } else { + info["MemTotal"] = mi.MemTotal + info["MenFree"] = mi.MemFree + info["SwapTotal"] = mi.SwapTotal + info["SwapFree"] = mi.SwapFree + } + hostDistributionInfo := getHostDistributionInfo() + info["Distribution"] = map[string]interface{}{ + "distribution": hostDistributionInfo["Distribution"], + "version": hostDistributionInfo["Version"], + } + + kv, err := readKernelVersion() + if err != nil { + logrus.Error(err, "error reading kernel version") + } + info["kernel"] = kv + + up, err := readUptime() + if err != nil { + logrus.Error(err, "error reading up time") + } + // Convert uptime in seconds to a human-readable format + upSeconds := up + "s" + upDuration, err := time.ParseDuration(upSeconds) + if err != nil { + logrus.Error(err, "error parsing system uptime") + } + + hoursFound := false + var timeBuffer bytes.Buffer + var hoursBuffer bytes.Buffer + for _, elem := range upDuration.String() { + timeBuffer.WriteRune(elem) + if elem == 'h' || elem == 'm' { + timeBuffer.WriteRune(' ') + if elem == 'h' { + hoursFound = true + } + } + if !hoursFound { + hoursBuffer.WriteRune(elem) + } + } + + info["uptime"] = timeBuffer.String() + if hoursFound { + hours, err := strconv.ParseFloat(hoursBuffer.String(), 64) + if err == nil { + days := hours / 24 + info["uptime"] = fmt.Sprintf("%s (Approximately %.2f days)", info["uptime"], days) + } + } + + host, err := os.Hostname() + if err != nil { + logrus.Error(err, "error getting hostname") + } + info["hostname"] = host + + return info, nil + +} + +// top-level "store" info +func storeInfo(store storage.Store) (map[string]interface{}, error) { + // lets say storage driver in use, number of images, number of containers + info := map[string]interface{}{} + info["GraphRoot"] = store.GraphRoot() + info["RunRoot"] = store.RunRoot() + info["GraphDriverName"] = store.GraphDriverName() + info["GraphOptions"] = store.GraphOptions() + statusPairs, err := store.Status() + if err != nil { + return nil, err + } + status := map[string]string{} + for _, pair := range statusPairs { + status[pair[0]] = pair[1] + } + info["GraphStatus"] = status + images, err := store.Images() + if err != nil { + logrus.Error(err, "error getting number of images") + } + info["ImageStore"] = map[string]interface{}{ + "number": len(images), + } + + containers, err := store.Containers() + if err != nil { + logrus.Error(err, "error getting number of containers") + } + info["ContainerStore"] = map[string]interface{}{ + "number": len(containers), + } + + return info, nil +} + +func readKernelVersion() (string, error) { + buf, err := ioutil.ReadFile("/proc/version") + if err != nil { + return "", err + } + f := bytes.Fields(buf) + if len(f) < 2 { + return string(bytes.TrimSpace(buf)), nil + } + return string(f[2]), nil +} + +func readUptime() (string, error) { + buf, err := ioutil.ReadFile("/proc/uptime") + if err != nil { + return "", err + } + f := bytes.Fields(buf) + if len(f) < 1 { + return "", fmt.Errorf("invalid uptime") + } + return string(f[0]), nil +} + +// getHostDistributionInfo returns a map containing the host's distribution and version +func getHostDistributionInfo() map[string]string { + dist := make(map[string]string) + + // Populate values in case we cannot find the values + // or the file + dist["Distribution"] = "unknown" + dist["Version"] = "unknown" + + f, err := os.Open("/etc/os-release") + if err != nil { + return dist + } + defer f.Close() + + l := bufio.NewScanner(f) + for l.Scan() { + if strings.HasPrefix(l.Text(), "ID=") { + dist["Distribution"] = strings.TrimPrefix(l.Text(), "ID=") + } + if strings.HasPrefix(l.Text(), "VERSION_ID=") { + dist["Version"] = strings.Trim(strings.TrimPrefix(l.Text(), "VERSION_ID="), "\"") + } + } + return dist +} diff --git a/vendor/github.com/containers/buildah/pkg/parse/parse.go b/vendor/github.com/containers/buildah/pkg/parse/parse.go index b87eb95c7..41fdea8b1 100644 --- a/vendor/github.com/containers/buildah/pkg/parse/parse.go +++ b/vendor/github.com/containers/buildah/pkg/parse/parse.go @@ -282,7 +282,7 @@ func SystemContextFromOptions(c *cli.Context) (*types.SystemContext, error) { DockerCertPath: c.String("cert-dir"), } if c.IsSet("tls-verify") { - ctx.DockerInsecureSkipTLSVerify = !c.BoolT("tls-verify") + ctx.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!c.BoolT("tls-verify")) ctx.OCIInsecureSkipTLSVerify = !c.BoolT("tls-verify") ctx.DockerDaemonInsecureSkipTLSVerify = !c.BoolT("tls-verify") } diff --git a/vendor/github.com/containers/buildah/util.go b/vendor/github.com/containers/buildah/util.go index 09aa7e1eb..66a4e535a 100644 --- a/vendor/github.com/containers/buildah/util.go +++ b/vendor/github.com/containers/buildah/util.go @@ -175,11 +175,11 @@ func (b *Builder) tarPath() func(path string) (io.ReadCloser, error) { // isRegistryInsecure checks if the named registry is marked as not secure func isRegistryInsecure(registry string, sc *types.SystemContext) (bool, error) { - registries, err := sysregistriesv2.GetRegistries(sc) + reginfo, err := sysregistriesv2.FindRegistry(sc, registry) if err != nil { return false, errors.Wrapf(err, "unable to parse the registries configuration (%s)", sysregistries.RegistriesConfPath(sc)) } - if reginfo := sysregistriesv2.FindRegistry(registry, registries); reginfo != nil { + if reginfo != nil { if reginfo.Insecure { logrus.Debugf("registry %q is marked insecure in registries configuration %q", registry, sysregistries.RegistriesConfPath(sc)) } else { @@ -193,11 +193,11 @@ func isRegistryInsecure(registry string, sc *types.SystemContext) (bool, error) // isRegistryBlocked checks if the named registry is marked as blocked func isRegistryBlocked(registry string, sc *types.SystemContext) (bool, error) { - registries, err := sysregistriesv2.GetRegistries(sc) + reginfo, err := sysregistriesv2.FindRegistry(sc, registry) if err != nil { return false, errors.Wrapf(err, "unable to parse the registries configuration (%s)", sysregistries.RegistriesConfPath(sc)) } - if reginfo := sysregistriesv2.FindRegistry(registry, registries); reginfo != nil { + if reginfo != nil { if reginfo.Blocked { logrus.Debugf("registry %q is marked as blocked in registries configuration %q", registry, sysregistries.RegistriesConfPath(sc)) } else { diff --git a/vendor/github.com/containers/buildah/util/util.go b/vendor/github.com/containers/buildah/util/util.go index b2451b78b..427c8db28 100644 --- a/vendor/github.com/containers/buildah/util/util.go +++ b/vendor/github.com/containers/buildah/util/util.go @@ -122,12 +122,11 @@ func ResolveName(name string, firstRegistry string, sc *types.SystemContext, sto // Figure out the list of registries. var registries []string - allRegistries, err := sysregistriesv2.GetRegistries(sc) + searchRegistries, err := sysregistriesv2.FindUnqualifiedSearchRegistries(sc) if err != nil { logrus.Debugf("unable to read configured registries to complete %q: %v", name, err) - registries = []string{} } - for _, registry := range sysregistriesv2.FindUnqualifiedSearchRegistries(allRegistries) { + for _, registry := range searchRegistries { if !registry.Blocked { registries = append(registries, registry.URL) } diff --git a/vendor/github.com/containers/buildah/vendor.conf b/vendor/github.com/containers/buildah/vendor.conf index 185cde449..acba0011e 100644 --- a/vendor/github.com/containers/buildah/vendor.conf +++ b/vendor/github.com/containers/buildah/vendor.conf @@ -3,7 +3,7 @@ github.com/blang/semver master github.com/BurntSushi/toml master github.com/containerd/continuity master github.com/containernetworking/cni v0.7.0-alpha1 -github.com/containers/image de7be82ee3c7fb676bf6cfdc9090be7cc28f404c +github.com/containers/image 63a1cbdc5e6537056695cf0d627c0a33b334df53 github.com/containers/libpod fe4f09493f41f675d24c969d1b60d1a6a45ddb9e github.com/containers/storage 3161726d1db0d0d4e86a9667dd476f09b997f497 github.com/docker/distribution 5f6282db7d65e6d72ad7c2cc66310724a57be716 diff --git a/vendor/github.com/containers/image/docker/docker_client.go b/vendor/github.com/containers/image/docker/docker_client.go index 6d2c5b670..ea1a8ec06 100644 --- a/vendor/github.com/containers/image/docker/docker_client.go +++ b/vendor/github.com/containers/image/docker/docker_client.go @@ -17,6 +17,7 @@ import ( "github.com/containers/image/docker/reference" "github.com/containers/image/pkg/docker/config" + "github.com/containers/image/pkg/sysregistriesv2" "github.com/containers/image/pkg/tlsclientconfig" "github.com/containers/image/types" "github.com/docker/distribution/registry/client" @@ -78,11 +79,13 @@ type bearerToken struct { // dockerClient is configuration for dealing with a single Docker registry. type dockerClient struct { // The following members are set by newDockerClient and do not change afterwards. - sys *types.SystemContext - registry string + sys *types.SystemContext + registry string + client *http.Client + insecureSkipTLSVerify bool + // The following members are not set by newDockerClient and must be set by callers if needed. username string password string - client *http.Client signatureBase signatureStorageBase scope authScope // The following members are detected registry properties: @@ -194,13 +197,26 @@ func newDockerClientFromRef(sys *types.SystemContext, ref dockerReference, write if err != nil { return nil, err } - remoteName := reference.Path(ref.ref) - return newDockerClientWithDetails(sys, registry, username, password, actions, sigBase, remoteName) + client, err := newDockerClient(sys, registry, ref.ref.Name()) + if err != nil { + return nil, err + } + client.username = username + client.password = password + client.signatureBase = sigBase + client.scope.actions = actions + client.scope.remoteName = reference.Path(ref.ref) + return client, nil } -// newDockerClientWithDetails returns a new dockerClient instance for the given parameters -func newDockerClientWithDetails(sys *types.SystemContext, registry, username, password, actions string, sigBase signatureStorageBase, remoteName string) (*dockerClient, error) { +// newDockerClient returns a new dockerClient instance for the given registry +// and reference. The reference is used to query the registry configuration +// and can either be a registry (e.g, "registry.com[:5000]"), a repository +// (e.g., "registry.com[:5000][/some/namespace]/repo"). +// Please note that newDockerClient does not set all members of dockerClient +// (e.g., username and password); those must be set by callers if necessary. +func newDockerClient(sys *types.SystemContext, registry, reference string) (*dockerClient, error) { hostName := registry if registry == dockerHostname { registry = dockerRegistry @@ -221,33 +237,43 @@ func newDockerClientWithDetails(sys *types.SystemContext, registry, username, pa return nil, err } - if sys != nil && sys.DockerInsecureSkipTLSVerify { - tr.TLSClientConfig.InsecureSkipVerify = true + // Check if TLS verification shall be skipped (default=false) which can + // either be specified in the sysregistriesv2 configuration or via the + // SystemContext, whereas the SystemContext is prioritized. + skipVerify := false + if sys != nil && sys.DockerInsecureSkipTLSVerify != types.OptionalBoolUndefined { + // Only use the SystemContext if the actual value is defined. + skipVerify = sys.DockerInsecureSkipTLSVerify == types.OptionalBoolTrue + } else { + reg, err := sysregistriesv2.FindRegistry(sys, reference) + if err != nil { + return nil, errors.Wrapf(err, "error loading registries") + } + if reg != nil { + skipVerify = reg.Insecure + } } + tr.TLSClientConfig.InsecureSkipVerify = skipVerify return &dockerClient{ - sys: sys, - registry: registry, - username: username, - password: password, - client: &http.Client{Transport: tr}, - signatureBase: sigBase, - scope: authScope{ - actions: actions, - remoteName: remoteName, - }, + sys: sys, + registry: registry, + client: &http.Client{Transport: tr}, + insecureSkipTLSVerify: skipVerify, }, nil } // CheckAuth validates the credentials by attempting to log into the registry // returns an error if an error occcured while making the http request or the status code received was 401 func CheckAuth(ctx context.Context, sys *types.SystemContext, username, password, registry string) error { - newLoginClient, err := newDockerClientWithDetails(sys, registry, username, password, "", nil, "") + client, err := newDockerClient(sys, registry, registry) if err != nil { return errors.Wrapf(err, "error creating new docker client") } + client.username = username + client.password = password - resp, err := newLoginClient.makeRequest(ctx, "GET", "/v2/", nil, nil, v2Auth) + resp, err := client.makeRequest(ctx, "GET", "/v2/", nil, nil, v2Auth) if err != nil { return err } @@ -299,16 +325,21 @@ func SearchRegistry(ctx context.Context, sys *types.SystemContext, registry, ima return nil, errors.Wrapf(err, "error getting username and password") } - // The /v2/_catalog endpoint has been disabled for docker.io therefore the call made to that endpoint will fail - // So using the v1 hostname for docker.io for simplicity of implementation and the fact that it returns search results + // The /v2/_catalog endpoint has been disabled for docker.io therefore + // the call made to that endpoint will fail. So using the v1 hostname + // for docker.io for simplicity of implementation and the fact that it + // returns search results. + hostname := registry if registry == dockerHostname { - registry = dockerV1Hostname + hostname = dockerV1Hostname } - client, err := newDockerClientWithDetails(sys, registry, username, password, "", nil, "") + client, err := newDockerClient(sys, hostname, registry) if err != nil { return nil, errors.Wrapf(err, "error creating new docker client") } + client.username = username + client.password = password // Only try the v1 search endpoint if the search query is not empty. If it is // empty skip to the v2 endpoint. @@ -530,7 +561,7 @@ func (c *dockerClient) detectProperties(ctx context.Context) error { return nil } err := ping("https") - if err != nil && c.sys != nil && c.sys.DockerInsecureSkipTLSVerify { + if err != nil && c.insecureSkipTLSVerify { err = ping("http") } if err != nil { @@ -554,7 +585,7 @@ func (c *dockerClient) detectProperties(ctx context.Context) error { return true } isV1 := pingV1("https") - if !isV1 && c.sys != nil && c.sys.DockerInsecureSkipTLSVerify { + if !isV1 && c.insecureSkipTLSVerify { isV1 = pingV1("http") } if isV1 { diff --git a/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go b/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go index 067f512ad..afc7312d1 100644 --- a/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go +++ b/vendor/github.com/containers/image/pkg/sysregistriesv2/system_registries_v2.go @@ -3,7 +3,7 @@ package sysregistriesv2 import ( "fmt" "io/ioutil" - "net/url" + "os" "path/filepath" "strings" "sync" @@ -82,8 +82,8 @@ func (e *InvalidRegistries) Error() string { } // parseURL parses the input string, performs some sanity checks and returns -// the sanitized input string. An error is returned in case parsing fails or -// or if URI scheme or user is set. +// the sanitized input string. An error is returned if the input string is +// empty or if contains an "http{s,}://" prefix. func parseURL(input string) (string, error) { trimmed := strings.TrimRight(input, "/") @@ -91,49 +91,11 @@ func parseURL(input string) (string, error) { return "", &InvalidRegistries{s: "invalid URL: cannot be empty"} } - // Ultimately, we expect input of the form example.com[/namespace/…], a prefix - // of a fully-expended reference (containers/image/docker/Reference.String()). - // c/image/docker/Reference does not currently provide such a parser. - // So, we use url.Parse("http://"+trimmed) below to ~verify the format, possibly - // letting some invalid input in, trading that off for a simpler parser. - // - // url.Parse("http://"+trimmed) is, sadly, too permissive, notably for - // trimmed == "http://example.com/…", url.Parse("http://http://example.com/…") - // is accepted and parsed as - // {Scheme: "http", Host: "http:", Path: "//example.com/…"}. - // - // So, first we do an explicit check for an unwanted scheme prefix: - - // This will parse trimmed=="http://example.com/…" with Scheme: "http". Perhaps surprisingly, - // it also succeeds for the input we want to accept, in different ways: - // "example.com" -> {Scheme:"", Opaque:"", Path:"example.com"} - // "example.com/repo" -> {Scheme:"", Opaque:"", Path:"example.com/repo"} - // "example.com:5000" -> {Scheme:"example.com", Opaque:"5000"} - // "example.com:5000/repo" -> {Scheme:"example.com", Opaque:"5000/repo"} - uri, err := url.Parse(trimmed) - if err != nil { - return "", &InvalidRegistries{s: fmt.Sprintf("invalid URL '%s': %v", input, err)} - } - - // Check if a URI Scheme is set. - // Note that URLs that do not start with a slash after the scheme are - // interpreted as `scheme:opaque[?query][#fragment]`; see above for examples. - if uri.Scheme != "" && uri.Opaque == "" { + if strings.HasPrefix(trimmed, "http://") || strings.HasPrefix(trimmed, "https://") { msg := fmt.Sprintf("invalid URL '%s': URI schemes are not supported", input) return "", &InvalidRegistries{s: msg} } - uri, err = url.Parse("http://" + trimmed) - if err != nil { - msg := fmt.Sprintf("invalid URL '%s': sanitized URL did not parse: %v", input, err) - return "", &InvalidRegistries{s: msg} - } - - if uri.User != nil { - msg := fmt.Sprintf("invalid URL '%s': user/password are not supported", trimmed) - return "", &InvalidRegistries{s: msg} - } - return trimmed, nil } @@ -279,7 +241,18 @@ var configMutex = sync.Mutex{} // are synchronized via configMutex. var configCache = make(map[string][]Registry) +// InvalidateCache invalidates the registry cache. This function is meant to be +// used for long-running processes that need to reload potential changes made to +// the cached registry config files. +func InvalidateCache() { + configMutex.Lock() + defer configMutex.Unlock() + configCache = make(map[string][]Registry) +} + // GetRegistries loads and returns the registries specified in the config. +// Note the parsed content of registry config files is cached. For reloading, +// use `InvalidateCache` and re-call `GetRegistries`. func GetRegistries(ctx *types.SystemContext) ([]Registry, error) { configPath := getConfigPath(ctx) @@ -293,6 +266,13 @@ func GetRegistries(ctx *types.SystemContext) ([]Registry, error) { // load the config config, err := loadRegistryConf(configPath) if err != nil { + // Return an empty []Registry if we use the default config, + // which implies that the config path of the SystemContext + // isn't set. Note: if ctx.SystemRegistriesConfPath points to + // the default config, we will still return an error. + if os.IsNotExist(err) && (ctx == nil || ctx.SystemRegistriesConfPath == "") { + return []Registry{}, nil + } return nil, err } @@ -323,23 +303,33 @@ func GetRegistries(ctx *types.SystemContext) ([]Registry, error) { // FindUnqualifiedSearchRegistries returns all registries that are configured // for unqualified image search (i.e., with Registry.Search == true). -func FindUnqualifiedSearchRegistries(registries []Registry) []Registry { +func FindUnqualifiedSearchRegistries(ctx *types.SystemContext) ([]Registry, error) { + registries, err := GetRegistries(ctx) + if err != nil { + return nil, err + } + unqualified := []Registry{} for _, reg := range registries { if reg.Search { unqualified = append(unqualified, reg) } } - return unqualified + return unqualified, nil } // FindRegistry returns the Registry with the longest prefix for ref. If no // Registry prefixes the image, nil is returned. -func FindRegistry(ref string, registries []Registry) *Registry { +func FindRegistry(ctx *types.SystemContext, ref string) (*Registry, error) { + registries, err := GetRegistries(ctx) + if err != nil { + return nil, err + } + reg := Registry{} prefixLen := 0 for _, r := range registries { - if strings.HasPrefix(ref, r.Prefix) { + if strings.HasPrefix(ref, r.Prefix+"/") || ref == r.Prefix { length := len(r.Prefix) if length > prefixLen { reg = r @@ -348,9 +338,9 @@ func FindRegistry(ref string, registries []Registry) *Registry { } } if prefixLen != 0 { - return ® + return ®, nil } - return nil + return nil, nil } // Reads the global registry file from the filesystem. Returns a byte array. diff --git a/vendor/github.com/containers/image/types/types.go b/vendor/github.com/containers/image/types/types.go index 5d05b711a..a552e4597 100644 --- a/vendor/github.com/containers/image/types/types.go +++ b/vendor/github.com/containers/image/types/types.go @@ -324,6 +324,30 @@ type DockerAuthConfig struct { Password string } +// OptionalBool is a boolean with an additional undefined value, which is meant +// to be used in the context of user input to distinguish between a +// user-specified value and a default value. +type OptionalBool byte + +const ( + // OptionalBoolUndefined indicates that the OptionalBoolean hasn't been written. + OptionalBoolUndefined OptionalBool = iota + // OptionalBoolTrue represents the boolean true. + OptionalBoolTrue + // OptionalBoolFalse represents the boolean false. + OptionalBoolFalse +) + +// NewOptionalBool converts the input bool into either OptionalBoolTrue or +// OptionalBoolFalse. The function is meant to avoid boilerplate code of users. +func NewOptionalBool(b bool) OptionalBool { + o := OptionalBoolFalse + if b == true { + o = OptionalBoolTrue + } + return o +} + // SystemContext allows parameterizing access to implicitly-accessed resources, // like configuration files in /etc and users' login state in their home directory. // Various components can share the same field only if their semantics is exactly @@ -376,7 +400,7 @@ type SystemContext struct { // Ignored if DockerCertPath is non-empty. DockerPerHostCertDirPath string // Allow contacting docker registries over HTTP, or HTTPS with failed TLS verification. Note that this does not affect other TLS connections. - DockerInsecureSkipTLSVerify bool + DockerInsecureSkipTLSVerify OptionalBool // if nil, the library tries to parse ~/.docker/config.json to retrieve credentials DockerAuthConfig *DockerAuthConfig // if not "", an User-Agent header is added to each request when contacting a registry. diff --git a/vendor/github.com/containers/image/vendor.conf b/vendor/github.com/containers/image/vendor.conf index 246c0096a..de6dcbecf 100644 --- a/vendor/github.com/containers/image/vendor.conf +++ b/vendor/github.com/containers/image/vendor.conf @@ -34,7 +34,7 @@ github.com/xeipuuv/gojsonschema master github.com/xeipuuv/gojsonreference master github.com/xeipuuv/gojsonpointer master github.com/tchap/go-patricia v2.2.6 -github.com/opencontainers/selinux ba1aefe8057f1d0cfb8e88d0ec1dc85925ef987d +github.com/opencontainers/selinux 077c8b6d1c18456fb7c792bc0de52295a0d1900e github.com/BurntSushi/toml b26d9c308763d68093482582cea63d69be07a0f0 github.com/ostreedev/ostree-go aeb02c6b6aa2889db3ef62f7855650755befd460 github.com/gogo/protobuf fcdc5011193ff531a548e9b0301828d5a5b97fd8 diff --git a/vendor/github.com/containers/storage/drivers/copy/copy.go b/vendor/github.com/containers/storage/drivers/copy/copy.go new file mode 100644 index 000000000..2617824c5 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/copy/copy.go @@ -0,0 +1,277 @@ +// +build linux + +package copy + +/* +#include <linux/fs.h> + +#ifndef FICLONE +#define FICLONE _IOW(0x94, 9, int) +#endif +*/ +import "C" +import ( + "container/list" + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/containers/storage/pkg/pools" + "github.com/containers/storage/pkg/system" + rsystem "github.com/opencontainers/runc/libcontainer/system" + "golang.org/x/sys/unix" +) + +// Mode indicates whether to use hardlink or copy content +type Mode int + +const ( + // Content creates a new file, and copies the content of the file + Content Mode = iota + // Hardlink creates a new hardlink to the existing file + Hardlink +) + +func copyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { + srcFile, err := os.Open(srcPath) + if err != nil { + return err + } + defer srcFile.Close() + + // If the destination file already exists, we shouldn't blow it away + dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, fileinfo.Mode()) + if err != nil { + return err + } + defer dstFile.Close() + + if *copyWithFileClone { + _, _, err = unix.Syscall(unix.SYS_IOCTL, dstFile.Fd(), C.FICLONE, srcFile.Fd()) + if err == nil { + return nil + } + + *copyWithFileClone = false + if err == unix.EXDEV { + *copyWithFileRange = false + } + } + if *copyWithFileRange { + err = doCopyWithFileRange(srcFile, dstFile, fileinfo) + // Trying the file_clone may not have caught the exdev case + // as the ioctl may not have been available (therefore EINVAL) + if err == unix.EXDEV || err == unix.ENOSYS { + *copyWithFileRange = false + } else { + return err + } + } + return legacyCopy(srcFile, dstFile) +} + +func doCopyWithFileRange(srcFile, dstFile *os.File, fileinfo os.FileInfo) error { + amountLeftToCopy := fileinfo.Size() + + for amountLeftToCopy > 0 { + n, err := unix.CopyFileRange(int(srcFile.Fd()), nil, int(dstFile.Fd()), nil, int(amountLeftToCopy), 0) + if err != nil { + return err + } + + amountLeftToCopy = amountLeftToCopy - int64(n) + } + + return nil +} + +func legacyCopy(srcFile io.Reader, dstFile io.Writer) error { + _, err := pools.Copy(dstFile, srcFile) + + return err +} + +func copyXattr(srcPath, dstPath, attr string) error { + data, err := system.Lgetxattr(srcPath, attr) + if err != nil { + return err + } + if data != nil { + if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil { + return err + } + } + return nil +} + +type fileID struct { + dev uint64 + ino uint64 +} + +type dirMtimeInfo struct { + dstPath *string + stat *syscall.Stat_t +} + +// DirCopy copies or hardlinks the contents of one directory to another, +// properly handling xattrs, and soft links +// +// Copying xattrs can be opted out of by passing false for copyXattrs. +func DirCopy(srcDir, dstDir string, copyMode Mode, copyXattrs bool) error { + copyWithFileRange := true + copyWithFileClone := true + + // This is a map of source file inodes to dst file paths + copiedFiles := make(map[fileID]string) + + dirsToSetMtimes := list.New() + err := filepath.Walk(srcDir, func(srcPath string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + relPath, err := filepath.Rel(srcDir, srcPath) + if err != nil { + return err + } + + dstPath := filepath.Join(dstDir, relPath) + if err != nil { + return err + } + + stat, ok := f.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("Unable to get raw syscall.Stat_t data for %s", srcPath) + } + + isHardlink := false + + switch f.Mode() & os.ModeType { + case 0: // Regular file + id := fileID{dev: stat.Dev, ino: stat.Ino} + if copyMode == Hardlink { + isHardlink = true + if err2 := os.Link(srcPath, dstPath); err2 != nil { + return err2 + } + } else if hardLinkDstPath, ok := copiedFiles[id]; ok { + if err2 := os.Link(hardLinkDstPath, dstPath); err2 != nil { + return err2 + } + } else { + if err2 := copyRegular(srcPath, dstPath, f, ©WithFileRange, ©WithFileClone); err2 != nil { + return err2 + } + copiedFiles[id] = dstPath + } + + case os.ModeDir: + if err := os.Mkdir(dstPath, f.Mode()); err != nil && !os.IsExist(err) { + return err + } + + case os.ModeSymlink: + link, err := os.Readlink(srcPath) + if err != nil { + return err + } + + if err := os.Symlink(link, dstPath); err != nil { + return err + } + + case os.ModeNamedPipe: + fallthrough + case os.ModeSocket: + if err := unix.Mkfifo(dstPath, stat.Mode); err != nil { + return err + } + + case os.ModeDevice: + if rsystem.RunningInUserNS() { + // cannot create a device if running in user namespace + return nil + } + if err := unix.Mknod(dstPath, stat.Mode, int(stat.Rdev)); err != nil { + return err + } + + default: + return fmt.Errorf("unknown file type for %s", srcPath) + } + + // Everything below is copying metadata from src to dst. All this metadata + // already shares an inode for hardlinks. + if isHardlink { + return nil + } + + if err := os.Lchown(dstPath, int(stat.Uid), int(stat.Gid)); err != nil { + return err + } + + if copyXattrs { + if err := doCopyXattrs(srcPath, dstPath); err != nil { + return err + } + } + + isSymlink := f.Mode()&os.ModeSymlink != 0 + + // There is no LChmod, so ignore mode for symlink. Also, this + // must happen after chown, as that can modify the file mode + if !isSymlink { + if err := os.Chmod(dstPath, f.Mode()); err != nil { + return err + } + } + + // system.Chtimes doesn't support a NOFOLLOW flag atm + // nolint: unconvert + if f.IsDir() { + dirsToSetMtimes.PushFront(&dirMtimeInfo{dstPath: &dstPath, stat: stat}) + } else if !isSymlink { + aTime := time.Unix(int64(stat.Atim.Sec), int64(stat.Atim.Nsec)) + mTime := time.Unix(int64(stat.Mtim.Sec), int64(stat.Mtim.Nsec)) + if err := system.Chtimes(dstPath, aTime, mTime); err != nil { + return err + } + } else { + ts := []syscall.Timespec{stat.Atim, stat.Mtim} + if err := system.LUtimesNano(dstPath, ts); err != nil { + return err + } + } + return nil + }) + if err != nil { + return err + } + for e := dirsToSetMtimes.Front(); e != nil; e = e.Next() { + mtimeInfo := e.Value.(*dirMtimeInfo) + ts := []syscall.Timespec{mtimeInfo.stat.Atim, mtimeInfo.stat.Mtim} + if err := system.LUtimesNano(*mtimeInfo.dstPath, ts); err != nil { + return err + } + } + + return nil +} + +func doCopyXattrs(srcPath, dstPath string) error { + if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil { + return err + } + + // We need to copy this attribute if it appears in an overlay upper layer, as + // this function is used to copy those. It is set by overlay if a directory + // is removed and then re-created and should not inherit anything from the + // same dir in the lower dir. + return copyXattr(srcPath, dstPath, "trusted.overlay.opaque") +} diff --git a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go index 2801dfdc5..b6f22e90a 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go @@ -2401,7 +2401,7 @@ func (devices *DeviceSet) MountDevice(hash, path string, moptions graphdriver.Mo addNouuid := strings.Contains("nouuid", mountOptions) mountOptions = strings.Join(moptions.Options, ",") if addNouuid { - mountOptions = fmt.Sprintf("nouuid,", mountOptions) + mountOptions = fmt.Sprintf("nouuid,%s", mountOptions) } } diff --git a/vendor/github.com/containers/storage/drivers/fsdiff.go b/vendor/github.com/containers/storage/drivers/fsdiff.go index 19da7d101..ab2c41e58 100644 --- a/vendor/github.com/containers/storage/drivers/fsdiff.go +++ b/vendor/github.com/containers/storage/drivers/fsdiff.go @@ -8,6 +8,7 @@ import ( "github.com/containers/storage/pkg/chrootarchive" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/ioutils" + rsystem "github.com/opencontainers/runc/libcontainer/system" "github.com/sirupsen/logrus" ) @@ -167,7 +168,9 @@ func (gdw *NaiveDiffDriver) ApplyDiff(id string, applyMappings *idtools.IDMappin } defer driver.Put(id) - options := &archive.TarOptions{} + options := &archive.TarOptions{ + InUserNS: rsystem.RunningInUserNS(), + } if applyMappings != nil { options.UIDMaps = applyMappings.UIDs() options.GIDMaps = applyMappings.GIDs() diff --git a/vendor/github.com/containers/storage/drivers/overlay/overlay.go b/vendor/github.com/containers/storage/drivers/overlay/overlay.go index a165a13a3..df736c0a9 100644 --- a/vendor/github.com/containers/storage/drivers/overlay/overlay.go +++ b/vendor/github.com/containers/storage/drivers/overlay/overlay.go @@ -29,6 +29,7 @@ import ( "github.com/containers/storage/pkg/parsers" "github.com/containers/storage/pkg/system" units "github.com/docker/go-units" + rsystem "github.com/opencontainers/runc/libcontainer/system" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -876,6 +877,7 @@ func (d *Driver) ApplyDiff(id string, idMappings *idtools.IDMappings, parent str UIDMaps: idMappings.UIDs(), GIDMaps: idMappings.GIDs(), WhiteoutFormat: d.getWhiteoutFormat(), + InUserNS: rsystem.RunningInUserNS(), }); err != nil { return 0, err } diff --git a/vendor/github.com/containers/storage/drivers/vfs/copy_linux.go b/vendor/github.com/containers/storage/drivers/vfs/copy_linux.go new file mode 100644 index 000000000..8137fcf67 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/vfs/copy_linux.go @@ -0,0 +1,7 @@ +package vfs + +import "github.com/containers/storage/drivers/copy" + +func dirCopy(srcDir, dstDir string) error { + return copy.DirCopy(srcDir, dstDir, copy.Content, false) +} diff --git a/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go b/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go new file mode 100644 index 000000000..8ac80ee1d --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go @@ -0,0 +1,9 @@ +// +build !linux + +package vfs // import "github.com/containers/storage/drivers/vfs" + +import "github.com/containers/storage/pkg/chrootarchive" + +func dirCopy(srcDir, dstDir string) error { + return chrootarchive.NewArchiver(nil).CopyWithTar(srcDir, dstDir) +} diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index e3a67a69b..f7f3c75ba 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/containers/storage/drivers" - "github.com/containers/storage/pkg/chrootarchive" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/ostree" "github.com/containers/storage/pkg/system" @@ -15,8 +14,8 @@ import ( ) var ( - // CopyWithTar defines the copy method to use. - CopyWithTar = chrootarchive.NewArchiver(nil).CopyWithTar + // CopyDir defines the copy method to use. + CopyDir = dirCopy ) func init() { @@ -141,7 +140,7 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, ro bool if err != nil { return fmt.Errorf("%s: %s", parent, err) } - if err := CopyWithTar(parentDir, dir); err != nil { + if err := dirCopy(parentDir, dir); err != nil { return err } } diff --git a/vendor/github.com/containers/storage/layers.go b/vendor/github.com/containers/storage/layers.go index 6c8f59b8b..0b532eb77 100644 --- a/vendor/github.com/containers/storage/layers.go +++ b/vendor/github.com/containers/storage/layers.go @@ -28,9 +28,8 @@ import ( ) const ( - tarSplitSuffix = ".tar-split.gz" - incompleteFlag = "incomplete" - compressionFlag = "diff-compression" + tarSplitSuffix = ".tar-split.gz" + incompleteFlag = "incomplete" ) // A Layer is a record of a copy-on-write layer that's stored by the lower diff --git a/vendor/github.com/containers/storage/pkg/archive/archive.go b/vendor/github.com/containers/storage/pkg/archive/archive.go index 4c4382625..8d6eaacf3 100644 --- a/vendor/github.com/containers/storage/pkg/archive/archive.go +++ b/vendor/github.com/containers/storage/pkg/archive/archive.go @@ -22,6 +22,7 @@ import ( "github.com/containers/storage/pkg/pools" "github.com/containers/storage/pkg/promise" "github.com/containers/storage/pkg/system" + rsystem "github.com/opencontainers/runc/libcontainer/system" "github.com/sirupsen/logrus" ) @@ -1054,6 +1055,7 @@ func (archiver *Archiver) TarUntar(src, dst string) error { GIDMaps: tarMappings.GIDs(), Compression: Uncompressed, CopyPass: true, + InUserNS: rsystem.RunningInUserNS(), } archive, err := TarWithOptions(src, options) if err != nil { @@ -1068,6 +1070,7 @@ func (archiver *Archiver) TarUntar(src, dst string) error { UIDMaps: untarMappings.UIDs(), GIDMaps: untarMappings.GIDs(), ChownOpts: archiver.ChownOpts, + InUserNS: rsystem.RunningInUserNS(), } return archiver.Untar(archive, dst, options) } @@ -1087,6 +1090,7 @@ func (archiver *Archiver) UntarPath(src, dst string) error { UIDMaps: untarMappings.UIDs(), GIDMaps: untarMappings.GIDs(), ChownOpts: archiver.ChownOpts, + InUserNS: rsystem.RunningInUserNS(), } return archiver.Untar(archive, dst, options) } @@ -1186,6 +1190,7 @@ func (archiver *Archiver) CopyFileWithTar(src, dst string) (err error) { UIDMaps: archiver.UntarIDMappings.UIDs(), GIDMaps: archiver.UntarIDMappings.GIDs(), ChownOpts: archiver.ChownOpts, + InUserNS: rsystem.RunningInUserNS(), } err = archiver.Untar(r, filepath.Dir(dst), options) if err != nil { diff --git a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go index b9fa228e6..dde8d44d3 100644 --- a/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go +++ b/vendor/github.com/containers/storage/pkg/chrootarchive/archive.go @@ -9,6 +9,7 @@ import ( "github.com/containers/storage/pkg/archive" "github.com/containers/storage/pkg/idtools" + rsystem "github.com/opencontainers/runc/libcontainer/system" ) // NewArchiver returns a new Archiver which uses chrootarchive.Untar @@ -52,6 +53,7 @@ func untarHandler(tarArchive io.Reader, dest string, options *archive.TarOptions } if options == nil { options = &archive.TarOptions{} + options.InUserNS = rsystem.RunningInUserNS() } if options.ExcludePatterns == nil { options.ExcludePatterns = []string{} diff --git a/vendor/github.com/containers/storage/vendor.conf b/vendor/github.com/containers/storage/vendor.conf index 059ae94f0..fa52584d7 100644 --- a/vendor/github.com/containers/storage/vendor.conf +++ b/vendor/github.com/containers/storage/vendor.conf @@ -9,7 +9,7 @@ github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062 github.com/opencontainers/go-digest master github.com/opencontainers/runc 6c22e77604689db8725fa866f0f2ec0b3e8c3a07 github.com/opencontainers/selinux 36a9bc45a08c85f2c52bd9eb32e20267876773bd -github.com/ostreedev/ostree-go aeb02c6b6aa2889db3ef62f7855650755befd460 +github.com/ostreedev/ostree-go master github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9 github.com/pkg/errors master github.com/pmezard/go-difflib v1.0.0 @@ -21,3 +21,5 @@ github.com/tchap/go-patricia v2.2.6 github.com/vbatts/tar-split v0.10.2 golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 golang.org/x/sys 07c182904dbd53199946ba614a412c61d3c548f5 +gotest.tools master +github.com/google/go-cmp master diff --git a/version/version.go b/version/version.go index 01b9b7a8d..45dc93d91 100644 --- a/version/version.go +++ b/version/version.go @@ -4,4 +4,4 @@ package version // NOTE: remember to bump the version at the top // of the top-level README.md file when this is // bumped. -const Version = "0.11.2-dev" +const Version = "0.12.2-dev" |