diff options
64 files changed, 1027 insertions, 317 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 663fb91b8..1665f3d9c 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -30,11 +30,12 @@ env: #### #### Cache-image names to test with (double-quotes around names are critical) ### - _BUILT_IMAGE_SUFFIX: "libpod-5751722641719296" + _BUILT_IMAGE_SUFFIX: "libpod-5664838702858240" FEDORA_CACHE_IMAGE_NAME: "fedora-30-${_BUILT_IMAGE_SUFFIX}" PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-29-${_BUILT_IMAGE_SUFFIX}" SPECIAL_FEDORA_CACHE_IMAGE_NAME: "xfedora-30-${_BUILT_IMAGE_SUFFIX}" - UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-${_BUILT_IMAGE_SUFFIX}" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-19-${_BUILT_IMAGE_SUFFIX}" + PRIOR_UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-${_BUILT_IMAGE_SUFFIX}" #### #### Variables for composing new cache-images (used in PR testing) from @@ -55,7 +56,7 @@ env: #### Credentials and other secret-sauces, decrypted at runtime when authorized. #### # Freenode IRC credentials for posting status messages - IRCID: ENCRYPTED[e87bba62a8e924dc70bdb2b66b16f6ab4a60d2870e6e5534ae9e2b0076f483c71c84091c655ca239101e6816c5ec0883] + IRCID: ENCRYPTED[1913f8a4572b6a6d2036232327789c4f6c0d98cde53f0336d860cd219b4cbd83863eefd93471aef8fa1079d4698e382d] # Needed to build GCE images, within a GCE VM SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a] # User ID for cirrus to ssh into VMs @@ -116,6 +117,12 @@ gating_task: - '/usr/local/bin/entrypoint.sh podman-remote-darwin |& ${TIMESTAMP}' - '/usr/local/bin/entrypoint.sh podman-remote-windows |& ${TIMESTAMP}' + # Verify some aspects of ci/related scripts + ci_script: + - '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/lib.sh.t |& ${TIMESTAMP}' + - '/usr/local/bin/entrypoint.sh -C ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/packer test' + - '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/cirrus_yaml_test.py |& ${TIMESTAMP}' + # Verify expected bash environment (-o pipefail) pipefail_enabledscript: 'if /bin/false | /bin/true; then echo "pipefail fault" && exit 72; fi' @@ -269,6 +276,7 @@ meta_task: ${PRIOR_FEDORA_CACHE_IMAGE_NAME} ${SPECIAL_FEDORA_CACHE_IMAGE_NAME} ${UBUNTU_CACHE_IMAGE_NAME} + ${PRIOR_UBUNTU_CACHE_IMAGE_NAME} ${IMAGE_BUILDER_CACHE_IMAGE_NAME} BUILDID: "${CIRRUS_BUILD_ID}" REPOREF: "${CIRRUS_CHANGE_IN_REPO}" @@ -325,7 +333,9 @@ testing_task: # Images are generated separately, from build_images_task (below) image_name: "${FEDORA_CACHE_IMAGE_NAME}" image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" - image_name: "${UBUNTU_CACHE_IMAGE_NAME}" + # Multiple test failures on Ubuntu 19 - Fixes TBD in future PR + # TODO: image_name: "${UBUNTU_CACHE_IMAGE_NAME}" + image_name: "${PRIOR_UBUNTU_CACHE_IMAGE_NAME}" timeout_in: 120m @@ -341,8 +351,14 @@ testing_task: integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}' + # For PRs this confirms uploading releases after merge, is functional. upload_release_archive_script: '$SCRIPT_BASE/upload_release_archive.sh |& ${TIMESTAMP}' + # When examining a particular run, provide convenient access to release files. + tar_artifacts: + path: "*.tar.gz" + type: "application/x-tar" + on_failure: failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' @@ -374,8 +390,6 @@ testing_crun_task: networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' install_crun_script: 'dnf install -y crun' - # FIXME: use the package once all the fixes are in a release - override_crun_script: 'setenforce 0; yum builddep -y crun && (git clone --depth=1 https://github.com/containers/crun && cd crun && ./autogen.sh && ./configure --prefix=/usr && make -j4 && make install) && rm -rf crun' unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}' integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}' system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}' @@ -472,6 +486,15 @@ special_testing_cross_task: on_failure: failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh' + # When examining a particular run, provide convenient access to release files. + zip_artifacts: + path: "*.zip" + type: "application/zip" + + msi_artifacts: + path: "*.msi" + type: "application/octet-stream" + special_testing_cgroupv2_task: @@ -532,6 +555,8 @@ test_building_snap_task: depends_on: - "gating" + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' + container: image: yakshaveinc/snapcraft:core18 snapcraft_script: @@ -544,8 +569,7 @@ test_build_cache_images_task: only_if: >- $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' && - $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' + $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' depends_on: - "gating" @@ -565,10 +589,7 @@ test_build_cache_images_task: - devstorage.full_control networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' - setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}' - df_script: '${DFCMD}' - journalctl_b_script: 'journalctl -b' on_failure: failed_df_script: '${DFCMD}' @@ -580,8 +601,7 @@ verify_test_built_images_task: only_if: >- $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' && - $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' + $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' depends_on: @@ -603,6 +623,7 @@ verify_test_built_images_task: PACKER_BUILDER_NAME: "fedora-30" PACKER_BUILDER_NAME: "xfedora-30" PACKER_BUILDER_NAME: "ubuntu-18" + # TODO support $UBUNTU_CACHE_IMAGE_NAME: PACKER_BUILDER_NAME: "ubuntu-19" networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' @@ -618,7 +639,7 @@ verify_test_built_images_task: $SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} build_release_script: >- [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \ - '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}' + $SCRIPT_BASE/build_release.sh |& ${TIMESTAMP} system_test_script: >- [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \ $SCRIPT_BASE/system_test.sh |& ${TIMESTAMP} @@ -626,6 +647,29 @@ verify_test_built_images_task: always: <<: *standardlogs + +upload_snap_task: + + # Only when PR or branch is merged into master + only_if: $CIRRUS_BRANCH == $DEST_BRANCH + + depends_on: + - "test_building_snap" + + container: + image: yakshaveinc/snapcraft:core18 + + env: + SNAPCRAFT_LOGIN: ENCRYPTED[d8e82eb31c6372fec07f405f413d57806026b1a9f8400033531ebcd54d6750a5e4a8b1f68e3ec65c98c65e0d9b2a6a75] + snapcraft_login_file: + path: /root/.snapcraft/login.cfg + variable_name: SNAPCRAFT_LOGIN + snapcraft_script: + - 'apt-get -y update' + - 'snapcraft login --with "/root/.snapcraft/login.cfg"' + - 'cd contrib/snapcraft && snapcraft && snapcraft push *.snap --release edge' + + # Post message to IRC if everything passed PR testing success_task: @@ -651,6 +695,7 @@ success_task: - "special_testing_endpoint" - "test_build_cache_images" - "test_building_snap" + - "upload_snap" - "verify_test_built_images" env: @@ -304,8 +304,43 @@ method AttachControl(name: [string](https://godoc.org/builtin#string)) </div> method BuildImage(build: [BuildInfo](#BuildInfo)) [MoreResponse](#MoreResponse)</div> BuildImage takes a [BuildInfo](#BuildInfo) structure and builds an image. At a minimum, you must provide the -'dockerfile' and 'tags' options in the BuildInfo structure. It will return a [MoreResponse](#MoreResponse) structure +contextDir tarball path, the 'dockerfiles' path, and 'output' option in the BuildInfo structure. The 'output' +options is the name of the of the resulting build. It will return a [MoreResponse](#MoreResponse) structure that contains the build logs and resulting image ID. +#### Example +~~~ +$ sudo varlink call -m unix:///run/podman/io.podman/io.podman.BuildImage '{"build":{"contextDir":"/tmp/t/context.tar","dockerfiles":["Dockerfile"], "output":"foobar"}}' +{ + "image": { + "id": "", + "logs": [ + "STEP 1: FROM alpine\n" + ] + } +} +{ + "image": { + "id": "", + "logs": [ + "STEP 2: COMMIT foobar\n" + ] + } +} +{ + "image": { + "id": "", + "logs": [ + "b7b28af77ffec6054d13378df4fdf02725830086c7444d9c278af25312aa39b9\n" + ] + } +} +{ + "image": { + "id": "b7b28af77ffec6054d13378df4fdf02725830086c7444d9c278af25312aa39b9", + "logs": [] + } +} +~~~ ### <a name="BuildImageHierarchyMap"></a>func BuildImageHierarchyMap <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;"> diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 07b2b3584..be13b6de3 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -347,9 +347,6 @@ tracking system. There is also a [mailing list](https://lists.podman.io/archives/) at `lists.podman.io`. You can subscribe by sending a message to `podman@lists.podman.io` with the subject `subscribe`. -[owners]: https://github.com/kubernetes/community/blob/master/contributors/guide/owners.md#owners - - ### Bot Interactions The primary human-interface is through comments in pull-requests. Some of these are outlined diff --git a/Dockerfile.fedora b/Dockerfile.fedora index 9b1568b0b..8769b5c18 100644 --- a/Dockerfile.fedora +++ b/Dockerfile.fedora @@ -2,8 +2,11 @@ FROM registry.fedoraproject.org/fedora:30 RUN dnf -y install btrfs-progs-devel \ atomic-registries \ + autoconf \ + automake \ bzip2 \ device-mapper-devel \ + file \ findutils \ git \ glib2-devel \ @@ -15,6 +18,7 @@ RUN dnf -y install btrfs-progs-devel \ libassuan-devel \ libseccomp-devel \ libselinux-devel \ + libtool \ containers-common \ runc \ make \ @@ -3,7 +3,7 @@ export GOPROXY=https://proxy.golang.org GO ?= go DESTDIR ?= -EPOCH_TEST_COMMIT ?= 2366fd7ac621ba15abe559832f024d06b3db3c9b +EPOCH_TEST_COMMIT ?= 0000afc1af06b04ececeb91637bb3d80d6f47e14 HEAD ?= HEAD CHANGELOG_BASE ?= HEAD~ CHANGELOG_TARGET ?= HEAD @@ -189,7 +189,7 @@ clean: ## Clean artifacts rm -rf \ .gopathok \ _output \ - release.txt + release.txt \ $(wildcard podman-remote*.zip) \ $(wildcard podman*.tar.gz) \ bin \ @@ -245,9 +245,6 @@ localunit: test/goecho/goecho varlink_generate --covermode atomic \ --tags "$(BUILDTAGS)" \ --succinct - $(MAKE) -C contrib/cirrus/packer test - ./contrib/cirrus/lib.sh.t - ./contrib/cirrus/cirrus_yaml_test.py ginkgo: ginkgo -v -tags "$(BUILDTAGS)" $(GINKGOTIMEOUT) -cover -flakeAttempts 3 -progress -trace -noColor -nodes 3 -debug test/e2e/. @@ -33,6 +33,23 @@ This project tests all builds against each supported version of Fedora, the late 1. Further work on the podman pod command 1. Further improvements on rootless containers +## Communications + +If you think you've identified a security issue in the project, please *DO NOT* report the issue publically via the Github issue tracker, mailing list, or IRC. +Instead, send an email with as many details as possible to `security@lists.podman.io`. This is a private mailing list for the core maintainers. + +For general questions and discussion, please use the +IRC `#podman` channel on `irc.freenode.net`. + +For discussions around issues/bugs and features, you can use the GitHub +[issues](https://github.com/containers/libpod/issues) +and +[PRs](https://github.com/containers/libpod/pulls) +tracking system. + +There is also a [mailing list](https://lists.podman.io/archives/) at `lists.podman.io`. +You can subscribe by sending a message to `podman@lists.podman.io` with the subject `subscribe`. + ## Rootless Podman can be easily run as a normal user, without requiring a setuid binary. When run without root, Podman containers use user namespaces to set root in the container to the user running Podman. diff --git a/RELEASE_NOTES.md b/RELEASE_NOTES.md index 5c9c06687..4fcdd68de 100644 --- a/RELEASE_NOTES.md +++ b/RELEASE_NOTES.md @@ -8,6 +8,7 @@ - The `podman volume rm` and `podman volume inspect` commands can now refer to volumes by an unambiguous partial name, in addition to full name (e.g. `podman volume rm myvol` to remove a volume named `myvolume`) ([#3891](https://github.com/containers/libpod/issues/3891)) - The `podman run` and `podman create` commands now support the `--pull` flag to allow forced re-pulling of images ([#3734](https://github.com/containers/libpod/issues/3734)) - Mounting volumes into a container using `--volume`, `--mount`, and `--tmpfs` now allows the `suid`, `dev`, and `exec` mount options (the inverse of `nosuid`, `nodev`, `noexec`) ([#3819](https://github.com/containers/libpod/issues/3819)) +- Mounting volumes into a container using `--mount` now allows the `relabel=Z` and `relabel=z` options to relabel mounts. - The `podman push` command now supports the `--digestfile` option to save a file containing the pushed digest - Pods can now have their hostname set via `podman pod create --hostname` or providing Pod YAML with a hostname set to `podman play kube` ([#3732](https://github.com/containers/libpod/issues/3732)) - The `podman image sign` command now supports the `--cert-dir` flag @@ -37,12 +38,18 @@ - Fixed a bug where `podman exec` would run as the wrong user when execing into a container was started from an image with Dockerfile `USER` (or a user specified via `podman run --user`) ([#3838](https://github.com/containers/libpod/issues/3838)) - Fixed a bug where images pulled using the `oci:` transport would be improperly named - Fixed a bug where `podman varlink` would hang when managed by systemd due to SD_NOTIFY support conflicting with Varlink ([#3572](https://github.com/containers/libpod/issues/3572)) +- Fixed a bug where mounts to the same destination would sometimes not trigger a conflict, causing a race as to which was actually mounted +- Fixed a bug where `podman exec --preserve-fds` caused Podman to hang ([#4020](https://github.com/containers/libpod/issues/4020)) +- Fixed a bug where removing an unmounted container that was unmounted might sometimes not properly clean up the container ([#4033](https://github.com/containers/libpod/issues/4033)) +- Fixed a bug where the Varlink server would freeze when run in a systemd unit file ([#4005](https://github.com/containers/libpod/issues/4005)) +- Fixed a bug where Podman would not properly set the `$HOME` environment variable when the OCI runtime did not set it ### Misc - Significant changes were made to Podman volumes in this release. If you have pre-existing volumes, it is strongly recommended to run `podman system renumber` after upgrading. - Version 0.8.1 or greater of the CNI Plugins is now required for Podman - Version 2.0.1 or greater of Conmon is strongly recommended - Updated vendored Buildah to v1.11.2 +- Updated vendored containers/storage library to v1.13.3 - Improved error messages when trying to run `podman pause` or `podman stats` on a rootless container on a system without CGroups V2 enabled - `TMPDIR` has been set to `/var/tmp` by default to better handle large temporary files - `podman wait` has been optimized to detect stopped containers more rapidly diff --git a/changelog.txt b/changelog.txt index c2c2a8ce9..1e9d17d06 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,49 @@ +- Changelog for v1.6.0-rc2 (2019-09-24) + * Add release notes for new-in-RC2 changes + * system tests: run test: reenable and fix + * play kube: Only support pod kind in k8s yaml + * runtime: fix logic to disable SDNotify + * add list mount tests + * Make netns bind mount shared + * Add Kata Containers support + * rootless: Rearrange setup of rootless containers + * Document the 'system' event types for 'podman events' + * Cirrus: Add upload_snap to success dependencies + * Cirrus: Add snapcraft credentials + * Cirrus: Upload snap only on merges to master + * Cirrus: Push snap continuously + * exec: set HOME also with exec sessions + * execuser: look at the source for /etc/{passwd,group} overrides + * We need to convert libpod.conf files in user homedir for cgroupv2 + * Cirrus: Temporarily disable testing on Ubuntu 19 + * Cirrus: disable Evil Units in base-images + * Cirrus: Add latest ubuntu + * Cirrus: More podbot/success improvements + * Cirrus: Fix success script + * Cirrus: Update podbot credentials + * container: make sure $HOME is always set + * Move rootless and Mac to Tutorials page + * fix trivial type for event logger + * Support podman-remote help on windows + * Clean destination paths during mount generation + * tests: use crun package + * Add a note on systemd shortcomings in rootless containers + * support non-standard ssh port for remote-client + * Add links to the Mac tutorial in the main tutorial + * Vendor c/storage 1.13.3 + * System-test: Temporarily disable 030-run + * Fix exit code failure + * exec: fix --preserve-fds + * networking: use --enable-sandbox if available + * Add 'relabel' to --mount options + * Bump Gitvalidation epoch + * Bump to v1.6.0-dev + * Unmounting a container that is already unmounted is OK + * Check for rootless before checking cgroups version in spec_test. + * Skip spec_test for rootless envs without cgroup v2. + * fix unit test to use Expect + * Cirrus: Prevent resident pollution + - Changelog for v1.6.0-rc1 (2019-09-16) * Fix default to pause in podman cp * Update release notes for v1.6.0 diff --git a/cmd/podman/cp.go b/cmd/podman/cp.go index 7205f9357..75a23afd6 100644 --- a/cmd/podman/cp.go +++ b/cmd/podman/cp.go @@ -290,7 +290,7 @@ func copy(src, destPath, dest string, idMappingOpts storage.IDMappingOptions, ch } destdir := destPath - if !srcfi.IsDir() && !strings.HasSuffix(dest, string(os.PathSeparator)) { + if !srcfi.IsDir() { destdir = filepath.Dir(destPath) } _, err = os.Stat(destdir) @@ -329,7 +329,7 @@ func copy(src, destPath, dest string, idMappingOpts storage.IDMappingOptions, ch destfi, err := os.Stat(destPath) if err != nil { - if !os.IsNotExist(err) { + if !os.IsNotExist(err) || strings.HasSuffix(dest, string(os.PathSeparator)) { return errors.Wrapf(err, "failed to get stat of dest path %s", destPath) } } diff --git a/cmd/podman/libpodruntime/runtime.go b/cmd/podman/libpodruntime/runtime.go index a133549ea..6dafeb0b0 100644 --- a/cmd/podman/libpodruntime/runtime.go +++ b/cmd/podman/libpodruntime/runtime.go @@ -171,7 +171,7 @@ func getRuntime(ctx context.Context, c *cliconfig.PodmanCommand, renumber, migra options = append(options, libpod.WithDefaultInfraCommand(infraCommand)) } - if withFDS { + if !withFDS { options = append(options, libpod.WithEnableSDNotify()) } if c.Flags().Changed("config") { diff --git a/cmd/podman/main.go b/cmd/podman/main.go index 992dbe1d5..344170ddd 100644 --- a/cmd/podman/main.go +++ b/cmd/podman/main.go @@ -107,15 +107,6 @@ func before(cmd *cobra.Command, args []string) error { os.Exit(1) } - if err := setupRootless(cmd, args); err != nil { - return err - } - - // check that global opts input is valid - if err := checkInput(); err != nil { - return err - } - // Set log level; if not log-level is provided, default to error logLevel := MainGlobalOpts.LogLevel if logLevel == "" { @@ -130,6 +121,15 @@ func before(cmd *cobra.Command, args []string) error { return err } + if err := setupRootless(cmd, args); err != nil { + return err + } + + // check that global opts input is valid + if err := checkInput(); err != nil { + return err + } + if err := setRLimits(); err != nil { return err } @@ -149,6 +149,8 @@ func main() { //cpuProfile := false if reexec.Init() { + // We were invoked with a different argv[0] indicating that we + // had a specific job to do as a subprocess, and it's done. return } // Hard code TMPDIR functions to use /var/tmp, if user did not override diff --git a/cmd/podman/varlink/io.podman.varlink b/cmd/podman/varlink/io.podman.varlink index 4692525e3..7239f5d2e 100644 --- a/cmd/podman/varlink/io.podman.varlink +++ b/cmd/podman/varlink/io.podman.varlink @@ -765,8 +765,43 @@ method ListImages() -> (images: []Image) method GetImage(id: string) -> (image: Image) # BuildImage takes a [BuildInfo](#BuildInfo) structure and builds an image. At a minimum, you must provide the -# 'dockerfile' and 'tags' options in the BuildInfo structure. It will return a [MoreResponse](#MoreResponse) structure +# contextDir tarball path, the 'dockerfiles' path, and 'output' option in the BuildInfo structure. The 'output' +# options is the name of the of the resulting build. It will return a [MoreResponse](#MoreResponse) structure # that contains the build logs and resulting image ID. +# #### Example +# ~~~ +# $ sudo varlink call -m unix:///run/podman/io.podman/io.podman.BuildImage '{"build":{"contextDir":"/tmp/t/context.tar","dockerfiles":["Dockerfile"], "output":"foobar"}}' +# { +# "image": { +# "id": "", +# "logs": [ +# "STEP 1: FROM alpine\n" +# ] +# } +# } +# { +# "image": { +# "id": "", +# "logs": [ +# "STEP 2: COMMIT foobar\n" +# ] +# } +# } +# { +# "image": { +# "id": "", +# "logs": [ +# "b7b28af77ffec6054d13378df4fdf02725830086c7444d9c278af25312aa39b9\n" +# ] +# } +# } +# { +# "image": { +# "id": "b7b28af77ffec6054d13378df4fdf02725830086c7444d9c278af25312aa39b9", +# "logs": [] +# } +# } +# ~~~ method BuildImage(build: BuildInfo) -> (image: MoreResponse) # This function is not implemented yet. diff --git a/contrib/cirrus/build_release.sh b/contrib/cirrus/build_release.sh index 287643f47..07db88f81 100755 --- a/contrib/cirrus/build_release.sh +++ b/contrib/cirrus/build_release.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + source $(dirname $0)/lib.sh req_env_var TEST_REMOTE_CLIENT OS_RELEASE_ID GOSRC @@ -13,18 +15,20 @@ fi if [[ -n "$CROSS_PLATFORM" ]] then + # Will fail if $CROSS_PLATFORM is unsupported cross-compile $GOOS value + make podman-remote-${CROSS_PLATFORM}-release + echo "Compiling podman-remote release archive for ${CROSS_PLATFORM}" - case "$CROSS_PLATFORM" in - linux) ;& - windows) ;& - darwin) - make podman-remote-${CROSS_PLATFORM}-release - ;; - *) - die 1 "Unknown/unsupported cross-compile platform '$CROSS_PLATFORM'" - ;; - esac + if [[ "$CROSS_PLATFORM" == "windows" ]] + then + # TODO: Remove next line, part of VM images next time they're built. + dnf install -y libmsi1 msitools pandoc + make podman.msi + fi else echo "Compiling release archive for $OS_RELEASE_ID" make podman-release fi + +echo "Preserving build details for later use." +mv -v release.txt actual_release.txt # Another 'make' during testing could overwrite it diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index 6230610cb..543f83a14 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -3,7 +3,7 @@ set -e source $(dirname $0)/lib.sh -BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE' +BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE PRIOR_UBUNTU_BASE_IMAGE' ENV_VARS="PACKER_BUILDS BUILT_IMAGE_SUFFIX $BASE_IMAGE_VARS SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO" req_env_var $ENV_VARS # Must also be made available through make, into packer process diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh index ad9a12f49..39f49d0a1 100755 --- a/contrib/cirrus/check_image.sh +++ b/contrib/cirrus/check_image.sh @@ -4,6 +4,8 @@ set -eo pipefail source $(dirname $0)/lib.sh +EVIL_UNITS="$($CIRRUS_WORKING_DIR/$PACKER_BASE/systemd_banish.sh --list)" + req_env_var PACKER_BUILDER_NAME TEST_REMOTE_CLIENT EVIL_UNITS OS_RELEASE_ID NFAILS=0 diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index 00c3b0ec3..110066ea7 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -37,11 +37,8 @@ case "$SPECIALMODE" in -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} ;; cgroupv2) - # FIXME: use the package once all the fixes are in a release - # yum install -y crun setenforce 0 - yum builddep -y crun - (git clone --depth=1 https://github.com/containers/crun && cd crun && ./autogen.sh && ./configure --prefix=/usr && make -j4 && make install) + dnf install -y crun export OCI_RUNTIME=/usr/bin/crun make make install PREFIX=/usr ETCDIR=/etc diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index f26eec87f..f81a8d501 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -36,7 +36,7 @@ SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_c AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv" cd $GOSRC -if type -P git &> /dev/null +if type -P git &> /dev/null && [[ -d "$GOSRC/.git" ]] then CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)} else # pick something unique and obviously not from Cirrus @@ -52,16 +52,17 @@ CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod} CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique # Vars. for image-building -PACKER_VER="1.3.5" +PACKER_VER="1.4.2" # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) # Base-images rarely change, define them here so they're out of the way. -export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-30,xfedora-30,fedora-29}" +export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-30,xfedora-30,fedora-29}" # Google-maintained base-image names -export UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a" +export UBUNTU_BASE_IMAGE="ubuntu-1904-disco-v20190724" +export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a" # Manually produced base-image names (see $SCRIPT_BASE/README.md) -export FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1559164849" -export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849" +export FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1565360543" +export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1565360543" export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" # IN_PODMAN container image IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest" @@ -81,9 +82,6 @@ ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(. # Unsafe env. vars for display SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(GC[EP]..+)|(SSH)' -# Names of systemd units which should never be running -EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean" - SPECIALMODE="${SPECIALMODE:-none}" TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}" export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} @@ -322,7 +320,7 @@ EOF sudo chmod 755 /usr/bin/git } -install_test_configs(){ +install_test_configs() { echo "Installing cni config, policy and registry config" req_env_var GOSRC SCRIPT_BASE cd $GOSRC @@ -342,7 +340,7 @@ install_test_configs(){ # of pulling in necessary prerequisites packages as the set can change over time. # For general CI testing however, calling this function makes sure the system # can only run the compiled source version. -remove_packaged_podman_files(){ +remove_packaged_podman_files() { echo "Removing packaged podman files to prevent conflicts with source build and testing." req_env_var OS_RELEASE_ID @@ -379,24 +377,11 @@ remove_packaged_podman_files(){ sync && echo 3 > /proc/sys/vm/drop_caches } -systemd_banish(){ - echo "Disabling periodic services that could destabilize testing (ignoring errors):" - set +e # Not all of these exist on every platform - for unit in $EVIL_UNITS - do - echo "Banishing $unit (ignoring errors)" - ( - sudo systemctl stop $unit - sudo systemctl disable $unit - sudo systemctl disable $unit.timer - sudo systemctl mask $unit - sudo systemctl mask $unit.timer - ) &> /dev/null - done - set -e +systemd_banish() { + $GOSRC/$PACKER_BASE/systemd_banish.sh } -_finalize(){ +_finalize() { set +e # Don't fail at the very end if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]] then @@ -419,7 +404,7 @@ _finalize(){ sudo fstrim -av } -rh_finalize(){ +rh_finalize() { set +e # Don't fail at the very end echo "Resetting to fresh-state for usage as cloud-image." PKG=$(type -P dnf || type -P yum || echo "") @@ -430,7 +415,7 @@ rh_finalize(){ _finalize } -ubuntu_finalize(){ +ubuntu_finalize() { set +e # Don't fail at the very end echo "Resetting to fresh-state for usage as cloud-image." $LILTO $SUDOAPTGET autoremove diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile index d03d22abe..947a2a1e9 100644 --- a/contrib/cirrus/packer/Makefile +++ b/contrib/cirrus/packer/Makefile @@ -3,7 +3,7 @@ # builder name(s) from applicable YAML file, # e.g for names see libpod_images.yml -PACKER_VER ?= 1.3.5 +PACKER_VER ?= 1.4.2 GOARCH=$(shell go env GOARCH) ARCH=$(uname -m) PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip @@ -15,6 +15,9 @@ PACKER_BASE ?= contrib/cirrus/packer SCRIPT_BASE ?= contrib/cirrus POST_MERGE_BUCKET_SUFFIX ?= +UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$UBUNTU_BASE_IMAGE") +PRIOR_UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$PRIOR_UBUNTU_BASE_IMAGE") + # For debugging nested-virt, use #TTYDEV := $(shell tty) TTYDEV := /dev/null @@ -22,6 +25,14 @@ TTYDEV := /dev/null .PHONY: all all: libpod_images +# Utility target for checking required parameters +.PHONY: guard-% +guard-%: + @if [[ -z "$($*)" ]]; then \ + echo "Missing or empty required make variable '$*'."; \ + exit 1; \ + fi; + %.json: %.yml @python3 -c 'import json,yaml; json.dump( yaml.load(open("$<").read()), open("$@","w"), indent=2);' @@ -44,10 +55,7 @@ test: libpod_base_images.json libpod_images.json packer @echo "All good" .PHONY: libpod_images -libpod_images: libpod_images.json packer -ifndef PACKER_BUILDS - $(error PACKER_BUILDS is undefined, expected builder-names CSV) -endif +libpod_images: guard-PACKER_BUILDS libpod_images.json packer ./packer build -only=${PACKER_BUILDS} \ -force \ -var GOSRC=$(GOSRC) \ @@ -72,16 +80,7 @@ cidata.iso: user-data meta-data # This is intended to be run by a human, with admin access to the libpod GCE project. .PHONY: libpod_base_images -libpod_base_images: libpod_base_images.json cidata.iso cidata.ssh packer -ifndef GCP_PROJECT_ID - $(error GCP_PROJECT_ID is undefined, expected complete GCP project ID string e.g. foobar-12345) -endif -ifndef GOOGLE_APPLICATION_CREDENTIALS - $(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json) -endif -ifndef PACKER_BUILDS - $(error PACKER_BUILDS is undefined, expected builder-names CSV) -endif +libpod_base_images: guard-GCP_PROJECT_ID guard-GOOGLE_APPLICATION_CREDENTIALS libpod_base_images.json cidata.iso cidata.ssh packer PACKER_CACHE_DIR=/tmp ./packer build \ -force \ -var TIMESTAMP=$(TIMESTAMP) \ @@ -91,5 +90,6 @@ endif -var GOSRC=$(GOSRC) \ -var PACKER_BASE=$(PACKER_BASE) \ -var SCRIPT_BASE=$(SCRIPT_BASE) \ - -only $(PACKER_BUILDS) \ + -var UBUNTU_BASE_IMAGE=$(UBUNTU_BASE_IMAGE) \ + -var PRIOR_UBUNTU_BASE_IMAGE=$(PRIOR_UBUNTU_BASE_IMAGE) \ libpod_base_images.json diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh index 788a54c34..29c23117f 100644 --- a/contrib/cirrus/packer/fedora_base-setup.sh +++ b/contrib/cirrus/packer/fedora_base-setup.sh @@ -8,8 +8,6 @@ set -e # Load in library (copied by packer, before this script was run) source $GOSRC/$SCRIPT_BASE/lib.sh -[[ "$1" == "post" ]] || exit 0 # nothing to do - install_ooe echo "Updating packages" diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 0e1a82cc0..f84645a04 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -24,6 +24,8 @@ ooe.sh sudo dnf config-manager --set-enabled updates-testing echo "Installing general build/test dependencies" ooe.sh sudo dnf install -y \ atomic-registries \ + autoconf \ + automake \ bats \ bridge-utils \ btrfs-progs-devel \ @@ -35,10 +37,14 @@ ooe.sh sudo dnf install -y \ device-mapper-devel \ emacs-nox \ findutils \ + fuse3 \ + fuse3-devel \ + gcc \ git \ glib2-devel \ glibc-static \ gnupg \ + go-md2man \ golang \ golang-github-cpuguy83-go-md2man \ gpgme-devel \ @@ -47,6 +53,7 @@ ooe.sh sudo dnf install -y \ jq \ libassuan-devel \ libcap-devel \ + libmsi1 \ libnet \ libnet-devel \ libnl3-devel \ @@ -56,9 +63,11 @@ ooe.sh sudo dnf install -y \ libvarlink-util \ lsof \ make \ + msitools \ nmap-ncat \ ostree \ ostree-devel \ + pandoc \ podman \ procps-ng \ protobuf \ @@ -81,6 +90,7 @@ ooe.sh sudo dnf install -y \ xz \ zip + # Ensure there are no disruptive periodic services enabled by default in image systemd_banish @@ -91,6 +101,7 @@ case "$PACKER_BUILDER_NAME" in xfedora*) echo "Configuring CGroups v2 enabled on next boot" sudo grubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=1" + sudo dnf install -y crun ;& # continue to next matching item *) echo "Finalizing $PACKER_BUILDER_NAME VM image" diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml index e519d2fba..bcca440ae 100644 --- a/contrib/cirrus/packer/libpod_base_images.yml +++ b/contrib/cirrus/packer/libpod_base_images.yml @@ -12,6 +12,10 @@ variables: # Required for output from qemu builders TTYDEV: + # Ubuntu releases are mearly copied to this project for control purposes + UBUNTU_BASE_IMAGE: + PRIOR_UBUNTU_BASE_IMAGE: + # Latest Fedora release FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2" FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM" @@ -78,50 +82,67 @@ builders: ssh_username: 'root' - <<: *nested_virt - name: 'prior_fedora' + name: 'prior-fedora' iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}' iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}' + - &imgcopy + name: 'ubuntu' + type: 'googlecompute' + image_name: '{{user `UBUNTU_BASE_IMAGE`}}' + image_family: '{{build_name}}-base' + source_image: '{{user `UBUNTU_BASE_IMAGE`}}' + source_image_project_id: 'ubuntu-os-cloud' + project_id: '{{user `GCP_PROJECT_ID`}}' + account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' + startup_script_file: "systemd_banish.sh" + zone: 'us-central1-a' + disk_size: 20 + communicator: 'none' + + - <<: *imgcopy + name: 'prior-ubuntu' + image_name: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' + source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' + provisioners: - type: 'shell' + only: ['fedora', 'prior-fedora'] inline: - 'mkdir -p /tmp/libpod/{{user `SCRIPT_BASE`}}' - 'mkdir -p /tmp/libpod/{{user `PACKER_BASE`}}' - type: 'file' + only: ['fedora', 'prior-fedora'] source: '{{user `GOSRC`}}/.cirrus.yml' destination: '/tmp/libpod/.cirrus.yml' - type: 'file' + only: ['fedora', 'prior-fedora'] source: '{{user `GOSRC`}}/{{user `SCRIPT_BASE`}}/' destination: '/tmp/libpod/{{user `SCRIPT_BASE`}}/' - type: 'file' + only: ['fedora', 'prior-fedora'] source: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/' destination: '/tmp/libpod/{{user `PACKER_BASE`}}/' - &shell_script + only: ['fedora', 'prior-fedora'] type: 'shell' inline: - - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh' - - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh pre' - expect_disconnect: true # Allow this to reboot the VM + - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/*.sh' + - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh' + expect_disconnect: true # Allow this to reboot the VM if needed environment_vars: - 'TIMESTAMP={{user `TIMESTAMP`}}' - 'GOSRC=/tmp/libpod' - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' - 'PACKER_BASE={{user `PACKER_BASE`}}' - - <<: *shell_script - inline: ['{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'] - expect_disconnect: false - pause_before: '10s' - inline: - - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh post' - post-processors: - - type: "compress" - only: ['fedora', 'prior_fedora'] + only: ['fedora', 'prior-fedora'] output: '/tmp/{{build_name}}/disk.raw.tar.gz' format: '.tar.gz' compression_level: 9 @@ -134,10 +155,10 @@ post-processors: gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}.tar.gz' image_name: "{{user `FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}' - image_family: '{{user `FEDORA_BASE_IMAGE_NAME`}}' + image_family: '{{build_name}}-base' - <<: *gcp_import - only: ['prior_fedora'] + only: ['prior-fedora'] image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}' - image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}' + image_family: '{{build_name}}-base' - type: 'manifest' diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml index cae5d4138..01a65d867 100644 --- a/contrib/cirrus/packer/libpod_images.yml +++ b/contrib/cirrus/packer/libpod_images.yml @@ -2,16 +2,17 @@ # All of these are required variables: - # Names of GCE Base images to start from, in .cirrus.yml - UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}' - FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}' - PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}' - BUILT_IMAGE_SUFFIX: '{{env `BUILT_IMAGE_SUFFIX`}}' GOSRC: '{{env `GOSRC`}}' PACKER_BASE: '{{env `PACKER_BASE`}}' SCRIPT_BASE: '{{env `SCRIPT_BASE`}}' + # Base-image names are required. Using image family-names breaks parallelism + UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}' + PRIOR_UBUNTU_BASE_IMAGE: '{{env `PRIOR_UBUNTU_BASE_IMAGE`}}' + FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}' + PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}' + # Protected credentials, decrypted by Cirrus at runtime GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}' GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}' @@ -28,11 +29,12 @@ sensitive-variables: builders: # v----- is a YAML anchor, allows referencing this object by name (below) - &gce_hosted_image - name: 'ubuntu-18' + name: 'ubuntu-19' type: 'googlecompute' image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}' - image_family: '{{build_name}}-libpod' - source_image: '{{user `UBUNTU_BASE_IMAGE`}}' + image_family: '{{build_name}}-cache' + source_image: '{{user `UBUNTU_BASE_IMAGE`}}' # precedence over family + source_image_family: 'ubuntu-base' # for ref. only disk_size: 20 # REQUIRED: Runtime allocation > this value project_id: '{{user `GCP_PROJECT_ID`}}' service_account_email: '{{user `SERVICE_ACCOUNT`}}' @@ -44,16 +46,24 @@ builders: # v----- is a YAML alias, allows partial re-use of the anchor object - <<: *gce_hosted_image + name: 'ubuntu-18' + source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' + source_image_family: 'prior-ubuntu-base' + + - <<: *gce_hosted_image name: 'fedora-30' source_image: '{{user `FEDORA_BASE_IMAGE`}}' + source_image_family: 'fedora-base' - <<: *gce_hosted_image name: 'xfedora-30' source_image: '{{user `FEDORA_BASE_IMAGE`}}' + source_image_family: 'fedora-base' - <<: *gce_hosted_image name: 'fedora-29' source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}' + source_image_family: 'prior-fedora-base' # The brains of the operation, making actual modifications to the base-image. provisioners: diff --git a/contrib/cirrus/packer/prior_fedora_base-setup.sh b/contrib/cirrus/packer/prior-fedora_base-setup.sh index 998a5d9fd..998a5d9fd 120000 --- a/contrib/cirrus/packer/prior_fedora_base-setup.sh +++ b/contrib/cirrus/packer/prior-fedora_base-setup.sh diff --git a/contrib/cirrus/packer/systemd_banish.sh b/contrib/cirrus/packer/systemd_banish.sh new file mode 100755 index 000000000..396cf906c --- /dev/null +++ b/contrib/cirrus/packer/systemd_banish.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set +e # Not all of these exist on every platform + +# This is intended to be executed on VMs as a startup script on initial-boot. +# Alternativly, it may be executed with the '--list' option to return the list +# of systemd units defined for disablement (useful for testing). + +EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean" + +if [[ "$1" == "--list" ]] +then + echo "$EVIL_UNITS" + exit 0 +fi + +echo "Disabling periodic services that could destabilize testing:" +for unit in $EVIL_UNITS +do + echo "Banishing $unit (ignoring errors)" + ( + sudo systemctl stop $unit + sudo systemctl disable $unit + sudo systemctl disable $unit.timer + sudo systemctl mask $unit + sudo systemctl mask $unit.timer + ) &> /dev/null +done diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index 00d92570f..da7d457a5 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -20,18 +20,25 @@ systemd_banish echo "Updating/configuring package repositories." $LILTO $SUDOAPTGET update + +echo "Upgrading all packages" +$BIGTO $SUDOAPTGET upgrade + +echo "Adding PPAs" $LILTO $SUDOAPTGET install software-properties-common -$LILTO $SUDOAPTADD ppa:longsleep/golang-backports $LILTO $SUDOAPTADD ppa:projectatomic/ppa $LILTO $SUDOAPTADD ppa:criu/ppa +if [[ "$OS_RELEASE_VER" -eq "18" ]] +then + $LILTO $SUDOAPTADD ppa:longsleep/golang-backports +fi -echo "Upgrading all packages" $LILTO $SUDOAPTGET update -$BIGTO $SUDOAPTGET upgrade echo "Installing general testing and system dependencies" $BIGTO $SUDOAPTGET install \ apparmor \ + aufs-tools \ autoconf \ automake \ bats \ @@ -46,6 +53,7 @@ $BIGTO $SUDOAPTGET install \ e2fslibs-dev \ emacs-nox \ gawk \ + gcc \ gettext \ go-md2man \ golang \ @@ -58,6 +66,7 @@ $BIGTO $SUDOAPTGET install \ libdevmapper-dev \ libdevmapper1.02.1 \ libfuse-dev \ + libfuse2 \ libglib2.0-dev \ libgpgme11-dev \ liblzma-dev \ @@ -66,7 +75,7 @@ $BIGTO $SUDOAPTGET install \ libnl-3-dev \ libostree-dev \ libvarlink \ - libprotobuf-c0-dev \ + libprotobuf-c-dev \ libprotobuf-dev \ libseccomp-dev \ libseccomp2 \ @@ -74,6 +83,7 @@ $BIGTO $SUDOAPTGET install \ libtool \ libudev-dev \ lsof \ + make \ netcat \ pkg-config \ podman \ @@ -87,19 +97,28 @@ $BIGTO $SUDOAPTGET install \ python3-psutil \ python3-pytoml \ python3-setuptools \ - slirp4netns \ skopeo \ + slirp4netns \ socat \ unzip \ vim \ xz-utils \ zip -echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting." -SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g' -ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* -ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub -ooe.sh sudo update-grub +if [[ "$OS_RELEASE_VER" -ge "19" ]] +then + echo "Installing Ubuntu > 18 packages" + $LILTO $SUDOAPTGET install fuse3 libfuse3-dev libbtrfs-dev +fi + +if [[ "$OS_RELEASE_VER" -eq "18" ]] +then + echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting." + SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g' + ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* + ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub + ooe.sh sudo update-grub +fi sudo /tmp/libpod/hack/install_catatonit.sh ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo diff --git a/contrib/cirrus/podbot.py b/contrib/cirrus/podbot.py index 1be41a8ed..9ca4915a7 100755 --- a/contrib/cirrus/podbot.py +++ b/contrib/cirrus/podbot.py @@ -12,7 +12,7 @@ import sys class IRC: - response_timeout = 10 # seconds + response_timeout = 30 # seconds irc = socket.socket() def __init__(self, server, nickname, channel): @@ -90,9 +90,16 @@ class IRC: if len(sys.argv) < 3: print("Error: Must pass desired nick and message as parameters") else: - irc = IRC("irc.freenode.net", sys.argv[1], "#podman") - err = irc.connect(*os.environ.get('IRCID', 'Big Bug').split(" ", 2)) - if not err: + for try_again in (True,False): + irc = IRC("irc.freenode.net", sys.argv[1], "#podman") + err = irc.connect(*os.environ.get('IRCID', 'Big Bug').split(" ", 2)) + if err and try_again: + print("Trying again in 5 seconds...") + time.sleep(5) + continue + elif err: + break irc.message(" ".join(sys.argv[2:])) time.sleep(5.0) # avoid join/quit spam irc.quit() + break diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index c1a01d6ab..df510deef 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -33,8 +33,8 @@ done # contrib/cirrus/packer/*_setup.sh to be incorporated into VM cache-images # (see docs). cd "${GOSRC}/" -case "${OS_REL_VER}" in - ubuntu-18) +case "${OS_RELEASE_ID}" in + ubuntu) CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc" if dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH" then @@ -42,14 +42,13 @@ case "${OS_REL_VER}" in ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc" fi ;; - fedora-30) ;& # continue to next item - fedora-29) - # All SELinux distros need this for systemd-in-a-container - setsebool container_manage_cgroup true + fedora) + # All SELinux distros need this for systemd-in-a-container + setsebool container_manage_cgroup true if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then bash "$SCRIPT_BASE/add_second_partition.sh"; fi ;; - centos-7) # Current VM is an image-builder-image no local podman/testing + centos) # Current VM is an image-builder-image no local podman/testing echo "No further setup required for VM image building" exit 0 ;; diff --git a/contrib/cirrus/success.sh b/contrib/cirrus/success.sh index 30d375d95..3b171757f 100755 --- a/contrib/cirrus/success.sh +++ b/contrib/cirrus/success.sh @@ -4,7 +4,7 @@ set -e source $(dirname $0)/lib.sh -req_env_var CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_REPO_FULL_NAME CIRRUS_BASE_SHA CIRRUS_CHANGE_IN_REPO +req_env_var CIRRUS_BRANCH CIRRUS_REPO_FULL_NAME CIRRUS_BASE_SHA CIRRUS_CHANGE_IN_REPO CIRRUS_CHANGE_MESSAGE cd $CIRRUS_WORKING_DIR @@ -18,18 +18,21 @@ then then SHARANGE="${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}" EXCLUDE_RE='merge-robot' + EMAILCSET='[:alnum:]-+_@.' AUTHOR_NICKS=$(egrep -v '(^[[:space:]]*$)|(^[[:space:]]*#)' "$AUTHOR_NICKS_FILEPATH" | sort -u) # Depending on branch-state, it's possible SHARANGE could be _WAY_ too big MAX_NICKS=10 # newline separated GITLOG="git log --format='%ae'" - COMMIT_AUTHORS=$($GITLOGt $SHARANGE || $GITLOG -1 HEAD | \ - sort -u | \ + COMMIT_AUTHORS=$($GITLOG $SHARANGE || $GITLOG -1 HEAD | \ + tr --delete --complement "$EMAILCSET[:space:]" | \ egrep -v "$EXCLUDE_RE" | \ + sort -u | \ tail -$MAX_NICKS) for c_email in $COMMIT_AUTHORS do + c_email=$(echo "$c_email" | tr --delete --complement "$EMAILCSET") echo -e "\tExamining $c_email" NICK=$(echo "$AUTHOR_NICKS" | grep -m 1 "$c_email" | \ awk --field-separator ',' '{print $2}' | tr -d '[[:blank:]]') @@ -40,8 +43,13 @@ then echo -e "\t\tNot found in $(basename $AUTHOR_NICKS_FILEPATH), using e-mail username." NICK=$(echo "$c_email" | cut -d '@' -f 1) fi - echo -e "\tUsing nick $NICK" - NICKS="${NICKS:+$NICKS, }$NICK" + if ! echo "$NICKS" | grep -q "$NICK" + then + echo -e "\tUsing nick $NICK" + NICKS="${NICKS:+$NICKS, }$NICK" + else + echo -e "\tNot re-adding duplicate nick $NICK" + fi done fi diff --git a/contrib/cirrus/upload_release_archive.sh b/contrib/cirrus/upload_release_archive.sh index 942255821..25107f0ef 100755 --- a/contrib/cirrus/upload_release_archive.sh +++ b/contrib/cirrus/upload_release_archive.sh @@ -9,6 +9,7 @@ req_env_var CI UPLDREL_IMAGE CIRRUS_BUILD_ID GOSRC RELEASE_GCPJSON RELEASE_GCPNA [[ "$CI" == "true" ]] || \ die 56 "$0 must be run under Cirrus-CI to function" +# We store "releases" for each PR, mostly to validate the process is functional unset PR_OR_BRANCH BUCKET if [[ -n "$CIRRUS_PR" ]] then @@ -22,31 +23,76 @@ else die 1 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty." fi -# Functional local podman required for uploading a release +echo "Parsing actual_release.txt contents: $(< actual_release.txt)" cd $GOSRC +RELEASETXT=$(<actual_release.txt) # see build_release.sh +[[ -n "$RELEASETXT" ]] || \ + die 3 "Could not obtain metadata from actual_release.txt" +RELEASE_INFO=$(echo "$RELEASETXT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/') +if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]] +then + die 4 "Metadata is empty or invalid: '$RELEASETXT'" +fi +# Format specified in Makefile +# e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64 +# or libpod-remote v1.3.1-166-g60df124e windows - amd64 +FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH" +read $FIELDS <<< $RELEASE_INFO +req_env_var $FIELDS + +# Functional local podman required for uploading +echo "Verifying a local, functional podman, building one if necessary." [[ -n "$(type -P podman)" ]] || \ - make install || \ + make install PREFIX=/usr || \ die 57 "$0 requires working podman binary on path to function" TMPF=$(mktemp -p '' $(basename $0)_XXXX.json) trap "rm -f $TMPF" EXIT set +x echo "$RELEASE_GCPJSON" > "$TMPF" +[[ "$OS_RELEASE_ID" == "ubuntu" ]] || \ + chcon -t container_file_t "$TMPF" unset RELEASE_GCPJSON cd $GOSRC -for filename in $(ls -1 *.tar.gz *.zip) +for filename in $(ls -1 *.tar.gz *.zip *.msi) do - echo "Running podman ... $UPLDREL_IMAGE $filename" + unset EXT + EXT=$(echo "$filename" | sed -r -e 's/.+\.(.+$)/\1/g') + if [[ -z "$EXT" ]] || [[ "$EXT" == "$filename" ]] + then + echo "Warning: Not processing $filename (invalid extension '$EXT')" + continue + fi + + [[ "$OS_RELEASE_ID" == "ubuntu" ]] || \ + chcon -t container_file_t "$filename" + # Form the generic "latest" file for this branch or pr + TO_PREFIX="${RELEASE_BASENAME}-latest-${PR_OR_BRANCH}-${RELEASE_DIST}" + # Form the fully-versioned filename for historical sake + ALSO_PREFIX="${RELEASE_BASENAME}-${RELEASE_VERSION}-${PR_OR_BRANCH}-${RELEASE_DIST}" + TO_SUFFIX="${RELEASE_ARCH}.${EXT}" + if [[ "$RELEASE_DIST" == "windows" ]] || [[ "$RELEASE_DIST" == "darwin" ]] + then + TO_FILENAME="${TO_PREFIX}-${TO_SUFFIX}" + ALSO_FILENAME="${ALSO_PREFIX}-${TO_SUFFIX}" + else + TO_FILENAME="${TO_PREFIX}-${RELEASE_DIST_VER}-${TO_SUFFIX}" + ALSO_FILENAME="${ALSO_PREFIX}-${TO_SUFFIX}" + fi + + echo "Running podman ... $UPLDREL_IMAGE for $filename -> $TO_FILENAME" + echo "Warning: upload failures are completely ignored, avoiding any needless holdup of PRs." podman run -i --rm \ -e "GCPNAME=$RELEASE_GCPNAME" \ -e "GCPPROJECT=$RELEASE_GCPROJECT" \ -e "GCPJSON_FILEPATH=$TMPF" \ - -e "REL_ARC_FILEPATH=/tmp/$filename" \ + -e "FROM_FILEPATH=/tmp/$filename" \ + -e "TO_FILENAME=$TO_FILENAME" \ + -e "ALSO_FILENAME=$ALSO_FILENAME" \ -e "PR_OR_BRANCH=$PR_OR_BRANCH" \ -e "BUCKET=$BUCKET" \ - --security-opt label=disable \ -v "$TMPF:$TMPF:ro" \ - -v "$GOSRC/$filename:/tmp/$filename:ro" \ - $UPLDREL_IMAGE + -v "$(realpath $GOSRC/$filename):/tmp/$filename:ro" \ + $UPLDREL_IMAGE || true done diff --git a/contrib/perftest/main.go b/contrib/perftest/main.go index f6c90914a..9b928a6b3 100644 --- a/contrib/perftest/main.go +++ b/contrib/perftest/main.go @@ -36,6 +36,9 @@ var helpMessage = ` ` func main() { + if reexec.Init() { + return + } ctx := context.Background() imageName := "" @@ -51,10 +54,6 @@ func main() { flag.Parse() - if reexec.Init() { - return - } - switch strings.ToLower(*logLevel) { case "error": logrus.SetLevel(logrus.ErrorLevel) diff --git a/contrib/upldrel/entrypoint.sh b/contrib/upldrel/entrypoint.sh index 985b828a0..6af6829c5 100755 --- a/contrib/upldrel/entrypoint.sh +++ b/contrib/upldrel/entrypoint.sh @@ -4,59 +4,22 @@ set -e source /usr/local/bin/lib_entrypoint.sh -req_env_var GCPJSON_FILEPATH GCPNAME GCPPROJECT REL_ARC_FILEPATH PR_OR_BRANCH BUCKET +req_env_var GCPJSON_FILEPATH GCPNAME GCPPROJECT BUCKET FROM_FILEPATH TO_FILENAME ALSO_FILENAME -[[ -r "$REL_ARC_FILEPATH" ]] || \ +[[ -r "$FROM_FILEPATH" ]] || \ die 2 ERROR Cannot read release archive file: "$REL_ARC_FILEPATH" [[ -r "$GCPJSON_FILEPATH" ]] || \ die 3 ERROR Cannot read GCP credentials file: "$GCPJSON_FILEPATH" -cd $TMPDIR -echo "Attempting to extract release.txt from tar or zip $REL_ARC_FILEPATH" -unset SFX -if tar xzf "$REL_ARC_FILEPATH" "./release.txt" -then - echo "It's a tarball" - SFX="tar.gz" -elif unzip "$REL_ARC_FILEPATH" release.txt -then - echo "It's a zip" - SFX="zip" -else - die 5 ERROR Could not extract release.txt from $REL_ARC_FILEPATH -fi - -echo "Parsing release.txt contents" -RELEASETXT=$(<release.txt) -cd - -[[ -n "$RELEASETXT" ]] || \ - die 3 ERROR Could not obtain metadata from release.txt in $REL_ARC_FILEPATH - -RELEASE_INFO=$(echo "$RELEASETXT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/') -if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]] -then - die 4 ERROR Metadata is empty or invalid: '$RELEASETXT' -fi - -# e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64 -# or libpod v1.3.1-166-g60df124e amd64 -FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH" -read $FIELDS <<< $RELEASE_INFO -for f in $FIELDS -do - [[ -n "${!f}" ]] || \ - die 5 ERROR Expecting $f to be non-empty in metadata: '$RELEASE_INFO' -done - +echo "Authenticating to google cloud for upload" gcloud_init "$GCPJSON_FILEPATH" -# Drop version number to enable "latest" representation -# (version available w/in zip-file comment) -RELEASE_ARCHIVE_NAME="${RELEASE_BASENAME}-${PR_OR_BRANCH}-${RELEASE_DIST}-${RELEASE_DIST_VER}-${RELEASE_ARCH}.${SFX}" - -echo "Uploading archive as $RELEASE_ARCHIVE_NAME" -gsutil cp "$REL_ARC_FILEPATH" "gs://$BUCKET/$RELEASE_ARCHIVE_NAME" +echo "Uploading archive as $TO_FILENAME" +gsutil cp "$FROM_FILEPATH" "gs://$BUCKET/$TO_FILENAME" +gsutil cp "$FROM_FILEPATH" "gs://$BUCKET/$ALSO_FILENAME" -echo "Release now available at:" -echo " https://storage.cloud.google.com/$BUCKET/$RELEASE_ARCHIVE_NAME" +echo "." +echo "Release now available for download at:" +echo " https://storage.cloud.google.com/$BUCKET/$TO_FILENAME" +echo " https://storage.cloud.google.com/$BUCKET/$ALSO_FILENAME" diff --git a/docs/podman-cp.1.md b/docs/podman-cp.1.md index 736bdb12a..0f54b2e8b 100644 --- a/docs/podman-cp.1.md +++ b/docs/podman-cp.1.md @@ -29,7 +29,7 @@ Assuming a path separator of /, a first argument of **src_path** and second argu - **dest_path** does not exist - the file is saved to a file created at **dest_path** - **dest_path** does not exist and ends with / - - **dest_path** is created as a directory and the file is copied into this directory using the basename from **src_path** + - Error condition: the destination directory must exist. - **dest_path** exists and is a file - the destination is overwritten with the source file's contents - **dest_path** exists and is a directory diff --git a/docs/podman-derivative-api.md b/docs/podman-derivative-api.md new file mode 100644 index 000000000..0342bb740 --- /dev/null +++ b/docs/podman-derivative-api.md @@ -0,0 +1,44 @@ +# How to use libpod for custom/derivative projects + +libpod today is a Golang library and a CLI. The choice of interface you make has advantages and disadvantages. + +Running as a subprocess +--- + +Advantages: + + - Many commands output JSON + - Works with languages other than Golang + - Easy to get started + +Disadvantages: + + - Error handling is harder + - May be slower + - Can't hook into or control low-level things like how images are pulled + +Vendoring into a Go project +--- + +Advantages: + + - Significant power and control + +Disadvantages: + + - You are now on the hook for container runtime security updates (partially, `runc`/`crun` are separate) + - Binary size + - Potential skew between multiple libpod versions operating on the same storage can cause problems + +Varlink +--- + +Some code exists for this; splits the difference. Future uncertain. + +Making the choice +--- + +A good question to ask first is: Do you want users to be able to use `podman` to manipulate the containers created by your project? +If so, that makes it more likely that you want to run `podman` as a subprocess. If you want a separate image store and a fundamentally +different experience; if what you're doing with containers is quite different from those created by the `podman` CLI, +that may drive you towards vendoring.
\ No newline at end of file diff --git a/docs/podman-events.1.md b/docs/podman-events.1.md index a5a715098..bb1923574 100644 --- a/docs/podman-events.1.md +++ b/docs/podman-events.1.md @@ -11,7 +11,9 @@ podman\-events - Monitor Podman events Monitor and print events that occur in Podman. Each event will include a timestamp, a type, a status, name (if applicable), and image (if applicable). The default logging mechanism is *journald*. This can be changed in libpod.conf by changing the `events_logger` -value to `file`. Only `file` and `journald` are the accepted. +value to `file`. Only `file` and `journald` are accepted. A `none` logger is also +available but this logging mechanism completely disables events; nothing will be reported by +`podman events`. The *container* event type will report the follow statuses: * attach @@ -54,6 +56,10 @@ The *image* event type will report the following statuses: * tag * untag +The *system* type will report the following statuses: + * refresh + * renumber + The *volume* type will report the following statuses: * create * prune diff --git a/docs/tutorials/README.md b/docs/tutorials/README.md index c340d683f..2cf9613b6 100644 --- a/docs/tutorials/README.md +++ b/docs/tutorials/README.md @@ -4,18 +4,18 @@ ## Links to a number of useful tutorials for the Podman utility. -**[Introduction Tutorial](https://github.com/containers/libpod/tree/master/docs/tutorials/podman_tutorial.md)** +**[Introduction Tutorial](podman_tutorial.md)** Learn how to setup Podman and perform some basic commands with the utility. -**[Basic Setup and Use of Podman in a Rootless environment.](https://github.com/containers/libpod/blob/master/docs/tutorials/rootless_tutorial.md).** +**[Basic Setup and Use of Podman in a Rootless environment](rootless_tutorial.md)** The steps required to setup rootless Podman are enumerated. -**[Setup on OS X](https://github.com/containers/libpod/blob/master/doc/tutorials/mac_client.md)** +**[Setup on OS X](mac_client.md)** -Special setup for running the Podman remote client on a Mac and connecting to Podman running on a Linux VM are documented +Special setup for running the Podman remote client on a Mac and connecting to Podman running on a Linux VM are documented. -**[Remote Client](https://github.com/containers/libpod/blob/master/doc/tutorials/remote_client.md)** +**[Remote Client](remote_client.md)** A brief how-to on using the Podman remote-client. diff --git a/docs/tutorials/podman_tutorial.md b/docs/tutorials/podman_tutorial.md index 559d25d6a..169cefc0e 100644 --- a/docs/tutorials/podman_tutorial.md +++ b/docs/tutorials/podman_tutorial.md @@ -5,7 +5,7 @@ Podman is a utility provided as part of the libpod library. It can be used to c containers. The following tutorial will teach you how to set up Podman and perform some basic commands with Podman. -If you are running on a Mac, you should instead follow the [Mac tutorial](https://github.com/containers/libpod/blob/master/mac_client.md) +If you are running on a Mac, you should instead follow the [Mac tutorial](https://github.com/containers/libpod/blob/master/docs/tutorials/mac_client.md) to set up the remote Podman client. **NOTE**: the code samples are intended to be run as a non-root user, and use `sudo` where diff --git a/hack/get_release_info.sh b/hack/get_release_info.sh index 29b4237b4..b75751170 100755 --- a/hack/get_release_info.sh +++ b/hack/get_release_info.sh @@ -33,7 +33,7 @@ case "$1" in OUTPUT="${GOARCH:-$(go env GOARCH 2> /dev/null)}" ;; BASENAME*) - OUTPUT="${CIRRUS_REPO_NAME:-$(basename $(git rev-parse --show-toplevel))}" + OUTPUT="podman" ;; REMOTENAME*) OUTPUT="$($0 BASENAME)-remote" diff --git a/libpod/container_internal.go b/libpod/container_internal.go index 42028c397..f1456548b 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -622,7 +622,7 @@ func (c *Container) refresh() error { return err } - return nil + return c.refreshCNI() } // Remove conmon attach socket and terminal resize FIFO diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index 4bbbef5db..2636fdb6c 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -78,15 +78,21 @@ func (c *Container) prepare() (Err error) { // Set up network namespace if not already set up if c.config.CreateNetNS && c.state.NetNS == nil && !c.config.PostConfigureNetNS { netNS, networkStatus, createNetNSErr = c.runtime.createNetNS(c) + if createNetNSErr != nil { + return + } tmpStateLock.Lock() defer tmpStateLock.Unlock() // Assign NetNS attributes to container - if createNetNSErr == nil { - c.state.NetNS = netNS - c.state.NetworkStatus = networkStatus - } + c.state.NetNS = netNS + c.state.NetworkStatus = networkStatus + } + + // handle rootless network namespace setup + if c.state.NetNS != nil && c.config.NetMode == "slirp4netns" && !c.config.PostConfigureNetNS { + createNetNSErr = c.runtime.setupRootlessNetNS(c) } }() // Mount storage if not mounted @@ -181,6 +187,30 @@ func (c *Container) cleanupNetwork() error { return nil } +func (c *Container) getUserOverrides() *lookup.Overrides { + var hasPasswdFile, hasGroupFile bool + overrides := lookup.Overrides{} + for _, m := range c.config.Spec.Mounts { + if m.Destination == "/etc/passwd" { + overrides.ContainerEtcPasswdPath = m.Source + hasPasswdFile = true + } + if m.Destination == "/etc/group" { + overrides.ContainerEtcGroupPath = m.Source + hasGroupFile = true + } + if m.Destination == "/etc" { + if !hasPasswdFile { + overrides.ContainerEtcPasswdPath = filepath.Join(m.Source, "passwd") + } + if !hasGroupFile { + overrides.ContainerEtcGroupPath = filepath.Join(m.Source, "group") + } + } + } + return &overrides +} + // Generate spec for a container // Accepts a map of the container's dependencies func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { @@ -188,7 +218,8 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { span.SetTag("type", "container") defer span.Finish() - execUser, err := lookup.GetUserGroupInfo(c.state.Mountpoint, c.config.User, nil) + overrides := c.getUserOverrides() + execUser, err := lookup.GetUserGroupInfo(c.state.Mountpoint, c.config.User, overrides) if err != nil { return nil, err } @@ -279,6 +310,17 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { } } + hasHomeSet := false + for _, s := range c.config.Spec.Process.Env { + if strings.HasPrefix(s, "HOME=") { + hasHomeSet = true + break + } + } + if !hasHomeSet { + c.config.Spec.Process.Env = append(c.config.Spec.Process.Env, fmt.Sprintf("HOME=%s", execUser.Home)) + } + if c.config.User != "" { // User and Group must go together g.SetProcessUID(uint32(execUser.Uid)) @@ -1283,3 +1325,10 @@ func (c *Container) copyOwnerAndPerms(source, dest string) error { } return nil } + +// Teardown CNI config on refresh +func (c *Container) refreshCNI() error { + // Let's try and delete any lingering network config... + podNetwork := c.runtime.getPodNetwork(c.ID(), c.config.Name, "", c.config.Networks, c.config.PortMappings, c.config.StaticIP) + return c.runtime.netPlugin.TearDownPod(podNetwork) +} diff --git a/libpod/container_internal_unsupported.go b/libpod/container_internal_unsupported.go index 6fa19a778..05a587c59 100644 --- a/libpod/container_internal_unsupported.go +++ b/libpod/container_internal_unsupported.go @@ -40,3 +40,7 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti func (c *Container) copyOwnerAndPerms(source, dest string) error { return nil } + +func (c *Container) refreshCNI() error { + return define.ErrNotImplemented +} diff --git a/libpod/image/image.go b/libpod/image/image.go index 0be6eeeb9..855da8611 100644 --- a/libpod/image/image.go +++ b/libpod/image/image.go @@ -29,7 +29,6 @@ import ( "github.com/containers/libpod/pkg/registries" "github.com/containers/libpod/pkg/util" "github.com/containers/storage" - "github.com/containers/storage/pkg/reexec" "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" ociv1 "github.com/opencontainers/image-spec/specs-go/v1" @@ -85,9 +84,6 @@ func NewImageRuntimeFromStore(store storage.Store) *Runtime { // NewImageRuntimeFromOptions creates an Image Runtime including the store given // store options func NewImageRuntimeFromOptions(options storage.StoreOptions) (*Runtime, error) { - if reexec.Init() { - return nil, errors.Errorf("unable to reexec") - } store, err := setStore(options) if err != nil { return nil, err diff --git a/libpod/image/image_test.go b/libpod/image/image_test.go index 5a6d095f6..ef39d09c3 100644 --- a/libpod/image/image_test.go +++ b/libpod/image/image_test.go @@ -11,6 +11,7 @@ import ( "github.com/containers/libpod/libpod/events" "github.com/containers/libpod/pkg/util" "github.com/containers/storage" + "github.com/containers/storage/pkg/reexec" "github.com/opencontainers/go-digest" "github.com/stretchr/testify/assert" ) @@ -70,6 +71,13 @@ func makeLocalMatrix(b, bg *Image) ([]localImageTest, error) { } +func TestMain(m *testing.M) { + if reexec.Init() { + return + } + os.Exit(m.Run()) +} + // TestImage_NewFromLocal tests finding the image locally by various names, // tags, and aliases func TestImage_NewFromLocal(t *testing.T) { diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go index 67dd0150b..d854a2de6 100644 --- a/libpod/networking_linux.go +++ b/libpod/networking_linux.go @@ -90,9 +90,6 @@ func (r *Runtime) configureNetNS(ctr *Container, ctrNS ns.NetNS) ([]*cnitypes.Re // Create and configure a new network namespace for a container func (r *Runtime) createNetNS(ctr *Container) (n ns.NetNS, q []*cnitypes.Result, err error) { - if rootless.IsRootless() { - return nil, nil, errors.New("cannot configure a new network namespace in rootless mode, only --network=slirp4netns is supported") - } ctrNS, err := netns.NewNS() if err != nil { return nil, nil, errors.Wrapf(err, "error creating network namespace for container %s", ctr.ID()) @@ -110,7 +107,10 @@ func (r *Runtime) createNetNS(ctr *Container) (n ns.NetNS, q []*cnitypes.Result, logrus.Debugf("Made network namespace at %s for container %s", ctrNS.Path(), ctr.ID()) - networkStatus, err := r.configureNetNS(ctr, ctrNS) + networkStatus := []*cnitypes.Result{} + if !rootless.IsRootless() { + networkStatus, err = r.configureNetNS(ctr, ctrNS) + } return ctrNS, networkStatus, err } @@ -138,9 +138,6 @@ func checkSlirpFlags(path string) (bool, bool, bool, error) { // Configure the network namespace for a rootless container func (r *Runtime) setupRootlessNetNS(ctr *Container) (err error) { - defer errorhandling.CloseQuiet(ctr.rootlessSlirpSyncR) - defer errorhandling.CloseQuiet(ctr.rootlessSlirpSyncW) - path := r.config.NetworkCmdPath if path == "" { @@ -164,7 +161,7 @@ func (r *Runtime) setupRootlessNetNS(ctr *Container) (err error) { cmdArgs := []string{} if havePortMapping { - cmdArgs = append(cmdArgs, "--api-socket", apiSocket, fmt.Sprintf("%d", ctr.state.PID)) + cmdArgs = append(cmdArgs, "--api-socket", apiSocket) } dhp, mtu, sandbox, err := checkSlirpFlags(path) if err != nil { @@ -179,13 +176,32 @@ func (r *Runtime) setupRootlessNetNS(ctr *Container) (err error) { if sandbox { cmdArgs = append(cmdArgs, "--enable-sandbox") } - cmdArgs = append(cmdArgs, "-c", "-e", "3", "-r", "4", fmt.Sprintf("%d", ctr.state.PID), "tap0") - cmd := exec.Command(path, cmdArgs...) + // the slirp4netns arguments being passed are describes as follows: + // from the slirp4netns documentation: https://github.com/rootless-containers/slirp4netns + // -c, --configure Brings up the tap interface + // -e, --exit-fd=FD specify the FD for terminating slirp4netns + // -r, --ready-fd=FD specify the FD to write to when the initialization steps are finished + cmdArgs = append(cmdArgs, "-c", "-e", "3", "-r", "4") + if !ctr.config.PostConfigureNetNS { + ctr.rootlessSlirpSyncR, ctr.rootlessSlirpSyncW, err = os.Pipe() + if err != nil { + return errors.Wrapf(err, "failed to create rootless network sync pipe") + } + cmdArgs = append(cmdArgs, "--netns-type=path", ctr.state.NetNS.Path(), "tap0") + } else { + defer errorhandling.CloseQuiet(ctr.rootlessSlirpSyncR) + defer errorhandling.CloseQuiet(ctr.rootlessSlirpSyncW) + cmdArgs = append(cmdArgs, fmt.Sprintf("%d", ctr.state.PID), "tap0") + } + cmd := exec.Command(path, cmdArgs...) + logrus.Debugf("slirp4netns command: %s", strings.Join(cmd.Args, " ")) cmd.SysProcAttr = &syscall.SysProcAttr{ Setpgid: true, } + + // Leak one end of the pipe in slirp4netns, the other will be sent to conmon cmd.ExtraFiles = append(cmd.ExtraFiles, ctr.rootlessSlirpSyncR, syncW) if err := cmd.Start(); err != nil { @@ -388,20 +404,22 @@ func (r *Runtime) teardownNetNS(ctr *Container) error { logrus.Debugf("Tearing down network namespace at %s for container %s", ctr.state.NetNS.Path(), ctr.ID()) - var requestedIP net.IP - if ctr.requestedIP != nil { - requestedIP = ctr.requestedIP - // cancel request for a specific IP in case the container is reused later - ctr.requestedIP = nil - } else { - requestedIP = ctr.config.StaticIP - } + // rootless containers do not use the CNI plugin + if !rootless.IsRootless() { + var requestedIP net.IP + if ctr.requestedIP != nil { + requestedIP = ctr.requestedIP + // cancel request for a specific IP in case the container is reused later + ctr.requestedIP = nil + } else { + requestedIP = ctr.config.StaticIP + } - podNetwork := r.getPodNetwork(ctr.ID(), ctr.Name(), ctr.state.NetNS.Path(), ctr.config.Networks, ctr.config.PortMappings, requestedIP) + podNetwork := r.getPodNetwork(ctr.ID(), ctr.Name(), ctr.state.NetNS.Path(), ctr.config.Networks, ctr.config.PortMappings, requestedIP) - // The network may have already been torn down, so don't fail here, just log - if err := r.netPlugin.TearDownPod(podNetwork); err != nil { - return errors.Wrapf(err, "error tearing down CNI namespace configuration for container %s", ctr.ID()) + if err := r.netPlugin.TearDownPod(podNetwork); err != nil { + return errors.Wrapf(err, "error tearing down CNI namespace configuration for container %s", ctr.ID()) + } } // First unmount the namespace diff --git a/libpod/oci_internal_linux.go b/libpod/oci_internal_linux.go index 4df1e4010..a5cce795b 100644 --- a/libpod/oci_internal_linux.go +++ b/libpod/oci_internal_linux.go @@ -131,9 +131,14 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Containe } if ctr.config.NetMode.IsSlirp4netns() { - ctr.rootlessSlirpSyncR, ctr.rootlessSlirpSyncW, err = os.Pipe() - if err != nil { - return errors.Wrapf(err, "failed to create rootless network sync pipe") + if ctr.config.PostConfigureNetNS { + ctr.rootlessSlirpSyncR, ctr.rootlessSlirpSyncW, err = os.Pipe() + if err != nil { + return errors.Wrapf(err, "failed to create rootless network sync pipe") + } + } else { + defer errorhandling.CloseQuiet(ctr.rootlessSlirpSyncR) + defer errorhandling.CloseQuiet(ctr.rootlessSlirpSyncW) } // Leak one end in conmon, the other one will be leaked into slirp4netns cmd.ExtraFiles = append(cmd.ExtraFiles, ctr.rootlessSlirpSyncW) @@ -200,13 +205,16 @@ func prepareProcessExec(c *Container, cmd, env []string, tty bool, cwd, user, se pspec.Cwd = cwd } + + overrides := c.getUserOverrides() + execUser, err := lookup.GetUserGroupInfo(c.state.Mountpoint, user, overrides) + if err != nil { + return nil, err + } + // If user was set, look it up in the container to get a UID to use on // the host if user != "" { - execUser, err := lookup.GetUserGroupInfo(c.state.Mountpoint, user, nil) - if err != nil { - return nil, err - } sgids := make([]uint32, 0, len(execUser.Sgids)) for _, sgid := range execUser.Sgids { sgids = append(sgids, uint32(sgid)) @@ -220,6 +228,17 @@ func prepareProcessExec(c *Container, cmd, env []string, tty bool, cwd, user, se pspec.User = processUser } + hasHomeSet := false + for _, s := range pspec.Env { + if strings.HasPrefix(s, "HOME=") { + hasHomeSet = true + break + } + } + if !hasHomeSet { + pspec.Env = append(pspec.Env, fmt.Sprintf("HOME=%s", execUser.Home)) + } + processJSON, err := json.Marshal(pspec) if err != nil { return nil, err diff --git a/libpod/runtime.go b/libpod/runtime.go index e2b9667be..675c92b7a 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -23,6 +23,7 @@ import ( "github.com/containers/libpod/libpod/events" "github.com/containers/libpod/libpod/image" "github.com/containers/libpod/libpod/lock" + "github.com/containers/libpod/pkg/cgroups" sysreg "github.com/containers/libpod/pkg/registries" "github.com/containers/libpod/pkg/rootless" "github.com/containers/libpod/pkg/util" @@ -254,6 +255,8 @@ type RuntimeConfig struct { // SDNotify tells Libpod to allow containers to notify the host // systemd of readiness using the SD_NOTIFY mechanism SDNotify bool + // CgroupCheck verifies if the cgroup check for correct OCI runtime has been done. + CgroupCheck bool `toml:"cgroup_check,omitempty"` } // runtimeConfiguredFrom is a struct used during early runtime init to help @@ -575,6 +578,10 @@ func newRuntimeFromConfig(ctx context.Context, userConfigPath string, options .. configPath) } + if err := cgroupV2Check(configPath, tmpConfig); err != nil { + return nil, err + } + if tmpConfig.StaticDir != "" { runtime.configuredFrom.libpodStaticDirSet = true } @@ -664,6 +671,14 @@ func newRuntimeFromConfig(ctx context.Context, userConfigPath string, options .. runtime.config.OCIRuntime = tmpConfig.OCIRuntime } + cgroupsV2, err := cgroups.IsCgroup2UnifiedMode() + if err != nil { + return nil, err + } + if cgroupsV2 { + runtime.config.CgroupCheck = true + } + break } } @@ -1451,3 +1466,37 @@ func (r *Runtime) ImageRuntime() *image.Runtime { func (r *Runtime) SystemContext() *types.SystemContext { return r.imageContext } + +// Since runc does not currently support cgroupV2 +// Change to default crun on first running of libpod.conf +// TODO Once runc has support for cgroups, this function should be removed. +func cgroupV2Check(configPath string, tmpConfig *RuntimeConfig) error { + if !tmpConfig.CgroupCheck && rootless.IsRootless() { + cgroupsV2, err := cgroups.IsCgroup2UnifiedMode() + if err != nil { + return err + } + if cgroupsV2 { + path, err := exec.LookPath("crun") + if err != nil { + logrus.Warnf("Can not find crun package on the host, containers might fail to run on cgroup V2 systems without crun: %q", err) + // Can't find crun path so do nothing + return nil + } + tmpConfig.CgroupCheck = true + tmpConfig.OCIRuntime = path + file, err := os.OpenFile(configPath, os.O_RDWR|os.O_CREATE, 0666) + if err != nil { + return errors.Wrapf(err, "cannot open file %s", configPath) + } + defer file.Close() + enc := toml.NewEncoder(file) + if err := enc.Encode(tmpConfig); err != nil { + if removeErr := os.Remove(configPath); removeErr != nil { + logrus.Debugf("unable to remove %s: %q", configPath, err) + } + } + } + } + return nil +} diff --git a/libpod/runtime_pod_infra_linux.go b/libpod/runtime_pod_infra_linux.go index ad6662f03..6a27c2800 100644 --- a/libpod/runtime_pod_infra_linux.go +++ b/libpod/runtime_pod_infra_linux.go @@ -99,7 +99,9 @@ func (r *Runtime) makeInfraContainer(ctx context.Context, p *Pod, imgName, imgID if isRootless { netmode = "slirp4netns" } - options = append(options, WithNetNS(p.config.InfraContainer.PortBindings, isRootless, netmode, networks)) + // PostConfigureNetNS should not be set since user namespace sharing is not implemented + // and rootless networking no longer supports post configuration setup + options = append(options, WithNetNS(p.config.InfraContainer.PortBindings, false, netmode, networks)) return r.newContainer(ctx, g.Config, options...) } diff --git a/pkg/adapter/containers_remote.go b/pkg/adapter/containers_remote.go index 01e008e87..6cecb92da 100644 --- a/pkg/adapter/containers_remote.go +++ b/pkg/adapter/containers_remote.go @@ -473,7 +473,12 @@ func (r *LocalRuntime) Run(ctx context.Context, c *cliconfig.RunValues, exitCode fmt.Println(cid) return 0, nil } - exitChan, errChan, err := r.attach(ctx, os.Stdin, os.Stdout, cid, true, c.String("detach-keys")) + inputStream := os.Stdin + // If -i is not set, clear stdin + if !c.Bool("interactive") { + inputStream = nil + } + exitChan, errChan, err := r.attach(ctx, inputStream, os.Stdout, cid, true, c.String("detach-keys")) if err != nil { return exitCode, err } diff --git a/pkg/adapter/pods.go b/pkg/adapter/pods.go index 70293a2c5..c8d57e2a2 100644 --- a/pkg/adapter/pods.go +++ b/pkg/adapter/pods.go @@ -467,8 +467,15 @@ func (r *LocalRuntime) PlayKubeYAML(ctx context.Context, c *cliconfig.KubePlayVa return nil, errors.Wrapf(err, "unable to read %s as YAML", yamlFile) } + if podYAML.Kind != "Pod" { + return nil, errors.Errorf("Invalid YAML kind: %s. Pod is the only supported Kubernetes YAML kind", podYAML.Kind) + } + // check for name collision between pod and container podName := podYAML.ObjectMeta.Name + if podName == "" { + return nil, errors.Errorf("pod does not have a name") + } for _, n := range podYAML.Spec.Containers { if n.Name == podName { fmt.Printf("a container exists with the same name (%s) as the pod in your YAML file; changing pod name to %s_pod\n", podName, podName) diff --git a/pkg/lookup/lookup.go b/pkg/lookup/lookup.go index 70b97144f..a249dd753 100644 --- a/pkg/lookup/lookup.go +++ b/pkg/lookup/lookup.go @@ -29,17 +29,30 @@ func GetUserGroupInfo(containerMount, containerUser string, override *Overrides) defaultExecUser *user.ExecUser err error ) - passwdPath := etcpasswd - groupPath := etcgroup if override != nil { // Check for an override /etc/passwd path if override.ContainerEtcPasswdPath != "" { - passwdPath = override.ContainerEtcPasswdPath + passwdDest = override.ContainerEtcPasswdPath } // Check for an override for /etc/group path if override.ContainerEtcGroupPath != "" { - groupPath = override.ContainerEtcGroupPath + groupDest = override.ContainerEtcGroupPath + } + } + + if passwdDest == "" { + // Make sure the /etc/passwd destination is not a symlink to something naughty + if passwdDest, err = securejoin.SecureJoin(containerMount, etcpasswd); err != nil { + logrus.Debug(err) + return nil, err + } + } + if groupDest == "" { + // Make sure the /etc/group destination is not a symlink to something naughty + if groupDest, err = securejoin.SecureJoin(containerMount, etcgroup); err != nil { + logrus.Debug(err) + return nil, err } } @@ -56,15 +69,6 @@ func GetUserGroupInfo(containerMount, containerUser string, override *Overrides) } - // Make sure the /etc/group and /etc/passwd destinations are not a symlink to something naughty - if passwdDest, err = securejoin.SecureJoin(containerMount, passwdPath); err != nil { - logrus.Debug(err) - return nil, err - } - if groupDest, err = securejoin.SecureJoin(containerMount, groupPath); err != nil { - logrus.Debug(err) - return nil, err - } return user.GetExecUserPath(containerUser, defaultExecUser, passwdDest, groupDest) } diff --git a/pkg/netns/netns_linux.go b/pkg/netns/netns_linux.go index 1d6fb873c..a62296549 100644 --- a/pkg/netns/netns_linux.go +++ b/pkg/netns/netns_linux.go @@ -23,23 +23,42 @@ import ( "fmt" "os" "path" + "path/filepath" "runtime" "strings" "sync" "github.com/containernetworking/plugins/pkg/ns" + "github.com/containers/libpod/pkg/rootless" + "github.com/containers/libpod/pkg/util" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) -const nsRunDir = "/var/run/netns" +// get NSRunDir returns the dir of where to create the netNS. When running +// rootless, it needs to be at a location writable by user. +func getNSRunDir() (string, error) { + if rootless.IsRootless() { + rootlessDir, err := util.GetRuntimeDir() + if err != nil { + return "", err + } + return filepath.Join(rootlessDir, "netns"), nil + } + return "/var/run/netns", nil +} // NewNS creates a new persistent (bind-mounted) network namespace and returns // an object representing that namespace, without switching to it. func NewNS() (ns.NetNS, error) { + nsRunDir, err := getNSRunDir() + if err != nil { + return nil, err + } + b := make([]byte, 16) - _, err := rand.Reader.Read(b) + _, err = rand.Reader.Read(b) if err != nil { return nil, fmt.Errorf("failed to generate random netns name: %v", err) } @@ -127,14 +146,15 @@ func NewNS() (ns.NetNS, error) { // Put this thread back to the orig ns, since it might get reused (pre go1.10) defer func() { if err := origNS.Set(); err != nil { - logrus.Errorf("unable to set namespace: %q", err) + logrus.Warnf("unable to set namespace: %q", err) } }() // bind mount the netns from the current thread (from /proc) onto the // mount point. This causes the namespace to persist, even when there - // are no threads in the ns. - err = unix.Mount(getCurrentThreadNetNSPath(), nsPath, "none", unix.MS_BIND, "") + // are no threads in the ns. Make this a shared mount; it needs to be + // back-propogated to the host + err = unix.Mount(getCurrentThreadNetNSPath(), nsPath, "none", unix.MS_BIND|unix.MS_SHARED|unix.MS_REC, "") if err != nil { err = fmt.Errorf("failed to bind mount ns at %s: %v", nsPath, err) } @@ -150,6 +170,11 @@ func NewNS() (ns.NetNS, error) { // UnmountNS unmounts the NS held by the netns object func UnmountNS(ns ns.NetNS) error { + nsRunDir, err := getNSRunDir() + if err != nil { + return err + } + nsPath := ns.Path() // Only unmount if it's been bind-mounted (don't touch namespaces in /proc...) if strings.HasPrefix(nsPath, nsRunDir) { diff --git a/pkg/rootless/rootless_linux.go b/pkg/rootless/rootless_linux.go index ecb84f6a9..6f6239e5f 100644 --- a/pkg/rootless/rootless_linux.go +++ b/pkg/rootless/rootless_linux.go @@ -365,7 +365,7 @@ func GetConfiguredMappings() ([]idtools.IDMap, []idtools.IDMap, error) { } mappings, err := idtools.NewIDMappings(username, username) if err != nil { - logrus.Warnf("cannot find mappings for user %s: %v", username, err) + logrus.Errorf("cannot find mappings for user %s: %v", username, err) } else { uids = mappings.UIDs() gids = mappings.GIDs() diff --git a/pkg/spec/createconfig.go b/pkg/spec/createconfig.go index c17172016..7c3195be4 100644 --- a/pkg/spec/createconfig.go +++ b/pkg/spec/createconfig.go @@ -275,7 +275,7 @@ func (c *CreateConfig) getContainerCreateOptions(runtime *libpod.Runtime, pod *l options = append(options, libpod.WithNetNSFrom(connectedCtr)) } else if !c.NetMode.IsHost() && !c.NetMode.IsNone() { hasUserns := c.UsernsMode.IsContainer() || c.UsernsMode.IsNS() || len(c.IDMappings.UIDMap) > 0 || len(c.IDMappings.GIDMap) > 0 - postConfigureNetNS := c.NetMode.IsSlirp4netns() || (hasUserns && !c.UsernsMode.IsHost()) + postConfigureNetNS := hasUserns && !c.UsernsMode.IsHost() options = append(options, libpod.WithNetNS(portBindings, postConfigureNetNS, string(c.NetMode), networks)) } diff --git a/pkg/spec/storage.go b/pkg/spec/storage.go index 3d59d70d8..93919dd0a 100644 --- a/pkg/spec/storage.go +++ b/pkg/spec/storage.go @@ -168,6 +168,9 @@ func (config *CreateConfig) parseVolumes(runtime *libpod.Runtime) ([]spec.Mount, if _, ok := baseMounts[dest]; ok { continue } + if _, ok := baseVolumes[dest]; ok { + continue + } localOpts := options if dest == "/run" { localOpts = append(localOpts, "noexec", "size=65536k") diff --git a/pkg/varlinkapi/attach.go b/pkg/varlinkapi/attach.go index 3bd487849..f8557ae0c 100644 --- a/pkg/varlinkapi/attach.go +++ b/pkg/varlinkapi/attach.go @@ -65,7 +65,9 @@ func (i *LibpodAPI) Attach(call iopodman.VarlinkCall, name string, detachKeys st } // ACK the client upgrade request - call.ReplyAttach() + if err := call.ReplyAttach(); err != nil { + return call.ReplyErrorOccurred(err.Error()) + } reader, writer, _, pw, streams := setupStreams(call) diff --git a/rootless.md b/rootless.md index 53463dccc..8cccb86eb 100644 --- a/rootless.md +++ b/rootless.md @@ -27,7 +27,6 @@ can easily fail * Can not use overlayfs driver, but does support fuse-overlayfs * Ubuntu supports non root overlay, but no other Linux distros do. * Only other supported driver is VFS. -* No KATA Container support * No CNI Support * CNI wants to modify IPTables, plus other network manipulation that requires CAP_SYS_ADMIN. * There is potential we could probably do some sort of blacklisting of the relevant plugins, and add a new plugin for rootless networking - slirp4netns as one example and there may be others diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go index 4e9881d59..1c2acf8ed 100644 --- a/test/e2e/common_test.go +++ b/test/e2e/common_test.go @@ -78,11 +78,15 @@ func (a testResultsSorted) Less(i, j int) bool { return a[i].length < a[j].lengt var testResults []testResult -// TestLibpod ginkgo master function -func TestLibpod(t *testing.T) { +func TestMain(m *testing.M) { if reexec.Init() { - os.Exit(1) + return } + os.Exit(m.Run()) +} + +// TestLibpod ginkgo master function +func TestLibpod(t *testing.T) { if os.Getenv("NOCACHE") == "1" { CACHE_IMAGES = []string{} RESTORE_IMAGES = []string{} diff --git a/test/e2e/cp_test.go b/test/e2e/cp_test.go index 9b0cb757d..3317683de 100644 --- a/test/e2e/cp_test.go +++ b/test/e2e/cp_test.go @@ -51,6 +51,10 @@ var _ = Describe("Podman cp", func() { err := ioutil.WriteFile(srcPath, fromHostToContainer, 0644) Expect(err).To(BeNil()) + session = podmanTest.Podman([]string{"cp", srcPath, name + ":foo/"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Not(Equal(0))) + session = podmanTest.Podman([]string{"cp", srcPath, name + ":foo"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -65,30 +69,29 @@ var _ = Describe("Podman cp", func() { }) It("podman cp file to dir", func() { - srcPath := filepath.Join(podmanTest.RunRoot, "cp_test.txt") - dstDir := filepath.Join(podmanTest.RunRoot, "receive") + name := "testctr" + setup := podmanTest.RunTopContainer(name) + setup.WaitWithDefaultTimeout() + Expect(setup.ExitCode()).To(Equal(0)) + + srcPath := "/tmp/cp_test.txt" fromHostToContainer := []byte("copy from host to container directory") + err := ioutil.WriteFile(srcPath, fromHostToContainer, 0644) + Expect(err).To(BeNil()) - session := podmanTest.Podman([]string{"create", ALPINE, "ls", "foodir/"}) + session := podmanTest.Podman([]string{"exec", name, "mkdir", "foodir"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - name := session.OutputToString() - - err := ioutil.WriteFile(srcPath, fromHostToContainer, 0644) - Expect(err).To(BeNil()) - err = os.Mkdir(dstDir, 0755) - Expect(err).To(BeNil()) session = podmanTest.Podman([]string{"cp", srcPath, name + ":foodir/"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"cp", name + ":foodir/cp_test.txt", dstDir}) + session = podmanTest.Podman([]string{"exec", name, "ls", "foodir/cp_test.txt"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - os.Remove("cp_test.txt") - os.RemoveAll("receive") + os.Remove("/tmp/cp_test.txt") }) It("podman cp dir to dir", func() { @@ -137,10 +140,18 @@ var _ = Describe("Podman cp", func() { session = podmanTest.Podman([]string{"cp", name + ":/foo.tar.gz", "-"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) + + os.RemoveAll(testDirPath) + os.Remove("file.tar.gz") }) It("podman cp tar", func() { - session := podmanTest.Podman([]string{"create", "--name", "testctr", ALPINE, "ls", "-l", "foo"}) + testctr := "testctr" + setup := podmanTest.RunTopContainer(testctr) + setup.WaitWithDefaultTimeout() + Expect(setup.ExitCode()).To(Equal(0)) + + session := podmanTest.Podman([]string{"exec", testctr, "mkdir", "foo"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -157,7 +168,7 @@ var _ = Describe("Podman cp", func() { session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - session = podmanTest.Podman([]string{"start", "-a", "testctr"}) + session = podmanTest.Podman([]string{"exec", testctr, "ls", "-l", "foo"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(session.OutputToString()).To(ContainSubstring("file.tar")) @@ -187,6 +198,15 @@ var _ = Describe("Podman cp", func() { _, err = os.Stat("/tmp/cp_test.txt") Expect(err).To(Not(BeNil())) + + session = podmanTest.Podman([]string{"exec", name, "ln", "-s", "/tmp/nonesuch", "/test1"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"cp", "--pause=false", srcPath, name + ":/test1/"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Not(Equal(0))) + }) It("podman cp volume", func() { session := podmanTest.Podman([]string{"volume", "create", "data"}) @@ -208,6 +228,9 @@ var _ = Describe("Podman cp", func() { session = podmanTest.Podman([]string{"cp", "container1" + ":/data/cp_vol1", "cp_vol2"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) + + os.Remove("cp_vol") + os.Remove("cp_vol2") }) It("podman cp from ctr chown ", func() { diff --git a/test/e2e/mount_test.go b/test/e2e/mount_test.go index 3197aa655..dda83ba31 100644 --- a/test/e2e/mount_test.go +++ b/test/e2e/mount_test.go @@ -156,4 +156,124 @@ var _ = Describe("Podman mount", func() { umount.WaitWithDefaultTimeout() Expect(umount.ExitCode()).To(Equal(0)) }) + + It("podman list mounted container", func() { + setup := podmanTest.Podman([]string{"create", ALPINE, "ls"}) + setup.WaitWithDefaultTimeout() + Expect(setup.ExitCode()).To(Equal(0)) + cid := setup.OutputToString() + + lmount := podmanTest.Podman([]string{"mount", "--notruncate"}) + lmount.WaitWithDefaultTimeout() + Expect(lmount.ExitCode()).To(Equal(0)) + Expect(lmount.OutputToString()).To(Equal("")) + + mount := podmanTest.Podman([]string{"mount", cid}) + mount.WaitWithDefaultTimeout() + Expect(mount.ExitCode()).To(Equal(0)) + + lmount = podmanTest.Podman([]string{"mount", "--notruncate"}) + lmount.WaitWithDefaultTimeout() + Expect(lmount.ExitCode()).To(Equal(0)) + Expect(lmount.OutputToString()).To(ContainSubstring(cid)) + + umount := podmanTest.Podman([]string{"umount", cid}) + umount.WaitWithDefaultTimeout() + Expect(umount.ExitCode()).To(Equal(0)) + }) + + It("podman list running container", func() { + SkipIfRootless() + + setup := podmanTest.Podman([]string{"run", "-dt", ALPINE, "top"}) + setup.WaitWithDefaultTimeout() + Expect(setup.ExitCode()).To(Equal(0)) + cid := setup.OutputToString() + + lmount := podmanTest.Podman([]string{"mount", "--notruncate"}) + lmount.WaitWithDefaultTimeout() + Expect(lmount.ExitCode()).To(Equal(0)) + Expect(lmount.OutputToString()).To(ContainSubstring(cid)) + + stop := podmanTest.Podman([]string{"stop", cid}) + stop.WaitWithDefaultTimeout() + Expect(stop.ExitCode()).To(Equal(0)) + + lmount = podmanTest.Podman([]string{"mount", "--notruncate"}) + lmount.WaitWithDefaultTimeout() + Expect(lmount.ExitCode()).To(Equal(0)) + Expect(lmount.OutputToString()).To(Equal("")) + }) + + It("podman list mulitple mounted containers", func() { + SkipIfRootless() + + setup := podmanTest.Podman([]string{"create", ALPINE, "ls"}) + setup.WaitWithDefaultTimeout() + Expect(setup.ExitCode()).To(Equal(0)) + cid1 := setup.OutputToString() + + setup = podmanTest.Podman([]string{"create", ALPINE, "ls"}) + setup.WaitWithDefaultTimeout() + Expect(setup.ExitCode()).To(Equal(0)) + cid2 := setup.OutputToString() + + setup = podmanTest.Podman([]string{"create", ALPINE, "ls"}) + setup.WaitWithDefaultTimeout() + Expect(setup.ExitCode()).To(Equal(0)) + cid3 := setup.OutputToString() + + lmount := podmanTest.Podman([]string{"mount", "--notruncate"}) + lmount.WaitWithDefaultTimeout() + Expect(lmount.ExitCode()).To(Equal(0)) + Expect(lmount.OutputToString()).To(Equal("")) + + mount := podmanTest.Podman([]string{"mount", cid1, cid3}) + mount.WaitWithDefaultTimeout() + Expect(mount.ExitCode()).To(Equal(0)) + + lmount = podmanTest.Podman([]string{"mount", "--notruncate"}) + lmount.WaitWithDefaultTimeout() + Expect(lmount.ExitCode()).To(Equal(0)) + Expect(lmount.OutputToString()).To(ContainSubstring(cid1)) + Expect(lmount.OutputToString()).ToNot(ContainSubstring(cid2)) + Expect(lmount.OutputToString()).To(ContainSubstring(cid3)) + + umount := podmanTest.Podman([]string{"umount", cid1, cid3}) + umount.WaitWithDefaultTimeout() + Expect(umount.ExitCode()).To(Equal(0)) + + lmount = podmanTest.Podman([]string{"mount", "--notruncate"}) + lmount.WaitWithDefaultTimeout() + Expect(lmount.ExitCode()).To(Equal(0)) + Expect(lmount.OutputToString()).To(Equal("")) + + }) + + It("podman list mounted container", func() { + SkipIfRootless() + + setup := podmanTest.Podman([]string{"create", ALPINE, "ls"}) + setup.WaitWithDefaultTimeout() + Expect(setup.ExitCode()).To(Equal(0)) + cid := setup.OutputToString() + + lmount := podmanTest.Podman([]string{"mount", "--notruncate"}) + lmount.WaitWithDefaultTimeout() + Expect(lmount.ExitCode()).To(Equal(0)) + Expect(lmount.OutputToString()).To(Equal("")) + + mount := podmanTest.Podman([]string{"mount", cid}) + mount.WaitWithDefaultTimeout() + Expect(mount.ExitCode()).To(Equal(0)) + + lmount = podmanTest.Podman([]string{"mount", "--notruncate"}) + lmount.WaitWithDefaultTimeout() + Expect(lmount.ExitCode()).To(Equal(0)) + Expect(lmount.OutputToString()).To(ContainSubstring(cid)) + + umount := podmanTest.Podman([]string{"umount", cid}) + umount.WaitWithDefaultTimeout() + Expect(umount.ExitCode()).To(Equal(0)) + }) }) diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go index 4d2cee8e3..1e6f1d97d 100644 --- a/test/e2e/run_test.go +++ b/test/e2e/run_test.go @@ -194,10 +194,28 @@ var _ = Describe("Podman run", func() { }) It("podman run environment test", func() { - session := podmanTest.Podman([]string{"run", "--rm", "--env", "FOO=BAR,BAZ", ALPINE, "printenv", "FOO"}) + session := podmanTest.Podman([]string{"run", "--rm", ALPINE, "printenv", "HOME"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) - match, _ := session.GrepString("BAR,BAZ") + match, _ := session.GrepString("/root") + Expect(match).Should(BeTrue()) + + session = podmanTest.Podman([]string{"run", "--rm", "--user", "2", ALPINE, "printenv", "HOME"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + match, _ = session.GrepString("/sbin") + Expect(match).Should(BeTrue()) + + session = podmanTest.Podman([]string{"run", "--rm", "--env", "HOME=/foo", ALPINE, "printenv", "HOME"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + match, _ = session.GrepString("/foo") + Expect(match).Should(BeTrue()) + + session = podmanTest.Podman([]string{"run", "--rm", "--env", "FOO=BAR,BAZ", ALPINE, "printenv", "FOO"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + match, _ = session.GrepString("BAR,BAZ") Expect(match).Should(BeTrue()) session = podmanTest.Podman([]string{"run", "--rm", "--env", "PATH=/bin", ALPINE, "printenv", "PATH"}) diff --git a/test/e2e/run_volume_test.go b/test/e2e/run_volume_test.go index fc1998ab2..bc3a14b66 100644 --- a/test/e2e/run_volume_test.go +++ b/test/e2e/run_volume_test.go @@ -270,4 +270,14 @@ var _ = Describe("Podman run with volumes", func() { Expect(separateVolumeSession.ExitCode()).To(Equal(0)) Expect(separateVolumeSession.OutputToString()).To(Equal(baselineOutput)) }) + + It("podman read-only tmpfs conflict with volume", func() { + session := podmanTest.Podman([]string{"run", "--rm", "-t", "-i", "--read-only", "-v", "tmp_volume:/run", ALPINE, "touch", "/run/a"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session2 := podmanTest.Podman([]string{"run", "--rm", "-t", "-i", "--read-only", "--tmpfs", "/run", ALPINE, "touch", "/run/a"}) + session2.WaitWithDefaultTimeout() + Expect(session2.ExitCode()).To(Equal(0)) + }) }) diff --git a/test/e2e/start_test.go b/test/e2e/start_test.go index fc1203ed1..06ab6aacd 100644 --- a/test/e2e/start_test.go +++ b/test/e2e/start_test.go @@ -6,6 +6,7 @@ import ( . "github.com/containers/libpod/test/utils" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" + . "github.com/onsi/gomega/gexec" ) var _ = Describe("Podman start", func() { @@ -107,32 +108,30 @@ var _ = Describe("Podman start", func() { start := podmanTest.Podman([]string{"start", "-l"}) start.WaitWithDefaultTimeout() - Expect(start.ExitCode()).To(Not(Equal(0))) + Expect(start.ExitCode()).Should(BeNumerically(">", 0)) - numContainers := podmanTest.NumberOfContainers() - Expect(numContainers).To(BeZero()) + Eventually(podmanTest.NumberOfContainers(), defaultWaitTimeout).Should(BeZero()) }) It("podman failed to start without --rm should NOT delete the container", func() { session := podmanTest.Podman([]string{"create", "-it", ALPINE, "foo"}) session.WaitWithDefaultTimeout() - Expect(session.ExitCode()).To(Equal(0)) + Expect(session).Should(Exit(0)) start := podmanTest.Podman([]string{"start", "-l"}) start.WaitWithDefaultTimeout() - Expect(start.ExitCode()).To(Not(Equal(0))) + Expect(start.ExitCode()).Should(BeNumerically(">", 0)) - numContainers := podmanTest.NumberOfContainers() - Expect(numContainers).To(Equal(1)) + Eventually(podmanTest.NumberOfContainers(), defaultWaitTimeout).Should(Equal(1)) }) It("podman start --sig-proxy should not work without --attach", func() { session := podmanTest.Podman([]string{"create", ALPINE, "ls"}) session.WaitWithDefaultTimeout() - Expect(session.ExitCode()).To(Equal(0)) + Expect(session).Should(Exit(0)) session = podmanTest.Podman([]string{"start", "-l", "--sig-proxy"}) session.WaitWithDefaultTimeout() - Expect(session.ExitCode()).To(Equal(125)) + Expect(session).Should(Exit(125)) }) }) diff --git a/test/system/030-run.bats b/test/system/030-run.bats index 65e13d559..7cbb60501 100644 --- a/test/system/030-run.bats +++ b/test/system/030-run.bats @@ -3,7 +3,6 @@ load helpers @test "podman run - basic tests" { - skip "Temporarily disabled during investigation into github issue 4044" rand=$(random_string 30) # 2019-09 Fedora 31 and rawhide (32) are switching from runc to crun @@ -28,6 +27,7 @@ echo $rand | 0 | $rand /etc | 126 | $err_no_exec_dir " + tests_run=0 while read cmd expected_rc expected_output; do if [ "$expected_output" = "''" ]; then expected_output=""; fi @@ -41,9 +41,24 @@ echo $rand | 0 | $rand # a way to do so. eval set "$cmd" - run_podman $expected_rc run $IMAGE "$@" - is "$output" "$expected_output" "podman run $cmd - output" + # FIXME: The </dev/null is a hack, necessary because as of 2019-09 + # podman-remote has a bug in which it silently slurps up stdin, + # including the output of parse_table (i.e. tests to be run). + run_podman $expected_rc run $IMAGE "$@" </dev/null + + # FIXME: remove conditional once podman-remote issue #4096 is fixed + if ! is_remote; then + is "$output" "$expected_output" "podman run $cmd - output" + fi + + tests_run=$(expr $tests_run + 1) done < <(parse_table "$tests") + + # Make sure we ran the expected number of tests! Until 2019-09-24 + # podman-remote was only running one test (the "true" one); all + # the rest were being silently ignored because of podman-remote + # bug #4095, in which it slurps up stdin. + is "$tests_run" "$(grep . <<<$tests | wc -l)" "Ran the full set of tests" } @test "podman run - uidmapping has no /sys/kernel mounts" { diff --git a/test/system/065-cp.bats b/test/system/065-cp.bats index 0ca730a50..38660a13c 100644 --- a/test/system/065-cp.bats +++ b/test/system/065-cp.bats @@ -144,12 +144,13 @@ load helpers trap 'exit 0' 15;while :;do sleep 0.5;done" # Copy file from host into container, into a file named 'x' - # Note that the second has a trailing slash; this will trigger mkdir + # Note that the second has a trailing slash, implying a directory. + # Since that destination directory doesn't exist, the cp will fail run_podman cp --pause=false $srcdir/$rand_filename1 cpcontainer:/tmp/d1/x is "$output" "" "output from podman cp 1" - run_podman cp --pause=false $srcdir/$rand_filename2 cpcontainer:/tmp/d2/x/ - is "$output" "" "output from podman cp 3" + run_podman 125 cp --pause=false $srcdir/$rand_filename2 cpcontainer:/tmp/d2/x/ + is "$output" "Error: failed to get stat of dest path .*stat.* no such file or directory" "cp will not create nonexistent destination directory" run_podman cp --pause=false $srcdir/$rand_filename3 cpcontainer:/tmp/d3/x is "$output" "" "output from podman cp 3" @@ -161,10 +162,8 @@ load helpers run_podman exec cpcontainer cat /tmp/nonesuch1 is "$output" "$rand_content1" "cp creates destination file" - # In the second case, podman creates a directory nonesuch2, then - # creates a file with the same name as the input file. THIS IS WEIRD! - run_podman exec cpcontainer cat /tmp/nonesuch2/$rand_filename2 - is "$output" "$rand_content2" "cp creates destination dir and file" + # cp into nonexistent directory should not mkdir nonesuch2 directory + run_podman 1 exec cpcontainer test -e /tmp/nonesuch2 # In the third case, podman (correctly imo) creates a file named 'x' run_podman exec cpcontainer cat /tmp/d3/x diff --git a/test/utils/utils.go b/test/utils/utils.go index 028107d46..2ae140fab 100644 --- a/test/utils/utils.go +++ b/test/utils/utils.go @@ -320,7 +320,7 @@ func (s *PodmanSession) IsJSONOutputValid() bool { // WaitWithDefaultTimeout waits for process finished with defaultWaitTimeout func (s *PodmanSession) WaitWithDefaultTimeout() { - s.Wait(defaultWaitTimeout) + Eventually(s, defaultWaitTimeout).Should(gexec.Exit()) os.Stdout.Sync() os.Stderr.Sync() fmt.Println("output:", s.OutputToString()) |