Merge pull request #5737 from cevich/add_f32

Cirrus: Add support for Fedora 32 beta

7 files changed, 54 insertions, 111 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 58f56798c..8ae1bb2f2 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -34,12 +34,12 @@ env:
     ####
     #### Cache-image names to test with (double-quotes around names are critical)
     ###
-    FEDORA_NAME: "fedora-31"
-    PRIOR_FEDORA_NAME: "fedora-30"
+    FEDORA_NAME: "fedora-32"
+    PRIOR_FEDORA_NAME: "fedora-31"
     UBUNTU_NAME: "ubuntu-19"
     PRIOR_UBUNTU_NAME: "ubuntu-18"
 
-    _BUILT_IMAGE_SUFFIX: "libpod-6465271544152064"
+    _BUILT_IMAGE_SUFFIX: "libpod-6301182083727360"
     FEDORA_CACHE_IMAGE_NAME: "${FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "${PRIOR_FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
     UBUNTU_CACHE_IMAGE_NAME: "${UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}"
@@ -401,16 +401,16 @@ testing_task:
         - name: "test ${FEDORA_NAME}"
           gce_instance:
               image_name: "${FEDORA_CACHE_IMAGE_NAME}"
-        # FIXME
-        #- name: "test ${PRIOR_FEDORA_NAME}"
-        #  gce_instance:
-        #      image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
-        # Multiple test failures on Ubuntu 19 - Fixes TBD in future PR
-        # TODO: image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
-        # FIXME
-        #- name: "test ${PRIOR_UBUNTU_NAME}"
-        #  gce_instance:
-        #      image_name: "${PRIOR_UBUNTU_CACHE_IMAGE_NAME}"
+        # TODO:
+        # - name: "test ${PRIOR_FEDORA_NAME}"
+        #   gce_instance:
+        #       image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
+        # - name: "test ${UBUNTU_NAME}"
+        #   gce_instance:
+        #       image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
+        # - name: "test ${PRIOR_UBUNTU_NAME}"
+        #   gce_instance:
+        #       image_name: "${PRIOR_UBUNTU_CACHE_IMAGE_NAME}"
 
     timeout_in: 120m
 
@@ -552,21 +552,17 @@ special_testing_in_podman_task:
         $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:DOCS.*'
 
     matrix:
-        # FIXME: Integration testing currently broken for F31 hosts
-        # Error: container_linux.go:345: starting container process caused "process_linux.go:281: applying cgroup configuration for process caused \"mountpoint for cgroup not found\"": OCI runtime error
-        # image_name: "${FEDORA_CACHE_IMAGE_NAME}"
         - name: "in-podman ${PRIOR_FEDORA_NAME}"
           gce_instance:
               image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
+        - name: "in-podman ${FEDORA_NAME}"
+          gce_instance:
+              image_name: "${FEDORA_CACHE_IMAGE_NAME}"
 
     env:
         ADD_SECOND_PARTITION: 'true'
         MOD_LIBPOD_CONF: 'false'  # Use existing/native setup
         SPECIALMODE: 'in_podman'  # See docs
-        # TODO: Support both runc and crun (cgroups v1 and v2 container images)
-        # matrix:
-        #   IN_PODMAN_IMAGE: "quay.io/libpod/in_podman:latest"
-        #   IN_PODMAN_IMAGE: "quay.io/libpod/in_podman_cgv2:latest"
 
     timeout_in: 60m
 
@@ -759,47 +755,6 @@ verify_test_built_images_task:
         <<: *standardlogs
 
 
-#test_building_snap_task:
-#
-#    depends_on:
-#        - "gating"
-#
-#    only_if: >-
-#        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:IMG.*' &&
-#        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:DOCS.*'
-#
-#    container:
-#        image: yakshaveinc/snapcraft:core18
-#    snapcraft_script:
-#        - 'apt-get -y update'
-#        - 'cd contrib/snapcraft && snapcraft'
-#
-#
-#upload_snap_task:
-#    only_if: >-
-#        $CIRRUS_BRANCH != $DEST_BRANCH &&
-#        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:IMG.*' &&
-#        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:DOCS.*'
-#
-#    # Only when PR or branch is merged into master
-#
-#    depends_on:
-#        - "test_building_snap"
-#
-#    container:
-#        image: yakshaveinc/snapcraft:core18
-#
-#    env:
-#        SNAPCRAFT_LOGIN: ENCRYPTED[d8e82eb31c6372fec07f405f413d57806026b1a9f8400033531ebcd54d6750a5e4a8b1f68e3ec65c98c65e0d9b2a6a75]
-#    snapcraft_login_file:
-#        path: /root/.snapcraft/login.cfg
-#        variable_name: SNAPCRAFT_LOGIN
-#    snapcraft_script:
-#        - 'apt-get -y update'
-#        - 'snapcraft login --with "/root/.snapcraft/login.cfg"'
-#        - 'cd contrib/snapcraft && snapcraft && snapcraft push *.snap --release edge'
-
-
 docs_task:
 
     # Don't run this when building/testing new VM images
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 2031432b9..04f14eeb3 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -34,7 +34,6 @@ PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer}
 # Important filepaths
 SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}"
 AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv"
-BUILDAH_PACKAGES_FILEPATH="./contrib/cirrus/packages.sh"  # in buildah repo.
 
 # Log remote-client system test varlink output here
 export VARLINK_LOG=/var/tmp/varlink.log
@@ -60,13 +59,13 @@ PACKER_VER="1.4.2"
 # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
 
 # Base-images rarely change, define them here so they're out of the way.
-export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-31,fedora-30}"
+export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-32,fedora-31}"
 # Manually produced base-image names (see $SCRIPT_BASE/README.md)
 export UBUNTU_BASE_IMAGE="ubuntu-1910-eoan-v20200211"
 export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20200218"
 # Manually produced base-image names (see $SCRIPT_BASE/README.md)
-export FEDORA_BASE_IMAGE="fedora-cloud-base-31-1-9-1578586410"
-export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1578586410"
+export FEDORA_BASE_IMAGE="fedora-cloud-base-32-n-0-1586202964"
+export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-31-1-9-1586202964"
 export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}"
 # IN_PODMAN container image
 IN_PODMAN_IMAGE="quay.io/libpod/in_podman:$DEST_BRANCH"
@@ -389,8 +388,7 @@ install_test_configs() {
     install -v -D -m 644 ./test/registries.conf /etc/containers/
 }
 
-# Remove all files (except conmon, for now) provided by the distro version of podman.
-# Except conmon, for now as it's expected to eventually be  packaged separately.
+# Remove all files provided by the distro version of podman.
 # All VM cache-images used for testing include the distro podman because (1) it's
 # required for podman-in-podman testing and (2) it somewhat simplifies the task
 # of pulling in necessary prerequisites packages as the set can change over time.
@@ -449,26 +447,6 @@ systemd_banish() {
     $GOSRC/$PACKER_BASE/systemd_banish.sh
 }
 
-install_buildah_packages() {
-    git clone https://github.com/containers/buildah.git /tmp/buildah
-    if [[ -r "$BUILDAH_PACKAGES_FILEPATH" ]]; then
-        source "$BUILDAH_PACKAGES_FILEPATH"
-        req_env_var UBUNTU_BUILDAH_PACKAGES FEDORA_BUILDAH_PACKAGES OS_RELEASE_ID
-        case "$OS_RELEASE_ID" in
-            fedora)
-                $BIGTO ooe.sh sudo dnf install -y ${FEDORA_BUILDAH_PACKAGES[@]}
-                ;;
-            ubuntu)
-                $LILTO $SUDOAPTGET update
-                $BIGTO $SUDOAPTGET install ${UBUNTU_BUILDAH_PACKAGES[@]}
-                ;;
-            *) bad_os_id_ver ;;
-        esac
-    else
-        warn "Could not find $BUILDAH_PACKAGES_FILEPATH in buildah repository root."
-    fi
-}
-
 _finalize() {
     set +e  # Don't fail at the very end
     if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]]
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index 81a46b13f..fcef7360b 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -12,6 +12,13 @@ req_env_var SCRIPT_BASE PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_I
 
 install_ooe
 
+if [[ $OS_RELEASE_VER -le 31 ]]; then
+    warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539"
+    warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447"
+    echo "mq-deadline" | sudo tee /sys/block/sda/queue/scheduler > /dev/null
+    sudo cat /sys/block/sda/queue/scheduler
+fi
+
 export GOPATH="$(mktemp -d)"
 trap "sudo rm -rf $GOPATH" EXIT
 
@@ -34,6 +41,7 @@ INSTALL_PACKAGES=(\
     bats
     bridge-utils
     btrfs-progs-devel
+    buildah
     bzip2
     conmon
     container-selinux
@@ -81,13 +89,11 @@ INSTALL_PACKAGES=(\
     protobuf-c
     protobuf-c-devel
     protobuf-devel
-    protobuf-python
     python
     python3-dateutil
     python3-psutil
     python3-pytoml
     rsync
-    runc
     selinux-policy-devel
     skopeo
     skopeo-containers
@@ -99,6 +105,7 @@ INSTALL_PACKAGES=(\
     xz
     zip
 )
+
 case "$OS_RELEASE_VER" in
     30)
         INSTALL_PACKAGES+=(\
@@ -113,6 +120,10 @@ case "$OS_RELEASE_VER" in
         INSTALL_PACKAGES+=(crun)
         REMOVE_PACKAGES+=(runc)
         ;;
+    32)
+        INSTALL_PACKAGES+=(crun)
+        REMOVE_PACKAGES+=(runc)
+        ;;
     *)
         bad_os_id_ver ;;
 esac
@@ -120,8 +131,6 @@ esac
 echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'"
 $BIGTO ooe.sh sudo dnf install -y ${INSTALL_PACKAGES[@]}
 
-install_buildah_packages
-
 [[ "${#REMOVE_PACKAGES[@]}" -eq "0" ]] || \
     $LILTO ooe.sh sudo dnf erase -y ${REMOVE_PACKAGES[@]}
 
diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml
index 255723d57..a66fac31c 100644
--- a/contrib/cirrus/packer/libpod_base_images.yml
+++ b/contrib/cirrus/packer/libpod_base_images.yml
@@ -17,14 +17,14 @@ variables:
     PRIOR_UBUNTU_BASE_IMAGE:
 
     # Latest Fedora release
-    FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-Base-31-1.9.x86_64.qcow2"
-    FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-31-1.9-x86_64-CHECKSUM"
-    FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-31-1-9'
+    FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/development/32/Cloud/x86_64/images/Fedora-Cloud-Base-32-20200406.n.0.x86_64.qcow2"
+    FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/development/32/Cloud/x86_64/images/Fedora-Cloud-32-x86_64-20200406.n.0-CHECKSUM"
+    FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-32-n-0'
 
     # Prior Fedora release
-    PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2"
-    PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM"
-    PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-30-1-2'
+    PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-Base-31-1.9.x86_64.qcow2"
+    PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-31-1.9-x86_64-CHECKSUM"
+    PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-31-1-9'
 
     # The name of the image in GCE used for packer build libpod_images.yml
     IBI_BASE_NAME: 'image-builder-image'
diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml
index 074a813af..c23439201 100644
--- a/contrib/cirrus/packer/libpod_images.yml
+++ b/contrib/cirrus/packer/libpod_images.yml
@@ -51,12 +51,12 @@ builders:
       source_image_family: 'prior-ubuntu-base'
 
     - <<: *gce_hosted_image
-      name: 'fedora-31'
+      name: 'fedora-32'
       source_image: '{{user `FEDORA_BASE_IMAGE`}}'
       source_image_family: 'fedora-base'
 
     - <<: *gce_hosted_image
-      name: 'fedora-30'
+      name: 'fedora-31'
       source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}'
       source_image_family: 'prior-fedora-base'
 
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index 46e7a620f..4b6e99358 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -52,6 +52,7 @@ INSTALL_PACKAGES=(\
     bash-completion
     bison
     build-essential
+    buildah
     bzip2
     conmon
     containernetworking-plugins
@@ -145,12 +146,6 @@ else
         /tmp/$(basename $BATS_URL)
         btrfs-tools
     )
-
-    echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting."
-    SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g'
-    ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/*
-    ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub
-    ooe.sh sudo update-grub
 fi
 
 echo "Installing general testing and system dependencies"
@@ -158,8 +153,6 @@ echo "Installing general testing and system dependencies"
 $LILTO $SUDOAPTGET update
 $BIGTO $SUDOAPTGET install ${INSTALL_PACKAGES[@]}
 
-install_buildah_packages
-
 echo "Installing cataonit and libseccomp.sudo"
 ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh
 ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo
@@ -174,6 +167,12 @@ then
     sudo ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc"
 fi
 
+echo "Making Ubuntu kernel to enable cgroup swap accounting as it is not the default."
+SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g'
+ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/*
+ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub
+ooe.sh sudo update-grub
+
 ubuntu_finalize
 
 echo "SUCCESS!"
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index eceb80b00..57c9ec52a 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -47,10 +47,12 @@ case "${OS_RELEASE_ID}" in
             bash "$SCRIPT_BASE/add_second_partition.sh"
         fi
 
-        warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539"
-        warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447"
-        echo "mq-deadline" > /sys/block/sda/queue/scheduler
-        cat /sys/block/sda/queue/scheduler
+        if [[ $OS_RELEASE_VER -le 31 ]]; then
+            warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539"
+            warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447"
+            echo "mq-deadline" > /sys/block/sda/queue/scheduler
+            cat /sys/block/sda/queue/scheduler
+        fi
 
         if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
             bash "$SCRIPT_BASE/add_second_partition.sh"