Cirrus: Support testing with F31

Signed-off-by: Chris Evich <cevich@redhat.com>

11 files changed, 215 insertions, 255 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 53b406804..d95db7240 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -30,10 +30,9 @@ env:
     ####
     #### Cache-image names to test with (double-quotes around names are critical)
     ###
-    _BUILT_IMAGE_SUFFIX: "libpod-5642998972416000"
-    FEDORA_CACHE_IMAGE_NAME: "fedora-30-${_BUILT_IMAGE_SUFFIX}"
+    _BUILT_IMAGE_SUFFIX: "libpod-5874660151656448"
+    FEDORA_CACHE_IMAGE_NAME: "fedora-31-${_BUILT_IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-30-${_BUILT_IMAGE_SUFFIX}"
-    SPECIAL_FEDORA_CACHE_IMAGE_NAME: "xfedora-30-${_BUILT_IMAGE_SUFFIX}"
     UBUNTU_CACHE_IMAGE_NAME: "ubuntu-19-${_BUILT_IMAGE_SUFFIX}"
     PRIOR_UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-${_BUILT_IMAGE_SUFFIX}"
 
@@ -143,11 +142,16 @@ gating_task:
     on_failure:
         failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'
 
+
 # This task checks to make sure that we can still build an rpm from the
 # source code using contrib/rpm/podman.spec.in
 rpmbuild_task:
 
-    only_if: $CIRRUS_BRANCH != $DEST_BRANCH
+    only_if: >-
+        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:IMG.*' &&
+        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:DOCS.*' &&
+        $CIRRUS_BRANCH != $DEST_BRANCH
+
     depends_on:
         - "gating"
     env:
@@ -246,12 +250,8 @@ build_each_commit_task:
         $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:DOCS.*'
 
     gce_instance:
-        image_project: "libpod-218412"
-        zone: "us-central1-a"  # Required by Cirrus for the time being
         cpu: 8
         memory: "8Gb"
-        disk: 200
-        image_name: "${FEDORA_CACHE_IMAGE_NAME}"
 
     timeout_in: 30m
 
@@ -279,12 +279,8 @@ build_without_cgo_task:
         $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:DOCS.*'
 
     gce_instance:
-        image_project: "libpod-218412"
-        zone: "us-central1-a"  # Required by Cirrus for the time being
         cpu: 8
         memory: "8Gb"
-        disk: 200
-        image_name: "${FEDORA_CACHE_IMAGE_NAME}"
 
     timeout_in: 30m
 
@@ -317,7 +313,6 @@ meta_task:
         IMGNAMES: >-
             ${FEDORA_CACHE_IMAGE_NAME}
             ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
-            ${SPECIAL_FEDORA_CACHE_IMAGE_NAME}
             ${UBUNTU_CACHE_IMAGE_NAME}
             ${PRIOR_UBUNTU_CACHE_IMAGE_NAME}
             ${IMAGE_BUILDER_CACHE_IMAGE_NAME}
@@ -377,8 +372,7 @@ testing_task:
 
     gce_instance:
         matrix:
-            # Images are generated separately, from build_images_task (below)
-            #image_name: "${FEDORA_CACHE_IMAGE_NAME}"
+            image_name: "${FEDORA_CACHE_IMAGE_NAME}"
             image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
             # Multiple test failures on Ubuntu 19 - Fixes TBD in future PR
             # TODO: image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
@@ -466,6 +460,12 @@ special_testing_in_podman_task:
         $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:IMG.*' &&
         $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:DOCS.*'
 
+    # Make sure F31 container images can run podman on F30 or F31 host
+    gce_instance:
+        matrix:
+            image_name: "${FEDORA_CACHE_IMAGE_NAME}"
+            image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
+
     env:
         ADD_SECOND_PARTITION: true
         SPECIALMODE: 'in_podman'  # See docs
@@ -519,39 +519,6 @@ special_testing_cross_task:
         type: "application/octet-stream"
 
 
-special_testing_cgroupv2_task:
-
-    depends_on:
-        - "gating"
-        - "varlink_api"
-        - "vendor"
-
-    only_if: >-
-        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:IMG.*' &&
-        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:DOCS.*'
-
-    gce_instance:
-        image_name: "${SPECIAL_FEDORA_CACHE_IMAGE_NAME}"
-
-    env:
-        SPECIALMODE: 'cgroupv2'  # See docs
-        matrix:
-            TEST_REMOTE_CLIENT: true
-            TEST_REMOTE_CLIENT: false
-
-    timeout_in: 120m
-
-    networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
-    setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
-    integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
-
-    on_failure:
-        failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'
-
-    always:
-        <<: *standardlogs
-
-
 special_testing_bindings_task:
 
     depends_on:
@@ -577,6 +544,7 @@ special_testing_bindings_task:
     always:
         <<: *standardlogs
 
+
 special_testing_endpoint_task:
 
     depends_on:
@@ -603,22 +571,6 @@ special_testing_endpoint_task:
         <<: *standardlogs
 
 
-test_building_snap_task:
-
-    depends_on:
-        - "gating"
-
-    only_if: >-
-        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:IMG.*' &&
-        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:DOCS.*'
-
-    container:
-        image: yakshaveinc/snapcraft:core18
-    snapcraft_script:
-        - 'apt-get -y update'
-        - 'cd contrib/snapcraft && snapcraft'
-
-
 # Test building of new cache-images for future PR testing, in this PR.
 test_build_cache_images_task:
 
@@ -677,12 +629,13 @@ verify_test_built_images_task:
         matrix:
             # Required env. var. by check_image_script
             PACKER_BUILDER_NAME: "fedora-30"
-            #PACKER_BUILDER_NAME: "fedora-31"
-            PACKER_BUILDER_NAME: "xfedora-30"
+            PACKER_BUILDER_NAME: "fedora-31"
             PACKER_BUILDER_NAME: "ubuntu-18"
-            # TODO support $UBUNTU_CACHE_IMAGE_NAME: PACKER_BUILDER_NAME: "ubuntu-19"
+            # Multiple test failures on ${UBUNTU_CACHE_IMAGE_NAME}
+            # PACKER_BUILDER_NAME: "ubuntu-19"
 
     networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
+    installed_packages_script: '$SCRIPT_BASE/logcollector.sh packages'
     environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
     # Verify expectations once per image
     check_image_script: >-
@@ -691,44 +644,60 @@ verify_test_built_images_task:
     # Note: A truncated form of normal testing. It only needs to confirm new images
     # "probably" work.  A full round of testing will happen again after $*_CACHE_IMAGE_NAME
     # are updated in this or another PR (w/o '***CIRRUS: TEST IMAGES***').
-    integration_test_script: >-
-        [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \
-            $SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}
+    integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
     build_release_script: '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}'
-    system_test_script: >-
-        [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \
-            $SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}
+    system_test_script: '$SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}'
 
     always:
         <<: *standardlogs
 
-        #upload_snap_task:
-        #    only_if: >-
-        #        $CIRRUS_BRANCH != $DEST_BRANCH &&
-        #        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:IMG.*' &&
-        #        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:DOCS.*'
-        #
-        #    # Only when PR or branch is merged into master
-        #
-        #    depends_on:
-        #        - "test_building_snap"
-        #
-        #    container:
-        #        image: yakshaveinc/snapcraft:core18
-        #
-        #    env:
-        #        SNAPCRAFT_LOGIN: ENCRYPTED[d8e82eb31c6372fec07f405f413d57806026b1a9f8400033531ebcd54d6750a5e4a8b1f68e3ec65c98c65e0d9b2a6a75]
-        #    snapcraft_login_file:
-        #        path: /root/.snapcraft/login.cfg
-        #        variable_name: SNAPCRAFT_LOGIN
-        #    snapcraft_script:
-        #        - 'apt-get -y update'
-        #        - 'snapcraft login --with "/root/.snapcraft/login.cfg"'
-        #        - 'cd contrib/snapcraft && snapcraft && snapcraft push *.snap --release edge'
+
+#test_building_snap_task:
+#
+#    depends_on:
+#        - "gating"
+#
+#    only_if: >-
+#        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:IMG.*' &&
+#        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:DOCS.*'
+#
+#    container:
+#        image: yakshaveinc/snapcraft:core18
+#    snapcraft_script:
+#        - 'apt-get -y update'
+#        - 'cd contrib/snapcraft && snapcraft'
+#
+#
+#upload_snap_task:
+#    only_if: >-
+#        $CIRRUS_BRANCH != $DEST_BRANCH &&
+#        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:IMG.*' &&
+#        $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:DOCS.*'
+#
+#    # Only when PR or branch is merged into master
+#
+#    depends_on:
+#        - "test_building_snap"
+#
+#    container:
+#        image: yakshaveinc/snapcraft:core18
+#
+#    env:
+#        SNAPCRAFT_LOGIN: ENCRYPTED[d8e82eb31c6372fec07f405f413d57806026b1a9f8400033531ebcd54d6750a5e4a8b1f68e3ec65c98c65e0d9b2a6a75]
+#    snapcraft_login_file:
+#        path: /root/.snapcraft/login.cfg
+#        variable_name: SNAPCRAFT_LOGIN
+#    snapcraft_script:
+#        - 'apt-get -y update'
+#        - 'snapcraft login --with "/root/.snapcraft/login.cfg"'
+#        - 'cd contrib/snapcraft && snapcraft && snapcraft push *.snap --release edge'
 
 
 docs_task:
 
+    # Don't run this when building/testing new VM images
+    only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:IMG.*'
+
     depends_on:
         - "gating"
 
@@ -757,12 +726,10 @@ success_task:
         - "rpmbuild"
         - "special_testing_rootless"
         - "special_testing_in_podman"
-        - "special_testing_cgroupv2"
         - "special_testing_cross"
         - "special_testing_endpoint"
         - "special_testing_bindings"
         - "test_build_cache_images"
-        - "test_building_snap"
         - "verify_test_built_images"
         - "docs"
 
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md
index 49f713a8f..3789965d6 100644
--- a/contrib/cirrus/README.md
+++ b/contrib/cirrus/README.md
@@ -216,10 +216,10 @@ the ``cache_images`` Task) some input parameters are required:
    to limit the base-images produced.  For example,
    ``PACKER_BUILDS=fedora,image-builder-image``.
 
-If there is an existing 'image-builder-image' within GCE, it may be utilized
-to produce base-images (in addition to cache-images).  However it must be
-created with support for nested-virtualization, and with elevated cloud
-privileges (to access GCE, from within the GCE VM).  For example:
+If there is no existing 'image-builder-image' within GCE, a new
+one may be bootstrapped by creating a CentOS 7 VM with support for
+nested-virtualization, and with elevated cloud privileges (to access
+GCE, from within the GCE VM).  For example:
 
 ```
 $ alias pgcloud='sudo podman run -it --rm -e AS_ID=$UID
@@ -229,34 +229,33 @@ $ URL=https://www.googleapis.com/auth
 $ SCOPES=$URL/userinfo.email,$URL/compute,$URL/devstorage.full_control
 
 # The --min-cpu-platform is critical for nested-virt.
-$ pgcloud compute instances create $USER-making-images \
-    --image-family image-builder-image \
+$ pgcloud compute instances create $USER-image-builder \
+    --image-family centos-7 \
     --boot-disk-size "200GB" \
     --min-cpu-platform "Intel Haswell" \
     --machine-type n1-standard-2 \
     --scopes $SCOPES
 ```
 
-Alternatively, if there is no image-builder-image available yet, a bare-metal
-CentOS 7 machine with network access to GCE is required.  Software dependencies
-can be obtained from the ``packer/image-builder-image_base_setup.sh`` script.
+Then from that VM, execute the
+``contrib/cirrus/packer/image-builder-image_base_setup.sh`` script.
+Shutdown the VM, and convert it into a new image-builder-image.
 
-In both cases, the following can be used to setup and build base-images.
+Building new base images is done by first creating a VM from an
+image-builder-image and copying the credentials json file to it.
 
 ```
-$ IP_ADDRESS=1.2.3.4  # EXTERNAL_IP from command output above
-$ rsync -av $PWD centos@$IP_ADDRESS:.
-$ scp $GOOGLE_APPLICATION_CREDENTIALS centos@$IP_ADDRESS:.
-$ ssh centos@$IP_ADDRESS
-...
+$ hack/get_ci_vm.sh image-builder-image-1541772081
+...in another terminal...
+$ pgcloud compute scp /path/to/gac.json $USER-image-builder-image-1541772081:.
 ```
 
-When ready, change to the ``packer`` sub-directory, and build the images:
+Then, on the VM, change to the ``packer`` sub-directory, and build the images:
 
 ```
 $ cd libpod/contrib/cirrus/packer
 $ make libpod_base_images GCP_PROJECT_ID=<VALUE> \
-    GOOGLE_APPLICATION_CREDENTIALS=<VALUE> \
+    GOOGLE_APPLICATION_CREDENTIALS=/path/to/gac.json \
     PACKER_BUILDS=<OPTIONAL>
 ```
 
@@ -283,7 +282,5 @@ values follows:
 * `rootless`: Causes a random, ordinary user account to be created
               and utilized for testing.
 * `in_podman`: Causes testing to occur within a container executed by
-               Podman on the host.
-* `cgroupv2`: The kernel on this VM was prepared with options to enable v2 cgroups
 * `windows`: See **darwin**
 * `darwin`: Signals the ``special_testing_cross`` task to cross-compile the remote client.
diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh
index 9fd79ab18..20e067c93 100755
--- a/contrib/cirrus/integration_test.sh
+++ b/contrib/cirrus/integration_test.sh
@@ -36,16 +36,6 @@ case "$SPECIALMODE" in
                 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
                 -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE}
         ;;
-    cgroupv2)
-        setenforce 0
-        dnf install -y crun
-        export OCI_RUNTIME=/usr/bin/crun
-        make
-        make install PREFIX=/usr ETCDIR=/etc
-        make install.config PREFIX=/usr
-        make test-binaries
-        make local${TESTSUITE}
-        ;;
     endpoint)
         make
         make install PREFIX=/usr ETCDIR=/etc
@@ -53,9 +43,9 @@ case "$SPECIALMODE" in
         make endpoint
         ;;
     bindings)
-	make
+	    make
         make install PREFIX=/usr ETCDIR=/etc
-	cd pkg/bindings/test && ginkgo -r
+	    cd pkg/bindings/test && ginkgo -r
 	;;
     none)
         make
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 1e237085f..71ad67c74 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -7,10 +7,10 @@
 source /etc/environment  # not always loaded under all circumstances
 
 # Under some contexts these values are not set, make sure they are.
-USER="$(whoami)"
-HOME="$(getent passwd $USER | cut -d : -f 6)"
-[[ -n "$UID" ]] || UID=$(getent passwd $USER | cut -d : -f 3)
-GID=$(getent passwd $USER | cut -d : -f 4)
+export USER="$(whoami)"
+export HOME="$(getent passwd $USER | cut -d : -f 6)"
+[[ -n "$UID" ]] || export UID=$(getent passwd $USER | cut -d : -f 3)
+export GID=$(getent passwd $USER | cut -d : -f 4)
 
 # Essential default paths, many are overridden when executing under Cirrus-CI
 export GOPATH="${GOPATH:-/var/tmp/go}"
@@ -59,13 +59,13 @@ PACKER_VER="1.4.2"
 # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
 
 # Base-images rarely change, define them here so they're out of the way.
-export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-30,xfedora-30,fedora-29}"
-# Google-maintained base-image names
+export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-31,fedora-30}"
+# Manually produced base-image names (see $SCRIPT_BASE/README.md)
 export UBUNTU_BASE_IMAGE="ubuntu-1904-disco-v20190724"
 export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a"
 # Manually produced base-image names (see $SCRIPT_BASE/README.md)
-export FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1578586410"
-export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245"
+export FEDORA_BASE_IMAGE="fedora-cloud-base-31-1-9-1578586410"
+export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1578586410"
 export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}"
 # IN_PODMAN container image
 IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest"
@@ -79,8 +79,8 @@ SUDOAPTADD="ooe.sh sudo -E add-apt-repository --yes"
 # Regex that finds enabled periodic apt configuration items
 PERIODIC_APT_RE='^(APT::Periodic::.+")1"\;'
 # Short-cuts for retrying/timeout calls
-LILTO="timeout_attempt_delay_command 24s 5 30s"
-BIGTO="timeout_attempt_delay_command 300s 5 30s"
+LILTO="timeout_attempt_delay_command 120s 5 30s"
+BIGTO="timeout_attempt_delay_command 300s 5 60s"
 
 # Safe env. vars. to transfer from root -> $ROOTLESS_USER  (go env handled separately)
 ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(TEST_REMOTE.*)'
@@ -178,8 +178,7 @@ die() {
 }
 
 warn() {
-    echo ">>>>> ${2:-WARNING (but no message given!) in ${FUNCNAME[1]}()}" > /dev/stderr
-    echo ${1:-1} > /dev/stdout
+    echo ">>>>> ${1:-WARNING (but no message given!) in ${FUNCNAME[1]}()}" > /dev/stderr
 }
 
 bad_os_id_ver() {
@@ -456,7 +455,6 @@ _finalize() {
         echo "Could not find any files in $CUSTOM_CLOUD_CONFIG_DEFAULTS"
     fi
     echo "Re-initializing so next boot does 'first-boot' setup again."
-    sudo history -c
     cd /
     sudo rm -rf /var/lib/cloud/instanc*
     sudo rm -rf /root/.ssh/*
diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile
index fa87d7019..a911cafdb 100644
--- a/contrib/cirrus/packer/Makefile
+++ b/contrib/cirrus/packer/Makefile
@@ -1,9 +1,4 @@
-
-# N/B: PACKER_BUILDS variable is required.  Should contain CSV of
-# builder name(s) from applicable YAML file,
-# e.g for names see libpod_images.yml
-
-PACKER_VER ?= 1.4.2
+PACKER_VER ?= 1.4.3
 GOARCH=$(shell go env GOARCH)
 ARCH=$(uname -m)
 PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip
@@ -56,8 +51,9 @@ test: libpod_base_images.json libpod_images.json packer
 
 .PHONY: libpod_images
 libpod_images: guard-PACKER_BUILDS libpod_images.json packer
-	./packer build -only=${PACKER_BUILDS} \
+	./packer build \
 		-force \
+		$(shell test -z "${PACKER_BUILDS}" || echo "-only=${PACKER_BUILDS}") \
 		-var GOSRC=$(GOSRC) \
 		-var PACKER_BASE=$(PACKER_BASE) \
 		-var SCRIPT_BASE=$(SCRIPT_BASE) \
@@ -82,6 +78,7 @@ cidata.iso: user-data meta-data
 .PHONY: libpod_base_images
 libpod_base_images: guard-GCP_PROJECT_ID guard-GOOGLE_APPLICATION_CREDENTIALS libpod_base_images.json cidata.iso cidata.ssh packer
 	PACKER_CACHE_DIR=/tmp ./packer build \
+		$(shell test -z "${PACKER_BUILDS}" || echo "-only=${PACKER_BUILDS}") \
 		-force \
 		-var TIMESTAMP=$(TIMESTAMP) \
 		-var TTYDEV=$(TTYDEV) \
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index 6cfaa05ce..591a59a05 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -15,100 +15,106 @@ install_ooe
 export GOPATH="$(mktemp -d)"
 trap "sudo rm -rf $GOPATH" EXIT
 
-ooe.sh sudo dnf update -y
+$BIGTO ooe.sh sudo dnf update -y
 
 echo "Enabling updates-testing repository"
-ooe.sh sudo dnf install -y 'dnf-command(config-manager)'
-ooe.sh sudo dnf config-manager --set-enabled updates-testing
+$LILTO ooe.sh sudo dnf install -y 'dnf-command(config-manager)'
+$LILTO ooe.sh sudo dnf config-manager --set-enabled updates-testing
 
-echo "Installing general build/test dependencies"
-ooe.sh sudo dnf install -y \
-    atomic-registries \
-    autoconf \
-    automake \
-    bash-completion \
-    bats \
-    bridge-utils \
-    btrfs-progs-devel \
-    bzip2 \
-    conmon \
-    container-selinux \
-    containernetworking-plugins \
-    containers-common \
-    criu \
-    device-mapper-devel \
-    emacs-nox \
-    file \
-    findutils \
-    fuse3 \
-    fuse3-devel \
-    gcc \
-    git \
-    glib2-devel \
-    glibc-static \
-    gnupg \
-    go-md2man \
-    golang \
-    golang-github-cpuguy83-go-md2man \
-    gpgme-devel \
-    iproute \
-    iptables \
-    jq \
-    libassuan-devel \
-    libcap-devel \
-    libmsi1 \
-    libnet \
-    libnet-devel \
-    libnl3-devel \
-    libseccomp \
-    libseccomp-devel \
-    libselinux-devel \
-    libtool \
-    libvarlink-util \
-    lsof \
-    make \
-    msitools \
-    nmap-ncat \
-    pandoc \
-    podman \
-    procps-ng \
-    protobuf \
-    protobuf-c \
-    protobuf-c-devel \
-    protobuf-compiler \
-    protobuf-devel \
-    protobuf-python \
-    python \
-    python2-future \
-    python3-dateutil \
-    python3-psutil \
-    python3-pytoml \
-    runc \
-    selinux-policy-devel \
-    slirp4netns \
-    unzip \
-    vim \
-    which \
-    xz \
+echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'"
+REMOVE_PACKAGES=()
+INSTALL_PACKAGES=(\
+    autoconf
+    automake
+    bash-completion
+    bats
+    bridge-utils
+    btrfs-progs-devel
+    bzip2
+    conmon
+    container-selinux
+    containernetworking-plugins
+    containers-common
+    criu
+    device-mapper-devel
+    dnsmasq
+    emacs-nox
+    file
+    findutils
+    fuse3
+    fuse3-devel
+    gcc
+    git
+    glib2-devel
+    glibc-static
+    gnupg
+    go-md2man
+    golang
+    gpgme-devel
+    iproute
+    iptables
+    jq
+    libassuan-devel
+    libcap-devel
+    libmsi1
+    libnet
+    libnet-devel
+    libnl3-devel
+    libseccomp
+    libseccomp-devel
+    libselinux-devel
+    libtool
+    libvarlink-util
+    lsof
+    make
+    msitools
+    nmap-ncat
+    pandoc
+    podman
+    procps-ng
+    protobuf
+    protobuf-c
+    protobuf-c-devel
+    protobuf-devel
+    protobuf-python
+    python
+    python3-dateutil
+    python3-psutil
+    python3-pytoml
+    selinux-policy-devel
+    skopeo
+    slirp4netns
+    unzip
+    vim
+    which
+    xz
     zip
+)
+case "$OS_RELEASE_VER" in
+    30)
+        INSTALL_PACKAGES+=(\
+            atomic-registries
+            golang-github-cpuguy83-go-md2man
+            python2-future
+            runc
+        )
+        ;;
+    31)
+        INSTALL_PACKAGES+=(crun)
+        REMOVE_PACKAGES+=(runc)
+        ;;
+    *)
+        bad_os_id_ver ;;
+esac
+$BIGTO ooe.sh sudo dnf install -y ${INSTALL_PACKAGES[@]}
 
+[[ "${#REMOVE_PACKAGES[@]}" -eq "0" ]] || \
+    $LILTO ooe.sh sudo dnf erase -y ${REMOVE_PACKAGES[@]}
 
 # Ensure there are no disruptive periodic services enabled by default in image
 systemd_banish
 
-sudo /tmp/libpod/hack/install_catatonit.sh
-
-# Same script is used for several related contexts
-case "$PACKER_BUILDER_NAME" in
-    xfedora*)
-        echo "Configuring CGroups v2 enabled on next boot"
-        sudo grubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=1"
-        sudo dnf install -y crun
-        ;&  # continue to next matching item
-    *)
-        echo "Finalizing $PACKER_BUILDER_NAME VM image"
-        ;;
-esac
+ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh
 
 rh_finalize
 
diff --git a/contrib/cirrus/packer/image-builder-image_base-setup.sh b/contrib/cirrus/packer/image-builder-image_base-setup.sh
index 43cfa7180..78772da09 100644
--- a/contrib/cirrus/packer/image-builder-image_base-setup.sh
+++ b/contrib/cirrus/packer/image-builder-image_base-setup.sh
@@ -31,10 +31,8 @@ ooe.sh sudo yum -y install \
     libvirt-client \
     libvirt-daemon \
     make \
-    python34 \
-    python34 \
-    python34-PyYAML \
-    python34-PyYAML \
+    python36 \
+    python36-PyYAML \
     qemu-img \
     qemu-kvm \
     qemu-kvm-tools \
diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml
index bcca440ae..21f3795f1 100644
--- a/contrib/cirrus/packer/libpod_base_images.yml
+++ b/contrib/cirrus/packer/libpod_base_images.yml
@@ -17,14 +17,14 @@ variables:
     PRIOR_UBUNTU_BASE_IMAGE:
 
     # Latest Fedora release
-    FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2"
-    FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM"
-    FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-30-1-2'
+    FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-Base-31-1.9.x86_64.qcow2"
+    FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-31-1.9-x86_64-CHECKSUM"
+    FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-31-1-9'
 
     # Prior Fedora release
-    PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2"
-    PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-29-1.2-x86_64-CHECKSUM"
-    PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-29-1-2'  # Name to use in GCE
+    PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2"
+    PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM"
+    PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-30-1-2'
 
     # The name of the image in GCE used for packer build libpod_images.yml
     IBI_BASE_NAME: 'image-builder-image'
diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml
index 01a65d867..074a813af 100644
--- a/contrib/cirrus/packer/libpod_images.yml
+++ b/contrib/cirrus/packer/libpod_images.yml
@@ -51,17 +51,12 @@ builders:
       source_image_family: 'prior-ubuntu-base'
 
     - <<: *gce_hosted_image
-      name: 'fedora-30'
-      source_image: '{{user `FEDORA_BASE_IMAGE`}}'
-      source_image_family: 'fedora-base'
-
-    - <<: *gce_hosted_image
-      name: 'xfedora-30'
+      name: 'fedora-31'
       source_image: '{{user `FEDORA_BASE_IMAGE`}}'
       source_image_family: 'fedora-base'
 
     - <<: *gce_hosted_image
-      name: 'fedora-29'
+      name: 'fedora-30'
       source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}'
       source_image_family: 'prior-fedora-base'
 
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index 118ee062a..7c39a76f8 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -59,6 +59,8 @@ $BIGTO $SUDOAPTGET install \
     cri-o-runc \
     criu \
     curl \
+    conmon \
+    dnsmasq \
     e2fslibs-dev \
     emacs-nox \
     file \
@@ -129,7 +131,7 @@ then
     ooe.sh sudo update-grub
 fi
 
-sudo /tmp/libpod/hack/install_catatonit.sh
+ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh
 ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo
 
 ubuntu_finalize
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index edd793bb9..4d4189aa6 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -47,6 +47,19 @@ case "${OS_RELEASE_ID}" in
         setsebool container_manage_cgroup true
         if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
             bash "$SCRIPT_BASE/add_second_partition.sh"; fi
+
+        if [[ "$OS_RELEASE_VER" == "31" ]]; then
+            warn "Testing with crun instead of runc"
+            X=$(echo "export export OCI_RUNTIME=/usr/bin/crun" | \
+                tee -a /etc/environment) && eval "$X" && echo "$X"
+            warn "Upgrading to the latest crun"
+            # Normally not something to do for stable testing
+            # but crun is new, and late-breaking fixes may be required
+            # on short notice
+            dnf update -y crun
+            warn "Setting SELinux into Permissive mode"
+            setenforce 0
+        fi
         ;;
     centos)  # Current VM is an image-builder-image no local podman/testing
         echo "No further setup required for VM image building"
@@ -62,9 +75,6 @@ source "$SCRIPT_BASE/lib.sh"
 make install.tools
 
 case "$SPECIALMODE" in
-    cgroupv2)
-        remove_packaged_podman_files  # we're building from source
-        ;;
     none)
         [[ -n "$CROSS_PLATFORM" ]] || \
             remove_packaged_podman_files