Merge pull request #5342 from cevich/runc_in_f30

Force using runc in F30

8 files changed, 88 insertions, 58 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 371f902c2..2106ac96d 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -30,7 +30,7 @@ env:
     ####
     #### Cache-image names to test with (double-quotes around names are critical)
     ###
-    _BUILT_IMAGE_SUFFIX: "libpod-5874660151656448"
+    _BUILT_IMAGE_SUFFIX: "libpod-5940307564953600"
     FEDORA_CACHE_IMAGE_NAME: "fedora-31-${_BUILT_IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-30-${_BUILT_IMAGE_SUFFIX}"
     UBUNTU_CACHE_IMAGE_NAME: "ubuntu-19-${_BUILT_IMAGE_SUFFIX}"
@@ -48,8 +48,9 @@ env:
     #### Default to NOT operating in any special-case testing mode
     ####
     SPECIALMODE: "none"  # don't do anything special
-    TEST_REMOTE_CLIENT: false  # don't test remote client by default
-    ADD_SECOND_PARTITION: false  # will certainly fail inside containers
+    TEST_REMOTE_CLIENT: 'false'  # don't test remote client by default
+    ADD_SECOND_PARTITION: 'false'  # will certainly fail inside containers
+    MOD_LIBPOD_CONF: 'true'  # Update libpod.conf runtime if required by OS environment
 
     ####
     #### Credentials and other secret-sauces, decrypted at runtime when authorized.
@@ -253,6 +254,9 @@ build_each_commit_task:
         cpu: 8
         memory: "8Gb"
 
+    env:
+        MOD_LIBPOD_CONF: 'false'
+
     timeout_in: 30m
 
     setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
@@ -282,6 +286,9 @@ build_without_cgo_task:
         cpu: 8
         memory: "8Gb"
 
+    env:
+        MOD_LIBPOD_CONF: 'false'
+
     timeout_in: 30m
 
     setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
@@ -381,10 +388,10 @@ testing_task:
     timeout_in: 120m
 
     env:
-        ADD_SECOND_PARTITION: true
+        ADD_SECOND_PARTITION: 'true'
         matrix:
-            TEST_REMOTE_CLIENT: true
-            TEST_REMOTE_CLIENT: false
+            TEST_REMOTE_CLIENT: 'true'
+            TEST_REMOTE_CLIENT: 'false'
 
     networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
     setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
@@ -428,11 +435,11 @@ special_testing_rootless_task:
         $CIRRUS_CHANGE_MESSAGE !=~ '.*CI:DOCS.*'
 
     env:
-        ADD_SECOND_PARTITION: true
+        ADD_SECOND_PARTITION: 'true'
         SPECIALMODE: 'rootless'  # See docs
         matrix:
-            TEST_REMOTE_CLIENT: true
-            TEST_REMOTE_CLIENT: false
+            TEST_REMOTE_CLIENT: 'true'
+            TEST_REMOTE_CLIENT: 'false'
 
     timeout_in: 60m
 
@@ -469,7 +476,8 @@ special_testing_in_podman_task:
             image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
 
     env:
-        ADD_SECOND_PARTITION: true
+        ADD_SECOND_PARTITION: 'true'
+        MOD_LIBPOD_CONF: 'false'  # Use existing/native setup
         SPECIALMODE: 'in_podman'  # See docs
         # TODO: Support both runc and crun (cgroups v1 and v2 container images)
         # matrix:
@@ -628,10 +636,10 @@ verify_test_built_images_task:
         image_name: "${PACKER_BUILDER_NAME}${BUILT_IMAGE_SUFFIX}"
 
     env:
-        ADD_SECOND_PARTITION: true
+        ADD_SECOND_PARTITION: 'true'
         matrix:
-            TEST_REMOTE_CLIENT: true
-            TEST_REMOTE_CLIENT: false
+            TEST_REMOTE_CLIENT: 'true'
+            TEST_REMOTE_CLIENT: 'false'
         matrix:
             # Required env. var. by check_image_script
             PACKER_BUILDER_NAME: "fedora-30"
diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh
index d5e6ec884..20e067c93 100755
--- a/contrib/cirrus/integration_test.sh
+++ b/contrib/cirrus/integration_test.sh
@@ -16,16 +16,6 @@ fi
 
 cd "$GOSRC"
 
-# Transition workaround: runc is still the default for upstream development
-handle_crun() {
-    # For systems with crun installed, assume CgroupsV2 and use it
-    if type -P crun &> /dev/null
-    then
-        warn "Replacing runc -> crun in libpod.conf"
-        sed -i -r -e 's/^runtime = "runc"/runtime = "crun"/' /usr/share/containers/libpod.conf
-    fi
-}
-
 case "$SPECIALMODE" in
     in_podman)
         ${CONTAINER_RUNTIME} run --rm --privileged --net=host \
@@ -49,7 +39,6 @@ case "$SPECIALMODE" in
     endpoint)
         make
         make install PREFIX=/usr ETCDIR=/etc
-        #handle_crun
         make test-binaries
         make endpoint
         ;;
@@ -63,7 +52,6 @@ case "$SPECIALMODE" in
         make install PREFIX=/usr ETCDIR=/etc
         make install.config PREFIX=/usr
         make test-binaries
-        handle_crun
         if [[ "$TEST_REMOTE_CLIENT" == "true" ]]
         then
             make remote${TESTSUITE} VARLINK_LOG=$VARLINK_LOG
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 71ad67c74..1ffe554e9 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -88,6 +88,7 @@ ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.
 SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(GC[EP]..+)|(SSH)'
 
 SPECIALMODE="${SPECIALMODE:-none}"
+MOD_LIBPOD_CONF="${MOD_LIBPOD_CONF:false}"
 TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}"
 export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
 
@@ -105,6 +106,8 @@ OS_RELEASE_ID="$(source /etc/os-release; echo $ID)"
 OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)"
 # Combined to ease soe usage
 OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
+# Type of filesystem used for cgroups
+CG_FS_TYPE="$(stat -f -c %T /sys/fs/cgroup)"
 
 # Installed into cache-images, supports overrides
 # by user-data in case of breakage or for debugging.
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index 591a59a05..20014e5f3 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -8,7 +8,7 @@ set -e
 # Load in library (copied by packer, before this script was run)
 source /tmp/libpod/$SCRIPT_BASE/lib.sh
 
-req_env_var SCRIPT_BASE PACKER_BUILDER_NAME GOSRC
+req_env_var SCRIPT_BASE PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER
 
 install_ooe
 
@@ -17,9 +17,14 @@ trap "sudo rm -rf $GOPATH" EXIT
 
 $BIGTO ooe.sh sudo dnf update -y
 
-echo "Enabling updates-testing repository"
-$LILTO ooe.sh sudo dnf install -y 'dnf-command(config-manager)'
-$LILTO ooe.sh sudo dnf config-manager --set-enabled updates-testing
+# Do not enable update-stesting on the previous Fedora release
+if [[ "$FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then
+    warn "Enabling updates-testing repository for image based on $FEDORA_BASE_IMAGE"
+    $LILTO ooe.sh sudo dnf install -y 'dnf-command(config-manager)'
+    $LILTO ooe.sh sudo dnf config-manager --set-enabled updates-testing
+else
+    warn "NOT enabling updates-testing repository for image based on $PRIOR_FEDORA_BASE_IMAGE"
+fi
 
 echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'"
 REMOVE_PACKAGES=()
@@ -98,6 +103,7 @@ case "$OS_RELEASE_VER" in
             python2-future
             runc
         )
+        REMOVE_PACKAGES+=(crun)
         ;;
     31)
         INSTALL_PACKAGES+=(crun)
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 5364dd510..d2e1b8767 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -6,15 +6,19 @@ source $(dirname $0)/lib.sh
 
 req_env_var USER HOME GOSRC SCRIPT_BASE SETUP_MARKER_FILEPATH
 
-show_env_vars
-
 # Ensure this script only executes successfully once and always logs ending timestamp
-[[ ! -e "$SETUP_MARKER_FILEPATH" ]] || exit 0
+if [[ -e "$SETUP_MARKER_FILEPATH" ]]; then
+    show_env_vars
+    exit 0
+fi
+
 exithandler() {
     RET=$?
     echo "."
     echo "$(basename $0) exit status: $RET"
     [[ "$RET" -eq "0" ]] && date +%s >> "$SETUP_MARKER_FILEPATH"
+    show_env_vars
+    [ "$RET" -eq "0" ]] || warn "Non-zero exit caused by error ABOVE env. var. display."
 }
 trap exithandler EXIT
 
@@ -46,42 +50,59 @@ case "${OS_RELEASE_ID}" in
         # All SELinux distros need this for systemd-in-a-container
         setsebool container_manage_cgroup true
         if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
-            bash "$SCRIPT_BASE/add_second_partition.sh"; fi
+            bash "$SCRIPT_BASE/add_second_partition.sh"
+        fi
 
-        if [[ "$OS_RELEASE_VER" == "31" ]]; then
-            warn "Switching io schedular to deadline to avoid RHBZ 1767539"
-            warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447"
-            echo "mq-deadline" > /sys/block/sda/queue/scheduler
-            cat /sys/block/sda/queue/scheduler
+        warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539"
+        warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447"
+        echo "mq-deadline" > /sys/block/sda/queue/scheduler
+        cat /sys/block/sda/queue/scheduler
 
-            warn "Forcing systemd cgroup manager"
-            X=$(echo "export CGROUP_MANAGER=systemd" | \
-                tee -a /etc/environment) && eval "$X" && echo "$X"
+        warn "Forcing systemd cgroup manager"
+        X=$(echo "export CGROUP_MANAGER=systemd" | \
+            tee -a /etc/environment) && eval "$X" && echo "$X"
+        ;;
+    centos)  # Current VM is an image-builder-image no local podman/testing
+        echo "No further setup required for VM image building"
+        exit 0
+        ;;
+    *) bad_os_id_ver ;;
+esac
 
-            warn "Testing with crun instead of runc"
-            X=$(echo "export OCI_RUNTIME=/usr/bin/crun" | \
-                tee -a /etc/environment) && eval "$X" && echo "$X"
+# Reload to incorporate any changes from above
+source "$SCRIPT_BASE/lib.sh"
+
+case "$CG_FS_TYPE" in
+    tmpfs)
+        warn "Forcing testing with runc instead of crun"
+        X=$(echo "export OCI_RUNTIME=/usr/bin/runc" | \
+            tee -a /etc/environment) && eval "$X" && echo "$X"
+        ;;
+    cgroup2fs)
+        # This is necessary since we've built/installed from source, which uses runc as the default.
+        warn "Forcing testing with crun instead of runc"
+        X=$(echo "export OCI_RUNTIME=/usr/bin/crun" | \
+            tee -a /etc/environment) && eval "$X" && echo "$X"
+
+        if [[ "$MOD_LIBPOD_CONF" == "true" ]]; then
+            warn "Updating runtime setting in repo. copy of libpod.conf"
+            sed -i -r -e 's/^runtime = "runc"/runtime = "crun"/' $GOSRC/libpod.conf
+            git diff $GOSRC/libpod.conf
+        fi
 
+        if [[ "$OS_RELEASE_ID" == "fedora" ]]; then
             warn "Upgrading to the latest crun"
             # Normally not something to do for stable testing
             # but crun is new, and late-breaking fixes may be required
             # on short notice
             dnf update -y crun
-
-            #warn "Setting SELinux into Permissive mode"
-            #setenforce 0
         fi
         ;;
-    centos)  # Current VM is an image-builder-image no local podman/testing
-        echo "No further setup required for VM image building"
-        exit 0
+    *)
+        die 110 "Unsure how to handle cgroup filesystem type '$CG_FS_TYPE'"
         ;;
-    *) bad_os_id_ver ;;
 esac
 
-# Reload to incorporate any changes from above
-source "$SCRIPT_BASE/lib.sh"
-
 # Must execute before possible setup_rootless()
 make install.tools
 
diff --git a/hack/get_ci_vm.sh b/hack/get_ci_vm.sh
index 768137213..7e31c19c6 100755
--- a/hack/get_ci_vm.sh
+++ b/hack/get_ci_vm.sh
@@ -96,7 +96,7 @@ env=yaml.load(open(".cirrus.yml"), Loader=yaml.SafeLoader)["env"]
 keys=[k for k in env if "ENCRYPTED" not in str(env[k])]
 for k,v in env.items():
     v=str(v)
-    if "ENCRYPTED" not in v:
+    if "ENCRYPTED" not in v and "ADD_SECOND_PARTITION" not in v:
         print("{0}=\"{1}\"".format(k, v)),
     '
 }
@@ -181,7 +181,7 @@ parse_args(){
     [[ -z "$ROOTLESS_USER" ]] || \
         ENVS="$ENVS ROOTLESS_USER=$ROOTLESS_USER"
 
-    SETUP_CMD="env $ENVS $GOSRC/contrib/cirrus/setup_environment.sh"
+    SETUP_CMD="env $ENVS ADD_SECOND_PARTITIO=True $GOSRC/contrib/cirrus/setup_environment.sh"
     VMNAME="${VMNAME:-${USER}-${IMAGE_NAME}}"
 
     CREATE_CMD="$PGCLOUD compute instances create --zone=$ZONE --image=${IMAGE_NAME} --custom-cpu=$CPUS --custom-memory=$MEMORY --boot-disk-size=$DISK --labels=in-use-by=$USER $IBI_ARGS $VMNAME"
diff --git a/pkg/bindings/test/common_test.go b/pkg/bindings/test/common_test.go
index 38f5014ca..1fc774074 100644
--- a/pkg/bindings/test/common_test.go
+++ b/pkg/bindings/test/common_test.go
@@ -240,3 +240,7 @@ func createCache() {
 	}
 	b.cleanup()
 }
+
+func isStopped(state string) bool {
+	return state == "exited" || state == "stopped"
+}
diff --git a/pkg/bindings/test/containers_test.go b/pkg/bindings/test/containers_test.go
index 6756e81c7..5a0bdebe6 100644
--- a/pkg/bindings/test/containers_test.go
+++ b/pkg/bindings/test/containers_test.go
@@ -232,7 +232,7 @@ var _ = Describe("Podman containers ", func() {
 		// Ensure container is stopped
 		data, err := containers.Inspect(connText, name, nil)
 		Expect(err).To(BeNil())
-		Expect(data.State.Status).To(Equal("exited"))
+		Expect(isStopped(data.State.Status)).To(BeTrue())
 	})
 
 	It("podman stop a running container by ID", func() {
@@ -247,7 +247,7 @@ var _ = Describe("Podman containers ", func() {
 		// Ensure container is stopped
 		data, err = containers.Inspect(connText, name, nil)
 		Expect(err).To(BeNil())
-		Expect(data.State.Status).To(Equal("exited"))
+		Expect(isStopped(data.State.Status)).To(BeTrue())
 	})
 
 })