Cirrus: Implement podman automation 2.0

Reimplement CI-automation to remove accumulated technical-debt and
optimize workflow.  The task-dependency graph designed goal was to
shorten it's depth and increase width (i.e. more parallelism).  A
reduction in redundant building (and 3rd party module download) was
also realized by caching `$GOPATH` and `$GOCACHE` early on.  This
cache is then reused in favor of a fresh clone of the repository
(when possible).

Note: The system tests typically execute MUCH faster than the
integration tests.  However, contrary to a fail-fast/fail-early
principal, they are executed last.  This was implemented due to
debug-ability related concerns/preferences of the primary
(golang-centric) project developers.

Signed-off-by: Chris Evich <cevich@redhat.com>

1 files changed, 149 insertions, 81 deletions

diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index b406d7b5c..c064b6840 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -2,119 +2,187 @@
 
 set -e
 
+# shellcheck source=./contrib/cirrus/lib.sh
 source $(dirname $0)/lib.sh
 
-req_env_var USER HOME GOSRC SCRIPT_BASE SETUP_MARKER_FILEPATH
+die_unknown() {
+    local var_name="$1"
+    req_env_vars var_name
+    local var_value="${!var_name}"
+    die "Unknown/unsupported \$$var_name '$var_value'"
+}
 
-# Ensure this script only executes successfully once and always logs ending timestamp
-if [[ -e "$SETUP_MARKER_FILEPATH" ]]; then
-    show_env_vars
-    exit 0
-fi
+msg "************************************************************"
+msg "Setting up runtime environment"
+msg "************************************************************"
+show_env_vars
 
-exithandler() {
-    RET=$?
-    echo "."
-    echo "$(basename $0) exit status: $RET"
-    [[ "$RET" -eq "0" ]] && date +%s >> "$SETUP_MARKER_FILEPATH"
-    show_env_vars
-    [[ "$RET" -eq "0" ]] || warn "Non-zero exit caused by error ABOVE env. var. display."
-}
-trap exithandler EXIT
+req_env_vars USER HOME GOSRC SCRIPT_BASE TEST_FLAVOR TEST_ENVIRON \
+             PODBIN_NAME PRIV_NAME DISTRO_NV
 
 # Verify basic dependencies
 for depbin in go rsync unzip sha256sum curl make python3 git
 do
     if ! type -P "$depbin" &> /dev/null
     then
-        echo "***** WARNING: $depbin binary not found in $PATH *****"
+        warn "$depbin binary not found in $PATH"
     fi
 done
 
-# Sometimes environment setup needs to vary between distros
-# Note: This should only be used for environment variables, and temporary workarounds.
+# This is a possible manual maintenance gaff, check to be sure everything matches.
+# shellcheck disable=SC2154
+[[ "$DISTRO_NV" == "$OS_REL_VER" ]] || \
+    die "Automation spec. '$DISTRO_NV'; actual host '$OS_REL_VER'"
+
+# Only allow this script to execute once
+if ((${SETUP_ENVIRONMENT:-0})); then
+    # Comes from automation library
+    # shellcheck disable=SC2154
+    warn "Not executing $SCRIPT_FILENAME again"
+    exit 0
+fi
+
 cd "${GOSRC}/"
-case "${OS_RELEASE_ID}" in
-    ubuntu)
-        ;;
-    fedora)
-        # All SELinux distros need this for systemd-in-a-container
-        setsebool container_manage_cgroup true
 
-        if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
-            bash "$SCRIPT_BASE/add_second_partition.sh"
+# Defined by lib.sh: Does the host support cgroups v1 or v2
+case "$CG_FS_TYPE" in
+    tmpfs)
+        if ((CONTAINER==0)); then
+            warn "Forcing testing with runc instead of crun"
+            if [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then
+                echo "export OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" >> /etc/environment
+            else
+                echo "export OCI_RUNTIME=/usr/bin/runc" >> /etc/environment
+            fi
         fi
-
-        warn "Forcing systemd cgroup manager"
-        X=$(echo "export CGROUP_MANAGER=systemd" | \
-            tee -a /etc/environment) && eval "$X" && echo "$X"
         ;;
-    centos)  # Current VM is an image-builder-image no local podman/testing
-        echo "No further setup required for VM image building"
-        exit 0
+    cgroup2fs)
+        if ((CONTAINER==0)); then
+            # This is necessary since we've built/installed from source,
+            # which uses runc as the default.
+            warn "Forcing testing with crun instead of runc"
+            echo "export OCI_RUNTIME=/usr/bin/crun" >> /etc/environment
+        fi
         ;;
-    *) bad_os_id_ver ;;
+    *) die_unknown CG_FS_TYPE
 esac
 
-# Reload to incorporate any changes from above
-source "$SCRIPT_BASE/lib.sh"
+# Required to be defined by caller: Which distribution are we testing on
+# shellcheck disable=SC2154
+case "$DISTRO_NV" in
+    ubuntu*) ;;
+    fedora*)
+        if ((CONTAINER==0)); then  # Not yet running inside a container
+            msg "Configuring / Expanding host storage."
+            # VM is setup to allow flexibility in testing alternate storage.
+            # For general use, simply make use of all available space.
+            ooe.sh bash "$SCRIPT_BASE/add_second_partition.sh"
 
-case "$CG_FS_TYPE" in
-    tmpfs)
-        warn "Forcing testing with runc instead of crun"
-        # On ubuntu, the default runc is usually not new enough.
-        if [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then
-            X=$(echo "export OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" | \
-                tee -a /etc/environment) && eval "$X" && echo "$X"
+            # All SELinux distros need this for systemd-in-a-container
+            msg "Enabling container_manage_cgroup"
+            setsebool container_manage_cgroup true
+        fi
+        ;;
+    *) die_unknown DISTRO_NV
+esac
+
+# Required to be defined by caller: The environment where primary testing happens
+# shellcheck disable=SC2154
+case "$TEST_ENVIRON" in
+    host)
+        if [[ "$OS_RELEASE_ID" == "fedora" ]]; then
+            # The e2e tests wrongly guess `--cgroup-manager cgroupfs`
+            msg "Forcing CGROUP_MANAGER=systemd"
+            _cgm="export CGROUP_MANAGER=systemd"
+            echo "$_cgm" >> /etc/environment
+            source /etc/environment
+        fi
+        ;;
+    container)
+        if ((CONTAINER==0)); then  # not yet inside a container
+            msg "Force loading iptables modules"
+            # Since CRIU 3.11, uses iptables to lock and unlock
+            # the network during checkpoint and restore.  Needs
+            # the following two modules loaded on the host.
+            modprobe ip6table_nat || :
+            modprobe iptable_nat || :
         else
-            X=$(echo "export OCI_RUNTIME=/usr/bin/runc" | \
-                tee -a /etc/environment) && eval "$X" && echo "$X"
+            # The e2e tests wrongly guess `--cgroup-manager systemd`
+            msg "Forcing CGROUP_MANAGER=cgroupfs"
+            _cgm="export CGROUP_MANAGER=cgroupfs"
+            echo "$_cgm" >> /etc/environment
+            source /etc/environment
         fi
         ;;
-    cgroup2fs)
-        # This is necessary since we've built/installed from source, which uses runc as the default.
-        warn "Forcing testing with crun instead of runc"
-        X=$(echo "export OCI_RUNTIME=/usr/bin/crun" | \
-            tee -a /etc/environment) && eval "$X" && echo "$X"
+    *) die_unknown TEST_ENVIRON
+esac
+
+# Required to be defined by caller: Are we testing as root or a regular user
+# shellcheck disable=SC2154
+case "$PRIV_NAME" in
+    root)
+        if [[ "$TEST_ENVIRON" == "container" ]] && ((container)); then
+            # There's no practical way to detect userns w/in a container
+            # affected/related tests are sensitive to this variable.
+            _suns='export SKIP_USERNS=1'
+            echo "$_suns" >> /etc/environment
+            source /etc/environment
+        fi
         ;;
-    *)
-        die 110 "Unsure how to handle cgroup filesystem type '$CG_FS_TYPE'"
+    rootless)
+        _ru="export ROOTLESS_USER='${ROOTLESS_USER:-some${RANDOM}dude}'"
+        echo "$_ru" >> /etc/environment
+        source /etc/environment
+        setup_rootless
         ;;
+    *) die_unknown PRIV_NAME
 esac
 
-# Must execute before possible setup_rootless()
-make install.tools
+# Required to be defined by caller: Are we testing podman or podman-remote client
+# shellcheck disable=SC2154
+case "$PODBIN_NAME" in
+    podman) ;;
+    remote) ;;
+    *) die_unknown PODBIN_NAME
+esac
 
-case "$SPECIALMODE" in
-    none)
-        [[ -n "$CROSS_PLATFORM" ]] || \
-            remove_packaged_podman_files
+# Required to be defined by caller: The primary type of testing that will be performed
+# shellcheck disable=SC2154
+case "$TEST_FLAVOR" in
+    ext_svc) ;;
+    smoke) ;&
+    validate)
+        # For some reason, this is also needed for validation
+        make .install.pre-commit
         ;;
+    automation) ;;
+    altbuild)
+        # Defined in .cirrus.yml
+        # shellcheck disable=SC2154
+        if [[ "$ALT_NAME" =~ RPM ]]; then
+            bigto dnf install -y glibc-minimal-langpack rpm-build
+        fi
+        ;&
+    docker-py) ;&
+    build) make clean ;;
+    unit) ;;
+    int) ;&
+    sys) ;&
+    bindings) ;&
+    swagger) ;&
     endpoint)
-        remove_packaged_podman_files
-        ;;
-    bindings)
-        remove_packaged_podman_files
-        ;;
-    rootless)
-        # Only do this once, even if ROOTLESS_USER (somehow) changes
-        if ! grep -q 'ROOTLESS_USER' /etc/environment
-        then
-            X=$(echo "export ROOTLESS_USER='${ROOTLESS_USER:-some${RANDOM}dude}'" | \
-                tee -a /etc/environment) && eval "$X" && echo "$X"
-            X=$(echo "export SPECIALMODE='${SPECIALMODE}'" | \
-                tee -a /etc/environment) && eval "$X" && echo "$X"
-            X=$(echo "export RCLI='${RCLI}'" | \
-                tee -a /etc/environment) && eval "$X" && echo "$X"
-            setup_rootless
+        # Use existing host bits when testing is to happen inside a container
+        # since this script will run again in that environment.
+        # shellcheck disable=SC2154
+        if ((CONTAINER==0)) && [[ "$TEST_ENVIRON" == "host" ]]; then
+            remove_packaged_podman_files
+            make install PREFIX=/usr ETCDIR=/etc
         fi
-        remove_packaged_podman_files
-        ;;
-    in_podman)  # Assumed to be Fedora
-        $SCRIPT_BASE/setup_container_environment.sh
         ;;
-    *)
-        die 111 "Unsupported \$SPECIALMODE: $SPECIALMODE"
+    vendor) make clean ;;
+    release) ;;
+    *) die_unknown TEST_FLAVOR
 esac
 
-install_test_configs
+# Must be the very last command.  Establishes successful setup.
+echo 'export SETUP_ENVIRONMENT=1' >> /etc/environment