Cirrus: Combine pre-test checks into build task

Previously, two tasks always ran first, prior to anything else.  One to
verify network and external-service connectivity.  Another to verify
certain important `.cirrus.yml` standards are met.  However, as the
total number of tasks continues to grow, the need to keep these basic
checks as dedicated prerequisites is of decreasing value/importance.
Fold these two checks into a new `pretesting_script` component of the
Fedora `build` task, on both `x86_64` and `aarch64`.

Signed-off-by: Chris Evich <cevich@redhat.com>

7 files changed, 123 insertions, 178 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 68fa158e4..230466b22 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -77,79 +77,11 @@ gcp_credentials: ENCRYPTED[a28959877b2c9c36f151781b0a05407218cda646c7d047fc556e4
 aws_credentials: ENCRYPTED[4ca070bffe28eb9b27d63c568b52970dd46f119c3a83b8e443241e895dbf1737580b4d84eed27a311a2b74287ef9f79f]
 
 
-# Attempt to prevent flakes by confirming all required external/3rd-party
-# services are available and functional.
-ext_svc_check_task:
-    alias: 'ext_svc_check'  # int. ref. name - required for depends_on reference
-    name: "Ext. services"  # Displayed Title - has no other significance
-    # Don't create this task for new tags so release process is more reliable
-    # Docs: ./contrib/cirrus/CIModes.md
-    only_if: $CIRRUS_TAG == ''
-    # Default/small container image to execute tasks with
-    container: &smallcontainer
-        image: ${CTR_FQIN}
-        # Resources are limited across ALL currently executing tasks
-        # ref: https://cirrus-ci.org/guide/linux/#linux-containers
-        cpu: 2
-        memory: 2
-    env:
-        TEST_FLAVOR: ext_svc
-        CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
-    # NOTE: The default way Cirrus-CI clones is *NOT* compatible with
-    #       environment expectations in contrib/cirrus/lib.sh.  Specifically
-    #       the 'origin' remote must be defined, and all remote branches/tags
-    #       must be available for reference from CI scripts.
-    clone_script: &full_clone |
-          cd /
-          rm -rf $CIRRUS_WORKING_DIR
-          mkdir -p $CIRRUS_WORKING_DIR
-          git clone --recursive --branch=$DEST_BRANCH https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR
-          cd $CIRRUS_WORKING_DIR
-          git remote update origin
-          if [[ -n "$CIRRUS_PR" ]]; then # running for a PR
-              git fetch origin pull/$CIRRUS_PR/head:pull/$CIRRUS_PR
-              git checkout pull/$CIRRUS_PR
-          else
-              git reset --hard $CIRRUS_CHANGE_IN_REPO
-          fi
-          # Some test operations & checks require a git "identity"
-          _gc='git config --file /root/.gitconfig'
-          $_gc user.email "TMcTestFace@example.com"
-          $_gc user.name "Testy McTestface"
-
-    setup_script: &setup '$GOSRC/$SCRIPT_BASE/setup_environment.sh'
-    main_script: &main '/usr/bin/time --verbose --output="$STATS_LOGFILE" $GOSRC/$SCRIPT_BASE/runner.sh'
-    always: &runner_stats
-        runner_stats_artifacts:
-            path: ./*-${STATS_LOGFILE_SFX}
-            type: text/plain
-
-
-# Execute some quick checks to confirm this YAML file and all
-# automation-related shell scripts are sane.
-automation_task:
-    alias: 'automation'
-    name: "Check Automation"
-    # This task is not needed for branches, tags, or cron runs.
-    # Docs: ./contrib/cirrus/CIModes.md
-    only_if: &is_pr "$CIRRUS_PR != ''"
-    container: *smallcontainer
-    env:
-        TEST_FLAVOR: automation
-        CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
-        TEST_ENVIRON: container
-    clone_script: *full_clone
-    setup_script: *setup
-    main_script: *main
-    always: *runner_stats
-
-
-# N/B: The two following tasks are critical. They build all binaries for all supported
-# OS platforms and versions on x86_64 and aarch64. On success, the contents of the repository
-# are preserved as an artifact.  This saves most subsequent tasks about
-# 3 minutes of otherwise duplicative effort.  It also ensures that the
-# exact same binaries used throughout CI testing, are available for
-# future consumption|inspection by the final 'artifacts' task.
+# N/B: This matrix of build tasks are critical to CI, along with the following
+# aarch64 task. They build binaries for all CI platforms, and versions. On
+# success, the contents of the repository are preserved as an artifact for
+# consumption by most subsequent CI tasks.  This saves about 3-5 minutes of
+# otherwise duplicative effort in most tasks.
 build_task:
     alias: 'build'
     name: 'Build for $DISTRO_NV'
@@ -186,9 +118,36 @@ build_task:
               CI_DESIRED_RUNTIME: runc
     env:
         TEST_FLAVOR: build
-    clone_script: *full_clone
-    setup_script: *setup
-    main_script: *main
+    # NOTE: The default way Cirrus-CI clones is *NOT* compatible with
+    #       environment expectations in contrib/cirrus/lib.sh.  Specifically
+    #       the 'origin' remote must be defined, and all remote branches/tags
+    #       must be available for reference from CI scripts.
+    clone_script: &full_clone |
+          cd /
+          rm -rf $CIRRUS_WORKING_DIR
+          mkdir -p $CIRRUS_WORKING_DIR
+          git clone --recursive --branch=$DEST_BRANCH https://x-access-token:${CIRRUS_REPO_CLONE_TOKEN}@github.com/${CIRRUS_REPO_FULL_NAME}.git $CIRRUS_WORKING_DIR
+          cd $CIRRUS_WORKING_DIR
+          git remote update origin
+          if [[ -n "$CIRRUS_PR" ]]; then # running for a PR
+              git fetch origin pull/$CIRRUS_PR/head:pull/$CIRRUS_PR
+              git checkout pull/$CIRRUS_PR
+          else
+              git reset --hard $CIRRUS_CHANGE_IN_REPO
+          fi
+          # Some test operations & checks require a git "identity"
+          _gc='git config --file /root/.gitconfig'
+          $_gc user.email "TMcTestFace@example.com"
+          $_gc user.name "Testy McTestface"
+    # Attempt to prevent flakes by confirming basic environment expectations,
+    # network service connectivity and essential container image availability.
+    prebuild_script: &prebuild $SCRIPT_BASE/prebuild.sh
+    # Standard setup stage call, used by nearly every task in CI.
+    setup_script: &setup '$GOSRC/$SCRIPT_BASE/setup_environment.sh'
+    # Attempt to prevent flakes by confirming automation environment and
+    # all required external/3rd-party services are available and functional.
+    # Standard main execution stage call, used by nearly every task in CI.
+    main_script: &main '/usr/bin/time --verbose --output="$STATS_LOGFILE" $GOSRC/$SCRIPT_BASE/runner.sh'
     # Cirrus-CI is very slow uploading one file at time, and the repo contains
     # thousands of files.  Speed this up by archiving into tarball first.
     repo_prep_script: &repo_prep >-
@@ -196,7 +155,10 @@ build_task:
     repo_artifacts: &repo_artifacts
         path: ./repo.tbz
         type: application/octet-stream
-    always: *runner_stats
+    always: &runner_stats
+        runner_stats_artifacts:
+            path: ./*-${STATS_LOGFILE_SFX}
+            type: text/plain
 
 
 build_aarch64_task:
@@ -218,6 +180,7 @@ build_aarch64_task:
         CI_DESIRED_RUNTIME: crun
         TEST_FLAVOR: build
     clone_script: *full_clone
+    prebuild_script: *prebuild
     setup_script: *setup
     main_script: *main
     # Cirrus-CI is very slow uploading one file at time, and the repo contains
@@ -242,10 +205,8 @@ validate_task:
     # to nonsequential PR merging practices, will be caught on a future PR,
     # build or test task failures.
     # Docs: ./contrib/cirrus/CIModes.md
-    only_if: *is_pr
+    only_if: &is_pr "$CIRRUS_PR != ''"
     depends_on:
-        - ext_svc_check
-        - automation
         - build
     # golangci-lint is a very, very hungry beast.
     gce_instance: &bigvm
@@ -281,8 +242,6 @@ validate_aarch64_task:
     # Docs: ./contrib/cirrus/CIModes.md
     only_if: *is_pr
     depends_on:
-        - ext_svc_check
-        - automation
         - build_aarch64
     # golangci-lint is a very, very hungry beast.
     ec2_instance: *standard_build_ec2_aarch64
@@ -377,7 +336,12 @@ consistency_task:
     only_if: *is_pr
     depends_on:
         - build
-    container: *smallcontainer
+    container: &smallcontainer
+        image: ${CTR_FQIN}
+        # Resources are limited across ALL currently executing tasks
+        # ref: https://cirrus-ci.org/guide/linux/#linux-containers
+        cpu: 2
+        memory: 2
     env:
         <<: *stdenvars
         TEST_FLAVOR: consistency
@@ -929,8 +893,6 @@ image_build_task: &image-build
     # this task to a specific Cirrus-Cron entry with this name.
     # Docs: ./contrib/cirrus/CIModes.md
     only_if: $CIRRUS_CRON == 'multiarch'
-    depends_on:
-        - ext_svc_check
     timeout_in: 120m  # emulation is sssllllooooowwww
     gce_instance:
         <<: *standardvm
@@ -1010,8 +972,6 @@ success_task:
     alias: success
     # N/B: ALL tasks must be listed here, minus their '_task' suffix.
     depends_on:
-        - ext_svc_check
-        - automation
         - build
         - build_aarch64
         - validate
diff --git a/contrib/cirrus/CIModes.md b/contrib/cirrus/CIModes.md
index 0b5a189a6..e7fcccf3c 100644
--- a/contrib/cirrus/CIModes.md
+++ b/contrib/cirrus/CIModes.md
@@ -43,8 +43,6 @@ of this document, it's not possible to override the behavior of `$CIRRUS_PR`.
 ## Cirrus Task contexts and runtime modes
 
 ### Intended general PR Tasks (*italic*: matrix)
-+ ext_svc_check
-+ automation
 + *build*
 + validate
 + bindings
@@ -76,8 +74,6 @@ of this document, it's not possible to override the behavior of `$CIRRUS_PR`.
 + release_test
 
 ### Intended `[CI:DOCS]` PR Tasks:
-+ ext_svc_check
-+ automation
 + *build*
 + validate
 + swagger
@@ -86,8 +82,6 @@ of this document, it's not possible to override the behavior of `$CIRRUS_PR`.
 + success
 
 ### Intended `[CI:COPR]` PR Tasks:
-+ ext_svc_check
-+ automation
 + *build*
 + validate
 + swagger
@@ -96,8 +90,6 @@ of this document, it's not possible to override the behavior of `$CIRRUS_PR`.
 + success
 
 ### Intend `[CI:BUILD]` PR Tasks:
-+ ext_svc_check
-+ automation
 + *build*
 + validate
 + consistency
@@ -109,8 +101,6 @@ of this document, it's not possible to override the behavior of `$CIRRUS_PR`.
 + artifacts
 
 ### Intended Branch tasks (and Cirrus-cron jobs, except "multiarch"):
-+ ext_svc_check
-+ *build*
 + swagger
 + *alt_build*
 + osx_alt_build
@@ -123,7 +113,6 @@ of this document, it's not possible to override the behavior of `$CIRRUS_PR`.
 + artifacts
 
 ### Intended for "multiarch" Cirrus-Cron (always a branch):
-+ ext_svc_check
 + image_build
 + meta
 + success
diff --git a/contrib/cirrus/ext_svc_check.sh b/contrib/cirrus/ext_svc_check.sh
deleted file mode 100755
index 146919c39..000000000
--- a/contrib/cirrus/ext_svc_check.sh
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-# This script attempts basic confirmation of functional networking
-# by connecting to a set of essential external servers and failing
-# if any cannot be reached.  It's intended for use early on in the
-# podman CI system, to help prevent wasting time on tests that can't
-# succeed due to some outage or another.
-
-# shellcheck source=./contrib/cirrus/lib.sh
-source $(dirname $0)/lib.sh
-
-cat ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/required_host_ports.txt | \
-    while read host port
-    do
-        if [[ "$port" -eq "443" ]]
-        then
-            echo "SSL/TLS to $host:$port"
-            echo -n '' | \
-                err_retry 9 1000 "" openssl s_client -quiet -no_ign_eof -connect $host:$port
-        else
-            echo "Connect to $host:$port"
-            err_retry 9 1000 1 nc -zv -w 13 $host $port
-        fi
-    done
-
-# Verify we can pull metadata from a few key testing images on quay.io
-# in the 'libpod' namespace.  This is mostly aimed at validating the
-# quay.io service is up and responsive.  Images were hand-picked with
-# egrep -ro 'quay.io/libpod/.+:latest' test | sort -u
-TEST_IMGS=(\
-    alpine:latest
-    busybox:latest
-    alpine_labels:latest
-    alpine_nginx:latest
-    alpine_healthcheck:latest
-    badhealthcheck:latest
-    cirros:latest
-)
-
-echo "Checking quay.io test image accessibility"
-for testimg in "${TEST_IMGS[@]}"; do
-    fqin="quay.io/libpod/$testimg"
-    echo "    $fqin"
-    skopeo inspect --retry-times 5 "docker://$fqin" | jq . > /dev/null
-done
diff --git a/contrib/cirrus/prebuild.sh b/contrib/cirrus/prebuild.sh
new file mode 100755
index 000000000..7695128de
--- /dev/null
+++ b/contrib/cirrus/prebuild.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# This script attempts to confirm functional networking and
+# connectivity to essential external servers.  It also verifies
+# some basic environmental expectations and shell-script sanity.
+# It's intended for use early on in the podman CI system, to help
+# prevent wasting time on tests that can't succeed due to some
+# outage, failure, or missed expectation.
+
+source /etc/automation_environment
+source $AUTOMATION_LIB_PATH/common_lib.sh
+
+req_env_vars CI DEST_BRANCH IMAGE_SUFFIX TEST_FLAVOR TEST_ENVIRON \
+             PODBIN_NAME PRIV_NAME DISTRO_NV AUTOMATION_LIB_PATH \
+             SCRIPT_BASE CIRRUS_WORKING_DIR FEDORA_NAME UBUNTU_NAME \
+             VM_IMAGE_NAME
+
+# There's no need to perform further checks on more than one
+# CI platform.  These variables are defined in .cirrus.yml
+# shellcheck disable=SC2154
+if [[ ! "${DISTRO_NV}" =~ ${FEDORA_NAME} ]]; then
+    echo "Skipping additional checks on $DISTRO_NV"
+    exit 0
+fi
+
+# shellcheck disable=SC2154
+$SCRIPT_BASE/cirrus_yaml_test.py
+
+ooe.sh dnf install -y ShellCheck  # small/quick addition
+
+shellcheck --color=always --format=tty \
+    --shell=bash --external-sources \
+    --enable add-default-case,avoid-nullary-conditions,check-unassigned-uppercase \
+    --exclude SC2046,SC2034,SC2090,SC2064 \
+    --wiki-link-count=0 --severity=warning \
+    $SCRIPT_BASE/*.sh hack/get_ci_vm.sh
+
+# shellcheck disable=SC2154
+cat ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/required_host_ports.txt | \
+    while read host port
+    do
+        if [[ "$port" -eq "443" ]]
+        then
+            echo "SSL/TLS to $host:$port"
+            echo -n '' | \
+                err_retry 9 1000 "" openssl s_client -quiet -no_ign_eof -connect $host:$port
+        else
+            echo "Connect to $host:$port"
+            err_retry 9 1000 1 nc -zv -w 13 $host $port
+        fi
+    done
+
+# Verify we can pull metadata from a few key testing images on quay.io
+# in the 'libpod' namespace.  This is mostly aimed at validating the
+# quay.io service is up and responsive.  Images were hand-picked with
+# egrep -ro 'quay.io/libpod/.+:latest' test | sort -u
+TEST_IMGS=(\
+    alpine:latest
+    busybox:latest
+    alpine_labels:latest
+    alpine_nginx:latest
+    alpine_healthcheck:latest
+    badhealthcheck:latest
+    cirros:latest
+)
+
+echo "Checking quay.io test image accessibility"
+for testimg in "${TEST_IMGS[@]}"; do
+    fqin="quay.io/libpod/$testimg"
+    echo "    $fqin"
+    skopeo inspect --retry-times 5 "docker://$fqin" | jq -e . > /dev/null
+done
diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh
index c44251e2f..d0657ab5f 100755
--- a/contrib/cirrus/runner.sh
+++ b/contrib/cirrus/runner.sh
@@ -19,21 +19,6 @@ set -eo pipefail
 # shellcheck source=contrib/cirrus/lib.sh
 source $(dirname $0)/lib.sh
 
-function _run_ext_svc() {
-    $SCRIPT_BASE/ext_svc_check.sh
-}
-
-function _run_automation() {
-    $SCRIPT_BASE/cirrus_yaml_test.py
-
-    req_env_vars CI DEST_BRANCH IMAGE_SUFFIX TEST_FLAVOR TEST_ENVIRON \
-                 PODBIN_NAME PRIV_NAME DISTRO_NV CONTAINER USER HOME \
-                 UID AUTOMATION_LIB_PATH SCRIPT_BASE OS_RELEASE_ID \
-                 CG_FS_TYPE
-    bigto ooe.sh dnf install -y ShellCheck  # small/quick addition
-    $SCRIPT_BASE/shellcheck.sh
-}
-
 function _run_validate() {
     # TODO: aarch64 images need python3-devel installed
     # https://github.com/containers/automation_images/issues/159
@@ -423,6 +408,8 @@ function _bail_if_test_can_be_skipped() {
         return 0
     fi
 
+    # Defined by Cirrus-CI for all tasks
+    # shellcheck disable=SC2154
     head=$CIRRUS_CHANGE_IN_REPO
     base=$(git merge-base $DEST_BRANCH $head)
     diffs=$(git diff --name-only $base $head)
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 4c86bbcfa..e43231640 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -228,13 +228,11 @@ esac
 # Required to be defined by caller: The primary type of testing that will be performed
 # shellcheck disable=SC2154
 case "$TEST_FLAVOR" in
-    ext_svc) ;;
     validate)
         dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
         # For some reason, this is also needed for validation
         make .install.pre-commit .install.gitvalidation
         ;;
-    automation) ;;
     altbuild)
         # Defined in .cirrus.yml
         # shellcheck disable=SC2154
diff --git a/contrib/cirrus/shellcheck.sh b/contrib/cirrus/shellcheck.sh
deleted file mode 100755
index 667d30c91..000000000
--- a/contrib/cirrus/shellcheck.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-# shellcheck source=./contrib/cirrus/lib.sh
-source $(dirname $0)/lib.sh
-
-cd $CIRRUS_WORKING_DIR
-shellcheck --color=always --format=tty \
-    --shell=bash --external-sources \
-    --enable add-default-case,avoid-nullary-conditions,check-unassigned-uppercase \
-    --exclude SC2046,SC2034,SC2090,SC2064 \
-    --wiki-link-count=0 --severity=warning \
-    $SCRIPT_BASE/*.sh hack/get_ci_vm.sh
-
-echo "Shellcheck: PASS"