Merge pull request #15782 from cevich/bundle_build_tasks

Cirrus: Task consolidation

8 files changed, 111 insertions, 108 deletions

diff --git a/contrib/cirrus/CIModes.md b/contrib/cirrus/CIModes.md
index 0b5a189a6..7d6a36cf3 100644
--- a/contrib/cirrus/CIModes.md
+++ b/contrib/cirrus/CIModes.md
@@ -43,13 +43,10 @@ of this document, it's not possible to override the behavior of `$CIRRUS_PR`.
 ## Cirrus Task contexts and runtime modes
 
 ### Intended general PR Tasks (*italic*: matrix)
-+ ext_svc_check
-+ automation
 + *build*
 + validate
 + bindings
 + swagger
-+ consistency
 + *alt_build*
 + osx_alt_build
 + docker-py_test
@@ -76,31 +73,22 @@ of this document, it's not possible to override the behavior of `$CIRRUS_PR`.
 + release_test
 
 ### Intended `[CI:DOCS]` PR Tasks:
-+ ext_svc_check
-+ automation
 + *build*
 + validate
 + swagger
-+ consistency
 + meta
 + success
 
 ### Intended `[CI:COPR]` PR Tasks:
-+ ext_svc_check
-+ automation
 + *build*
 + validate
 + swagger
-+ consistency
 + meta
 + success
 
 ### Intend `[CI:BUILD]` PR Tasks:
-+ ext_svc_check
-+ automation
 + *build*
 + validate
-+ consistency
 + *alt_build*
 + osx_alt_build
 + test_image_build
@@ -109,7 +97,6 @@ of this document, it's not possible to override the behavior of `$CIRRUS_PR`.
 + artifacts
 
 ### Intended Branch tasks (and Cirrus-cron jobs, except "multiarch"):
-+ ext_svc_check
 + *build*
 + swagger
 + *alt_build*
@@ -123,7 +110,6 @@ of this document, it's not possible to override the behavior of `$CIRRUS_PR`.
 + artifacts
 
 ### Intended for "multiarch" Cirrus-Cron (always a branch):
-+ ext_svc_check
 + image_build
 + meta
 + success
diff --git a/contrib/cirrus/check_go_changes.sh b/contrib/cirrus/check_go_changes.sh
index 3c35ce51a..a92ab03af 100755
--- a/contrib/cirrus/check_go_changes.sh
+++ b/contrib/cirrus/check_go_changes.sh
@@ -36,6 +36,8 @@ then
     exit 0
 fi
 
+# Defined by/in Cirrus-CI config.
+# shellcheck disable=SC2154
 base=$(git merge-base $DEST_BRANCH $CIRRUS_CHANGE_IN_REPO)
 diffs=$(git diff $base $CIRRUS_CHANGE_IN_REPO -- '*.go' ':^vendor/')
 
diff --git a/contrib/cirrus/ext_svc_check.sh b/contrib/cirrus/ext_svc_check.sh
deleted file mode 100755
index 146919c39..000000000
--- a/contrib/cirrus/ext_svc_check.sh
+++ /dev/null
@@ -1,47 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-# This script attempts basic confirmation of functional networking
-# by connecting to a set of essential external servers and failing
-# if any cannot be reached.  It's intended for use early on in the
-# podman CI system, to help prevent wasting time on tests that can't
-# succeed due to some outage or another.
-
-# shellcheck source=./contrib/cirrus/lib.sh
-source $(dirname $0)/lib.sh
-
-cat ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/required_host_ports.txt | \
-    while read host port
-    do
-        if [[ "$port" -eq "443" ]]
-        then
-            echo "SSL/TLS to $host:$port"
-            echo -n '' | \
-                err_retry 9 1000 "" openssl s_client -quiet -no_ign_eof -connect $host:$port
-        else
-            echo "Connect to $host:$port"
-            err_retry 9 1000 1 nc -zv -w 13 $host $port
-        fi
-    done
-
-# Verify we can pull metadata from a few key testing images on quay.io
-# in the 'libpod' namespace.  This is mostly aimed at validating the
-# quay.io service is up and responsive.  Images were hand-picked with
-# egrep -ro 'quay.io/libpod/.+:latest' test | sort -u
-TEST_IMGS=(\
-    alpine:latest
-    busybox:latest
-    alpine_labels:latest
-    alpine_nginx:latest
-    alpine_healthcheck:latest
-    badhealthcheck:latest
-    cirros:latest
-)
-
-echo "Checking quay.io test image accessibility"
-for testimg in "${TEST_IMGS[@]}"; do
-    fqin="quay.io/libpod/$testimg"
-    echo "    $fqin"
-    skopeo inspect --retry-times 5 "docker://$fqin" | jq . > /dev/null
-done
diff --git a/contrib/cirrus/postbuild.sh b/contrib/cirrus/postbuild.sh
new file mode 100755
index 000000000..47cb558e3
--- /dev/null
+++ b/contrib/cirrus/postbuild.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# This script attempts to confirm all included go modules from
+# other sources match what is expected in `vendor/modules.txt`
+# vs `go.mod`.  Also make sure that the generated bindings in
+# `pkg/bindings/...` are in sync with the code.  It's intended
+# for use after successfully building podman, to prevent wasting
+# time on tests that might otherwise succeed with bad/ugly/invalid
+# code.
+
+source /etc/automation_environment
+source $AUTOMATION_LIB_PATH/common_lib.sh
+
+# Defined by the CI system
+# shellcheck disable=SC2154
+cd $CIRRUS_WORKING_DIR
+
+showrun make .install.goimports
+showrun make vendor
+SUGGESTION="run 'make vendor' and commit all changes" ./hack/tree_status.sh
+showrun make generate-bindings
+SUGGESTION="run 'make generate-bindings' and commit all changes" ./hack/tree_status.sh
+showrun make completions
+SUGGESTION="run 'make completions' and commit all changes" ./hack/tree_status.sh
+
+# Defined in Cirrus-CI config.
+# shellcheck disable=SC2154
+$SCRIPT_BASE/check_go_changes.sh
diff --git a/contrib/cirrus/prebuild.sh b/contrib/cirrus/prebuild.sh
new file mode 100755
index 000000000..ea05d90dc
--- /dev/null
+++ b/contrib/cirrus/prebuild.sh
@@ -0,0 +1,77 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# This script attempts to confirm functional networking and
+# connectivity to essential external servers.  It also verifies
+# some basic environmental expectations and shell-script sanity.
+# It's intended for use early on in the podman CI system, to help
+# prevent wasting time on tests that can't succeed due to some
+# outage, failure, or missed expectation.
+
+source /etc/automation_environment
+source $AUTOMATION_LIB_PATH/common_lib.sh
+
+req_env_vars CI DEST_BRANCH IMAGE_SUFFIX TEST_FLAVOR TEST_ENVIRON \
+             PODBIN_NAME PRIV_NAME DISTRO_NV AUTOMATION_LIB_PATH \
+             SCRIPT_BASE CIRRUS_WORKING_DIR FEDORA_NAME UBUNTU_NAME \
+             VM_IMAGE_NAME
+
+# Defined by the CI system
+# shellcheck disable=SC2154
+cd $CIRRUS_WORKING_DIR
+
+# Defined by CI config.
+# shellcheck disable=SC2154
+showrun $SCRIPT_BASE/cirrus_yaml_test.py
+
+# Defined by CI config.
+# shellcheck disable=SC2154
+if [[ "${DISTRO_NV}" =~ fedora ]]; then
+    showrun ooe.sh dnf install -y ShellCheck  # small/quick addition
+    showrun shellcheck --color=always --format=tty \
+        --shell=bash --external-sources \
+        --enable add-default-case,avoid-nullary-conditions,check-unassigned-uppercase \
+        --exclude SC2046,SC2034,SC2090,SC2064 \
+        --wiki-link-count=0 --severity=warning \
+        $SCRIPT_BASE/*.sh hack/get_ci_vm.sh
+fi
+
+msg "Checking 3rd party network service connectivity"
+# shellcheck disable=SC2154
+cat ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/required_host_ports.txt | \
+    while read host port
+    do
+        if [[ "$port" -eq "443" ]]
+        then
+            echo "SSL/TLS to $host:$port"
+            echo -n '' | \
+                err_retry 9 1000 "" openssl s_client -quiet -no_ign_eof -connect $host:$port
+        else
+            echo "Connect to $host:$port"
+            err_retry 9 1000 1 nc -zv -w 13 $host $port
+        fi
+    done
+
+# Verify we can pull metadata from a few key testing images on quay.io
+# in the 'libpod' namespace.  This is mostly aimed at validating the
+# quay.io service is up and responsive.  Images were hand-picked with
+# egrep -ro 'quay.io/libpod/.+:latest' test | sort -u
+TEST_IMGS=(\
+    alpine:latest
+    busybox:latest
+    alpine_labels:latest
+    alpine_nginx:latest
+    alpine_healthcheck:latest
+    badhealthcheck:latest
+    cirros:latest
+)
+
+msg "Checking quay.io test image accessibility"
+for testimg in "${TEST_IMGS[@]}"; do
+    fqin="quay.io/libpod/$testimg"
+    echo "    $fqin"
+    # Belt-and-suspenders: Catch skopeo (somehow) returning False or null
+    # in addition to "bad" (invalid) JSON.
+    skopeo inspect --retry-times 5 "docker://$fqin" | jq -e . > /dev/null
+done
diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh
index c44251e2f..d360f6a04 100755
--- a/contrib/cirrus/runner.sh
+++ b/contrib/cirrus/runner.sh
@@ -19,21 +19,6 @@ set -eo pipefail
 # shellcheck source=contrib/cirrus/lib.sh
 source $(dirname $0)/lib.sh
 
-function _run_ext_svc() {
-    $SCRIPT_BASE/ext_svc_check.sh
-}
-
-function _run_automation() {
-    $SCRIPT_BASE/cirrus_yaml_test.py
-
-    req_env_vars CI DEST_BRANCH IMAGE_SUFFIX TEST_FLAVOR TEST_ENVIRON \
-                 PODBIN_NAME PRIV_NAME DISTRO_NV CONTAINER USER HOME \
-                 UID AUTOMATION_LIB_PATH SCRIPT_BASE OS_RELEASE_ID \
-                 CG_FS_TYPE
-    bigto ooe.sh dnf install -y ShellCheck  # small/quick addition
-    $SCRIPT_BASE/shellcheck.sh
-}
-
 function _run_validate() {
     # TODO: aarch64 images need python3-devel installed
     # https://github.com/containers/automation_images/issues/159
@@ -226,16 +211,6 @@ eof
     rm -f $envvarsfile
 }
 
-function _run_consistency() {
-    make vendor
-    SUGGESTION="run 'make vendor' and commit all changes" ./hack/tree_status.sh
-    make generate-bindings
-    SUGGESTION="run 'make generate-bindings' and commit all changes" ./hack/tree_status.sh
-    make completions
-    SUGGESTION="run 'make completions' and commit all changes" ./hack/tree_status.sh
-    $SCRIPT_BASE/check_go_changes.sh
-}
-
 function _run_build() {
     # Ensure always start from clean-slate with all vendor modules downloaded
     make clean
@@ -423,6 +398,8 @@ function _bail_if_test_can_be_skipped() {
         return 0
     fi
 
+    # Defined by Cirrus-CI for all tasks
+    # shellcheck disable=SC2154
     head=$CIRRUS_CHANGE_IN_REPO
     base=$(git merge-base $DEST_BRANCH $head)
     diffs=$(git diff --name-only $base $head)
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 4c86bbcfa..ca1e16544 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -228,13 +228,11 @@ esac
 # Required to be defined by caller: The primary type of testing that will be performed
 # shellcheck disable=SC2154
 case "$TEST_FLAVOR" in
-    ext_svc) ;;
     validate)
         dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
         # For some reason, this is also needed for validation
         make .install.pre-commit .install.gitvalidation
         ;;
-    automation) ;;
     altbuild)
         # Defined in .cirrus.yml
         # shellcheck disable=SC2154
@@ -366,10 +364,6 @@ case "$TEST_FLAVOR" in
             docker.io/gitlab/gitlab-runner-helper:x86_64-latest-pwsh
         ;;
     swagger) ;&  # use next item
-    consistency)
-        make clean
-        make .install.goimports
-        ;;
     release) ;;
     *) die_unknown TEST_FLAVOR
 esac
diff --git a/contrib/cirrus/shellcheck.sh b/contrib/cirrus/shellcheck.sh
deleted file mode 100755
index 667d30c91..000000000
--- a/contrib/cirrus/shellcheck.sh
+++ /dev/null
@@ -1,16 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-# shellcheck source=./contrib/cirrus/lib.sh
-source $(dirname $0)/lib.sh
-
-cd $CIRRUS_WORKING_DIR
-shellcheck --color=always --format=tty \
-    --shell=bash --external-sources \
-    --enable add-default-case,avoid-nullary-conditions,check-unassigned-uppercase \
-    --exclude SC2046,SC2034,SC2090,SC2064 \
-    --wiki-link-count=0 --severity=warning \
-    $SCRIPT_BASE/*.sh hack/get_ci_vm.sh
-
-echo "Shellcheck: PASS"