Merge pull request #14016 from cevich/fix_caching

Cirrus: Simplify use of cache in automation

6 files changed, 144 insertions, 156 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 958c6b0c7..324fd32f6 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -29,7 +29,7 @@ env:
     UBUNTU_NAME: "ubuntu-2110"
 
     # Google-cloud VM Images
-    IMAGE_SUFFIX: "c4955393725038592"
+    IMAGE_SUFFIX: "c6211193021923328"
     FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}"
     UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}"
@@ -51,6 +51,11 @@ env:
     VM_IMAGE_NAME:           # One of the "Google-cloud VM Images" (above)
     CTR_FQIN:                # One of the "Container FQIN's" (above)
 
+    # Curl-command prefix for downloading task artifacts, simply add the
+    # the url-encoded task name, artifact name, and path as a suffix.
+    ARTCURL: >-
+        curl --fail --location -O
+        --url https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}
 
 # Default timeout for each task
 timeout_in: 60m
@@ -96,7 +101,6 @@ ext_svc_check_task:
           _gc='git config --file /root/.gitconfig'
           $_gc user.email "TMcTestFace@example.com"
           $_gc user.name "Testy McTestface"
-          make install.tools
 
     setup_script: &setup '$GOSRC/$SCRIPT_BASE/setup_environment.sh'
     main_script: &main '/usr/bin/time --verbose --output="$STATS_LOGFILE" $GOSRC/$SCRIPT_BASE/runner.sh'
@@ -123,14 +127,12 @@ automation_task:
     always: *runner_stats
 
 
-# N/B: This task is critical.  It builds all binaries and release archives
-# for the project, using all primary OS platforms and versions.  Assuming
-# the builds are successful, a cache is stored of the entire `$GOPATH`
-# contents.  For all subsequent tasks, the _BUILD_CACHE_HANDLE value
-# is used as a key to reuse this cache, saving both time and money.
-# The only exceptions are tasks which only run inside a container, they
-# will not have access the cache and therefore must rely on cloning the
-# repository.
+# N/B: This task is critical.  It builds all binaries for all supported
+# OS platforms and versions.  On success, the contents of the repository
+# are preserved as an artifact.  This saves most subsequent tasks about
+# 3 minutes of otherwise duplicative effort.  It also ensures that the
+# exact same binaries used throughout CI testing, are available for
+# future consumption|inspection by the final 'artifacts' task.
 build_task:
     alias: 'build'
     name: 'Build for $DISTRO_NV'
@@ -151,37 +153,27 @@ build_task:
               VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
               CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
               # ID for re-use of build output
-              _BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
         - env: &priorfedora_envvars
               DISTRO_NV: ${PRIOR_FEDORA_NAME}
               VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
               CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
-              _BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
         - env: &ubuntu_envvars
               DISTRO_NV: ${UBUNTU_NAME}
               VM_IMAGE_NAME: ${UBUNTU_CACHE_IMAGE_NAME}
               CTR_FQIN: ${UBUNTU_CONTAINER_FQIN}
-              _BUILD_CACHE_HANDLE: ${UBUNTU_NAME}-build-${CIRRUS_BUILD_ID}
     env:
         TEST_FLAVOR: build
-    # Ref: https://cirrus-ci.org/guide/writing-tasks/#cache-instruction
-    gopath_cache:  &gopath_cache
-        folder: *gopath  # Required hard-coded path, no variables.
-        fingerprint_script: echo "$_BUILD_CACHE_HANDLE"
-        # Cheat: Clone here when cache is empty, guaranteeing consistency.
-        populate_script: *full_clone
-    # A normal clone would invalidate useful cache
-    clone_script: &noop mkdir -p $CIRRUS_WORKING_DIR
+    clone_script: *full_clone
     setup_script: *setup
     main_script: *main
-    always: &binary_artifacts
-        <<: *runner_stats
-        gosrc_artifacts:
-            path: ./*  # Grab everything in top-level $GOSRC
-            type: application/octet-stream
-        binary_artifacts:
-            path: ./bin/*
-            type: application/octet-stream
+    # Cirrus-CI is very slow uploading one file at time, and the repo contains
+    # thousands of files.  Speed this up by archiving into tarball first.
+    repo_prep_script: &repo_prep >-
+        tar cjf /tmp/repo.tbz -C $GOSRC . && mv /tmp/repo.tbz $GOSRC/
+    repo_artifacts: &repo_artifacts
+        path: ./repo.tbz
+        type: application/octet-stream
+    always: *runner_stats
 
 
 # Confirm the result of building on at least one platform appears sane.
@@ -208,10 +200,12 @@ validate_task:
     env:
         <<: *stdenvars
         TEST_FLAVOR: validate
-    gopath_cache: &ro_gopath_cache
-        <<: *gopath_cache
-        reupload_on_changes: false
-    clone_script: *noop
+    # N/B: This script depends on ${DISTRO_NV} being defined for the task.
+    clone_script: &get_gosrc |
+        cd /tmp
+        echo "$ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tbz"
+        time $ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tbz
+        time tar xjf /tmp/repo.tbz -C $GOSRC
     setup_script: *setup
     main_script: *main
     always: *runner_stats
@@ -231,8 +225,7 @@ bindings_task:
     env:
         <<: *stdenvars
         TEST_FLAVOR: bindings
-    gopath_cache: *ro_gopath_cache
-    clone_script: *noop  # Comes from cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always: &logs_artifacts
@@ -260,19 +253,18 @@ swagger_task:
     env:
         <<: *stdenvars
         TEST_FLAVOR: swagger
-        # TODO: Due to podman 3.0 activity (including new images), avoid
-        # disturbing the status-quo just to incorporate this one new
-        # container image.  Uncomment line below when CI activities normalize.
-        #CTR_FQIN: 'quay.io/libpod/gcsupld:${IMAGE_SUFFIX}'
-        CTR_FQIN: 'quay.io/libpod/gcsupld:c4813063494828032'
+        CTR_FQIN: 'quay.io/libpod/gcsupld:${IMAGE_SUFFIX}'
         GCPJSON: ENCRYPTED[927dc01e755eaddb4242b0845cf86c9098d1e3dffac38c70aefb1487fd8b4fe6dd6ae627b3bffafaba70e2c63172664e]
         GCPNAME: ENCRYPTED[c145e9c16b6fb88d476944a454bf4c1ccc84bb4ecaca73bdd28bdacef0dfa7959ebc8171a27b2e4064d66093b2cdba49]
         GCPPROJECT: 'libpod-218412'
-    gopath_cache: *ro_gopath_cache
-    clone_script: *noop  # Comes from cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
-    always: *binary_artifacts
+    always:
+        <<: *runner_stats
+        swagger_artifacts:
+            path: ./swagger.yaml
+            type: text/plain
 
 
 # Check that all included go modules from other sources match
@@ -291,7 +283,7 @@ consistency_task:
         TEST_FLAVOR: consistency
         TEST_ENVIRON: container
         CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
-    clone_script: *full_clone  # build-cache not available to container tasks
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always: *runner_stats
@@ -322,11 +314,14 @@ alt_build_task:
             ALT_NAME: 'Test build RPM'
       - env:
             ALT_NAME: 'Alt Arch. Cross'
-    gopath_cache: *ro_gopath_cache
-    clone_script: *noop  # Comes from cache
+    # This task cannot make use of the shared repo.tbz artifact.
+    clone_script: *full_clone
     setup_script: *setup
     main_script: *main
-    always: *binary_artifacts
+    # Produce a new repo.tbz artifact for consumption by 'artifacts' task.
+    repo_prep_script: *repo_prep
+    repo_artifacts: *repo_artifacts
+    always: *runner_stats
 
 
 # Confirm building the remote client, natively on a Mac OS-X VM.
@@ -348,10 +343,14 @@ osx_alt_build_task:
         - brew install go-md2man
         - go version
     build_amd64_script:
-        - make podman-remote-release-darwin_amd64.zip
+        - make podman-remote-release-darwin_amd64.zip GOARCH=amd64
     build_arm64_script:
         - make podman-remote-release-darwin_arm64.zip GOARCH=arm64
-    always: *binary_artifacts
+    # This task cannot make use of the shared repo.tbz artifact and must
+    # produce a new repo.tbz artifact for consumption by 'artifacts' task.
+    repo_prep_script: *repo_prep
+    repo_artifacts: *repo_artifacts
+    always: *runner_stats
 
 
 # Verify podman is compatible with the docker python-module.
@@ -367,8 +366,7 @@ docker-py_test_task:
         <<: *stdenvars
         TEST_FLAVOR: docker-py
         TEST_ENVIRON: container
-    gopath_cache: *ro_gopath_cache
-    clone_script: *noop  # Comes from cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always: *runner_stats
@@ -395,8 +393,7 @@ unit_test_task:
     gce_instance: *standardvm
     env:
         TEST_FLAVOR: unit
-    clone_script: *noop  # Comes from cache
-    gopath_cache: *ro_gopath_cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always: *logs_artifacts
@@ -416,8 +413,7 @@ apiv2_test_task:
     env:
         <<: *stdenvars
         TEST_FLAVOR: apiv2
-    clone_script: *noop  # Comes from cache
-    gopath_cache: *ro_gopath_cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always: *logs_artifacts
@@ -431,8 +427,6 @@ compose_test_task:
     depends_on:
         - validate
     gce_instance: *standardvm
-    env:
-        <<: *stdenvars
     matrix:
       - env:
             TEST_FLAVOR: compose
@@ -446,8 +440,9 @@ compose_test_task:
       - env:
             TEST_FLAVOR: compose_v2
             PRIV_NAME: rootless
-    clone_script: *noop  # Comes from cache
-    gopath_cache: *ro_gopath_cache
+    env:
+        <<: *stdenvars
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always: *logs_artifacts
@@ -469,8 +464,7 @@ local_integration_test_task: &local_integration_test_task
     timeout_in: 90m
     env:
         TEST_FLAVOR: int
-    clone_script: *noop  # Comes from cache
-    gopath_cache: *ro_gopath_cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always: &int_logs_artifacts
@@ -516,8 +510,7 @@ container_integration_test_task:
     env:
         TEST_FLAVOR: int
         TEST_ENVIRON: container
-    clone_script: *noop  # Comes from cache
-    gopath_cache: *ro_gopath_cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always: *int_logs_artifacts
@@ -537,8 +530,7 @@ rootless_integration_test_task:
     env:
         TEST_FLAVOR: int
         PRIV_NAME: rootless
-    clone_script: *noop  # Comes from cache
-    gopath_cache: *ro_gopath_cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always: *int_logs_artifacts
@@ -559,8 +551,7 @@ local_system_test_task: &local_system_test_task
     gce_instance: *standardvm
     env:
         TEST_FLAVOR: sys
-    clone_script: *noop  # Comes from cache
-    gopath_cache: *ro_gopath_cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always: *logs_artifacts
@@ -619,8 +610,7 @@ buildah_bud_test_task:
             PODBIN_NAME: remote
     gce_instance: *standardvm
     timeout_in: 45m
-    clone_script: *noop
-    gopath_cache: *ro_gopath_cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always: *int_logs_artifacts
@@ -638,8 +628,7 @@ rootless_system_test_task:
     env:
         TEST_FLAVOR: sys
         PRIV_NAME: rootless
-    clone_script: *noop  # Comes from cache
-    gopath_cache: *ro_gopath_cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always: *logs_artifacts
@@ -661,8 +650,7 @@ rootless_gitlab_test_task:
         <<: *ubuntu_envvars
         TEST_FLAVOR: 'gitlab'
         PRIV_NAME: rootless
-    clone_script: *noop  # Comes from cache
-    gopath_cache: *ro_gopath_cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always:
@@ -694,8 +682,7 @@ upgrade_test_task:
         VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
         # ID for re-use of build output
         _BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
-    clone_script: *noop
-    gopath_cache: *ro_gopath_cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
     always: *logs_artifacts
@@ -715,11 +702,6 @@ image_build_task: &image-build
         image_name: build-push-${IMAGE_SUFFIX}
         # More muscle required for parallel multi-arch build
         type: "n2-standard-4"
-    env:
-        PODMAN_USERNAME: ENCRYPTED[b9f0f2550029dd2196e086d9dd6c2d1fec7e328630b15990d9bb610f9fcccb5baab8b64a8c3e72b0c1d0f5917cf65aa1]
-        PODMAN_PASSWORD: ENCRYPTED[e3444f6072853f0c8db7f964ead5e2204116af485469fa0de367f26b9316b460fd842a9882f552b9e9a83bbaf650d8b4]
-        CONTAINERS_USERNAME: ENCRYPTED[54a372d5f22f424173c114c6fb25c3214956cad323d5b285c7393a71041884ce96471d0ff733774e5dab9fa5a3c8795c]
-        CONTAINERS_PASSWORD: ENCRYPTED[4ecc3fb534935095a99fb1f2e320ac6bc87f3e7e186746e41cbcc4b5f5379a014b9fc8cc90e1f3d5abdbaf31580a4ab9]
     matrix:
         - env:
             CTXDIR: contrib/podmanimage/upstream
@@ -729,6 +711,12 @@ image_build_task: &image-build
             CTXDIR: contrib/podmanimage/stable
         - env:
             CTXDIR: contrib/hello
+    env:
+        PODMAN_USERNAME: ENCRYPTED[b9f0f2550029dd2196e086d9dd6c2d1fec7e328630b15990d9bb610f9fcccb5baab8b64a8c3e72b0c1d0f5917cf65aa1]
+        PODMAN_PASSWORD: ENCRYPTED[e3444f6072853f0c8db7f964ead5e2204116af485469fa0de367f26b9316b460fd842a9882f552b9e9a83bbaf650d8b4]
+        CONTAINERS_USERNAME: ENCRYPTED[54a372d5f22f424173c114c6fb25c3214956cad323d5b285c7393a71041884ce96471d0ff733774e5dab9fa5a3c8795c]
+        CONTAINERS_PASSWORD: ENCRYPTED[4ecc3fb534935095a99fb1f2e320ac6bc87f3e7e186746e41cbcc4b5f5379a014b9fc8cc90e1f3d5abdbaf31580a4ab9]
+    clone_script: &noop mkdir -p $CIRRUS_WORKING_DIR
     script:
         - set -a; source /etc/automation_environment; set +a
         - main.sh $CIRRUS_REPO_CLONE_URL $CTXDIR
@@ -828,42 +816,41 @@ artifacts_task:
     env:
         CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
         TEST_ENVIRON: container
-        CURL: "curl --fail --location -O https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}"
     # In order to keep the download URL and Cirrus-CI artifact.zip contents
     # simple, nothing should exist in $CIRRUS_WORKING_DIR except for artifacts.
     clone_script: *noop
-    script:
-        # Assume the latest Fedora release build is most useful
-        - $CURL/Build%20for%20$FEDORA_NAME/binary/bin/podman
-        - $CURL/Build%20for%20$FEDORA_NAME/binary/bin/podman-remote
-        - $CURL/Build%20for%20$FEDORA_NAME/binary/bin/rootlessport
-        - chmod +x podman* rootlessport
-        # Architecture in filename & can't use wildcards in a URL
+    fedora_binaries_script:
+        - mkdir -p /tmp/fed
+        - cd /tmp/fed
+        - $ARTCURL/Build%20for%20${FEDORA_NAME}/repo/repo.tbz
+        - tar xjf repo.tbz
+        - cp ./bin/* $CIRRUS_WORKING_DIR/
+    alt_binaries_script:
         - mkdir -p /tmp/alt
         - cd /tmp/alt
-        - $CURL/Alt%20Arch.%20Cross/gosrc.zip
-        - unzip gosrc.zip
-        - cd $CIRRUS_WORKING_DIR
-        - mv /tmp/alt/*.tar.gz ./
-        # Windows MSI filename has version number
+        - $ARTCURL/Alt%20Arch.%20Cross/repo/repo.tbz
+        - tar xjf repo.tbz
+        - mv ./*.tar.gz $CIRRUS_WORKING_DIR/
+    win_binaries_script:
         - mkdir -p /tmp/win
         - cd /tmp/win
-        - $CURL/Windows%20Cross/gosrc.zip
-        - unzip gosrc.zip
-        - cd $CIRRUS_WORKING_DIR
-        - mv /tmp/win/podman-remote*.zip /tmp/win/*.msi ./
-        # OSX
-        - $CURL/OSX%20Cross/gosrc/podman-remote-release-darwin_amd64.zip
-        - $CURL/OSX%20Cross/gosrc/podman-remote-release-darwin_arm64.zip
-    # Always show contents to assist in debugging
+        - $ARTCURL/Windows%20Cross/repo/repo.tbz
+        - tar xjf repo.tbz
+        - mv ./podman-remote*.zip ./*.msi $CIRRUS_WORKING_DIR/
+    osx_binaries_script:
+        - mkdir -p /tmp/osx
+        - cd /tmp/osx
+        - $ARTCURL/OSX%20Cross/repo/repo.tbz
+        - tar xjf repo.tbz
+        - mv ./podman-remote-release-darwin_*.zip $CIRRUS_WORKING_DIR/
     always:
-        contents_script: ls -1 $CIRRUS_WORKING_DIR
-    # Produce downloadable files and an automatic zip-file accessible
-    # by a consistent URL, based on contents of $CIRRUS_WORKING_DIR
-    # Ref: https://cirrus-ci.org/guide/writing-tasks/#latest-build-artifacts
-    binary_artifacts:
-        path: ./*
-        type: application/octet-stream
+      contents_script: ls -la $CIRRUS_WORKING_DIR
+      # Produce downloadable files and an automatic zip-file accessible
+      # by a consistent URL, based on contents of $CIRRUS_WORKING_DIR
+      # Ref: https://cirrus-ci.org/guide/writing-tasks/#latest-build-artifacts
+      binary_artifacts:
+          path: ./*
+          type: application/octet-stream
 
 
 # When a new tag is pushed, confirm that the code and commits
@@ -878,11 +865,9 @@ release_task:
     env:
         <<: *stdenvars
         TEST_FLAVOR: release
-    gopath_cache: *ro_gopath_cache
-    clone_script: *noop  # Comes from cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
-    always: *binary_artifacts
 
 
 # When preparing to release a new version, this task may be manually
@@ -905,8 +890,6 @@ release_test_task:
     env:
         <<: *stdenvars
         TEST_FLAVOR: release
-    gopath_cache: *ro_gopath_cache
-    clone_script: *noop  # Comes from cache
+    clone_script: *get_gosrc
     setup_script: *setup
     main_script: *main
-    always: *binary_artifacts
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
index 3416a93ee..ca023d18a 100644
--- a/.pre-commit-config.yaml
+++ b/.pre-commit-config.yaml
@@ -17,3 +17,4 @@ repos:
       - id: check-byte-order-marker
       - id: check-executables-have-shebangs
       - id: check-merge-conflict
+      - id: check-yaml
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 46b245a45..a7f39e7af 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -36,11 +36,6 @@ fi
 # Managed by setup_environment.sh; holds task-specific definitions.
 if [[ -r "/etc/ci_environment" ]]; then source /etc/ci_environment; fi
 
-OS_RELEASE_ID="$(source /etc/os-release; echo $ID)"
-# GCE image-name compatible string representation of distribution _major_ version
-OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | tr -d '.')"
-# Combined to ease some usage
-OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
 # This is normally set from .cirrus.yml but default is necessary when
 # running under hack/get_ci_vm.sh since it cannot infer the value.
 DISTRO_NV="${DISTRO_NV:-$OS_REL_VER}"
@@ -182,30 +177,21 @@ setup_rootless() {
     cat $HOME/.ssh/*.pub /home/$ROOTLESS_USER/.ssh/*.pub >> $HOME/.ssh/authorized_keys
     cat $HOME/.ssh/*.pub /home/$ROOTLESS_USER/.ssh/*.pub >> /home/$ROOTLESS_USER/.ssh/authorized_keys
 
-    msg "Ensure the ssh daemon is up and running within 5 minutes"
-    systemctl start sshd
-    lilto systemctl is-active sshd
-
     msg "Configure ssh file permissions"
     chmod -R 700 "$HOME/.ssh"
     chmod -R 700 "/home/$ROOTLESS_USER/.ssh"
     chown -R $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.ssh"
 
+    # N/B: We're clobbering the known_hosts here on purpose.  There should
+    # never be any non-localhost connections made from tests (using strict-mode).
+    # If there are, it's either a security problem or a broken test, both of which
+    # we want to lead to test failures.
     msg "   setup known_hosts for $USER"
-    ssh -q root@localhost \
-        -o UserKnownHostsFile=/root/.ssh/known_hosts \
-        -o UpdateHostKeys=yes \
-        -o StrictHostKeyChecking=no \
-        -o CheckHostIP=no \
-        true
-
+    ssh-keyscan localhost > /root/.ssh/known_hosts
     msg "   setup known_hosts for $ROOTLESS_USER"
-    su $ROOTLESS_USER -c "ssh -q $ROOTLESS_USER@localhost \
-        -o UserKnownHostsFile=/home/$ROOTLESS_USER/.ssh/known_hosts \
-        -o UpdateHostKeys=yes \
-        -o StrictHostKeyChecking=no \
-        -o CheckHostIP=no \
-        true"
+    # Maintain access-permission consistency with all other .ssh files.
+    install -Z -m 700 -o $ROOTLESS_USER -g $ROOTLESS_USER \
+        /root/.ssh/known_hosts /home/$ROOTLESS_USER/.ssh/known_hosts
 }
 
 install_test_configs() {
@@ -270,6 +256,8 @@ remove_packaged_podman_files() {
         done
     done
 
+    # OS_RELEASE_ID is defined by automation-library
+    # shellcheck disable=SC2154
     if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]]
     then
         LISTING_CMD="dpkg-query -L podman"
diff --git a/contrib/cirrus/pr-should-include-tests b/contrib/cirrus/pr-should-include-tests
index 0d39047a6..57ca39d9b 100755
--- a/contrib/cirrus/pr-should-include-tests
+++ b/contrib/cirrus/pr-should-include-tests
@@ -30,19 +30,20 @@ fi
 # Nothing changed under test subdirectory.
 #
 # This is OK if the only files being touched are "safe" ones.
-filtered_changes=$(git diff --name-only $base $head   |
-                       fgrep -vx .cirrus.yml          |
-                       fgrep -vx .gitignore           |
-                       fgrep -vx Makefile             |
-                       fgrep -vx go.mod               |
-                       fgrep -vx go.sum               |
-                       egrep -v  '^[^/]+\.md$'        |
-                       egrep -v  '^.github'           |
-                       egrep -v  '^contrib/'          |
-                       egrep -v  '^docs/'             |
-                       egrep -v  '^hack/'             |
-                       egrep -v  '^nix/'              |
-                       egrep -v  '^vendor/'           |
+filtered_changes=$(git diff --name-only $base $head      |
+                       fgrep -vx .cirrus.yml             |
+                       fgrep -vx .pre-commit-config.yaml |
+                       fgrep -vx .gitignore              |
+                       fgrep -vx Makefile                |
+                       fgrep -vx go.mod                  |
+                       fgrep -vx go.sum                  |
+                       egrep -v  '^[^/]+\.md$'           |
+                       egrep -v  '^.github'              |
+                       egrep -v  '^contrib/'             |
+                       egrep -v  '^docs/'                |
+                       egrep -v  '^hack/'                |
+                       egrep -v  '^nix/'                 |
+                       egrep -v  '^vendor/'              |
                        egrep -v  '^version/')
 if [[ -z "$filtered_changes" ]]; then
     exit 0
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index e3eb46783..832cb58ab 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -121,6 +121,9 @@ case "$OS_RELEASE_ID" in
         # CNI networking available.  Upgrading from one to the other is
         # not supported at this time.  Support execution of the upgrade
         # tests in F36 and later, by disabling Netavark and enabling CNI.
+        #
+        # OS_RELEASE_VER is defined by automation-library
+        # shellcheck disable=SC2154
         if [[ "$OS_RELEASE_VER" -ge 36 ]] && \
            [[ "$TEST_FLAVOR" != "upgrade_test" ]];
         then
@@ -217,6 +220,7 @@ case "$TEST_FLAVOR" in
     validate)
         dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
         # For some reason, this is also needed for validation
+        make install.tools
         make .install.pre-commit
         ;;
     automation) ;;
@@ -226,10 +230,12 @@ case "$TEST_FLAVOR" in
         if [[ "$ALT_NAME" =~ RPM ]]; then
             bigto dnf install -y glibc-minimal-langpack go-rpm-macros rpkg rpm-build shadow-utils-subid-devel
         fi
+        make install.tools
         ;;
     docker-py)
         remove_packaged_podman_files
-        make && make install PREFIX=/usr ETCDIR=/etc
+        make install.tools
+        make install PREFIX=/usr ETCDIR=/etc
 
         msg "Installing previously downloaded/cached packages"
         dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
@@ -239,13 +245,17 @@ case "$TEST_FLAVOR" in
         pip install --requirement $GOSRC/test/python/requirements.txt
         ;;
     build) make clean ;;
-    unit) ;;
+    unit)
+        make install.tools
+        ;;
     compose_v2)
+        make install.tools
         dnf -y remove docker-compose
         curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
         chmod +x /usr/local/bin/docker-compose
         ;& # Continue with next item
     apiv2)
+        make install.tools
         msg "Installing previously downloaded/cached packages"
         dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
         virtualenv .venv/requests
@@ -254,6 +264,7 @@ case "$TEST_FLAVOR" in
         pip install --requirement $GOSRC/test/apiv2/python/requirements.txt
         ;&  # continue with next item
     compose)
+        make install.tools
         rpm -ivh $PACKAGE_DOWNLOAD_DIR/podman-docker*
         ;&  # continue with next item
     int) ;&
@@ -262,6 +273,7 @@ case "$TEST_FLAVOR" in
     bud) ;&
     bindings) ;&
     endpoint)
+        make install.tools
         # Use existing host bits when testing is to happen inside a container
         # since this script will run again in that environment.
         # shellcheck disable=SC2154
@@ -270,11 +282,11 @@ case "$TEST_FLAVOR" in
                 die "Refusing to config. host-test in container";
             fi
             remove_packaged_podman_files
-            make && make install PREFIX=/usr ETCDIR=/etc
+            make install PREFIX=/usr ETCDIR=/etc
         elif [[ "$TEST_ENVIRON" == "container" ]]; then
             if ((CONTAINER)); then
                 remove_packaged_podman_files
-                make && make install PREFIX=/usr ETCDIR=/etc
+                make install PREFIX=/usr ETCDIR=/etc
             fi
         else
             die "Invalid value for \$TEST_ENVIRON=$TEST_ENVIRON"
@@ -291,7 +303,7 @@ case "$TEST_FLAVOR" in
         # Ref: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27270#note_499585550
 
         remove_packaged_podman_files
-        make && make install PREFIX=/usr ETCDIR=/etc
+        make install PREFIX=/usr ETCDIR=/etc
 
         msg "Installing docker and containerd"
         # N/B: Tests check/expect `docker info` output, and this `!= podman info`
@@ -324,7 +336,10 @@ case "$TEST_FLAVOR" in
             docker.io/gitlab/gitlab-runner-helper:x86_64-latest-pwsh
         ;;
     swagger) ;&  # use next item
-    consistency) make clean ;;
+    consistency)
+        make clean
+        make install.tools
+        ;;
     release) ;;
     *) die_unknown TEST_FLAVOR
 esac
diff --git a/hack/get_ci_vm.sh b/hack/get_ci_vm.sh
index ef7069a81..6632a0178 100755
--- a/hack/get_ci_vm.sh
+++ b/hack/get_ci_vm.sh
@@ -61,7 +61,7 @@ else
         -e NAME="$USER" \
         -e SRCDIR=/src \
         -e GCLOUD_ZONE="$GCLOUD_ZONE" \
-        -e DEBUG="${DEBUG:-0}" \
+        -e A_DEBUG="${A_DEBUG:-0}" \
         -v $REPO_DIRPATH:/src:O \
         -v $HOME/.config/gcloud:/root/.config/gcloud:z \
         -v $HOME/.config/gcloud/ssh:/root/.ssh:z \