Merge pull request #14016 from cevich/fix_caching

Cirrus: Simplify use of cache in automation

3 files changed, 45 insertions, 41 deletions

diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 46b245a45..a7f39e7af 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -36,11 +36,6 @@ fi
 # Managed by setup_environment.sh; holds task-specific definitions.
 if [[ -r "/etc/ci_environment" ]]; then source /etc/ci_environment; fi
 
-OS_RELEASE_ID="$(source /etc/os-release; echo $ID)"
-# GCE image-name compatible string representation of distribution _major_ version
-OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | tr -d '.')"
-# Combined to ease some usage
-OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
 # This is normally set from .cirrus.yml but default is necessary when
 # running under hack/get_ci_vm.sh since it cannot infer the value.
 DISTRO_NV="${DISTRO_NV:-$OS_REL_VER}"
@@ -182,30 +177,21 @@ setup_rootless() {
     cat $HOME/.ssh/*.pub /home/$ROOTLESS_USER/.ssh/*.pub >> $HOME/.ssh/authorized_keys
     cat $HOME/.ssh/*.pub /home/$ROOTLESS_USER/.ssh/*.pub >> /home/$ROOTLESS_USER/.ssh/authorized_keys
 
-    msg "Ensure the ssh daemon is up and running within 5 minutes"
-    systemctl start sshd
-    lilto systemctl is-active sshd
-
     msg "Configure ssh file permissions"
     chmod -R 700 "$HOME/.ssh"
     chmod -R 700 "/home/$ROOTLESS_USER/.ssh"
     chown -R $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.ssh"
 
+    # N/B: We're clobbering the known_hosts here on purpose.  There should
+    # never be any non-localhost connections made from tests (using strict-mode).
+    # If there are, it's either a security problem or a broken test, both of which
+    # we want to lead to test failures.
     msg "   setup known_hosts for $USER"
-    ssh -q root@localhost \
-        -o UserKnownHostsFile=/root/.ssh/known_hosts \
-        -o UpdateHostKeys=yes \
-        -o StrictHostKeyChecking=no \
-        -o CheckHostIP=no \
-        true
-
+    ssh-keyscan localhost > /root/.ssh/known_hosts
     msg "   setup known_hosts for $ROOTLESS_USER"
-    su $ROOTLESS_USER -c "ssh -q $ROOTLESS_USER@localhost \
-        -o UserKnownHostsFile=/home/$ROOTLESS_USER/.ssh/known_hosts \
-        -o UpdateHostKeys=yes \
-        -o StrictHostKeyChecking=no \
-        -o CheckHostIP=no \
-        true"
+    # Maintain access-permission consistency with all other .ssh files.
+    install -Z -m 700 -o $ROOTLESS_USER -g $ROOTLESS_USER \
+        /root/.ssh/known_hosts /home/$ROOTLESS_USER/.ssh/known_hosts
 }
 
 install_test_configs() {
@@ -270,6 +256,8 @@ remove_packaged_podman_files() {
         done
     done
 
+    # OS_RELEASE_ID is defined by automation-library
+    # shellcheck disable=SC2154
     if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]]
     then
         LISTING_CMD="dpkg-query -L podman"
diff --git a/contrib/cirrus/pr-should-include-tests b/contrib/cirrus/pr-should-include-tests
index 0d39047a6..57ca39d9b 100755
--- a/contrib/cirrus/pr-should-include-tests
+++ b/contrib/cirrus/pr-should-include-tests
@@ -30,19 +30,20 @@ fi
 # Nothing changed under test subdirectory.
 #
 # This is OK if the only files being touched are "safe" ones.
-filtered_changes=$(git diff --name-only $base $head   |
-                       fgrep -vx .cirrus.yml          |
-                       fgrep -vx .gitignore           |
-                       fgrep -vx Makefile             |
-                       fgrep -vx go.mod               |
-                       fgrep -vx go.sum               |
-                       egrep -v  '^[^/]+\.md$'        |
-                       egrep -v  '^.github'           |
-                       egrep -v  '^contrib/'          |
-                       egrep -v  '^docs/'             |
-                       egrep -v  '^hack/'             |
-                       egrep -v  '^nix/'              |
-                       egrep -v  '^vendor/'           |
+filtered_changes=$(git diff --name-only $base $head      |
+                       fgrep -vx .cirrus.yml             |
+                       fgrep -vx .pre-commit-config.yaml |
+                       fgrep -vx .gitignore              |
+                       fgrep -vx Makefile                |
+                       fgrep -vx go.mod                  |
+                       fgrep -vx go.sum                  |
+                       egrep -v  '^[^/]+\.md$'           |
+                       egrep -v  '^.github'              |
+                       egrep -v  '^contrib/'             |
+                       egrep -v  '^docs/'                |
+                       egrep -v  '^hack/'                |
+                       egrep -v  '^nix/'                 |
+                       egrep -v  '^vendor/'              |
                        egrep -v  '^version/')
 if [[ -z "$filtered_changes" ]]; then
     exit 0
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index e3eb46783..832cb58ab 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -121,6 +121,9 @@ case "$OS_RELEASE_ID" in
         # CNI networking available.  Upgrading from one to the other is
         # not supported at this time.  Support execution of the upgrade
         # tests in F36 and later, by disabling Netavark and enabling CNI.
+        #
+        # OS_RELEASE_VER is defined by automation-library
+        # shellcheck disable=SC2154
         if [[ "$OS_RELEASE_VER" -ge 36 ]] && \
            [[ "$TEST_FLAVOR" != "upgrade_test" ]];
         then
@@ -217,6 +220,7 @@ case "$TEST_FLAVOR" in
     validate)
         dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
         # For some reason, this is also needed for validation
+        make install.tools
         make .install.pre-commit
         ;;
     automation) ;;
@@ -226,10 +230,12 @@ case "$TEST_FLAVOR" in
         if [[ "$ALT_NAME" =~ RPM ]]; then
             bigto dnf install -y glibc-minimal-langpack go-rpm-macros rpkg rpm-build shadow-utils-subid-devel
         fi
+        make install.tools
         ;;
     docker-py)
         remove_packaged_podman_files
-        make && make install PREFIX=/usr ETCDIR=/etc
+        make install.tools
+        make install PREFIX=/usr ETCDIR=/etc
 
         msg "Installing previously downloaded/cached packages"
         dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
@@ -239,13 +245,17 @@ case "$TEST_FLAVOR" in
         pip install --requirement $GOSRC/test/python/requirements.txt
         ;;
     build) make clean ;;
-    unit) ;;
+    unit)
+        make install.tools
+        ;;
     compose_v2)
+        make install.tools
         dnf -y remove docker-compose
         curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose
         chmod +x /usr/local/bin/docker-compose
         ;& # Continue with next item
     apiv2)
+        make install.tools
         msg "Installing previously downloaded/cached packages"
         dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm
         virtualenv .venv/requests
@@ -254,6 +264,7 @@ case "$TEST_FLAVOR" in
         pip install --requirement $GOSRC/test/apiv2/python/requirements.txt
         ;&  # continue with next item
     compose)
+        make install.tools
         rpm -ivh $PACKAGE_DOWNLOAD_DIR/podman-docker*
         ;&  # continue with next item
     int) ;&
@@ -262,6 +273,7 @@ case "$TEST_FLAVOR" in
     bud) ;&
     bindings) ;&
     endpoint)
+        make install.tools
         # Use existing host bits when testing is to happen inside a container
         # since this script will run again in that environment.
         # shellcheck disable=SC2154
@@ -270,11 +282,11 @@ case "$TEST_FLAVOR" in
                 die "Refusing to config. host-test in container";
             fi
             remove_packaged_podman_files
-            make && make install PREFIX=/usr ETCDIR=/etc
+            make install PREFIX=/usr ETCDIR=/etc
         elif [[ "$TEST_ENVIRON" == "container" ]]; then
             if ((CONTAINER)); then
                 remove_packaged_podman_files
-                make && make install PREFIX=/usr ETCDIR=/etc
+                make install PREFIX=/usr ETCDIR=/etc
             fi
         else
             die "Invalid value for \$TEST_ENVIRON=$TEST_ENVIRON"
@@ -291,7 +303,7 @@ case "$TEST_FLAVOR" in
         # Ref: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27270#note_499585550
 
         remove_packaged_podman_files
-        make && make install PREFIX=/usr ETCDIR=/etc
+        make install PREFIX=/usr ETCDIR=/etc
 
         msg "Installing docker and containerd"
         # N/B: Tests check/expect `docker info` output, and this `!= podman info`
@@ -324,7 +336,10 @@ case "$TEST_FLAVOR" in
             docker.io/gitlab/gitlab-runner-helper:x86_64-latest-pwsh
         ;;
     swagger) ;&  # use next item
-    consistency) make clean ;;
+    consistency)
+        make clean
+        make install.tools
+        ;;
     release) ;;
     *) die_unknown TEST_FLAVOR
 esac