Merge pull request #6193 from cevich/conmon_ci_packages

Cirrus: Include packages for containers/conmon CI

7 files changed, 129 insertions, 79 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 5b9dbdab8..4897265d4 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -39,7 +39,7 @@ env:
     UBUNTU_NAME: "ubuntu-20"
     PRIOR_UBUNTU_NAME: "ubuntu-19"
 
-    _BUILT_IMAGE_SUFFIX: "libpod-6268069335007232"  # From the packer output of 'build_vm_images_script'
+    _BUILT_IMAGE_SUFFIX: "libpod-6508632441356288"
     FEDORA_CACHE_IMAGE_NAME: "${FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "${PRIOR_FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
     UBUNTU_CACHE_IMAGE_NAME: "${UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}"
@@ -72,10 +72,6 @@ env:
     GCE_SSH_USERNAME: cirrus-ci
     # Name where this repositories cloud resources are located
     GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
-    RELEASE_GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46]
-    RELEASE_GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1]
-    RELEASE_GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
-
 
 
 # Default VM to use unless set or modified by task
@@ -600,6 +596,7 @@ special_testing_bindings_task:
 
     env:
         SPECIALMODE: 'bindings'  # See docs
+        ADD_SECOND_PARTITION: 'true'  # More root fs space is required
 
     timeout_in: 40m
 
@@ -700,20 +697,16 @@ verify_test_built_images_task:
               env:
                   TEST_REMOTE_CLIENT: 'false'
         matrix:
-            # Required env. var. by check_image_script
             PACKER_BUILDER_NAME: "${FEDORA_NAME}"
             PACKER_BUILDER_NAME: "${PRIOR_FEDORA_NAME}"
+            PACKER_BUILDER_NAME: "${UBUNTU_NAME}"
             PACKER_BUILDER_NAME: "${PRIOR_UBUNTU_NAME}"
-            # Multiple test failures on ${UBUNTU_CACHE_IMAGE_NAME}
-            # PACKER_BUILDER_NAME: "${UBUNTU_NAME}"
 
     networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
     installed_packages_script: '$SCRIPT_BASE/logcollector.sh packages'
     environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
-    # Verify expectations once per image
-    check_image_script: >-
-        [[ "$TEST_REMOTE_CLIENT" == "false" ]] || \
-            $SCRIPT_BASE/check_image.sh |& ${TIMESTAMP}
+    # Verify expectations of built images
+    check_image_script: '$SCRIPT_BASE/check_image.sh |& ${TIMESTAMP}'
     # Note: A truncated form of normal testing. It only needs to confirm new images
     # "probably" work.  A full round of testing will happen again after $*_CACHE_IMAGE_NAME
     # are updated in this or another PR (w/o '***CIRRUS: TEST IMAGES***').
@@ -731,6 +724,10 @@ docs_task:
 
     depends_on:
         - "gating"
+    env:
+        RELEASE_GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46]
+        RELEASE_GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1]
+        RELEASE_GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
 
     script:
         - "$SCRIPT_BASE/build_swagger.sh |& ${TIMESTAMP}"
diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh
index 5423f67d6..0d33e55bf 100755
--- a/contrib/cirrus/check_image.sh
+++ b/contrib/cirrus/check_image.sh
@@ -6,7 +6,7 @@ source $(dirname $0)/lib.sh
 
 EVIL_UNITS="$($CIRRUS_WORKING_DIR/$PACKER_BASE/systemd_banish.sh --list)"
 
-req_env_var PACKER_BUILDER_NAME TEST_REMOTE_CLIENT EVIL_UNITS OS_RELEASE_ID
+req_env_var PACKER_BUILDER_NAME TEST_REMOTE_CLIENT EVIL_UNITS OS_RELEASE_ID CG_FS_TYPE
 
 NFAILS=0
 echo "Validating VM image"
@@ -22,7 +22,8 @@ item_test 'Minimum available memory' $MEM_FREE -ge $MIN_MEM_MB || let "NFAILS+=1
 
 # We're testing a custom-built podman; make sure there isn't a distro-provided
 # binary anywhere; that could potentially taint our results.
-item_test "remove_packaged_podman_files() did it's job" -z "$(type -P podman)" || let "NFAILS+=1"
+remove_packaged_podman_files
+item_test "remove_packaged_podman_files() does it's job" -z "$(type -P podman)" || let "NFAILS+=1"
 
 # Integration Tests require varlink in Fedora
 item_test "The varlink executable is present" -x "$(type -P varlink)" || let "NFAILS+=1"
@@ -39,8 +40,10 @@ for REQ_UNIT in google-accounts-daemon.service \
                 google-shutdown-scripts.service \
                 google-startup-scripts.service
 do
-    item_test "required $REQ_UNIT enabled" \
-        "$(systemctl list-unit-files --no-legend $REQ_UNIT)" = "$REQ_UNIT enabled" || let "NFAILS+=1"
+    # enabled/disabled appears at the end of the line, on some Ubuntu's it appears twice
+    service_status=$(systemctl list-unit-files --no-legend $REQ_UNIT | tac -s ' ' | head -1)
+    item_test "required $REQ_UNIT status is enabled" \
+        "$service_status" = "enabled" || let "NFAILS+=1"
 done
 
 for evil_unit in $EVIL_UNITS
@@ -50,19 +53,28 @@ do
     item_test "No $evil_unit unit is present or active:" "$unit_status" -ne "0" || let "NFAILS+=1"
 done
 
-if [[ "$OS_RELEASE_ID" == "ubuntu" ]] && [[ -x "/usr/lib/cri-o-runc/sbin/runc" ]]
-then
-    SAMESAME=$(diff --brief /usr/lib/cri-o-runc/sbin/runc /usr/bin/runc &> /dev/null; echo $?)
-    item_test "On ubuntu /usr/bin/runc is /usr/lib/cri-o-runc/sbin/runc" "$SAMESAME" -eq "0" || let "NFAILS+=1"
-fi
-
-if [[ "$OS_RELEASE_ID" == "ubuntu" ]]
-then
-    item_test "On ubuntu, no periodic apt crap is enabled" -z "$(egrep $PERIODIC_APT_RE /etc/apt/apt.conf.d/*)"
-fi
-
 echo "Checking items specific to ${PACKER_BUILDER_NAME}${BUILT_IMAGE_SUFFIX}"
 case "$PACKER_BUILDER_NAME" in
+    ubuntu*)
+        item_test "On ubuntu, no periodic apt crap is enabled" -z "$(egrep $PERIODIC_APT_RE /etc/apt/apt.conf.d/*)"
+        ;;
+    fedora*)
+        # Only runc -OR- crun should be installed, never both
+        case "$CG_FS_TYPE" in
+            tmpfs)
+                HAS=runc
+                HAS_NOT=crun
+                ;;
+            cgroup2fs)
+                HAS=crun
+                HAS_NOT=runc
+                ;;
+        esac
+        HAS_RC=$(rpm -qV $HAS &> /dev/null; echo $?)
+        HAS_NOT_RC=$(rpm -qV $HAS_NOT &> /dev/null; echo $?)
+        item_test "With a cgroups-fs type $CG_FS_TYPE, the $HAS package is installed" $HAS_RC -eq 0
+        item_test "With a cgroups-fs type $CG_FS_TYPE, the $HAS_NOT package is not installed" $HAS_NOT_RC -ne 0
+        ;;
     xfedora*)
         echo "Kernel Command-line: $(cat /proc/cmdline)"
         item_test \
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index cc5a3ffa7..66e8060cf 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -39,6 +39,8 @@ PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer}
 # Important filepaths
 SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}"
 AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv"
+# Downloaded, but not installed packages.
+PACKAGE_DOWNLOAD_DIR=/var/cache/download
 
 # Log remote-client system test varlink output here
 export VARLINK_LOG=/var/tmp/varlink.log
@@ -422,7 +424,7 @@ remove_packaged_podman_files() {
     then
         LISTING_CMD="$SUDO dpkg-query -L podman"
     else
-        LISTING_CMD='$SUDO rpm -ql podman'
+        LISTING_CMD="$SUDO rpm -ql podman"
     fi
 
     # yum/dnf/dpkg may list system directories, only remove files
@@ -437,6 +439,14 @@ remove_packaged_podman_files() {
     sync && echo 3 > /proc/sys/vm/drop_caches
 }
 
+# The version of CRI-O and Kubernetes must always match
+get_kubernetes_version(){
+    # TODO: Look up the kube RPM/DEB version installed, or in $PACKAGE_DOWNLOAD_DIR
+    #       and retrieve the major-minor version directly.
+    local KUBERNETES_VERSION="1.15"
+    echo "$KUBERNETES_VERSION"
+}
+
 canonicalize_image_names() {
     req_env_var IMGNAMES
     echo "Adding all current base images to \$IMGNAMES for timestamp update"
@@ -479,6 +489,7 @@ _finalize() {
     fi
     echo "Re-initializing so next boot does 'first-boot' setup again."
     cd /
+    $SUDO rm -rf $GOPATH/src  # Actual source will be cloned at runtime
     $SUDO rm -rf /var/lib/cloud/instanc*
     $SUDO rm -rf /root/.ssh/*
     $SUDO rm -rf /etc/ssh/*key*
diff --git a/contrib/cirrus/packer/fedora_packaging.sh b/contrib/cirrus/packer/fedora_packaging.sh
index e80d48bc8..aecaaef93 100644
--- a/contrib/cirrus/packer/fedora_packaging.sh
+++ b/contrib/cirrus/packer/fedora_packaging.sh
@@ -11,6 +11,8 @@ echo "Updating/Installing repos and packages for $OS_REL_VER"
 
 source $GOSRC/$SCRIPT_BASE/lib.sh
 
+req_env_var GOSRC SCRIPT_BASE BIGTO INSTALL_AUTOMATION_VERSION FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE
+
 # Pre-req. to install automation tooing
 $LILTO $SUDO dnf install -y git
 
@@ -35,7 +37,7 @@ fi
 
 $BIGTO ooe.sh $SUDO dnf update -y
 
-REMOVE_PACKAGES=()
+REMOVE_PACKAGES=(runc)
 INSTALL_PACKAGES=(\
     autoconf
     automake
@@ -50,8 +52,11 @@ INSTALL_PACKAGES=(\
     containernetworking-plugins
     containers-common
     criu
+    crun
+    curl
     device-mapper-devel
     dnsmasq
+    e2fsprogs-devel
     emacs-nox
     file
     findutils
@@ -60,16 +65,26 @@ INSTALL_PACKAGES=(\
     gcc
     git
     glib2-devel
+    glibc-devel
     glibc-static
     gnupg
     go-md2man
     golang
+    gpgme
     gpgme-devel
+    grubby
+    hostname
     iproute
     iptables
     jq
+    krb5-workstation
+    libassuan
     libassuan-devel
+    libblkid-devel
     libcap-devel
+    libffi-devel
+    libgpg-error-devel
+    libguestfs-tools
     libmsi1
     libnet
     libnet-devel
@@ -79,56 +94,60 @@ INSTALL_PACKAGES=(\
     libselinux-devel
     libtool
     libvarlink-util
+    libxml2-devel
+    libxslt-devel
     lsof
     make
+    mlocate
     msitools
+    nfs-utils
     nmap-ncat
+    openssl
+    openssl-devel
     ostree-devel
     pandoc
+    pkgconfig
     podman
+    policycoreutils
     procps-ng
     protobuf
     protobuf-c
     protobuf-c-devel
     protobuf-devel
-    python
+    python2
+    python3-PyYAML
     python3-dateutil
     python3-psutil
     python3-pytoml
+    python3-libsemanage
+    python3-libselinux
+    python3-libvirt
+    redhat-rpm-config
+    rpcbind
     rsync
+    sed
     selinux-policy-devel
     skopeo
     skopeo-containers
     slirp4netns
+    socat
+    tar
     unzip
     vim
     wget
     which
     xz
     zip
+    zlib-devel
+)
+DOWNLOAD_PACKAGES=(\
+    "cri-o-$(get_kubernetes_version)*"
+    cri-tools
+    "kubernetes-$(get_kubernetes_version)*"
+    runc
+    oci-umount
+    parallel
 )
-
-case "$OS_RELEASE_VER" in
-    30)
-        INSTALL_PACKAGES+=(\
-            atomic-registries
-            golang-github-cpuguy83-go-md2man
-            python2-future
-            runc
-        )
-        REMOVE_PACKAGES+=(crun)
-        ;;
-    31)
-        INSTALL_PACKAGES+=(crun)
-        REMOVE_PACKAGES+=(runc)
-        ;;
-    32)
-        INSTALL_PACKAGES+=(crun)
-        REMOVE_PACKAGES+=(runc)
-        ;;
-    *)
-        bad_os_id_ver ;;
-esac
 
 echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'"
 $BIGTO ooe.sh $SUDO dnf install -y ${INSTALL_PACKAGES[@]}
@@ -136,6 +155,18 @@ $BIGTO ooe.sh $SUDO dnf install -y ${INSTALL_PACKAGES[@]}
 [[ ${#REMOVE_PACKAGES[@]} -eq 0 ]] || \
     $LILTO ooe.sh $SUDO dnf erase -y ${REMOVE_PACKAGES[@]}
 
-export GOPATH="$(mktemp -d)"
-trap "$SUDO rm -rf $GOPATH" EXIT
-ooe.sh $SUDO $GOSRC/hack/install_catatonit.sh
+if [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then
+    echo "Downloading packages for optional installation at runtime, as needed."
+    # Required for cri-o
+    ooe.sh $SUDO dnf -y module enable cri-o:$(get_kubernetes_version)
+    $SUDO mkdir -p "$PACKAGE_DOWNLOAD_DIR"
+    cd "$PACKAGE_DOWNLOAD_DIR"
+    $LILTO ooe.sh $SUDO dnf download -y --resolve ${DOWNLOAD_PACKAGES[@]}
+    ls -la "$PACKAGE_DOWNLOAD_DIR/"
+fi
+
+echo "Installing runtime tooling"
+# Save some runtime by having these already available
+cd $GOSRC
+$SUDO make install.tools
+$SUDO $GOSRC/hack/install_catatonit.sh
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index 3830b3bc4..25b568e8a 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -12,11 +12,11 @@ req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NA
 
 workaround_bfq_bug
 
-# Do not enable update-stesting on the previous Fedora release
-if [[ "$FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then
-    DISABLE_UPDATES_TESTING=0
-else
+# Do not enable updates-testing on the previous Fedora release
+if [[ "$PRIOR_FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then
     DISABLE_UPDATES_TESTING=1
+else
+    DISABLE_UPDATES_TESTING=0
 fi
 
 bash $PACKER_BASE/fedora_packaging.sh
diff --git a/contrib/cirrus/packer/ubuntu_packaging.sh b/contrib/cirrus/packer/ubuntu_packaging.sh
index fd0280230..09f9aab9f 100644
--- a/contrib/cirrus/packer/ubuntu_packaging.sh
+++ b/contrib/cirrus/packer/ubuntu_packaging.sh
@@ -11,6 +11,8 @@ echo "Updating/Installing repos and packages for $OS_REL_VER"
 
 source $GOSRC/$SCRIPT_BASE/lib.sh
 
+req_env_var GOSRC SCRIPT_BASE BIGTO SUDOAPTGET INSTALL_AUTOMATION_VERSION
+
 echo "Updating/configuring package repositories."
 $BIGTO $SUDOAPTGET update
 
@@ -99,6 +101,7 @@ INSTALL_PACKAGES=(\
     protobuf-c-compiler
     protobuf-compiler
     python-protobuf
+    python2
     python3-dateutil
     python3-pip
     python3-psutil
@@ -118,6 +121,11 @@ INSTALL_PACKAGES=(\
     zip
     zlib1g-dev
 )
+DOWNLOAD_PACKAGES=(\
+    cri-o-$(get_kubernetes_version)
+    cri-tools
+    parallel
+)
 
 # These aren't resolvable on Ubuntu 20
 if [[ "$OS_RELEASE_VER" -le 19 ]]; then
@@ -137,16 +145,15 @@ echo "Installing general testing and system dependencies"
 $LILTO ooe.sh $SUDOAPTGET update
 $BIGTO ooe.sh $SUDOAPTGET install ${INSTALL_PACKAGES[@]}
 
-export GOPATH="$(mktemp -d)"
-trap "$SUDO rm -rf $GOPATH" EXIT
-echo "Installing cataonit and libseccomp.sudo"
-cd $GOSRC
-ooe.sh $SUDO hack/install_catatonit.sh
-ooe.sh $SUDO make install.libseccomp.sudo
-
-CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc"
-if $SUDO dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH"
-then
-    echo "Linking $CRIO_RUNC_PATH to /usr/bin/runc for ease of testing."
-    $SUDO ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc"
+if [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then
+    echo "Downloading packages for optional installation at runtime, as needed."
+    $SUDO ln -s /var/cache/apt/archives "$PACKAGE_DOWNLOAD_DIR"
+    $LILTO ooe.sh $SUDOAPTGET install --download-only ${DOWNLOAD_PACKAGES[@]}
+    ls -la "$PACKAGE_DOWNLOAD_DIR/"
 fi
+
+echo "Installing runtime tooling"
+cd $GOSRC
+$SUDO hack/install_catatonit.sh
+$SUDO make install.libseccomp.sudo
+$SUDO make install.tools
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 4066b813b..323e7c35b 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -89,14 +89,6 @@ case "$CG_FS_TYPE" in
         warn "Forcing testing with crun instead of runc"
         X=$(echo "export OCI_RUNTIME=/usr/bin/crun" | \
             tee -a /etc/environment) && eval "$X" && echo "$X"
-
-        if [[ "$OS_RELEASE_ID" == "fedora" ]]; then
-            warn "Upgrading to the latest crun"
-            # Normally not something to do for stable testing
-            # but crun is new, and late-breaking fixes may be required
-            # on short notice
-            dnf update -y crun containers-common
-        fi
         ;;
     *)
         die 110 "Unsure how to handle cgroup filesystem type '$CG_FS_TYPE'"