Cirrus: Run tests on test-built cache-images

Also, add jq and catatonit installs to images.

Signed-off-by: Chris Evich <cevich@redhat.com>

11 files changed, 148 insertions, 99 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 6035284c8..08875a354 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -26,6 +26,7 @@ env:
     # Command to prefix every output line with a timestamp
     # (can't do inline awk script, Cirrus-CI or YAML mangles quoting)
     TIMESTAMP: "awk --file ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk"
+    BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}"
 
     ####
     #### Cache-image names to test with
@@ -89,22 +90,6 @@ env:
     # Name where this repositories cloud resources are located
     GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
 
-    # Space separated list of environment variables to unset before testing
-    UNSET_ENV_VARS: >-
-        GCP_PROJECT_ID GCE_SSH_USERNAME SERVICE_ACCOUNT RHSM_COMMAND BUILT_IMAGE_SUFFIX
-        IRCID RHEL_BASE_IMAGE FAH_BASE_IMAGE FEDORA_BASE_IMAGE CENTOS_BASE_IMAGE
-        UBUNTU_BASE_IMAGE PACKER_VER PACKER_BUILDS RUNC_COMMIT CRIU_COMMIT
-        CRIO_COMMIT CNI_COMMIT FEDORA_CNI_COMMIT PACKER_BASE SCRIPT_BASE
-        CIRRUS_SHELL CIRRUS_WORKING_DIR ENVLIB BUILT_IMAGE_SUFFIX CIRRUS_CI
-        CI_NODE_INDEX CI_NODE_TOTAL CIRRUS_BASE_BRANCH CIRRUS_BASE_SHA
-        CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO CIRRUS_CLONE_DEPTH
-        CIRRUS_COMMIT_MESSAGE CIRRUS_CHANGE_MESSAGE CIRRUS_REPO_CLONE_HOST
-        CIRRUS_DEFAULT_BRANCH CIRRUS_PR CIRRUS_TAG CIRRUS_OS CIRRUS_TASK_NAME
-        CIRRUS_TASK_ID CIRRUS_REPO_NAME CIRRUS_REPO_OWNER CIRRUS_REPO_FULL_NAME
-        CIRRUS_REPO_CLONE_URL CIRRUS_SHELL CIRRUS_USER_COLLABORATOR CIRRUS_USER_PERMISSION
-        CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST PACKER_BUILDS BUILT_IMAGE_SUFFIX
-        XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID ROOTLESS_USER
-
 
 # Default VM to use unless set or modified by task
 gce_instance:
@@ -401,10 +386,6 @@ test_build_cache_images_task:
     # VMs created by packer are not cleaned up by cirrus, must allow task to complete
     auto_cancellation: $CI != "true"
 
-    env:
-        # Unique to this specific run of _all_ tasks
-        BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}"
-
     gce_instance:
         image_project: "libpod-218412"
         zone: "us-central1-a"
@@ -421,7 +402,7 @@ test_build_cache_images_task:
 
 
 # Test building of new cache-images for future PR testing, in this PR.
-test_built_images_task:
+verify_test_built_images_task:
 
     only_if: >-
         $CIRRUS_BRANCH != 'master' &&
@@ -433,13 +414,6 @@ test_built_images_task:
         - "gating"
         - "test_build_cache_images"
 
-    # VMs created by packer are not cleaned up by cirrus, must allow task to complete
-    auto_cancellation: $CI != "true"
-
-    env:
-        # Unique to this specific run of _all_ tasks
-        BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}"
-
     gce_instance:
         matrix:
             # Images are generated separately, from build_images_task (below)
@@ -455,11 +429,7 @@ test_built_images_task:
     environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
     unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}'
     integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
-    rootless_testing_script:
-        - export ROOTLESS_USER=yarrpilferingpirate
-        - export ROOTLESS_UID=234567
-        - export ROOTLESS_GID=234567
-        - cd $GOSRC && $SCRIPT_BASE/setup_and_run_rootless.sh |& ${TIMESTAMP}
+    rootless_testing_script: '$SCRIPT_BASE/resetup_and_run_rootless.sh |& ${TIMESTAMP}'
 
 
 # Build new cache-images for future PR testing, but only after a PR merge.
@@ -524,6 +494,7 @@ success_task:
         - "special_testing"
         - "optional_testing"
         - "test_build_cache_images"
+        - "verify_test_built_images"
         - "build_cache_images"
 
     env:
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md
index ce3ea7777..1d62f71ce 100644
--- a/contrib/cirrus/README.md
+++ b/contrib/cirrus/README.md
@@ -124,29 +124,54 @@ in the PR description, add the magic string:  ``***CIRRUS: TEST IMAGES***``
        This is a retained as a secondary method for tracking/auditing
        creation of VM images, should it ever be needed.
 
-***Manual Steps:***  In order to utilize built images, their names must be upated
-in ``.cirrus.yml``.  For example, if the image ``blah-1234`` was produced above:
+### ``verify_test_built_images`` Task
+
+Only runs following successful ``test_build_cache_images_task`` task.  Uses
+images following the standard naming format; ***however, only runs a limited
+sub-set of automated tests***.  Validating newly built images fully, requires
+updating ``.cirrus.yml``.
+
+***Manual Steps:***  Assuming `verify_test_built_images` passes, then
+you'll find the new image names displayed at the end of the
+`test_build_cache_images_task` in the `build_vm_images` output.
+For example:
+
+```
+...cut...
+==> Builds finished. The artifacts of successful builds are:
+--> rhel-7: A disk image was created: rhel-7-libpod-5699523102900224
+--> rhel-7:
+--> ubuntu-18: A disk image was created: ubuntu-18-libpod-5699523102900224
+--> ubuntu-18:
+--> fedora-29: A disk image was created: fedora-29-libpod-5699523102900224
+--> fedora-29:
+--> fedora-28: A disk image was created: fedora-28-libpod-5699523102900224
+```
+
+An updated (or new) pull-request with this change, will utilize
+the new cache-images:
+
 
 ```yaml
 env:
+    ...cut...
     ####
     #### Cache-image names to test with
     ###
-    BLAH_CACHE_IMAGE_NAME: "blah-1234"
+    FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5699523102900224"
+    PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5699523102900224"
+    UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5699523102900224"
+    PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-5699523102900224"
+    ...cut...
 ```
 
-A new pull-request with that change, will run tasks utilizing that image.
-
-
-### ``test_built_images`` Task
-
-Only runs following successful ``test_build_cache_images_task`` task.  Uses
-images following the standard naming format, with execution of
-the 'gate', 'testing' and 'rootless_testing' scripts.  Validating the images
-suitability for wide-spread use.
+Take care to also update the PR description if any 'magic' phrases were used
+(they affect which tests run).  In other words, you'll likely want to
+remove the ``***CIRRUS: TEST IMAGES***`` string - otherwise Cirrus-CI will
+simply build new and test again.
 
 
-### ``cache_images`` Task
+### ``build_cache_images`` Task  *(Deprecated)*
 
 Exactly the same as ``test_build_cache_images_task`` task, but only runs on
 the master branch.  Requires a magic string to be in the `HEAD`
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 565646c12..fb1685104 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -19,9 +19,13 @@ CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-HEAD}
 CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-FETCH_HEAD}
 SPECIALMODE="${SPECIALMODE:-none}"
 export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
-ROOTLESS_USER="${ROOTLESS_USER:-}"
-ROOTLESS_UID="${ROOTLESS_UID:-}"
-ROOTLESS_GID="${ROOTLESS_GID:-}"
+
+if [[ "$USER" == "root" ]]
+then
+    ROOTLESS_USER="${ROOTLESS_USER:-}"
+else
+    ROOTLESS_USER="${ROOTLESS_USER:-$USER}"
+fi
 
 if ! [[ "$PATH" =~ "/usr/local/bin" ]]
 then
@@ -35,6 +39,23 @@ then
     source "$HOME/$ENVLIB"
 fi
 
+# Space separated list of environment variables to unset before testing
+UNSET_ENV_VARS='
+    GCP_PROJECT_ID GCE_SSH_USERNAME SERVICE_ACCOUNT RHSM_COMMAND BUILT_IMAGE_SUFFIX
+    IRCID RHEL_BASE_IMAGE FAH_BASE_IMAGE FEDORA_BASE_IMAGE CENTOS_BASE_IMAGE
+    UBUNTU_BASE_IMAGE PACKER_VER PACKER_BUILDS RUNC_COMMIT CRIU_COMMIT
+    CRIO_COMMIT CNI_COMMIT FEDORA_CNI_COMMIT PACKER_BASE SCRIPT_BASE
+    CIRRUS_SHELL CIRRUS_WORKING_DIR ENVLIB CIRRUS_CI
+    CI_NODE_INDEX CI_NODE_TOTAL CIRRUS_BASE_BRANCH CIRRUS_BASE_SHA
+    CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO CIRRUS_CLONE_DEPTH
+    CIRRUS_COMMIT_MESSAGE CIRRUS_CHANGE_MESSAGE CIRRUS_REPO_CLONE_HOST
+    CIRRUS_DEFAULT_BRANCH CIRRUS_PR CIRRUS_TAG CIRRUS_OS CIRRUS_TASK_NAME
+    CIRRUS_TASK_ID CIRRUS_REPO_NAME CIRRUS_REPO_OWNER CIRRUS_REPO_FULL_NAME
+    CIRRUS_REPO_CLONE_URL CIRRUS_SHELL CIRRUS_USER_COLLABORATOR CIRRUS_USER_PERMISSION
+    CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST PACKER_BUILDS
+    XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID ROOTLESS_USER
+'
+
 # Pass in a list of one or more envariable names; exit non-zero with
 # helpful error message if any value is empty
 req_env_var() {
@@ -126,12 +147,14 @@ os_release_id() {
     eval "$(egrep -m 1 '^ID=' /etc/os-release | tr -d \' | tr -d \")"
     echo "$ID"
 }
+export OS_RELEASE_ID="$(os_release_id)"
 
 # Return a GCE image-name compatible string representation of distribution major version
 os_release_ver() {
     eval "$(egrep -m 1 '^VERSION_ID=' /etc/os-release | tr -d \' | tr -d \")"
     echo "$VERSION_ID" | cut -d '.' -f 1
 }
+export OS_RELEASE_VER="$(os_release_VER)"
 
 bad_os_id_ver() {
     echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $ARGS"
@@ -158,6 +181,10 @@ ircmsg() {
 setup_rootless() {
     req_env_var ROOTLESS_USER GOSRC ENVLIB
 
+    make install.catatonit
+    go get github.com/onsi/ginkgo/ginkgo
+    go get github.com/onsi/gomega/...
+
     if passwd --status $ROOTLESS_USER
     then
         echo "Updating $ROOTLESS_USER user permissions on possibly changed libpod code"
@@ -180,7 +207,8 @@ setup_rootless() {
     chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOSRC"
 
     echo "creating ssh keypair for $USER"
-    ssh-keygen -P "" -f $HOME/.ssh/id_rsa
+    [[ -r "$HOME/.ssh/id_rsa" ]] || \
+        ssh-keygen -P "" -f "$HOME/.ssh/id_rsa"
 
     echo "Allowing ssh key for $ROOTLESS_USER"
     (umask 077 && mkdir "/home/$ROOTLESS_USER/.ssh")
@@ -199,11 +227,14 @@ setup_rootless() {
     install -o $ROOTLESS_USER -g $ROOTLESS_USER -m 0700 \
         "$HOME/$ENVLIB" "/home/$ROOTLESS_USER/$ENVLIB"
 
+    # Allow the tests to run
+    echo "export ROOTLESS_USER=$ROOTLESS_USER" >> "/home/$ROOTLESS_USER/$ENVLIB"
+
     echo "Configuring user's go environment variables"
     su --login --command 'go env' $ROOTLESS_USER | \
         while read envline
         do
-            X=$(echo "export $envline" | tee -a "/home/$ROOTLESS_USER/.bash_profile") && echo "$X"
+            X=$(echo "export $envline" >> "/home/$ROOTLESS_USER/$ENVLIB")
         done
 }
 
@@ -242,6 +273,10 @@ install_cni_plugins() {
 }
 
 install_runc_from_git(){
+    req_env_var "
+        GOPATH $GOPATH
+        OS_RELEASE_ID $OS_RELEASE_ID
+    "
     wd=$(pwd)
     DEST="$GOPATH/src/github.com/opencontainers/runc"
     rm -rf "$DEST"
@@ -249,7 +284,12 @@ install_runc_from_git(){
     cd "$DEST"
     ooe.sh git fetch origin --tags
     ooe.sh git checkout -q "$RUNC_COMMIT"
-    ooe.sh make static BUILDTAGS="seccomp apparmor selinux"
+    if [[ "${OS_RELEASE_ID}" == "ubuntu" ]]
+    then
+        ooe.sh make static BUILDTAGS="seccomp apparmor"
+    else
+        ooe.sh make BUILDTAGS="seccomp selinux"
+    fi
     sudo install -m 755 runc /usr/bin/runc
     cd $wd
 }
diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh
index b86ba1a1c..f9728f2f7 100644
--- a/contrib/cirrus/packer/centos_setup.sh
+++ b/contrib/cirrus/packer/centos_setup.sh
@@ -59,7 +59,6 @@ ooe.sh sudo yum -y install \
     python3-dateutil \
     python3-psutil \
     python3-pytoml \
-    runc \
     skopeo-containers \
     unzip \
     vim \
@@ -68,14 +67,16 @@ ooe.sh sudo yum -y install \
 
 install_scl_git
 
-install_cni_plugins
-
-install_buildah
-
 install_conmon
 
+install_runc
+
 install_criu
 
+install_cni_plugins
+
+install_buildah
+
 install_packer_copied_files
 
 sudo /tmp/libpod/hack/install_catatonit.sh
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index fef74179d..d9229c668 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -60,7 +60,6 @@ ooe.sh sudo dnf install -y \
     python3-dateutil \
     python3-psutil \
     python3-pytoml \
-    runc \
     skopeo-containers \
     slirp4netns \
     unzip \
@@ -70,13 +69,15 @@ ooe.sh sudo dnf install -y \
 
 install_varlink
 
+install_conmon
+
+install_runc
+
 CNI_COMMIT=$FEDORA_CNI_COMMIT
 install_cni_plugins
 
 install_buildah
 
-install_conmon
-
 install_packer_copied_files
 
 sudo /tmp/libpod/hack/install_catatonit.sh
diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh
index b6240be80..dee35a102 100644
--- a/contrib/cirrus/packer/rhel_setup.sh
+++ b/contrib/cirrus/packer/rhel_setup.sh
@@ -22,6 +22,9 @@ ooe.sh sudo subscription-manager repos \
     --enable=rhel-7-server-extras-rpms \
     --enable=rhel-server-rhscl-7-rpms
 
+sudo sed -r -i -e 's/^notify_only=.+/notify_only=0/g' \
+    /etc/yum/pluginconf.d/search-disabled-repos.conf
+
 ooe.sh sudo yum -y update
 
 ooe.sh sudo yum -y install \
@@ -66,7 +69,6 @@ ooe.sh sudo yum -y install \
     python34-psutil \
     python34-pytoml \
     python34-PyYAML \
-    runc \
     skopeo-containers \
     unzip \
     vim \
@@ -75,14 +77,16 @@ ooe.sh sudo yum -y install \
 
 install_scl_git
 
-install_cni_plugins
-
-install_buildah
-
 install_conmon
 
+install_runc
+
 install_criu
 
+install_cni_plugins
+
+install_buildah
+
 install_packer_copied_files
 
 sudo /tmp/libpod/hack/install_catatonit.sh
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index 93dbd0fa0..d02b71e12 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -90,10 +90,10 @@ ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/*
 ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub
 ooe.sh sudo update-grub
 
-install_runc
-
 install_conmon
 
+install_runc
+
 install_criu
 
 install_cni_plugins
diff --git a/contrib/cirrus/resetup_and_run_rootless.sh b/contrib/cirrus/resetup_and_run_rootless.sh
new file mode 100755
index 000000000..1c387fdf5
--- /dev/null
+++ b/contrib/cirrus/resetup_and_run_rootless.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+# N/B: This script is only intended to be used for the special-case of
+#      setting up and executing the rootless tests AFTER normal tests complete
+#      while testing a freshly built image.
+
+source $(dirname $0)/lib.sh
+
+# must be after source lib.sh b/c it loads $ENVLIB
+export ROOTLESS_USER="pilferingpirate$RANDOM"
+
+req_env_var "
+CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR
+GOSRC $GOSRC
+SCRIPT_BASE $SCRIPT_BASE
+ROOTLESS_USER $ROOTLESS_USER
+"
+
+if ! run_rootless
+then
+    die 86 "Error: Expected rootless env. var not set or empty"
+fi
+
+cd $GOSRC
+make clean
+setup_rootless
+
+ssh $ROOTLESS_USER@localhost \
+            -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \
+            $CIRRUS_WORKING_DIR/$SCRIPT_BASE/rootless_test.sh
diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh
index 13386bb09..d4cc6965e 100755
--- a/contrib/cirrus/rootless_test.sh
+++ b/contrib/cirrus/rootless_test.sh
@@ -1,12 +1,10 @@
 #!/bin/bash
 
-set -ex
+set -e
 
-source "$HOME/.bash_profile"
-source $GOSRC/$SCRIPT_BASE/lib.sh
-cd $GOSRC
+source $(dirname $0)/lib.sh
 
-req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER
+req_env_var GOSRC ROOTLESS_USER
 
 if [[ "$UID" == "0" ]]
 then
@@ -17,6 +15,10 @@ fi
 echo "."
 echo "Hello, my name is $USER and I live in $PWD can I be your friend?"
 
+show_env_vars
+
+export PODMAN_VARLINK_ADDRESS=unix:/tmp/podman-$(id -u)
+
 cd "$GOSRC"
 make
 make varlink_generate
diff --git a/contrib/cirrus/setup_and_run_rootless.sh b/contrib/cirrus/setup_and_run_rootless.sh
deleted file mode 100755
index 7ca34a6ac..000000000
--- a/contrib/cirrus/setup_and_run_rootless.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-source $(dirname $0)/lib.sh
-
-req_env_var "
-CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR
-GOSRC $GOSRC
-SCRIPT_BASE $SCRIPT_BASE
-ROOTLESS_USER $ROOTLESS_USER
-ROOTLESS_UID $ROOTLESS_UID
-ROOTLESS_GID $ROOTLESS_GID
-"
-
-if run_rootless
-then
-    die 86 "Error: Expected rootless env. vars not set or empty"
-fi
-
-cd $GOSRC
-setup_rootless
-
-ssh $ROOTLESS_USER@localhost \
-            -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \
-            $CIRRUS_WORKING_DIR/$SCRIPT_BASE/rootless_test.sh
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 1a89a5b71..b3319a3e1 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -37,7 +37,6 @@ then
         "export OS_RELEASE_VER=\"$(os_release_ver)\"" \
         "export OS_REL_VER=\"$(os_release_id)-$(os_release_ver)\"" \
         "export TEST_REMOTE_CLIENT=\"$TEST_REMOTE_CLIENT\"" \
-        "export BUILT_IMAGE_SUFFIX=\"-$CIRRUS_REPO_NAME-${CIRRUS_CHANGE_IN_REPO:0:8}\"" \
         "export GOPATH=\"/var/tmp/go\"" \
         'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \
         'export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"'