Cirrus: Notify on IRC if post-merge testing fails

Until recently it was very difficult to execute any scripts if part of a
task failed.  A new feature in Cirrus-CI makes this easy.  Use it to
post a notice on IRC when any task fails.

Also: Add quotes around yaml-string values for consistency and
syntax-highlighting correctness.

Signed-off-by: Chris Evich <cevich@redhat.com>

3 files changed, 59 insertions, 19 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 8e26ce72e..14ba3fc6e 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -140,6 +140,10 @@ gating_task:
         - '/usr/local/bin/entrypoint.sh clean podman BUILDTAGS="exclude_graphdriver_devicemapper selinux seccomp"'
         - '/usr/local/bin/entrypoint.sh clean podman-remote-darwin'
 
+    on_failure:
+        master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
+
+
 build_each_commit_task:
 
     depends_on:
@@ -160,9 +164,12 @@ build_each_commit_task:
     timeout_in: 30m
 
     script:
-        - $SCRIPT_BASE/setup_environment.sh
-        - git fetch --depth $CIRRUS_CLONE_DEPTH origin $CIRRUS_BASE_BRANCH
-        - env GOPATH=/var/tmp/go/ make build-all-new-commits GIT_BASE_BRANCH=origin/$CIRRUS_BASE_BRANCH
+        - '$SCRIPT_BASE/setup_environment.sh'
+        - 'git fetch --depth $CIRRUS_CLONE_DEPTH origin $CIRRUS_BASE_BRANCH'
+        - 'env GOPATH=/var/tmp/go/ make build-all-new-commits GIT_BASE_BRANCH=origin/$CIRRUS_BASE_BRANCH'
+
+    on_failure:
+        master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
 
 
 # Update metadata on VM images referenced by this repository state
@@ -186,7 +193,7 @@ meta_task:
         GCPPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
         CIRRUS_CLONE_DEPTH: 1  # source not used
 
-    script: /usr/local/bin/entrypoint.sh
+    script: '/usr/local/bin/entrypoint.sh'
 
 
 # This task does the unit and integration testing for every platform
@@ -219,14 +226,12 @@ testing_task:
 
     # Every *_script runs in sequence, for each task. The name prefix is for
     # WebUI reference.  The values may be strings...
-    setup_environment_script: $SCRIPT_BASE/setup_environment.sh
+    setup_environment_script: '$SCRIPT_BASE/setup_environment.sh'
+    unit_test_script: '$SCRIPT_BASE/unit_test.sh'
+    integration_test_script: '$SCRIPT_BASE/integration_test.sh'
 
-    # ...or lists of strings
-    unit_test_script:
-        - go version
-        - $SCRIPT_BASE/unit_test.sh
-
-    integration_test_script: $SCRIPT_BASE/integration_test.sh
+    on_failure:
+        master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
 
 
 # This task executes tests as a regular user on a system
@@ -252,12 +257,15 @@ rootless_testing_task:
 
     timeout_in: 120m
 
-    setup_environment_script: $SCRIPT_BASE/setup_environment.sh
+    setup_environment_script: '$SCRIPT_BASE/setup_environment.sh'
     rootless_test_script: >-
         ssh $ROOTLESS_USER@localhost
         -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no
         $CIRRUS_WORKING_DIR/$SCRIPT_BASE/rootless_test.sh
 
+    on_failure:
+        master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
+
 
 # Because system tests are stored within the repository, it is sometimes
 # necessary to execute them within a PR to validate changes.
@@ -283,8 +291,8 @@ optional_testing_task:
 
     timeout_in: 60m
 
-    setup_environment_script: $SCRIPT_BASE/setup_environment.sh
-    system_test_script: $SCRIPT_BASE/system_test.sh
+    setup_environment_script: '$SCRIPT_BASE/setup_environment.sh'
+    system_test_script: '$SCRIPT_BASE/system_test.sh'
 
 
 # Build new cache-images for future PR testing, but only after a PR merge.
@@ -317,8 +325,8 @@ cache_images_task:
         scopes:
             - compute
             - devstorage.full_control
-    environment_script: $SCRIPT_BASE/setup_environment.sh
-    build_vm_images_script: $SCRIPT_BASE/build_vm_images.sh
+    environment_script: '$SCRIPT_BASE/setup_environment.sh'
+    build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh'
 
     # TODO,Continuous Delivery: Automatically open a libpod PR after using 'sed' to replace
     #                           the image_names with the new (just build) images.  That will
@@ -331,6 +339,9 @@ cache_images_task:
     #   - modify_cirrus_yaml_image_names.sh
     #   - commit_and_create_upstream_pr.sh
 
+    on_failure:
+        master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
+
 
 # Post message to IRC if everything passed
 success_task:
@@ -350,4 +361,4 @@ success_task:
         cpu: 1
         memory: 1
 
-    success_script: $SCRIPT_BASE/success.sh
+    success_script: '$SCRIPT_BASE/success.sh'
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 9419dad05..773f4f484 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -103,6 +103,15 @@ clean_env() {
     unset -v UNSET_ENV_VARS $UNSET_ENV_VARS || true  # don't fail on read-only
 }
 
+die() {
+    req_env_var "
+        1 $1
+        2 $2
+    "
+    echo "$2"
+    exit $1
+}
+
 # Return a GCE image-name compatible string representation of distribution name
 os_release_id() {
     eval "$(egrep -m 1 '^ID=' /etc/os-release | tr -d \' | tr -d \")"
@@ -136,14 +145,14 @@ stub() {
 ircmsg() {
     req_env_var "
         CIRRUS_TASK_ID $CIRRUS_TASK_ID
-        1 $1
+        @ $@
     "
     # Sometimes setup_environment.sh didn't run
     SCRIPT="$(dirname $0)/podbot.py"
     NICK="podbot_$CIRRUS_TASK_ID"
     NICK="${NICK:0:15}"  # Any longer will break things
     set +e
-    $SCRIPT $NICK $1
+    $SCRIPT $NICK $@
     echo "Ignoring exit($?)"
     set -e
 }
diff --git a/contrib/cirrus/notice_master_failure.sh b/contrib/cirrus/notice_master_failure.sh
new file mode 100755
index 000000000..e18460a54
--- /dev/null
+++ b/contrib/cirrus/notice_master_failure.sh
@@ -0,0 +1,20 @@
+#!/bin/bash
+
+set -e
+
+source $(dirname $0)/lib.sh
+
+# mIRC "escape" codes are the most standard, for a non-standard client-side interpretation.
+ETX="$(echo -n -e '\x03')"
+RED="${ETX}4"
+NOR="$(echo -n -e '\x0f')"
+
+if [[ "$CIRRUS_BRANCH" =~ "master" ]]
+then
+    BURL="https://cirrus-ci.com/build/$CIRRUS_BUILD_ID"
+    echo "Monitoring execution of $CIRRUS_TASK_NAME and notifying on failure"
+    MSG="${RED}[Action Recommended]: ${NOR}Post-merge testing ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on $(os_release_id)-$(os_release_ver): $BURL.  Please investigate, and re-run if appropriate."
+fi
+
+# This script assumed to be executed on failure
+die 1 "Testing Failed"