Github-Actions: Send e-mail on Cirrus cron failure

This repository has a number of automaticly triggered branch-level
testing enabled.  However, other than remembering to go look at a
specific WebUI, there is no way for anybody to notice if/when these jobs
fail.

This commit introduces a github-action workflow which runs periodically,
checking for failed cron-triggered Cirrus-CI jobs.  When it finds any, it
formats a simple report for e-mail delivery.  The list of destination
addresses is configurable at any time by merging changes to a
simple CSV file.

Signed-off-by: Chris Evich <cevich@redhat.com>

3 files changed, 198 insertions, 0 deletions

diff --git a/.github/actions/check_cirrus_cron/cron_failures.sh b/.github/actions/check_cirrus_cron/cron_failures.sh
new file mode 100755
index 000000000..2693df417
--- /dev/null
+++ b/.github/actions/check_cirrus_cron/cron_failures.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Intended to be executed from a github action workflow step.
+# Outputs the Cirrus cron names and IDs of any failed builds
+
+err() {
+    # Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions
+    echo "::error file=${BASH_SOURCE[0]},line=${BASH_LINENO[0]}::${1:-No error message given}"
+    exit 1
+}
+
+_errfmt="Expecting %s value to not be empty"
+if [[ -z "$GITHUB_REPOSITORY" ]]; then
+    err $(printf "$_errfmt" "\$GITHUB_REPOSITORY")
+elif [[ -z "$NAME_ID_FILEPATH" ]]; then
+    err $(printf "$_errfmt" "\$NAME_ID_FILEPATH")
+fi
+
+mkdir -p artifacts
+cat > ./artifacts/query_raw.json << "EOF"
+{"query":"
+  query CronNameStatus($owner: String!, $repo: String!) {
+    githubRepository(owner: $owner, name: $repo) {
+      cronSettings {
+        name
+        lastInvocationBuild {
+          id
+          status
+        }
+      }
+    }
+  }
+",
+"variables":"{
+  \"owner\": \"@@OWNER@@\",
+  \"repo\": \"@@REPO@@\"
+}"}
+EOF
+# Makes for easier copy/pasting query to/from
+# https://cirrus-ci.com/explorer
+owner=$(cut -d '/' -f 1 <<<"$GITHUB_REPOSITORY")
+repo=$(cut -d '/' -f 2 <<<"$GITHUB_REPOSITORY")
+sed -i -r -e "s/@@OWNER@@/$owner/g" -e "s/@@REPO@@/$repo/g" ./artifacts/query_raw.json
+
+echo "::group::Posting GraphQL Query"
+# Easier to debug in error-reply when query is compacted
+tr -d '\n' < ./artifacts/query_raw.json | tr -s ' ' | tee ./artifacts/query.json | \
+    jq --indent 4 --color-output .
+
+if grep -q '@@' ./artifacts/query.json; then
+    err "Found unreplaced substitution token in raw query JSON"
+fi
+curl \
+  --request POST \
+  --silent \
+  --location \
+  --header 'content-type: application/json' \
+  --url 'https://api.cirrus-ci.com/graphql' \
+  --data @./artifacts/query.json \
+  --output ./artifacts/reply.json
+echo "::endgroup::"
+
+echo "::group::Received GraphQL Reply"
+jq --indent 4 --color-output . <./artifacts/reply.json || \
+    cat ./artifacts/reply.json
+echo "::endgroup::"
+
+# Desireable to catch non-JSON encoded errors in reply.
+if grep -qi 'error' ./artifacts/reply.json; then
+    err "Found the word 'error' in reply"
+fi
+
+# e.x. reply.json
+# {
+#   "data": {
+#     "githubRepository": {
+#       "cronSettings": [
+#         {
+#           "name": "Keepalive_v2.0",
+#           "lastInvocationBuild": {
+#             "id": "5776050544181248",
+#             "status": "EXECUTING"
+#           }
+#         },
+#         {
+#           "name": "Keepalive_v1.9",
+#           "lastInvocationBuild": {
+#             "id": "5962921081569280",
+#             "status": "COMPLETED"
+#           }
+#         },
+#         {
+#           "name": "Keepalive_v2.0.5-rhel",
+#           "lastInvocationBuild": {
+#             "id": "5003065549914112",
+#             "status": "FAILED"
+#           }
+#         }
+#       ]
+#     }
+#   }
+# }
+_filt='.data.githubRepository.cronSettings | map(select(.lastInvocationBuild.status=="FAILED") | { name:.name, id:.lastInvocationBuild.id} | join(" ")) | join("\n")'
+jq --raw-output "$_filt" ./artifacts/reply.json > "$NAME_ID_FILEPATH"
+
+echo "<Cron Name> <Failed Build ID>"
+cat "$NAME_ID_FILEPATH"
+
+# Don't rely on a newline present for zero/one output line, always count words
+records=$(wc --words "$NAME_ID_FILEPATH" | cut -d ' ' -f 1)
+# Always two words per record
+failures=$((records/2))
+echo "::set-output name=failures::$failures"
+echo "Total failed Cirrus-CI cron builds: $failures"
diff --git a/.github/workflows/check_cirrus_cron.yml b/.github/workflows/check_cirrus_cron.yml
new file mode 100644
index 000000000..86f8c26dc
--- /dev/null
+++ b/.github/workflows/check_cirrus_cron.yml
@@ -0,0 +1,81 @@
+---
+
+# Format Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions
+
+# Required to un-FUBAR default ${{github.workflow}} value
+name: check_cirrus_cron
+
+on:
+    schedule:
+        # Assume cirrus cron jobs runs at least once per day
+        - cron:  '59 23 * * *'
+    # Debug: Allow triggering job manually in github-actions WebUI
+    workflow_dispatch: {}
+
+env:
+    # Debug-mode can reveal secrets, only enable by a secret value.
+    # Ref: https://help.github.com/en/actions/configuring-and-managing-workflows/managing-a-workflow-run#enabling-step-debug-logging
+    ACTIONS_STEP_DEBUG: '${{ secrets.ACTIONS_STEP_DEBUG }}'
+    # File with CSV listing of zero or more e-mail addresses for delivery
+    # of daily failure notice e-mails.
+    FAILMAILCSV: './contrib/cirrus/cron-fail_addrs.csv'
+    # Filename for table of cron-name to build-id data
+    # (must be in $GITHUB_WORKSPACE/artifacts/)
+    NAME_ID_FILEPATH: './artifacts/name_id.txt'
+
+jobs:
+    cron_failures:
+        runs-on: ubuntu-latest
+        steps:
+            - uses: actions/checkout@v2
+              with:
+                  ref: master
+                  persist-credentials: false
+
+            - name: Get failed cron names and Build IDs
+              id: cron
+              run: './.github/actions/${{ github.workflow }}/${{ github.job }}.sh'
+
+            - if: steps.cron.outputs.failures > 0
+              shell: bash
+              # Must be inline, since context expressions are used.
+              # Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions
+              run: |
+                set -eo pipefail
+                (
+                echo "Detected one or more Cirrus-CI cron-triggered jobs have failed recently:"
+                echo ""
+
+                while read -r NAME BID; do
+                    echo "Cron build '$NAME' Failed: https://cirrus-ci.com/build/$BID"
+                done < "$NAME_ID_FILEPATH"
+
+                echo ""
+                echo "# Source: ${{ github.workflow }} workflow on ${{ github.repository }}."
+                # Separate content from sendgrid.com automatic footer.
+                echo ""
+                ) > ./artifacts/email_body.txt
+
+            - if: steps.cron.outputs.failures > 0
+              id: mailto
+              run: printf "::set-output name=csv::%s\n" $(cat "$FAILMAILCSV")
+
+            - if: steps.mailto.outputs.csv != ''
+              name: Send failure notification e-mail
+              # Ref: https://github.com/dawidd6/action-send-mail
+              uses: dawidd6/action-send-mail@v2.2.2
+              with:
+                server_address: ${{secrets.ACTION_MAIL_SERVER}}
+                server_port: 465
+                username: ${{secrets.ACTION_MAIL_USERNAME}}
+                password: ${{secrets.ACTION_MAIL_PASSWORD}}
+                subject: Cirrus-CI cron build failures on ${{github.repository}}
+                to: ${{steps.mailto.outputs.csv}}
+                from: ${{secrets.ACTION_MAIL_SENDER}}
+                body: file://./artifacts/email_body.txt
+
+            - if: always()
+              uses: actions/upload-artifact@v2
+              with:
+                  name: ${{ github.job }}_artifacts
+                  path: artifacts/*
diff --git a/contrib/cirrus/cron-fail_addrs.csv b/contrib/cirrus/cron-fail_addrs.csv
new file mode 100644
index 000000000..c25fc1226
--- /dev/null
+++ b/contrib/cirrus/cron-fail_addrs.csv
@@ -0,0 +1 @@
+rh.container.bot@gmail.com