summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChris Evich <cevich@redhat.com>2020-10-07 16:43:36 -0400
committerChris Evich <cevich@redhat.com>2020-11-18 15:34:01 -0500
commit887f88c490a63e759a34eaf642a4eb47e8087512 (patch)
tree615ca61a428cae6deade313ced9b46a471538827
parent4434bd797842c3015c0e6132eaf2509fed370c26 (diff)
downloadpodman-887f88c490a63e759a34eaf642a4eb47e8087512.tar.gz
podman-887f88c490a63e759a34eaf642a4eb47e8087512.tar.bz2
podman-887f88c490a63e759a34eaf642a4eb47e8087512.zip
Github-Actions: Send e-mail on Cirrus cron failure
This repository has a number of automaticly triggered branch-level testing enabled. However, other than remembering to go look at a specific WebUI, there is no way for anybody to notice if/when these jobs fail. This commit introduces a github-action workflow which runs periodically, checking for failed cron-triggered Cirrus-CI jobs. When it finds any, it formats a simple report for e-mail delivery. The list of destination addresses is configurable at any time by merging changes to a simple CSV file. Signed-off-by: Chris Evich <cevich@redhat.com>
-rwxr-xr-x.github/actions/check_cirrus_cron/cron_failures.sh116
-rw-r--r--.github/workflows/check_cirrus_cron.yml81
-rw-r--r--contrib/cirrus/cron-fail_addrs.csv1
3 files changed, 198 insertions, 0 deletions
diff --git a/.github/actions/check_cirrus_cron/cron_failures.sh b/.github/actions/check_cirrus_cron/cron_failures.sh
new file mode 100755
index 000000000..2693df417
--- /dev/null
+++ b/.github/actions/check_cirrus_cron/cron_failures.sh
@@ -0,0 +1,116 @@
+#!/bin/bash
+
+set -eo pipefail
+
+# Intended to be executed from a github action workflow step.
+# Outputs the Cirrus cron names and IDs of any failed builds
+
+err() {
+ # Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions
+ echo "::error file=${BASH_SOURCE[0]},line=${BASH_LINENO[0]}::${1:-No error message given}"
+ exit 1
+}
+
+_errfmt="Expecting %s value to not be empty"
+if [[ -z "$GITHUB_REPOSITORY" ]]; then
+ err $(printf "$_errfmt" "\$GITHUB_REPOSITORY")
+elif [[ -z "$NAME_ID_FILEPATH" ]]; then
+ err $(printf "$_errfmt" "\$NAME_ID_FILEPATH")
+fi
+
+mkdir -p artifacts
+cat > ./artifacts/query_raw.json << "EOF"
+{"query":"
+ query CronNameStatus($owner: String!, $repo: String!) {
+ githubRepository(owner: $owner, name: $repo) {
+ cronSettings {
+ name
+ lastInvocationBuild {
+ id
+ status
+ }
+ }
+ }
+ }
+",
+"variables":"{
+ \"owner\": \"@@OWNER@@\",
+ \"repo\": \"@@REPO@@\"
+}"}
+EOF
+# Makes for easier copy/pasting query to/from
+# https://cirrus-ci.com/explorer
+owner=$(cut -d '/' -f 1 <<<"$GITHUB_REPOSITORY")
+repo=$(cut -d '/' -f 2 <<<"$GITHUB_REPOSITORY")
+sed -i -r -e "s/@@OWNER@@/$owner/g" -e "s/@@REPO@@/$repo/g" ./artifacts/query_raw.json
+
+echo "::group::Posting GraphQL Query"
+# Easier to debug in error-reply when query is compacted
+tr -d '\n' < ./artifacts/query_raw.json | tr -s ' ' | tee ./artifacts/query.json | \
+ jq --indent 4 --color-output .
+
+if grep -q '@@' ./artifacts/query.json; then
+ err "Found unreplaced substitution token in raw query JSON"
+fi
+curl \
+ --request POST \
+ --silent \
+ --location \
+ --header 'content-type: application/json' \
+ --url 'https://api.cirrus-ci.com/graphql' \
+ --data @./artifacts/query.json \
+ --output ./artifacts/reply.json
+echo "::endgroup::"
+
+echo "::group::Received GraphQL Reply"
+jq --indent 4 --color-output . <./artifacts/reply.json || \
+ cat ./artifacts/reply.json
+echo "::endgroup::"
+
+# Desireable to catch non-JSON encoded errors in reply.
+if grep -qi 'error' ./artifacts/reply.json; then
+ err "Found the word 'error' in reply"
+fi
+
+# e.x. reply.json
+# {
+# "data": {
+# "githubRepository": {
+# "cronSettings": [
+# {
+# "name": "Keepalive_v2.0",
+# "lastInvocationBuild": {
+# "id": "5776050544181248",
+# "status": "EXECUTING"
+# }
+# },
+# {
+# "name": "Keepalive_v1.9",
+# "lastInvocationBuild": {
+# "id": "5962921081569280",
+# "status": "COMPLETED"
+# }
+# },
+# {
+# "name": "Keepalive_v2.0.5-rhel",
+# "lastInvocationBuild": {
+# "id": "5003065549914112",
+# "status": "FAILED"
+# }
+# }
+# ]
+# }
+# }
+# }
+_filt='.data.githubRepository.cronSettings | map(select(.lastInvocationBuild.status=="FAILED") | { name:.name, id:.lastInvocationBuild.id} | join(" ")) | join("\n")'
+jq --raw-output "$_filt" ./artifacts/reply.json > "$NAME_ID_FILEPATH"
+
+echo "<Cron Name> <Failed Build ID>"
+cat "$NAME_ID_FILEPATH"
+
+# Don't rely on a newline present for zero/one output line, always count words
+records=$(wc --words "$NAME_ID_FILEPATH" | cut -d ' ' -f 1)
+# Always two words per record
+failures=$((records/2))
+echo "::set-output name=failures::$failures"
+echo "Total failed Cirrus-CI cron builds: $failures"
diff --git a/.github/workflows/check_cirrus_cron.yml b/.github/workflows/check_cirrus_cron.yml
new file mode 100644
index 000000000..86f8c26dc
--- /dev/null
+++ b/.github/workflows/check_cirrus_cron.yml
@@ -0,0 +1,81 @@
+---
+
+# Format Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions
+
+# Required to un-FUBAR default ${{github.workflow}} value
+name: check_cirrus_cron
+
+on:
+ schedule:
+ # Assume cirrus cron jobs runs at least once per day
+ - cron: '59 23 * * *'
+ # Debug: Allow triggering job manually in github-actions WebUI
+ workflow_dispatch: {}
+
+env:
+ # Debug-mode can reveal secrets, only enable by a secret value.
+ # Ref: https://help.github.com/en/actions/configuring-and-managing-workflows/managing-a-workflow-run#enabling-step-debug-logging
+ ACTIONS_STEP_DEBUG: '${{ secrets.ACTIONS_STEP_DEBUG }}'
+ # File with CSV listing of zero or more e-mail addresses for delivery
+ # of daily failure notice e-mails.
+ FAILMAILCSV: './contrib/cirrus/cron-fail_addrs.csv'
+ # Filename for table of cron-name to build-id data
+ # (must be in $GITHUB_WORKSPACE/artifacts/)
+ NAME_ID_FILEPATH: './artifacts/name_id.txt'
+
+jobs:
+ cron_failures:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ ref: master
+ persist-credentials: false
+
+ - name: Get failed cron names and Build IDs
+ id: cron
+ run: './.github/actions/${{ github.workflow }}/${{ github.job }}.sh'
+
+ - if: steps.cron.outputs.failures > 0
+ shell: bash
+ # Must be inline, since context expressions are used.
+ # Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions
+ run: |
+ set -eo pipefail
+ (
+ echo "Detected one or more Cirrus-CI cron-triggered jobs have failed recently:"
+ echo ""
+
+ while read -r NAME BID; do
+ echo "Cron build '$NAME' Failed: https://cirrus-ci.com/build/$BID"
+ done < "$NAME_ID_FILEPATH"
+
+ echo ""
+ echo "# Source: ${{ github.workflow }} workflow on ${{ github.repository }}."
+ # Separate content from sendgrid.com automatic footer.
+ echo ""
+ ) > ./artifacts/email_body.txt
+
+ - if: steps.cron.outputs.failures > 0
+ id: mailto
+ run: printf "::set-output name=csv::%s\n" $(cat "$FAILMAILCSV")
+
+ - if: steps.mailto.outputs.csv != ''
+ name: Send failure notification e-mail
+ # Ref: https://github.com/dawidd6/action-send-mail
+ uses: dawidd6/action-send-mail@v2.2.2
+ with:
+ server_address: ${{secrets.ACTION_MAIL_SERVER}}
+ server_port: 465
+ username: ${{secrets.ACTION_MAIL_USERNAME}}
+ password: ${{secrets.ACTION_MAIL_PASSWORD}}
+ subject: Cirrus-CI cron build failures on ${{github.repository}}
+ to: ${{steps.mailto.outputs.csv}}
+ from: ${{secrets.ACTION_MAIL_SENDER}}
+ body: file://./artifacts/email_body.txt
+
+ - if: always()
+ uses: actions/upload-artifact@v2
+ with:
+ name: ${{ github.job }}_artifacts
+ path: artifacts/*
diff --git a/contrib/cirrus/cron-fail_addrs.csv b/contrib/cirrus/cron-fail_addrs.csv
new file mode 100644
index 000000000..c25fc1226
--- /dev/null
+++ b/contrib/cirrus/cron-fail_addrs.csv
@@ -0,0 +1 @@
+rh.container.bot@gmail.com