diff options
Diffstat (limited to '.github')
| -rwxr-xr-x | .github/actions/check_cirrus_cron/cron_failures.sh | 116 | ||||
| -rw-r--r-- | .github/workflows/check_cirrus_cron.yml | 81 |
2 files changed, 197 insertions, 0 deletions
diff --git a/.github/actions/check_cirrus_cron/cron_failures.sh b/.github/actions/check_cirrus_cron/cron_failures.sh new file mode 100755 index 000000000..2693df417 --- /dev/null +++ b/.github/actions/check_cirrus_cron/cron_failures.sh @@ -0,0 +1,116 @@ +#!/bin/bash + +set -eo pipefail + +# Intended to be executed from a github action workflow step. +# Outputs the Cirrus cron names and IDs of any failed builds + +err() { + # Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-commands-for-github-actions + echo "::error file=${BASH_SOURCE[0]},line=${BASH_LINENO[0]}::${1:-No error message given}" + exit 1 +} + +_errfmt="Expecting %s value to not be empty" +if [[ -z "$GITHUB_REPOSITORY" ]]; then + err $(printf "$_errfmt" "\$GITHUB_REPOSITORY") +elif [[ -z "$NAME_ID_FILEPATH" ]]; then + err $(printf "$_errfmt" "\$NAME_ID_FILEPATH") +fi + +mkdir -p artifacts +cat > ./artifacts/query_raw.json << "EOF" +{"query":" + query CronNameStatus($owner: String!, $repo: String!) { + githubRepository(owner: $owner, name: $repo) { + cronSettings { + name + lastInvocationBuild { + id + status + } + } + } + } +", +"variables":"{ + \"owner\": \"@@OWNER@@\", + \"repo\": \"@@REPO@@\" +}"} +EOF +# Makes for easier copy/pasting query to/from +# https://cirrus-ci.com/explorer +owner=$(cut -d '/' -f 1 <<<"$GITHUB_REPOSITORY") +repo=$(cut -d '/' -f 2 <<<"$GITHUB_REPOSITORY") +sed -i -r -e "s/@@OWNER@@/$owner/g" -e "s/@@REPO@@/$repo/g" ./artifacts/query_raw.json + +echo "::group::Posting GraphQL Query" +# Easier to debug in error-reply when query is compacted +tr -d '\n' < ./artifacts/query_raw.json | tr -s ' ' | tee ./artifacts/query.json | \ + jq --indent 4 --color-output . + +if grep -q '@@' ./artifacts/query.json; then + err "Found unreplaced substitution token in raw query JSON" +fi +curl \ + --request POST \ + --silent \ + --location \ + --header 'content-type: application/json' \ + --url 'https://api.cirrus-ci.com/graphql' \ + --data @./artifacts/query.json \ + --output ./artifacts/reply.json +echo "::endgroup::" + +echo "::group::Received GraphQL Reply" +jq --indent 4 --color-output . <./artifacts/reply.json || \ + cat ./artifacts/reply.json +echo "::endgroup::" + +# Desireable to catch non-JSON encoded errors in reply. +if grep -qi 'error' ./artifacts/reply.json; then + err "Found the word 'error' in reply" +fi + +# e.x. reply.json +# { +# "data": { +# "githubRepository": { +# "cronSettings": [ +# { +# "name": "Keepalive_v2.0", +# "lastInvocationBuild": { +# "id": "5776050544181248", +# "status": "EXECUTING" +# } +# }, +# { +# "name": "Keepalive_v1.9", +# "lastInvocationBuild": { +# "id": "5962921081569280", +# "status": "COMPLETED" +# } +# }, +# { +# "name": "Keepalive_v2.0.5-rhel", +# "lastInvocationBuild": { +# "id": "5003065549914112", +# "status": "FAILED" +# } +# } +# ] +# } +# } +# } +_filt='.data.githubRepository.cronSettings | map(select(.lastInvocationBuild.status=="FAILED") | { name:.name, id:.lastInvocationBuild.id} | join(" ")) | join("\n")' +jq --raw-output "$_filt" ./artifacts/reply.json > "$NAME_ID_FILEPATH" + +echo "<Cron Name> <Failed Build ID>" +cat "$NAME_ID_FILEPATH" + +# Don't rely on a newline present for zero/one output line, always count words +records=$(wc --words "$NAME_ID_FILEPATH" | cut -d ' ' -f 1) +# Always two words per record +failures=$((records/2)) +echo "::set-output name=failures::$failures" +echo "Total failed Cirrus-CI cron builds: $failures" diff --git a/.github/workflows/check_cirrus_cron.yml b/.github/workflows/check_cirrus_cron.yml new file mode 100644 index 000000000..86f8c26dc --- /dev/null +++ b/.github/workflows/check_cirrus_cron.yml @@ -0,0 +1,81 @@ +--- + +# Format Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions + +# Required to un-FUBAR default ${{github.workflow}} value +name: check_cirrus_cron + +on: + schedule: + # Assume cirrus cron jobs runs at least once per day + - cron: '59 23 * * *' + # Debug: Allow triggering job manually in github-actions WebUI + workflow_dispatch: {} + +env: + # Debug-mode can reveal secrets, only enable by a secret value. + # Ref: https://help.github.com/en/actions/configuring-and-managing-workflows/managing-a-workflow-run#enabling-step-debug-logging + ACTIONS_STEP_DEBUG: '${{ secrets.ACTIONS_STEP_DEBUG }}' + # File with CSV listing of zero or more e-mail addresses for delivery + # of daily failure notice e-mails. + FAILMAILCSV: './contrib/cirrus/cron-fail_addrs.csv' + # Filename for table of cron-name to build-id data + # (must be in $GITHUB_WORKSPACE/artifacts/) + NAME_ID_FILEPATH: './artifacts/name_id.txt' + +jobs: + cron_failures: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + with: + ref: master + persist-credentials: false + + - name: Get failed cron names and Build IDs + id: cron + run: './.github/actions/${{ github.workflow }}/${{ github.job }}.sh' + + - if: steps.cron.outputs.failures > 0 + shell: bash + # Must be inline, since context expressions are used. + # Ref: https://docs.github.com/en/free-pro-team@latest/actions/reference/context-and-expression-syntax-for-github-actions + run: | + set -eo pipefail + ( + echo "Detected one or more Cirrus-CI cron-triggered jobs have failed recently:" + echo "" + + while read -r NAME BID; do + echo "Cron build '$NAME' Failed: https://cirrus-ci.com/build/$BID" + done < "$NAME_ID_FILEPATH" + + echo "" + echo "# Source: ${{ github.workflow }} workflow on ${{ github.repository }}." + # Separate content from sendgrid.com automatic footer. + echo "" + ) > ./artifacts/email_body.txt + + - if: steps.cron.outputs.failures > 0 + id: mailto + run: printf "::set-output name=csv::%s\n" $(cat "$FAILMAILCSV") + + - if: steps.mailto.outputs.csv != '' + name: Send failure notification e-mail + # Ref: https://github.com/dawidd6/action-send-mail + uses: dawidd6/action-send-mail@v2.2.2 + with: + server_address: ${{secrets.ACTION_MAIL_SERVER}} + server_port: 465 + username: ${{secrets.ACTION_MAIL_USERNAME}} + password: ${{secrets.ACTION_MAIL_PASSWORD}} + subject: Cirrus-CI cron build failures on ${{github.repository}} + to: ${{steps.mailto.outputs.csv}} + from: ${{secrets.ACTION_MAIL_SENDER}} + body: file://./artifacts/email_body.txt + + - if: always() + uses: actions/upload-artifact@v2 + with: + name: ${{ github.job }}_artifacts + path: artifacts/* |