From 0e769210487056f20b7a4c47d195cc939ad5c7db Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Thu, 14 Mar 2019 11:57:46 -0400 Subject: Cirrus: Notify on IRC if post-merge testing fails Until recently it was very difficult to execute any scripts if part of a task failed. A new feature in Cirrus-CI makes this easy. Use it to post a notice on IRC when any task fails. Also: Add quotes around yaml-string values for consistency and syntax-highlighting correctness. Signed-off-by: Chris Evich --- .cirrus.yml | 45 ++++++++++++++++++++------------- contrib/cirrus/lib.sh | 13 ++++++++-- contrib/cirrus/notice_master_failure.sh | 20 +++++++++++++++ 3 files changed, 59 insertions(+), 19 deletions(-) create mode 100755 contrib/cirrus/notice_master_failure.sh diff --git a/.cirrus.yml b/.cirrus.yml index 8e26ce72e..14ba3fc6e 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -140,6 +140,10 @@ gating_task: - '/usr/local/bin/entrypoint.sh clean podman BUILDTAGS="exclude_graphdriver_devicemapper selinux seccomp"' - '/usr/local/bin/entrypoint.sh clean podman-remote-darwin' + on_failure: + master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' + + build_each_commit_task: depends_on: @@ -160,9 +164,12 @@ build_each_commit_task: timeout_in: 30m script: - - $SCRIPT_BASE/setup_environment.sh - - git fetch --depth $CIRRUS_CLONE_DEPTH origin $CIRRUS_BASE_BRANCH - - env GOPATH=/var/tmp/go/ make build-all-new-commits GIT_BASE_BRANCH=origin/$CIRRUS_BASE_BRANCH + - '$SCRIPT_BASE/setup_environment.sh' + - 'git fetch --depth $CIRRUS_CLONE_DEPTH origin $CIRRUS_BASE_BRANCH' + - 'env GOPATH=/var/tmp/go/ make build-all-new-commits GIT_BASE_BRANCH=origin/$CIRRUS_BASE_BRANCH' + + on_failure: + master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' # Update metadata on VM images referenced by this repository state @@ -186,7 +193,7 @@ meta_task: GCPPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f] CIRRUS_CLONE_DEPTH: 1 # source not used - script: /usr/local/bin/entrypoint.sh + script: '/usr/local/bin/entrypoint.sh' # This task does the unit and integration testing for every platform @@ -219,14 +226,12 @@ testing_task: # Every *_script runs in sequence, for each task. The name prefix is for # WebUI reference. The values may be strings... - setup_environment_script: $SCRIPT_BASE/setup_environment.sh + setup_environment_script: '$SCRIPT_BASE/setup_environment.sh' + unit_test_script: '$SCRIPT_BASE/unit_test.sh' + integration_test_script: '$SCRIPT_BASE/integration_test.sh' - # ...or lists of strings - unit_test_script: - - go version - - $SCRIPT_BASE/unit_test.sh - - integration_test_script: $SCRIPT_BASE/integration_test.sh + on_failure: + master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' # This task executes tests as a regular user on a system @@ -252,12 +257,15 @@ rootless_testing_task: timeout_in: 120m - setup_environment_script: $SCRIPT_BASE/setup_environment.sh + setup_environment_script: '$SCRIPT_BASE/setup_environment.sh' rootless_test_script: >- ssh $ROOTLESS_USER@localhost -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no $CIRRUS_WORKING_DIR/$SCRIPT_BASE/rootless_test.sh + on_failure: + master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' + # Because system tests are stored within the repository, it is sometimes # necessary to execute them within a PR to validate changes. @@ -283,8 +291,8 @@ optional_testing_task: timeout_in: 60m - setup_environment_script: $SCRIPT_BASE/setup_environment.sh - system_test_script: $SCRIPT_BASE/system_test.sh + setup_environment_script: '$SCRIPT_BASE/setup_environment.sh' + system_test_script: '$SCRIPT_BASE/system_test.sh' # Build new cache-images for future PR testing, but only after a PR merge. @@ -317,8 +325,8 @@ cache_images_task: scopes: - compute - devstorage.full_control - environment_script: $SCRIPT_BASE/setup_environment.sh - build_vm_images_script: $SCRIPT_BASE/build_vm_images.sh + environment_script: '$SCRIPT_BASE/setup_environment.sh' + build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh' # TODO,Continuous Delivery: Automatically open a libpod PR after using 'sed' to replace # the image_names with the new (just build) images. That will @@ -331,6 +339,9 @@ cache_images_task: # - modify_cirrus_yaml_image_names.sh # - commit_and_create_upstream_pr.sh + on_failure: + master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' + # Post message to IRC if everything passed success_task: @@ -350,4 +361,4 @@ success_task: cpu: 1 memory: 1 - success_script: $SCRIPT_BASE/success.sh + success_script: '$SCRIPT_BASE/success.sh' diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 9419dad05..773f4f484 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -103,6 +103,15 @@ clean_env() { unset -v UNSET_ENV_VARS $UNSET_ENV_VARS || true # don't fail on read-only } +die() { + req_env_var " + 1 $1 + 2 $2 + " + echo "$2" + exit $1 +} + # Return a GCE image-name compatible string representation of distribution name os_release_id() { eval "$(egrep -m 1 '^ID=' /etc/os-release | tr -d \' | tr -d \")" @@ -136,14 +145,14 @@ stub() { ircmsg() { req_env_var " CIRRUS_TASK_ID $CIRRUS_TASK_ID - 1 $1 + @ $@ " # Sometimes setup_environment.sh didn't run SCRIPT="$(dirname $0)/podbot.py" NICK="podbot_$CIRRUS_TASK_ID" NICK="${NICK:0:15}" # Any longer will break things set +e - $SCRIPT $NICK $1 + $SCRIPT $NICK $@ echo "Ignoring exit($?)" set -e } diff --git a/contrib/cirrus/notice_master_failure.sh b/contrib/cirrus/notice_master_failure.sh new file mode 100755 index 000000000..e18460a54 --- /dev/null +++ b/contrib/cirrus/notice_master_failure.sh @@ -0,0 +1,20 @@ +#!/bin/bash + +set -e + +source $(dirname $0)/lib.sh + +# mIRC "escape" codes are the most standard, for a non-standard client-side interpretation. +ETX="$(echo -n -e '\x03')" +RED="${ETX}4" +NOR="$(echo -n -e '\x0f')" + +if [[ "$CIRRUS_BRANCH" =~ "master" ]] +then + BURL="https://cirrus-ci.com/build/$CIRRUS_BUILD_ID" + echo "Monitoring execution of $CIRRUS_TASK_NAME and notifying on failure" + MSG="${RED}[Action Recommended]: ${NOR}Post-merge testing ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on $(os_release_id)-$(os_release_ver): $BURL. Please investigate, and re-run if appropriate." +fi + +# This script assumed to be executed on failure +die 1 "Testing Failed" -- cgit v1.2.3-54-g00ecf