diff options
Diffstat (limited to 'contrib/cirrus')
-rw-r--r-- | contrib/cirrus/README.md | 75 | ||||
-rwxr-xr-x | contrib/cirrus/build_vm_images.sh | 4 | ||||
-rw-r--r-- | contrib/cirrus/lib.sh | 39 | ||||
-rwxr-xr-x | contrib/cirrus/optional_system_test.sh | 24 | ||||
-rw-r--r-- | contrib/cirrus/packer/fedora_setup.sh | 3 | ||||
-rw-r--r-- | contrib/cirrus/packer/ubuntu_setup.sh | 7 |
6 files changed, 134 insertions, 18 deletions
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md new file mode 100644 index 000000000..0d315c4f5 --- /dev/null +++ b/contrib/cirrus/README.md @@ -0,0 +1,75 @@ +![PODMAN logo](../../logo/podman-logo-source.svg) + +# Cirrus-CI + +Similar to other integrated github CI/CD services, Cirrus utilizes a simple +YAML-based configuration/description file: ``.cirrus.yml``. Ref: https://cirrus-ci.org/ + +## Workflow + +All tasks execute in parallel, unless there are conditions or dependencies +which alter this behavior. Within each task, each script executes in sequence, +so long as any previous script exited successfully. The overall state of each +task (pass or fail) is set based on the exit status of the last script to execute. + +### ``full_vm_testing`` Task + +1. Unconditionally, spin up one VM per ``matrix: image_name`` item defined + in ``.cirrus.yml``. Once accessible, ``ssh`` into each VM and run the following + scripts. + +2. ``setup_environment.sh``: Configure root's ``.bash_profile`` + for all subsequent scripts (each run in a new shell). Any + distribution-specific environment variables are also defined + here. For example, setting tags/flags to use compiling. + +3. ``verify_source.sh``: Perform per-distribution source + verification, lint-checking, etc. This acts as a minimal + gate, blocking extended use of VMs when a PR's code or commits + would otherwise not be accepted. Should run for less than a minute. + +4. ``unit_test.sh``: Execute unit-testing, as defined by the ``Makefile``. + This should execute within 10-minutes, but often much faster. + +5. ``integration_test.sh``: Execute integration-testing. This is + much more involved, and relies on access to external + resources like container images and code from other repositories. + Total execution time is capped at 2-hours (includes all the above) + but this script normally completes in less than an hour. + +### ``build_vm_images`` Task + +1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), run another + round of the ``full_vm_testing`` task (above). + +2. After confirming the tests all pass post-merge, spin up a special VM + capable of communicating with the GCE API. Once accessible, ``ssh`` into + the special VM and run the following scripts. + +3. ``setup_environment.sh``: Configure root's ``.bash_profile`` + for all subsequent scripts (each run in a new shell). Any + distribution-specific environment variables are also defined + here. For example, setting tags/flags to use compiling. + +4. ``build_vm_images.sh``: Examine the merged PR's description on github. + If it contains the magic string ``***CIRRUS: REBUILD IMAGES***``, then + continue. Otherwise display a message, take no further action, and + exit successfully. This prevents production of new VM images unless + they are called for, thereby saving the cost of needlessly storing them. + +5. If the magic string was found, utilize [the packer tool](http://packer.io/docs/) + to produce new VM images. Create a new VM from each base-image, connect + to them with ``ssh``, and perform these steps as defined by the + ``libpod_images.json`` file. + + 1. Copy the current state of the repository into ``/tmp/libpod``. + 2. Execute distribution-specific scripts to prepare the image for + use by the ``full_vm_testing`` task (above). + 3. If successful, shut down each VM and create a new GCE Image + named after the base image and the commit sha of the merge. + +***Note:*** The ``.cirrus.yml`` file must be manually updated with the new +images names, then the change sent in via a secondary pull-request. This +ensures that all the ``full_vm_testing`` tasks can pass with the new images, +before subjecting all future PRs to them. A workflow to automate this +process is described in comments at the end of the ``.cirrus.yml`` file. diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index 80c689a6c..ffbb2d5d5 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -22,9 +22,7 @@ SCRIPT_BASE $SCRIPT_BASE PACKER_BASE $PACKER_BASE " -# TODO: Skip building images if $CIRRUS_BRANCH =~ "master" and -# commit message of $CIRRUS_CHANGE_IN_REPO contains a magic word -# produced by 'commit_and_create_upstream_pr.sh' script (see .cirrus.yml) +require_regex '\*\*\*\s*CIRRUS:\s*REBUILD\s*IMAGES\s*\*\*\*' 'Not re-building VM images' show_env_vars diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 1e0052a65..4a3efb8ff 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -106,7 +106,10 @@ ircmsg() { SCRIPT="$GOSRC/$SCRIPT_BASE/podbot.py" NICK="podbot_$CIRRUS_TASK_ID" NICK="${NICK:0:15}" # Any longer will break things + set +e $SCRIPT $NICK $1 + echo "Ignoring exit($?)" + set -e } # Run sudo in directory with GOPATH set @@ -117,6 +120,22 @@ cdsudo() { sudo --preserve-env=GOPATH --non-interactive bash -c "$CMD" } +# Skip a build if $1 does not match in the PR Title/Description with message $2 +require_regex() { + req_env_var " + CIRRUS_CHANGE_MESSAGE $CIRRUS_CHANGE_MESSAGE + 1 $1 + 2 $2 + " + regex="$1" + msg="$2" + if ! echo "$CIRRUS_CHANGE_MESSAGE" | egrep -q "$regex" + then + echo "***** The PR Title/Description did not match the regular expression: $MAGIC_RE" + echo "***** $msg" + exit 0 + fi +} # Helper/wrapper script to only show stderr/stdout on non-zero exit install_ooe() { @@ -142,8 +161,8 @@ EOF install_cni_plugins() { echo "Installing CNI Plugins from commit $CNI_COMMIT" req_env_var " - GOPATH $GOPATH - CNI_COMMIT $CNI_COMMIT + GOPATH $GOPATH + CNI_COMMIT $CNI_COMMIT " DEST="$GOPATH/src/github.com/containernetworking/plugins" rm -rf "$DEST" @@ -160,9 +179,9 @@ install_runc(){ echo "Installing RunC from commit $RUNC_COMMIT" echo "Platform is $OS_RELEASE_ID" req_env_var " - GOPATH $GOPATH - RUNC_COMMIT $RUNC_COMMIT - OS_RELEASE_ID $OS_RELEASE_ID + GOPATH $GOPATH + RUNC_COMMIT $RUNC_COMMIT + OS_RELEASE_ID $OS_RELEASE_ID " if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]]; then echo "Running make install.libseccomp.sudo for ubuntu" @@ -202,8 +221,8 @@ install_buildah() { install_conmon(){ echo "Installing conmon from commit $CRIO_COMMIT" req_env_var " - GOPATH $GOPATH - CRIO_COMMIT $CRIO_COMMIT + GOPATH $GOPATH + CRIO_COMMIT $CRIO_COMMIT " DEST="$GOPATH/src/github.com/kubernetes-sigs/cri-o.git" rm -rf "$DEST" @@ -234,8 +253,8 @@ install_criu(){ install_testing_dependencies() { echo "Installing ginkgo, gomega, and easyjson into \$GOPATH=$GOPATH" req_env_var " - GOPATH $GOPATH - GOSRC $GOSRC + GOPATH $GOPATH + GOSRC $GOSRC " cd "$GOSRC" ooe.sh go get -u github.com/onsi/ginkgo/ginkgo @@ -263,7 +282,7 @@ install_varlink(){ _finalize(){ echo "Removing leftover giblets from cloud-init" cd / - sudo rm -rf /var/lib/cloud + sudo rm -rf /var/lib/cloud/instance? sudo rm -rf /root/.ssh/* sudo rm -rf /home/* } diff --git a/contrib/cirrus/optional_system_test.sh b/contrib/cirrus/optional_system_test.sh new file mode 100755 index 000000000..705dda5ad --- /dev/null +++ b/contrib/cirrus/optional_system_test.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +set -e +source $(dirname $0)/lib.sh + +MAGIC_RE='\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*' +if ! echo "$CIRRUS_CHANGE_MESSAGE" | egrep -q "$MAGIC_RE" +then + echo "Skipping system-testing because PR title or description" + echo "does not match regular expression: $MAGIC_RE" + exit 0 +fi + +req_env_var " +GOSRC $GOSRC +OS_RELEASE_ID $OS_RELEASE_ID +OS_RELEASE_VER $OS_RELEASE_VER +" + +show_env_vars + +set -x +cd "$GOSRC" +make localsystem diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 16b6e4e6b..f9fea04a7 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -21,8 +21,7 @@ install_ooe export GOPATH="$(mktemp -d)" trap "sudo rm -rf $GOPATH" EXIT -# breaks networking on f28/29 in GCE -# ooe.sh sudo dnf update -y +ooe.sh sudo dnf update -y ooe.sh sudo dnf install -y \ atomic-registries \ diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index ff20944dc..4cf1f335b 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -21,9 +21,10 @@ install_ooe export GOPATH="$(mktemp -d)" trap "sudo rm -rf $GOPATH" EXIT -ooe.sh sudo apt-get -qq update -ooe.sh sudo apt-get -qq update # sometimes it needs to get it twice :S -ooe.sh sudo apt-get -qq upgrade +# Try twice as workaround for minor networking problems +echo "Updating system and installing package dependencies" +ooe.sh sudo apt-get -qq update || sudo apt-get -qq update +ooe.sh sudo apt-get -qq upgrade || sudo apt-get -qq upgrade ooe.sh sudo apt-get -qq install --no-install-recommends \ apparmor \ autoconf \ |