diff options
Diffstat (limited to 'contrib/cirrus')
| -rw-r--r-- | contrib/cirrus/README.md | 82 | ||||
| -rwxr-xr-x | contrib/cirrus/build_vm_images.sh | 4 | ||||
| -rw-r--r-- | contrib/cirrus/lib.sh | 46 | ||||
| -rw-r--r-- | contrib/cirrus/packer/fedora_setup.sh | 3 | ||||
| -rw-r--r-- | contrib/cirrus/packer/ubuntu_setup.sh | 11 | ||||
| -rwxr-xr-x | contrib/cirrus/setup_environment.sh | 2 | ||||
| -rwxr-xr-x | contrib/cirrus/system_test.sh | 33 |
7 files changed, 152 insertions, 29 deletions
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md new file mode 100644 index 000000000..fa233a2cb --- /dev/null +++ b/contrib/cirrus/README.md @@ -0,0 +1,82 @@ + + +# Cirrus-CI + +Similar to other integrated github CI/CD services, Cirrus utilizes a simple +YAML-based configuration/description file: ``.cirrus.yml``. Ref: https://cirrus-ci.org/ + +## Workflow + +All tasks execute in parallel, unless there are conditions or dependencies +which alter this behavior. Within each task, each script executes in sequence, +so long as any previous script exited successfully. The overall state of each +task (pass or fail) is set based on the exit status of the last script to execute. + +### ``full_vm_testing`` Task + +1. Unconditionally, spin up one VM per ``matrix: image_name`` item defined + in ``.cirrus.yml``. Once accessible, ``ssh`` into each VM and run the following + scripts. + +2. ``setup_environment.sh``: Configure root's ``.bash_profile`` + for all subsequent scripts (each run in a new shell). Any + distribution-specific environment variables are also defined + here. For example, setting tags/flags to use compiling. + +3. ``verify_source.sh``: Perform per-distribution source + verification, lint-checking, etc. This acts as a minimal + gate, blocking extended use of VMs when a PR's code or commits + would otherwise not be accepted. Should run for less than a minute. + +4. ``unit_test.sh``: Execute unit-testing, as defined by the ``Makefile``. + This should execute within 10-minutes, but often much faster. + +5. ``integration_test.sh``: Execute integration-testing. This is + much more involved, and relies on access to external + resources like container images and code from other repositories. + Total execution time is capped at 2-hours (includes all the above) + but this script normally completes in less than an hour. + +### ``optional_system_testing`` Task + +1. Optionally executes in parallel with ``full_vm_testing``. Requires + **prior** to job-start, the magic string ``***CIRRUS: SYSTEM TEST***`` + is found in the pull-request *description*. The *description* is the first + text-box under the main *summary* line in the github WebUI. + +2. ``setup_environment.sh``: Same as for other tasks. + +3. ``system_test.sh``: Build both dependencies and libpod, install them, + then execute `make localsystem` from the repository root. + +### ``build_vm_images`` Task + +1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), Cirrus + checks the last commit message. If it contains the magic string + ``***CIRRUS: REBUILD IMAGES***``, then this task continues. + +2. Execute run another round of the ``full_vm_testing`` task (above). + After the tests pass (post-merge), spin up a special VM + (from the `image-builder-image`) capable of communicating with the + GCE API. Once accessible, ``ssh`` into the VM and run the following scripts. + +3. ``setup_environment.sh``: Same as for other tasks. + +4. ``build_vm_images.sh``: Utilize [the packer tool](http://packer.io/docs/) + to produce new VM images. Create a new VM from each base-image, connect + to them with ``ssh``, and perform the steps as defined by the + ``$PACKER_BASE/libpod_images.json`` file: + + 1. On a base-image VM, as root, copy the current state of the repository + into ``/tmp/libpod``. + 2. Execute distribution-specific scripts to prepare the image for + use by the ``full_vm_testing`` task (above). These scripts all + end with the suffix `_setup.sh` within the `$PACKER_BASE` directory. + 3. If successful, shut down each VM and create a new GCE Image + named after the base image and the commit sha of the merge. + +***Note:*** The ``.cirrus.yml`` file must be manually updated with the new +images names, then the change sent in via a secondary pull-request. This +ensures that all the ``full_vm_testing`` tasks can pass with the new images, +before subjecting all future PRs to them. A workflow to automate this +process is described in comments at the end of the ``.cirrus.yml`` file. diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index 80c689a6c..c8ff55445 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -22,10 +22,6 @@ SCRIPT_BASE $SCRIPT_BASE PACKER_BASE $PACKER_BASE " -# TODO: Skip building images if $CIRRUS_BRANCH =~ "master" and -# commit message of $CIRRUS_CHANGE_IN_REPO contains a magic word -# produced by 'commit_and_create_upstream_pr.sh' script (see .cirrus.yml) - show_env_vars # Everything here is running on the 'image-builder-image' GCE image diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 1e0052a65..04314e5fe 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -106,7 +106,10 @@ ircmsg() { SCRIPT="$GOSRC/$SCRIPT_BASE/podbot.py" NICK="podbot_$CIRRUS_TASK_ID" NICK="${NICK:0:15}" # Any longer will break things + set +e $SCRIPT $NICK $1 + echo "Ignoring exit($?)" + set -e } # Run sudo in directory with GOPATH set @@ -117,7 +120,6 @@ cdsudo() { sudo --preserve-env=GOPATH --non-interactive bash -c "$CMD" } - # Helper/wrapper script to only show stderr/stdout on non-zero exit install_ooe() { req_env_var "SCRIPT_BASE $SCRIPT_BASE" @@ -142,8 +144,8 @@ EOF install_cni_plugins() { echo "Installing CNI Plugins from commit $CNI_COMMIT" req_env_var " - GOPATH $GOPATH - CNI_COMMIT $CNI_COMMIT + GOPATH $GOPATH + CNI_COMMIT $CNI_COMMIT " DEST="$GOPATH/src/github.com/containernetworking/plugins" rm -rf "$DEST" @@ -155,14 +157,27 @@ install_cni_plugins() { sudo cp bin/* /usr/libexec/cni } +install_runc_from_git(){ + wd=$(pwd) + DEST="$GOPATH/src/github.com/opencontainers/runc" + rm -rf "$DEST" + ooe.sh git clone https://github.com/opencontainers/runc.git "$DEST" + cd "$DEST" + ooe.sh git fetch origin --tags + ooe.sh git checkout -q "$RUNC_COMMIT" + ooe.sh make static BUILDTAGS="seccomp selinux" + sudo install -m 755 runc /usr/bin/runc + cd $wd +} + install_runc(){ OS_RELEASE_ID=$(os_release_id) echo "Installing RunC from commit $RUNC_COMMIT" echo "Platform is $OS_RELEASE_ID" req_env_var " - GOPATH $GOPATH - RUNC_COMMIT $RUNC_COMMIT - OS_RELEASE_ID $OS_RELEASE_ID + GOPATH $GOPATH + RUNC_COMMIT $RUNC_COMMIT + OS_RELEASE_ID $OS_RELEASE_ID " if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]]; then echo "Running make install.libseccomp.sudo for ubuntu" @@ -177,14 +192,7 @@ install_runc(){ cd "$GOPATH/src/github.com/containers/libpod" ooe.sh sudo make install.libseccomp.sudo fi - DEST="$GOPATH/src/github.com/opencontainers/runc" - rm -rf "$DEST" - ooe.sh git clone https://github.com/opencontainers/runc.git "$DEST" - cd "$DEST" - ooe.sh git fetch origin --tags - ooe.sh git checkout -q "$RUNC_COMMIT" - ooe.sh make static BUILDTAGS="seccomp selinux" - sudo install -m 755 runc /usr/bin/runc + install_runc_from_git } install_buildah() { @@ -202,8 +210,8 @@ install_buildah() { install_conmon(){ echo "Installing conmon from commit $CRIO_COMMIT" req_env_var " - GOPATH $GOPATH - CRIO_COMMIT $CRIO_COMMIT + GOPATH $GOPATH + CRIO_COMMIT $CRIO_COMMIT " DEST="$GOPATH/src/github.com/kubernetes-sigs/cri-o.git" rm -rf "$DEST" @@ -234,8 +242,8 @@ install_criu(){ install_testing_dependencies() { echo "Installing ginkgo, gomega, and easyjson into \$GOPATH=$GOPATH" req_env_var " - GOPATH $GOPATH - GOSRC $GOSRC + GOPATH $GOPATH + GOSRC $GOSRC " cd "$GOSRC" ooe.sh go get -u github.com/onsi/ginkgo/ginkgo @@ -263,7 +271,7 @@ install_varlink(){ _finalize(){ echo "Removing leftover giblets from cloud-init" cd / - sudo rm -rf /var/lib/cloud + sudo rm -rf /var/lib/cloud/instance? sudo rm -rf /root/.ssh/* sudo rm -rf /home/* } diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 16b6e4e6b..f9fea04a7 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -21,8 +21,7 @@ install_ooe export GOPATH="$(mktemp -d)" trap "sudo rm -rf $GOPATH" EXIT -# breaks networking on f28/29 in GCE -# ooe.sh sudo dnf update -y +ooe.sh sudo dnf update -y ooe.sh sudo dnf install -y \ atomic-registries \ diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index ff20944dc..ef209a4a4 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -21,10 +21,13 @@ install_ooe export GOPATH="$(mktemp -d)" trap "sudo rm -rf $GOPATH" EXIT -ooe.sh sudo apt-get -qq update -ooe.sh sudo apt-get -qq update # sometimes it needs to get it twice :S -ooe.sh sudo apt-get -qq upgrade -ooe.sh sudo apt-get -qq install --no-install-recommends \ +export DEBIAN_FRONTEND=noninteractive + +# Try twice as workaround for minor networking problems +echo "Updating system and installing package dependencies" +ooe.sh sudo -E apt-get -qq update || sudo -E apt-get -qq update +ooe.sh sudo -E apt-get -qq upgrade || sudo -E apt-get -qq upgrade +ooe.sh sudo -E apt-get -qq install --no-install-recommends \ apparmor \ autoconf \ automake \ diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 167db127f..2563b5f43 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -53,6 +53,8 @@ then # Some setup needs to vary between distros case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in ubuntu-18) + # Always install runc on Ubuntu + install_runc_from_git envstr='export BUILDTAGS="seccomp $($GOSRC/hack/btrfs_tag.sh) $($GOSRC/hack/btrfs_installed_tag.sh) $($GOSRC/hack/ostree_tag.sh) varlink exclude_graphdriver_devicemapper"' ;; fedora-28) ;& # Continue to the next item diff --git a/contrib/cirrus/system_test.sh b/contrib/cirrus/system_test.sh new file mode 100755 index 000000000..7c727d336 --- /dev/null +++ b/contrib/cirrus/system_test.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +set -e +source $(dirname $0)/lib.sh + +req_env_var " +GOSRC $GOSRC +OS_RELEASE_ID $OS_RELEASE_ID +OS_RELEASE_VER $OS_RELEASE_VER +" + +show_env_vars + +set -x +cd "$GOSRC" + +case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in + ubuntu-18) + make install.tools "BUILDTAGS=$BUILDTAGS" + make "BUILDTAGS=$BUILDTAGS" + make test-binaries "BUILDTAGS=$BUILDTAGS" + ;; + fedora-28) ;& + centos-7) ;& + rhel-7) + make install.tools + make + make test-binaries + ;; + *) bad_os_id_ver ;; +esac + +make localsystem |