diff options
| -rw-r--r-- | .cirrus.yml | 3 | ||||
| -rw-r--r-- | contrib/cirrus/README.md | 75 | ||||
| -rwxr-xr-x | contrib/cirrus/build_vm_images.sh | 4 | ||||
| -rw-r--r-- | contrib/cirrus/lib.sh | 36 | ||||
| -rw-r--r-- | contrib/cirrus/packer/fedora_setup.sh | 3 | ||||
| -rw-r--r-- | contrib/cirrus/packer/ubuntu_setup.sh | 7 | ||||
| -rw-r--r-- | contrib/python/pypodman/pypodman/lib/actions/history_action.py | 2 | ||||
| -rw-r--r-- | contrib/python/pypodman/pypodman/lib/actions/images_action.py | 2 |
8 files changed, 112 insertions, 20 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index ae660394b..c5d35141e 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -47,6 +47,7 @@ full_vm_testing_task: #image_name: "rhel-server-ec2-7-5-165-1-libpod-fce09afe" #image_name: "centos-7-v20180911-libpod-fce09afe" #image_name: "fedora-cloud-base-28-1-1-7-libpod-fce09afe" + timeout_in: 120m # Every *_script runs in sequence, for each task. The name prefix is for @@ -99,6 +100,8 @@ build_vm_images_task: # Version of packer to use PACKER_VER: "1.3.1" + # VMs created by packer are not cleaned up by cirrus + auto_cancellation: $CI != "true" gce_instance: image_name: "image-builder-image" # Simply CentOS 7 + packer dependencies diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md new file mode 100644 index 000000000..0d315c4f5 --- /dev/null +++ b/contrib/cirrus/README.md @@ -0,0 +1,75 @@ + + +# Cirrus-CI + +Similar to other integrated github CI/CD services, Cirrus utilizes a simple +YAML-based configuration/description file: ``.cirrus.yml``. Ref: https://cirrus-ci.org/ + +## Workflow + +All tasks execute in parallel, unless there are conditions or dependencies +which alter this behavior. Within each task, each script executes in sequence, +so long as any previous script exited successfully. The overall state of each +task (pass or fail) is set based on the exit status of the last script to execute. + +### ``full_vm_testing`` Task + +1. Unconditionally, spin up one VM per ``matrix: image_name`` item defined + in ``.cirrus.yml``. Once accessible, ``ssh`` into each VM and run the following + scripts. + +2. ``setup_environment.sh``: Configure root's ``.bash_profile`` + for all subsequent scripts (each run in a new shell). Any + distribution-specific environment variables are also defined + here. For example, setting tags/flags to use compiling. + +3. ``verify_source.sh``: Perform per-distribution source + verification, lint-checking, etc. This acts as a minimal + gate, blocking extended use of VMs when a PR's code or commits + would otherwise not be accepted. Should run for less than a minute. + +4. ``unit_test.sh``: Execute unit-testing, as defined by the ``Makefile``. + This should execute within 10-minutes, but often much faster. + +5. ``integration_test.sh``: Execute integration-testing. This is + much more involved, and relies on access to external + resources like container images and code from other repositories. + Total execution time is capped at 2-hours (includes all the above) + but this script normally completes in less than an hour. + +### ``build_vm_images`` Task + +1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), run another + round of the ``full_vm_testing`` task (above). + +2. After confirming the tests all pass post-merge, spin up a special VM + capable of communicating with the GCE API. Once accessible, ``ssh`` into + the special VM and run the following scripts. + +3. ``setup_environment.sh``: Configure root's ``.bash_profile`` + for all subsequent scripts (each run in a new shell). Any + distribution-specific environment variables are also defined + here. For example, setting tags/flags to use compiling. + +4. ``build_vm_images.sh``: Examine the merged PR's description on github. + If it contains the magic string ``***CIRRUS: REBUILD IMAGES***``, then + continue. Otherwise display a message, take no further action, and + exit successfully. This prevents production of new VM images unless + they are called for, thereby saving the cost of needlessly storing them. + +5. If the magic string was found, utilize [the packer tool](http://packer.io/docs/) + to produce new VM images. Create a new VM from each base-image, connect + to them with ``ssh``, and perform these steps as defined by the + ``libpod_images.json`` file. + + 1. Copy the current state of the repository into ``/tmp/libpod``. + 2. Execute distribution-specific scripts to prepare the image for + use by the ``full_vm_testing`` task (above). + 3. If successful, shut down each VM and create a new GCE Image + named after the base image and the commit sha of the merge. + +***Note:*** The ``.cirrus.yml`` file must be manually updated with the new +images names, then the change sent in via a secondary pull-request. This +ensures that all the ``full_vm_testing`` tasks can pass with the new images, +before subjecting all future PRs to them. A workflow to automate this +process is described in comments at the end of the ``.cirrus.yml`` file. diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index 80c689a6c..ffbb2d5d5 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -22,9 +22,7 @@ SCRIPT_BASE $SCRIPT_BASE PACKER_BASE $PACKER_BASE " -# TODO: Skip building images if $CIRRUS_BRANCH =~ "master" and -# commit message of $CIRRUS_CHANGE_IN_REPO contains a magic word -# produced by 'commit_and_create_upstream_pr.sh' script (see .cirrus.yml) +require_regex '\*\*\*\s*CIRRUS:\s*REBUILD\s*IMAGES\s*\*\*\*' 'Not re-building VM images' show_env_vars diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 1e0052a65..2fa91258b 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -117,6 +117,22 @@ cdsudo() { sudo --preserve-env=GOPATH --non-interactive bash -c "$CMD" } +# Skip a build if $1 does not match in the PR Title/Description with message $2 +require_regex() { + req_env_var " + CIRRUS_CHANGE_MESSAGE $CIRRUS_CHANGE_MESSAGE + 1 $1 + 2 $2 + " + regex="$1" + msg="$2" + if ! echo "$CIRRUS_CHANGE_MESSAGE" | egrep -q "$regex" + then + echo "***** The PR Title/Description did not match the regular expression: $MAGIC_RE" + echo "***** $msg" + exit 0 + fi +} # Helper/wrapper script to only show stderr/stdout on non-zero exit install_ooe() { @@ -142,8 +158,8 @@ EOF install_cni_plugins() { echo "Installing CNI Plugins from commit $CNI_COMMIT" req_env_var " - GOPATH $GOPATH - CNI_COMMIT $CNI_COMMIT + GOPATH $GOPATH + CNI_COMMIT $CNI_COMMIT " DEST="$GOPATH/src/github.com/containernetworking/plugins" rm -rf "$DEST" @@ -160,9 +176,9 @@ install_runc(){ echo "Installing RunC from commit $RUNC_COMMIT" echo "Platform is $OS_RELEASE_ID" req_env_var " - GOPATH $GOPATH - RUNC_COMMIT $RUNC_COMMIT - OS_RELEASE_ID $OS_RELEASE_ID + GOPATH $GOPATH + RUNC_COMMIT $RUNC_COMMIT + OS_RELEASE_ID $OS_RELEASE_ID " if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]]; then echo "Running make install.libseccomp.sudo for ubuntu" @@ -202,8 +218,8 @@ install_buildah() { install_conmon(){ echo "Installing conmon from commit $CRIO_COMMIT" req_env_var " - GOPATH $GOPATH - CRIO_COMMIT $CRIO_COMMIT + GOPATH $GOPATH + CRIO_COMMIT $CRIO_COMMIT " DEST="$GOPATH/src/github.com/kubernetes-sigs/cri-o.git" rm -rf "$DEST" @@ -234,8 +250,8 @@ install_criu(){ install_testing_dependencies() { echo "Installing ginkgo, gomega, and easyjson into \$GOPATH=$GOPATH" req_env_var " - GOPATH $GOPATH - GOSRC $GOSRC + GOPATH $GOPATH + GOSRC $GOSRC " cd "$GOSRC" ooe.sh go get -u github.com/onsi/ginkgo/ginkgo @@ -263,7 +279,7 @@ install_varlink(){ _finalize(){ echo "Removing leftover giblets from cloud-init" cd / - sudo rm -rf /var/lib/cloud + sudo rm -rf /var/lib/cloud/instance? sudo rm -rf /root/.ssh/* sudo rm -rf /home/* } diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 16b6e4e6b..f9fea04a7 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -21,8 +21,7 @@ install_ooe export GOPATH="$(mktemp -d)" trap "sudo rm -rf $GOPATH" EXIT -# breaks networking on f28/29 in GCE -# ooe.sh sudo dnf update -y +ooe.sh sudo dnf update -y ooe.sh sudo dnf install -y \ atomic-registries \ diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index ff20944dc..4cf1f335b 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -21,9 +21,10 @@ install_ooe export GOPATH="$(mktemp -d)" trap "sudo rm -rf $GOPATH" EXIT -ooe.sh sudo apt-get -qq update -ooe.sh sudo apt-get -qq update # sometimes it needs to get it twice :S -ooe.sh sudo apt-get -qq upgrade +# Try twice as workaround for minor networking problems +echo "Updating system and installing package dependencies" +ooe.sh sudo apt-get -qq update || sudo apt-get -qq update +ooe.sh sudo apt-get -qq upgrade || sudo apt-get -qq upgrade ooe.sh sudo apt-get -qq install --no-install-recommends \ apparmor \ autoconf \ diff --git a/contrib/python/pypodman/pypodman/lib/actions/history_action.py b/contrib/python/pypodman/pypodman/lib/actions/history_action.py index 3e3f539fc..f9aaa54f6 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/history_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/history_action.py @@ -60,7 +60,7 @@ class History(AbstractActionBase): if self._args.human: fields.update({ 'size': - humanize.naturalsize(details.size, binary=True), + humanize.naturalsize(details.size), 'created': humanize.naturaldate( podman.datetime_parse(details.created)), diff --git a/contrib/python/pypodman/pypodman/lib/actions/images_action.py b/contrib/python/pypodman/pypodman/lib/actions/images_action.py index d28e32db9..b8f5ccc78 100644 --- a/contrib/python/pypodman/pypodman/lib/actions/images_action.py +++ b/contrib/python/pypodman/pypodman/lib/actions/images_action.py @@ -65,7 +65,7 @@ class Images(AbstractActionBase): 'created': humanize.naturaldate(podman.datetime_parse(image.created)), 'size': - humanize.naturalsize(int(image.size), binary=True), + humanize.naturalsize(int(image.size)), 'repoDigests': ' '.join(image.repoDigests), }) |