Merge pull request #1603 from cevich/fix_cirrus_image_build

Fix Cirrus/Packer VM image building

4 files changed, 105 insertions, 15 deletions

diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md
new file mode 100644
index 000000000..0d315c4f5
--- /dev/null
+++ b/contrib/cirrus/README.md
@@ -0,0 +1,75 @@
+![PODMAN logo](../../logo/podman-logo-source.svg)
+
+# Cirrus-CI
+
+Similar to other integrated github CI/CD services, Cirrus utilizes a simple
+YAML-based configuration/description file: ``.cirrus.yml``.  Ref: https://cirrus-ci.org/
+
+## Workflow
+
+All tasks execute in parallel, unless there are conditions or dependencies
+which alter this behavior.  Within each task, each script executes in sequence,
+so long as any previous script exited successfully.  The overall state of each
+task (pass or fail) is set based on the exit status of the last script to execute.
+
+### ``full_vm_testing`` Task
+
+1. Unconditionally, spin up one VM per ``matrix: image_name`` item defined
+   in ``.cirrus.yml``.  Once accessible, ``ssh`` into each VM and run the following
+   scripts.
+
+2. ``setup_environment.sh``: Configure root's ``.bash_profile``
+   for all subsequent scripts (each run in a new shell).  Any
+   distribution-specific environment variables are also defined
+   here.  For example, setting tags/flags to use compiling.
+
+3. ``verify_source.sh``: Perform per-distribution source
+   verification, lint-checking, etc.  This acts as a minimal
+   gate, blocking extended use of VMs when a PR's code or commits
+   would otherwise not be accepted.  Should run for less than a minute.
+
+4. ``unit_test.sh``: Execute unit-testing, as defined by the ``Makefile``.
+   This should execute within 10-minutes, but often much faster.
+
+5. ``integration_test.sh``: Execute integration-testing.  This is
+   much more involved, and relies on access to external
+   resources like container images and code from other repositories.
+   Total execution time is capped at 2-hours (includes all the above)
+   but this script normally completes in less than an hour.
+
+### ``build_vm_images`` Task
+
+1. When a PR is merged (``$CIRRUS_BRANCH`` == ``master``), run another
+   round of the ``full_vm_testing`` task (above).
+
+2. After confirming the tests all pass post-merge, spin up a special VM
+   capable of communicating with the GCE API.  Once accessible, ``ssh`` into
+   the special VM and run the following scripts.
+
+3. ``setup_environment.sh``: Configure root's ``.bash_profile``
+   for all subsequent scripts (each run in a new shell).  Any
+   distribution-specific environment variables are also defined
+   here.  For example, setting tags/flags to use compiling.
+
+4. ``build_vm_images.sh``: Examine the merged PR's description on github.
+   If it contains the magic string ``***CIRRUS: REBUILD IMAGES***``, then
+   continue.  Otherwise display a message, take no further action, and
+   exit successfully.  This prevents production of new VM images unless
+   they are called for, thereby saving the cost of needlessly storing them.
+
+5. If the magic string was found, utilize [the packer tool](http://packer.io/docs/)
+   to produce new VM images.  Create a new VM from each base-image, connect
+   to them with ``ssh``, and perform these steps as defined by the
+   ``libpod_images.json`` file.
+
+    1. Copy the current state of the repository into ``/tmp/libpod``.
+    2. Execute distribution-specific scripts to prepare the image for
+       use by the ``full_vm_testing`` task (above).
+    3. If successful, shut down each VM and create a new GCE Image
+       named after the base image and the commit sha of the merge.
+
+***Note:*** The ``.cirrus.yml`` file must be manually updated with the new
+images names, then the change sent in via a secondary pull-request.  This
+ensures that all the ``full_vm_testing`` tasks can pass with the new images,
+before subjecting all future PRs to them.  A workflow to automate this
+process is described in comments at the end of the ``.cirrus.yml`` file.
diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh
index 80c689a6c..ffbb2d5d5 100755
--- a/contrib/cirrus/build_vm_images.sh
+++ b/contrib/cirrus/build_vm_images.sh
@@ -22,9 +22,7 @@ SCRIPT_BASE $SCRIPT_BASE
 PACKER_BASE $PACKER_BASE
 "
 
-# TODO: Skip building images if $CIRRUS_BRANCH =~ "master" and
-# commit message of $CIRRUS_CHANGE_IN_REPO contains a magic word
-# produced by 'commit_and_create_upstream_pr.sh' script (see .cirrus.yml)
+require_regex '\*\*\*\s*CIRRUS:\s*REBUILD\s*IMAGES\s*\*\*\*' 'Not re-building VM images'
 
 show_env_vars
 
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 1e0052a65..12d24042f 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -117,6 +117,22 @@ cdsudo() {
     sudo --preserve-env=GOPATH --non-interactive bash -c "$CMD"
 }
 
+# Skip a build if $1 does not match in the PR Title/Description with message $2
+require_regex() {
+    req_env_var "
+        CIRRUS_CHANGE_MESSAGE $CIRRUS_CHANGE_MESSAGE
+        1 $1
+        2 $2
+    "
+    regex="$1"
+    msg="$2"
+    if ! echo "$CIRRUS_CHANGE_MESSAGE" | egrep -q "$regex"
+    then
+        echo "***** The PR Title/Description did not match the regular expression: $MAGIC_RE"
+        echo "***** $msg"
+        exit 0
+    fi
+}
 
 # Helper/wrapper script to only show stderr/stdout on non-zero exit
 install_ooe() {
@@ -142,8 +158,8 @@ EOF
 install_cni_plugins() {
     echo "Installing CNI Plugins from commit $CNI_COMMIT"
     req_env_var "
-    GOPATH $GOPATH
-    CNI_COMMIT $CNI_COMMIT
+        GOPATH $GOPATH
+        CNI_COMMIT $CNI_COMMIT
     "
     DEST="$GOPATH/src/github.com/containernetworking/plugins"
     rm -rf "$DEST"
@@ -160,9 +176,9 @@ install_runc(){
     echo "Installing RunC from commit $RUNC_COMMIT"
     echo "Platform is $OS_RELEASE_ID"
     req_env_var "
-    GOPATH $GOPATH
-    RUNC_COMMIT $RUNC_COMMIT
-    OS_RELEASE_ID $OS_RELEASE_ID
+        GOPATH $GOPATH
+        RUNC_COMMIT $RUNC_COMMIT
+        OS_RELEASE_ID $OS_RELEASE_ID
     "
     if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]]; then
         echo "Running make install.libseccomp.sudo for ubuntu"
@@ -202,8 +218,8 @@ install_buildah() {
 install_conmon(){
     echo "Installing conmon from commit $CRIO_COMMIT"
     req_env_var "
-    GOPATH $GOPATH
-    CRIO_COMMIT $CRIO_COMMIT
+        GOPATH $GOPATH
+        CRIO_COMMIT $CRIO_COMMIT
     "
     DEST="$GOPATH/src/github.com/kubernetes-sigs/cri-o.git"
     rm -rf "$DEST"
@@ -234,8 +250,8 @@ install_criu(){
 install_testing_dependencies() {
     echo "Installing ginkgo, gomega, and easyjson into \$GOPATH=$GOPATH"
     req_env_var "
-    GOPATH $GOPATH
-    GOSRC $GOSRC
+        GOPATH $GOPATH
+        GOSRC $GOSRC
     "
     cd "$GOSRC"
     ooe.sh go get -u github.com/onsi/ginkgo/ginkgo
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index ff20944dc..4cf1f335b 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -21,9 +21,10 @@ install_ooe
 export GOPATH="$(mktemp -d)"
 trap "sudo rm -rf $GOPATH" EXIT
 
-ooe.sh sudo apt-get -qq update
-ooe.sh sudo apt-get -qq update  # sometimes it needs to get it twice :S
-ooe.sh sudo apt-get -qq upgrade
+# Try twice as workaround for minor networking problems
+echo "Updating system and installing package dependencies"
+ooe.sh sudo apt-get -qq update || sudo apt-get -qq update
+ooe.sh sudo apt-get -qq upgrade || sudo apt-get -qq upgrade
 ooe.sh sudo apt-get -qq install --no-install-recommends \
     apparmor \
     autoconf \