Cirrus: Use Makefile for image-building

The packer tool takes JSON as input for the details of producing VM
images to be used for PR CI-testing.  JSON is not a very human-friendly
format, without support for comments and frequently containing lots of
duplicate data.

Fix this by using a Makefile + simple python one-liner to convert
from a human-friendly YAML format into packer-native JSON.  This allows
use of anchors/aliases to reduce duplication, and allows inline comments
for easier maintainability.  This also allows separating the 'test'
action from the 'build' action, for earlier and better syntax problem
detection.

Lastly, there are some minor ``lib.sh`` and ``integration_test.sh``
updates to support future work, and slightly improve the build and
test environments.

Signed-off-by: Chris Evich <cevich@redhat.com>

10 files changed, 174 insertions, 155 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 625b96fdd..6259a3ed7 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -113,13 +113,12 @@ build_vm_images_task:
     env:
         # CSV of packer builder names to enable (see $PACKER_BASE/libpod_images.json)
         PACKER_BUILDS: "ubuntu-18"
-        # TODO: Make these work (also full_vm_testing_task above)
-        # PACKER_BUILDS: "rhel-7,centos-7,fedora-28,ubuntu-18"
-        CENTOS_BASE_IMAGE: "centos-7-v20180911"
-        RHEL_BASE_IMAGE: "rhel-server-ec2-7-5-165-1"
-        FEDORA_BASE_IMAGE: "fedora-cloud-base-28-1-1-7"
+        # TODO: PACKER_BUILDS: "rhel-7,centos-7,fedora-29,fah-29,ubuntu-18"
         UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20180911"
-        # low-level base VM image name inputs to packer
+        CENTOS_BASE_IMAGE: "centos-7-v20180911"
+        RHEL_BASE_IMAGE: "rhel-server-ec2-7-5-165-1"  # Manually imported into GCE
+        FEDORA_BASE_IMAGE:  "fedora-cloud-base-29-1-2-1541186745"  # see $PACKER_BASE/Makefile
+        FAH_BASE_IMAGE:  "fedora-atomichost-29-20181025-1"  # See $PACKER_BASE/Makefile
 
         # Command to register a RHEL VM
         RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187]
diff --git a/Makefile b/Makefile
index 195d105b7..a6482aaac 100644
--- a/Makefile
+++ b/Makefile
@@ -172,6 +172,7 @@ testunit: libpodimage
 
 localunit: test/goecho/goecho varlink_generate
 	$(GO) test -tags "$(BUILDTAGS)" -cover $(PACKAGES)
+	$(MAKE) -C contrib/cirrus/packer test
 
 ginkgo:
 	ginkgo -v -tags "$(BUILDTAGS)" -cover -flakeAttempts 3 -progress -trace -noColor test/e2e/.
diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh
index c8ff55445..818097e2c 100755
--- a/contrib/cirrus/build_vm_images.sh
+++ b/contrib/cirrus/build_vm_images.sh
@@ -11,9 +11,9 @@ PACKER_BUILDS $PACKER_BUILDS
 CENTOS_BASE_IMAGE $CENTOS_BASE_IMAGE
 UBUNTU_BASE_IMAGE $UBUNTU_BASE_IMAGE
 FEDORA_BASE_IMAGE $FEDORA_BASE_IMAGE
+FAH_BASE_IMAGE $FAH_BASE_IMAGE
 RHEL_BASE_IMAGE $RHEL_BASE_IMAGE
 RHSM_COMMAND $RHSM_COMMAND
-BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX
 SERVICE_ACCOUNT $SERVICE_ACCOUNT
 GCE_SSH_USERNAME $GCE_SSH_USERNAME
 GCP_PROJECT_ID $GCP_PROJECT_ID
@@ -43,13 +43,10 @@ fi
 
 set -x
 
-cd "$GOSRC"
-# N/B: /usr/sbin/packer is a DIFFERENT tool, and will exit 0 given the args below :(
-TEMPLATE="./$PACKER_BASE/libpod_images.json"
-
-$HOME/packer/packer inspect "$TEMPLATE"
-
-#$HOME/packer/packer build -machine-readable "-only=$PACKER_BUILDS" "$TEMPLATE" | tee /tmp/packer_log.csv
-$HOME/packer/packer build "-only=$PACKER_BUILDS" "$TEMPLATE"
-
-# TODO: Report back to PR names of built images
+cd "$GOSRC/$PACKER_BASE"
+make libpod_images \
+    PACKER_BUILDS=$PACKER_BUILDS \
+    PACKER_VER=$PACKER_VER \
+    GOSRC=$GOSRC \
+    SCRIPT_BASE=$SCRIPT_BASE \
+    PACKER_BASE=$PACKER_BASE
diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh
index 226053724..dc43f8e9d 100755
--- a/contrib/cirrus/integration_test.sh
+++ b/contrib/cirrus/integration_test.sh
@@ -22,7 +22,9 @@ case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
     fedora-28) ;&  # Continue to the next item
     centos-7) ;&
     rhel-7)
-        stub 'integration testing not working on $OS_RELEASE_ID'
+        make install PREFIX=/usr ETCDIR=/etc
+        make test-binaries
+        make localintegration
         ;;
     *) bad_os_id_ver ;;
 esac
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 04314e5fe..ff5925d5d 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -4,8 +4,8 @@
 # to be sourced by other scripts, not called directly.
 
 # Under some contexts these values are not set, make sure they are.
-USER="$(whoami)"
-HOME="$(getent passwd $USER | cut -d : -f 6)"
+export USER="$(whoami)"
+export HOME="$(getent passwd $USER | cut -d : -f 6)"
 if ! [[ "$PATH" =~ "/usr/local/bin" ]]
 then
     export PATH="$PATH:/usr/local/bin"
@@ -274,6 +274,10 @@ _finalize(){
     sudo rm -rf /var/lib/cloud/instance?
     sudo rm -rf /root/.ssh/*
     sudo rm -rf /home/*
+    sudo rm -rf /tmp/*
+    sudo rm -rf /tmp/.??*
+    sync
+    sudo fstrim -av
 }
 
 rh_finalize(){
diff --git a/contrib/cirrus/packer/.gitignore b/contrib/cirrus/packer/.gitignore
new file mode 100644
index 000000000..6080c9639
--- /dev/null
+++ b/contrib/cirrus/packer/.gitignore
@@ -0,0 +1,3 @@
+*json
+packer
+packer*zip
diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile
new file mode 100644
index 000000000..d3a34877f
--- /dev/null
+++ b/contrib/cirrus/packer/Makefile
@@ -0,0 +1,56 @@
+
+# N/B: PACKER_BUILDS variable is required.  Should contain CSV of
+# builder name(s) from applicable YAML file,
+# e.g for names see libpod_images.yml
+
+PACKER_VER ?= 1.3.1
+PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_amd64.zip
+
+# Only needed for libpod_base_images target
+TIMESTAMP := $(shell date +%s)
+GOSRC ?= $(shell realpath "./../../../")
+PACKER_BASE ?= contrib/cirrus/packer
+SCRIPT_BASE ?= contrib/cirrus
+
+# For debugging nested-virt, use
+#TTYDEV := $(shell tty)
+TTYDEV := /dev/null
+
+.PHONY: all
+all: libpod_images
+
+%.json: %.yml
+	@python3 -c 'import json,yaml; json.dump( yaml.load(open("$<").read()), open("$@","w"), indent=2);'
+
+${PACKER_DIST_FILENAME}:
+	@curl -L --silent --show-error \
+		-O https://releases.hashicorp.com/packer/${PACKER_VER}/${PACKER_DIST_FILENAME}
+
+packer: ${PACKER_DIST_FILENAME}
+	@curl -L --silent --show-error \
+		https://releases.hashicorp.com/packer/${PACKER_VER}/packer_${PACKER_VER}_SHA256SUMS \
+		| grep 'linux_amd64' > /tmp/packer_sha256sums
+	@sha256sum --check /tmp/packer_sha256sums
+	@unzip -o ${PACKER_DIST_FILENAME}
+	@touch --reference=Makefile ${PACKER_DIST_FILENAME}
+
+.PHONY: test
+test: libpod_base_images.json libpod_images.json packer
+	./packer inspect libpod_base_images.json > /dev/null
+	./packer inspect libpod_images.json > /dev/null
+	@echo "All good"
+
+.PHONY: libpod_images
+libpod_images: libpod_images.json packer
+ifndef PACKER_BUILDS
+	$(error PACKER_BUILDS is undefined, expected builder-names CSV)
+endif
+	./packer build -only=${PACKER_BUILDS} \
+		-var GOSRC=$(GOSRC) \
+		-var PACKER_BASE=$(PACKER_BASE) \
+		-var SCRIPT_BASE=$(SCRIPT_BASE) \
+		libpod_images.json
+	@echo ""
+	@echo "Finished.  The images mentioned above, and in packer-manifest.json"
+	@echo "can be used in .cirrus.yml as values for the 'image_name' keys"
+	@echo ""
diff --git a/contrib/cirrus/packer/libpod_images.json b/contrib/cirrus/packer/libpod_images.json
deleted file mode 100644
index 9dac3e8ea..000000000
--- a/contrib/cirrus/packer/libpod_images.json
+++ /dev/null
@@ -1,130 +0,0 @@
-{
-  "variables": {
-    "FEDORA_CNI_COMMIT": "{{env `FEDORA_CNI_COMMIT`}}",
-    "CNI_COMMIT": "{{env `CNI_COMMIT`}}",
-    "CRIO_COMMIT": "{{env `CRIO_COMMIT`}}",
-    "CRIU_COMMIT": "{{env `CRIU_COMMIT`}}",
-    "RUNC_COMMIT": "{{env `RUNC_COMMIT`}}",
-
-    "CENTOS_BASE_IMAGE": "{{env `CENTOS_BASE_IMAGE`}}" ,
-    "UBUNTU_BASE_IMAGE": "{{env `UBUNTU_BASE_IMAGE`}}",
-    "FEDORA_BASE_IMAGE": "{{env `FEDORA_BASE_IMAGE`}}",
-    "RHEL_BASE_IMAGE": "{{env `RHEL_BASE_IMAGE`}}",
-
-    "GOSRC": "{{env `GOSRC`}}",
-    "PACKER_BASE": "{{env `PACKER_BASE`}}",
-    "SCRIPT_BASE": "{{env `SCRIPT_BASE`}}",
-
-    "SERVICE_ACCOUNT": "{{env `SERVICE_ACCOUNT`}}",
-    "GCP_PROJECT_ID": "{{env `GCP_PROJECT_ID`}}",
-    "BUILT_IMAGE_SUFFIX": "{{env `BUILT_IMAGE_SUFFIX`}}",
-    "GCE_SSH_USERNAME": "{{env `GCE_SSH_USERNAME`}}",
-    "RHSM_COMMAND": "{{env `RHSM_COMMAND`}}"
-  },
-  "sensitive-variables": [
-    "GCP_PROJECT_ID", "SERVICE_ACCOUNT", "GCE_SSH_USERNAME", "RHSM_COMMAND"
-  ],
-  "builders": [
-    {
-      "name": "rhel-7",
-      "type": "googlecompute",
-      "project_id": "{{user `GCP_PROJECT_ID`}}",
-      "zone": "us-central1-a",
-      "source_image": "{{user `RHEL_BASE_IMAGE`}}",
-      "image_name": "{{user `RHEL_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
-      "image_family": "{{user `RHEL_BASE_IMAGE`}}-libpod",
-      "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
-      "communicator": "ssh",
-      "ssh_username": "ec2-user",
-      "ssh_pty": "true"
-    },{
-      "name": "centos-7",
-      "type": "googlecompute",
-      "project_id": "{{user `GCP_PROJECT_ID`}}",
-      "zone": "us-central1-a",
-      "source_image": "{{user `CENTOS_BASE_IMAGE`}}",
-      "image_name": "{{user `CENTOS_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
-      "image_family": "{{user `CENTOS_BASE_IMAGE`}}-libpod",
-      "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
-      "communicator": "ssh",
-      "ssh_username": "{{user `GCE_SSH_USERNAME`}}",
-      "ssh_pty": "true"
-    },{
-      "name": "fedora-28",
-      "type": "googlecompute",
-      "project_id": "{{user `GCP_PROJECT_ID`}}",
-      "zone": "us-central1-a",
-      "source_image": "{{user `FEDORA_BASE_IMAGE`}}",
-      "image_name": "{{user `FEDORA_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
-      "image_family": "{{user `FEDORA_BASE_IMAGE`}}-libpod",
-      "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
-      "communicator": "ssh",
-      "ssh_username": "fedora",
-      "ssh_pty": "true"
-    },{
-      "name": "ubuntu-18",
-      "type": "googlecompute",
-      "project_id": "{{user `GCP_PROJECT_ID`}}",
-      "zone": "us-central1-a",
-      "source_image": "{{user `UBUNTU_BASE_IMAGE`}}",
-      "image_name": "{{user `UBUNTU_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
-      "image_family": "{{user `UBUNTU_BASE_IMAGE`}}-libpod",
-      "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
-      "communicator": "ssh",
-      "ssh_username": "{{user `GCE_SSH_USERNAME`}}",
-      "ssh_pty": "true"
-    }
-  ],
-  "provisioners": [
-    {
-      "type": "file",
-      "source": "{{user `GOSRC`}}",
-      "destination": "/tmp/libpod"
-    },{
-      "type": "shell",
-      "only": ["rhel-7"],
-      "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/rhel_setup.sh",
-      "environment_vars": [
-        "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `CNI_COMMIT`}}",
-        "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
-        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
-        "RUNC_COMMIT={{user `RUNC_COMMIT`}}",
-        "RHSM_COMMAND={{user `RHSM_COMMAND`}}"
-      ]
-    },{
-      "type": "shell",
-      "only": ["centos-7"],
-      "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/centos_setup.sh",
-      "environment_vars": [
-        "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `CNI_COMMIT`}}",
-        "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
-        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
-        "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
-      ]
-    },{
-      "type": "shell",
-      "only": ["fedora-28"],
-      "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/fedora_setup.sh",
-      "environment_vars": [
-        "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}",
-        "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
-        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
-        "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
-      ]
-    },{
-      "type": "shell",
-      "only": ["ubuntu-18"],
-      "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/ubuntu_setup.sh",
-      "environment_vars": [
-        "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `CNI_COMMIT`}}",
-        "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
-        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
-        "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
-      ]
-    }
-  ]
-}
diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml
new file mode 100644
index 000000000..1e85e8522
--- /dev/null
+++ b/contrib/cirrus/packer/libpod_images.yml
@@ -0,0 +1,89 @@
+---
+
+# All of these are required
+variables:
+    # Names of GCE Base images to start from, in .cirrus.yml
+    RHEL_BASE_IMAGE: '{{env `RHEL_BASE_IMAGE`}}'
+    CENTOS_BASE_IMAGE: '{{env `CENTOS_BASE_IMAGE`}}'
+    UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}'
+    FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}'
+    FAH_BASE_IMAGE: '{{env `FAH_BASE_IMAGE`}}'
+
+    # libpod dependencies to build and install into images
+    FEDORA_CNI_COMMIT: "{{env `FEDORA_CNI_COMMIT`}}"
+    CNI_COMMIT: "{{env `CNI_COMMIT`}}"
+    CRIO_COMMIT: "{{env `CRIO_COMMIT`}}"
+    CRIU_COMMIT: "{{env `CRIU_COMMIT`}}"
+    RUNC_COMMIT: "{{env `RUNC_COMMIT`}}"
+
+    CIRRUS_BUILD_ID: '{{env `CIRRUS_BUILD_ID`}}'
+    GOSRC: '{{env `GOSRC`}}'
+    PACKER_BASE: '{{env `PACKER_BASE`}}'
+    SCRIPT_BASE: '{{env `SCRIPT_BASE`}}'
+
+    # Protected credentials, decrypted by Cirrus at runtime
+    GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}'
+    GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}'
+    RHSM_COMMAND: '{{env `RHSM_COMMAND`}}'
+    SERVICE_ACCOUNT: '{{env `SERVICE_ACCOUNT`}}'
+    GOOGLE_APPLICATION_CREDENTIALS: '{{env `GOOGLE_APPLICATION_CREDENTIALS`}}'
+
+# Don't leak sensitive values in error messages / output
+sensitive-variables:
+    - 'GCE_SSH_USERNAME'
+    - 'GCP_PROJECT_ID'
+    - 'RHSM_COMMAND'
+    - 'SERVICE_ACCOUNT'
+
+# What images to produce in which cloud
+builders:
+    #  v----- is a YAML anchor, allows referencing this object by name (below)
+    - &gce_hosted_image
+      name: 'ubuntu-18'
+      type: 'googlecompute'
+      image_name: '{{user `UBUNTU_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}'
+      image_family: '{{user `UBUNTU_BASE_IMAGE`}}-libpod'
+      source_image: '{{user `UBUNTU_BASE_IMAGE`}}'
+      project_id: '{{user `GCP_PROJECT_ID`}}'
+      service_account_email: '{{user `SERVICE_ACCOUNT`}}'
+      communicator: 'ssh'
+      ssh_username: '{{user `GCE_SSH_USERNAME`}}'
+      ssh_pty: 'true'
+      # The only supported zone in Cirrus-CI, as of addition of this comment
+      zone: 'us-central1-a'
+
+    #      v----- is a YAML alias, allows partial re-use of the anchor object
+    - <<: *gce_hosted_image
+      name: 'rhel-7'
+      image_name: '{{user `RHEL_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}'
+      image_family: '{{user `RHEL_BASE_IMAGE`}}-libpod'
+      source_image: '{{user `RHEL_BASE_IMAGE`}}'
+      ssh_username: 'ec2-user'
+
+    - <<: *gce_hosted_image
+      name: 'centos-7'
+      image_family: '{{user `CENTOS_BASE_IMAGE`}}-libpod'
+      image_name: '{{user `CENTOS_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}'
+      source_image: '{{user `CENTOS_BASE_IMAGE`}}'
+
+
+# The brains of the operation, making actual modifications to the base-image.
+provisioners:
+    - type: 'file'
+      source: '{{user `GOSRC`}}'
+      destination: '/tmp/libpod'
+
+    - type: 'shell'
+      script: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{split build_name "-" 0}}_setup.sh'
+      environment_vars:
+          - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
+          - 'CNI_COMMIT={{user `CNI_COMMIT`}}'
+          - 'FEDORA_CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}'
+          - 'CRIO_COMMIT={{user `CRIO_COMMIT`}}'
+          - 'CRIU_COMMIT={{user `CRIU_COMMIT`}}'
+          - 'RUNC_COMMIT={{user `RUNC_COMMIT`}}'
+          - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}'
+
+post-processors:
+    - - type: 'manifest'
+        output: '/tmp/packer-manifest.json'
diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh
index cacc23045..6bb601e77 100755
--- a/contrib/cirrus/unit_test.sh
+++ b/contrib/cirrus/unit_test.sh
@@ -18,13 +18,11 @@ case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
         make localunit "BUILDTAGS=$BUILDTAGS"
         make "BUILDTAGS=$BUILDTAGS"
         ;;
-    fedora-28)
+    fedora-28) ;&  # Continue to the next item
+    centos-7) ;&
+    rhel-7)
         make localunit
         make
         ;;
-    centos-7) ;&  # Continue to the next item
-    rhel-7)
-        stub 'unit testing not working on $OS_RELEASE_ID'
-        ;;
     *) bad_os_id_ver ;;
 esac