Cirrus: Add latest ubuntu

Add the latest Ubuntu version into the testing matrix
and image-build workflow. This is also needed to support
other containers projects which share use of VM images
from this one.

Update package lists to include needs for contianers/storage
use of images.

Signed-off-by: Chris Evich <cevich@redhat.com>

9 files changed, 114 insertions, 45 deletions

diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh
index 6230610cb..543f83a14 100755
--- a/contrib/cirrus/build_vm_images.sh
+++ b/contrib/cirrus/build_vm_images.sh
@@ -3,7 +3,7 @@
 set -e
 source $(dirname $0)/lib.sh
 
-BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE'
+BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE PRIOR_UBUNTU_BASE_IMAGE'
 ENV_VARS="PACKER_BUILDS BUILT_IMAGE_SUFFIX $BASE_IMAGE_VARS SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO"
 req_env_var $ENV_VARS
 # Must also be made available through make, into packer process
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index f26eec87f..ec2cc683b 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -52,13 +52,14 @@ CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod}
 CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)}  # difficult to reliably discover
 CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)}  # must be short and unique
 # Vars. for image-building
-PACKER_VER="1.3.5"
+PACKER_VER="1.4.2"
 # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
 
 # Base-images rarely change, define them here so they're out of the way.
-export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-30,xfedora-30,fedora-29}"
+export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-30,xfedora-30,fedora-29}"
 # Google-maintained base-image names
-export UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a"
+export UBUNTU_BASE_IMAGE="ubuntu-1904-disco-v20190724"
+export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a"
 # Manually produced base-image names (see $SCRIPT_BASE/README.md)
 export FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1559164849"
 export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849"
diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile
index d03d22abe..947a2a1e9 100644
--- a/contrib/cirrus/packer/Makefile
+++ b/contrib/cirrus/packer/Makefile
@@ -3,7 +3,7 @@
 # builder name(s) from applicable YAML file,
 # e.g for names see libpod_images.yml
 
-PACKER_VER ?= 1.3.5
+PACKER_VER ?= 1.4.2
 GOARCH=$(shell go env GOARCH)
 ARCH=$(uname -m)
 PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip
@@ -15,6 +15,9 @@ PACKER_BASE ?= contrib/cirrus/packer
 SCRIPT_BASE ?= contrib/cirrus
 POST_MERGE_BUCKET_SUFFIX ?=
 
+UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$UBUNTU_BASE_IMAGE")
+PRIOR_UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$PRIOR_UBUNTU_BASE_IMAGE")
+
 # For debugging nested-virt, use
 #TTYDEV := $(shell tty)
 TTYDEV := /dev/null
@@ -22,6 +25,14 @@ TTYDEV := /dev/null
 .PHONY: all
 all: libpod_images
 
+# Utility target for checking required parameters
+.PHONY: guard-%
+guard-%:
+	@if [[ -z "$($*)" ]]; then \
+		echo "Missing or empty required make variable '$*'."; \
+		exit 1; \
+	fi;
+
 %.json: %.yml
 	@python3 -c 'import json,yaml; json.dump( yaml.load(open("$<").read()), open("$@","w"), indent=2);'
 
@@ -44,10 +55,7 @@ test: libpod_base_images.json libpod_images.json packer
 	@echo "All good"
 
 .PHONY: libpod_images
-libpod_images: libpod_images.json packer
-ifndef PACKER_BUILDS
-	$(error PACKER_BUILDS is undefined, expected builder-names CSV)
-endif
+libpod_images: guard-PACKER_BUILDS libpod_images.json packer
 	./packer build -only=${PACKER_BUILDS} \
 		-force \
 		-var GOSRC=$(GOSRC) \
@@ -72,16 +80,7 @@ cidata.iso: user-data meta-data
 
 # This is intended to be run by a human, with admin access to the libpod GCE project.
 .PHONY: libpod_base_images
-libpod_base_images: libpod_base_images.json cidata.iso cidata.ssh packer
-ifndef GCP_PROJECT_ID
-	$(error GCP_PROJECT_ID is undefined, expected complete GCP project ID string e.g. foobar-12345)
-endif
-ifndef GOOGLE_APPLICATION_CREDENTIALS
-	$(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json)
-endif
-ifndef PACKER_BUILDS
-	$(error PACKER_BUILDS is undefined, expected builder-names CSV)
-endif
+libpod_base_images: guard-GCP_PROJECT_ID guard-GOOGLE_APPLICATION_CREDENTIALS libpod_base_images.json cidata.iso cidata.ssh packer
 	PACKER_CACHE_DIR=/tmp ./packer build \
 		-force \
 		-var TIMESTAMP=$(TIMESTAMP) \
@@ -91,5 +90,6 @@ endif
 		-var GOSRC=$(GOSRC) \
 		-var PACKER_BASE=$(PACKER_BASE) \
 		-var SCRIPT_BASE=$(SCRIPT_BASE) \
-		-only $(PACKER_BUILDS) \
+		-var UBUNTU_BASE_IMAGE=$(UBUNTU_BASE_IMAGE) \
+		-var PRIOR_UBUNTU_BASE_IMAGE=$(PRIOR_UBUNTU_BASE_IMAGE) \
 		libpod_base_images.json
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index 0e1a82cc0..6709eecaf 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -24,6 +24,8 @@ ooe.sh sudo dnf config-manager --set-enabled updates-testing
 echo "Installing general build/test dependencies"
 ooe.sh sudo dnf install -y \
     atomic-registries \
+    autoconf \
+    automake \
     bats \
     bridge-utils \
     btrfs-progs-devel \
@@ -35,10 +37,14 @@ ooe.sh sudo dnf install -y \
     device-mapper-devel \
     emacs-nox \
     findutils \
+    fuse3 \
+    fuse3-devel \
+    gcc \
     git \
     glib2-devel \
     glibc-static \
     gnupg \
+    go-md2man \
     golang \
     golang-github-cpuguy83-go-md2man \
     gpgme-devel \
@@ -81,6 +87,7 @@ ooe.sh sudo dnf install -y \
     xz \
     zip
 
+
 # Ensure there are no disruptive periodic services enabled by default in image
 systemd_banish
 
@@ -91,6 +98,7 @@ case "$PACKER_BUILDER_NAME" in
     xfedora*)
         echo "Configuring CGroups v2 enabled on next boot"
         sudo grubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=1"
+        sudo dnf install -y crun
         ;&  # continue to next matching item
     *)
         echo "Finalizing $PACKER_BUILDER_NAME VM image"
diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml
index e519d2fba..722822837 100644
--- a/contrib/cirrus/packer/libpod_base_images.yml
+++ b/contrib/cirrus/packer/libpod_base_images.yml
@@ -12,6 +12,10 @@ variables:
     # Required for output from qemu builders
     TTYDEV:
 
+    # Ubuntu releases are mearly copied to this project for control purposes
+    UBUNTU_BASE_IMAGE:
+    PRIOR_UBUNTU_BASE_IMAGE:
+
     # Latest Fedora release
     FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2"
     FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM"
@@ -78,29 +82,52 @@ builders:
       ssh_username: 'root'
 
     - <<: *nested_virt
-      name: 'prior_fedora'
+      name: 'prior-fedora'
       iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}'
       iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}'
 
+    - &imgcopy
+      name: 'ubuntu'
+      type: 'googlecompute'
+      image_name: '{{user `UBUNTU_BASE_IMAGE`}}'
+      image_family: '{{build_name}}-base'
+      source_image: '{{user `UBUNTU_BASE_IMAGE`}}'
+      source_image_project_id: 'ubuntu-os-cloud'
+      project_id: '{{user `GCP_PROJECT_ID`}}'
+      account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}'
+      zone: 'us-central1-a'
+      disk_size: 20
+      communicator: 'none'
+
+    - <<: *imgcopy
+      name: 'prior-ubuntu'
+      image_name: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}'
+      source_image: '{{user `UBUNTU_BASE_IMAGE`}}'
+
 provisioners:
     - type: 'shell'
+      only: ['fedora', 'prior-fedora']
       inline:
         - 'mkdir -p /tmp/libpod/{{user `SCRIPT_BASE`}}'
         - 'mkdir -p /tmp/libpod/{{user `PACKER_BASE`}}'
 
     - type: 'file'
+      only: ['fedora', 'prior-fedora']
       source: '{{user `GOSRC`}}/.cirrus.yml'
       destination: '/tmp/libpod/.cirrus.yml'
 
     - type: 'file'
+      only: ['fedora', 'prior-fedora']
       source: '{{user `GOSRC`}}/{{user `SCRIPT_BASE`}}/'
       destination: '/tmp/libpod/{{user `SCRIPT_BASE`}}/'
 
     - type: 'file'
+      only: ['fedora', 'prior-fedora']
       source: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/'
       destination: '/tmp/libpod/{{user `PACKER_BASE`}}/'
 
     - &shell_script
+      only: ['fedora', 'prior-fedora']
       type: 'shell'
       inline:
         - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'
@@ -121,7 +148,7 @@ provisioners:
 
 post-processors:
     - - type: "compress"
-        only: ['fedora', 'prior_fedora']
+        only: ['fedora', 'prior-fedora']
         output: '/tmp/{{build_name}}/disk.raw.tar.gz'
         format: '.tar.gz'
         compression_level: 9
@@ -134,10 +161,10 @@ post-processors:
         gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}.tar.gz'
         image_name: "{{user `FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
         image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}'
-        image_family: '{{user `FEDORA_BASE_IMAGE_NAME`}}'
+        image_family: '{{build_name}}-base'
       - <<: *gcp_import
-        only: ['prior_fedora']
+        only: ['prior-fedora']
         image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
         image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}'
-        image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}'
+        image_family: '{{build_name}}-base'
       - type: 'manifest'
diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml
index cae5d4138..42ccba186 100644
--- a/contrib/cirrus/packer/libpod_images.yml
+++ b/contrib/cirrus/packer/libpod_images.yml
@@ -12,6 +12,12 @@ variables:
     PACKER_BASE: '{{env `PACKER_BASE`}}'
     SCRIPT_BASE: '{{env `SCRIPT_BASE`}}'
 
+    # Base-image names are required.  Using image family-names breaks parallelism
+    UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}'
+    PRIOR_UBUNTU_BASE_IMAGE: '{{env `PRIOR_UBUNTU_BASE_IMAGE`}}'
+    FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}'
+    PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}'
+
     # Protected credentials, decrypted by Cirrus at runtime
     GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}'
     GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}'
@@ -28,11 +34,12 @@ sensitive-variables:
 builders:
     #  v----- is a YAML anchor, allows referencing this object by name (below)
     - &gce_hosted_image
-      name: 'ubuntu-18'
+      name: 'ubuntu-19'
       type: 'googlecompute'
       image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}'
-      image_family: '{{build_name}}-libpod'
-      source_image: '{{user `UBUNTU_BASE_IMAGE`}}'
+      image_family: '{{build_name}}-cache'
+      source_image: '{{user `UBUNTU_BASE_IMAGE`}}'  # precedence over family
+      source_image_family: 'ubuntu-base'  # for ref. only
       disk_size: 20  # REQUIRED: Runtime allocation > this value
       project_id: '{{user `GCP_PROJECT_ID`}}'
       service_account_email: '{{user `SERVICE_ACCOUNT`}}'
@@ -44,16 +51,24 @@ builders:
 
     #      v----- is a YAML alias, allows partial re-use of the anchor object
     - <<: *gce_hosted_image
+      name: 'ubuntu-18'
+      source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}'
+      source_image_family: 'prior-ubuntu-base'
+
+    - <<: *gce_hosted_image
       name: 'fedora-30'
       source_image: '{{user `FEDORA_BASE_IMAGE`}}'
+      source_image_family: 'fedora-base'
 
     - <<: *gce_hosted_image
       name: 'xfedora-30'
       source_image: '{{user `FEDORA_BASE_IMAGE`}}'
+      source_image_family: 'fedora-base'
 
     - <<: *gce_hosted_image
       name: 'fedora-29'
       source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}'
+      source_image_family: 'prior-fedora-base'
 
 # The brains of the operation, making actual modifications to the base-image.
 provisioners:
diff --git a/contrib/cirrus/packer/prior_fedora_base-setup.sh b/contrib/cirrus/packer/prior-fedora_base-setup.sh
index 998a5d9fd..998a5d9fd 120000
--- a/contrib/cirrus/packer/prior_fedora_base-setup.sh
+++ b/contrib/cirrus/packer/prior-fedora_base-setup.sh
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index 00d92570f..da7d457a5 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -20,18 +20,25 @@ systemd_banish
 
 echo "Updating/configuring package repositories."
 $LILTO $SUDOAPTGET update
+
+echo "Upgrading all packages"
+$BIGTO $SUDOAPTGET upgrade
+
+echo "Adding PPAs"
 $LILTO $SUDOAPTGET install software-properties-common
-$LILTO $SUDOAPTADD ppa:longsleep/golang-backports
 $LILTO $SUDOAPTADD ppa:projectatomic/ppa
 $LILTO $SUDOAPTADD ppa:criu/ppa
+if [[ "$OS_RELEASE_VER" -eq "18" ]]
+then
+    $LILTO $SUDOAPTADD ppa:longsleep/golang-backports
+fi
 
-echo "Upgrading all packages"
 $LILTO $SUDOAPTGET update
-$BIGTO $SUDOAPTGET upgrade
 
 echo "Installing general testing and system dependencies"
 $BIGTO $SUDOAPTGET install \
     apparmor \
+    aufs-tools \
     autoconf \
     automake \
     bats \
@@ -46,6 +53,7 @@ $BIGTO $SUDOAPTGET install \
     e2fslibs-dev \
     emacs-nox \
     gawk \
+    gcc \
     gettext \
     go-md2man \
     golang \
@@ -58,6 +66,7 @@ $BIGTO $SUDOAPTGET install \
     libdevmapper-dev \
     libdevmapper1.02.1 \
     libfuse-dev \
+    libfuse2 \
     libglib2.0-dev \
     libgpgme11-dev \
     liblzma-dev \
@@ -66,7 +75,7 @@ $BIGTO $SUDOAPTGET install \
     libnl-3-dev \
     libostree-dev \
     libvarlink \
-    libprotobuf-c0-dev \
+    libprotobuf-c-dev \
     libprotobuf-dev \
     libseccomp-dev \
     libseccomp2 \
@@ -74,6 +83,7 @@ $BIGTO $SUDOAPTGET install \
     libtool \
     libudev-dev \
     lsof \
+    make \
     netcat \
     pkg-config \
     podman \
@@ -87,19 +97,28 @@ $BIGTO $SUDOAPTGET install \
     python3-psutil \
     python3-pytoml \
     python3-setuptools \
-    slirp4netns \
     skopeo \
+    slirp4netns \
     socat \
     unzip \
     vim \
     xz-utils \
     zip
 
-echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting."
-SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g'
-ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/*
-ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub
-ooe.sh sudo update-grub
+if [[ "$OS_RELEASE_VER" -ge "19" ]]
+then
+    echo "Installing Ubuntu > 18 packages"
+    $LILTO $SUDOAPTGET install fuse3 libfuse3-dev libbtrfs-dev
+fi
+
+if [[ "$OS_RELEASE_VER" -eq "18" ]]
+then
+    echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting."
+    SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g'
+    ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/*
+    ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub
+    ooe.sh sudo update-grub
+fi
 
 sudo /tmp/libpod/hack/install_catatonit.sh
 ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index c1a01d6ab..df510deef 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -33,8 +33,8 @@ done
 #       contrib/cirrus/packer/*_setup.sh to be incorporated into VM cache-images
 #       (see docs).
 cd "${GOSRC}/"
-case "${OS_REL_VER}" in
-    ubuntu-18)
+case "${OS_RELEASE_ID}" in
+    ubuntu)
         CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc"
         if dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH"
         then
@@ -42,14 +42,13 @@ case "${OS_REL_VER}" in
             ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc"
         fi
         ;;
-    fedora-30) ;&  # continue to next item
-    fedora-29)
-	# All SELinux distros need this for systemd-in-a-container
-	setsebool container_manage_cgroup true
+    fedora)
+        # All SELinux distros need this for systemd-in-a-container
+        setsebool container_manage_cgroup true
         if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
             bash "$SCRIPT_BASE/add_second_partition.sh"; fi
         ;;
-    centos-7)  # Current VM is an image-builder-image no local podman/testing
+    centos)  # Current VM is an image-builder-image no local podman/testing
         echo "No further setup required for VM image building"
         exit 0
         ;;