From 9a2ae2442dc018444f73009e2faa513008f01128 Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Wed, 7 Aug 2019 10:25:23 -0400
Subject: Cirrus: Add latest ubuntu

Add the latest Ubuntu version into the testing matrix
and image-build workflow. This is also needed to support
other containers projects which share use of VM images
from this one.

Update package lists to include needs for contianers/storage
use of images.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 .cirrus.yml                                      | 22 +++++++------
 contrib/cirrus/build_vm_images.sh                |  2 +-
 contrib/cirrus/lib.sh                            |  7 +++--
 contrib/cirrus/packer/Makefile                   | 32 +++++++++----------
 contrib/cirrus/packer/fedora_setup.sh            |  8 +++++
 contrib/cirrus/packer/libpod_base_images.yml     | 37 +++++++++++++++++++---
 contrib/cirrus/packer/libpod_images.yml          | 21 +++++++++++--
 contrib/cirrus/packer/prior-fedora_base-setup.sh |  1 +
 contrib/cirrus/packer/prior_fedora_base-setup.sh |  1 -
 contrib/cirrus/packer/ubuntu_setup.sh            | 39 ++++++++++++++++++------
 contrib/cirrus/setup_environment.sh              | 13 ++++----
 11 files changed, 127 insertions(+), 56 deletions(-)
 create mode 120000 contrib/cirrus/packer/prior-fedora_base-setup.sh
 delete mode 120000 contrib/cirrus/packer/prior_fedora_base-setup.sh

diff --git a/.cirrus.yml b/.cirrus.yml
index 77d5ba702..42ec10594 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -30,11 +30,12 @@ env:
     ####
     #### Cache-image names to test with (double-quotes around names are critical)
     ###
-    _BUILT_IMAGE_SUFFIX: "libpod-5751722641719296"
+    _BUILT_IMAGE_SUFFIX: "libpod-5664838702858240"
     FEDORA_CACHE_IMAGE_NAME: "fedora-30-${_BUILT_IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-29-${_BUILT_IMAGE_SUFFIX}"
     SPECIAL_FEDORA_CACHE_IMAGE_NAME: "xfedora-30-${_BUILT_IMAGE_SUFFIX}"
-    UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-${_BUILT_IMAGE_SUFFIX}"
+    UBUNTU_CACHE_IMAGE_NAME: "ubuntu-19-${_BUILT_IMAGE_SUFFIX}"
+    PRIOR_UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-${_BUILT_IMAGE_SUFFIX}"
 
     ####
     #### Variables for composing new cache-images (used in PR testing) from
@@ -269,6 +270,7 @@ meta_task:
             ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
             ${SPECIAL_FEDORA_CACHE_IMAGE_NAME}
             ${UBUNTU_CACHE_IMAGE_NAME}
+            ${PRIOR_UBUNTU_CACHE_IMAGE_NAME}
             ${IMAGE_BUILDER_CACHE_IMAGE_NAME}
         BUILDID: "${CIRRUS_BUILD_ID}"
         REPOREF: "${CIRRUS_CHANGE_IN_REPO}"
@@ -326,6 +328,7 @@ testing_task:
             image_name: "${FEDORA_CACHE_IMAGE_NAME}"
             image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
             image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
+            image_name: "${PRIOR_UBUNTU_CACHE_IMAGE_NAME}"
 
     timeout_in: 120m
 
@@ -530,6 +533,8 @@ test_building_snap_task:
     depends_on:
         - "gating"
 
+    only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
+
     container:
         image: yakshaveinc/snapcraft:core18
     snapcraft_script:
@@ -542,8 +547,7 @@ test_build_cache_images_task:
 
     only_if: >-
         $CIRRUS_BRANCH != $DEST_BRANCH &&
-        $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' &&
-        $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*'
+        $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
 
     depends_on:
         - "gating"
@@ -563,10 +567,7 @@ test_build_cache_images_task:
             - devstorage.full_control
 
     networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
-    setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
     build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}'
-    df_script: '${DFCMD}'
-    journalctl_b_script: 'journalctl -b'
 
     on_failure:
         failed_df_script: '${DFCMD}'
@@ -578,8 +579,7 @@ verify_test_built_images_task:
 
     only_if: >-
         $CIRRUS_BRANCH != $DEST_BRANCH &&
-        $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' &&
-        $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*'
+        $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*'
 
 
     depends_on:
@@ -601,6 +601,7 @@ verify_test_built_images_task:
             PACKER_BUILDER_NAME: "fedora-30"
             PACKER_BUILDER_NAME: "xfedora-30"
             PACKER_BUILDER_NAME: "ubuntu-18"
+            PACKER_BUILDER_NAME: "ubuntu-19"
 
     networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
     environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
@@ -616,7 +617,7 @@ verify_test_built_images_task:
             $SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}
     build_release_script: >-
         [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \
-            '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}'
+            $SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}
     system_test_script: >-
         [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \
             $SCRIPT_BASE/system_test.sh |& ${TIMESTAMP}
@@ -624,6 +625,7 @@ verify_test_built_images_task:
     always:
         <<: *standardlogs
 
+
 # Post message to IRC if everything passed PR testing
 success_task:
 
diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh
index 6230610cb..543f83a14 100755
--- a/contrib/cirrus/build_vm_images.sh
+++ b/contrib/cirrus/build_vm_images.sh
@@ -3,7 +3,7 @@
 set -e
 source $(dirname $0)/lib.sh
 
-BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE'
+BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE PRIOR_UBUNTU_BASE_IMAGE'
 ENV_VARS="PACKER_BUILDS BUILT_IMAGE_SUFFIX $BASE_IMAGE_VARS SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO"
 req_env_var $ENV_VARS
 # Must also be made available through make, into packer process
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index f26eec87f..ec2cc683b 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -52,13 +52,14 @@ CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod}
 CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)}  # difficult to reliably discover
 CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)}  # must be short and unique
 # Vars. for image-building
-PACKER_VER="1.3.5"
+PACKER_VER="1.4.2"
 # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
 
 # Base-images rarely change, define them here so they're out of the way.
-export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-30,xfedora-30,fedora-29}"
+export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-30,xfedora-30,fedora-29}"
 # Google-maintained base-image names
-export UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a"
+export UBUNTU_BASE_IMAGE="ubuntu-1904-disco-v20190724"
+export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a"
 # Manually produced base-image names (see $SCRIPT_BASE/README.md)
 export FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1559164849"
 export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849"
diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile
index d03d22abe..947a2a1e9 100644
--- a/contrib/cirrus/packer/Makefile
+++ b/contrib/cirrus/packer/Makefile
@@ -3,7 +3,7 @@
 # builder name(s) from applicable YAML file,
 # e.g for names see libpod_images.yml
 
-PACKER_VER ?= 1.3.5
+PACKER_VER ?= 1.4.2
 GOARCH=$(shell go env GOARCH)
 ARCH=$(uname -m)
 PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip
@@ -15,6 +15,9 @@ PACKER_BASE ?= contrib/cirrus/packer
 SCRIPT_BASE ?= contrib/cirrus
 POST_MERGE_BUCKET_SUFFIX ?=
 
+UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$UBUNTU_BASE_IMAGE")
+PRIOR_UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$PRIOR_UBUNTU_BASE_IMAGE")
+
 # For debugging nested-virt, use
 #TTYDEV := $(shell tty)
 TTYDEV := /dev/null
@@ -22,6 +25,14 @@ TTYDEV := /dev/null
 .PHONY: all
 all: libpod_images
 
+# Utility target for checking required parameters
+.PHONY: guard-%
+guard-%:
+	@if [[ -z "$($*)" ]]; then \
+		echo "Missing or empty required make variable '$*'."; \
+		exit 1; \
+	fi;
+
 %.json: %.yml
 	@python3 -c 'import json,yaml; json.dump( yaml.load(open("$<").read()), open("$@","w"), indent=2);'
 
@@ -44,10 +55,7 @@ test: libpod_base_images.json libpod_images.json packer
 	@echo "All good"
 
 .PHONY: libpod_images
-libpod_images: libpod_images.json packer
-ifndef PACKER_BUILDS
-	$(error PACKER_BUILDS is undefined, expected builder-names CSV)
-endif
+libpod_images: guard-PACKER_BUILDS libpod_images.json packer
 	./packer build -only=${PACKER_BUILDS} \
 		-force \
 		-var GOSRC=$(GOSRC) \
@@ -72,16 +80,7 @@ cidata.iso: user-data meta-data
 
 # This is intended to be run by a human, with admin access to the libpod GCE project.
 .PHONY: libpod_base_images
-libpod_base_images: libpod_base_images.json cidata.iso cidata.ssh packer
-ifndef GCP_PROJECT_ID
-	$(error GCP_PROJECT_ID is undefined, expected complete GCP project ID string e.g. foobar-12345)
-endif
-ifndef GOOGLE_APPLICATION_CREDENTIALS
-	$(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json)
-endif
-ifndef PACKER_BUILDS
-	$(error PACKER_BUILDS is undefined, expected builder-names CSV)
-endif
+libpod_base_images: guard-GCP_PROJECT_ID guard-GOOGLE_APPLICATION_CREDENTIALS libpod_base_images.json cidata.iso cidata.ssh packer
 	PACKER_CACHE_DIR=/tmp ./packer build \
 		-force \
 		-var TIMESTAMP=$(TIMESTAMP) \
@@ -91,5 +90,6 @@ endif
 		-var GOSRC=$(GOSRC) \
 		-var PACKER_BASE=$(PACKER_BASE) \
 		-var SCRIPT_BASE=$(SCRIPT_BASE) \
-		-only $(PACKER_BUILDS) \
+		-var UBUNTU_BASE_IMAGE=$(UBUNTU_BASE_IMAGE) \
+		-var PRIOR_UBUNTU_BASE_IMAGE=$(PRIOR_UBUNTU_BASE_IMAGE) \
 		libpod_base_images.json
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index 0e1a82cc0..6709eecaf 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -24,6 +24,8 @@ ooe.sh sudo dnf config-manager --set-enabled updates-testing
 echo "Installing general build/test dependencies"
 ooe.sh sudo dnf install -y \
     atomic-registries \
+    autoconf \
+    automake \
     bats \
     bridge-utils \
     btrfs-progs-devel \
@@ -35,10 +37,14 @@ ooe.sh sudo dnf install -y \
     device-mapper-devel \
     emacs-nox \
     findutils \
+    fuse3 \
+    fuse3-devel \
+    gcc \
     git \
     glib2-devel \
     glibc-static \
     gnupg \
+    go-md2man \
     golang \
     golang-github-cpuguy83-go-md2man \
     gpgme-devel \
@@ -81,6 +87,7 @@ ooe.sh sudo dnf install -y \
     xz \
     zip
 
+
 # Ensure there are no disruptive periodic services enabled by default in image
 systemd_banish
 
@@ -91,6 +98,7 @@ case "$PACKER_BUILDER_NAME" in
     xfedora*)
         echo "Configuring CGroups v2 enabled on next boot"
         sudo grubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=1"
+        sudo dnf install -y crun
         ;&  # continue to next matching item
     *)
         echo "Finalizing $PACKER_BUILDER_NAME VM image"
diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml
index e519d2fba..722822837 100644
--- a/contrib/cirrus/packer/libpod_base_images.yml
+++ b/contrib/cirrus/packer/libpod_base_images.yml
@@ -12,6 +12,10 @@ variables:
     # Required for output from qemu builders
     TTYDEV:
 
+    # Ubuntu releases are mearly copied to this project for control purposes
+    UBUNTU_BASE_IMAGE:
+    PRIOR_UBUNTU_BASE_IMAGE:
+
     # Latest Fedora release
     FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2"
     FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM"
@@ -78,29 +82,52 @@ builders:
       ssh_username: 'root'
 
     - <<: *nested_virt
-      name: 'prior_fedora'
+      name: 'prior-fedora'
       iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}'
       iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}'
 
+    - &imgcopy
+      name: 'ubuntu'
+      type: 'googlecompute'
+      image_name: '{{user `UBUNTU_BASE_IMAGE`}}'
+      image_family: '{{build_name}}-base'
+      source_image: '{{user `UBUNTU_BASE_IMAGE`}}'
+      source_image_project_id: 'ubuntu-os-cloud'
+      project_id: '{{user `GCP_PROJECT_ID`}}'
+      account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}'
+      zone: 'us-central1-a'
+      disk_size: 20
+      communicator: 'none'
+
+    - <<: *imgcopy
+      name: 'prior-ubuntu'
+      image_name: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}'
+      source_image: '{{user `UBUNTU_BASE_IMAGE`}}'
+
 provisioners:
     - type: 'shell'
+      only: ['fedora', 'prior-fedora']
       inline:
         - 'mkdir -p /tmp/libpod/{{user `SCRIPT_BASE`}}'
         - 'mkdir -p /tmp/libpod/{{user `PACKER_BASE`}}'
 
     - type: 'file'
+      only: ['fedora', 'prior-fedora']
       source: '{{user `GOSRC`}}/.cirrus.yml'
       destination: '/tmp/libpod/.cirrus.yml'
 
     - type: 'file'
+      only: ['fedora', 'prior-fedora']
       source: '{{user `GOSRC`}}/{{user `SCRIPT_BASE`}}/'
       destination: '/tmp/libpod/{{user `SCRIPT_BASE`}}/'
 
     - type: 'file'
+      only: ['fedora', 'prior-fedora']
       source: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/'
       destination: '/tmp/libpod/{{user `PACKER_BASE`}}/'
 
     - &shell_script
+      only: ['fedora', 'prior-fedora']
       type: 'shell'
       inline:
         - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'
@@ -121,7 +148,7 @@ provisioners:
 
 post-processors:
     - - type: "compress"
-        only: ['fedora', 'prior_fedora']
+        only: ['fedora', 'prior-fedora']
         output: '/tmp/{{build_name}}/disk.raw.tar.gz'
         format: '.tar.gz'
         compression_level: 9
@@ -134,10 +161,10 @@ post-processors:
         gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}.tar.gz'
         image_name: "{{user `FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
         image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}'
-        image_family: '{{user `FEDORA_BASE_IMAGE_NAME`}}'
+        image_family: '{{build_name}}-base'
       - <<: *gcp_import
-        only: ['prior_fedora']
+        only: ['prior-fedora']
         image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
         image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}'
-        image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}'
+        image_family: '{{build_name}}-base'
       - type: 'manifest'
diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml
index cae5d4138..42ccba186 100644
--- a/contrib/cirrus/packer/libpod_images.yml
+++ b/contrib/cirrus/packer/libpod_images.yml
@@ -12,6 +12,12 @@ variables:
     PACKER_BASE: '{{env `PACKER_BASE`}}'
     SCRIPT_BASE: '{{env `SCRIPT_BASE`}}'
 
+    # Base-image names are required.  Using image family-names breaks parallelism
+    UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}'
+    PRIOR_UBUNTU_BASE_IMAGE: '{{env `PRIOR_UBUNTU_BASE_IMAGE`}}'
+    FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}'
+    PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}'
+
     # Protected credentials, decrypted by Cirrus at runtime
     GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}'
     GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}'
@@ -28,11 +34,12 @@ sensitive-variables:
 builders:
     #  v----- is a YAML anchor, allows referencing this object by name (below)
     - &gce_hosted_image
-      name: 'ubuntu-18'
+      name: 'ubuntu-19'
       type: 'googlecompute'
       image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}'
-      image_family: '{{build_name}}-libpod'
-      source_image: '{{user `UBUNTU_BASE_IMAGE`}}'
+      image_family: '{{build_name}}-cache'
+      source_image: '{{user `UBUNTU_BASE_IMAGE`}}'  # precedence over family
+      source_image_family: 'ubuntu-base'  # for ref. only
       disk_size: 20  # REQUIRED: Runtime allocation > this value
       project_id: '{{user `GCP_PROJECT_ID`}}'
       service_account_email: '{{user `SERVICE_ACCOUNT`}}'
@@ -43,17 +50,25 @@ builders:
       zone: 'us-central1-a'
 
     #      v----- is a YAML alias, allows partial re-use of the anchor object
+    - <<: *gce_hosted_image
+      name: 'ubuntu-18'
+      source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}'
+      source_image_family: 'prior-ubuntu-base'
+
     - <<: *gce_hosted_image
       name: 'fedora-30'
       source_image: '{{user `FEDORA_BASE_IMAGE`}}'
+      source_image_family: 'fedora-base'
 
     - <<: *gce_hosted_image
       name: 'xfedora-30'
       source_image: '{{user `FEDORA_BASE_IMAGE`}}'
+      source_image_family: 'fedora-base'
 
     - <<: *gce_hosted_image
       name: 'fedora-29'
       source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}'
+      source_image_family: 'prior-fedora-base'
 
 # The brains of the operation, making actual modifications to the base-image.
 provisioners:
diff --git a/contrib/cirrus/packer/prior-fedora_base-setup.sh b/contrib/cirrus/packer/prior-fedora_base-setup.sh
new file mode 120000
index 000000000..998a5d9fd
--- /dev/null
+++ b/contrib/cirrus/packer/prior-fedora_base-setup.sh
@@ -0,0 +1 @@
+fedora_base-setup.sh
\ No newline at end of file
diff --git a/contrib/cirrus/packer/prior_fedora_base-setup.sh b/contrib/cirrus/packer/prior_fedora_base-setup.sh
deleted file mode 120000
index 998a5d9fd..000000000
--- a/contrib/cirrus/packer/prior_fedora_base-setup.sh
+++ /dev/null
@@ -1 +0,0 @@
-fedora_base-setup.sh
\ No newline at end of file
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index 00d92570f..da7d457a5 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -20,18 +20,25 @@ systemd_banish
 
 echo "Updating/configuring package repositories."
 $LILTO $SUDOAPTGET update
+
+echo "Upgrading all packages"
+$BIGTO $SUDOAPTGET upgrade
+
+echo "Adding PPAs"
 $LILTO $SUDOAPTGET install software-properties-common
-$LILTO $SUDOAPTADD ppa:longsleep/golang-backports
 $LILTO $SUDOAPTADD ppa:projectatomic/ppa
 $LILTO $SUDOAPTADD ppa:criu/ppa
+if [[ "$OS_RELEASE_VER" -eq "18" ]]
+then
+    $LILTO $SUDOAPTADD ppa:longsleep/golang-backports
+fi
 
-echo "Upgrading all packages"
 $LILTO $SUDOAPTGET update
-$BIGTO $SUDOAPTGET upgrade
 
 echo "Installing general testing and system dependencies"
 $BIGTO $SUDOAPTGET install \
     apparmor \
+    aufs-tools \
     autoconf \
     automake \
     bats \
@@ -46,6 +53,7 @@ $BIGTO $SUDOAPTGET install \
     e2fslibs-dev \
     emacs-nox \
     gawk \
+    gcc \
     gettext \
     go-md2man \
     golang \
@@ -58,6 +66,7 @@ $BIGTO $SUDOAPTGET install \
     libdevmapper-dev \
     libdevmapper1.02.1 \
     libfuse-dev \
+    libfuse2 \
     libglib2.0-dev \
     libgpgme11-dev \
     liblzma-dev \
@@ -66,7 +75,7 @@ $BIGTO $SUDOAPTGET install \
     libnl-3-dev \
     libostree-dev \
     libvarlink \
-    libprotobuf-c0-dev \
+    libprotobuf-c-dev \
     libprotobuf-dev \
     libseccomp-dev \
     libseccomp2 \
@@ -74,6 +83,7 @@ $BIGTO $SUDOAPTGET install \
     libtool \
     libudev-dev \
     lsof \
+    make \
     netcat \
     pkg-config \
     podman \
@@ -87,19 +97,28 @@ $BIGTO $SUDOAPTGET install \
     python3-psutil \
     python3-pytoml \
     python3-setuptools \
-    slirp4netns \
     skopeo \
+    slirp4netns \
     socat \
     unzip \
     vim \
     xz-utils \
     zip
 
-echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting."
-SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g'
-ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/*
-ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub
-ooe.sh sudo update-grub
+if [[ "$OS_RELEASE_VER" -ge "19" ]]
+then
+    echo "Installing Ubuntu > 18 packages"
+    $LILTO $SUDOAPTGET install fuse3 libfuse3-dev libbtrfs-dev
+fi
+
+if [[ "$OS_RELEASE_VER" -eq "18" ]]
+then
+    echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting."
+    SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g'
+    ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/*
+    ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub
+    ooe.sh sudo update-grub
+fi
 
 sudo /tmp/libpod/hack/install_catatonit.sh
 ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index c1a01d6ab..df510deef 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -33,8 +33,8 @@ done
 #       contrib/cirrus/packer/*_setup.sh to be incorporated into VM cache-images
 #       (see docs).
 cd "${GOSRC}/"
-case "${OS_REL_VER}" in
-    ubuntu-18)
+case "${OS_RELEASE_ID}" in
+    ubuntu)
         CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc"
         if dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH"
         then
@@ -42,14 +42,13 @@ case "${OS_REL_VER}" in
             ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc"
         fi
         ;;
-    fedora-30) ;&  # continue to next item
-    fedora-29)
-	# All SELinux distros need this for systemd-in-a-container
-	setsebool container_manage_cgroup true
+    fedora)
+        # All SELinux distros need this for systemd-in-a-container
+        setsebool container_manage_cgroup true
         if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
             bash "$SCRIPT_BASE/add_second_partition.sh"; fi
         ;;
-    centos-7)  # Current VM is an image-builder-image no local podman/testing
+    centos)  # Current VM is an image-builder-image no local podman/testing
         echo "No further setup required for VM image building"
         exit 0
         ;;
-- 
cgit v1.2.3-54-g00ecf


From 831a64393e1a0440efa3890410ff61ad908de6bd Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Fri, 9 Aug 2019 09:16:09 -0400
Subject: Cirrus: disable Evil Units in base-images

Also, minor update to prevent harmless 'Fatal: not a git repo' error.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 contrib/cirrus/check_image.sh                |  2 ++
 contrib/cirrus/lib.sh                        | 36 ++++++++--------------------
 contrib/cirrus/packer/fedora_base-setup.sh   |  2 --
 contrib/cirrus/packer/libpod_base_images.yml | 16 ++++---------
 contrib/cirrus/packer/libpod_images.yml      |  5 ----
 contrib/cirrus/packer/systemd_banish.sh      | 28 ++++++++++++++++++++++
 6 files changed, 45 insertions(+), 44 deletions(-)
 create mode 100755 contrib/cirrus/packer/systemd_banish.sh

diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh
index ad9a12f49..39f49d0a1 100755
--- a/contrib/cirrus/check_image.sh
+++ b/contrib/cirrus/check_image.sh
@@ -4,6 +4,8 @@ set -eo pipefail
 
 source $(dirname $0)/lib.sh
 
+EVIL_UNITS="$($CIRRUS_WORKING_DIR/$PACKER_BASE/systemd_banish.sh --list)"
+
 req_env_var PACKER_BUILDER_NAME TEST_REMOTE_CLIENT EVIL_UNITS OS_RELEASE_ID
 
 NFAILS=0
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index ec2cc683b..f81a8d501 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -36,7 +36,7 @@ SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_c
 AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv"
 
 cd $GOSRC
-if type -P git &> /dev/null
+if type -P git &> /dev/null && [[ -d "$GOSRC/.git" ]]
 then
     CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)}
 else # pick something unique and obviously not from Cirrus
@@ -61,8 +61,8 @@ export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-30,xfedora-30,
 export UBUNTU_BASE_IMAGE="ubuntu-1904-disco-v20190724"
 export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a"
 # Manually produced base-image names (see $SCRIPT_BASE/README.md)
-export FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1559164849"
-export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849"
+export FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1565360543"
+export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1565360543"
 export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}"
 # IN_PODMAN container image
 IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest"
@@ -82,9 +82,6 @@ ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.
 # Unsafe env. vars for display
 SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(GC[EP]..+)|(SSH)'
 
-# Names of systemd units which should never be running
-EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean"
-
 SPECIALMODE="${SPECIALMODE:-none}"
 TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}"
 export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
@@ -323,7 +320,7 @@ EOF
     sudo chmod 755 /usr/bin/git
 }
 
-install_test_configs(){
+install_test_configs() {
     echo "Installing cni config, policy and registry config"
     req_env_var GOSRC SCRIPT_BASE
     cd $GOSRC
@@ -343,7 +340,7 @@ install_test_configs(){
 # of pulling in necessary prerequisites packages as the set can change over time.
 # For general CI testing however, calling this function makes sure the system
 # can only run the compiled source version.
-remove_packaged_podman_files(){
+remove_packaged_podman_files() {
     echo "Removing packaged podman files to prevent conflicts with source build and testing."
     req_env_var OS_RELEASE_ID
 
@@ -380,24 +377,11 @@ remove_packaged_podman_files(){
     sync && echo 3 > /proc/sys/vm/drop_caches
 }
 
-systemd_banish(){
-    echo "Disabling periodic services that could destabilize testing (ignoring errors):"
-    set +e  # Not all of these exist on every platform
-    for unit in $EVIL_UNITS
-    do
-        echo "Banishing $unit (ignoring errors)"
-        (
-            sudo systemctl stop $unit
-            sudo systemctl disable $unit
-            sudo systemctl disable $unit.timer
-            sudo systemctl mask $unit
-            sudo systemctl mask $unit.timer
-        ) &> /dev/null
-    done
-    set -e
+systemd_banish() {
+    $GOSRC/$PACKER_BASE/systemd_banish.sh
 }
 
-_finalize(){
+_finalize() {
     set +e  # Don't fail at the very end
     if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]]
     then
@@ -420,7 +404,7 @@ _finalize(){
     sudo fstrim -av
 }
 
-rh_finalize(){
+rh_finalize() {
     set +e  # Don't fail at the very end
     echo "Resetting to fresh-state for usage as cloud-image."
     PKG=$(type -P dnf || type -P yum || echo "")
@@ -431,7 +415,7 @@ rh_finalize(){
     _finalize
 }
 
-ubuntu_finalize(){
+ubuntu_finalize() {
     set +e  # Don't fail at the very end
     echo "Resetting to fresh-state for usage as cloud-image."
     $LILTO $SUDOAPTGET autoremove
diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh
index 788a54c34..29c23117f 100644
--- a/contrib/cirrus/packer/fedora_base-setup.sh
+++ b/contrib/cirrus/packer/fedora_base-setup.sh
@@ -8,8 +8,6 @@ set -e
 # Load in library (copied by packer, before this script was run)
 source $GOSRC/$SCRIPT_BASE/lib.sh
 
-[[ "$1" == "post" ]] || exit 0  # nothing to do
-
 install_ooe
 
 echo "Updating packages"
diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml
index 722822837..bcca440ae 100644
--- a/contrib/cirrus/packer/libpod_base_images.yml
+++ b/contrib/cirrus/packer/libpod_base_images.yml
@@ -95,6 +95,7 @@ builders:
       source_image_project_id: 'ubuntu-os-cloud'
       project_id: '{{user `GCP_PROJECT_ID`}}'
       account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}'
+      startup_script_file: "systemd_banish.sh"
       zone: 'us-central1-a'
       disk_size: 20
       communicator: 'none'
@@ -102,7 +103,7 @@ builders:
     - <<: *imgcopy
       name: 'prior-ubuntu'
       image_name: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}'
-      source_image: '{{user `UBUNTU_BASE_IMAGE`}}'
+      source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}'
 
 provisioners:
     - type: 'shell'
@@ -130,22 +131,15 @@ provisioners:
       only: ['fedora', 'prior-fedora']
       type: 'shell'
       inline:
-        - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'
-        - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh pre'
-      expect_disconnect: true  # Allow this to reboot the VM
+        - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/*.sh'
+        - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'
+      expect_disconnect: true  # Allow this to reboot the VM if needed
       environment_vars:
           - 'TIMESTAMP={{user `TIMESTAMP`}}'
           - 'GOSRC=/tmp/libpod'
           - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
           - 'PACKER_BASE={{user `PACKER_BASE`}}'
 
-    - <<: *shell_script
-      inline: ['{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh']
-      expect_disconnect: false
-      pause_before: '10s'
-      inline:
-        - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh post'
-
 post-processors:
     - - type: "compress"
         only: ['fedora', 'prior-fedora']
diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml
index 42ccba186..01a65d867 100644
--- a/contrib/cirrus/packer/libpod_images.yml
+++ b/contrib/cirrus/packer/libpod_images.yml
@@ -2,11 +2,6 @@
 
 # All of these are required
 variables:
-    # Names of GCE Base images to start from, in .cirrus.yml
-    UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}'
-    FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}'
-    PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}'
-
     BUILT_IMAGE_SUFFIX: '{{env `BUILT_IMAGE_SUFFIX`}}'
     GOSRC: '{{env `GOSRC`}}'
     PACKER_BASE: '{{env `PACKER_BASE`}}'
diff --git a/contrib/cirrus/packer/systemd_banish.sh b/contrib/cirrus/packer/systemd_banish.sh
new file mode 100755
index 000000000..396cf906c
--- /dev/null
+++ b/contrib/cirrus/packer/systemd_banish.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set +e  # Not all of these exist on every platform
+
+# This is intended to be executed on VMs as a startup script on initial-boot.
+# Alternativly, it may be executed with the '--list' option to return the list
+# of systemd units defined for disablement (useful for testing).
+
+EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean"
+
+if [[ "$1" == "--list" ]]
+then
+    echo "$EVIL_UNITS"
+    exit 0
+fi
+
+echo "Disabling periodic services that could destabilize testing:"
+for unit in $EVIL_UNITS
+do
+    echo "Banishing $unit (ignoring errors)"
+    (
+        sudo systemctl stop $unit
+        sudo systemctl disable $unit
+        sudo systemctl disable $unit.timer
+        sudo systemctl mask $unit
+        sudo systemctl mask $unit.timer
+    ) &> /dev/null
+done
-- 
cgit v1.2.3-54-g00ecf


From a6534a7c9617047396252b66fdcd1caa49ce49d5 Mon Sep 17 00:00:00 2001
From: Chris Evich <cevich@redhat.com>
Date: Wed, 14 Aug 2019 13:51:07 -0400
Subject: Cirrus: Temporarily disable testing on Ubuntu 19

The images build correctly but neither integration or remote client
tests pass.  Temporarily disable Ubuntu 19 testing until both are
ready to be supported.

Signed-off-by: Chris Evich <cevich@redhat.com>
---
 .cirrus.yml | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/.cirrus.yml b/.cirrus.yml
index 42ec10594..95092a900 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -327,7 +327,8 @@ testing_task:
             # Images are generated separately, from build_images_task (below)
             image_name: "${FEDORA_CACHE_IMAGE_NAME}"
             image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
-            image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
+            # Multiple test failures on Ubuntu 19 - Fixes TBD in future PR
+            # TODO: image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
             image_name: "${PRIOR_UBUNTU_CACHE_IMAGE_NAME}"
 
     timeout_in: 120m
@@ -601,7 +602,7 @@ verify_test_built_images_task:
             PACKER_BUILDER_NAME: "fedora-30"
             PACKER_BUILDER_NAME: "xfedora-30"
             PACKER_BUILDER_NAME: "ubuntu-18"
-            PACKER_BUILDER_NAME: "ubuntu-19"
+            # TODO support $UBUNTU_CACHE_IMAGE_NAME: PACKER_BUILDER_NAME: "ubuntu-19"
 
     networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
     environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
-- 
cgit v1.2.3-54-g00ecf