14 files changed, 231 insertions, 44 deletions
diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh
index f5d53a92e..74b10158c 100755
--- a/contrib/cirrus/build_vm_images.sh
+++ b/contrib/cirrus/build_vm_images.sh
@@ -3,7 +3,8 @@
 set -e
 source $(dirname $0)/lib.sh
 
-ENV_VARS='PACKER_BUILDS BUILT_IMAGE_SUFFIX UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE'
+BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE'
+ENV_VARS="PACKER_BUILDS BUILT_IMAGE_SUFFIX $BASE_IMAGE_VARS SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO"
 req_env_var $ENV_VARS
 # Must also be made available through make, into packer process
 export $ENV_VARS
@@ -24,6 +25,20 @@ then
 fi
 
 cd "$GOSRC/$PACKER_BASE"
+# Add/update labels on base-images used in this build to prevent premature deletion
+ARGS="
+"
+for base_image_var in $BASE_IMAGE_VARS
+do
+    # See entrypoint.sh in contrib/imgts and contrib/imgprune
+    # These updates can take a while, run them in the background, check later
+    gcloud compute images update "$image" \
+        --update-labels=last-used=$(date +%s) \
+        --update-labels=build-id=$CIRRUS_BUILD_ID \
+        --update-labels=repo-ref=$CIRRUS_CHANGE_IN_REPO \
+        --update-labels=project=$GCP_PROJECT_ID \
+        ${!base_image_var} &
+done
 
 make libpod_images \
     PACKER_BUILDS=$PACKER_BUILDS \
@@ -33,9 +48,31 @@ make libpod_images \
     PACKER_BASE=$PACKER_BASE \
     BUILT_IMAGE_SUFFIX=$BUILT_IMAGE_SUFFIX
 
+# Separate PR-produced images from those produced on master.
+if [[ "${CIRRUS_BRANCH:-}" == "master" ]]
+then
+    POST_MERGE_BUCKET_SUFFIX="-master"
+else
+    POST_MERGE_BUCKET_SUFFIX=""
+fi
+
 # When successful, upload manifest of produced images using a filename unique
 # to this build.
 URI="gs://packer-import${POST_MERGE_BUCKET_SUFFIX}/manifest${BUILT_IMAGE_SUFFIX}.json"
 gsutil cp packer-manifest.json "$URI"
 
+# Ensure any background 'gcloud compute images update' processes finish
+set +e  # need 'wait' exit code to avoid race
+while [[ -n "$(jobs)" ]]
+do
+    wait -n
+    RET=$?
+    if [[ "$RET" -eq "127" ]] || \   # Avoid TOCTOU race w/ jobs + wait
+       [[ "$RET" -eq "0" ]]
+    then
+        continue
+    fi
+    die $RET "Required base-image metadata update failed"
+done
+
 echo "Finished. A JSON manifest of produced images is available at $URI"
diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh
index 948039234..690a38119 100755
--- a/contrib/cirrus/check_image.sh
+++ b/contrib/cirrus/check_image.sh
@@ -16,7 +16,9 @@ MIN_MEM_MB=2000
 read JUNK TOTAL USED MEM_FREE JUNK <<<$(free -tm | tail -1)
 item_test 'Minimum available memory' $MEM_FREE -ge $MIN_MEM_MB || let "RET+=1"
 
-item_test "podman command NOT found on path" -z "$(type -P podman)" || let "RET+=1"
+# We're testing a custom-built podman; make sure there isn't a distro-provided
+# binary anywhere; that could potentially taint our results.
+item_test "remove_packaged_podman_files() did it's job" -z "$(type -P podman)" || let "RET+=1"
 
 MIN_ZIP_VER='3.0'
 VER_RE='.+([[:digit:]]+\.[[:digit:]]+).+'
@@ -34,4 +36,10 @@ do
         "$(systemctl list-unit-files --no-legend $REQ_UNIT)" = "$REQ_UNIT enabled" || let "RET+=1"
 done
 
+# Exits zero if any unit matching pattern is running
+UNIT_STATUS=$(systemctl is-active $EVIL_UNITS; echo $?)
+item_test "No interfering background units are active:" \
+    "$UNIT_STATUS" -ne "0" || let "RET+=1"
+
+echo "Total failed tests: $RET"
 exit $RET
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index ea0f9e326..a9da3f4ce 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -45,6 +45,7 @@ fi
 # Defaults when not running under CI
 export CI="${CI:-false}"
 CIRRUS_CI="${CIRRUS_CI:-false}"
+DEST_BRANCH="${DEST_BRANCH:-master}"
 CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}"
 CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod}
 CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)}  # difficult to reliably discover
@@ -77,6 +78,9 @@ ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.
 # Unsafe env. vars for display
 SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)'
 
+# Names of systemd units which should never be running
+EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean"
+
 SPECIALMODE="${SPECIALMODE:-none}"
 TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}"
 export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
@@ -323,8 +327,15 @@ install_test_configs(){
                            /etc/containers/registries.conf
 }
 
+# Remove all files (except conmon, for now) provided by the distro version of podman.
+# Except conmon, for now as it's expected to eventually be  packaged separately.
+# All VM cache-images used for testing include the distro podman because (1) it's
+# required for podman-in-podman testing and (2) it somewhat simplifies the task
+# of pulling in necessary prerequisites packages as the set can change over time.
+# For general CI testing however, calling this function makes sure the system
+# can only run the compiled source version.
 remove_packaged_podman_files(){
-    show_and_store_warning "Removing packaged podman files to prevent conflicts with source build and testing."
+    echo "Removing packaged podman files to prevent conflicts with source build and testing."
     req_env_var OS_RELEASE_ID
     if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]]
     then
@@ -342,6 +353,20 @@ remove_packaged_podman_files(){
     done
 }
 
+systemd_banish(){
+    echo "Disabling periodic services that could destabilize testing:"
+    set +e  # Not all of these exist on every platform
+    for unit in $EVIL_UNITS
+    do
+        ooe.sh sudo systemctl stop $unit
+        ooe.sh sudo systemctl disable $unit
+        ooe.sh sudo systemctl disable $unit.timer
+        ooe.sh sudo systemctl mask $unit
+        ooe.sh sudo systemctl mask $unit.timer
+    done
+    set -e
+}
+
 _finalize(){
     set +e  # Don't fail at the very end
     set +e  # make errors non-fatal
diff --git a/contrib/cirrus/notice_master_failure.sh b/contrib/cirrus/notice_branch_failure.sh
index 1fc15cdf9..f030c12e5 100755
--- a/contrib/cirrus/notice_master_failure.sh
+++ b/contrib/cirrus/notice_branch_failure.sh
@@ -9,10 +9,10 @@ ETX="$(echo -n -e '\x03')"
 RED="${ETX}4"
 NOR="$(echo -n -e '\x0f')"
 
-if [[ "$CIRRUS_BRANCH" =~ "master" ]]
+if [[ "$CIRRUS_BRANCH" = "$DEST_BRANCH" ]]
 then
     BURL="https://cirrus-ci.com/build/$CIRRUS_BUILD_ID"
-    ircmsg "${RED}[Action Recommended]: ${NOR}Post-merge testing ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on $(OS_RELEASE_ID)-$(OS_RELEASE_VER): $BURL.  Please investigate, and re-run if appropriate."
+    ircmsg "${RED}[Action Recommended]: ${NOR}Post-merge testing on ${RED}$CIRRUS_BRANCH failed${NOR} in $CIRRUS_TASK_NAME on ${OS_RELEASE_ID}-${OS_RELEASE_VER}: $BURL.  Please investigate, and re-run if appropriate."
 fi
 
 # This script assumed to be executed on failure
diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh
index a425b2b57..788a54c34 100644
--- a/contrib/cirrus/packer/fedora_base-setup.sh
+++ b/contrib/cirrus/packer/fedora_base-setup.sh
@@ -27,6 +27,9 @@ ooe.sh systemctl enable rngd
 echo "Setting cloud-init service to start after google-network-daemon.service"
 cp -v $GOSRC/$PACKER_BASE/cloud-init/fedora/cloud-init.service /etc/systemd/system/
 
+# Ensure there are no disruptive periodic services enabled by default in image
+systemd_banish
+
 rh_finalize
 
 echo "SUCCESS!"
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index eb95db907..1e25a1a3c 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -76,6 +76,9 @@ ooe.sh sudo dnf install -y \
     xz \
     zip
 
+# Ensure there are no disruptive periodic services enabled by default in image
+systemd_banish
+
 sudo /tmp/libpod/hack/install_catatonit.sh
 
 rh_finalize
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index 6209f2f89..dba191ad2 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -100,6 +100,9 @@ ooe.sh sudo update-grub
 sudo /tmp/libpod/hack/install_catatonit.sh
 ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo
 
+# Ensure there are no disruptive periodic services enabled by default in image
+systemd_banish
+
 ubuntu_finalize
 
 echo "SUCCESS!"
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index f312e593a..e49bb98fe 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -52,7 +52,9 @@ install_test_configs
 make install.tools
 
 case "$SPECIALMODE" in
-    none) ;;  # Do the normal thing
+    none)
+        remove_packaged_podman_files  # we're building from source
+        ;;
     rootless)
         # Only do this once, even if ROOTLESS_USER (somehow) changes
         if ! grep -q 'ROOTLESS_USER' /etc/environment
@@ -65,9 +67,9 @@ case "$SPECIALMODE" in
                 tee -a /etc/environment) && eval "$X" && echo "$X"
             setup_rootless
         fi
+        remove_packaged_podman_files
         ;;
     in_podman)  # Assumed to be Fedora
-        dnf install -y podman
         $SCRIPT_BASE/setup_container_environment.sh
         ;;
     windows) ;&  # for podman-remote building only
diff --git a/contrib/imgprune/Dockerfile b/contrib/imgprune/Dockerfile
new file mode 100644
index 000000000..26329e828
--- /dev/null
+++ b/contrib/imgprune/Dockerfile
@@ -0,0 +1,7 @@
+FROM libpod/imgts:latest
+
+RUN yum -y update && \
+    yum clean all
+
+COPY /contrib/imgprune/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod 755 /usr/local/bin/entrypoint.sh
diff --git a/contrib/imgprune/README.md b/contrib/imgprune/README.md
new file mode 100644
index 000000000..48abc2028
--- /dev/null
+++ b/contrib/imgprune/README.md
@@ -0,0 +1,11 @@
+![PODMAN logo](../../logo/podman-logo-source.svg)
+
+A container image for maintaining the collection of
+VM images used by CI/CD on this project and several others.
+Acts upon metadata maintained by the imgts container.
+
+Example build (from repository root):
+
+```bash
+sudo podman build -t $IMAGE_NAME -f contrib/imgprune/Dockerfile .
+```
diff --git a/contrib/imgprune/entrypoint.sh b/contrib/imgprune/entrypoint.sh
new file mode 100755
index 000000000..a4b77523b
--- /dev/null
+++ b/contrib/imgprune/entrypoint.sh
@@ -0,0 +1,67 @@
+#!/bin/bash
+
+set -e
+
+source /usr/local/bin/lib_entrypoint.sh
+
+req_env_var GCPJSON GCPNAME GCPPROJECT IMGNAMES
+
+gcloud_init
+
+# For safety's sake + limit nr background processes
+PRUNE_LIMIT=10
+THEFUTURE=$(date --date='+1 hour' +%s)
+TOO_OLD='90 days ago'
+THRESHOLD=$(date --date="$TOO_OLD" +%s)
+# Format Ref: https://cloud.google.com/sdk/gcloud/reference/topic/formats
+FORMAT='value[quote](name,selfLink,creationTimestamp,labels)'
+PROJRE="/v1/projects/$GCPPROJECT/global/"
+BASE_IMAGE_RE='cloud-base'
+RECENTLY=$(date --date='30 days ago' --iso-8601=date)
+EXCLUDE="$IMGNAMES $IMAGE_BUILDER_CACHE_IMAGE_NAME" # whitespace separated values
+# Filter Ref: https://cloud.google.com/sdk/gcloud/reference/topic/filters
+FILTER="selfLink~$PROJRE AND creationTimestamp<$RECENTLY AND NOT name=($EXCLUDE)"
+TODELETE=$(mktemp -p '' todelete.XXXXXX)
+
+echo "Searching images for pruning candidates older than $TOO_OLD ($THRESHOLD):"
+$GCLOUD compute images list --format="$FORMAT" --filter="$FILTER" | \
+    while read name selfLink creationTimestamp labels
+    do
+        created_ymd=$(date --date=$creationTimestamp --iso-8601=date)
+        last_used=$(egrep --only-matching --max-count=1 'last-used=[[:digit:]]+' <<< $labels || true)
+        markmsgpfx="Marking $name (created $created_ymd) for deletion"
+        if [[ -z "$last_used" ]]
+        then # image pre-dates addition of tracking labels
+            echo "$markmsgpfx: Missing 'last-used' metadata, labels: '$labels'"
+            echo "$name" >> $TODELETE
+            continue
+        fi
+
+        last_used_timestamp=$(date --date=@$(cut -d= -f2 <<< $last_used || true) +%s || true)
+        last_used_ymd=$(date --date=@$last_used_timestamp --iso-8601=date)
+        if [[ -z "$last_used_timestamp" ]] || [[ "$last_used_timestamp" -ge "$THEFUTURE" ]]
+        then
+            echo "$markmsgpfx: Missing or invalid last-used timestamp: '$last_used_timestamp'"
+            echo "$name" >> $TODELETE
+            continue
+        fi
+
+        if [[ "$last_used_timestamp" -le "$THRESHOLD" ]]
+        then
+            echo "$markmsgpfx: Used over $TOO_OLD on $last_used_ymd"
+            echo "$name" >> $TODELETE
+            continue
+        fi
+
+        echo "NOT $markmsgpfx: last used on $last_used_ymd)"
+    done
+
+echo "Pruning up to $PRUNE_LIMIT images that were marked for deletion:"
+for image_name in $(tail -$PRUNE_LIMIT $TODELETE | sort --random-sort)
+do
+    # This can take quite some time (minutes), run in parallel disconnected from terminal
+    echo "TODO: Would have: $GCLOUD compute images delete $image_name &"
+    sleep "$[1+RANDOM/1000]s" &  # Simlate background operation
+done
+
+wait || true  # Nothing to delete: No background jobs
diff --git a/contrib/imgts/Dockerfile b/contrib/imgts/Dockerfile
index 0746eca4c..deaadb899 100644
--- a/contrib/imgts/Dockerfile
+++ b/contrib/imgts/Dockerfile
@@ -7,14 +7,14 @@ RUN yum -y update && \
     yum -y install google-cloud-sdk && \
     yum clean all
 
-COPY /contrib/imgts/entrypoint.sh /usr/local/bin/entrypoint.sh
 ENV GCPJSON="__unknown__" \
     GCPNAME="__unknown__" \
     GCPPROJECT="__unknown__" \
     IMGNAMES="__unknown__" \
-    TIMESTAMP="__unknown__" \
     BUILDID="__unknown__" \
     REPOREF="__unknown__"
+
+COPY ["/contrib/imgts/entrypoint.sh", "/contrib/imgts/lib_entrypoint.sh", "/usr/local/bin/"]
 RUN chmod 755 /usr/local/bin/entrypoint.sh
 
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
diff --git a/contrib/imgts/entrypoint.sh b/contrib/imgts/entrypoint.sh
index 610e1f3b6..9c653eda0 100755
--- a/contrib/imgts/entrypoint.sh
+++ b/contrib/imgts/entrypoint.sh
@@ -2,45 +2,22 @@
 
 set -e
 
-RED="\e[1;36;41m"
-YEL="\e[1;33;44m"
-NOR="\e[0m"
+source /usr/local/bin/lib_entrypoint.sh
 
-die() {
-    echo -e "$2" >&2
-    exit "$1"
-}
+req_env_var GCPJSON GCPNAME GCPPROJECT IMGNAMES BUILDID REPOREF
 
-SENTINEL="__unknown__"  # default set in dockerfile
+gcloud_init
 
-[[ "$GCPJSON" != "$SENTINEL" ]] || \
-    die 1 "Must specify service account JSON in \$GCPJSON"
-[[ "$GCPNAME" != "$SENTINEL" ]] || \
-    die 2 "Must specify service account name in \$GCPNAME"
-[[ "$GCPPROJECT" != "$SENTINEL" ]] || \
-    die 4 "Must specify GCP Project ID in \$GCPPROJECT"
-[[ -n "$GCPPROJECT" ]] || \
-    die 5 "Must specify non-empty GCP Project ID in \$GCPPROJECT"
-[[ "$IMGNAMES" != "$SENTINEL" ]] || \
-    die 6 "Must specify space separated list of GCE image names in \$IMGNAMES"
-[[ "$BUILDID" != "$SENTINEL" ]] || \
-    die 7 "Must specify the number of current build in \$BUILDID"
-[[ "$REPOREF" != "$SENTINEL" ]] || \
-    die 8 "Must specify a PR number or Branch name in \$REPOREF"
+ARGS="
+    --update-labels=last-used=$(date +%s)
+    --update-labels=build-id=$BUILDID
+    --update-labels=repo-ref=$REPOREF
+    --update-labels=project=$GCPPROJECT
+"
 
-ARGS="--update-labels=last-used=$(date +%s)"
-# optional
-[[ -z "$BUILDID" ]] || ARGS="$ARGS --update-labels=build-id=$BUILDID"
-[[ -z "$REPOREF" ]] || ARGS="$ARGS --update-labels=repo-ref=$REPOREF"
-[[ -z "$GCPPROJECT" ]] || ARGS="$ARGS --update-labels=project=$GCPPROJECT"
-
-gcloud config set account "$GCPNAME"
-gcloud config set project "$GCPPROJECT"
-echo "$GCPJSON" > /tmp/gcp.json
-gcloud auth activate-service-account --key-file=/tmp/gcp.json || rm /tmp/gcp.json
 for image in $IMGNAMES
 do
-    gcloud compute images update "$image" $ARGS &
+    $GCLOUD compute images update "$image" $ARGS &
 done
-set +e  # Actual update failures are only warnings
-wait || die 0 "${RED}WARNING:$NOR ${YEL}Failed to update labels on one or more images:$NOR '$IMGNAMES'"
+
+wait || echo "Warning: No \$IMGNAMES were specified."
diff --git a/contrib/imgts/lib_entrypoint.sh b/contrib/imgts/lib_entrypoint.sh
new file mode 100644
index 000000000..7b76c823f
--- /dev/null
+++ b/contrib/imgts/lib_entrypoint.sh
@@ -0,0 +1,44 @@
+#!/bin/bash
+
+set -e
+
+RED="\e[1;36;41m"
+YEL="\e[1;33;44m"
+NOR="\e[0m"
+SENTINEL="__unknown__"  # default set in dockerfile
+# Disable all input prompts
+# https://cloud.google.com/sdk/docs/scripting-gcloud
+GCLOUD="gcloud --quiet"
+
+die() {
+    EXIT=$1
+    PFX=$2
+    shift 2
+    MSG="$@"
+    echo -e "${RED}${PFX}:${NOR} ${YEL}$MSG${NOR}"
+    [[ "$EXIT" -eq "0" ]] || exit "$EXIT"
+}
+
+# Pass in a list of one or more envariable names; exit non-zero with
+# helpful error message if any value is empty
+req_env_var() {
+    for i; do
+        if [[ -z "${!i}" ]]
+        then
+            die 1 FATAL entrypoint.sh requires \$$i to be non-empty.
+        elif [[ "${!i}" == "$SENTINEL" ]]
+        then
+            die 2 FATAL entrypoint.sh requires \$$i to be explicitly set.
+        fi
+    done
+}
+
+gcloud_init() {
+    set +xe
+    TMPF=$(mktemp -p '' .$(uuidgen)XXXX)
+    trap "rm -f $TMPF" EXIT
+    echo "$GCPJSON" > $TMPF  && \
+    $GCLOUD auth activate-service-account --project "$GCPPROJECT" --key-file=$TMPF || \
+        die 5 FATAL auth
+    rm -f $TMPF
+}