diff options
| -rw-r--r-- | .cirrus.yml | 21 | ||||
| -rw-r--r-- | contrib/imgprune/Dockerfile | 7 | ||||
| -rw-r--r-- | contrib/imgprune/README.md | 11 | ||||
| -rwxr-xr-x | contrib/imgprune/entrypoint.sh | 106 | ||||
| -rw-r--r-- | contrib/imgts/Dockerfile | 20 | ||||
| -rw-r--r-- | contrib/imgts/README.md | 11 | ||||
| -rwxr-xr-x | contrib/imgts/entrypoint.sh | 23 | ||||
| -rw-r--r-- | contrib/imgts/google-cloud-sdk.repo | 8 | ||||
| -rw-r--r-- | contrib/imgts/lib_entrypoint.sh | 49 | ||||
| -rw-r--r-- | contrib/upldrel/Dockerfile | 9 | ||||
| -rw-r--r-- | contrib/upldrel/README.md | 9 | ||||
| -rwxr-xr-x | contrib/upldrel/entrypoint.sh | 27 |
12 files changed, 10 insertions, 291 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index d2a8cb660..6f8cd0a30 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -34,16 +34,16 @@ env: #### #### Cache-image names to test with (double-quotes around names are critical) ### - FEDORA_NAME: "fedora" - PRIOR_FEDORA_NAME: "prior-fedora" - UBUNTU_NAME: "ubuntu" - PRIOR_UBUNTU_NAME: "prior-ubuntu" + FEDORA_NAME: "fedora-32" + PRIOR_FEDORA_NAME: "fedora-31" + UBUNTU_NAME: "ubuntu-20" + PRIOR_UBUNTU_NAME: "ubuntu-19" - _BUILT_IMAGE_SUFFIX: "c6110627968057344" - FEDORA_CACHE_IMAGE_NAME: "${FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}" - PRIOR_FEDORA_CACHE_IMAGE_NAME: "${PRIOR_FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}" - UBUNTU_CACHE_IMAGE_NAME: "${UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}" - PRIOR_UBUNTU_CACHE_IMAGE_NAME: "${PRIOR_UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}" + _BUILT_IMAGE_SUFFIX: "c4948709391728640" + FEDORA_CACHE_IMAGE_NAME: "fedora-${_BUILT_IMAGE_SUFFIX}" + PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${_BUILT_IMAGE_SUFFIX}" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${_BUILT_IMAGE_SUFFIX}" + PRIOR_UBUNTU_CACHE_IMAGE_NAME: "prior-ubuntu-${_BUILT_IMAGE_SUFFIX}" #### #### Default to NOT operating in any special-case testing mode @@ -290,7 +290,7 @@ build_without_cgo_task: meta_task: container: - image: "quay.io/libpod/imgts:master" # see contrib/imgts + image: "quay.io/libpod/imgts:${_BUILT_IMAGE_SUFFIX}" cpu: 1 memory: 1 @@ -301,7 +301,6 @@ meta_task: ${PRIOR_FEDORA_CACHE_IMAGE_NAME} ${UBUNTU_CACHE_IMAGE_NAME} ${PRIOR_UBUNTU_CACHE_IMAGE_NAME} - ${IMAGE_BUILDER_CACHE_IMAGE_NAME} BUILDID: "${CIRRUS_BUILD_ID}" REPOREF: "${CIRRUS_CHANGE_IN_REPO}" GCPJSON: ENCRYPTED[3a198350077849c8df14b723c0f4c9fece9ebe6408d35982e7adf2105a33f8e0e166ed3ed614875a0887e1af2b8775f4] diff --git a/contrib/imgprune/Dockerfile b/contrib/imgprune/Dockerfile deleted file mode 100644 index b0dc77da5..000000000 --- a/contrib/imgprune/Dockerfile +++ /dev/null @@ -1,7 +0,0 @@ -FROM quay.io/libpod/imgts:latest - -RUN yum -y update && \ - yum clean all - -COPY /contrib/imgprune/entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod 755 /usr/local/bin/entrypoint.sh diff --git a/contrib/imgprune/README.md b/contrib/imgprune/README.md deleted file mode 100644 index 48abc2028..000000000 --- a/contrib/imgprune/README.md +++ /dev/null @@ -1,11 +0,0 @@ - - -A container image for maintaining the collection of -VM images used by CI/CD on this project and several others. -Acts upon metadata maintained by the imgts container. - -Example build (from repository root): - -```bash -sudo podman build -t $IMAGE_NAME -f contrib/imgprune/Dockerfile . -``` diff --git a/contrib/imgprune/entrypoint.sh b/contrib/imgprune/entrypoint.sh deleted file mode 100755 index fd80d9b26..000000000 --- a/contrib/imgprune/entrypoint.sh +++ /dev/null @@ -1,106 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source /usr/local/bin/lib_entrypoint.sh - -req_env_var GCPJSON GCPNAME GCPPROJECT IMGNAMES - -unset BASE_IMAGES -# When executing under Cirrus-CI, script have access to current source -LIB="$CIRRUS_WORKING_DIR/$SCRIPT_BASE/lib.sh" -if [[ "$CI" == "true" ]] && [[ -r "$LIB" ]] -then - # Avoid importing anything that might conflict - for env in $(sed -ne 's/^[^#]\+_BASE_IMAGE=/img=/p' "$LIB") - do - eval $env - BASE_IMAGES="$BASE_IMAGES $img" - done -else - # metadata labeling may have broken for some reason in the future - echo "Warning: Running outside of Cirrus-CI, very minor-risk of base-image deletion." -fi - -gcloud_init - -# For safety's sake + limit nr background processes -PRUNE_LIMIT=5 -THEFUTURE=$(date --date='+1 hour' +%s) -TOO_OLD='30 days ago' -THRESHOLD=$(date --date="$TOO_OLD" +%s) -# Format Ref: https://cloud.google.com/sdk/gcloud/reference/topic/formats -FORMAT='value[quote](name,selfLink,creationTimestamp,labels)' -PROJRE="/v1/projects/$GCPPROJECT/global/" -RECENTLY=$(date --date='3 days ago' --iso-8601=date) -# Filter Ref: https://cloud.google.com/sdk/gcloud/reference/topic/filters -FILTER="selfLink~$PROJRE AND creationTimestamp<$RECENTLY AND NOT name=($IMGNAMES $BASE_IMAGES)" -TODELETE=$(mktemp -p '' todelete.XXXXXX) -IMGCOUNT=$(mktemp -p '' imgcount.XXXXXX) - -# Search-loop runs in a sub-process, must store count in file -echo "0" > "$IMGCOUNT" -count_image() { - local count - count=$(<"$IMGCOUNT") - let 'count+=1' - echo "$count" > "$IMGCOUNT" -} - -echo "Using filter: $FILTER" -echo "Searching images for pruning candidates older than $TOO_OLD ($(date --date="$TOO_OLD" --iso-8601=date)):" -$GCLOUD compute images list --format="$FORMAT" --filter="$FILTER" | \ - while read name selfLink creationTimestamp labels - do - count_image - created_ymd=$(date --date=$creationTimestamp --iso-8601=date) - last_used=$(egrep --only-matching --max-count=1 'last-used=[[:digit:]]+' <<< $labels || true) - markmsgpfx="Marking $name (created $created_ymd) for deletion" - if [[ -z "$last_used" ]] - then # image pre-dates addition of tracking labels - echo "$markmsgpfx: Missing 'last-used' metadata, labels: '$labels'" - echo "$name" >> $TODELETE - continue - fi - - last_used_timestamp=$(date --date=@$(cut -d= -f2 <<< $last_used || true) +%s || true) - last_used_ymd=$(date --date=@$last_used_timestamp --iso-8601=date) - if [[ -z "$last_used_timestamp" ]] || [[ "$last_used_timestamp" -ge "$THEFUTURE" ]] - then - echo "$markmsgpfx: Missing or invalid last-used timestamp: '$last_used_timestamp'" - echo "$name" >> $TODELETE - continue - fi - - if [[ "$last_used_timestamp" -le "$THRESHOLD" ]] - then - echo "$markmsgpfx: Used over $TOO_OLD on $last_used_ymd" - echo "$name" >> $TODELETE - continue - fi - done - -COUNT=$(<"$IMGCOUNT") -echo "########################################################################" -echo "Deleting up to $PRUNE_LIMIT images marked ($(wc -l < $TODELETE)) of all searched ($COUNT):" - -# Require a minimum number of images to exist -NEED="$[$PRUNE_LIMIT*2]" -if [[ "$COUNT" -lt "$NEED" ]] -then - die 0 Safety-net Insufficient images \($COUNT\) to process deletions \($NEED\) - exit 0 -fi - -for image_name in $(sort --random-sort $TODELETE | tail -$PRUNE_LIMIT) -do - if echo "$IMGNAMES $BASE_IMAGES" | grep -q "$image_name" - then - # double-verify in-use images were filtered out in search loop above - die 8 FATAL ATTEMPT TO DELETE IN-USE IMAGE \'$image_name\' - THIS SHOULD NEVER HAPPEN - fi - echo "Deleting $image_name in parallel..." - $GCLOUD compute images delete $image_name & -done - -wait || true # Nothing to delete: No background jobs diff --git a/contrib/imgts/Dockerfile b/contrib/imgts/Dockerfile deleted file mode 100644 index deaadb899..000000000 --- a/contrib/imgts/Dockerfile +++ /dev/null @@ -1,20 +0,0 @@ -FROM centos:7 - -# Only needed for installing build-time dependencies -COPY /contrib/imgts/google-cloud-sdk.repo /etc/yum.repos.d/google-cloud-sdk.repo -RUN yum -y update && \ - yum -y install epel-release && \ - yum -y install google-cloud-sdk && \ - yum clean all - -ENV GCPJSON="__unknown__" \ - GCPNAME="__unknown__" \ - GCPPROJECT="__unknown__" \ - IMGNAMES="__unknown__" \ - BUILDID="__unknown__" \ - REPOREF="__unknown__" - -COPY ["/contrib/imgts/entrypoint.sh", "/contrib/imgts/lib_entrypoint.sh", "/usr/local/bin/"] -RUN chmod 755 /usr/local/bin/entrypoint.sh - -ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] diff --git a/contrib/imgts/README.md b/contrib/imgts/README.md deleted file mode 100644 index ad5ed4172..000000000 --- a/contrib/imgts/README.md +++ /dev/null @@ -1,11 +0,0 @@ - - -A container image for tracking automation metadata. -Currently this is used to update last-used timestamps on -VM images. - -Example build (from repository root): - -```bash -sudo podman build -t $IMAGE_NAME -f contrib/imgts/Dockerfile . -``` diff --git a/contrib/imgts/entrypoint.sh b/contrib/imgts/entrypoint.sh deleted file mode 100755 index b089e1e9b..000000000 --- a/contrib/imgts/entrypoint.sh +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source /usr/local/bin/lib_entrypoint.sh - -req_env_var GCPJSON GCPNAME GCPPROJECT IMGNAMES BUILDID REPOREF - -gcloud_init - -ARGS=" - --update-labels=last-used=$(date +%s) - --update-labels=build-id=$BUILDID - --update-labels=repo-ref=$REPOREF - --update-labels=project=$GCPPROJECT -" - -for image in $IMGNAMES -do - $GCLOUD compute images update "$image" $ARGS & -done - -wait || echo "Warning: No \$IMGNAMES were specified." diff --git a/contrib/imgts/google-cloud-sdk.repo b/contrib/imgts/google-cloud-sdk.repo deleted file mode 100644 index 45b1e43bb..000000000 --- a/contrib/imgts/google-cloud-sdk.repo +++ /dev/null @@ -1,8 +0,0 @@ -[google-cloud-sdk] -name=Google Cloud SDK -baseurl=https://packages.cloud.google.com/yum/repos/cloud-sdk-el7-x86_64 -enabled=1 -gpgcheck=1 -repo_gpgcheck=1 -gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg - https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg diff --git a/contrib/imgts/lib_entrypoint.sh b/contrib/imgts/lib_entrypoint.sh deleted file mode 100644 index 6eb5cdc2f..000000000 --- a/contrib/imgts/lib_entrypoint.sh +++ /dev/null @@ -1,49 +0,0 @@ -#!/usr/bin/env bash - -set -e - -RED="\e[1;36;41m" -YEL="\e[1;33;44m" -NOR="\e[0m" -SENTINEL="__unknown__" # default set in dockerfile -# Disable all input prompts -# https://cloud.google.com/sdk/docs/scripting-gcloud -GCLOUD="gcloud --quiet" - -die() { - EXIT=$1 - PFX=$2 - shift 2 - MSG="$@" - echo -e "${RED}${PFX}:${NOR} ${YEL}$MSG${NOR}" - [[ "$EXIT" -eq "0" ]] || exit "$EXIT" -} - -# Pass in a list of one or more envariable names; exit non-zero with -# helpful error message if any value is empty -req_env_var() { - for i; do - if [[ -z "${!i}" ]] - then - die 1 FATAL entrypoint.sh requires \$$i to be non-empty. - elif [[ "${!i}" == "$SENTINEL" ]] - then - die 2 FATAL entrypoint.sh requires \$$i to be explicitly set. - fi - done -} - -gcloud_init() { - set +xe - if [[ -n "$1" ]] && [[ -r "$1" ]] - then - TMPF="$1" - else - TMPF=$(mktemp -p '' .$(uuidgen)_XXXX.json) - trap "rm -f $TMPF &> /dev/null" EXIT - echo "$GCPJSON" > $TMPF - fi - $GCLOUD auth activate-service-account --project="$GCPPROJECT" --key-file="$TMPF" || \ - die 5 FATAL auth - rm -f $TMPF &> /dev/null || true # ignore any read-only error -} diff --git a/contrib/upldrel/Dockerfile b/contrib/upldrel/Dockerfile deleted file mode 100644 index 54a58c521..000000000 --- a/contrib/upldrel/Dockerfile +++ /dev/null @@ -1,9 +0,0 @@ -FROM quay.io/libpod/imgts:latest - -RUN yum -y update && \ - yum -y install unzip && \ - rpm -V unzip && \ - yum clean all - -COPY /contrib/upldrel/entrypoint.sh /usr/local/bin/entrypoint.sh -RUN chmod 755 /usr/local/bin/entrypoint.sh diff --git a/contrib/upldrel/README.md b/contrib/upldrel/README.md deleted file mode 100644 index 41f5ffef0..000000000 --- a/contrib/upldrel/README.md +++ /dev/null @@ -1,9 +0,0 @@ - - -A container image for canonical-naming and uploading of -libpod and remote-client archives. Only intended to ever -be used by CI/CD, and depends heavily on an embedded -`release.txt` file produced by `make`. - -Build script: [../cirrus/build_release.sh](../cirrus/build_release.sh) -Upload script: [../cirrus/upload_release_archive.sh](../cirrus/upload_release_archive.sh) diff --git a/contrib/upldrel/entrypoint.sh b/contrib/upldrel/entrypoint.sh deleted file mode 100755 index 6eb1b8f94..000000000 --- a/contrib/upldrel/entrypoint.sh +++ /dev/null @@ -1,27 +0,0 @@ -#!/usr/bin/env bash - -set -e - -source /usr/local/bin/lib_entrypoint.sh - -req_env_var GCPJSON_FILEPATH GCPNAME GCPPROJECT BUCKET FROM_FILEPATH TO_FILENAME - -[[ -r "$FROM_FILEPATH" ]] || \ - die 2 ERROR Cannot read release archive file: "$FROM_FILEPATH" - -[[ -r "$GCPJSON_FILEPATH" ]] || \ - die 3 ERROR Cannot read GCP credentials file: "$GCPJSON_FILEPATH" - -echo "Authenticating to google cloud for upload" -gcloud_init "$GCPJSON_FILEPATH" - -echo "Uploading archive as $TO_FILENAME" -gsutil cp "$FROM_FILEPATH" "gs://$BUCKET/$TO_FILENAME" -[[ -z "$ALSO_FILENAME" ]] || \ - gsutil cp "$FROM_FILEPATH" "gs://$BUCKET/$ALSO_FILENAME" - -echo "." -echo "Release now available for download at:" -echo " https://storage.googleapis.com/$BUCKET/$TO_FILENAME" -[[ -z "$ALSO_FILENAME" ]] || \ - echo " https://storage.googleapis.com/$BUCKET/$ALSO_FILENAME" |