Merge pull request #3710 from cevich/release_redo

Release redo

13 files changed, 192 insertions, 186 deletions

diff --git a/contrib/cirrus/build_release.sh b/contrib/cirrus/build_release.sh
new file mode 100755
index 000000000..287643f47
--- /dev/null
+++ b/contrib/cirrus/build_release.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+source $(dirname $0)/lib.sh
+
+req_env_var TEST_REMOTE_CLIENT OS_RELEASE_ID GOSRC
+
+cd $GOSRC
+
+if [[ "$TEST_REMOTE_CLIENT" == "true" ]] && [[ -z "$CROSS_PLATFORM" ]]
+then
+    CROSS_PLATFORM=linux
+fi
+
+if [[ -n "$CROSS_PLATFORM" ]]
+then
+    echo "Compiling podman-remote release archive for ${CROSS_PLATFORM}"
+    case "$CROSS_PLATFORM" in
+        linux) ;&
+        windows) ;&
+        darwin)
+            make podman-remote-${CROSS_PLATFORM}-release
+            ;;
+        *)
+            die 1 "Unknown/unsupported cross-compile platform '$CROSS_PLATFORM'"
+            ;;
+    esac
+else
+    echo "Compiling release archive for $OS_RELEASE_ID"
+    make podman-release
+fi
diff --git a/contrib/cirrus/cache_release_archive.sh b/contrib/cirrus/cache_release_archive.sh
deleted file mode 100755
index 2365f7593..000000000
--- a/contrib/cirrus/cache_release_archive.sh
+++ /dev/null
@@ -1,140 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-source $(dirname $0)/lib.sh
-
-req_env_var GOSRC
-
-RELEASE_ARCHIVE_NAMES=""
-
-handle_archive() {  # Assumed to be called with set +e
-        TASK_NUMBER=$1
-        PR_OR_BRANCH=$2
-        CACHE_URL=$3
-        ARCHIVE_NAME="$(basename $CACHE_URL)"
-        req_env_var TASK_NUMBER PR_OR_BRANCH CACHE_URL ARCHIVE_NAME
-
-        cd /tmp
-        curl -sO "$CACHE_URL" || return $(warn 0 "Couldn't download file, skipping.")
-        [[ -r "/tmp/$ARCHIVE_NAME" ]] || return $(warn 0 "Unreadable archive '/tmp/$ARCHIVE_NAME', skipping.")
-
-        ZIPCOMMENT=$(unzip -qqz "$ARCHIVE_NAME" 2>/dev/null)  # noisy bugger
-        if [[ "$?" -ne "0" ]] || [[ -z "$ZIPCOMMENT" ]]
-        then
-            return $(warn 0 "Could not unzip metadata from downloaded '/tmp/$ARCHIVE_NAME', skipping.")
-        fi
-
-        RELEASE_INFO=$(echo "$ZIPCOMMENT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/')
-        if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]]
-        then
-            return $(warn 0 "Metadata empty or invalid: '$ZIPCOMMENT', skipping.")
-        fi
-
-        # e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64
-        # or   libpod v1.3.1-166-g60df124e   amd64
-        FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH"
-        read $FIELDS <<< $RELEASE_INFO
-        for f in $FIELDS
-        do
-            [[ -n "${!f}" ]] || return $(warn 0 "Expecting $f to be non-empty in metadata: '$RELEASE_INFO', skipping.")
-        done
-
-        echo -n "Preparing $RELEASE_BASENAME archive: "
-        # Drop version number to enable "latest" representation
-        # (version available w/in zip-file comment)
-        RELEASE_ARCHIVE_NAME="${RELEASE_BASENAME}-${PR_OR_BRANCH}-${RELEASE_DIST}-${RELEASE_DIST_VER}-${RELEASE_ARCH}.zip"
-        # Allow uploading all gathered files in parallel, later with gsutil.
-        mv -v "$ARCHIVE_NAME" "/$RELEASE_ARCHIVE_NAME"
-        RELEASE_ARCHIVE_NAMES="$RELEASE_ARCHIVE_NAMES $RELEASE_ARCHIVE_NAME"
-}
-
-make_release() {
-    ARCHIVE_NAME="$1"
-    req_env_var ARCHIVE_NAME
-
-    # There's no actual testing of windows/darwin targets yet
-    # but we still want to cross-compile and publish binaries
-    if [[ "$SPECIALMODE" == "windows" ]] || [[ "$SPECIALMODE" == "darwin" ]]
-    then
-        RELFILE="podman-remote-${SPECIALMODE}.zip"
-    elif [[ "$SPECIALMODE" == "none" ]]
-    then
-        RELFILE="podman.zip"
-    else
-        die 55 "$(basename $0) unable to handle \$SPECIALMODE=$SPECIALMODE for $ARCHIVE_NAME"
-    fi
-    echo "Calling make $RELFILE"
-    cd $GOSRC
-    make "$RELFILE"
-    echo "Renaming archive so it can be identified/downloaded for publishing"
-    mv -v "$RELFILE" "$ARCHIVE_NAME"
-    echo "Success!"
-}
-
-[[ "$CI" == "true" ]] || \
-    die 56 "$0 requires a Cirrus-CI cross-task cache to function"
-
-cd $GOSRC
-# Same script re-used for both uploading and downloading to avoid duplication
-if [[ "$(basename $0)" == "cache_release_archive.sh" ]]
-then
-    # ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables
-    req_env_var CI_NODE_INDEX CIRRUS_BUILD_ID
-    # Use unique names for uncache_release_archives.sh to find/download them all
-    ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${CI_NODE_INDEX}.zip"
-    make_release "$ARCHIVE_NAME"
-
-    # ref: https://cirrus-ci.org/guide/writing-tasks/#http-cache
-    URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}"
-    echo "Uploading $ARCHIVE_NAME to Cirrus-CI cache at $URL"
-    curl -s -X POST --data-binary "@$ARCHIVE_NAME" "$URL"
-elif [[ "$(basename $0)" == "uncache_release_archives.sh" ]]
-then
-    req_env_var CIRRUS_BUILD_ID CI_NODE_TOTAL GCPJSON GCPNAME GCPROJECT
-    [[ "${CI_NODE_INDEX}" -eq  "$[CI_NODE_TOTAL-1]" ]] || \
-        die 0 "WARNING: This task depends on cache data from other tasks, otherwise it is a no-op."
-
-    if [[ -n "$CIRRUS_PR" ]]
-    then
-        PR_OR_BRANCH="pr$CIRRUS_PR"
-        BUCKET="libpod-pr-releases"
-    elif [[ -n "$CIRRUS_BRANCH" ]]
-    then
-        PR_OR_BRANCH="$CIRRUS_BRANCH"
-        BUCKET="libpod-$CIRRUS_BRANCH-releases"
-    else
-        die 10 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty."
-    fi
-
-    echo "Blindly downloading Cirrus-CI cache files for task (some will fail)."
-    set +e  # Don't stop looping until all task's cache is attempted
-    for (( task_number = 0 ; task_number < $CI_NODE_TOTAL ; task_number++ ))
-    do
-        ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${task_number}.zip"
-        URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}"
-        echo "Attempting to download cached archive from $URL"
-        handle_archive "$task_number" "$PR_OR_BRANCH" "$URL"
-        echo "----------------------------------------"
-    done
-    set -e
-
-    [[ -n "$RELEASE_ARCHIVE_NAMES" ]] || \
-        die 67 "Error: No release archives found in CI cache, expecting at least one."
-
-    echo "Preparing to upload release archives."
-    gcloud config set project "$GCPROJECT"
-    echo "$GCPJSON" > /tmp/gcp.json
-    gcloud auth activate-service-account --key-file=/tmp/gcp.json
-    rm /tmp/gcp.json
-    # handle_archive() placed all uploadable files under /
-    gsutil -m cp /*.zip "gs://$BUCKET"  # Upload in parallel
-    echo "Successfully uploaded archives:"
-    for ARCHIVE_NAME in $RELEASE_ARCHIVE_NAMES
-    do
-        echo "    https://storage.cloud.google.com/$BUCKET/$ARCHIVE_NAME"
-    done
-    echo "These will remain available until automatic pruning by bucket policy."
-else
-    die 9 "I don't know what to do when called $0"
-fi
diff --git a/contrib/cirrus/cirrus_yaml_test.py b/contrib/cirrus/cirrus_yaml_test.py
index c8faee65f..c2ff8e69e 100755
--- a/contrib/cirrus/cirrus_yaml_test.py
+++ b/contrib/cirrus/cirrus_yaml_test.py
@@ -26,7 +26,6 @@ class TestCaseBase(unittest.TestCase):
 class TestDependsOn(TestCaseBase):
 
     ALL_TASK_NAMES = None
-    SUCCESS_RELEASE = set(['success', 'release'])
 
     def setUp(self):
         super().setUp()
@@ -34,34 +33,22 @@ class TestDependsOn(TestCaseBase):
                                    for key, _ in self.CIRRUS_YAML.items()
                                    if key.endswith('_task')])
 
-    def test_dicts(self):
+    def test_00_dicts(self):
         """Expected dictionaries are present and non-empty"""
-        for name in ('success_task', 'release_task'):
-            # tests all names then show specific failures
-            with self.subTest(name=name):
-                self.assertIn(name, self.CIRRUS_YAML)
-                self.assertIn(name.replace('_task', ''), self.ALL_TASK_NAMES)
-                self.assertIn('depends_on', self.CIRRUS_YAML[name])
-                self.assertGreater(len(self.CIRRUS_YAML[name]['depends_on']), 0)
-
-    def _check_dep(self, name, task_name, deps):
-        # name includes '_task' suffix, task_name does not
-        msg=('Please add "{0}" to the "depends_on" list in "{1}"'
-             "".format(task_name, name))
-        self.assertIn(task_name, deps, msg=msg)
-
-    def test_depends(self):
-        """Success and Release tasks depend on all other tasks"""
-        for name in ('success_task', 'release_task'):
-            deps = set(self.CIRRUS_YAML[name]['depends_on'])
-            for task_name in self.ALL_TASK_NAMES - self.SUCCESS_RELEASE:
-                with self.subTest(name=name, task_name=task_name):
-                    self._check_dep(name, task_name, deps)
-
-    def test_release(self):
-        """Release task must always execute last"""
-        deps = set(self.CIRRUS_YAML['release_task']['depends_on'])
-        self._check_dep('release_task', 'success', deps)
+        self.assertIn('success_task', self.CIRRUS_YAML)
+        self.assertIn('success_task'.replace('_task', ''), self.ALL_TASK_NAMES)
+        self.assertIn('depends_on', self.CIRRUS_YAML['success_task'])
+        self.assertGreater(len(self.CIRRUS_YAML['success_task']['depends_on']), 0)
+
+    def test_01_depends(self):
+        """Success task depends on all other tasks"""
+        success_deps = set(self.CIRRUS_YAML['success_task']['depends_on'])
+        for task_name in self.ALL_TASK_NAMES - set(['success']):
+            with self.subTest(task_name=task_name):
+                msg=('Please add "{0}" to the "depends_on" list in "success_task"'
+                     "".format(task_name))
+                self.assertIn(task_name, success_deps, msg=msg)
+
 
 
 if __name__ == "__main__":
diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh
index a3d18d440..552f2ba73 100755
--- a/contrib/cirrus/integration_test.sh
+++ b/contrib/cirrus/integration_test.sh
@@ -65,10 +65,6 @@ case "$SPECIALMODE" in
             make local${TESTSUITE}
         fi
         ;;
-    windows) ;&  # for podman-remote building only
-    darwin)
-        warn '' "No $SPECIALMODE remote client integration tests configured"
-        ;;
     *)
         die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE"
 esac
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 47fee878a..f66e63140 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -64,6 +64,8 @@ export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849"
 export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}"
 # IN_PODMAN container image
 IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest"
+# Image for uploading releases
+UPLDREL_IMAGE="quay.io/libpod/upldrel:latest"
 
 # Avoid getting stuck waiting for user input
 export DEBIAN_FRONTEND="noninteractive"
@@ -76,7 +78,7 @@ BIGTO="timeout_attempt_delay_command 300s 5 30s"
 # Safe env. vars. to transfer from root -> $ROOTLESS_USER  (go env handled separetly)
 ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(TEST_REMOTE.*)'
 # Unsafe env. vars for display
-SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)'
+SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(GC[EP]..+)|(SSH)'
 
 # Names of systemd units which should never be running
 EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean"
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 463647d2f..7c7659169 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -69,7 +69,8 @@ case "$SPECIALMODE" in
         remove_packaged_podman_files  # we're building from source
         ;;
     none)
-        remove_packaged_podman_files
+        [[ -n "$CROSS_PLATFORM" ]] || \
+            remove_packaged_podman_files
         ;;
     endpoint)
         remove_packaged_podman_files
@@ -91,8 +92,6 @@ case "$SPECIALMODE" in
     in_podman)  # Assumed to be Fedora
         $SCRIPT_BASE/setup_container_environment.sh
         ;;
-    windows) ;&  # for podman-remote building only
-    darwin) ;;
     *)
         die 111 "Unsupported \$SPECIALMODE: $SPECIALMODE"
 esac
diff --git a/contrib/cirrus/uncache_release_archives.sh b/contrib/cirrus/uncache_release_archives.sh
deleted file mode 120000
index e9fc6edff..000000000
--- a/contrib/cirrus/uncache_release_archives.sh
+++ /dev/null
@@ -1 +0,0 @@
-cache_release_archive.sh
\ No newline at end of file
diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh
index 004839f17..c6c77d17e 100755
--- a/contrib/cirrus/unit_test.sh
+++ b/contrib/cirrus/unit_test.sh
@@ -16,10 +16,6 @@ case "$SPECIALMODE" in
     none)
         make
         ;;
-    windows) ;&
-    darwin)
-        make podman-remote-$SPECIALMODE
-        ;;
     *)
         die 109 "Unsupported \$SPECIAL_MODE: $SPECIALMODE"
 esac
diff --git a/contrib/cirrus/upload_release_archive.sh b/contrib/cirrus/upload_release_archive.sh
new file mode 100755
index 000000000..942255821
--- /dev/null
+++ b/contrib/cirrus/upload_release_archive.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -eo pipefail
+
+source $(dirname $0)/lib.sh
+
+req_env_var CI UPLDREL_IMAGE CIRRUS_BUILD_ID GOSRC RELEASE_GCPJSON RELEASE_GCPNAME RELEASE_GCPROJECT
+
+[[ "$CI" == "true" ]] || \
+    die 56 "$0 must be run under Cirrus-CI to function"
+
+unset PR_OR_BRANCH BUCKET
+if [[ -n "$CIRRUS_PR" ]]
+then
+    PR_OR_BRANCH="pr$CIRRUS_PR"
+    BUCKET="libpod-pr-releases"
+elif [[ -n "$CIRRUS_BRANCH" ]]
+then
+    PR_OR_BRANCH="$CIRRUS_BRANCH"
+    BUCKET="libpod-$CIRRUS_BRANCH-releases"
+else
+    die 1 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty."
+fi
+
+# Functional local podman required for uploading a release
+cd $GOSRC
+[[ -n "$(type -P podman)" ]] || \
+    make install || \
+    die 57 "$0 requires working podman binary on path to function"
+
+TMPF=$(mktemp -p '' $(basename $0)_XXXX.json)
+trap "rm -f $TMPF" EXIT
+set +x
+echo "$RELEASE_GCPJSON" > "$TMPF"
+unset RELEASE_GCPJSON
+
+cd $GOSRC
+for filename in $(ls -1 *.tar.gz *.zip)
+do
+    echo "Running podman ... $UPLDREL_IMAGE $filename"
+    podman run -i --rm \
+        -e "GCPNAME=$RELEASE_GCPNAME" \
+        -e "GCPPROJECT=$RELEASE_GCPROJECT" \
+        -e "GCPJSON_FILEPATH=$TMPF" \
+        -e "REL_ARC_FILEPATH=/tmp/$filename" \
+        -e "PR_OR_BRANCH=$PR_OR_BRANCH" \
+        -e "BUCKET=$BUCKET" \
+        --security-opt label=disable \
+        -v "$TMPF:$TMPF:ro" \
+        -v "$GOSRC/$filename:/tmp/$filename:ro" \
+        $UPLDREL_IMAGE
+done
diff --git a/contrib/imgts/lib_entrypoint.sh b/contrib/imgts/lib_entrypoint.sh
index 7b76c823f..3f6b11128 100644
--- a/contrib/imgts/lib_entrypoint.sh
+++ b/contrib/imgts/lib_entrypoint.sh
@@ -35,10 +35,15 @@ req_env_var() {
 
 gcloud_init() {
     set +xe
-    TMPF=$(mktemp -p '' .$(uuidgen)XXXX)
-    trap "rm -f $TMPF" EXIT
-    echo "$GCPJSON" > $TMPF  && \
-    $GCLOUD auth activate-service-account --project "$GCPPROJECT" --key-file=$TMPF || \
+    if [[ -n "$1" ]] && [[ -r "$1" ]]
+    then
+        TMPF="$1"
+    else
+        TMPF=$(mktemp -p '' .$(uuidgen)_XXXX.json)
+        trap "rm -f $TMPF &> /dev/null" EXIT
+        echo "$GCPJSON" > $TMPF
+    fi
+    $GCLOUD auth activate-service-account --project="$GCPPROJECT" --key-file="$TMPF" || \
         die 5 FATAL auth
-    rm -f $TMPF
+    rm -f $TMPF &> /dev/null || true  # ignore any read-only error
 }
diff --git a/contrib/upldrel/Dockerfile b/contrib/upldrel/Dockerfile
new file mode 100644
index 000000000..54a58c521
--- /dev/null
+++ b/contrib/upldrel/Dockerfile
@@ -0,0 +1,9 @@
+FROM quay.io/libpod/imgts:latest
+
+RUN yum -y update && \
+    yum -y install unzip && \
+    rpm -V unzip && \
+    yum clean all
+
+COPY /contrib/upldrel/entrypoint.sh /usr/local/bin/entrypoint.sh
+RUN chmod 755 /usr/local/bin/entrypoint.sh
diff --git a/contrib/upldrel/README.md b/contrib/upldrel/README.md
new file mode 100644
index 000000000..41f5ffef0
--- /dev/null
+++ b/contrib/upldrel/README.md
@@ -0,0 +1,9 @@
+![PODMAN logo](../../logo/podman-logo-source.svg)
+
+A container image for canonical-naming and uploading of
+libpod and remote-client archives.  Only intended to ever
+be used by CI/CD, and depends heavily on an embedded
+`release.txt` file produced by `make`.
+
+Build  script: [../cirrus/build_release.sh](../cirrus/build_release.sh)
+Upload script: [../cirrus/upload_release_archive.sh](../cirrus/upload_release_archive.sh)
diff --git a/contrib/upldrel/entrypoint.sh b/contrib/upldrel/entrypoint.sh
new file mode 100755
index 000000000..985b828a0
--- /dev/null
+++ b/contrib/upldrel/entrypoint.sh
@@ -0,0 +1,62 @@
+#!/bin/bash
+
+set -e
+
+source /usr/local/bin/lib_entrypoint.sh
+
+req_env_var GCPJSON_FILEPATH GCPNAME GCPPROJECT REL_ARC_FILEPATH PR_OR_BRANCH BUCKET
+
+[[ -r "$REL_ARC_FILEPATH" ]] || \
+    die 2 ERROR Cannot read release archive file: "$REL_ARC_FILEPATH"
+
+[[ -r "$GCPJSON_FILEPATH" ]] || \
+    die 3 ERROR Cannot read GCP credentials file: "$GCPJSON_FILEPATH"
+
+cd $TMPDIR
+echo "Attempting to extract release.txt from tar or zip $REL_ARC_FILEPATH"
+unset SFX
+if tar xzf "$REL_ARC_FILEPATH" "./release.txt"
+then
+    echo "It's a tarball"
+    SFX="tar.gz"
+elif unzip "$REL_ARC_FILEPATH" release.txt
+then
+    echo "It's a zip"
+    SFX="zip"
+else
+    die 5 ERROR Could not extract release.txt from $REL_ARC_FILEPATH
+fi
+
+echo "Parsing release.txt contents"
+RELEASETXT=$(<release.txt)
+cd -
+[[ -n "$RELEASETXT" ]] || \
+    die 3 ERROR Could not obtain metadata from release.txt in $REL_ARC_FILEPATH
+
+RELEASE_INFO=$(echo "$RELEASETXT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/')
+if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]]
+then
+    die 4 ERROR Metadata is empty or invalid: '$RELEASETXT'
+fi
+
+# e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64
+# or   libpod v1.3.1-166-g60df124e   amd64
+FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH"
+read $FIELDS <<< $RELEASE_INFO
+for f in $FIELDS
+do
+    [[ -n "${!f}" ]] || \
+        die 5 ERROR Expecting $f to be non-empty in metadata: '$RELEASE_INFO'
+done
+
+gcloud_init "$GCPJSON_FILEPATH"
+
+# Drop version number to enable "latest" representation
+# (version available w/in zip-file comment)
+RELEASE_ARCHIVE_NAME="${RELEASE_BASENAME}-${PR_OR_BRANCH}-${RELEASE_DIST}-${RELEASE_DIST_VER}-${RELEASE_ARCH}.${SFX}"
+
+echo "Uploading archive as $RELEASE_ARCHIVE_NAME"
+gsutil cp "$REL_ARC_FILEPATH" "gs://$BUCKET/$RELEASE_ARCHIVE_NAME"
+
+echo "Release now available at:"
+echo "    https://storage.cloud.google.com/$BUCKET/$RELEASE_ARCHIVE_NAME"