summaryrefslogtreecommitdiff
path: root/contrib/cirrus
diff options
context:
space:
mode:
authorChris Evich <cevich@redhat.com>2019-08-01 07:31:04 -0400
committerChris Evich <cevich@redhat.com>2019-08-28 11:54:06 -0400
commit370b1a887cbf6db8ac893c39118cf8c6c2fd663c (patch)
tree33fb752adc3957916890103fcc8ff68a57b8a1a5 /contrib/cirrus
parent8e46106f420dfc6125750c12e13c5ae39be9d6f1 (diff)
downloadpodman-370b1a887cbf6db8ac893c39118cf8c6c2fd663c.tar.gz
podman-370b1a887cbf6db8ac893c39118cf8c6c2fd663c.tar.bz2
podman-370b1a887cbf6db8ac893c39118cf8c6c2fd663c.zip
Cirrus: Reimplement release archive + upload
The initial implementation was far more complicated than necessary. Strip out the complexities in favor of a simpler and more direct approach. Signed-off-by: Chris Evich <cevich@redhat.com>
Diffstat (limited to 'contrib/cirrus')
-rwxr-xr-xcontrib/cirrus/build_release.sh30
-rwxr-xr-xcontrib/cirrus/cache_release_archive.sh140
-rwxr-xr-xcontrib/cirrus/cirrus_yaml_test.py43
-rwxr-xr-xcontrib/cirrus/integration_test.sh4
-rw-r--r--contrib/cirrus/lib.sh4
-rwxr-xr-xcontrib/cirrus/setup_environment.sh5
l---------contrib/cirrus/uncache_release_archives.sh1
-rwxr-xr-xcontrib/cirrus/unit_test.sh4
-rwxr-xr-xcontrib/cirrus/upload_release_archive.sh52
9 files changed, 102 insertions, 181 deletions
diff --git a/contrib/cirrus/build_release.sh b/contrib/cirrus/build_release.sh
new file mode 100755
index 000000000..287643f47
--- /dev/null
+++ b/contrib/cirrus/build_release.sh
@@ -0,0 +1,30 @@
+#!/bin/bash
+
+source $(dirname $0)/lib.sh
+
+req_env_var TEST_REMOTE_CLIENT OS_RELEASE_ID GOSRC
+
+cd $GOSRC
+
+if [[ "$TEST_REMOTE_CLIENT" == "true" ]] && [[ -z "$CROSS_PLATFORM" ]]
+then
+ CROSS_PLATFORM=linux
+fi
+
+if [[ -n "$CROSS_PLATFORM" ]]
+then
+ echo "Compiling podman-remote release archive for ${CROSS_PLATFORM}"
+ case "$CROSS_PLATFORM" in
+ linux) ;&
+ windows) ;&
+ darwin)
+ make podman-remote-${CROSS_PLATFORM}-release
+ ;;
+ *)
+ die 1 "Unknown/unsupported cross-compile platform '$CROSS_PLATFORM'"
+ ;;
+ esac
+else
+ echo "Compiling release archive for $OS_RELEASE_ID"
+ make podman-release
+fi
diff --git a/contrib/cirrus/cache_release_archive.sh b/contrib/cirrus/cache_release_archive.sh
deleted file mode 100755
index 2365f7593..000000000
--- a/contrib/cirrus/cache_release_archive.sh
+++ /dev/null
@@ -1,140 +0,0 @@
-#!/bin/bash
-
-set -eo pipefail
-
-source $(dirname $0)/lib.sh
-
-req_env_var GOSRC
-
-RELEASE_ARCHIVE_NAMES=""
-
-handle_archive() { # Assumed to be called with set +e
- TASK_NUMBER=$1
- PR_OR_BRANCH=$2
- CACHE_URL=$3
- ARCHIVE_NAME="$(basename $CACHE_URL)"
- req_env_var TASK_NUMBER PR_OR_BRANCH CACHE_URL ARCHIVE_NAME
-
- cd /tmp
- curl -sO "$CACHE_URL" || return $(warn 0 "Couldn't download file, skipping.")
- [[ -r "/tmp/$ARCHIVE_NAME" ]] || return $(warn 0 "Unreadable archive '/tmp/$ARCHIVE_NAME', skipping.")
-
- ZIPCOMMENT=$(unzip -qqz "$ARCHIVE_NAME" 2>/dev/null) # noisy bugger
- if [[ "$?" -ne "0" ]] || [[ -z "$ZIPCOMMENT" ]]
- then
- return $(warn 0 "Could not unzip metadata from downloaded '/tmp/$ARCHIVE_NAME', skipping.")
- fi
-
- RELEASE_INFO=$(echo "$ZIPCOMMENT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/')
- if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]]
- then
- return $(warn 0 "Metadata empty or invalid: '$ZIPCOMMENT', skipping.")
- fi
-
- # e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64
- # or libpod v1.3.1-166-g60df124e amd64
- FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH"
- read $FIELDS <<< $RELEASE_INFO
- for f in $FIELDS
- do
- [[ -n "${!f}" ]] || return $(warn 0 "Expecting $f to be non-empty in metadata: '$RELEASE_INFO', skipping.")
- done
-
- echo -n "Preparing $RELEASE_BASENAME archive: "
- # Drop version number to enable "latest" representation
- # (version available w/in zip-file comment)
- RELEASE_ARCHIVE_NAME="${RELEASE_BASENAME}-${PR_OR_BRANCH}-${RELEASE_DIST}-${RELEASE_DIST_VER}-${RELEASE_ARCH}.zip"
- # Allow uploading all gathered files in parallel, later with gsutil.
- mv -v "$ARCHIVE_NAME" "/$RELEASE_ARCHIVE_NAME"
- RELEASE_ARCHIVE_NAMES="$RELEASE_ARCHIVE_NAMES $RELEASE_ARCHIVE_NAME"
-}
-
-make_release() {
- ARCHIVE_NAME="$1"
- req_env_var ARCHIVE_NAME
-
- # There's no actual testing of windows/darwin targets yet
- # but we still want to cross-compile and publish binaries
- if [[ "$SPECIALMODE" == "windows" ]] || [[ "$SPECIALMODE" == "darwin" ]]
- then
- RELFILE="podman-remote-${SPECIALMODE}.zip"
- elif [[ "$SPECIALMODE" == "none" ]]
- then
- RELFILE="podman.zip"
- else
- die 55 "$(basename $0) unable to handle \$SPECIALMODE=$SPECIALMODE for $ARCHIVE_NAME"
- fi
- echo "Calling make $RELFILE"
- cd $GOSRC
- make "$RELFILE"
- echo "Renaming archive so it can be identified/downloaded for publishing"
- mv -v "$RELFILE" "$ARCHIVE_NAME"
- echo "Success!"
-}
-
-[[ "$CI" == "true" ]] || \
- die 56 "$0 requires a Cirrus-CI cross-task cache to function"
-
-cd $GOSRC
-# Same script re-used for both uploading and downloading to avoid duplication
-if [[ "$(basename $0)" == "cache_release_archive.sh" ]]
-then
- # ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables
- req_env_var CI_NODE_INDEX CIRRUS_BUILD_ID
- # Use unique names for uncache_release_archives.sh to find/download them all
- ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${CI_NODE_INDEX}.zip"
- make_release "$ARCHIVE_NAME"
-
- # ref: https://cirrus-ci.org/guide/writing-tasks/#http-cache
- URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}"
- echo "Uploading $ARCHIVE_NAME to Cirrus-CI cache at $URL"
- curl -s -X POST --data-binary "@$ARCHIVE_NAME" "$URL"
-elif [[ "$(basename $0)" == "uncache_release_archives.sh" ]]
-then
- req_env_var CIRRUS_BUILD_ID CI_NODE_TOTAL GCPJSON GCPNAME GCPROJECT
- [[ "${CI_NODE_INDEX}" -eq "$[CI_NODE_TOTAL-1]" ]] || \
- die 0 "WARNING: This task depends on cache data from other tasks, otherwise it is a no-op."
-
- if [[ -n "$CIRRUS_PR" ]]
- then
- PR_OR_BRANCH="pr$CIRRUS_PR"
- BUCKET="libpod-pr-releases"
- elif [[ -n "$CIRRUS_BRANCH" ]]
- then
- PR_OR_BRANCH="$CIRRUS_BRANCH"
- BUCKET="libpod-$CIRRUS_BRANCH-releases"
- else
- die 10 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty."
- fi
-
- echo "Blindly downloading Cirrus-CI cache files for task (some will fail)."
- set +e # Don't stop looping until all task's cache is attempted
- for (( task_number = 0 ; task_number < $CI_NODE_TOTAL ; task_number++ ))
- do
- ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${task_number}.zip"
- URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}"
- echo "Attempting to download cached archive from $URL"
- handle_archive "$task_number" "$PR_OR_BRANCH" "$URL"
- echo "----------------------------------------"
- done
- set -e
-
- [[ -n "$RELEASE_ARCHIVE_NAMES" ]] || \
- die 67 "Error: No release archives found in CI cache, expecting at least one."
-
- echo "Preparing to upload release archives."
- gcloud config set project "$GCPROJECT"
- echo "$GCPJSON" > /tmp/gcp.json
- gcloud auth activate-service-account --key-file=/tmp/gcp.json
- rm /tmp/gcp.json
- # handle_archive() placed all uploadable files under /
- gsutil -m cp /*.zip "gs://$BUCKET" # Upload in parallel
- echo "Successfully uploaded archives:"
- for ARCHIVE_NAME in $RELEASE_ARCHIVE_NAMES
- do
- echo " https://storage.cloud.google.com/$BUCKET/$ARCHIVE_NAME"
- done
- echo "These will remain available until automatic pruning by bucket policy."
-else
- die 9 "I don't know what to do when called $0"
-fi
diff --git a/contrib/cirrus/cirrus_yaml_test.py b/contrib/cirrus/cirrus_yaml_test.py
index c8faee65f..c2ff8e69e 100755
--- a/contrib/cirrus/cirrus_yaml_test.py
+++ b/contrib/cirrus/cirrus_yaml_test.py
@@ -26,7 +26,6 @@ class TestCaseBase(unittest.TestCase):
class TestDependsOn(TestCaseBase):
ALL_TASK_NAMES = None
- SUCCESS_RELEASE = set(['success', 'release'])
def setUp(self):
super().setUp()
@@ -34,34 +33,22 @@ class TestDependsOn(TestCaseBase):
for key, _ in self.CIRRUS_YAML.items()
if key.endswith('_task')])
- def test_dicts(self):
+ def test_00_dicts(self):
"""Expected dictionaries are present and non-empty"""
- for name in ('success_task', 'release_task'):
- # tests all names then show specific failures
- with self.subTest(name=name):
- self.assertIn(name, self.CIRRUS_YAML)
- self.assertIn(name.replace('_task', ''), self.ALL_TASK_NAMES)
- self.assertIn('depends_on', self.CIRRUS_YAML[name])
- self.assertGreater(len(self.CIRRUS_YAML[name]['depends_on']), 0)
-
- def _check_dep(self, name, task_name, deps):
- # name includes '_task' suffix, task_name does not
- msg=('Please add "{0}" to the "depends_on" list in "{1}"'
- "".format(task_name, name))
- self.assertIn(task_name, deps, msg=msg)
-
- def test_depends(self):
- """Success and Release tasks depend on all other tasks"""
- for name in ('success_task', 'release_task'):
- deps = set(self.CIRRUS_YAML[name]['depends_on'])
- for task_name in self.ALL_TASK_NAMES - self.SUCCESS_RELEASE:
- with self.subTest(name=name, task_name=task_name):
- self._check_dep(name, task_name, deps)
-
- def test_release(self):
- """Release task must always execute last"""
- deps = set(self.CIRRUS_YAML['release_task']['depends_on'])
- self._check_dep('release_task', 'success', deps)
+ self.assertIn('success_task', self.CIRRUS_YAML)
+ self.assertIn('success_task'.replace('_task', ''), self.ALL_TASK_NAMES)
+ self.assertIn('depends_on', self.CIRRUS_YAML['success_task'])
+ self.assertGreater(len(self.CIRRUS_YAML['success_task']['depends_on']), 0)
+
+ def test_01_depends(self):
+ """Success task depends on all other tasks"""
+ success_deps = set(self.CIRRUS_YAML['success_task']['depends_on'])
+ for task_name in self.ALL_TASK_NAMES - set(['success']):
+ with self.subTest(task_name=task_name):
+ msg=('Please add "{0}" to the "depends_on" list in "success_task"'
+ "".format(task_name))
+ self.assertIn(task_name, success_deps, msg=msg)
+
if __name__ == "__main__":
diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh
index a3d18d440..552f2ba73 100755
--- a/contrib/cirrus/integration_test.sh
+++ b/contrib/cirrus/integration_test.sh
@@ -65,10 +65,6 @@ case "$SPECIALMODE" in
make local${TESTSUITE}
fi
;;
- windows) ;& # for podman-remote building only
- darwin)
- warn '' "No $SPECIALMODE remote client integration tests configured"
- ;;
*)
die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE"
esac
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index a20ee5a62..555f3e717 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -64,6 +64,8 @@ export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849"
export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}"
# IN_PODMAN container image
IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest"
+# Image for uploading releases
+UPLDREL_IMAGE="quay.io/libpod/upldrel:latest"
# Avoid getting stuck waiting for user input
export DEBIAN_FRONTEND="noninteractive"
@@ -76,7 +78,7 @@ BIGTO="timeout_attempt_delay_command 300s 5 30s"
# Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly)
ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(TEST_REMOTE.*)'
# Unsafe env. vars for display
-SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)'
+SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(GC[EP]..+)|(SSH)'
# Names of systemd units which should never be running
EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean"
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 2579229a5..323a05489 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -66,7 +66,8 @@ case "$SPECIALMODE" in
remove_packaged_podman_files # we're building from source
;;
none)
- remove_packaged_podman_files
+ [[ -n "$CROSS_PLATFORM" ]] || \
+ remove_packaged_podman_files
;;
endpoint)
remove_packaged_podman_files
@@ -88,8 +89,6 @@ case "$SPECIALMODE" in
in_podman) # Assumed to be Fedora
$SCRIPT_BASE/setup_container_environment.sh
;;
- windows) ;& # for podman-remote building only
- darwin) ;;
*)
die 111 "Unsupported \$SPECIALMODE: $SPECIALMODE"
esac
diff --git a/contrib/cirrus/uncache_release_archives.sh b/contrib/cirrus/uncache_release_archives.sh
deleted file mode 120000
index e9fc6edff..000000000
--- a/contrib/cirrus/uncache_release_archives.sh
+++ /dev/null
@@ -1 +0,0 @@
-cache_release_archive.sh \ No newline at end of file
diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh
index 004839f17..c6c77d17e 100755
--- a/contrib/cirrus/unit_test.sh
+++ b/contrib/cirrus/unit_test.sh
@@ -16,10 +16,6 @@ case "$SPECIALMODE" in
none)
make
;;
- windows) ;&
- darwin)
- make podman-remote-$SPECIALMODE
- ;;
*)
die 109 "Unsupported \$SPECIAL_MODE: $SPECIALMODE"
esac
diff --git a/contrib/cirrus/upload_release_archive.sh b/contrib/cirrus/upload_release_archive.sh
new file mode 100755
index 000000000..942255821
--- /dev/null
+++ b/contrib/cirrus/upload_release_archive.sh
@@ -0,0 +1,52 @@
+#!/bin/bash
+
+set -eo pipefail
+
+source $(dirname $0)/lib.sh
+
+req_env_var CI UPLDREL_IMAGE CIRRUS_BUILD_ID GOSRC RELEASE_GCPJSON RELEASE_GCPNAME RELEASE_GCPROJECT
+
+[[ "$CI" == "true" ]] || \
+ die 56 "$0 must be run under Cirrus-CI to function"
+
+unset PR_OR_BRANCH BUCKET
+if [[ -n "$CIRRUS_PR" ]]
+then
+ PR_OR_BRANCH="pr$CIRRUS_PR"
+ BUCKET="libpod-pr-releases"
+elif [[ -n "$CIRRUS_BRANCH" ]]
+then
+ PR_OR_BRANCH="$CIRRUS_BRANCH"
+ BUCKET="libpod-$CIRRUS_BRANCH-releases"
+else
+ die 1 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty."
+fi
+
+# Functional local podman required for uploading a release
+cd $GOSRC
+[[ -n "$(type -P podman)" ]] || \
+ make install || \
+ die 57 "$0 requires working podman binary on path to function"
+
+TMPF=$(mktemp -p '' $(basename $0)_XXXX.json)
+trap "rm -f $TMPF" EXIT
+set +x
+echo "$RELEASE_GCPJSON" > "$TMPF"
+unset RELEASE_GCPJSON
+
+cd $GOSRC
+for filename in $(ls -1 *.tar.gz *.zip)
+do
+ echo "Running podman ... $UPLDREL_IMAGE $filename"
+ podman run -i --rm \
+ -e "GCPNAME=$RELEASE_GCPNAME" \
+ -e "GCPPROJECT=$RELEASE_GCPROJECT" \
+ -e "GCPJSON_FILEPATH=$TMPF" \
+ -e "REL_ARC_FILEPATH=/tmp/$filename" \
+ -e "PR_OR_BRANCH=$PR_OR_BRANCH" \
+ -e "BUCKET=$BUCKET" \
+ --security-opt label=disable \
+ -v "$TMPF:$TMPF:ro" \
+ -v "$GOSRC/$filename:/tmp/$filename:ro" \
+ $UPLDREL_IMAGE
+done