diff options
Diffstat (limited to 'contrib/cirrus')
| -rwxr-xr-x | contrib/cirrus/build_release.sh | 30 | ||||
| -rwxr-xr-x | contrib/cirrus/cache_release_archive.sh | 140 | ||||
| -rwxr-xr-x | contrib/cirrus/cirrus_yaml_test.py | 43 | ||||
| -rwxr-xr-x | contrib/cirrus/integration_test.sh | 4 | ||||
| -rw-r--r-- | contrib/cirrus/lib.sh | 4 | ||||
| -rwxr-xr-x | contrib/cirrus/setup_environment.sh | 5 | ||||
| l--------- | contrib/cirrus/uncache_release_archives.sh | 1 | ||||
| -rwxr-xr-x | contrib/cirrus/unit_test.sh | 4 | ||||
| -rwxr-xr-x | contrib/cirrus/upload_release_archive.sh | 52 |
9 files changed, 102 insertions, 181 deletions
diff --git a/contrib/cirrus/build_release.sh b/contrib/cirrus/build_release.sh new file mode 100755 index 000000000..287643f47 --- /dev/null +++ b/contrib/cirrus/build_release.sh @@ -0,0 +1,30 @@ +#!/bin/bash + +source $(dirname $0)/lib.sh + +req_env_var TEST_REMOTE_CLIENT OS_RELEASE_ID GOSRC + +cd $GOSRC + +if [[ "$TEST_REMOTE_CLIENT" == "true" ]] && [[ -z "$CROSS_PLATFORM" ]] +then + CROSS_PLATFORM=linux +fi + +if [[ -n "$CROSS_PLATFORM" ]] +then + echo "Compiling podman-remote release archive for ${CROSS_PLATFORM}" + case "$CROSS_PLATFORM" in + linux) ;& + windows) ;& + darwin) + make podman-remote-${CROSS_PLATFORM}-release + ;; + *) + die 1 "Unknown/unsupported cross-compile platform '$CROSS_PLATFORM'" + ;; + esac +else + echo "Compiling release archive for $OS_RELEASE_ID" + make podman-release +fi diff --git a/contrib/cirrus/cache_release_archive.sh b/contrib/cirrus/cache_release_archive.sh deleted file mode 100755 index 2365f7593..000000000 --- a/contrib/cirrus/cache_release_archive.sh +++ /dev/null @@ -1,140 +0,0 @@ -#!/bin/bash - -set -eo pipefail - -source $(dirname $0)/lib.sh - -req_env_var GOSRC - -RELEASE_ARCHIVE_NAMES="" - -handle_archive() { # Assumed to be called with set +e - TASK_NUMBER=$1 - PR_OR_BRANCH=$2 - CACHE_URL=$3 - ARCHIVE_NAME="$(basename $CACHE_URL)" - req_env_var TASK_NUMBER PR_OR_BRANCH CACHE_URL ARCHIVE_NAME - - cd /tmp - curl -sO "$CACHE_URL" || return $(warn 0 "Couldn't download file, skipping.") - [[ -r "/tmp/$ARCHIVE_NAME" ]] || return $(warn 0 "Unreadable archive '/tmp/$ARCHIVE_NAME', skipping.") - - ZIPCOMMENT=$(unzip -qqz "$ARCHIVE_NAME" 2>/dev/null) # noisy bugger - if [[ "$?" -ne "0" ]] || [[ -z "$ZIPCOMMENT" ]] - then - return $(warn 0 "Could not unzip metadata from downloaded '/tmp/$ARCHIVE_NAME', skipping.") - fi - - RELEASE_INFO=$(echo "$ZIPCOMMENT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/') - if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]] - then - return $(warn 0 "Metadata empty or invalid: '$ZIPCOMMENT', skipping.") - fi - - # e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64 - # or libpod v1.3.1-166-g60df124e amd64 - FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH" - read $FIELDS <<< $RELEASE_INFO - for f in $FIELDS - do - [[ -n "${!f}" ]] || return $(warn 0 "Expecting $f to be non-empty in metadata: '$RELEASE_INFO', skipping.") - done - - echo -n "Preparing $RELEASE_BASENAME archive: " - # Drop version number to enable "latest" representation - # (version available w/in zip-file comment) - RELEASE_ARCHIVE_NAME="${RELEASE_BASENAME}-${PR_OR_BRANCH}-${RELEASE_DIST}-${RELEASE_DIST_VER}-${RELEASE_ARCH}.zip" - # Allow uploading all gathered files in parallel, later with gsutil. - mv -v "$ARCHIVE_NAME" "/$RELEASE_ARCHIVE_NAME" - RELEASE_ARCHIVE_NAMES="$RELEASE_ARCHIVE_NAMES $RELEASE_ARCHIVE_NAME" -} - -make_release() { - ARCHIVE_NAME="$1" - req_env_var ARCHIVE_NAME - - # There's no actual testing of windows/darwin targets yet - # but we still want to cross-compile and publish binaries - if [[ "$SPECIALMODE" == "windows" ]] || [[ "$SPECIALMODE" == "darwin" ]] - then - RELFILE="podman-remote-${SPECIALMODE}.zip" - elif [[ "$SPECIALMODE" == "none" ]] - then - RELFILE="podman.zip" - else - die 55 "$(basename $0) unable to handle \$SPECIALMODE=$SPECIALMODE for $ARCHIVE_NAME" - fi - echo "Calling make $RELFILE" - cd $GOSRC - make "$RELFILE" - echo "Renaming archive so it can be identified/downloaded for publishing" - mv -v "$RELFILE" "$ARCHIVE_NAME" - echo "Success!" -} - -[[ "$CI" == "true" ]] || \ - die 56 "$0 requires a Cirrus-CI cross-task cache to function" - -cd $GOSRC -# Same script re-used for both uploading and downloading to avoid duplication -if [[ "$(basename $0)" == "cache_release_archive.sh" ]] -then - # ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables - req_env_var CI_NODE_INDEX CIRRUS_BUILD_ID - # Use unique names for uncache_release_archives.sh to find/download them all - ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${CI_NODE_INDEX}.zip" - make_release "$ARCHIVE_NAME" - - # ref: https://cirrus-ci.org/guide/writing-tasks/#http-cache - URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}" - echo "Uploading $ARCHIVE_NAME to Cirrus-CI cache at $URL" - curl -s -X POST --data-binary "@$ARCHIVE_NAME" "$URL" -elif [[ "$(basename $0)" == "uncache_release_archives.sh" ]] -then - req_env_var CIRRUS_BUILD_ID CI_NODE_TOTAL GCPJSON GCPNAME GCPROJECT - [[ "${CI_NODE_INDEX}" -eq "$[CI_NODE_TOTAL-1]" ]] || \ - die 0 "WARNING: This task depends on cache data from other tasks, otherwise it is a no-op." - - if [[ -n "$CIRRUS_PR" ]] - then - PR_OR_BRANCH="pr$CIRRUS_PR" - BUCKET="libpod-pr-releases" - elif [[ -n "$CIRRUS_BRANCH" ]] - then - PR_OR_BRANCH="$CIRRUS_BRANCH" - BUCKET="libpod-$CIRRUS_BRANCH-releases" - else - die 10 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty." - fi - - echo "Blindly downloading Cirrus-CI cache files for task (some will fail)." - set +e # Don't stop looping until all task's cache is attempted - for (( task_number = 0 ; task_number < $CI_NODE_TOTAL ; task_number++ )) - do - ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${task_number}.zip" - URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}" - echo "Attempting to download cached archive from $URL" - handle_archive "$task_number" "$PR_OR_BRANCH" "$URL" - echo "----------------------------------------" - done - set -e - - [[ -n "$RELEASE_ARCHIVE_NAMES" ]] || \ - die 67 "Error: No release archives found in CI cache, expecting at least one." - - echo "Preparing to upload release archives." - gcloud config set project "$GCPROJECT" - echo "$GCPJSON" > /tmp/gcp.json - gcloud auth activate-service-account --key-file=/tmp/gcp.json - rm /tmp/gcp.json - # handle_archive() placed all uploadable files under / - gsutil -m cp /*.zip "gs://$BUCKET" # Upload in parallel - echo "Successfully uploaded archives:" - for ARCHIVE_NAME in $RELEASE_ARCHIVE_NAMES - do - echo " https://storage.cloud.google.com/$BUCKET/$ARCHIVE_NAME" - done - echo "These will remain available until automatic pruning by bucket policy." -else - die 9 "I don't know what to do when called $0" -fi diff --git a/contrib/cirrus/cirrus_yaml_test.py b/contrib/cirrus/cirrus_yaml_test.py index c8faee65f..c2ff8e69e 100755 --- a/contrib/cirrus/cirrus_yaml_test.py +++ b/contrib/cirrus/cirrus_yaml_test.py @@ -26,7 +26,6 @@ class TestCaseBase(unittest.TestCase): class TestDependsOn(TestCaseBase): ALL_TASK_NAMES = None - SUCCESS_RELEASE = set(['success', 'release']) def setUp(self): super().setUp() @@ -34,34 +33,22 @@ class TestDependsOn(TestCaseBase): for key, _ in self.CIRRUS_YAML.items() if key.endswith('_task')]) - def test_dicts(self): + def test_00_dicts(self): """Expected dictionaries are present and non-empty""" - for name in ('success_task', 'release_task'): - # tests all names then show specific failures - with self.subTest(name=name): - self.assertIn(name, self.CIRRUS_YAML) - self.assertIn(name.replace('_task', ''), self.ALL_TASK_NAMES) - self.assertIn('depends_on', self.CIRRUS_YAML[name]) - self.assertGreater(len(self.CIRRUS_YAML[name]['depends_on']), 0) - - def _check_dep(self, name, task_name, deps): - # name includes '_task' suffix, task_name does not - msg=('Please add "{0}" to the "depends_on" list in "{1}"' - "".format(task_name, name)) - self.assertIn(task_name, deps, msg=msg) - - def test_depends(self): - """Success and Release tasks depend on all other tasks""" - for name in ('success_task', 'release_task'): - deps = set(self.CIRRUS_YAML[name]['depends_on']) - for task_name in self.ALL_TASK_NAMES - self.SUCCESS_RELEASE: - with self.subTest(name=name, task_name=task_name): - self._check_dep(name, task_name, deps) - - def test_release(self): - """Release task must always execute last""" - deps = set(self.CIRRUS_YAML['release_task']['depends_on']) - self._check_dep('release_task', 'success', deps) + self.assertIn('success_task', self.CIRRUS_YAML) + self.assertIn('success_task'.replace('_task', ''), self.ALL_TASK_NAMES) + self.assertIn('depends_on', self.CIRRUS_YAML['success_task']) + self.assertGreater(len(self.CIRRUS_YAML['success_task']['depends_on']), 0) + + def test_01_depends(self): + """Success task depends on all other tasks""" + success_deps = set(self.CIRRUS_YAML['success_task']['depends_on']) + for task_name in self.ALL_TASK_NAMES - set(['success']): + with self.subTest(task_name=task_name): + msg=('Please add "{0}" to the "depends_on" list in "success_task"' + "".format(task_name)) + self.assertIn(task_name, success_deps, msg=msg) + if __name__ == "__main__": diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index a3d18d440..552f2ba73 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -65,10 +65,6 @@ case "$SPECIALMODE" in make local${TESTSUITE} fi ;; - windows) ;& # for podman-remote building only - darwin) - warn '' "No $SPECIALMODE remote client integration tests configured" - ;; *) die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE" esac diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 47fee878a..f66e63140 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -64,6 +64,8 @@ export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849" export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" # IN_PODMAN container image IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest" +# Image for uploading releases +UPLDREL_IMAGE="quay.io/libpod/upldrel:latest" # Avoid getting stuck waiting for user input export DEBIAN_FRONTEND="noninteractive" @@ -76,7 +78,7 @@ BIGTO="timeout_attempt_delay_command 300s 5 30s" # Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly) ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(TEST_REMOTE.*)' # Unsafe env. vars for display -SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' +SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(GC[EP]..+)|(SSH)' # Names of systemd units which should never be running EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean" diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 463647d2f..7c7659169 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -69,7 +69,8 @@ case "$SPECIALMODE" in remove_packaged_podman_files # we're building from source ;; none) - remove_packaged_podman_files + [[ -n "$CROSS_PLATFORM" ]] || \ + remove_packaged_podman_files ;; endpoint) remove_packaged_podman_files @@ -91,8 +92,6 @@ case "$SPECIALMODE" in in_podman) # Assumed to be Fedora $SCRIPT_BASE/setup_container_environment.sh ;; - windows) ;& # for podman-remote building only - darwin) ;; *) die 111 "Unsupported \$SPECIALMODE: $SPECIALMODE" esac diff --git a/contrib/cirrus/uncache_release_archives.sh b/contrib/cirrus/uncache_release_archives.sh deleted file mode 120000 index e9fc6edff..000000000 --- a/contrib/cirrus/uncache_release_archives.sh +++ /dev/null @@ -1 +0,0 @@ -cache_release_archive.sh
\ No newline at end of file diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh index 004839f17..c6c77d17e 100755 --- a/contrib/cirrus/unit_test.sh +++ b/contrib/cirrus/unit_test.sh @@ -16,10 +16,6 @@ case "$SPECIALMODE" in none) make ;; - windows) ;& - darwin) - make podman-remote-$SPECIALMODE - ;; *) die 109 "Unsupported \$SPECIAL_MODE: $SPECIALMODE" esac diff --git a/contrib/cirrus/upload_release_archive.sh b/contrib/cirrus/upload_release_archive.sh new file mode 100755 index 000000000..942255821 --- /dev/null +++ b/contrib/cirrus/upload_release_archive.sh @@ -0,0 +1,52 @@ +#!/bin/bash + +set -eo pipefail + +source $(dirname $0)/lib.sh + +req_env_var CI UPLDREL_IMAGE CIRRUS_BUILD_ID GOSRC RELEASE_GCPJSON RELEASE_GCPNAME RELEASE_GCPROJECT + +[[ "$CI" == "true" ]] || \ + die 56 "$0 must be run under Cirrus-CI to function" + +unset PR_OR_BRANCH BUCKET +if [[ -n "$CIRRUS_PR" ]] +then + PR_OR_BRANCH="pr$CIRRUS_PR" + BUCKET="libpod-pr-releases" +elif [[ -n "$CIRRUS_BRANCH" ]] +then + PR_OR_BRANCH="$CIRRUS_BRANCH" + BUCKET="libpod-$CIRRUS_BRANCH-releases" +else + die 1 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty." +fi + +# Functional local podman required for uploading a release +cd $GOSRC +[[ -n "$(type -P podman)" ]] || \ + make install || \ + die 57 "$0 requires working podman binary on path to function" + +TMPF=$(mktemp -p '' $(basename $0)_XXXX.json) +trap "rm -f $TMPF" EXIT +set +x +echo "$RELEASE_GCPJSON" > "$TMPF" +unset RELEASE_GCPJSON + +cd $GOSRC +for filename in $(ls -1 *.tar.gz *.zip) +do + echo "Running podman ... $UPLDREL_IMAGE $filename" + podman run -i --rm \ + -e "GCPNAME=$RELEASE_GCPNAME" \ + -e "GCPPROJECT=$RELEASE_GCPROJECT" \ + -e "GCPJSON_FILEPATH=$TMPF" \ + -e "REL_ARC_FILEPATH=/tmp/$filename" \ + -e "PR_OR_BRANCH=$PR_OR_BRANCH" \ + -e "BUCKET=$BUCKET" \ + --security-opt label=disable \ + -v "$TMPF:$TMPF:ro" \ + -v "$GOSRC/$filename:/tmp/$filename:ro" \ + $UPLDREL_IMAGE +done |