diff options
Diffstat (limited to 'contrib')
| -rw-r--r-- | contrib/cirrus/README.md | 36 | ||||
| -rwxr-xr-x | contrib/cirrus/cache_release_archive.sh | 140 | ||||
| -rwxr-xr-x | contrib/cirrus/check_image.sh | 4 | ||||
| -rwxr-xr-x | contrib/cirrus/integration_test.sh | 72 | ||||
| -rw-r--r-- | contrib/cirrus/lib.sh | 19 | ||||
| -rwxr-xr-x | contrib/cirrus/rootless_test.sh | 2 | ||||
| -rwxr-xr-x | contrib/cirrus/setup_environment.sh | 8 | ||||
| l--------- | contrib/cirrus/uncache_release_archives.sh | 1 | ||||
| -rwxr-xr-x | contrib/cirrus/unit_test.sh | 15 |
9 files changed, 251 insertions, 46 deletions
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index 5ff4f290f..18ef3e7f7 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -13,7 +13,6 @@ which alter this behavior. Within each task, each script executes in sequence, so long as any previous script exited successfully. The overall state of each task (pass or fail) is set based on the exit status of the last script to execute. - ### ``gating`` Task ***N/B: Steps below are performed by automation*** @@ -64,6 +63,12 @@ task (pass or fail) is set based on the exit status of the last script to execut but this script normally completes in less than an hour. +### ``special_testing_cross`` Task + +Confirm that cross-compile of podman-remote functions for both `windows` +and `darwin` targets. + + ### ``test_build_cache_images_task`` Task Modifying the contents of cache-images is tested by making changes to @@ -142,8 +147,22 @@ the magic ``***CIRRUS: TEST IMAGES***`` string. Keeping it and `--force` pushing would needlessly cause Cirrus-CI to build and test images again. +### `release` Task + +Gathers up zip files uploaded by other tasks, from the local Cirrus-CI caching service. +Depending on the execution context (a PR or a branch), this task uploads the files +found to storage buckets at: + +* [https://storage.cloud.google.com/libpod-pr-releases](https://storage.cloud.google.com/libpod-pr-releases) +* [https://storage.cloud.google.com/libpod-master-releases](https://storage.cloud.google.com/libpod-master-releases) -### Base-images +***Note:*** Repeated builds from the same PR or branch, will clobber previous archives + *by design*. This is intended so that the "latest" archive is always + available at a consistent URL. The precise details regarding a particular + build is encoded within the zip-archive comment. + + +## Base-images Base-images are VM disk-images specially prepared for executing as GCE VMs. In particular, they run services on startup similar in purpose/function @@ -236,3 +255,16 @@ console output. Simply set the ``TTYDEV`` parameter, for example: $ make libpod_base_images ... TTYDEV=$(tty) ... ``` + +## `$SPECIALMODE` + +Some tasks alter their behavior based on this value. A summary of supported +values follows: + +* `none`: Operate as normal, this is the default value if unspecified. +* `rootless`: Causes a random, ordinary user account to be created + and utilized for testing. +* `in_podman`: Causes testing to occur within a container executed by + podman on the host. +* `windows`: See **darwin** +* `darwin`: Signals the ``special_testing_cross`` task to cross-compile the remote client. diff --git a/contrib/cirrus/cache_release_archive.sh b/contrib/cirrus/cache_release_archive.sh new file mode 100755 index 000000000..639bc9801 --- /dev/null +++ b/contrib/cirrus/cache_release_archive.sh @@ -0,0 +1,140 @@ +#!/bin/bash + +set -eo pipefail + +source $(dirname $0)/lib.sh + +req_env_var GOSRC + +RELEASE_ARCHIVE_NAMES="" + +handle_archive() { # Assumed to be called with set +e + TASK_NUMBER=$1 + PR_OR_BRANCH=$2 + CACHE_URL=$3 + ARCHIVE_NAME="$(basename $CACHE_URL)" + req_env_var TASK_NUMBER PR_OR_BRANCH CACHE_URL ARCHIVE_NAME + + cd /tmp + curl -sO "$CACHE_URL" || return $(warn 0 "Couldn't download file, skipping.") + [[ -r "/tmp/$ARCHIVE_NAME" ]] || return $(warn 0 "Unreadable archive '/tmp/$ARCHIVE_NAME', skipping.") + + ZIPCOMMENT=$(unzip -qqz "$ARCHIVE_NAME" 2>/dev/null) # noisy bugger + if [[ "$?" -ne "0" ]] || [[ -z "$ZIPCOMMENT" ]] + then + return $(warn 0 "Could not unzip metadata from downloaded '/tmp/$ARCHIVE_NAME', skipping.") + fi + + RELEASE_INFO=$(echo "$ZIPCOMMENT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/') + if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]] + then + return $(warn 0 "Metadata empty or invalid: '$ZIPCOMMENT', skipping.") + fi + + # e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64 + # or libpod v1.3.1-166-g60df124e amd64 + FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH" + read $FIELDS <<< $RELEASE_INFO + for f in $FIELDS + do + [[ -n "${!f}" ]] || return $(warn 0 "Expecting $f to be non-empty in metadata: '$RELEASE_INFO', skipping.") + done + + echo -n "Preparing $RELEASE_BASENAME archive: " + # Drop version number to enable "latest" representation + # (version available w/in zip-file comment) + RELEASE_ARCHIVE_NAME="${RELEASE_BASENAME}-${PR_OR_BRANCH}-${RELEASE_DIST}-${RELEASE_DIST_VER}-${RELEASE_ARCH}.zip" + # Allow uploading all gathered files in parallel, later with gsutil. + mv -v "$ARCHIVE_NAME" "/$RELEASE_ARCHIVE_NAME" + RELEASE_ARCHIVE_NAMES="$RELEASE_ARCHIVE_NAMES $RELEASE_ARCHIVE_NAME" +} + +make_release() { + ARCHIVE_NAME="$1" + req_env_var ARCHIVE_NAME + + # There's no actual testing of windows/darwin targets yet + # but we still want to cross-compile and publish binaries + if [[ "$SPECIALMODE" == "windows" ]] || [[ "$SPECIALMODE" == "darwin" ]] + then + RELFILE="podman-remote-${SPECIALMODE}.zip" + elif [[ "$SPECIALMODE" == "none" ]] + then + RELFILE="podman.zip" + else + die 55 "$(basename $0) unable to handle \$SPECIALMODE=$SPECIALMODE for $ARCHIVE_NAME" + fi + echo "Calling make $RELFILE" + cd $GOSRC + make "$RELFILE" + echo "Renaming archive so it can be identified/downloaded for publishing" + mv -v "$RELFILE" "$ARCHIVE_NAME" + echo "Success!" +} + +[[ "$CI" == "true" ]] || \ + die 56 "$0 requires a Cirrus-CI cross-task cache to function" + +cd $GOSRC +# Same script re-used for both uploading and downloading to avoid duplication +if [[ "$(basename $0)" == "cache_release_archive.sh" ]] +then + # ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables + req_env_var CI_NODE_INDEX CIRRUS_BUILD_ID + # Use unique names for uncache_release_archives.sh to find/download them all + ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${CI_NODE_INDEX}.zip" + make_release "$ARCHIVE_NAME" + + # ref: https://cirrus-ci.org/guide/writing-tasks/#http-cache + URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}" + echo "Uploading $ARCHIVE_NAME to Cirrus-CI cache at $URL" + curl -s -X POST --data-binary "@$ARCHIVE_NAME" "$URL" +elif [[ "$(basename $0)" == "uncache_release_archives.sh" ]] +then + req_env_var CIRRUS_BUILD_ID CI_NODE_TOTAL GCPJSON GCPNAME GCPROJECT + [[ "${CI_NODE_INDEX}" -eq "$[CI_NODE_TOTAL-1]" ]] || \ + die 8 "The release task must be executed last to guarantee archive cache is complete" + + if [[ -n "$CIRRUS_PR" ]] + then + PR_OR_BRANCH="pr$CIRRUS_PR" + BUCKET="libpod-pr-releases" + elif [[ -n "$CIRRUS_BRANCH" ]] + then + PR_OR_BRANCH="$CIRRUS_BRANCH" + BUCKET="libpod-$CIRRUS_BRANCH-releases" + else + die 10 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty." + fi + + echo "Blindly downloading Cirrus-CI cache files for task (some will fail)." + set +e # Don't stop looping until all task's cache is attempted + for (( task_number = 0 ; task_number < $CI_NODE_TOTAL ; task_number++ )) + do + ARCHIVE_NAME="build-${CIRRUS_BUILD_ID}-task-${task_number}.zip" + URL="http://$CIRRUS_HTTP_CACHE_HOST/${ARCHIVE_NAME}" + echo "Attempting to download cached archive from $URL" + handle_archive "$task_number" "$PR_OR_BRANCH" "$URL" + echo "----------------------------------------" + done + set -e + + [[ -n "$RELEASE_ARCHIVE_NAMES" ]] || \ + die 67 "Error: No release archives found in CI cache, expecting at least one." + + echo "Preparing to upload release archives." + gcloud config set project "$GCPROJECT" + echo "$GCPJSON" > /tmp/gcp.json + gcloud auth activate-service-account --key-file=/tmp/gcp.json + rm /tmp/gcp.json + # handle_archive() placed all uploadable files under / + gsutil -m cp /*.zip "gs://$BUCKET" # Upload in parallel + echo "Successfully uploaded archives:" + for ARCHIVE_NAME in $RELEASE_ARCHIVE_NAMES + do + echo " https://storage.cloud.google.com/$BUCKET/$ARCHIVE_NAME" + done + echo "These will remain available until automatic pruning by bucket policy." +else + die 9 "I don't know what to do when called $0" +fi diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh index 948039234..67e807d61 100755 --- a/contrib/cirrus/check_image.sh +++ b/contrib/cirrus/check_image.sh @@ -16,7 +16,9 @@ MIN_MEM_MB=2000 read JUNK TOTAL USED MEM_FREE JUNK <<<$(free -tm | tail -1) item_test 'Minimum available memory' $MEM_FREE -ge $MIN_MEM_MB || let "RET+=1" -item_test "podman command NOT found on path" -z "$(type -P podman)" || let "RET+=1" +# We're testing a custom-built podman; make sure there isn't a distro-provided +# binary anywhere; that could potentially taint our results. +item_test "remove_packaged_podman_files() did it's job" -z "$(type -P podman)" || let "RET+=1" MIN_ZIP_VER='3.0' VER_RE='.+([[:digit:]]+\.[[:digit:]]+).+' diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index 959bf3c43..cfaf33b85 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -16,39 +16,41 @@ fi cd "$GOSRC" -if [[ "$SPECIALMODE" == "in_podman" ]] -then - ${CONTAINER_RUNTIME} run --rm --privileged --net=host \ - -v $GOSRC:$GOSRC:Z \ - --workdir $GOSRC \ - -e "CGROUP_MANAGER=cgroupfs" \ - -e "STORAGE_OPTIONS=--storage-driver=vfs" \ - -e "CRIO_ROOT=$GOSRC" \ - -e "PODMAN_BINARY=/usr/bin/podman" \ - -e "CONMON_BINARY=/usr/libexec/podman/conmon" \ - -e "DIST=$OS_RELEASE_ID" \ - -e "CONTAINER_RUNTIME=$CONTAINER_RUNTIME" \ - $IN_PODMAN_IMAGE bash $GOSRC/$SCRIPT_BASE/container_test.sh -b -i -t -elif [[ "$SPECIALMODE" == "rootless" ]] -then - req_env_var ROOTLESS_USER - - if [[ "$USER" == "$ROOTLESS_USER" ]] - then - $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} - else +case "$SPECIALMODE" in + in_podman) + ${CONTAINER_RUNTIME} run --rm --privileged --net=host \ + -v $GOSRC:$GOSRC:Z \ + --workdir $GOSRC \ + -e "CGROUP_MANAGER=cgroupfs" \ + -e "STORAGE_OPTIONS=--storage-driver=vfs" \ + -e "CRIO_ROOT=$GOSRC" \ + -e "PODMAN_BINARY=/usr/bin/podman" \ + -e "CONMON_BINARY=/usr/libexec/podman/conmon" \ + -e "DIST=$OS_RELEASE_ID" \ + -e "CONTAINER_RUNTIME=$CONTAINER_RUNTIME" \ + $IN_PODMAN_IMAGE bash $GOSRC/$SCRIPT_BASE/container_test.sh -b -i -t + ;; + rootless) + req_env_var ROOTLESS_USER ssh $ROOTLESS_USER@localhost \ - -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \ - $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} - fi -else - make - make install PREFIX=/usr ETCDIR=/etc - make test-binaries - if [[ "$TEST_REMOTE_CLIENT" == "true" ]] - then - make remote${TESTSUITE} - else - make local${TESTSUITE} - fi -fi + -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \ + -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} + ;; + none) + make + make install PREFIX=/usr ETCDIR=/etc + make test-binaries + if [[ "$TEST_REMOTE_CLIENT" == "true" ]] + then + make remote${TESTSUITE} + else + make local${TESTSUITE} + fi + ;; + windows) ;& # for podman-remote building only + darwin) + warn '' "No $SPECIALMODE remote client integration tests configured" + ;; + *) + die 110 "Unsupported \$SPECIAL_MODE: $SPECIALMODE" +esac diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index e19763bfb..b2fcaa749 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -18,9 +18,8 @@ if type -P go &> /dev/null then # required for go 1.12+ export GOCACHE="${GOCACHE:-$HOME/.cache/go-build}" - eval "$(go env)" - # required by make and other tools - export $(go env | cut -d '=' -f 1) + # called processes like `make` and other tools need these vars. + eval "export $(go env)" # Ensure compiled tooling is reachable export PATH="$PATH:$GOPATH/bin" @@ -165,6 +164,11 @@ die() { exit ${1:-1} } +warn() { + echo ">>>>> ${2:-WARNING (but no message given!) in ${FUNCNAME[1]}()}" > /dev/stderr + echo ${1:-1} > /dev/stdout +} + bad_os_id_ver() { echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)" exit 42 @@ -319,8 +323,15 @@ install_test_configs(){ /etc/containers/registries.conf } +# Remove all files (except conmon, for now) provided by the distro version of podman. +# Except conmon, for now as it's expected to eventually be packaged separately. +# All VM cache-images used for testing include the distro podman because (1) it's +# required for podman-in-podman testing and (2) it somewhat simplifies the task +# of pulling in necessary prerequisites packages as the set can change over time. +# For general CI testing however, calling this function makes sure the system +# can only run the compiled source version. remove_packaged_podman_files(){ - show_and_store_warning "Removing packaged podman files to prevent conflicts with source build and testing." + echo "Removing packaged podman files to prevent conflicts with source build and testing." req_env_var OS_RELEASE_ID if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]] then diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh index b5744671b..3f45aac84 100755 --- a/contrib/cirrus/rootless_test.sh +++ b/contrib/cirrus/rootless_test.sh @@ -5,7 +5,7 @@ set -e remote=0 # The TEST_REMOTE_CLIENT environment variable decides whether -# to test varlinke +# to test varlink if [[ "$TEST_REMOTE_CLIENT" == "true" ]]; then remote=1 fi diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 6beecaa6a..e49bb98fe 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -52,7 +52,9 @@ install_test_configs make install.tools case "$SPECIALMODE" in - none) ;; # Do the normal thing + none) + remove_packaged_podman_files # we're building from source + ;; rootless) # Only do this once, even if ROOTLESS_USER (somehow) changes if ! grep -q 'ROOTLESS_USER' /etc/environment @@ -65,11 +67,13 @@ case "$SPECIALMODE" in tee -a /etc/environment) && eval "$X" && echo "$X" setup_rootless fi + remove_packaged_podman_files ;; in_podman) # Assumed to be Fedora - dnf install -y podman $SCRIPT_BASE/setup_container_environment.sh ;; + windows) ;& # for podman-remote building only + darwin) ;; *) die 111 "Unsupported \$SPECIAL_MODE: $SPECIALMODE" esac diff --git a/contrib/cirrus/uncache_release_archives.sh b/contrib/cirrus/uncache_release_archives.sh new file mode 120000 index 000000000..e9fc6edff --- /dev/null +++ b/contrib/cirrus/uncache_release_archives.sh @@ -0,0 +1 @@ +cache_release_archive.sh
\ No newline at end of file diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh index 56310bc36..004839f17 100755 --- a/contrib/cirrus/unit_test.sh +++ b/contrib/cirrus/unit_test.sh @@ -9,4 +9,17 @@ req_env_var GOSRC cd "$GOSRC" make install.tools make localunit -make + +case "$SPECIALMODE" in + in_podman) ;& + rootless) ;& + none) + make + ;; + windows) ;& + darwin) + make podman-remote-$SPECIALMODE + ;; + *) + die 109 "Unsupported \$SPECIAL_MODE: $SPECIALMODE" +esac |