diff options
Diffstat (limited to 'contrib/cirrus')
| -rw-r--r-- | contrib/cirrus/README.md | 45 | ||||
| -rwxr-xr-x | contrib/cirrus/build_release.sh | 24 | ||||
| -rwxr-xr-x | contrib/cirrus/check_image.sh | 5 | ||||
| -rw-r--r-- | contrib/cirrus/lib.sh | 2 | ||||
| -rw-r--r-- | contrib/cirrus/packer/fedora_setup.sh | 4 | ||||
| -rw-r--r-- | contrib/cirrus/packer/ubuntu_setup.sh | 11 | ||||
| -rwxr-xr-x | contrib/cirrus/upload_release_archive.sh | 62 |
7 files changed, 117 insertions, 36 deletions
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index 7aa8881d6..779f95d95 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -124,35 +124,46 @@ you'll find the new image names displayed at the end of the ``` ...cut... -==> Builds finished. The artifacts of successful builds are: ---> ubuntu-18: A disk image was created: ubuntu-18-libpod-5699523102900224 ---> ubuntu-18: ---> fedora-29: A disk image was created: fedora-29-libpod-5699523102900224 ---> fedora-29: ---> fedora-28: A disk image was created: fedora-28-libpod-5699523102900224 + +[+0747s] ==> Builds finished. The artifacts of successful builds are: +[+0747s] --> ubuntu-18: A disk image was created: ubuntu-18-libpod-5664838702858240 +[+0747s] --> fedora-29: A disk image was created: fedora-29-libpod-5664838702858240 +[+0747s] --> fedora-30: A disk image was created: fedora-30-libpod-5664838702858240 +[+0747s] --> ubuntu-19: A disk image was created: ubuntu-19-libpod-5664838702858240 ``` -Now edit `.cirrus.yml`, updating the `*_IMAGE_NAME` lines to reflect the -images from above: +Notice the suffix on all the image names comes from the env. var. set in +*.cirrus.yml*: `BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}"`. +Edit `.cirrus.yml`, in the top-level `env` section, update the suffix variable +used at runtime to launch VMs for testing: ```yaml env: ...cut... #### - #### Cache-image names to test with + #### Cache-image names to test with (double-quotes around names are critical) ### - FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5699523102900224" - PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5699523102900224" - UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5699523102900224" + _BUILT_IMAGE_SUFFIX: "libpod-5664838702858240" + FEDORA_CACHE_IMAGE_NAME: "fedora-30-${_BUILT_IMAGE_SUFFIX}" + PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-29-${_BUILT_IMAGE_SUFFIX}" ...cut... ``` -***NOTE:*** If re-using the same PR with new images in `.cirrus.yml`, -take care to also *update the PR description* to remove -the magic ``***CIRRUS: TEST IMAGES***`` string. Keeping it and -`--force` pushing would needlessly cause Cirrus-CI to build -and test images again. +***NOTES:*** +* If re-using the same PR with new images in `.cirrus.yml`, + take care to also *update the PR description* to remove + the magic ``***CIRRUS: TEST IMAGES***`` string. Keeping it and + `--force` pushing would needlessly cause Cirrus-CI to build + and test images again. +* In the future, if you need to review the log from the build that produced + the referenced image: + + * Note the Build ID from the image name (for example `5664838702858240`). + * Go to that build in the Cirrus-CI WebUI, using the build ID in the URL. + (For example `https://cirrus-ci.com/build/5664838702858240`. + * Choose the *test_build_cache_images* task. + * Open the *build_vm_images* script section. ### `release` Task diff --git a/contrib/cirrus/build_release.sh b/contrib/cirrus/build_release.sh index 287643f47..07db88f81 100755 --- a/contrib/cirrus/build_release.sh +++ b/contrib/cirrus/build_release.sh @@ -1,5 +1,7 @@ #!/bin/bash +set -e + source $(dirname $0)/lib.sh req_env_var TEST_REMOTE_CLIENT OS_RELEASE_ID GOSRC @@ -13,18 +15,20 @@ fi if [[ -n "$CROSS_PLATFORM" ]] then + # Will fail if $CROSS_PLATFORM is unsupported cross-compile $GOOS value + make podman-remote-${CROSS_PLATFORM}-release + echo "Compiling podman-remote release archive for ${CROSS_PLATFORM}" - case "$CROSS_PLATFORM" in - linux) ;& - windows) ;& - darwin) - make podman-remote-${CROSS_PLATFORM}-release - ;; - *) - die 1 "Unknown/unsupported cross-compile platform '$CROSS_PLATFORM'" - ;; - esac + if [[ "$CROSS_PLATFORM" == "windows" ]] + then + # TODO: Remove next line, part of VM images next time they're built. + dnf install -y libmsi1 msitools pandoc + make podman.msi + fi else echo "Compiling release archive for $OS_RELEASE_ID" make podman-release fi + +echo "Preserving build details for later use." +mv -v release.txt actual_release.txt # Another 'make' during testing could overwrite it diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh index 39f49d0a1..5423f67d6 100755 --- a/contrib/cirrus/check_image.sh +++ b/contrib/cirrus/check_image.sh @@ -56,6 +56,11 @@ then item_test "On ubuntu /usr/bin/runc is /usr/lib/cri-o-runc/sbin/runc" "$SAMESAME" -eq "0" || let "NFAILS+=1" fi +if [[ "$OS_RELEASE_ID" == "ubuntu" ]] +then + item_test "On ubuntu, no periodic apt crap is enabled" -z "$(egrep $PERIODIC_APT_RE /etc/apt/apt.conf.d/*)" +fi + echo "Checking items specific to ${PACKER_BUILDER_NAME}${BUILT_IMAGE_SUFFIX}" case "$PACKER_BUILDER_NAME" in xfedora*) diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index f81a8d501..fe4c25e73 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -73,6 +73,8 @@ UPLDREL_IMAGE="quay.io/libpod/upldrel:latest" export DEBIAN_FRONTEND="noninteractive" SUDOAPTGET="ooe.sh sudo -E apt-get -qq --yes" SUDOAPTADD="ooe.sh sudo -E add-apt-repository --yes" +# Regex that finds enabled periodic apt configuration items +PERIODIC_APT_RE='^(APT::Periodic::.+")1"\;' # Short-cuts for retrying/timeout calls LILTO="timeout_attempt_delay_command 24s 5 30s" BIGTO="timeout_attempt_delay_command 300s 5 30s" diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index d948a0afa..679ad3b8d 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -26,6 +26,7 @@ ooe.sh sudo dnf install -y \ atomic-registries \ autoconf \ automake \ + bash-completion \ bats \ bridge-utils \ btrfs-progs-devel \ @@ -54,6 +55,7 @@ ooe.sh sudo dnf install -y \ jq \ libassuan-devel \ libcap-devel \ + libmsi1 \ libnet \ libnet-devel \ libnl3-devel \ @@ -64,9 +66,11 @@ ooe.sh sudo dnf install -y \ libvarlink-util \ lsof \ make \ + msitools \ nmap-ncat \ ostree \ ostree-devel \ + pandoc \ podman \ procps-ng \ protobuf \ diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index d20e7e005..2f54da9ed 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -18,8 +18,16 @@ trap "sudo rm -rf $GOPATH" EXIT # Ensure there are no disruptive periodic services enabled by default in image systemd_banish +# Stop disruption upon boot ASAP after booting +echo "Disabling all packaging activity on boot" +# Don't let sed process sed's temporary files +_FILEPATHS=$(sudo ls -1 /etc/apt/apt.conf.d) +for filename in $_FILEPATHS; do \ + echo "Checking/Patching $filename" + sudo sed -i -r -e "s/$PERIODIC_APT_RE/"'\10"\;/' "/etc/apt/apt.conf.d/$filename"; done + echo "Updating/configuring package repositories." -$LILTO $SUDOAPTGET update +$BIGTO $SUDOAPTGET update echo "Upgrading all packages" $BIGTO $SUDOAPTGET upgrade @@ -41,6 +49,7 @@ $BIGTO $SUDOAPTGET install \ aufs-tools \ autoconf \ automake \ + bash-completion \ bats \ bison \ btrfs-tools \ diff --git a/contrib/cirrus/upload_release_archive.sh b/contrib/cirrus/upload_release_archive.sh index 942255821..25107f0ef 100755 --- a/contrib/cirrus/upload_release_archive.sh +++ b/contrib/cirrus/upload_release_archive.sh @@ -9,6 +9,7 @@ req_env_var CI UPLDREL_IMAGE CIRRUS_BUILD_ID GOSRC RELEASE_GCPJSON RELEASE_GCPNA [[ "$CI" == "true" ]] || \ die 56 "$0 must be run under Cirrus-CI to function" +# We store "releases" for each PR, mostly to validate the process is functional unset PR_OR_BRANCH BUCKET if [[ -n "$CIRRUS_PR" ]] then @@ -22,31 +23,76 @@ else die 1 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty." fi -# Functional local podman required for uploading a release +echo "Parsing actual_release.txt contents: $(< actual_release.txt)" cd $GOSRC +RELEASETXT=$(<actual_release.txt) # see build_release.sh +[[ -n "$RELEASETXT" ]] || \ + die 3 "Could not obtain metadata from actual_release.txt" +RELEASE_INFO=$(echo "$RELEASETXT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/') +if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]] +then + die 4 "Metadata is empty or invalid: '$RELEASETXT'" +fi +# Format specified in Makefile +# e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64 +# or libpod-remote v1.3.1-166-g60df124e windows - amd64 +FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH" +read $FIELDS <<< $RELEASE_INFO +req_env_var $FIELDS + +# Functional local podman required for uploading +echo "Verifying a local, functional podman, building one if necessary." [[ -n "$(type -P podman)" ]] || \ - make install || \ + make install PREFIX=/usr || \ die 57 "$0 requires working podman binary on path to function" TMPF=$(mktemp -p '' $(basename $0)_XXXX.json) trap "rm -f $TMPF" EXIT set +x echo "$RELEASE_GCPJSON" > "$TMPF" +[[ "$OS_RELEASE_ID" == "ubuntu" ]] || \ + chcon -t container_file_t "$TMPF" unset RELEASE_GCPJSON cd $GOSRC -for filename in $(ls -1 *.tar.gz *.zip) +for filename in $(ls -1 *.tar.gz *.zip *.msi) do - echo "Running podman ... $UPLDREL_IMAGE $filename" + unset EXT + EXT=$(echo "$filename" | sed -r -e 's/.+\.(.+$)/\1/g') + if [[ -z "$EXT" ]] || [[ "$EXT" == "$filename" ]] + then + echo "Warning: Not processing $filename (invalid extension '$EXT')" + continue + fi + + [[ "$OS_RELEASE_ID" == "ubuntu" ]] || \ + chcon -t container_file_t "$filename" + # Form the generic "latest" file for this branch or pr + TO_PREFIX="${RELEASE_BASENAME}-latest-${PR_OR_BRANCH}-${RELEASE_DIST}" + # Form the fully-versioned filename for historical sake + ALSO_PREFIX="${RELEASE_BASENAME}-${RELEASE_VERSION}-${PR_OR_BRANCH}-${RELEASE_DIST}" + TO_SUFFIX="${RELEASE_ARCH}.${EXT}" + if [[ "$RELEASE_DIST" == "windows" ]] || [[ "$RELEASE_DIST" == "darwin" ]] + then + TO_FILENAME="${TO_PREFIX}-${TO_SUFFIX}" + ALSO_FILENAME="${ALSO_PREFIX}-${TO_SUFFIX}" + else + TO_FILENAME="${TO_PREFIX}-${RELEASE_DIST_VER}-${TO_SUFFIX}" + ALSO_FILENAME="${ALSO_PREFIX}-${TO_SUFFIX}" + fi + + echo "Running podman ... $UPLDREL_IMAGE for $filename -> $TO_FILENAME" + echo "Warning: upload failures are completely ignored, avoiding any needless holdup of PRs." podman run -i --rm \ -e "GCPNAME=$RELEASE_GCPNAME" \ -e "GCPPROJECT=$RELEASE_GCPROJECT" \ -e "GCPJSON_FILEPATH=$TMPF" \ - -e "REL_ARC_FILEPATH=/tmp/$filename" \ + -e "FROM_FILEPATH=/tmp/$filename" \ + -e "TO_FILENAME=$TO_FILENAME" \ + -e "ALSO_FILENAME=$ALSO_FILENAME" \ -e "PR_OR_BRANCH=$PR_OR_BRANCH" \ -e "BUCKET=$BUCKET" \ - --security-opt label=disable \ -v "$TMPF:$TMPF:ro" \ - -v "$GOSRC/$filename:/tmp/$filename:ro" \ - $UPLDREL_IMAGE + -v "$(realpath $GOSRC/$filename):/tmp/$filename:ro" \ + $UPLDREL_IMAGE || true done |