From 3baa9da4edec72a70aec4aa25a29b4afd82ed689 Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Tue, 9 Feb 2021 14:33:19 -0700 Subject: WIP: run buildah bud tests using podman Set of scripts to run buildah's bud.bats test using podman build in podman CI. podman build is not 100% compatible with buildah bud. In particular: * podman defaults to --layers=true; buildah to false * podman defaults to --force-rm=true; buildah to false * podman error exit status is 125; buildah is 2 * differences in error messages, command-line arguments Some of the above can be dealt with programmatically, by tweaking the buildah helpers.bash (BATS helpers). Some need to be tweaked by patching bud.bats itself. This PR includes a patch that will, I fear, need to be periodically maintained over time. There will likely be failures when vendoring in a new buildah, possibly because new tests were added for new features that don't exist in podman, possibly (I hope unlikely) if existing tests are changed in ways that make the patch file fail to apply. I've tried to write good instructions and to write the run script in such a way that it will offer helpful hints on failure. My instructions and code will be imperfect; I hope they will be good enough to merit continued use of this test (possibly with improvements to the instructions as we learn more about real-world failures). Signed-off-by: Ed Santiago --- test/buildah-bud/run-buildah-bud-tests | 166 +++++++++++++++++++++++++++++++++ 1 file changed, 166 insertions(+) create mode 100755 test/buildah-bud/run-buildah-bud-tests (limited to 'test/buildah-bud/run-buildah-bud-tests') diff --git a/test/buildah-bud/run-buildah-bud-tests b/test/buildah-bud/run-buildah-bud-tests new file mode 100755 index 000000000..67c8fdfa4 --- /dev/null +++ b/test/buildah-bud/run-buildah-bud-tests @@ -0,0 +1,166 @@ +#!/bin/bash + +ME=$(basename $0) + +############################################################################### +# BEGIN user-customizable section + +# Buildah main repository; unlikely to change often +BUILDAH_REPO=github.com/containers/buildah + +# Tag name used to identify the base checkout +BASE_TAG=buildah-bud-in-podman + +# END user-customizable section +############################################################################### + +usage="Usage: $ME [--help] [--no-checkout] [--no-test] +" + +# Parse command-line options (used in development only, not in CI) +do_checkout=y +do_test=y +for i; do + case "$i" in + --no-checkout) do_checkout= ; shift;; + --no-test) do_test= ; shift;; + -h|--help) echo "$usage"; exit 0;; + *) echo "$ME: Unrecognized option '$i'" >&2; exit 1;; + esac +done + +# Patches helpers.bash and potentially other files (bud.bats? Dockerfiles?) +# +# The patch file is horrible to generate: +# 1) cd to the checked-out buildah/tests directory +# 2) make your edits +# 3) git commit -asm 'blah blah blah' +# 3a) if checked-out directory already includes earlier patches, +# you may need to 'git commit --amend' instead +# 4) git format-patch HEAD^ +# 5) sed -e 's/ \+$//' 0001* >../PATCH-FILE-PATH +# 6) vim that file, remove trailing empty newlines +# 7) cd back out of buildah directory, and git-commit this new patch file +# +# FIXME: this makes me nervous. The diff will probably need tweaking +# over time. I don't think we need to version it, because we +# *have* to be in lockstep with a specific buildah version, +# so problems should only arise when we re-vendor. +# But I'm still nervous and can't put my finger on the reason. +# +# Complicated invocation needed because we 'cd' down below. +BUD_TEST_DIR=$(realpath $(dirname ${BASH_SOURCE[0]})) +PATCHES=${BUD_TEST_DIR}/buildah-tests.diff + +# Friendlier relative path to our buildah-tests dir +BUD_TEST_DIR_REL=$(dirname $(git ls-files --full-name ${BASH_SOURCE[0]})) +# Path to podman binary; again, do it before we cd +PODMAN_BINARY=$(pwd)/bin/podman +REMOTE= +# If remote, start server & change path +if [[ "${PODBIN_NAME:-}" = "remote" ]]; then + REMOTE=1 + echo "$ME: remote tests are not working yet" >&2 + exit 1 +fi + +function die() { + failhint= + echo "$ME: $*" >&2 + exit 1 +} + +# From here on out, any unexpected abort will try to offer helpful hints +failhint= +trap 'if [[ $? != 0 ]]; then if [[ -n $failhint ]]; then echo;echo "***************************************";echo $failhint;echo;echo "Please see $BUD_TEST_DIR_REL/README.md for advice";fi;fi' 0 + +# Find the version of buildah we've vendored in, so we can run the right tests +buildah_version=$(awk "\$1 == \"$BUILDAH_REPO\" { print \$2 }" make-new-buildah-diffs + chmod 755 make-new-buildah-diffs +else + # Called with --no-checkout + test -d $buildah_dir || die "Called with --no-checkout, but $buildah_dir does not exist" + + cd $buildah_dir +fi + +if [[ -n $do_test ]]; then + failhint="Error running buildah bud tests under podman." + if [[ -n $is_revendor ]]; then + failhint+=" + +It looks like you're vendoring in a new buildah. The likely failure +here is that there's a new test in bud.bats that uses functionality +not (yet) in podman build. You will likely need to 'skip' that test. +" + else + failhint+=" + +Is it possible that your PR breaks podman build in some way? Please +review the test failure and double-check your changes. +" + fi + + (set -x;sudo env TMPDIR=/var/tmp \ + PODMAN_BINARY=$PODMAN_BINARY \ + BUILDAH_BINARY=$(pwd)/bin/buildah \ + bats tests/bud.bats) +fi -- cgit v1.2.3-54-g00ecf