WIP: run buildah bud tests using podman

Set of scripts to run buildah's bud.bats test using
podman build in podman CI.

podman build is not 100% compatible with buildah bud.
In particular:

  * podman defaults to --layers=true; buildah to false
  * podman defaults to --force-rm=true; buildah to false
  * podman error exit status is 125; buildah is 2
  * differences in error messages, command-line arguments

Some of the above can be dealt with programmatically,
by tweaking the buildah helpers.bash (BATS helpers).
Some need to be tweaked by patching bud.bats itself.
This PR includes a patch that will, I fear, need to
be periodically maintained over time.

There will likely be failures when vendoring in a
new buildah, possibly because new tests were added
for new features that don't exist in podman, possibly
(I hope unlikely) if existing tests are changed in
ways that make the patch file fail to apply. I've
tried to write good instructions and to write the run
script in such a way that it will offer helpful hints
on failure. My instructions and code will be imperfect;
I hope they will be good enough to merit continued use
of this test (possibly with improvements to the instructions
as we learn more about real-world failures).

Signed-off-by: Ed Santiago <santiago@redhat.com>

1 files changed, 192 insertions, 0 deletions

diff --git a/test/buildah-bud/buildah-tests.diff b/test/buildah-bud/buildah-tests.diff
new file mode 100644
index 000000000..92adabcac
--- /dev/null
+++ b/test/buildah-bud/buildah-tests.diff
@@ -0,0 +1,192 @@
+From c85882a8f7fb6efbf4d59dfe8340bfbef57ccd48 Mon Sep 17 00:00:00 2001
+From: Ed Santiago <santiago@redhat.com>
+Date: Tue, 9 Feb 2021 17:28:05 -0700
+Subject: [PATCH] tweaks for running buildah tests under podman
+
+Signed-off-by: Ed Santiago <santiago@redhat.com>
+---
+ tests/bud.bats     | 26 ++++++++++++++++----------
+ tests/helpers.bash | 28 ++++++++++++++++++++++++----
+ 2 files changed, 40 insertions(+), 14 deletions(-)
+
+diff --git a/tests/bud.bats b/tests/bud.bats
+index 1efc3c58..9a39d594 100644
+--- a/tests/bud.bats
++++ b/tests/bud.bats
+@@ -4,7 +4,7 @@ load helpers
+
+ @test "bud with a path to a Dockerfile (-f) containing a non-directory entry" {
+   run_buildah 125 bud -f ${TESTSDIR}/bud/non-directory-in-path/non-directory/Dockerfile
+-  expect_output --substring "non-directory/Dockerfile: not a directory"
++  expect_output --substring "Error: context must be a directory:"
+ }
+
+ @test "bud with --dns* flags" {
+@@ -95,6 +95,7 @@ symlink(subdir)"
+ }
+
+ @test "bud-flags-order-verification" {
++  skip "N/A under podman"
+   run_buildah 125 bud /tmp/tmpdockerfile/ -t blabla
+   check_options_flag_err "-t"
+
+@@ -1324,13 +1325,13 @@ function _test_http() {
+ @test "bud with dir for file but no Dockerfile in dir" {
+   target=alpine-image
+   run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json -t ${target} -f ${TESTSDIR}/bud/empty-dir ${TESTSDIR}/bud/empty-dir
+-  expect_output --substring "no such file or directory"
++  expect_output --substring "Error: context must be a directory:"
+ }
+
+ @test "bud with bad dir Dockerfile" {
+   target=alpine-image
+   run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json -t ${target} -f ${TESTSDIR}/baddirname ${TESTSDIR}/baddirname
+-  expect_output --substring "no such file or directory"
++  expect_output --substring "Error: context must be a directory:"
+ }
+
+ @test "bud with ARG before FROM default value" {
+@@ -1742,7 +1743,9 @@ _EOF
+   run_buildah bud --signature-policy ${TESTSDIR}/policy.json --layers -t test-img-2 --build-arg TEST=foo -f Dockerfile4 ${TESTSDIR}/bud/build-arg
+   run_buildah inspect -f '{{.FromImageID}}' test-img-2
+   argsid="$output"
+-  [[ "$argsid" != "$initialid" ]]
++  if [[ "$argsid" == "$initialid" ]]; then
++      die ".FromImageID of test-img-2 ($argsid) == same as test-img, it should be different"
++  fi
+
+   # Set the build-arg via an ENV in the local environment and verify that the cached layers are not used
+   export TEST=bar
+@@ -1795,6 +1798,7 @@ _EOF
+ }
+
+ @test "bud without any arguments should succeed" {
++  skip "does not work under podman"
+   cd ${TESTSDIR}/bud/from-scratch
+   run_buildah bud --signature-policy ${TESTSDIR}/policy.json
+ }
+@@ -1802,7 +1806,7 @@ _EOF
+ @test "bud without any arguments should fail when no Dockerfile exist" {
+   cd $(mktemp -d)
+   run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json
+-  expect_output --substring "no such file or directory"
++  expect_output "Error: no context directory and no Containerfile specified"
+ }
+
+ @test "bud with specified context should fail if directory contains no Dockerfile" {
+@@ -1815,16 +1819,17 @@ _EOF
+   DIR=$(mktemp -d)
+   mkdir -p "$DIR"/Dockerfile
+   run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json "$DIR"
+-  expect_output --substring "is not a file"
++  expect_output --substring "Error: open .*: no such file or directory"
+ }
+
+ @test "bud with specified context should fail if context contains not-existing Dockerfile" {
+   DIR=$(mktemp -d)
+   run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json "$DIR"/Dockerfile
+-  expect_output --substring "no such file or directory"
++  expect_output --substring "context must be a directory"
+ }
+
+ @test "bud with specified context should succeed if context contains existing Dockerfile" {
++  skip "podman requires a directory, not a Dockerfile"
+   DIR=$(mktemp -d)
+   echo "FROM alpine" > "$DIR"/Dockerfile
+   run_buildah 0 bud --signature-policy ${TESTSDIR}/policy.json "$DIR"/Dockerfile
+@@ -1876,7 +1881,7 @@ _EOF
+
+ @test "bud-squash-hardlinks" {
+   _prefetch busybox
+-  run_buildah bud --signature-policy ${TESTSDIR}/policy.json --squash ${TESTSDIR}/bud/layers-squash/Dockerfile.hardlinks
++  run_buildah bud --signature-policy ${TESTSDIR}/policy.json --squash -f Dockerfile.hardlinks ${TESTSDIR}/bud/layers-squash
+ }
+
+ @test "bud with additional directory of devices" {
+@@ -2023,6 +2028,7 @@ _EOF
+ }
+
+ @test "bud pull never" {
++  skip "FIXME: podman issue #9573"
+   target=pull
+   run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json -t ${target} --pull-never ${TESTSDIR}/bud/pull
+   expect_output --substring "pull policy is \"never\" but \""
+@@ -2042,6 +2048,7 @@ _EOF
+ }
+
+ @test "bud with Containerfile should fail with nonexistent authfile" {
++  skip "FIXME: podman issue #9572"
+   target=alpine-image
+   run_buildah 125 bud --authfile /tmp/nonexistent --signature-policy ${TESTSDIR}/policy.json -t ${target} ${TESTSDIR}/bud/containerfile
+ }
+@@ -2169,6 +2176,7 @@ EOM
+ }
+
+ @test "bud with encrypted FROM image" {
++  skip "Too much effort to spin up a local registry"
+   _prefetch busybox
+   mkdir ${TESTDIR}/tmp
+   openssl genrsa -out ${TESTDIR}/tmp/mykey.pem 1024
+@@ -2241,8 +2249,6 @@ EOM
+   _prefetch alpine
+   run_buildah bud --timestamp=0 --quiet --pull=false --signature-policy ${TESTSDIR}/policy.json -t timestamp -f Dockerfile.1 ${TESTSDIR}/bud/cache-stages
+   cid=$output
+-  run_buildah inspect --format '{{ .Docker.Created }}' timestamp
+-  expect_output --substring "1970-01-01"
+   run_buildah inspect --format '{{ .OCIv1.Created }}' timestamp
+   expect_output --substring "1970-01-01"
+   run_buildah inspect --format '{{ .History }}' timestamp
+diff --git a/tests/helpers.bash b/tests/helpers.bash
+index 5623a0e7..9683360f 100644
+--- a/tests/helpers.bash
++++ b/tests/helpers.bash
+@@ -70,7 +70,7 @@ function _prefetch() {
+             mkdir -p ${_BUILDAH_IMAGE_CACHEDIR}
+         fi
+
+-        local _podman_opts="--root ${TESTDIR}/root --storage-driver ${STORAGE_DRIVER}"
++        local _podman_opts="--root ${TESTDIR}/root --runroot ${TESTDIR}/runroot --storage-driver ${STORAGE_DRIVER}"
+
+         for img in "$@"; do
+             echo "# [checking for: $img]" >&2
+@@ -138,15 +138,35 @@ function run_buildah() {
+         --retry)         retry=3;        shift;;  # retry network flakes
+     esac
+
++    local podman_or_buildah=${BUILDAH_BINARY}
++    if [[ $1 == "bud" || $1 == "build-using-dockerfile" ]]; then
++        shift
++        # podman defaults to --layers=true; buildah to --false.
++        # If command line includes explicit --layers, leave it untouched,
++        # but otherwise update command line so podman mimics buildah default.
++        if [[ "$*" =~ --layers || "$*" =~ --squash ]]; then
++            set "build" "--force-rm=false" "$@"
++        else
++            set "build" "--force-rm=false" "--layers=false" "$@"
++        fi
++        podman_or_buildah=${PODMAN_BINARY}
++
++        # podman always exits 125 where buildah exits 1 or 2
++        case $expected_rc in
++            1|2)   expected_rc=125 ;;
++        esac
++    fi
++    local cmd_basename=$(basename ${podman_or_buildah})
++
+     # Remember command args, for possible use in later diagnostic messages
+-    MOST_RECENT_BUILDAH_COMMAND="buildah $*"
++    MOST_RECENT_BUILDAH_COMMAND="$cmd_basename $*"
+
+     while [ $retry -gt 0 ]; do
+         retry=$(( retry - 1 ))
+
+         # stdout is only emitted upon error; this echo is to help a debugger
+-        echo "\$ $BUILDAH_BINARY $*"
+-        run timeout --foreground --kill=10 $BUILDAH_TIMEOUT ${BUILDAH_BINARY} --registries-conf ${TESTSDIR}/registries.conf --root ${TESTDIR}/root --runroot ${TESTDIR}/runroot --storage-driver ${STORAGE_DRIVER} "$@"
++        echo "\$ $cmd_basename $*"
++        run timeout --foreground --kill=10 $BUILDAH_TIMEOUT ${podman_or_buildah} --registries-conf ${TESTSDIR}/registries.conf --root ${TESTDIR}/root --runroot ${TESTDIR}/runroot --storage-driver ${STORAGE_DRIVER} "$@"
+         # without "quotes", multiple lines are glommed together into one
+         if [ -n "$output" ]; then
+             echo "$output"
+--
+2.30.2