buildah-bud tests: simplify

Experience this week has shown that managing .diff files
is too difficult for humans, and too fragile. Opportunities
for errors abound. So, let's try to minimize the diffs.

We can't eliminate the diffs to helpers.bash: those are
true code changes that are absolutely required for running
tests using podman instead of buildah. We need to carry
those ourselves: they are not appropriate for the buildah
repo itself.

What we can do is simplify the patching of bud.bats. That
is fragile, because bud.bats changes often, and context-
sensitive git patch files can easily get confused.

Recognizing that the changes to bud.bats fall under two types:

  - tests that are skipped
  - tests in which podman error messages differ from buildah's

...we now have a new script, apply-podman-deltas, which
is (I hope) much user-friendlier. It understands two directives:

  errmsg - alter the expected error message
  skip   - skip a test

Both operate based on a bats test name. The test name must
match exactly. These directives use 'sed' to update bud.bats.
If any directive fails, the script will keep going (so you
get as many errors as possible in a run), then exits failure.

Instructions (README.md) now explain the process for dealing
with all expected test failures.

(Sneak checkin: add '--filter=NAME' option to test runner,
allowing for targeted and much shorter test runs).

Signed-off-by: Ed Santiago <santiago@redhat.com>

1 files changed, 2 insertions, 119 deletions

diff --git a/test/buildah-bud/buildah-tests.diff b/test/buildah-bud/buildah-tests.diff
index 141bd9266..bba737848 100644
--- a/test/buildah-bud/buildah-tests.diff
+++ b/test/buildah-bud/buildah-tests.diff
@@ -1,130 +1,13 @@
-From 47cc8f54d272039bdab389b29c8b2b5bb085e923 Mon Sep 17 00:00:00 2001
+From b948e99cb6cb4765987711e8d8948841f6d3f7e2 Mon Sep 17 00:00:00 2001
 From: Ed Santiago <santiago@redhat.com>
 Date: Tue, 9 Feb 2021 17:28:05 -0700
 Subject: [PATCH] tweaks for running buildah tests under podman
 
 Signed-off-by: Ed Santiago <santiago@redhat.com>
 ---
- tests/bud.bats     | 22 +++++++++++++---------
  tests/helpers.bash | 28 ++++++++++++++++++++++++----
- 2 files changed, 37 insertions(+), 13 deletions(-)
+ 1 file changed, 24 insertions(+), 4 deletions(-)
 
-diff --git a/tests/bud.bats b/tests/bud.bats
-index fe3af27a..d1ccced5 100644
---- a/tests/bud.bats
-+++ b/tests/bud.bats
-@@ -4,7 +4,7 @@ load helpers
-
- @test "bud with a path to a Dockerfile (-f) containing a non-directory entry" {
-   run_buildah 125 bud -f ${TESTSDIR}/bud/non-directory-in-path/non-directory/Dockerfile
--  expect_output --substring "non-directory/Dockerfile: not a directory"
-+  expect_output --substring "Error: context must be a directory:"
- }
-
- @test "bud with --dns* flags" {
-@@ -117,6 +117,7 @@ symlink(subdir)"
- }
-
- @test "bud-flags-order-verification" {
-+  skip "N/A under podman"
-   run_buildah 125 bud /tmp/tmpdockerfile/ -t blabla
-   check_options_flag_err "-t"
-
-@@ -1436,13 +1437,13 @@ function _test_http() {
- @test "bud with dir for file but no Dockerfile in dir" {
-   target=alpine-image
-   run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json -t ${target} -f ${TESTSDIR}/bud/empty-dir ${TESTSDIR}/bud/empty-dir
--  expect_output --substring "no such file or directory"
-+  expect_output --substring "Error: context must be a directory:"
- }
-
- @test "bud with bad dir Dockerfile" {
-   target=alpine-image
-   run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json -t ${target} -f ${TESTSDIR}/baddirname ${TESTSDIR}/baddirname
--  expect_output --substring "no such file or directory"
-+  expect_output --substring "Error: context must be a directory:"
- }
-
- @test "bud with ARG before FROM default value" {
-@@ -1913,6 +1914,7 @@ _EOF
- }
-
- @test "bud without any arguments should succeed" {
-+  skip "does not work under podman"
-   cd ${TESTSDIR}/bud/from-scratch
-   run_buildah bud --signature-policy ${TESTSDIR}/policy.json
- }
-@@ -1920,7 +1922,7 @@ _EOF
- @test "bud without any arguments should fail when no Dockerfile exist" {
-   cd $(mktemp -d)
-   run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json
--  expect_output --substring "no such file or directory"
-+  expect_output "Error: no context directory and no Containerfile specified"
- }
-
- @test "bud with specified context should fail if directory contains no Dockerfile" {
-@@ -1933,16 +1935,17 @@ _EOF
-   DIR=$(mktemp -d)
-   mkdir -p "$DIR"/Dockerfile
-   run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json "$DIR"
--  expect_output --substring "is not a file"
-+  expect_output --substring "Error: open .*: no such file or directory"
- }
-
- @test "bud with specified context should fail if context contains not-existing Dockerfile" {
-   DIR=$(mktemp -d)
-   run_buildah 125 bud --signature-policy ${TESTSDIR}/policy.json "$DIR"/Dockerfile
--  expect_output --substring "no such file or directory"
-+  expect_output --substring "context must be a directory"
- }
-
- @test "bud with specified context should succeed if context contains existing Dockerfile" {
-+  skip "podman requires a directory, not a Dockerfile"
-   DIR=$(mktemp -d)
-   echo "FROM alpine" > "$DIR"/Dockerfile
-   run_buildah 0 bud --signature-policy ${TESTSDIR}/policy.json "$DIR"/Dockerfile
-@@ -1994,7 +1997,7 @@ _EOF
-
- @test "bud-squash-hardlinks" {
-   _prefetch busybox
--  run_buildah bud --signature-policy ${TESTSDIR}/policy.json --squash ${TESTSDIR}/bud/layers-squash/Dockerfile.hardlinks
-+  run_buildah bud --signature-policy ${TESTSDIR}/policy.json --squash -f Dockerfile.hardlinks ${TESTSDIR}/bud/layers-squash
- }
-
- @test "bud with additional directory of devices" {
-@@ -2159,6 +2162,7 @@ _EOF
- }
-
- @test "bud with Containerfile should fail with nonexistent authfile" {
-+  skip "FIXME: podman issue #9572"
-   target=alpine-image
-   run_buildah 125 bud --authfile /tmp/nonexistent --signature-policy ${TESTSDIR}/policy.json -t ${target} ${TESTSDIR}/bud/containerfile
- }
-@@ -2286,6 +2290,7 @@ EOM
- }
-
- @test "bud with encrypted FROM image" {
-+  skip "Too much effort to spin up a local registry"
-   _prefetch busybox
-   mkdir ${TESTDIR}/tmp
-   openssl genrsa -out ${TESTDIR}/tmp/mykey.pem 1024
-@@ -2358,8 +2363,6 @@ EOM
-   _prefetch alpine
-   run_buildah bud --timestamp=0 --quiet --pull=false --signature-policy ${TESTSDIR}/policy.json -t timestamp -f Dockerfile.1 ${TESTSDIR}/bud/cache-stages
-   cid=$output
--  run_buildah inspect --format '{{ .Docker.Created }}' timestamp
--  expect_output --substring "1970-01-01"
-   run_buildah inspect --format '{{ .OCIv1.Created }}' timestamp
-   expect_output --substring "1970-01-01"
-   run_buildah inspect --format '{{ .History }}' timestamp
-@@ -2619,6 +2622,7 @@ _EOF
- }
-
- @test "bud with --arch flag" {
-+  skip "FIXME: for podman, emergency skip to get CI going"
-   _prefetch alpine
-   mytmpdir=${TESTDIR}/my-dir
-   mkdir -p ${mytmpdir}
 diff --git a/tests/helpers.bash b/tests/helpers.bash
 index 99c290af..c5572840 100644
 --- a/tests/helpers.bash