path: root/test/buildah-bud/apply-podman-deltas

#!/bin/bash
#
# *** NOTE TO READER: Please skip down to "user-customizable section" below!
#
# Not all tests in buildah/tests/bud.bats work under podman.
# Some work, but emit different error messages.
#
# This script is used to skip the former, and munge expect_output messages
# for the latter.
#
ME=$(basename $0)

BUD=tests/bud.bats

if [[ ! -e $BUD ]]; then
    echo "$ME: $BUD not found: please run me from buildah subdir" >&2
    exit 1
fi

###############################################################################
# BEGIN handlers
#
# *** NOTE TO READER (again): Please skip down to "user-customizable section"
#
# You almost certainly don't care about anything in this section.
#
set -e

RC=0

ECHO=':'
if [[ -n $DEBUG_PODMAN_DELTAS ]]; then
    ECHO='echo'
fi

# Issue a warning, and set exit status (but do not exit now)
function warn() {
    echo "$ME: ERROR: $*" >&2
    RC=1
}

# errmsg: used to change the text of a message, probably in expect_output()
function errmsg() {
    local msg_orig=${1//\//\\/}; shift
    local msg_new=${1//\//\\/};  shift

    for t in "$@"; do
        if fgrep -qx "@test \"$t\" {" $BUD; then
            $ECHO "@test \"$t\" : updating to \"$msg_new\""
            t=${t//\//\\/}
            # FIXME: emit error if msg_orig not found
            sed -i -e "/^\@test \"$t\" {/,/^}/s/\"$msg_orig\"/\"$msg_new\"/" $BUD
        else
            warn "[errmsg] Did not find test \"$t\" in $BUD"
        fi
    done
}

# _skip: used to add a 'skip' or 'skip_if_remote' to one specific test
function _skip() {
    local skip=$1;   shift
    local reason=$1; shift

    # All further arguments are test names. Make sure we're invoked with some!
    if [[ -z "$*" ]]; then
        echo "$ME: FATAL: Invalid use of '${FUNCNAME[1]}' at line ${BASH_LINENO[1]}: missing test-name argument(s)." >&2
        exit 1
    fi

    for t in "$@"; do
        if fgrep -qx "@test \"$t\" {" $BUD; then
            $ECHO "@test \"$t\" : $skip \"$reason\""
            t=${t//\//\\/}
            sed -i -e "/^\@test \"$t\" {/ a \ \ $skip \"$reason\"" $BUD
        else
            warn "[$skip] Did not find test \"$t\" in $BUD"
        fi
    done
}

function skip() {
    _skip "skip" "$@"
}

function skip_if_remote() {
    _skip "skip_if_remote" "$@"
}

# END   handlers
###############################################################################
# BEGIN user-customizable section
#
# These are the hand-maintained exceptions. This is what you want to edit
# or update as needed.
#
# There are three directives you can use below:
#
#    errmsg "old-message" "new-message" "test name" ["test name"...]
#
# This replaced "old-message" with "new-message" in @test "test name".
# It is used when a podman error message differs from buildah's.
#
#    [skip | skip_if_remote] "reason" "test name" ["test name"...]
#
# This adds a 'skip' statement as the first line of @test "test name".
# It is used when a test does not work in podman, either for permanent
# design-related reasons or for hopefully-temporary bug-in-podman reasons.
# (If the latter, please file an issue before adding the skip, and include
# the issue number in your skip message. This makes it possible to remove
# the skip once the issue is fixed).
#
# For both cases, you can list multiple "test names" at the end. This
# is not used much right now, but will be once I file my podman-remote PR
# because there are some cases where the same issue affects up to fifty
# different bud.bats tests.
#

###############################################################################
# BEGIN differences in error messages between buildah and podman

errmsg "non-directory/Dockerfile: not a directory" \
       "Error: context must be a directory:" \
       "bud with a path to a Dockerfile (-f) containing a non-directory entry"

errmsg "no such file or directory" \
       "Error: context must be a directory:" \
       "bud with dir for file but no Dockerfile in dir" \
       "bud with bad dir Dockerfile"

errmsg "no such file or directory" \
       "Error: no context directory and no Containerfile specified" \
       "bud without any arguments should fail when no Dockerfile exists"

errmsg "is not a file" \
       "Error: containerfile: .* cannot be path to a directory" \
       "bud with specified context should fail if assumed Dockerfile is a directory"

errmsg "no such file or directory" \
       "context must be a directory" \
       "bud with specified context should fail if context contains not-existing Dockerfile"

# 2022-04-26 after buildah PR 3926 (where Ed added error-message checks"
errmsg "no FROM statement found" \
       "Error: no FROM statement found" \
       "bud with Dockerfile from invalid URL"

errmsg "no contents in .*" \
       "Error: context must be a directory: .*" \
       "bud with specified context should fail if context contains empty Dockerfile"

errmsg "checking authfile: stat /tmp/nonexistent: no such file or directory" \
       "Error: checking authfile: stat /tmp/nonexistent: no such file or directory" \
       "bud with Containerfile should fail with nonexistent authfile"

errmsg "cannot find Containerfile or Dockerfile" \
       "no such file or directory" \
       "bud-github-context-from-commit"

###############################################################################
# BEGIN tests that don't make sense under podman due to fundamental differences

# Fails with "Error: no context directory and no Containerfile specified"
skip "does not work under podman" \
     "bud without any arguments should succeed"

# ...or due to a fundamental arg-parsing difference between buildah and podman
# which we could and perhaps should fix in the buildah repo via:
#    - ... ${TESTSDIR}/bud/layers-squash/Dockerfile.hardlinks
#    + ... -f Dockerfile.hardlinks ${TESTSDIR}/bud/layers-squash
skip "argument-order incompatible with podman" \
     "bud-squash-hardlinks"

# Fails with "Error: context must be a directory: /path/to/Dockerfile"
skip "podman-build fails with 'context must be a directory'" \
     "bud with specified context should succeed if context contains existing Dockerfile"

###############################################################################
# BEGIN tests which are skipped because they make no sense under podman-remote

skip_if_remote "--runtime-flags does not work with podman-remote" \
               "bud - invalid runtime flags test"

skip_if_remote "--target does not work with podman-remote" \
               "bud-target"

skip_if_remote "--runtime not meaningful under podman-remote" \
               "bud with --runtime and --runtime-flag"

skip_if_remote "secret files not implemented under podman-remote" \
               "bud with containerfile secret" \
               "bud with containerfile secret accessed on second RUN" \
               "bud with containerfile secret options" \
               "bud with containerfile env secret" \
               "bud with containerfile env secret priority"

skip_if_remote "volumes don't work with podman-remote" \
               "buildah bud --volume" \
               "buildah-bud-policy"

skip_if_remote "--build-context option not implemented in podman-remote" \
               "build-with-additional-build-context and COPY, additional context from host" \
               "build-with-additional-build-context and RUN --mount=from=, additional-context not image and also test conflict with stagename" \

skip_if_remote "env-variable for Containerfile.in pre-processing is not propagated on remote" \
               "bud with Containerfile.in, via envariable" \

# Requires a local file outside context dir
skip_if_remote "local keyfile not sent to podman-remote" \
               "bud with encrypted FROM image"

# See podman #9890 for discussion
skip_if_remote "--stdin option will not be implemented in podman-remote" \
               "bud test no --stdin"

# https://github.com/containers/buildah/pull/3823
# If this is possible with podman-remote, it'll take way more Go skills
# to implement than what Ed can do.
skip_if_remote "--output option not implemented in podman-remote" \
               "build with custom build output and output rootfs to directory" \
               "build with custom build output and output rootfs to tar" \
               "build with custom build output and output rootfs to tar by pipe" \
               "build with custom build output must fail for bad input" \
               "build with custom build output and output rootfs to tar with no additional step" \
               "build with custom build output for single-stage-cached and output rootfs to directory" \
               "build with custom build output for multi-stage-cached and output rootfs to directory" \
               "build with custom build output for multi-stage and output rootfs to directory"

# https://github.com/containers/podman/issues/14544
skip_if_remote "logfile not implemented on remote" "bud-logfile-with-split-logfile-by-platform"

skip_if_remote "envariables do not automatically work with -remote." \
               "build proxy"

# 2022-07-04 this is a new test in buildah; it's failing in treadmill
skip_if_remote "FIXME FIXME FIXME: does this test make sense in remote?" \
               "build-test with OCI prestart hook"

# 2022-08-17 buildah PR 4190
skip_if_remote "Explicit request in buildah PR 4190 to skip this on remote" \
               "build: test race in updating image name while performing parallel commits"

###############################################################################
# BEGIN tests which are skipped due to actual podman or podman-remote bugs.

skip_if_remote "different error messages between podman & podman-remote" \
               "bud with .dockerignore #2"

# These two tests, new in 2022-01, invoke podman (create, export) in ways
# that don't work with podman-remote due to the use of --registries-conf
skip_if_remote "FIXME FIXME FIXME: find a way to clean up their podman calls" \
               "bud with run should not leave mounts behind cleanup test" \
               "bud with custom files in /run/ should persist cleanup test"

# Under podman-remote, the "Ignoring <stdin>:5:2: error: #error" message
# is never seen. (Not even as stdout/stderr on the server; Ed checked).
skip_if_remote "FIXME FIXME FIXME: 'Ignoring' warning is never seen" \
               "bud with preprocessor error"

# END   tests which are skipped due to actual podman or podman-remote bugs.
###############################################################################

exit $RC