system tests: add assert(), and start using it

Problem: the system test 'is()' checker was poorly thought out.
For example, there is no way to check for inequality or for
absence of a substring.

Solution, step 1: introduce new assert(), copied almost verbatim
from buildah, where it has been successful in addressing the
gaps in is().

The logical next step is to search the tests for 'die' and
for 'run', looking for negative assertions which we can
replace with assert(). There were a lot, and in the process
I found a number of ugly bugs in the tests themselves. I've
taken the liberty of fixing these.

Important note: at this time we have both assert() and is().
Replacing all instances of is() would be impossible to review.

Signed-off-by: Ed Santiago <santiago@redhat.com>

1 files changed, 99 insertions, 12 deletions

diff --git a/test/system/helpers.bash b/test/system/helpers.bash
index 0d336592f..b41be53bc 100644
--- a/test/system/helpers.bash
+++ b/test/system/helpers.bash
@@ -500,19 +500,107 @@ function die() {
     false
 }
 
-
-########
-#  is  #  Compare actual vs expected string; fail w/diagnostic if mismatch
-########
+############
+#  assert  #  Compare actual vs expected string; fail if mismatch
+############
 #
-# Compares given string against expectations, using 'expr' to allow patterns.
+# Compares string (default: $output) against the given string argument.
+# By default we do an exact-match comparison against $output, but there
+# are two different ways to invoke us, each with an optional description:
+#
+#      assert               "EXPECT" [DESCRIPTION]
+#      assert "RESULT" "OP" "EXPECT" [DESCRIPTION]
+#
+# The first form (one or two arguments) does an exact-match comparison
+# of "$output" against "EXPECT". The second (three or four args) compares
+# the first parameter against EXPECT, using the given OPerator. If present,
+# DESCRIPTION will be displayed on test failure.
 #
 # Examples:
 #
-#   is "$actual" "$expected" "descriptive test name"
-#   is "apple" "orange"  "name of a test that will fail in most universes"
-#   is "apple" "[a-z]\+" "this time it should pass"
+#   assert "this is exactly what we expect"
+#   assert "${lines[0]}" =~ "^abc"  "first line begins with abc"
 #
+function assert() {
+    local actual_string="$output"
+    local operator='=='
+    local expect_string="$1"
+    local testname="$2"
+
+    case "${#*}" in
+        0)   die "Internal error: 'assert' requires one or more arguments" ;;
+        1|2) ;;
+        3|4) actual_string="$1"
+             operator="$2"
+             expect_string="$3"
+             testname="$4"
+             ;;
+        *)   die "Internal error: too many arguments to 'assert'" ;;
+    esac
+
+    # Comparisons.
+    # Special case: there is no !~ operator, so fake it via '! x =~ y'
+    local not=
+    local actual_op="$operator"
+    if [[ $operator == '!~' ]]; then
+        not='!'
+        actual_op='=~'
+    fi
+    if [[ $operator == '=' || $operator == '==' ]]; then
+        # Special case: we can't use '=' or '==' inside [[ ... ]] because
+        # the right-hand side is treated as a pattern... and '[xy]' will
+        # not compare literally. There seems to be no way to turn that off.
+        if [ "$actual_string" = "$expect_string" ]; then
+            return
+        fi
+    elif [[ $operator == '!=' ]]; then
+        # Same special case as above
+        if [ "$actual_string" != "$expect_string" ]; then
+            return
+        fi
+    else
+        if eval "[[ $not \$actual_string $actual_op \$expect_string ]]"; then
+            return
+        elif [ $? -gt 1 ]; then
+            die "Internal error: could not process 'actual' $operator 'expect'"
+        fi
+    fi
+
+    # Test has failed. Get a descriptive test name.
+    if [ -z "$testname" ]; then
+        testname="${MOST_RECENT_PODMAN_COMMAND:-[no test name given]}"
+    fi
+
+    # Display optimization: the typical case for 'expect' is an
+    # exact match ('='), but there are also '=~' or '!~' or '-ge'
+    # and the like. Omit the '=' but show the others; and always
+    # align subsequent output lines for ease of comparison.
+    local op=''
+    local ws=''
+    if [ "$operator" != '==' ]; then
+        op="$operator "
+        ws=$(printf "%*s" ${#op} "")
+    fi
+
+    # This is a multi-line message, which may in turn contain multi-line
+    # output, so let's format it ourself, readably
+    local actual_split
+    IFS=$'\n' read -rd '' -a actual_split <<<"$actual_string" || true
+    printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n"    >&2
+    printf "#|     FAIL: %s\n" "$testname"                        >&2
+    printf "#| expected: %s'%s'\n" "$op" "$expect_string"         >&2
+    printf "#|   actual: %s'%s'\n" "$ws" "${actual_split[0]}"     >&2
+    local line
+    for line in "${actual_split[@]:1}"; do
+        printf "#|         > %s'%s'\n" "$ws" "$line"              >&2
+    done
+    printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n"   >&2
+    false
+}
+
+########
+#  is  #  **DEPRECATED**; see assert() above
+########
 function is() {
     local actual="$1"
     local expect="$2"
@@ -716,10 +804,9 @@ function remove_same_dev_warning() {
 # return that list.
 function _podman_commands() {
     dprint "$@"
-    run_podman help "$@" |
-        awk '/^Available Commands:/{ok=1;next}/^Options:/{ok=0}ok { print $1 }' |
-        grep .
-    "$output"
+    # &>/dev/null prevents duplicate output
+    run_podman help "$@" &>/dev/null
+    awk '/^Available Commands:/{ok=1;next}/^Options:/{ok=0}ok { print $1 }' <<<"$output" | grep .
 }
 
 # END   miscellaneous tools