diff options
Diffstat (limited to 'test')
| -rw-r--r-- | test/system/001-basic.bats | 38 | ||||
| -rw-r--r-- | test/system/005-info.bats | 54 | ||||
| -rw-r--r-- | test/system/010-images.bats | 46 | ||||
| -rw-r--r-- | test/system/015-help.bats | 62 | ||||
| -rw-r--r-- | test/system/030-run.bats | 32 | ||||
| -rw-r--r-- | test/system/040-ps.bats | 36 | ||||
| -rw-r--r-- | test/system/050-stop.bats | 65 | ||||
| -rw-r--r-- | test/system/060-mount.bats | 35 | ||||
| -rw-r--r-- | test/system/110-history.bats | 49 | ||||
| -rw-r--r-- | test/system/README.md | 65 | ||||
| -rw-r--r-- | test/system/helpers.bash | 315 | ||||
| -rwxr-xr-x | test/system/helpers.t | 145 | ||||
| -rw-r--r-- | test/system/libpod_suite_test.go | 217 | ||||
| -rw-r--r-- | test/system/version_test.go | 51 |
14 files changed, 942 insertions, 268 deletions
diff --git a/test/system/001-basic.bats b/test/system/001-basic.bats new file mode 100644 index 000000000..15d5f38db --- /dev/null +++ b/test/system/001-basic.bats @@ -0,0 +1,38 @@ +#!/usr/bin/env bats +# +# Simplest set of podman tests. If any of these fail, we have serious problems. +# + +load helpers + +# Override standard setup! We don't yet trust podman-images or podman-rm +function setup() { + : +} + +@test "podman version emits reasonable output" { + run_podman version + + is "${lines[0]}" "Version:[ ]\+[1-9][0-9.]\+" "Version line 1" + + is "$output" ".*Go Version: \+" "'Go Version' in output" + + # FIXME: enable for 1.1 +# is "$output" ".*RemoteAPI Version: \+" "API version in output" +} + + +@test "podman can pull an image" { + run_podman pull $PODMAN_TEST_IMAGE_FQN +} + +# This is for development only; it's intended to make sure our timeout +# in run_podman continues to work. This test should never run in production +# because it will, by definition, fail. +@test "timeout" { + if [ -z "$PODMAN_RUN_TIMEOUT_TEST" ]; then + skip "define \$PODMAN_RUN_TIMEOUT_TEST to enable this test" + fi + PODMAN_TIMEOUT=10 run_podman run $PODMAN_TEST_IMAGE_FQN sleep 90 + echo "*** SHOULD NEVER GET HERE" +} diff --git a/test/system/005-info.bats b/test/system/005-info.bats new file mode 100644 index 000000000..c3eb43063 --- /dev/null +++ b/test/system/005-info.bats @@ -0,0 +1,54 @@ +#!/usr/bin/env bats + +load helpers + +@test "podman info - basic test" { + run_podman info + + expected_keys=" +BuildahVersion: *[0-9.]\\\+ +Conmon:\\\s\\\+package: +Distribution: +OCIRuntime:\\\s\\\+package: +os: +rootless: +insecure registries: +store: +GraphDriverName: +GraphRoot: +GraphStatus: +ImageStore:\\\s\\\+number: 1 +RunRoot: +" + while read expect; do + is "$output" ".*$expect" "output includes '$expect'" + done < <(parse_table "$expected_keys") +} + +@test "podman info - json" { + type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'" + + run_podman info --format=json + + expr_nvr="[a-z0-9-]\\\+-[a-z0-9.]\\\+-[a-z0-9]\\\+\." + expr_path="/[a-z0-9\\\/.]\\\+\\\$" + + tests=" +host.BuildahVersion | [0-9.] +host.Conmon.package | $expr_nvr +host.Conmon.path | $expr_path +host.OCIRuntime.package | $expr_nvr +host.OCIRuntime.path | $expr_path +store.ConfigFile | $expr_path +store.GraphDriverName | [a-z0-9]\\\+\\\$ +store.GraphRoot | $expr_path +store.ImageStore.number | 1 +" + + parse_table "$tests" | while read field expect; do + actual=$(echo "$output" | jq -r ".$field") + dprint "# actual=<$actual> expect=<$expect>" + is "$actual" "$expect" "jq .$field" + done + +} diff --git a/test/system/010-images.bats b/test/system/010-images.bats new file mode 100644 index 000000000..b88f6c2e1 --- /dev/null +++ b/test/system/010-images.bats @@ -0,0 +1,46 @@ +#!/usr/bin/env bats + +load helpers + +@test "podman images - basic output" { + run_podman images -a + + is "${lines[0]}" "REPOSITORY *TAG *IMAGE ID *CREATED *SIZE" "header line" + is "${lines[1]}" "$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME *$PODMAN_TEST_IMAGE_TAG *[0-9a-f]\+" "podman images output" +} + +@test "podman images - custom formats" { + tests=" +--format {{.ID}} | [0-9a-f]\\\{12\\\} +--format {{.ID}} --no-trunc | sha256:[0-9a-f]\\\{64\\\} +--format {{.Repository}}:{{.Tag}} | $PODMAN_TEST_IMAGE_FQN +" + + parse_table "$tests" | while read fmt expect; do + run_podman images $fmt + is "$output" "$expect\$" "podman images $fmt" + done + +} + + +@test "podman images - json" { + type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'" + + tests=" +names[0] | $PODMAN_TEST_IMAGE_FQN +id | [0-9a-f]\\\{64\\\} +digest | sha256:[0-9a-f]\\\{64\\\} +created | [0-9-]\\\+T[0-9:]\\\+\\\.[0-9]\\\+Z +size | [0-9]\\\+ +" + + run_podman images -a --format json + + parse_table "$tests" | while read field expect; do + actual=$(echo "$output" | jq -r ".[0].$field") + dprint "# actual=<$actual> expect=<$expect}>" + is "$actual" "$expect" "jq .$field" + done + +} diff --git a/test/system/015-help.bats b/test/system/015-help.bats new file mode 100644 index 000000000..ac737908d --- /dev/null +++ b/test/system/015-help.bats @@ -0,0 +1,62 @@ +#!/usr/bin/env bats +# +# Tests based on 'podman help' +# +# Find all commands listed by 'podman --help'. Run each one, make sure it +# provides its own --help output. If the usage message ends in '[command]', +# treat it as a subcommand, and recurse into its own list of sub-subcommands. +# +# Any usage message that ends in '[flags]' is interpreted as a command +# that takes no further arguments; we confirm by running with 'invalid-arg' +# and confirming that it exits with error status and message. +# +load helpers + +# run 'podman help', parse the output looking for 'Available Commands'; +# return that list. +function podman_commands() { + dprint "$@" + run_podman help "$@" |\ + awk '/^Available Commands:/{ok=1;next}/^Flags:/{ok=0}ok { print $1 }' |\ + grep . + "$output" +} + + +function check_help() { + count=0 + for cmd in $(podman_commands "$@"); do + dprint "podman $@ $cmd --help" + run_podman "$@" $cmd --help + + # FIXME FIXME FIXME + usage=$(echo "$output" | grep -A2 '^Usage:' | grep . | tail -1) + # dprint "$usage" + [ -n "$usage" ] || die "podman $cmd: no Usage message found" + + # if ends in '[command]', recurse into subcommands + if expr "$usage" : '.*\[command\]$' >/dev/null; then + check_help "$@" $cmd + continue + fi + + # if ends in '[flag]' FIXME + if expr "$usage" : '.*\[flags\]$' >/dev/null; then + if [ "$cmd" != "help" ]; then + run_podman 125 "$@" $cmd invalid-arg + is "$output" "Error: .* takes no arguments" \ + "'podman $@ $cmd' with extra (invalid) arguments" + fi + fi + + count=$(expr $count + 1) + done + + [ $count -gt 0 ] || \ + die "Internal error: no commands found in 'podman help $@' list" +} + + +@test "podman help - basic tests" { + check_help +} diff --git a/test/system/030-run.bats b/test/system/030-run.bats new file mode 100644 index 000000000..d1f87d554 --- /dev/null +++ b/test/system/030-run.bats @@ -0,0 +1,32 @@ +#!/usr/bin/env bats + +load helpers + +@test "podman run - basic tests" { + rand=$(random_string 30) + tests=" +true | 0 | +false | 1 | +sh -c 'exit 32' | 32 | +echo $rand | 0 | $rand +/no/such/command | 127 | Error: container create failed:.*exec:.* no such file or dir +/etc | 126 | Error: container create failed:.*exec:.* permission denied +" + + while read cmd expected_rc expected_output; do + if [ "$expected_output" = "''" ]; then expected_output=""; fi + + # THIS IS TRICKY: this is what lets us handle a quoted command. + # Without this incantation (and the "$@" below), the cmd string + # gets passed on as individual tokens: eg "sh" "-c" "'exit" "32'" + # (note unmatched opening and closing single-quotes in the last 2). + # That results in a bizarre and hard-to-understand failure + # in the BATS 'run' invocation. + # This should really be done inside parse_table; I can't find + # a way to do so. + eval set "$cmd" + + run_podman $expected_rc run $PODMAN_TEST_IMAGE_FQN "$@" + is "$output" "$expected_output" "podman run $cmd - output" + done < <(parse_table "$tests") +} diff --git a/test/system/040-ps.bats b/test/system/040-ps.bats new file mode 100644 index 000000000..775df4af6 --- /dev/null +++ b/test/system/040-ps.bats @@ -0,0 +1,36 @@ +#!/usr/bin/env bats + +load helpers + +@test "podman ps - basic tests" { + rand_name=$(random_string 30) + + run_podman run -d --name $rand_name $PODMAN_TEST_IMAGE_FQN sleep 5 + cid=$output + is "$cid" "[0-9a-f]\{64\}$" + + # Special case: formatted ps + run_podman ps --no-trunc \ + --format '{{.ID}} {{.Image}} {{.Command}} {{.Names}}' + is "$output" "$cid $PODMAN_TEST_IMAGE_FQN sleep 5 $rand_name" "podman ps" + + + # Plain old regular ps + run_podman ps + is "${lines[1]}" \ + "${cid:0:12} \+$PODMAN_TEST_IMAGE_FQN \+sleep [0-9]\+ .*second.* $cname"\ + "output from podman ps" + + # OK. Wait for sleep to finish... + run_podman wait $cid + + # ...then make sure container shows up as stopped + run_podman ps -a + is "${lines[1]}" \ + "${cid:0:12} \+$PODMAN_TEST_IMAGE_FQN *sleep .* Exited .* $rand_name" \ + "podman ps -a" + + + + run_podman rm $cid +} diff --git a/test/system/050-stop.bats b/test/system/050-stop.bats new file mode 100644 index 000000000..d04e22606 --- /dev/null +++ b/test/system/050-stop.bats @@ -0,0 +1,65 @@ +#!/usr/bin/env bats + +load helpers + +# Very simple test +@test "podman stop - basic test" { + run_podman run -d $PODMAN_TEST_IMAGE_FQN sleep 60 + cid="$output" + + # Run 'stop'. Time how long it takes. + t0=$SECONDS + run_podman stop $cid + t1=$SECONDS + + # Confirm that container is stopped + run_podman inspect --format '{{.State.Status}} {{.State.ExitCode}}' $cid + is "$output" "exited \+137" "Status and exit code of stopped container" + + # The initial SIGTERM is ignored, so this operation should take + # exactly 10 seconds. Give it some leeway. + delta_t=$(( $t1 - $t0 )) + [ $delta_t -gt 8 ] ||\ + die "podman stop: ran too quickly! ($delta_t seconds; expected >= 10)" + [ $delta_t -le 14 ] ||\ + die "podman stop: took too long ($delta_t seconds; expected ~10)" + + run_podman rm $cid +} + + +# Test fallback + + +# Regression test for #2472 +@test "podman stop - can trap signal" { + # Because the --time and --timeout options can be wonky, try three + # different variations of this test. + for t_opt in '' '--time=5' '--timeout=5'; do + # Run a simple container that logs output on SIGTERM + run_podman run -d $PODMAN_TEST_IMAGE_FQN sh -c \ + "trap 'echo Received SIGTERM, finishing; exit' SIGTERM; echo READY; while :; do sleep 1; done" + cid="$output" + wait_for_ready $cid + + # Run 'stop' against it... + t0=$SECONDS + run_podman stop $t_opt $cid + t1=$SECONDS + + # ...the container should trap the signal, log it, and exit. + run_podman logs $cid + is "$output" ".*READY.*Received SIGTERM, finishing" "podman stop $t_opt" + + # Exit code should be 0, because container did its own exit + run_podman inspect --format '{{.State.ExitCode}}' $cid + is "$output" "0" "Exit code of stopped container" + + # The 'stop' command should return almost instantaneously + delta_t=$(( $t1 - $t0 )) + [ $delta_t -le 2 ] ||\ + die "podman stop: took too long ($delta_t seconds; expected <= 2)" + + run_podman rm $cid + done +} diff --git a/test/system/060-mount.bats b/test/system/060-mount.bats new file mode 100644 index 000000000..7e86c94dc --- /dev/null +++ b/test/system/060-mount.bats @@ -0,0 +1,35 @@ +#!/usr/bin/env bats + +load helpers + + +@test "podman mount - basic test" { + # Only works with root (FIXME: does it work with rootless + vfs?) + skip_if_rootless + + f_path=/tmp/tmpfile_$(random_string 8) + f_content=$(random_string 30) + + c_name=mount_test_$(random_string 5) + run_podman run --name $c_name $PODMAN_TEST_IMAGE_FQN \ + sh -c "echo $f_content > $f_path" + + run_podman mount $c_name + mount_path=$output + + test -d $mount_path + test -e "$mount_path/$f_path" + is $(< "$mount_path/$f_path") "$f_content" "contents of file, as read via fs" + + # Make sure that 'podman mount' (no args) returns the expected path + run_podman mount --notruncate + # FIXME: is it worth the effort to validate the CID ($1) ? + reported_mountpoint=$(echo "$output" | awk '{print $2}') + is $reported_mountpoint $mount_path "mountpoint reported by 'podman mount'" + + # umount, and make sure files are gone + run_podman umount $c_name + if [ -e "$mount_path/$f_path" ]; then + die "Mounted file exists even after umount: $mount_path/$f_path" + fi +} diff --git a/test/system/110-history.bats b/test/system/110-history.bats new file mode 100644 index 000000000..c00949bd1 --- /dev/null +++ b/test/system/110-history.bats @@ -0,0 +1,49 @@ +#!/usr/bin/env bats + +load helpers + +@test "podman history - basic tests" { + tests=" + | .*[0-9a-f]\\\{12\\\} .* CMD .* LABEL +--format '{{.ID}} {{.Created}}' | .*[0-9a-f]\\\{12\\\} .* ago +--human=false | .*[0-9a-f]\\\{12\\\} *[0-9-]\\\+T[0-9:]\\\+Z +-qH | .*[0-9a-f]\\\{12\\\} +--no-trunc | .*[0-9a-f]\\\{64\\\} +" + + parse_table "$tests" | while read options expect; do + if [ "$options" = "''" ]; then options=; fi + + eval set -- "$options" + + run_podman history "$@" $PODMAN_TEST_IMAGE_FQN + is "$output" "$expect" "podman history $options" + done +} + +@test "podman history - json" { + type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'" + + tests=" +id | [0-9a-f]\\\{64\\\} +created | [0-9-]\\\+T[0-9:]\\\+\\\.[0-9]\\\+Z +size | -\\\?[0-9]\\\+ +" + + run_podman history --format json $PODMAN_TEST_IMAGE_FQN + + parse_table "$tests" | while read field expect; do + # HACK: we can't include '|' in the table + if [ "$field" = "id" ]; then expect="$expect\|<missing>";fi + + # output is an array of dicts; check each one + count=$(echo "$output" | jq '. | length') + i=0 + while [ $i -lt $count ]; do + actual=$(echo "$output" | jq -r ".[$i].$field") + is "$actual" "$expect\$" "jq .[$i].$field" + i=$(expr $i + 1) + done + done + +} diff --git a/test/system/README.md b/test/system/README.md new file mode 100644 index 000000000..6853b68f3 --- /dev/null +++ b/test/system/README.md @@ -0,0 +1,65 @@ +Quick overview of podman system tests. The idea is to use BATS, +but with a framework for making it easy to add new tests and to +debug failures. + +Quick Start +=========== + +Look at [030-run.bats](030-run.bats) for a simple but packed example. +This introduces the basic set of helper functions: + +* `setup` (implicit) - resets container storage so there's +one and only one (standard) image, and no running containers. + +* `parse_table` - you can define tables of inputs and expected results, +then read those in a `while` loop. This makes it easy to add new tests. +Because bash is not a programming language, the caller of `parse_table` +sometimes needs to massage the returned values; `015-run.bats` offers +examples of how to deal with the more typical such issues. + +* `run_podman` - runs command defined in `$PODMAN` (default: 'podman' +but could also be 'podman-remote'), with a timeout. Checks its exit status. + +* `is` - compare actual vs expected output. Emits a useful diagnostic +on failure. + +* `random_string` - returns a pseudorandom alphanumeric string + +Test files are of the form `NNN-name.bats` where NNN is a three-digit +number. Please preserve this convention, it simplifies viewing the +directory and understanding test order. Most of the time it's not +important but `00x` should be reserved for the times when it matters. + + +Analyzing test failures +======================= + +The top priority for this scheme is to make it easy to diagnose +what went wrong. To that end, `podman_run` always logs all invoked +commands, their output and exit codes. In a normal run you will never +see this, but BATS will display it on failure. The goal here is to +give you everything you need to diagnose without having to rerun tests. + +The `is` comparison function is designed to emit useful diagnostics, +in particular, the actual and expected strings. Please do not use +the horrible BATS standard of `[ x = y ]`; that's nearly useless +for tracking down failures. + +If the above are not enough to help you track down a failure: + + +Debugging tests +--------------- + +Some functions have `dprint` statements. To see the output of these, +set `PODMAN_TEST_DEBUG="funcname"` where `funcname` is the name of +the function or perhaps just a substring. + + +Further Details +=============== + +TBD. For now, look in [helpers.bash](helpers.bash); each helper function +has (what are intended to be) helpful header comments. For even more +examples, see and/or run `helpers.t`; that's a regression test +and provides a thorough set of examples of how the helpers work. diff --git a/test/system/helpers.bash b/test/system/helpers.bash new file mode 100644 index 000000000..fd204e1cf --- /dev/null +++ b/test/system/helpers.bash @@ -0,0 +1,315 @@ +# -*- bash -*- + +# Podman command to run; may be podman-remote +PODMAN=${PODMAN:-podman} + +# Standard image to use for most tests +PODMAN_TEST_IMAGE_REGISTRY=${PODMAN_TEST_IMAGE_REGISTRY:-"quay.io"} +PODMAN_TEST_IMAGE_USER=${PODMAN_TEST_IMAGE_USER:-"libpod"} +PODMAN_TEST_IMAGE_NAME=${PODMAN_TEST_IMAGE_NAME:-"alpine_labels"} +PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"latest"} +PODMAN_TEST_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG" + +# Default timeout for a podman command. +PODMAN_TIMEOUT=${PODMAN_TIMEOUT:-60} + +############################################################################### +# BEGIN setup/teardown tools + +# Provide common setup and teardown functions, but do not name them such! +# That way individual tests can override with their own setup/teardown, +# while retaining the ability to include these if they so desire. + +# Setup helper: establish a test environment with exactly the images needed +function basic_setup() { + # Clean up all containers + run_podman rm --all --force + + # Clean up all images except those desired + found_needed_image= + run_podman images --all --format '{{.Repository}}:{{.Tag}} {{.ID}}' + for line in "${lines[@]}"; do + set $line + if [ "$1" == "$PODMAN_TEST_IMAGE_FQN" ]; then + found_needed_image=1 + else + echo "# setup_standard_environment: podman rmi $1 & $2" >&3 + podman rmi --force "$1" >/dev/null 2>&1 || true + podman rmi --force "$2" >/dev/null 2>&1 || true + fi + done + + # Make sure desired images are present + if [ -z "$found_needed_image" ]; then + run_podman pull "$PODMAN_TEST_IMAGE_FQN" + fi +} + +# Basic teardown: remove all containers +function basic_teardown() { + run_podman rm --all --force +} + + +# Provide the above as default methods. +function setup() { + basic_setup +} + +function teardown() { + basic_teardown +} + + +# Helpers useful for tests running rmi +function archive_image() { + local image=$1 + + # FIXME: refactor? + archive_basename=$(echo $1 | tr -c a-zA-Z0-9._- _) + archive=$BATS_TMPDIR/$archive_basename.tar + + run_podman save -o $archive $image +} + +function restore_image() { + local image=$1 + + archive_basename=$(echo $1 | tr -c a-zA-Z0-9._- _) + archive=$BATS_TMPDIR/$archive_basename.tar + + run_podman restore $archive +} + +# END setup/teardown tools +############################################################################### +# BEGIN podman helpers + +################ +# run_podman # Invoke $PODMAN, with timeout, using BATS 'run' +################ +# +# This is the preferred mechanism for invoking podman: first, it +# invokes $PODMAN, which may be 'podman-remote' or '/some/path/podman'. +# +# Second, we use 'timeout' to abort (with a diagnostic) if something +# takes too long; this is preferable to a CI hang. +# +# Third, we log the command run and its output. This doesn't normally +# appear in BATS output, but it will if there's an error. +# +# Next, we check exit status. Since the normal desired code is 0, +# that's the default; but the first argument can override: +# +# run_podman 125 nonexistent-subcommand +# run_podman '?' some-other-command # let our caller check status +# +# Since we use the BATS 'run' mechanism, $output and $status will be +# defined for our caller. +# +function run_podman() { + # Number as first argument = expected exit code; default 0 + expected_rc=0 + case "$1" in + [0-9]) expected_rc=$1; shift;; + [1-9][0-9]) expected_rc=$1; shift;; + [12][0-9][0-9]) expected_rc=$1; shift;; + '?') expected_rc= ; shift;; # ignore exit code + esac + + # stdout is only emitted upon error; this echo is to help a debugger + echo "\$ $PODMAN $@" + run timeout --foreground -v --kill=10 $PODMAN_TIMEOUT $PODMAN "$@" + # without "quotes", multiple lines are glommed together into one + echo "$output" + if [ "$status" -ne 0 ]; then + echo -n "[ rc=$status "; + if [ -n "$expected_rc" ]; then + if [ "$status" -eq "$expected_rc" ]; then + echo -n "(expected) "; + else + echo -n "(** EXPECTED $expected_rc **) "; + fi + fi + echo "]" + fi + + if [ "$status" -eq 124 ]; then + if expr "$output" : ".*timeout: sending" >/dev/null; then + echo "*** TIMED OUT ***" + false + fi + fi + + if [ -n "$expected_rc" ]; then + if [ "$status" -ne "$expected_rc" ]; then + die "FAIL: exit code is $status; expected $expected_rc" + fi + fi +} + + +# Wait for 'READY' in container output +function wait_for_ready { + local cid= + local sleep_delay=5 + local how_long=60 + + # Arg processing. A single-digit number is how long to sleep between + # iterations; a 2- or 3-digit number is the total time to wait; anything + # else is the container ID or name to wait on. + local i + for i in "$@"; do + if expr "$i" : '[0-9]\+$' >/dev/null; then + if [ $i -le 9 ]; then + sleep_delay=$i + else + how_long=$i + fi + else + cid=$i + fi + done + + [ -n "$cid" ] || die "FATAL: wait_for_ready: no container name/ID in '$*'" + + t1=$(expr $SECONDS + $how_long) + while [ $SECONDS -lt $t1 ]; do + run_podman logs $cid + if expr "$output" : ".*READY" >/dev/null; then + return + fi + + sleep $sleep_delay + done + + die "FAIL: timed out waiting for READY from $cid" +} + +# END podman helpers +############################################################################### +# BEGIN miscellaneous tools + +###################### +# skip_if_rootless # ...with an optional message +###################### +function skip_if_rootless() { + if [ "$(id -u)" -eq 0 ]; then + return + fi + + skip "${1:-not applicable under rootless podman}" +} + + +######### +# die # Abort with helpful message +######### +function die() { + echo "# $*" >&2 + false +} + + +######## +# is # Compare actual vs expected string; fail w/diagnostic if mismatch +######## +# +# Compares given string against expectations, using 'expr' to allow patterns. +# +# Examples: +# +# is "$actual" "$expected" "descriptive test name" +# is "apple" "orange" "name of a test that will fail in most universes" +# is "apple" "[a-z]\+" "this time it should pass" +# +function is() { + local actual="$1" + local expect="$2" + local testname="${3:-FIXME}" + + if [ -z "$expect" ]; then + if [ -z "$actual" ]; then + return + fi + die "$testname:\n# expected no output; got %q\n" "$actual" + fi + + if expr "$actual" : "$expect" >/dev/null; then + return + fi + + # This is a multi-line message, so let's format it ourself (not via die) + printf "# $testname:\n# expected: %q\n# actual: %q\n" \ + "$expect" "$actual" >&2 + false +} + + +############ +# dprint # conditional debug message +############ +# +# Set PODMAN_TEST_DEBUG to the name of one or more functions you want to debug +# +# Examples: +# +# $ PODMAN_TEST_DEBUG=parse_table bats . +# $ PODMAN_TEST_DEBUG="test_podman_images test_podman_run" bats . +# +function dprint() { + test -z "$PODMAN_TEST_DEBUG" && return + + caller="${FUNCNAME[1]}" + + # PODMAN_TEST_DEBUG is a space-separated list of desired functions + # e.g. "parse_table test_podman_images" (or even just "table") + for want in $PODMAN_TEST_DEBUG; do + # Check if our calling function matches any of the desired strings + if expr "$caller" : ".*$want" >/dev/null; then + echo "# ${FUNCNAME[1]}() : $*" >&3 + return + fi + done +} + + +################# +# parse_table # Split a table on '|' delimiters; return space-separated +################# +# +# See sample .bats scripts for examples. The idea is to list a set of +# tests in a table, then use simple logic to iterate over each test. +# Columns are separated using '|' (pipe character) because sometimes +# we need spaces in our fields. +# +function parse_table() { + while read line; do + test -z "$line" && continue + + declare -a row=() + while read col; do + dprint "col=<<$col>>" + row+=("$col") + done < <(echo "$line" | tr '|' '\012' | sed -e 's/^ *//' -e 's/\\/\\\\/g') + + printf "%q " "${row[@]}" + printf "\n" + done <<<"$1" +} + + +################### +# random_string # Returns a pseudorandom human-readable string +################### +# +# Numeric argument, if present, is desired length of string +# +function random_string() { + local length=${1:-10} + + head /dev/urandom | tr -dc a-zA-Z0-9 | head -c$length +} + +# END miscellaneous tools +############################################################################### diff --git a/test/system/helpers.t b/test/system/helpers.t new file mode 100755 index 000000000..7b4e48a84 --- /dev/null +++ b/test/system/helpers.t @@ -0,0 +1,145 @@ +#!/bin/bash +# +# regression tests for helpers.bash +# +# Some of those helper functions are fragile, and we don't want to break +# anything if we have to mess with them. +# + +source $(dirname $0)/helpers.bash + +die() { + echo "$(basename $0): $*" >&2 + exit 1 +} + +# Iterator and return code; updated in check_result() +testnum=0 +rc=0 + +############################################################################### +# BEGIN test the parse_table helper + +function check_result { + testnum=$(expr $testnum + 1) + if [ "$1" = "$2" ]; then + echo "ok $testnum $3 = $1" + else + echo "not ok $testnum $3" + echo "# expected: $2" + echo "# actual: $1" + rc=1 + fi +} + +# IMPORTANT NOTE: you have to do +# this: while ... done < <(parse_table) +# and not: parse_table | while read ... +# +# ...because piping to 'while' makes it a subshell, hence testnum and rc +# will not be updated. +# +while read x y z; do + check_result "$x" "a" "parse_table simple: column 1" + check_result "$y" "b" "parse_table simple: column 2" + check_result "$z" "c" "parse_table simple: column 3" +done < <(parse_table "a | b | c") + +# More complicated example, with spaces +while read x y z; do + check_result "$x" "a b" "parse_table with spaces: column 1" + check_result "$y" "c d" "parse_table with spaces: column 2" + check_result "$z" "e f g" "parse_table with spaces: column 3" +done < <(parse_table "a b | c d | e f g") + +# Multi-row, with spaces and with blank lines +table=" +a | b | c d e +d e f | g h | i j +" +declare -A expect=( + [0,0]="a" + [0,1]="b" + [0,2]="c d e" + [1,0]="d e f" + [1,1]="g h" + [1,2]="i j" +) +row=0 +while read x y z;do + check_result "$x" "${expect[$row,0]}" "parse_table multi_row[$row,0]" + check_result "$y" "${expect[$row,1]}" "parse_table multi_row[$row,1]" + check_result "$z" "${expect[$row,2]}" "parse_table multi_row[$row,2]" + row=$(expr $row + 1) +done < <(parse_table "$table") + +# Backslash handling. The first element should have none, the second some +while read x y;do + check_result "$x" '[0-9]{2}' "backslash test - no backslashes" + check_result "$y" '[0-9]\{3\}' "backslash test - one backslash each" +done < <(parse_table "[0-9]{2} | [0-9]\\\{3\\\}") + +# Empty strings. I wish we could convert those to real empty strings. +while read x y z; do + check_result "$x" "''" "empty string - left-hand" + check_result "$y" "''" "empty string - middle" + check_result "$z" "''" "empty string - right" +done < <(parse_table " | |") + +# Quotes +while read x y z;do + check_result "$x" "a 'b c'" "single quotes" + check_result "$y" "d \"e f\" g" "double quotes" + check_result "$z" "h" "no quotes" + + # FIXME FIXME FIXME: this is the only way I can find to get bash-like + # splitting of tokens. It really should be done inside parse_table + # but I can't find any way of doing so. If you can find a way, please + # update this test and any BATS tests that rely on quoting. + eval set "$x" + check_result "$1" "a" "single quotes - token split - 1" + check_result "$2" "b c" "single quotes - token split - 2" + check_result "$3" "" "single quotes - token split - 3" + + eval set "$y" + check_result "$1" "d" "double quotes - token split - 1" + check_result "$2" "e f" "double quotes - token split - 2" + check_result "$3" "g" "double quotes - token split - 3" +done < <(parse_table "a 'b c' | d \"e f\" g | h") + +# END test the parse_table helper +############################################################################### +# BEGIN dprint + +function dprint_test_1() { + dprint "$*" +} + +# parse_table works, might as well use it +# +# <value of PODMAN_TEST_DEBUG> | <blank for no msg, - for msg> | <desc> +# +table=" + | | debug unset +dprint_test | - | substring match +dprint_test_1 | - | exact match +dprint_test_10 | | caller name mismatch +xxx yyy zzz | | multiple callers, no match +dprint_test_1 xxx yyy zzz | - | multiple callers, match at start +xxx dprint_test_1 yyy zzz | - | multiple callers, match in middle +xxx yyy zzz dprint_test_1 | - | multiple callers, match at end +" +while read var expect name; do + random_string=$(random_string 20) + PODMAN_TEST_DEBUG="$var" result=$(dprint_test_1 "$random_string" 3>&1) + expect_full="" + if [ -n "$expect" -a "$expect" != "''" ]; then + expect_full="# dprint_test_1() : $random_string" + fi + check_result "$result" "$expect_full" "DEBUG='$var' - $name" +done < <(parse_table "$table") + +# END dprint +############################################################################### + +exit $rc diff --git a/test/system/libpod_suite_test.go b/test/system/libpod_suite_test.go deleted file mode 100644 index 5de50e4e7..000000000 --- a/test/system/libpod_suite_test.go +++ /dev/null @@ -1,217 +0,0 @@ -package system - -import ( - "fmt" - "os" - "strings" - "testing" - - . "github.com/containers/libpod/test/utils" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var ( - PODMAN_BINARY string - GLOBALOPTIONS = []string{"--cgroup-manager", - "--cni-config-dir", - "--config", "-c", - "--conmon", - "--cpu-profile", - "--log-level", - "--root", - "--tmpdir", - "--runroot", - "--runtime", - "--storage-driver", - "--storage-opt", - "--syslog", - } - PODMAN_SUBCMD = []string{"attach", - "commit", - "container", - "build", - "create", - "diff", - "exec", - "export", - "history", - "image", - "images", - "import", - "info", - "inspect", - "kill", - "load", - "login", - "logout", - "logs", - "mount", - "pause", - "ps", - "pod", - "port", - "pull", - "push", - "restart", - "rm", - "rmi", - "run", - "save", - "search", - "start", - "stats", - "stop", - "tag", - "top", - "umount", - "unpause", - "version", - "wait", - "h", - } - INTEGRATION_ROOT string - ARTIFACT_DIR = "/tmp/.artifacts" - ALPINE = "docker.io/library/alpine:latest" - BB = "docker.io/library/busybox:latest" - BB_GLIBC = "docker.io/library/busybox:glibc" - fedoraMinimal = "registry.fedoraproject.org/fedora-minimal:latest" - nginx = "quay.io/baude/alpine_nginx:latest" - redis = "docker.io/library/redis:alpine" - registry = "docker.io/library/registry:2" - infra = "k8s.gcr.io/pause:3.1" - defaultWaitTimeout = 90 -) - -// PodmanTestSystem struct for command line options -type PodmanTestSystem struct { - PodmanTest - GlobalOptions map[string]string - PodmanCmdOptions map[string][]string -} - -// TestLibpod ginkgo master function -func TestLibpod(t *testing.T) { - RegisterFailHandler(Fail) - RunSpecs(t, "Libpod Suite") -} - -var _ = BeforeSuite(func() { -}) - -// PodmanTestCreate creates a PodmanTestSystem instance for the tests -func PodmanTestCreate(tempDir string) *PodmanTestSystem { - var envKey string - globalOptions := make(map[string]string) - podmanCmdOptions := make(map[string][]string) - - for _, n := range GLOBALOPTIONS { - envKey = strings.Replace(strings.ToUpper(strings.Trim(n, "-")), "-", "_", -1) - if isEnvSet(envKey) { - globalOptions[n] = os.Getenv(envKey) - } - } - - for _, n := range PODMAN_SUBCMD { - envKey = strings.Replace("PODMAN_SUBCMD_OPTIONS", "SUBCMD", strings.ToUpper(n), -1) - if isEnvSet(envKey) { - podmanCmdOptions[n] = strings.Split(os.Getenv(envKey), " ") - } - } - - podmanBinary := "podman" - if os.Getenv("PODMAN_BINARY") != "" { - podmanBinary = os.Getenv("PODMAN_BINARY") - } - - p := &PodmanTestSystem{ - PodmanTest: PodmanTest{ - PodmanBinary: podmanBinary, - ArtifactPath: ARTIFACT_DIR, - TempDir: tempDir, - }, - GlobalOptions: globalOptions, - PodmanCmdOptions: podmanCmdOptions, - } - - p.PodmanMakeOptions = p.makeOptions - - return p -} - -func (p *PodmanTestSystem) Podman(args []string) *PodmanSession { - return p.PodmanBase(args) -} - -//MakeOptions assembles all the podman options -func (p *PodmanTestSystem) makeOptions(args []string) []string { - var addOptions, subArgs []string - for _, n := range GLOBALOPTIONS { - if p.GlobalOptions[n] != "" { - addOptions = append(addOptions, n, p.GlobalOptions[n]) - } - } - - if len(args) == 0 { - return addOptions - } - - subCmd := args[0] - addOptions = append(addOptions, subCmd) - if subCmd == "unmount" { - subCmd = "umount" - } - if subCmd == "help" { - subCmd = "h" - } - - if _, ok := p.PodmanCmdOptions[subCmd]; ok { - m := make(map[string]bool) - subArgs = p.PodmanCmdOptions[subCmd] - for i := 0; i < len(subArgs); i++ { - m[subArgs[i]] = true - } - for i := 1; i < len(args); i++ { - if _, ok := m[args[i]]; !ok { - subArgs = append(subArgs, args[i]) - } - } - } else { - subArgs = args[1:] - } - - addOptions = append(addOptions, subArgs...) - - return addOptions -} - -// Cleanup cleans up the temporary store -func (p *PodmanTestSystem) Cleanup() { - // Remove all containers - stopall := p.Podman([]string{"stop", "-a", "--timeout", "0"}) - stopall.WaitWithDefaultTimeout() - - session := p.Podman([]string{"rm", "-fa"}) - session.Wait(90) - // Nuke tempdir - if err := os.RemoveAll(p.TempDir); err != nil { - fmt.Printf("%q\n", err) - } -} - -// CleanupPod cleans up the temporary store -func (p *PodmanTestSystem) CleanupPod() { - // Remove all containers - session := p.Podman([]string{"pod", "rm", "-fa"}) - session.Wait(90) - // Nuke tempdir - if err := os.RemoveAll(p.TempDir); err != nil { - fmt.Printf("%q\n", err) - } -} - -// Check if the key is set in Env -func isEnvSet(key string) bool { - _, set := os.LookupEnv(key) - return set -} diff --git a/test/system/version_test.go b/test/system/version_test.go deleted file mode 100644 index ada0093b7..000000000 --- a/test/system/version_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package system - -import ( - "fmt" - "os" - "regexp" - - . "github.com/containers/libpod/test/utils" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" -) - -var _ = Describe("Podman version test", func() { - var ( - tempdir string - err error - podmanTest *PodmanTestSystem - ) - - BeforeEach(func() { - tempdir, err = CreateTempDirInTempDir() - if err != nil { - os.Exit(1) - } - podmanTest = PodmanTestCreate(tempdir) - }) - - AfterEach(func() { - podmanTest.Cleanup() - f := CurrentGinkgoTestDescription() - timedResult := fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds()) - GinkgoWriter.Write([]byte(timedResult)) - }) - - It("Smoking test: podman version with extra args", func() { - logc := podmanTest.Podman([]string{"version", "anything", "-", "--"}) - logc.WaitWithDefaultTimeout() - Expect(logc.ExitCode()).To(Equal(0)) - ver := logc.OutputToString() - Expect(regexp.MatchString("Version:.*?Go Version:.*?OS/Arch", ver)).To(BeTrue()) - }) - - It("Negative test: podman version with extra flag", func() { - logc := podmanTest.Podman([]string{"version", "--foo"}) - logc.WaitWithDefaultTimeout() - Expect(logc.ExitCode()).NotTo(Equal(0)) - err, _ := logc.GrepString("Incorrect Usage: flag provided but not defined: -foo") - Expect(err).To(BeTrue()) - }) - -}) |