From 681eae9bcc856f8dad107765a97c29d0fe093d4a Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Wed, 20 Feb 2019 13:19:20 -0700 Subject: new system tests under BATS Initial attempt at writing a framework for podman system tests. The idea is to define a useful set of primitives that will make it easy to write actual tests and to interpret results of failing ones. This is a proof-of-concept right now; only a small number of tests, by no means comprehensive. I am requesting review in order to find showstopper problems: reasons why this approach cannot work. Should there be none, we can work toward running these as gating tests for Fedora and RHEL8. Signed-off-by: Ed Santiago --- test/system/helpers.bash | 315 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 315 insertions(+) create mode 100644 test/system/helpers.bash (limited to 'test/system/helpers.bash') diff --git a/test/system/helpers.bash b/test/system/helpers.bash new file mode 100644 index 000000000..fd204e1cf --- /dev/null +++ b/test/system/helpers.bash @@ -0,0 +1,315 @@ +# -*- bash -*- + +# Podman command to run; may be podman-remote +PODMAN=${PODMAN:-podman} + +# Standard image to use for most tests +PODMAN_TEST_IMAGE_REGISTRY=${PODMAN_TEST_IMAGE_REGISTRY:-"quay.io"} +PODMAN_TEST_IMAGE_USER=${PODMAN_TEST_IMAGE_USER:-"libpod"} +PODMAN_TEST_IMAGE_NAME=${PODMAN_TEST_IMAGE_NAME:-"alpine_labels"} +PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"latest"} +PODMAN_TEST_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG" + +# Default timeout for a podman command. +PODMAN_TIMEOUT=${PODMAN_TIMEOUT:-60} + +############################################################################### +# BEGIN setup/teardown tools + +# Provide common setup and teardown functions, but do not name them such! +# That way individual tests can override with their own setup/teardown, +# while retaining the ability to include these if they so desire. + +# Setup helper: establish a test environment with exactly the images needed +function basic_setup() { + # Clean up all containers + run_podman rm --all --force + + # Clean up all images except those desired + found_needed_image= + run_podman images --all --format '{{.Repository}}:{{.Tag}} {{.ID}}' + for line in "${lines[@]}"; do + set $line + if [ "$1" == "$PODMAN_TEST_IMAGE_FQN" ]; then + found_needed_image=1 + else + echo "# setup_standard_environment: podman rmi $1 & $2" >&3 + podman rmi --force "$1" >/dev/null 2>&1 || true + podman rmi --force "$2" >/dev/null 2>&1 || true + fi + done + + # Make sure desired images are present + if [ -z "$found_needed_image" ]; then + run_podman pull "$PODMAN_TEST_IMAGE_FQN" + fi +} + +# Basic teardown: remove all containers +function basic_teardown() { + run_podman rm --all --force +} + + +# Provide the above as default methods. +function setup() { + basic_setup +} + +function teardown() { + basic_teardown +} + + +# Helpers useful for tests running rmi +function archive_image() { + local image=$1 + + # FIXME: refactor? + archive_basename=$(echo $1 | tr -c a-zA-Z0-9._- _) + archive=$BATS_TMPDIR/$archive_basename.tar + + run_podman save -o $archive $image +} + +function restore_image() { + local image=$1 + + archive_basename=$(echo $1 | tr -c a-zA-Z0-9._- _) + archive=$BATS_TMPDIR/$archive_basename.tar + + run_podman restore $archive +} + +# END setup/teardown tools +############################################################################### +# BEGIN podman helpers + +################ +# run_podman # Invoke $PODMAN, with timeout, using BATS 'run' +################ +# +# This is the preferred mechanism for invoking podman: first, it +# invokes $PODMAN, which may be 'podman-remote' or '/some/path/podman'. +# +# Second, we use 'timeout' to abort (with a diagnostic) if something +# takes too long; this is preferable to a CI hang. +# +# Third, we log the command run and its output. This doesn't normally +# appear in BATS output, but it will if there's an error. +# +# Next, we check exit status. Since the normal desired code is 0, +# that's the default; but the first argument can override: +# +# run_podman 125 nonexistent-subcommand +# run_podman '?' some-other-command # let our caller check status +# +# Since we use the BATS 'run' mechanism, $output and $status will be +# defined for our caller. +# +function run_podman() { + # Number as first argument = expected exit code; default 0 + expected_rc=0 + case "$1" in + [0-9]) expected_rc=$1; shift;; + [1-9][0-9]) expected_rc=$1; shift;; + [12][0-9][0-9]) expected_rc=$1; shift;; + '?') expected_rc= ; shift;; # ignore exit code + esac + + # stdout is only emitted upon error; this echo is to help a debugger + echo "\$ $PODMAN $@" + run timeout --foreground -v --kill=10 $PODMAN_TIMEOUT $PODMAN "$@" + # without "quotes", multiple lines are glommed together into one + echo "$output" + if [ "$status" -ne 0 ]; then + echo -n "[ rc=$status "; + if [ -n "$expected_rc" ]; then + if [ "$status" -eq "$expected_rc" ]; then + echo -n "(expected) "; + else + echo -n "(** EXPECTED $expected_rc **) "; + fi + fi + echo "]" + fi + + if [ "$status" -eq 124 ]; then + if expr "$output" : ".*timeout: sending" >/dev/null; then + echo "*** TIMED OUT ***" + false + fi + fi + + if [ -n "$expected_rc" ]; then + if [ "$status" -ne "$expected_rc" ]; then + die "FAIL: exit code is $status; expected $expected_rc" + fi + fi +} + + +# Wait for 'READY' in container output +function wait_for_ready { + local cid= + local sleep_delay=5 + local how_long=60 + + # Arg processing. A single-digit number is how long to sleep between + # iterations; a 2- or 3-digit number is the total time to wait; anything + # else is the container ID or name to wait on. + local i + for i in "$@"; do + if expr "$i" : '[0-9]\+$' >/dev/null; then + if [ $i -le 9 ]; then + sleep_delay=$i + else + how_long=$i + fi + else + cid=$i + fi + done + + [ -n "$cid" ] || die "FATAL: wait_for_ready: no container name/ID in '$*'" + + t1=$(expr $SECONDS + $how_long) + while [ $SECONDS -lt $t1 ]; do + run_podman logs $cid + if expr "$output" : ".*READY" >/dev/null; then + return + fi + + sleep $sleep_delay + done + + die "FAIL: timed out waiting for READY from $cid" +} + +# END podman helpers +############################################################################### +# BEGIN miscellaneous tools + +###################### +# skip_if_rootless # ...with an optional message +###################### +function skip_if_rootless() { + if [ "$(id -u)" -eq 0 ]; then + return + fi + + skip "${1:-not applicable under rootless podman}" +} + + +######### +# die # Abort with helpful message +######### +function die() { + echo "# $*" >&2 + false +} + + +######## +# is # Compare actual vs expected string; fail w/diagnostic if mismatch +######## +# +# Compares given string against expectations, using 'expr' to allow patterns. +# +# Examples: +# +# is "$actual" "$expected" "descriptive test name" +# is "apple" "orange" "name of a test that will fail in most universes" +# is "apple" "[a-z]\+" "this time it should pass" +# +function is() { + local actual="$1" + local expect="$2" + local testname="${3:-FIXME}" + + if [ -z "$expect" ]; then + if [ -z "$actual" ]; then + return + fi + die "$testname:\n# expected no output; got %q\n" "$actual" + fi + + if expr "$actual" : "$expect" >/dev/null; then + return + fi + + # This is a multi-line message, so let's format it ourself (not via die) + printf "# $testname:\n# expected: %q\n# actual: %q\n" \ + "$expect" "$actual" >&2 + false +} + + +############ +# dprint # conditional debug message +############ +# +# Set PODMAN_TEST_DEBUG to the name of one or more functions you want to debug +# +# Examples: +# +# $ PODMAN_TEST_DEBUG=parse_table bats . +# $ PODMAN_TEST_DEBUG="test_podman_images test_podman_run" bats . +# +function dprint() { + test -z "$PODMAN_TEST_DEBUG" && return + + caller="${FUNCNAME[1]}" + + # PODMAN_TEST_DEBUG is a space-separated list of desired functions + # e.g. "parse_table test_podman_images" (or even just "table") + for want in $PODMAN_TEST_DEBUG; do + # Check if our calling function matches any of the desired strings + if expr "$caller" : ".*$want" >/dev/null; then + echo "# ${FUNCNAME[1]}() : $*" >&3 + return + fi + done +} + + +################# +# parse_table # Split a table on '|' delimiters; return space-separated +################# +# +# See sample .bats scripts for examples. The idea is to list a set of +# tests in a table, then use simple logic to iterate over each test. +# Columns are separated using '|' (pipe character) because sometimes +# we need spaces in our fields. +# +function parse_table() { + while read line; do + test -z "$line" && continue + + declare -a row=() + while read col; do + dprint "col=<<$col>>" + row+=("$col") + done < <(echo "$line" | tr '|' '\012' | sed -e 's/^ *//' -e 's/\\/\\\\/g') + + printf "%q " "${row[@]}" + printf "\n" + done <<<"$1" +} + + +################### +# random_string # Returns a pseudorandom human-readable string +################### +# +# Numeric argument, if present, is desired length of string +# +function random_string() { + local length=${1:-10} + + head /dev/urandom | tr -dc a-zA-Z0-9 | head -c$length +} + +# END miscellaneous tools +############################################################################### -- cgit v1.2.3-54-g00ecf From 589248d2f359dea73fc763ac587e2927f005b300 Mon Sep 17 00:00:00 2001 From: Ed Santiago Date: Tue, 5 Mar 2019 09:58:30 -0700 Subject: Implement review feedback - document a recommended convention for fail-fast tests - document the requirement for jq. (And, add a fail-fast test for its presence; remove the duplicated checks in subtests) - add further sanity checks to 'help' test. Add missing documentation. Remove a no-longer-needed workaround for usage-message bug fixed in #2486 - add a documented TEMPLATE - and, since we're at 1.1, enable 'Remote API' check in version test - better diagnostics in setup/teardown; add vim filetype hint; better formatting of actual-vs-expect errors - new pod-top, logs, build tests - improve error messages - add $IMAGE alias for ridiculous $PODMAN_TEST_IMAGE_FQN - final cleanup, in prep for merge Signed-off-by: Ed Santiago --- test/system/000-TEMPLATE | 114 +++++++++++++++++++++++++++++++++++++++++++ test/system/001-basic.bats | 24 ++++++--- test/system/005-info.bats | 4 +- test/system/010-images.bats | 4 +- test/system/015-help.bats | 26 +++++++--- test/system/030-run.bats | 4 +- test/system/035-logs.bats | 24 +++++++++ test/system/040-ps.bats | 10 ++-- test/system/050-stop.bats | 6 ++- test/system/060-mount.bats | 4 +- test/system/070-build.bats | 28 +++++++++++ test/system/110-history.bats | 8 +-- test/system/200-pod-top.bats | 37 ++++++++++++++ test/system/README.md | 23 +++++++-- test/system/helpers.bash | 82 ++++++++++++++++++++++--------- 15 files changed, 343 insertions(+), 55 deletions(-) create mode 100644 test/system/000-TEMPLATE create mode 100644 test/system/035-logs.bats create mode 100644 test/system/070-build.bats create mode 100644 test/system/200-pod-top.bats (limited to 'test/system/helpers.bash') diff --git a/test/system/000-TEMPLATE b/test/system/000-TEMPLATE new file mode 100644 index 000000000..296ed4d58 --- /dev/null +++ b/test/system/000-TEMPLATE @@ -0,0 +1,114 @@ +#!/usr/bin/env bats -*- bats -*- +# +# FIXME: short description of the purpose of this module +# +# FIXME: copy this file to 'NNN-yourtestname.bats' and edit as needed. +# + +load helpers + +@test "podman subcmd - description of this particular test" { + args="some sort of argument list" + run_podman subcmd $args + is "$output" "what we expect" "output from 'podman subcmd $args'" +} + +# vim: filetype=sh + +############################################################################### +# +# FIXME FIXME FIXME: Most of the time you can cut from here on down. +# FIXME FIXME FIXME: The above template is probably enough for many tests. +# FIXME FIXME FIXME: +# FIXME FIXME FIXME: If you need anything more complicated, read on. +# +# FIXME: This is a bloated test template. It provides mostly stuff for you +# FIXME: to remove, plus stuff for you to base your tests on. +# FIXME: +# FIXME: copy this file to 'NNN-yourtestname.bats' and edit as needed. +# FIXME: Read all FIXMEs, act on them as needed, then remove them. +# FIXME: test w/ $ PODMAN=./bin/podman bats test/system/NNN-yourtestname.bats +# + +load helpers + +# FIXME: DELETE THESE LINES UNLESS YOU ABSOLUTELY NEED THEM. +# FIXME: Most tests will not need a custom setup/teardown: they are +# FIXME: provided by helpers.bash. +# FIXME: But if you have to do anything special, these give you the +# FIXME: names of the standard setup/teardown so you can call them +# FIXME: before or after your own additions. +function setup() { + basic_setup + # FIXME: you almost certainly want to do your own setup _after_ basic. +} +function teardown() { + # FIXME: you almost certainly want to do your own teardown _before_ basic. + basic_teardown +} + + +# FIXME: very basic one-pass example +@test "podman FOO - description of test" { + # FIXME: please try to remove this line; that is, try to write tests + # that will pass as both root and rootless. + skip_if_rootless + + # FIXME: template for run commands. Always use 'run_podman'! + # FIXME: The '?' means 'ignore exit status'; use a number if you + # FIXME: expect a precise nonzero code, or omit for 0 (usual case). + # FIXME: NEVER EVER RUN 'podman' DIRECTLY. See helpers.bash for why. + run_podman '?' run -d $IMAGE sh -c 'prep..; echo READY' + cid="$output" + wait_for_ready $cid + + run_podman logs $cid + # FIXME: example of dprint. This will trigger if PODMAN_TEST_DEBUG=FOO + # FIXME: ...or anything that matches the name assigned in the @test line. + dprint "podman logs $cid -> '$output'" + is "$output" "what are we expecting?" "description of this check" + + # Clean up + run_podman rm $cid +} + + +# FIXME: another example, this time with a test table loop +@test "podman FOO - json - template for playing with json output" { + # FIXME: Define a multiline string in tabular form, using '|' as separator. + # FIXME: Each row defines one test. Each column (there may be as many as + # FIXME: you want) is one field. In the case below we have two, a + # FIXME: json field descriptor and an expected value. + tests=" +id | [0-9a-f]\\\{64\\\} +created | [0-9-]\\\+T[0-9:]\\\+\\\.[0-9]\\\+Z +size | -\\\?[0-9]\\\+ +" + + # FIXME: Run a basic podman command. We'll check $output multiple times + # FIXME: in the while loop below. + run_podman history --format json $IMAGE + + # FIXME: parse_table is what does all the work, giving us test cases. + parse_table "$tests" | while read field expect; do + # FIXME: this shows a drawback of BATS and bash: we can't include '|' + # FIXME: in the table, but we need to because some images don't + # FIXME: have a CID. So, yeah, this is ugly -- but rare. + if [ "$field" = "id" ]; then expect="$expect\|";fi + + # output is an array of dicts; check each one + count=$(echo "$output" | jq '. | length') + i=0 + while [ $i -lt $count ]; do + actual=$(echo "$output" | jq -r ".[$i].$field") + # FIXME: please be sure to note the third field! + # FIXME: that's the test name. Make it something useful! Include + # FIXME: loop variables whenever possible. Don't just say "my test" + is "$actual" "$expect\$" "jq .[$i].$field" + i=$(expr $i + 1) + done + done +} + + +# vim: filetype=sh diff --git a/test/system/001-basic.bats b/test/system/001-basic.bats index 15d5f38db..85b9bc1ca 100644 --- a/test/system/001-basic.bats +++ b/test/system/001-basic.bats @@ -14,16 +14,13 @@ function setup() { run_podman version is "${lines[0]}" "Version:[ ]\+[1-9][0-9.]\+" "Version line 1" - is "$output" ".*Go Version: \+" "'Go Version' in output" - - # FIXME: enable for 1.1 -# is "$output" ".*RemoteAPI Version: \+" "API version in output" + is "$output" ".*RemoteAPI Version: \+" "API version in output" } @test "podman can pull an image" { - run_podman pull $PODMAN_TEST_IMAGE_FQN + run_podman pull $IMAGE } # This is for development only; it's intended to make sure our timeout @@ -33,6 +30,21 @@ function setup() { if [ -z "$PODMAN_RUN_TIMEOUT_TEST" ]; then skip "define \$PODMAN_RUN_TIMEOUT_TEST to enable this test" fi - PODMAN_TIMEOUT=10 run_podman run $PODMAN_TEST_IMAGE_FQN sleep 90 + PODMAN_TIMEOUT=10 run_podman run $IMAGE sleep 90 echo "*** SHOULD NEVER GET HERE" } + + +# Too many tests rely on jq for parsing JSON. +# +# If absolutely necessary, one could establish a convention such as +# defining PODMAN_TEST_SKIP_JQ=1 and adding a skip_if_no_jq() helper. +# For now, let's assume this is not absolutely necessary. +@test "jq is installed and produces reasonable output" { + type -path jq >/dev/null || die "FATAL: 'jq' tool not found." + + run jq -r .a.b < <(echo '{ "a": { "b" : "you found me" } }') + is "$output" "you found me" "sample invocation of 'jq'" +} + +# vim: filetype=sh diff --git a/test/system/005-info.bats b/test/system/005-info.bats index c3eb43063..7dcc78838 100644 --- a/test/system/005-info.bats +++ b/test/system/005-info.bats @@ -26,8 +26,6 @@ RunRoot: } @test "podman info - json" { - type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'" - run_podman info --format=json expr_nvr="[a-z0-9-]\\\+-[a-z0-9.]\\\+-[a-z0-9]\\\+\." @@ -52,3 +50,5 @@ store.ImageStore.number | 1 done } + +# vim: filetype=sh diff --git a/test/system/010-images.bats b/test/system/010-images.bats index b88f6c2e1..1c9577e34 100644 --- a/test/system/010-images.bats +++ b/test/system/010-images.bats @@ -25,8 +25,6 @@ load helpers @test "podman images - json" { - type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'" - tests=" names[0] | $PODMAN_TEST_IMAGE_FQN id | [0-9a-f]\\\{64\\\} @@ -44,3 +42,5 @@ size | [0-9]\\\+ done } + +# vim: filetype=sh diff --git a/test/system/015-help.bats b/test/system/015-help.bats index ac737908d..b648599f7 100644 --- a/test/system/015-help.bats +++ b/test/system/015-help.bats @@ -24,23 +24,26 @@ function podman_commands() { function check_help() { - count=0 + local count=0 + local subcommands_found=0 + for cmd in $(podman_commands "$@"); do dprint "podman $@ $cmd --help" run_podman "$@" $cmd --help - # FIXME FIXME FIXME - usage=$(echo "$output" | grep -A2 '^Usage:' | grep . | tail -1) - # dprint "$usage" + # The line immediately after 'Usage:' gives us a 1-line synopsis + usage=$(echo "$output" | grep -A1 '^Usage:' | tail -1) [ -n "$usage" ] || die "podman $cmd: no Usage message found" - # if ends in '[command]', recurse into subcommands + # If usage ends in '[command]', recurse into subcommands if expr "$usage" : '.*\[command\]$' >/dev/null; then + subcommands_found=$(expr $subcommands_found + 1) check_help "$@" $cmd continue fi - # if ends in '[flag]' FIXME + # If usage ends in '[flag]', command takes no more arguments. + # Confirm that by running with 'invalid-arg' and expecting failure. if expr "$usage" : '.*\[flags\]$' >/dev/null; then if [ "$cmd" != "help" ]; then run_podman 125 "$@" $cmd invalid-arg @@ -52,11 +55,22 @@ function check_help() { count=$(expr $count + 1) done + # This can happen if the output of --help changes, such as between + # the old command parser and cobra. [ $count -gt 0 ] || \ die "Internal error: no commands found in 'podman help $@' list" + + # At least the top level must have some subcommands + if [ -z "$*" -a $subcommands_found -eq 0 ]; then + die "Internal error: did not find any podman subcommands" + fi } @test "podman help - basic tests" { + # Called with no args -- start with 'podman --help'. check_help() will + # recurse for any subcommands. check_help } + +# vim: filetype=sh diff --git a/test/system/030-run.bats b/test/system/030-run.bats index d1f87d554..8ae68f33d 100644 --- a/test/system/030-run.bats +++ b/test/system/030-run.bats @@ -26,7 +26,9 @@ echo $rand | 0 | $rand # a way to do so. eval set "$cmd" - run_podman $expected_rc run $PODMAN_TEST_IMAGE_FQN "$@" + run_podman $expected_rc run $IMAGE "$@" is "$output" "$expected_output" "podman run $cmd - output" done < <(parse_table "$tests") } + +# vim: filetype=sh diff --git a/test/system/035-logs.bats b/test/system/035-logs.bats new file mode 100644 index 000000000..debec29b6 --- /dev/null +++ b/test/system/035-logs.bats @@ -0,0 +1,24 @@ +#!/usr/bin/env bats -*- bats -*- +# +# Basic tests for podman logs +# + +load helpers + +@test "podman logs - basic test" { + rand_string=$(random_string 40) + + run_podman create $IMAGE echo $rand_string + cid="$output" + + run_podman logs $cid + is "$output" "" "logs on created container: empty" + + run_podman start --attach --interactive $cid + is "$output" "$rand_string" "output from podman-start on created ctr" + is "$output" "$rand_string" "logs of started container" + + run_podman rm $cid +} + +# vim: filetype=sh diff --git a/test/system/040-ps.bats b/test/system/040-ps.bats index 775df4af6..dec2df4d5 100644 --- a/test/system/040-ps.bats +++ b/test/system/040-ps.bats @@ -5,20 +5,20 @@ load helpers @test "podman ps - basic tests" { rand_name=$(random_string 30) - run_podman run -d --name $rand_name $PODMAN_TEST_IMAGE_FQN sleep 5 + run_podman run -d --name $rand_name $IMAGE sleep 5 cid=$output is "$cid" "[0-9a-f]\{64\}$" # Special case: formatted ps run_podman ps --no-trunc \ --format '{{.ID}} {{.Image}} {{.Command}} {{.Names}}' - is "$output" "$cid $PODMAN_TEST_IMAGE_FQN sleep 5 $rand_name" "podman ps" + is "$output" "$cid $IMAGE sleep 5 $rand_name" "podman ps" # Plain old regular ps run_podman ps is "${lines[1]}" \ - "${cid:0:12} \+$PODMAN_TEST_IMAGE_FQN \+sleep [0-9]\+ .*second.* $cname"\ + "${cid:0:12} \+$IMAGE \+sleep [0-9]\+ .*second.* $cname"\ "output from podman ps" # OK. Wait for sleep to finish... @@ -27,10 +27,12 @@ load helpers # ...then make sure container shows up as stopped run_podman ps -a is "${lines[1]}" \ - "${cid:0:12} \+$PODMAN_TEST_IMAGE_FQN *sleep .* Exited .* $rand_name" \ + "${cid:0:12} \+$IMAGE *sleep .* Exited .* $rand_name" \ "podman ps -a" run_podman rm $cid } + +# vim: filetype=sh diff --git a/test/system/050-stop.bats b/test/system/050-stop.bats index d04e22606..093606ece 100644 --- a/test/system/050-stop.bats +++ b/test/system/050-stop.bats @@ -4,7 +4,7 @@ load helpers # Very simple test @test "podman stop - basic test" { - run_podman run -d $PODMAN_TEST_IMAGE_FQN sleep 60 + run_podman run -d $IMAGE sleep 60 cid="$output" # Run 'stop'. Time how long it takes. @@ -37,7 +37,7 @@ load helpers # different variations of this test. for t_opt in '' '--time=5' '--timeout=5'; do # Run a simple container that logs output on SIGTERM - run_podman run -d $PODMAN_TEST_IMAGE_FQN sh -c \ + run_podman run -d $IMAGE sh -c \ "trap 'echo Received SIGTERM, finishing; exit' SIGTERM; echo READY; while :; do sleep 1; done" cid="$output" wait_for_ready $cid @@ -63,3 +63,5 @@ load helpers run_podman rm $cid done } + +# vim: filetype=sh diff --git a/test/system/060-mount.bats b/test/system/060-mount.bats index 7e86c94dc..3601b7b84 100644 --- a/test/system/060-mount.bats +++ b/test/system/060-mount.bats @@ -11,7 +11,7 @@ load helpers f_content=$(random_string 30) c_name=mount_test_$(random_string 5) - run_podman run --name $c_name $PODMAN_TEST_IMAGE_FQN \ + run_podman run --name $c_name $IMAGE \ sh -c "echo $f_content > $f_path" run_podman mount $c_name @@ -33,3 +33,5 @@ load helpers die "Mounted file exists even after umount: $mount_path/$f_path" fi } + +# vim: filetype=sh diff --git a/test/system/070-build.bats b/test/system/070-build.bats new file mode 100644 index 000000000..25eb36c58 --- /dev/null +++ b/test/system/070-build.bats @@ -0,0 +1,28 @@ +#!/usr/bin/env bats -*- bats -*- +# +# Tests for podman build +# + +load helpers + +@test "podman build - basic test" { + rand_filename=$(random_string 20) + rand_content=$(random_string 50) + + tmpdir=$PODMAN_TMPDIR/build-test + run mkdir -p $tmpdir || die "Could not mkdir $tmpdir" + dockerfile=$tmpdir/Dockerfile + cat >$dockerfile < /$rand_filename +EOF + + run_podman build -t build_test --format=docker $tmpdir + + run_podman run --rm build_test cat /$rand_filename + is "$output" "$rand_content" "reading generated file in image" + + run_podman rmi build_test +} + +# vim: filetype=sh diff --git a/test/system/110-history.bats b/test/system/110-history.bats index c00949bd1..84a1e42b4 100644 --- a/test/system/110-history.bats +++ b/test/system/110-history.bats @@ -16,21 +16,19 @@ load helpers eval set -- "$options" - run_podman history "$@" $PODMAN_TEST_IMAGE_FQN + run_podman history "$@" $IMAGE is "$output" "$expect" "podman history $options" done } @test "podman history - json" { - type -path jq >/dev/null || die "FAIL: please 'dnf -y install jq'" - tests=" id | [0-9a-f]\\\{64\\\} created | [0-9-]\\\+T[0-9:]\\\+\\\.[0-9]\\\+Z size | -\\\?[0-9]\\\+ " - run_podman history --format json $PODMAN_TEST_IMAGE_FQN + run_podman history --format json $IMAGE parse_table "$tests" | while read field expect; do # HACK: we can't include '|' in the table @@ -47,3 +45,5 @@ size | -\\\?[0-9]\\\+ done } + +# vim: filetype=sh diff --git a/test/system/200-pod-top.bats b/test/system/200-pod-top.bats new file mode 100644 index 000000000..81c4be3ff --- /dev/null +++ b/test/system/200-pod-top.bats @@ -0,0 +1,37 @@ +#!/usr/bin/env bats + +load helpers + +@test "podman pod top - containers in different PID namespaces" { + skip_if_rootless + + run_podman pod create + podid="$output" + + # Start two containers... + run_podman run -d --pod $podid $IMAGE top -d 2 + cid1="$output" + run_podman run -d --pod $podid $IMAGE top -d 2 + cid2="$output" + + # ...and wait for them to actually start. + wait_for_output "PID \+PPID \+USER " $cid1 + wait_for_output "PID \+PPID \+USER " $cid2 + + # Both containers have emitted at least one top-like line. + # Now run 'pod top', and expect two 'top -d 2' processes running. + run_podman pod top $podid + is "$output" ".*root.*top -d 2.*root.*top -d 2" "two 'top' containers" + + # There should be a /pause container + # FIXME: sometimes there is, sometimes there isn't. If anyone ever + # actually figures this out, please either reenable this line or + # remove it entirely. + #is "$output" ".*0 \+1 \+0 \+[0-9. ?s]\+/pause" "there is a /pause container" + + # Clean up + run_podman pod rm -f $podid +} + + +# vim: filetype=sh diff --git a/test/system/README.md b/test/system/README.md index 6853b68f3..6ac408f4e 100644 --- a/test/system/README.md +++ b/test/system/README.md @@ -18,17 +18,28 @@ sometimes needs to massage the returned values; `015-run.bats` offers examples of how to deal with the more typical such issues. * `run_podman` - runs command defined in `$PODMAN` (default: 'podman' -but could also be 'podman-remote'), with a timeout. Checks its exit status. +but could also be './bin/podman' or 'podman-remote'), with a timeout. +Checks its exit status. * `is` - compare actual vs expected output. Emits a useful diagnostic on failure. +* `die` - output a properly-formatted message to stderr, and fail test + +* `skip_if_rootless` - if rootless, skip this test with a helpful message. + * `random_string` - returns a pseudorandom alphanumeric string Test files are of the form `NNN-name.bats` where NNN is a three-digit number. Please preserve this convention, it simplifies viewing the -directory and understanding test order. Most of the time it's not -important but `00x` should be reserved for the times when it matters. +directory and understanding test order. In particular, `00x` tests +should be reserved for a first-pass fail-fast subset of tests: + + bats test/system/00*.bats || exit 1 + bats test/system + +...the goal being to provide quick feedback on catastrophic failures +without having to wait for the entire test suite. Analyzing test failures @@ -56,6 +67,12 @@ set `PODMAN_TEST_DEBUG="funcname"` where `funcname` is the name of the function or perhaps just a substring. +Requirements +============ + +The `jq` tool is needed for parsing JSON output. + + Further Details =============== diff --git a/test/system/helpers.bash b/test/system/helpers.bash index fd204e1cf..431228498 100644 --- a/test/system/helpers.bash +++ b/test/system/helpers.bash @@ -10,6 +10,9 @@ PODMAN_TEST_IMAGE_NAME=${PODMAN_TEST_IMAGE_NAME:-"alpine_labels"} PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"latest"} PODMAN_TEST_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG" +# Because who wants to spell that out each time? +IMAGE=$PODMAN_TEST_IMAGE_FQN + # Default timeout for a podman command. PODMAN_TIMEOUT=${PODMAN_TIMEOUT:-60} @@ -33,9 +36,9 @@ function basic_setup() { if [ "$1" == "$PODMAN_TEST_IMAGE_FQN" ]; then found_needed_image=1 else - echo "# setup_standard_environment: podman rmi $1 & $2" >&3 - podman rmi --force "$1" >/dev/null 2>&1 || true - podman rmi --force "$2" >/dev/null 2>&1 || true + echo "# setup(): removing stray images" >&3 + run_podman rmi --force "$1" >/dev/null 2>&1 || true + run_podman rmi --force "$2" >/dev/null 2>&1 || true fi done @@ -43,11 +46,22 @@ function basic_setup() { if [ -z "$found_needed_image" ]; then run_podman pull "$PODMAN_TEST_IMAGE_FQN" fi + + # Argh. Although BATS provides $BATS_TMPDIR, it's just /tmp! + # That's bloody worthless. Let's make our own, in which subtests + # can write whatever they like and trust that it'll be deleted + # on cleanup. + # TODO: do this outside of setup, so it carries across tests? + PODMAN_TMPDIR=$(mktemp -d --tmpdir=${BATS_TMPDIR:-/tmp} podman_bats.XXXXXX) } -# Basic teardown: remove all containers +# Basic teardown: remove all pods and containers function basic_teardown() { - run_podman rm --all --force + echo "# [teardown]" >&2 + run_podman '?' pod rm --all --force + run_podman '?' rm --all --force + + /bin/rm -rf $PODMAN_TMPDIR } @@ -118,10 +132,12 @@ function run_podman() { esac # stdout is only emitted upon error; this echo is to help a debugger - echo "\$ $PODMAN $@" + echo "\$ $PODMAN $*" run timeout --foreground -v --kill=10 $PODMAN_TIMEOUT $PODMAN "$@" # without "quotes", multiple lines are glommed together into one - echo "$output" + if [ -n "$output" ]; then + echo "$output" + fi if [ "$status" -ne 0 ]; then echo -n "[ rc=$status "; if [ -n "$expected_rc" ]; then @@ -143,21 +159,22 @@ function run_podman() { if [ -n "$expected_rc" ]; then if [ "$status" -ne "$expected_rc" ]; then - die "FAIL: exit code is $status; expected $expected_rc" + die "exit code is $status; expected $expected_rc" fi fi } -# Wait for 'READY' in container output -function wait_for_ready { - local cid= +# Wait for certain output from a container, indicating that it's ready. +function wait_for_output { local sleep_delay=5 - local how_long=60 + local how_long=$PODMAN_TIMEOUT + local expect= + local cid= # Arg processing. A single-digit number is how long to sleep between - # iterations; a 2- or 3-digit number is the total time to wait; anything - # else is the container ID or name to wait on. + # iterations; a 2- or 3-digit number is the total time to wait; all + # else are, in order, the string to expect and the container name/ID. local i for i in "$@"; do if expr "$i" : '[0-9]\+$' >/dev/null; then @@ -166,6 +183,8 @@ function wait_for_ready { else how_long=$i fi + elif [ -z "$expect" ]; then + expect=$i else cid=$i fi @@ -176,14 +195,19 @@ function wait_for_ready { t1=$(expr $SECONDS + $how_long) while [ $SECONDS -lt $t1 ]; do run_podman logs $cid - if expr "$output" : ".*READY" >/dev/null; then + if expr "$output" : ".*$expect" >/dev/null; then return fi sleep $sleep_delay done - die "FAIL: timed out waiting for READY from $cid" + die "timed out waiting for '$expect' from $cid" +} + +# Shortcut for the lazy +function wait_for_ready { + wait_for_output 'READY' "$@" } # END podman helpers @@ -206,7 +230,9 @@ function skip_if_rootless() { # die # Abort with helpful message ######### function die() { - echo "# $*" >&2 + echo "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv" >&2 + echo "#| FAIL: $*" >&2 + echo "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^" >&2 false } @@ -232,16 +258,24 @@ function is() { if [ -z "$actual" ]; then return fi - die "$testname:\n# expected no output; got %q\n" "$actual" - fi - - if expr "$actual" : "$expect" >/dev/null; then + expect='[no output]' + elif expr "$actual" : "$expect" >/dev/null; then return fi - # This is a multi-line message, so let's format it ourself (not via die) - printf "# $testname:\n# expected: %q\n# actual: %q\n" \ - "$expect" "$actual" >&2 + # This is a multi-line message, which may in turn contain multi-line + # output, so let's format it ourself, readably + local -a actual_split + readarray -t actual_split <<<"$actual" + printf "#/vvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvvv\n" >&2 + printf "#| FAIL: $testname\n" >&2 + printf "#| expected: '%s'\n" "$expect" >&2 + printf "#| actual: '%s'\n" "${actual_split[0]}" >&2 + local line + for line in "${actual_split[@]:1}"; do + printf "#| > '%s'\n" "$line" >&2 + done + printf "#\\^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n" >&2 false } -- cgit v1.2.3-54-g00ecf