Merge pull request #8553 from baude/composeci

add compose regression to ci

33 files changed, 603 insertions, 1 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 0fa51be63..8507aa3d2 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -30,7 +30,7 @@ env:
     PRIOR_UBUNTU_NAME: "ubuntu-19"
 
     # Google-cloud VM Images
-    IMAGE_SUFFIX: "c5402398833246208"
+    IMAGE_SUFFIX: "c4704091098054656"
     FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}"
     UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}"
@@ -434,6 +434,21 @@ apiv2_test_task:
         podman_system_info_script: '$SCRIPT_BASE/logcollector.sh podman'
         time_script: '$SCRIPT_BASE/logcollector.sh time'
 
+compose_test_task:
+    name: "compose test on $DISTRO_NV"
+    alias: compose_test
+    depends_on:
+        - validate
+    gce_instance: *standardvm
+    env:
+        <<: *stdenvars
+        TEST_FLAVOR: compose
+    clone_script: *noop  # Comes from cache
+    gopath_cache: *ro_gopath_cache
+    setup_script: *setup
+    main_script: *main
+    always: *logs_artifacts
+
 
 # Execute the podman integration tests on all primary platforms and release
 # versions, as root, without involving the podman-remote client.
@@ -619,6 +634,7 @@ success_task:
         - docker-py_test
         - unit_test
         - apiv2_test
+        - compose_test
         - local_integration_test
         - remote_integration_test
         - rootless_integration_test
diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh
index fa921f3e4..cc6d155f9 100755
--- a/contrib/cirrus/runner.sh
+++ b/contrib/cirrus/runner.sh
@@ -73,6 +73,10 @@ function _run_apiv2() {
     make localapiv2 |& logformatter
 }
 
+function _run_compose() {
+    ./test/compose/test-compose |& logformatter
+}
+
 function _run_int() {
     dotest integration
 }
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index c32b45a4f..a3c0f9a13 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -195,6 +195,7 @@ case "$TEST_FLAVOR" in
     build) make clean ;;
     unit) ;;
     apiv2) ;&  # use next item
+    compose) ;&
     int) ;&
     sys) ;&
     bindings) ;&
diff --git a/test/compose/README.md b/test/compose/README.md
new file mode 100644
index 000000000..863decf2c
--- /dev/null
+++ b/test/compose/README.md
@@ -0,0 +1,47 @@
+Tests for docker-compose
+========================
+
+This directory contains tests for docker-compose under podman.
+
+Each subdirectory must contain one docker-compose.yml file along with
+all necessary infrastructure for it (e.g. Containerfile, any files
+to be copied into the container, and so on.
+
+The `test-compose` script will, for each test subdirectory:
+
+* set up a fresh podman root under an empty working directory;
+* run a podman server rooted therein;
+* cd to the test subdirectory, and run `docker-compose up -d`;
+* source `tests.sh`;
+* run `docker-compose down`.
+
+As a special case, `setup.sh` and `teardown.sh` in the test directory
+will contain commands to be executed prior to `docker-compose up` and
+after `docker-compose down` respectively.
+
+tests.sh will probably contain commands of the form
+
+     test_port 12345 = 'hello there'
+
+Where 12345 is the port to curl to; '=' checks equality, '~' uses `expr`
+to check substrings; and 'hello there' is a string to look for in
+the curl results.
+
+Usage:
+
+    $ sudo test/compose/test-compose [pattern]
+
+By default, all subdirs will be run. If given a pattern, only those
+subdirectories matching 'pattern' will be run.
+
+If `$COMPOSE_WAIT` is set, `test-compose` will pause before running
+`docker-compose down`. This can be helpful for you to debug failing tests:
+
+    $ env COMPOSE_WAIT=1 sudo --preserve-env=COMPOSE_WAIT test/compose/test-compose
+
+Then, in another window,
+
+    # ls -lt /var/tmp/
+    # X=/var/tmp/test-compose.tmp.XXXXXX <--- most recent results of above
+    # podman --root $X/root --runroot $X/runroot ps -a
+    # podman --root $X/root --runroot $X/runroot logs -l
diff --git a/test/compose/env_and_volume/README.md b/test/compose/env_and_volume/README.md
new file mode 100644
index 000000000..e7d74976b
--- /dev/null
+++ b/test/compose/env_and_volume/README.md
@@ -0,0 +1,12 @@
+environment variable and volume
+===============
+
+This test creates two containers both of which are running flask.  The first container has
+an environment variable called PODMAN_MSG.  That container pipes the contents of PODMAN_MSG
+to a file on a shared volume between the containers.  The second container then reads the
+file are returns the PODMAN_MSG value via flask (http).
+
+Validation
+------------
+* curl http://localhost:5000 and verify message
+* curl http://localhost:5001 and verify message
diff --git a/test/compose/env_and_volume/docker-compose.yml b/test/compose/env_and_volume/docker-compose.yml
new file mode 100644
index 000000000..df906e170
--- /dev/null
+++ b/test/compose/env_and_volume/docker-compose.yml
@@ -0,0 +1,18 @@
+version: '3'
+services:
+  writer:
+    environment:
+      - PODMAN_MSG=podman_rulez
+    build: write
+    ports:
+      - '5000:5000'
+    volumes:
+      - data:/data
+  reader:
+    build: read
+    ports:
+      - '5001:5000'
+    volumes:
+      - data:/data
+volumes:
+  data:
diff --git a/test/compose/env_and_volume/read/Dockerfile b/test/compose/env_and_volume/read/Dockerfile
new file mode 100644
index 000000000..8d5c45401
--- /dev/null
+++ b/test/compose/env_and_volume/read/Dockerfile
@@ -0,0 +1,5 @@
+FROM quay.io/libpod/podman_python
+WORKDIR /app
+COPY . /app
+ENTRYPOINT ["python3"]
+CMD ["app.py"]
diff --git a/test/compose/env_and_volume/read/app.py b/test/compose/env_and_volume/read/app.py
new file mode 100644
index 000000000..71fbbb26a
--- /dev/null
+++ b/test/compose/env_and_volume/read/app.py
@@ -0,0 +1,10 @@
+from flask import Flask
+app = Flask(__name__)
+
+@app.route('/')
+def hello():
+    f = open("/data/message", "r")
+    return f.read()
+
+if __name__ == '__main__':
+	app.run(host='0.0.0.0')
diff --git a/test/compose/env_and_volume/tests.sh b/test/compose/env_and_volume/tests.sh
new file mode 100644
index 000000000..a4c8bed30
--- /dev/null
+++ b/test/compose/env_and_volume/tests.sh
@@ -0,0 +1,4 @@
+# -*- bash -*-
+
+test_port 5000 = "done"
+test_port 5001 = "podman_rulez"
diff --git a/test/compose/env_and_volume/write/Dockerfile b/test/compose/env_and_volume/write/Dockerfile
new file mode 100644
index 000000000..8d5c45401
--- /dev/null
+++ b/test/compose/env_and_volume/write/Dockerfile
@@ -0,0 +1,5 @@
+FROM quay.io/libpod/podman_python
+WORKDIR /app
+COPY . /app
+ENTRYPOINT ["python3"]
+CMD ["app.py"]
diff --git a/test/compose/env_and_volume/write/app.py b/test/compose/env_and_volume/write/app.py
new file mode 100644
index 000000000..b6ad6fe63
--- /dev/null
+++ b/test/compose/env_and_volume/write/app.py
@@ -0,0 +1,13 @@
+from flask import Flask
+import os
+app = Flask(__name__)
+
+@app.route('/')
+def hello():
+    f = open("/data/message", "w")
+    f.write(os.getenv("PODMAN_MSG"))
+    f.close()
+    return "done"
+
+if __name__ == '__main__':
+	app.run(host='0.0.0.0')
diff --git a/test/compose/images/README.md b/test/compose/images/README.md
new file mode 100644
index 000000000..f25fbdc24
--- /dev/null
+++ b/test/compose/images/README.md
@@ -0,0 +1,5 @@
+images
+======
+
+Use these directories for images that are needed for the compose testing.  These
+images should be then pushed to `quay.io/libpod` for consumption.
diff --git a/test/compose/images/podman-python/Containerfile b/test/compose/images/podman-python/Containerfile
new file mode 100644
index 000000000..47f90afaa
--- /dev/null
+++ b/test/compose/images/podman-python/Containerfile
@@ -0,0 +1,3 @@
+FROM alpine
+WORKDIR /app
+RUN apk update && apk add py3-pip && pip3 install flask && rm -fr /var/cache/apk/*
diff --git a/test/compose/mount_and_label/README.md b/test/compose/mount_and_label/README.md
new file mode 100644
index 000000000..623b38cac
--- /dev/null
+++ b/test/compose/mount_and_label/README.md
@@ -0,0 +1,9 @@
+mount and label
+===============
+
+This test creates a container with a mount (not volume) and also adds a label to the container.
+
+Validation
+------------
+* curl http://localhost:5000 and verify message
+* inspect the container to make the label exists on it
diff --git a/test/compose/mount_and_label/docker-compose.yml b/test/compose/mount_and_label/docker-compose.yml
new file mode 100644
index 000000000..112d7e134
--- /dev/null
+++ b/test/compose/mount_and_label/docker-compose.yml
@@ -0,0 +1,10 @@
+version: '3'
+services:
+  web:
+    build: frontend
+    ports:
+      - '5000:5000'
+    volumes:
+      - /tmp/data:/data:ro
+    labels:
+      - "io.podman=the_best"
diff --git a/test/compose/mount_and_label/frontend/Dockerfile b/test/compose/mount_and_label/frontend/Dockerfile
new file mode 100644
index 000000000..8d5c45401
--- /dev/null
+++ b/test/compose/mount_and_label/frontend/Dockerfile
@@ -0,0 +1,5 @@
+FROM quay.io/libpod/podman_python
+WORKDIR /app
+COPY . /app
+ENTRYPOINT ["python3"]
+CMD ["app.py"]
diff --git a/test/compose/mount_and_label/frontend/app.py b/test/compose/mount_and_label/frontend/app.py
new file mode 100644
index 000000000..bd2794d94
--- /dev/null
+++ b/test/compose/mount_and_label/frontend/app.py
@@ -0,0 +1,10 @@
+from flask import Flask
+app = Flask(__name__)
+
+@app.route('/')
+def hello():
+    f = open("/data/message")
+    return f.read()
+
+if __name__ == '__main__':
+	app.run(host='0.0.0.0')
diff --git a/test/compose/mount_and_label/setup.sh b/test/compose/mount_and_label/setup.sh
new file mode 100644
index 000000000..8633d65d5
--- /dev/null
+++ b/test/compose/mount_and_label/setup.sh
@@ -0,0 +1,2 @@
+mkdir -p /tmp/data
+echo "Podman rulez!" > /tmp/data/message
diff --git a/test/compose/mount_and_label/teardown.sh b/test/compose/mount_and_label/teardown.sh
new file mode 100644
index 000000000..57867c28a
--- /dev/null
+++ b/test/compose/mount_and_label/teardown.sh
@@ -0,0 +1 @@
+rm /tmp/data/message
diff --git a/test/compose/mount_and_label/tests.sh b/test/compose/mount_and_label/tests.sh
new file mode 100644
index 000000000..07ff089b5
--- /dev/null
+++ b/test/compose/mount_and_label/tests.sh
@@ -0,0 +1,4 @@
+# -*- bash -*-
+
+test_port 5000 = "Podman rulez!"
+podman container inspect -l --format '{{.Config.Labels}}' | grep "the_best"
diff --git a/test/compose/port_map_diff_port/README.md b/test/compose/port_map_diff_port/README.md
new file mode 100644
index 000000000..13ece72ad
--- /dev/null
+++ b/test/compose/port_map_diff_port/README.md
@@ -0,0 +1,9 @@
+port map on different port
+===============
+
+This test creates a container that runs flask on different ports for the container
+and the host
+
+Validation
+------------
+* curl http://localhost:5001 and verify message
diff --git a/test/compose/port_map_diff_port/docker-compose.yml b/test/compose/port_map_diff_port/docker-compose.yml
new file mode 100644
index 000000000..3003c52f4
--- /dev/null
+++ b/test/compose/port_map_diff_port/docker-compose.yml
@@ -0,0 +1,6 @@
+version: '3'
+services:
+  web:
+    build: frontend
+    ports:
+      - '5001:5000'
diff --git a/test/compose/port_map_diff_port/frontend/Dockerfile b/test/compose/port_map_diff_port/frontend/Dockerfile
new file mode 100644
index 000000000..8d5c45401
--- /dev/null
+++ b/test/compose/port_map_diff_port/frontend/Dockerfile
@@ -0,0 +1,5 @@
+FROM quay.io/libpod/podman_python
+WORKDIR /app
+COPY . /app
+ENTRYPOINT ["python3"]
+CMD ["app.py"]
diff --git a/test/compose/port_map_diff_port/frontend/app.py b/test/compose/port_map_diff_port/frontend/app.py
new file mode 100644
index 000000000..895556a89
--- /dev/null
+++ b/test/compose/port_map_diff_port/frontend/app.py
@@ -0,0 +1,9 @@
+from flask import Flask
+app = Flask(__name__)
+
+@app.route('/')
+def hello():
+	return "Podman rulez!"
+
+if __name__ == '__main__':
+	app.run(host='0.0.0.0')
diff --git a/test/compose/port_map_diff_port/tests.sh b/test/compose/port_map_diff_port/tests.sh
new file mode 100644
index 000000000..5a468aadc
--- /dev/null
+++ b/test/compose/port_map_diff_port/tests.sh
@@ -0,0 +1,3 @@
+# -*- bash -*-
+
+test_port 5001 = "Podman rulez!"
diff --git a/test/compose/setup.sh.example b/test/compose/setup.sh.example
new file mode 100644
index 000000000..9004b1e76
--- /dev/null
+++ b/test/compose/setup.sh.example
@@ -0,0 +1,3 @@
+# -*- bash -*-
+
+export ENV_PASSTHRU=$(random_string 20)
diff --git a/test/compose/simple_port_map/README.md b/test/compose/simple_port_map/README.md
new file mode 100644
index 000000000..f28d71c3e
--- /dev/null
+++ b/test/compose/simple_port_map/README.md
@@ -0,0 +1,9 @@
+simple port map to host
+===============
+
+This test creates a container that runs flask on and maps to the same
+host port
+
+Validation
+------------
+* curl http://localhost:5000 and verify message
diff --git a/test/compose/simple_port_map/docker-compose.yml b/test/compose/simple_port_map/docker-compose.yml
new file mode 100644
index 000000000..e7eab1047
--- /dev/null
+++ b/test/compose/simple_port_map/docker-compose.yml
@@ -0,0 +1,6 @@
+version: '3'
+services:
+  web:
+    build: frontend
+    ports:
+      - '5000:5000'
diff --git a/test/compose/simple_port_map/frontend/Dockerfile b/test/compose/simple_port_map/frontend/Dockerfile
new file mode 100644
index 000000000..2595828ff
--- /dev/null
+++ b/test/compose/simple_port_map/frontend/Dockerfile
@@ -0,0 +1,6 @@
+FROM alpine
+WORKDIR /app
+RUN apk update && apk add py3-pip && pip3 install flask
+COPY . /app
+ENTRYPOINT ["python3"]
+CMD ["app.py"]
diff --git a/test/compose/simple_port_map/frontend/app.py b/test/compose/simple_port_map/frontend/app.py
new file mode 100644
index 000000000..e4f84068c
--- /dev/null
+++ b/test/compose/simple_port_map/frontend/app.py
@@ -0,0 +1,10 @@
+from flask import Flask
+import os
+app = Flask(__name__)
+
+@app.route('/')
+def hello():
+    return "Podman rulez!"
+
+if __name__ == '__main__':
+    app.run(host='0.0.0.0')
diff --git a/test/compose/simple_port_map/tests.sh b/test/compose/simple_port_map/tests.sh
new file mode 100644
index 000000000..ccb2b6a3d
--- /dev/null
+++ b/test/compose/simple_port_map/tests.sh
@@ -0,0 +1,3 @@
+# -*- bash -*-
+
+test_port 5000 = "Podman rulez!"
diff --git a/test/compose/teardown.sh.example b/test/compose/teardown.sh.example
new file mode 100644
index 000000000..3f8153fa0
--- /dev/null
+++ b/test/compose/teardown.sh.example
@@ -0,0 +1,4 @@
+# -*- bash -*-
+
+# FIXME: this is completely unnecessary; it's just an example of a teardown
+unset ENV_PASSTHRU
diff --git a/test/compose/test-compose b/test/compose/test-compose
new file mode 100755
index 000000000..9558fbf58
--- /dev/null
+++ b/test/compose/test-compose
@@ -0,0 +1,345 @@
+#!/usr/bin/env bash
+#
+# Usage: test-compose [testname]
+#
+ME=$(basename $0)
+
+###############################################################################
+# BEGIN stuff you can but probably shouldn't customize
+
+# Directory where this script and all subtests live
+TEST_ROOTDIR=$(realpath $(dirname $0))
+
+# Podman executable
+PODMAN_BIN=$(realpath $TEST_ROOTDIR/../../bin)/podman
+
+# Local path to docker socket (we will add the unix:/ prefix when we need it)
+DOCKER_SOCK=/var/run/docker.sock
+
+# END   stuff you can but probably shouldn't customize
+###############################################################################
+# BEGIN setup
+
+export TMPDIR=${TMPDIR:-/var/tmp}
+WORKDIR=$(mktemp --tmpdir -d $ME.tmp.XXXXXX)
+
+# Log of all HTTP requests and responses; always make '.log' point to latest
+LOGBASE=${TMPDIR}/$ME.log
+LOG=${LOGBASE}.$(date +'%Y%m%dT%H%M%S')
+ln -sf $LOG $LOGBASE
+
+# Keep track of test count and failures in files, not variables, because
+# variables don't carry back up from subshells.
+testcounter_file=$WORKDIR/.testcounter
+failures_file=$WORKDIR/.failures
+
+echo 0 >$testcounter_file
+echo 0 >$failures_file
+
+# END   setup
+###############################################################################
+# BEGIN infrastructure code - the helper functions used in tests themselves
+
+#########
+#  die  #  Exit error with a message to stderr
+#########
+function die() {
+    echo "$ME: $*" >&2
+    exit 1
+}
+
+########
+#  is  #  Simple comparison
+########
+function is() {
+    local actual=$1
+    local expect=$2
+    local testname=$3
+
+    if [[ $actual = $expect ]]; then
+        # On success, include expected value; this helps readers understand
+        _show_ok 1 "$testname=$expect"
+        return
+    fi
+    _show_ok 0 "$testname" "$expect" "$actual"
+}
+
+##########
+#  like  #  Compare, but allowing patterns
+##########
+function like() {
+    local actual=$1
+    local expect=$2
+    local testname=$3
+
+    # "is" (equality) is a subset of "like", but one that expr fails on if
+    # the expected result has shell-special characters like '['. Treat it
+    # as a special case.
+
+    if [[ "$actual" = "$expect" ]]; then
+        _show_ok 1 "$testname=$expect"
+        return
+    fi
+
+    if expr "$actual" : ".*$expect" &>/dev/null; then
+        # On success, include expected value; this helps readers understand
+        _show_ok 1 "$testname ('$actual') ~ $expect"
+        return
+    fi
+    _show_ok 0 "$testname" "~ $expect" "$actual"
+}
+
+##############
+#  _show_ok  #  Helper for is() and like(): displays 'ok' or 'not ok'
+##############
+function _show_ok() {
+    local ok=$1
+    local testname=$2
+
+    # If output is a tty, colorize pass/fail
+    local red=
+    local green=
+    local reset=
+    local bold=
+    if [ -t 1 ]; then
+        red='\e[31m'
+        green='\e[32m'
+        reset='\e[0m'
+        bold='\e[1m'
+    fi
+
+    _bump $testcounter_file
+    count=$(<$testcounter_file)
+
+    # "skip" is a special case of "ok". Assume that our caller has included
+    # the magical '# skip - reason" comment string.
+    if [[ $ok == "skip" ]]; then
+        # colon-plus: replace green with yellow, but only if green is non-null
+        green="${green:+\e[33m}"
+        ok=1
+    fi
+    if [ $ok -eq 1 ]; then
+        echo -e "${green}ok $count $testname${reset}"
+        echo    "ok $count $testname" >>$LOG
+        return
+    fi
+
+    # Failed
+    local expect=$3
+    local actual=$4
+    printf "${red}not ok $count $testname${reset}\n"
+    printf "${red}#  expected: %s${reset}\n" "$expect"
+    printf "${red}#    actual: ${bold}%s${reset}\n" "$actual"
+
+    echo    "not ok $count $testname" >>$LOG
+    echo    "  expected: $expect"                     >>$LOG
+
+    _bump $failures_file
+}
+
+###########
+#  _bump  #  Increment a counter in a file
+###########
+function _bump() {
+    local file=$1
+
+    count=$(<$file)
+    echo $(( $count + 1 )) >| $file
+}
+
+###############
+#  test_port  #  Run curl against a port, check results against expectation
+###############
+function test_port() {
+    local port="$1"              # e.g. 5000
+    local op="$2"                # '=' or '~'
+    local expect="$3"            # what to expect from curl output
+
+    local actual=$(curl --retry 5 --retry-connrefused -s http://127.0.0.1:$port/)
+    local curl_rc=$?
+    if [ $curl_rc -ne 0 ]; then
+        _show_ok 0 "$testname - curl failed with status $curl_rc"
+###        docker-compose down >>$logfile 2>&1
+###        exit 1
+    fi
+
+    case "$op" in
+        '=')   is   "$actual" "$expect" "$testname : port $port" ;;
+        '~')   like "$actual" "$expect" "$testname : port $port" ;;
+        *)     die "Invalid operator '$op'" ;;
+    esac
+}
+
+
+###################
+#  start_service  #  Run the socket listener
+###################
+service_pid=
+function start_service() {
+    test -x $PODMAN_BIN || die "Not found: $PODMAN_BIN"
+
+    # FIXME: use ${testname} subdir but we can't: 50-char limit in runroot
+    rm -rf            $WORKDIR/{root,runroot,cni}
+    mkdir --mode 0755 $WORKDIR/{root,runroot,cni}
+    chcon --reference=/var/lib/containers $WORKDIR/root
+    cp /etc/cni/net.d/*podman*conflist $WORKDIR/cni/
+
+    $PODMAN_BIN \
+        --root $WORKDIR/root \
+        --runroot $WORKDIR/runroot \
+        --cgroup-manager=systemd \
+        --cni-config-dir $WORKDIR/cni \
+        system service \
+        --time 0 unix:/$DOCKER_SOCK \
+        &> $WORKDIR/server.log &
+    service_pid=$!
+
+    # Wait (FIXME: how do we test the socket?)
+    local _timeout=5
+    while [ $_timeout -gt 0 ]; do
+        # FIXME: should we actually try a read or write?
+        test -S $DOCKER_SOCK && return
+        sleep 1
+        _timeout=$(( $_timeout - 1 ))
+    done
+    cat $WORKDIR/server.log
+    die "Timed out waiting for service"
+}
+
+############
+#  podman  #  Needed by some test scripts to invoke the actual podman binary
+############
+function podman() {
+    echo "\$ podman $*"           >>$WORKDIR/output.log
+    $PODMAN_BIN \
+        --root    $WORKDIR/root    \
+        --runroot $WORKDIR/runroot \
+        "$@"    >>$WORKDIR/output.log 2>&1
+}
+
+###################
+#  random_string  #  Returns a pseudorandom human-readable string
+###################
+function random_string() {
+    # Numeric argument, if present, is desired length of string
+    local length=${1:-10}
+
+    head /dev/urandom | tr -dc a-zA-Z0-9 | head -c$length
+}
+
+# END   infrastructure code
+###############################################################################
+# BEGIN sanity checks
+
+for tool in curl docker-compose; do
+    type $tool &>/dev/null || die "$ME: Required tool '$tool' not found"
+done
+
+# END   sanity checks
+###############################################################################
+# BEGIN entry handler (subtest invoker)
+
+# Identify the tests to run. If called with args, use those as globs.
+tests_to_run=()
+if [ -n "$*" ]; then
+    shopt -s nullglob
+    for i; do
+        match=(${TEST_ROOTDIR}/*${i}*/docker-compose.yml)
+        if [ ${#match} -eq 0 ]; then
+            die "No match for $TEST_ROOTDIR/*$i*.curl"
+        fi
+        tests_to_run+=("${match[@]}")
+    done
+    shopt -u nullglob
+else
+    tests_to_run=(${TEST_ROOTDIR}/*/docker-compose.yml)
+fi
+
+# Too hard to precompute the number of tests; just spit it out at the end.
+n_tests=0
+for t in ${tests_to_run[@]}; do
+    testdir="$(dirname $t)"
+    testname="$(basename $testdir)"
+
+    if [ -e $test_dir/SKIP ]; then
+        local reason="$(<$test_dir/SKIP)"
+        if [ -n "$reason" ]; then
+            reason=" - $reason"
+        fi
+        _show_ok skip "$testname # skip$reason"
+        continue
+    fi
+
+    start_service
+
+    logfile=$WORKDIR/$testname.log
+    (
+        cd $testdir || die "Cannot cd $testdir"
+
+        # setup file may be used for creating temporary directories/files.
+        # We source it so that envariables defined in it will get back to us.
+        if [ -e setup.sh ]; then
+            . setup.sh
+        fi
+        if [ -e teardown.sh ]; then
+            trap '. teardown.sh' 0
+        fi
+
+        docker-compose up -d     &> $logfile
+        docker_compose_rc=$?
+        if [[ $docker_compose_rc -ne 0 ]]; then
+            _show_ok 0 "$testname - up" "[ok]" "status=$docker_compose_rc"
+            sed -e 's/^/#  /' <$logfile
+            docker-compose down >>$logfile 2>&1      # No status check here
+            exit 1
+        fi
+        _show_ok 1 "$testname - up"
+
+        # Run tests. This is likely to be a series of 'test_port' checks
+        # but may also include podman commands to inspect labels, state
+        if [ -e tests.sh ]; then
+            . tests.sh
+        fi
+        # FIXME: if any tests fail, try 'podman logs' on container?
+
+        if [ -n "$COMPOSE_WAIT" ]; then
+            echo -n "Pausing due to \$COMPOSE_WAIT. Press ENTER to continue: "
+            read keepgoing
+        fi
+
+        # Done. Clean up.
+        docker-compose down     &> $logfile
+        rc=$?
+        if [[ $rc -eq 0 ]]; then
+            _show_ok 1 "$testname - down"
+        else
+            _show_ok 0 "$testname - down" "[ok]" "rc=$rc"
+            # FIXME: show error
+        fi
+    )
+
+    kill $service_pid
+    wait $service_pid
+
+    # FIXME: otherwise we get EBUSY
+    umount $WORKDIR/root/overlay  &>/dev/null
+
+    # FIXME: run 'podman ps'?
+#    rm -rf $WORKDIR/${testname}
+done
+
+# END   entry handler
+###############################################################################
+
+# Clean up
+
+test_count=$(<$testcounter_file)
+failure_count=$(<$failures_file)
+
+#if [ -z "$PODMAN_TESTS_KEEP_WORKDIR" ]; then
+#    rm -rf $WORKDIR
+#fi
+
+echo "1..${test_count}"
+
+exit $failure_count