Tests for API v2

Initial framework for testing the version 2 (HTTP) API.
Includes a collection of tests for some of the existing
endpoints. Not all tests are currently passing.

Signed-off-by: Ed Santiago <santiago@redhat.com>

8 files changed, 555 insertions, 0 deletions

diff --git a/test/apiv2/00-TEMPLATE b/test/apiv2/00-TEMPLATE
new file mode 100644
index 000000000..e256371ca
--- /dev/null
+++ b/test/apiv2/00-TEMPLATE
@@ -0,0 +1,6 @@
+# -*- sh -*-
+#
+# FIXME: one-line description of the purpose of this file
+#
+
+# vim: filetype=sh
diff --git a/test/apiv2/01-basic.at b/test/apiv2/01-basic.at
new file mode 100644
index 000000000..e87ec534c
--- /dev/null
+++ b/test/apiv2/01-basic.at
@@ -0,0 +1,49 @@
+# -*- sh -*-
+#
+# The earliest most basic tests. If any of these fail, life is bad
+#
+
+# NOTE: paths with a leading slash will be interpreted as-is;
+#       paths without will have '/v1.40/' prepended.
+t GET  /_ping        200  OK
+t HEAD /_ping        200
+t GET  /libpod/_ping 200 OK
+
+for i in /version version; do
+    t GET  $i      200                    \
+      .Components[0].Name="Podman Engine"       \
+      .Components[0].Details.APIVersion=1.40    \
+      .Components[0].Details.MinAPIVersion=1.24 \
+      .Components[0].Details.Os=linux           \
+      .ApiVersion=1.40                          \
+      .MinAPIVersion=1.24                       \
+      .Os=linux
+done
+
+#
+# Garbage tests - requests that should yield errors
+#
+t GET  /nonesuch                       404
+t POST /nonesuch ''                    404
+t GET  container/nonesuch/json         404
+t GET  libpod/containers/nonesuch/json 404
+t GET 'libpod/containers/json?a=b'     400
+
+# Method not allowed
+t POST   /_ping                 '' 405
+t DELETE /_ping                    405
+t POST   libpod/containers/json '' 405
+t POST   libpod/pods/abc        '' 405
+t POST   info                   '' 405
+t GET    libpod/containers/create  405
+
+#
+# system info
+#
+# FIXME: run 'podman info --format=json', and compare select fields
+t GET info 200           \
+  .OSType=linux          \
+  .DefaultRuntime=runc   \
+  .MemTotal~[0-9]\\+
+
+# vim: filetype=sh
diff --git a/test/apiv2/10-images.at b/test/apiv2/10-images.at
new file mode 100644
index 000000000..243b35e9f
--- /dev/null
+++ b/test/apiv2/10-images.at
@@ -0,0 +1,36 @@
+# -*- sh -*-
+#
+# Tests for image-related endpoints
+#
+
+# FIXME: API doesn't support pull yet, so use podman
+podman pull -q $IMAGE
+
+# We want the SHA without the "sha256:" prefix
+full_iid=$(podman images --no-trunc --format '{{.ID}}' $IMAGE)
+iid=${full_iid##sha256:}
+
+t GET libpod/images/$iid/exists                     204
+t GET libpod/images/$PODMAN_TEST_IMAGE_NAME/exists  204
+
+# FIXME: compare to actual podman info
+t GET libpod/images/json 200  \
+  .[0].Id=${iid}
+
+t GET libpod/images/$iid/json 200 \
+  .Id=$iid \
+  .RepoTags[0]=$IMAGE
+
+# Same thing, but with abbreviated image id
+t GET libpod/images/${iid:0:12}/json 200 \
+  .Id=$iid \
+  .RepoTags[0]=$IMAGE
+
+# FIXME: docker API incompatibility: libpod returns 'id', docker 'sha256:id'
+t GET images/$iid/json 200 \
+  .Id=sha256:$iid \
+  .RepoTags[0]=$IMAGE
+
+#t POST images/create fromImage=alpine 201 foo
+
+# vim: filetype=sh
diff --git a/test/apiv2/20-containers.at b/test/apiv2/20-containers.at
new file mode 100644
index 000000000..5f0a145f0
--- /dev/null
+++ b/test/apiv2/20-containers.at
@@ -0,0 +1,29 @@
+# -*- sh -*-
+#
+# test container-related endpoints
+#
+
+podman pull $IMAGE &>/dev/null
+
+# Unimplemented
+#t POST libpod/containers/create '' 201 'sdf'
+
+# Ensure clean slate
+podman rm -a -f &>/dev/null
+
+t GET libpod/containers/json 200 []
+
+podman run $IMAGE true
+
+t GET libpod/containers/json 200 \
+  .[0].ID~[0-9a-f]\\{12\\} \
+  .[0].Image=$IMAGE \
+  .[0].Command=true \
+  .[0].State=4 \
+  .[0].IsInfra=false
+
+cid=$(jq -r '.[0].ID' <<<"$output")
+
+t DELETE libpod/containers/$cid 204
+
+# vim: filetype=sh
diff --git a/test/apiv2/30-volumes.at b/test/apiv2/30-volumes.at
new file mode 100644
index 000000000..b599680e3
--- /dev/null
+++ b/test/apiv2/30-volumes.at
@@ -0,0 +1,14 @@
+# -*- sh -*-
+#
+# volume-related tests
+#
+
+#
+# FIXME: endpoints seem to be unimplemented, return 404
+#
+if false; then
+t GET  libpod/volumes/json            200 null
+t POST libpod/volumes/create name=foo 201
+fi
+
+# vim: filetype=sh
diff --git a/test/apiv2/40-pods.at b/test/apiv2/40-pods.at
new file mode 100644
index 000000000..1c25a3822
--- /dev/null
+++ b/test/apiv2/40-pods.at
@@ -0,0 +1,33 @@
+# -*- sh -*-
+#
+# test pod-related endpoints
+#
+
+t GET  libpod/pods/json            200 null
+t POST libpod/pods/create name=foo 201 '{"id":"machine.slice"}'   # FIXME!
+t GET  libpod/pods/foo/exists      204
+t GET  libpod/pods/notfoo/exists   404
+t GET  libpod/pods/foo/json        200     .Config.name=foo     .Containers=null
+t GET  libpod/pods/json            200 .[0].Config.name=foo .[0].Containers=null
+
+# Cannot create a dup pod with the same name (FIXME: should that be 409?)
+t POST libpod/pods/create name=foo 500 .cause="pod already exists"
+
+#t POST libpod/pods/create a=b 400 .cause='bad parameter'  # FIXME: unimplemented
+
+t POST libpod/pods/foo/pause   '' 204
+t POST libpod/pods/foo/unpause '' 200
+t POST libpod/pods/foo/unpause '' 200   # (2nd time)
+t POST libpod/pods/foo/stop    '' 304
+t POST libpod/pods/foo/restart '' 500 .cause="no such container"
+
+t POST libpod/pods/bar/restart '' 404
+
+#t POST libpod/pods/prune ''    200     # FIXME: unimplemented, returns 500
+#t POST libpod/pods/prune 'a=b' 400     # FIXME: unimplemented, returns 500
+
+# Clean up; and try twice, making sure that the second time fails
+t DELETE libpod/pods/foo 204
+t DELETE libpod/pods/foo 404
+
+# vim: filetype=sh
diff --git a/test/apiv2/README.md b/test/apiv2/README.md
new file mode 100644
index 000000000..252d6454e
--- /dev/null
+++ b/test/apiv2/README.md
@@ -0,0 +1,63 @@
+API v2 tests
+============
+
+This directory contains tests for the podman version 2 API (HTTP).
+
+Tests themselves are in files of the form 'NN-NAME.at' where NN is a
+two-digit number, NAME is a descriptive name, and '.at' is just
+an extension I picked.
+
+Running Tests
+=============
+
+The main test runner is `test-apiv2`. Usage is:
+
+    $ sudo ./test-apiv2 [NAME [...]]
+
+...where NAME is one or more optional test names, e.g. 'image' or 'pod'
+or both. By default, `test-apiv2` will invoke all `*.at` tests.
+
+`test-apiv2` connects to *localhost only* and *via TCP*. There is
+no support here for remote hosts or for UNIX sockets. This is a
+framework for testing the API, not all possible protocols.
+
+`test-apiv2` will start the service if it isn't already running.
+
+
+Writing Tests
+=============
+
+The main test function is `t`. It runs `curl` against the server,
+with POST parameters if present, and compares return status and
+(optionally) string results from the server:
+
+    t GET /_ping 200 OK
+      ^^^ ^^^^^^ ^^^ ^^
+      |   |      |   +--- expected string result
+      |   |      +------- expected return code
+      |   +-------------- endpoint to access
+      +------------------ method (GET, POST, DELETE, HEAD)
+
+
+    t POST libpod/volumes/create name=foo 201 .ID~[0-9a-f]\\{12\\}
+           ^^^^^^^^^^^^^^^^^^^^^ ^^^^^^^^ ^^^ ^^^^^^^^^^^^^^^^^^^^
+           |                     |        |   JSON '.ID': expect 12-char hex
+           |                     |        +-- expected code
+           |                     +----------- POST params
+           +--------------------------------- note the missing slash
+
+Notes:
+
+* If the endpoint has a leading slash (`/_ping`), `t` leaves it unchanged.
+If there's no leading slash, `t` prepends `/v1.40`. This is a simple
+convenience for simplicity of writing tests.
+
+* When method is POST, the argument after the endpoint must be a series
+of POST arguments in the form 'key=value', separated by commas. `t` will
+convert those to JSON form for passing to the server.
+
+* The final arguments are one or more expected string results. If an
+argument starts with a dot, `t` will invoke `jq` on the output to
+fetch that field, and will compare it to the right-hand side of
+the argument. If the separator is `=` (equals), `t` will require
+an exact match; if `~` (tilde), `t` will use `expr` to compare.
diff --git a/test/apiv2/test-apiv2 b/test/apiv2/test-apiv2
new file mode 100755
index 000000000..786c976d6
--- /dev/null
+++ b/test/apiv2/test-apiv2
@@ -0,0 +1,325 @@
+#!/bin/bash
+#
+# Usage: test-apiv2 [PORT]
+#
+# DEVELOPER NOTE: you almost certainly don't need to play in here. See README.
+#
+ME=$(basename $0)
+
+###############################################################################
+# BEGIN stuff you can but probably shouldn't customize
+
+PODMAN_TEST_IMAGE_REGISTRY=${PODMAN_TEST_IMAGE_REGISTRY:-"quay.io"}
+PODMAN_TEST_IMAGE_USER=${PODMAN_TEST_IMAGE_USER:-"libpod"}
+PODMAN_TEST_IMAGE_NAME=${PODMAN_TEST_IMAGE_NAME:-"alpine_labels"}
+PODMAN_TEST_IMAGE_TAG=${PODMAN_TEST_IMAGE_TAG:-"latest"}
+PODMAN_TEST_IMAGE_FQN="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/$PODMAN_TEST_IMAGE_NAME:$PODMAN_TEST_IMAGE_TAG"
+
+IMAGE=$PODMAN_TEST_IMAGE_FQN
+
+# END   stuff you can but probably shouldn't customize
+###############################################################################
+# BEGIN setup
+
+TMPDIR=${TMPDIR:-/tmp}
+WORKDIR=$(mktemp --tmpdir -d $ME.tmp.XXXXXX)
+
+# Log of all HTTP requests and responses
+LOG=${TMPDIR}/$ME.log.$(date +'%Y%m%dT%H%M%S')
+
+HOST=localhost
+PORT=${PODMAN_SERVICE_PORT:-8081}
+
+# Keep track of test count and failures in files, not variables, because
+# variables don't carry back up from subshells.
+testcounter_file=$WORKDIR/.testcounter
+failures_file=$WORKDIR/.failures
+
+echo 0 >$testcounter_file
+echo 0 >$failures_file
+
+# Where the tests live
+TESTS_DIR=$(realpath $(dirname $0))
+
+# END   setup
+###############################################################################
+# BEGIN infrastructure code - the helper functions used in tests themselves
+
+#########
+#  die  #  Exit error with a message to stderr
+#########
+function die() {
+    echo "$ME: $*" >&2
+    exit 1
+}
+
+########
+#  is  #  Simple comparison
+########
+function is() {
+    local actual=$1
+    local expect=$2
+    local testname=$3
+
+    if [ "$actual" = "$expect" ]; then
+        # On success, include expected value; this helps readers understand
+        _show_ok 1 "$testname=$expect"
+        return
+    fi
+    _show_ok 0 "$testname" "$expect" "$actual"
+}
+
+##########
+#  like  #  Compare, but allowing patterns
+##########
+function like() {
+    local actual=$1
+    local expect=$2
+    local testname=$3
+
+    if expr "$actual" : "$expect" &>/dev/null; then
+        # On success, include expected value; this helps readers understand
+        _show_ok 1 "$testname~$expect"
+        return
+    fi
+    _show_ok 0 "$testname" "~ $expect" "$actual"
+}
+
+##############
+#  _show_ok  #  Helper for is() and like(): displays 'ok' or 'not ok'
+##############
+function _show_ok() {
+    local ok=$1
+    local testname=$2
+
+    # If output is a tty, colorize pass/fail
+    local red=
+    local green=
+    local reset=
+    local bold=
+    if [ -t 3 ]; then
+        red='\e[31m'
+        green='\e[32m'
+        reset='\e[0m'
+        bold='\e[1m'
+    fi
+
+    _bump $testcounter_file
+    count=$(<$testcounter_file)
+    if [ $ok -eq 1 ]; then
+        echo -e "${green}ok $count $testname${reset}" >&3
+        return
+    fi
+
+    # Failed
+    local expect=$3
+    local actual=$4
+    echo -e "${red}not ok $count $testname${reset}"       >&3
+    echo -e "${red}#  expected: $expect${reset}"          >&3
+    echo -e "${red}#    actual: ${bold}$actual${reset}"   >&3
+
+    _bump $failures_file
+}
+
+###########
+#  _bump  #  Increment a counter in a file
+###########
+function _bump() {
+    local file=$1
+
+    count=$(<$file)
+    echo $(( $count + 1 )) >| $file
+}
+
+#############
+#  jsonify  #  convert 'foo=bar,x=y' to json {"foo":"bar","x":"y"}
+#############
+function jsonify() {
+    # split by comma
+    local -a settings_in
+    read -ra settings_in <<<"$1"
+
+    # convert each to double-quoted form
+    local -a settings_out
+    for i in ${settings_in[*]}; do
+        settings_out+=$(sed -e 's/\(.*\)=\(.*\)/"\1":"\2"/' <<<$i)
+    done
+
+    # ...and wrap inside braces.
+    # FIXME: handle commas
+    echo "{${settings_out[*]}}"
+}
+
+#######
+#  t  #  Main test helper
+#######
+function t() {
+    local method=$1; shift
+    local path=$1; shift
+    local curl_args
+
+    local testname="$method $path"
+    # POST requests require an extra params arg
+    if [[ $method = "POST" ]]; then
+        curl_args="-d $(jsonify $1)"
+        testname="$testname [$1]"
+        shift
+    fi
+    # curl -X HEAD but without --head seems to wait for output anyway
+    if [[ $method == "HEAD" ]]; then
+        curl_args="--head"
+    fi
+    local expected_code=$1; shift
+
+    # If given path begins with /, use it as-is; otherwise prepend /version/
+    local url=http://$HOST:$PORT
+    if expr "$path" : "/" >/dev/null; then
+        url="$url$path"
+    else
+        url="$url/v1.40/$path"
+    fi
+
+    # Log every action we do
+    echo "-------------------------------------------------------------" >>$LOG
+    echo "\$ $testname"                                                  >>$LOG
+    rm -f $WORKDIR/curl.*
+    curl -s -X $method ${curl_args}                   \
+         -H 'Content-type: application/json'          \
+         --dump-header $WORKDIR/curl.headers.out       \
+         -o $WORKDIR/curl.result.out "$url"
+
+    if [[ $? -eq 7 ]]; then
+        echo "FATAL: curl failure on $url - cannot continue" >&2
+        exit 1
+    fi
+
+    cat $WORKDIR/curl.headers.out $WORKDIR/curl.result.out >>$LOG 2>/dev/null || true
+
+    # Test return code
+    actual_code=$(head -n1 $WORKDIR/curl.headers.out | awk '/^HTTP/ { print $2}')
+    is "$actual_code" "$expected_code" "$testname : status"
+
+    output=$(< $WORKDIR/curl.result.out)
+
+    for i; do
+        case "$i" in
+            # Exact match on json field
+            .*=*)
+                json_field=$(expr "$i" : "\([^=]*\)=")
+                expect=$(expr "$i" : '[^=]*=\(.*\)')
+                actual=$(jq -r "$json_field" <<<"$output")
+                is "$actual" "$expect" "$testname : $json_field"
+                ;;
+            # regex match on json field
+            .*~*)
+                json_field=$(expr "$i" : "\([^~]*\)~")
+                expect=$(expr "$i" : '[^~]*~\(.*\)')
+                actual=$(jq -r "$json_field" <<<"$output")
+                like "$actual" "$expect" "$testname : $json_field"
+                ;;
+            # Direct string comparison
+            *)
+                is "$output" "$i" "$testname : output"
+                ;;
+        esac
+    done
+}
+
+###################
+#  start_service  #  Run the socket listener
+###################
+service_pid=
+function start_service() {
+    # If there's a listener on the port, nothing for us to do
+    echo -n >/dev/tcp/$HOST/$PORT &>/dev/null && return
+
+    if [ "$HOST" != "localhost" ]; then
+        die "Cannot start service on non-localhost ($HOST)"
+    fi
+
+    if [ $(id -u) -ne 0 ]; then
+        echo "$ME: WARNING: running service rootless is unlikely to work!" >&2
+    fi
+
+    # Find the binary
+    SERVICE_BIN=${SERVICE_BIN:-${TESTS_DIR}/../../bin/service}
+    test -x $SERVICE_BIN || die "Not found: $SERVICE_BIN"
+
+    systemd-socket-activate -l 127.0.0.1:$PORT \
+                            $SERVICE_BIN --root $WORKDIR/root \
+        &> $WORKDIR/server.log &
+    service_pid=$!
+
+    # Wait
+    local _timeout=5
+    while [ $_timeout -gt 0 ]; do
+        echo -n >/dev/tcp/$HOST/$PORT &>/dev/null && return
+        sleep 1
+        _timeout=$(( $_timeout - 1 ))
+    done
+    die "Timed out waiting for service"
+}
+
+# END   infrastructure code
+###############################################################################
+# BEGIN sanity checks
+
+for tool in curl jq podman; do
+    type $tool &>/dev/null || die "$ME: Required tool '$tool' not found"
+done
+
+# END   sanity checks
+###############################################################################
+# BEGIN entry handler (subtest invoker)
+
+# Identify the tests to run. If called with args, use those as globs.
+tests_to_run=()
+if [ -n "$*" ]; then
+    shopt -s nullglob
+    for i; do
+        match=(${TESTS_DIR}/*${i}*.at)
+        if [ ${#match} -eq 0 ]; then
+            die "No match for $TESTS_DIR/*$i*.at"
+        fi
+        tests_to_run+=("${match[@]}")
+    done
+    shopt -u nullglob
+else
+    tests_to_run=($TESTS_DIR/*.at)
+fi
+
+# Because subtests may run podman or other commands that emit stderr;
+# redirect all those and use fd 3 for all output
+exec 3>&1 &>$WORKDIR/output.log
+
+start_service
+
+for i in ${tests_to_run[@]}; do
+    source $i
+done
+
+# END   entry handler
+###############################################################################
+
+# Clean up
+
+if [ -n "$service_pid" ]; then
+    # Yep, has to be -9. It ignores everything else.
+    kill -9 $service_pid
+fi
+
+test_count=$(<$testcounter_file)
+failure_count=$(<$failures_file)
+
+if [ $failure_count -gt 0 -a -s "$WORKDIR/output.log" ]; then
+    echo "# Collected stdout/stderr:"         >&3
+    sed -e 's/^/# /' < $WORKDIR/output.log    >&3
+fi
+
+if [ -z "$PODMAN_TESTS_KEEP_WORKDIR" ]; then
+    rm -rf $WORKDIR
+fi
+
+echo "1..${test_count}" >&3
+
+exit $failure_count