Add system test for shell completion

There exists a unit test to ensure that shell completion functions are
defined. However there was no check about the quality of the provided
shell completions. Lets change that.

The idea is to create a general test that makes sure we are suggesting
containers,pods,images... for the correct commands. This works by
reading the command use line and checking for each arg if we provide
the correct suggestions for this arg.

It includes the following tests:
- flag suggestions if [options] is set
- container, pod, image, network, volume, registry completion
- path completion for the appropriate arg KEYWORDS (`PATH`,`CONTEXT`,etc.)
- no completion if there are no args
- completion for more than one arg if it ends with `...]`

The test does not cover completion values for flags and not every arg KEYWORD
is supported. This is still a huge improvement and covers most use cases.

This test spotted several inconsistencies between the completion and the
command use line. All of them have been adjusted to make the test pass.

The biggest advantage is that the completions always match the latest
command changes. So if someone changes the arguments for a command this
ensures that the completions must be adjusted.

Signed-off-by: Paul Holzinger <paul.holzinger@web.de>

3 files changed, 283 insertions, 12 deletions

diff --git a/test/system/015-help.bats b/test/system/015-help.bats
index 22db8be8a..5f38c34a1 100644
--- a/test/system/015-help.bats
+++ b/test/system/015-help.bats
@@ -12,22 +12,11 @@
 #
 load helpers
 
-# run 'podman help', parse the output looking for 'Available Commands';
-# return that list.
-function podman_commands() {
-    dprint "$@"
-    run_podman help "$@" |\
-        awk '/^Available Commands:/{ok=1;next}/^Options:/{ok=0}ok { print $1 }' |\
-        grep .
-    "$output"
-}
-
-
 function check_help() {
     local count=0
     local -A found
 
-    for cmd in $(podman_commands "$@"); do
+    for cmd in $(_podman_commands "$@"); do
         # Human-readable podman command string, with multiple spaces collapsed
         command_string="podman $* $cmd"
         command_string=${command_string//  / }  # 'podman  x' -> 'podman x'
diff --git a/test/system/600-completion.bats b/test/system/600-completion.bats
new file mode 100644
index 000000000..1e43cdc41
--- /dev/null
+++ b/test/system/600-completion.bats
@@ -0,0 +1,272 @@
+#!/usr/bin/env bats   -*- bats -*-
+#
+# Test podman shell completion
+#
+# Shell completion is provided via the cobra library
+# It is implement by calling a hidden subcommand called "__complete"
+#
+
+load helpers
+
+function check_shell_completion() {
+    local count=0
+
+    # Newline character; used for confirming string output
+    local nl="
+"
+
+    for cmd in $(_podman_commands "$@"); do
+        # Human-readable podman command string, with multiple spaces collapsed
+        name="podman"
+        if is_remote; then
+            name="podman-remote"
+        fi
+        command_string="$name $* $cmd"
+        command_string=${command_string//  / } # 'podman  x' -> 'podman x'
+
+        run_podman "$@" $cmd --help
+        local full_help="$output"
+
+        # The line immediately after 'Usage:' gives us a 1-line synopsis
+        usage=$(echo "$full_help" | grep -A1 '^Usage:' | tail -1)
+        [ -n "$usage" ] || die "podman $cmd: no Usage message found"
+
+        # If usage ends in '[command]', recurse into subcommands
+        if expr "$usage" : '.*\[command\]$' >/dev/null; then
+            check_shell_completion "$@" $cmd
+            continue
+        fi
+
+        # Trim to command path so we only have the args
+        args="${usage/$command_string/}"
+        # Trim leading whitespaces
+        args="${args#"${args%%[![:space:]]*}"}"
+
+        # Extra args is used to match the correct argument number for the command
+        # This is important because some commands provide different suggestions based
+        # on the number of arguments.
+        extra_args=()
+
+        for arg in $args; do
+
+            match=false
+            i=0
+            while true; do
+
+                case $arg in
+
+                # If we have options than we need to check if we are getting flag completion
+                "[options]")
+                    # skip this for remote it fails if a command only has the latest flag e.g podman top
+                    if ! is_remote; then
+                        run_completion "$@" $cmd "--"
+                        # If this fails there is most likely a problem with the cobra library
+                        is "${lines[0]}" "--.*" "Found flag in suggestions"
+                        [ ${#lines[@]} -gt 2 ] || die "No flag suggestions"
+                        _check_completion_end NoFileComp
+                    fi
+                    # continue the outer for args loop
+                    continue 2
+                    ;;
+
+                *CONTAINER*)
+                    run_completion "$@" $cmd "${extra_args[@]}" ""
+                    is "$output" ".*-$random_container_name${nl}" "Found expected container in suggestions"
+
+                    match=true
+                    # resume
+                    ;;&
+
+                *POD*)
+                    run_completion "$@" $cmd "${extra_args[@]}" ""
+                    is "$output" ".*-$random_pod_name${nl}" "Found expected pod in suggestions"
+                    _check_completion_end NoFileComp
+
+                    match=true
+                    # resume
+                    ;;&
+
+                *IMAGE*)
+                    run_completion "$@" $cmd "${extra_args[@]}" ""
+                    is "$output" ".*localhost/$random_image_name:$random_image_tag${nl}" "Found expected image in suggestions"
+
+                    # check that we complete the image with and without tag after at least one char is typed
+                    run_completion "$@" $cmd "${extra_args[@]}" "${random_image_name:0:1}"
+                    is "$output" ".*$random_image_name:$random_image_tag${nl}" "Found expected image with tag in suggestions"
+                    is "$output" ".*$random_image_name${nl}" "Found expected image without tag in suggestions"
+
+                    # check that we complete the image id after at least two chars are typed
+                    run_completion "$@" $cmd "${extra_args[@]}" "${random_image_id:0:2}"
+                    is "$output" ".*$random_image_id${nl}" "Found expected image id in suggestions"
+
+                    match=true
+                    # resume
+                    ;;&
+
+                *NETWORK*)
+                    run_completion "$@" $cmd "${extra_args[@]}" ""
+                    is "$output" ".*$random_network_name${nl}" "Found network in suggestions"
+                    _check_completion_end NoFileComp
+
+                    match=true
+                    # resume
+                    ;;&
+
+                *VOLUME*)
+                    run_completion "$@" $cmd "${extra_args[@]}" ""
+                    is "$output" ".*$random_volume_name${nl}" "Found volume in suggestions"
+                    _check_completion_end NoFileComp
+
+                    match=true
+                    # resume
+                    ;;&
+
+                *REGISTRY*)
+                    run_completion "$@" $cmd "${extra_args[@]}" ""
+                    ### FIXME how can we get the configured registries?
+                    _check_completion_end NoFileComp
+                    ### FIXME this fails if no registries are configured
+                    [[ ${#lines[@]} -gt 2 ]] || die "No registries found in suggestions"
+
+                    match=true
+                    # resume
+                    ;;&
+
+                *PATH* | *CONTEXT* | *KUBEFILE* | *COMMAND* | *ARG...* | *URI*)
+                    # default shell completion should be done for everthing which accepts a path
+                    run_completion "$@" $cmd "${extra_args[@]}" ""
+
+                    # cp is a special case it returns ShellCompDirectiveNoSpace
+                    if [[ "$cmd" == "cp" ]]; then
+                        _check_completion_end NoSpace
+                    else
+                        _check_completion_end Default
+                        [[ ${#lines[@]} -eq 2 ]] || die "Suggestions are in the output"
+                    fi
+                    ;;
+
+                *)
+                    if [[ "$match" == "false" ]]; then
+                        dprint "UNKNOWN arg: $arg for $command_string ${extra_args[*]}"
+                    fi
+                    ;;
+
+                esac
+
+                # Increment the argument array
+                extra_args+=("arg")
+
+                i=$(($i + 1))
+                # If the argument ends with ...] than we accept 0...n args
+                # Loop three times to make sure we are not only completing the first arg
+                if [[ ! ${arg} =~ "..." ]] || [[ i -gt 3 ]]; then
+                    break
+                fi
+
+            done
+
+        done
+
+        # If the command takes no more parameters make sure we are getting no completion
+        if [[ ! ${args##* } =~ "..." ]]; then
+            run_completion "$@" $cmd "${extra_args[@]}" ""
+            _check_completion_end NoFileComp
+            if [ ${#lines[@]} -gt 2 ]; then
+                # checking for line count is not enough since we may inlcude additional debug output
+                # lines starting with [Debug] are allowed
+                i=0
+                length=$(( ${#lines[@]} - 2 ))
+                while [[ i -lt length ]]; do
+                    [[ "${lines[$i]:0:7}" == "[Debug]" ]] || die "Suggestions are in the output"
+                    i=$(( i + 1 ))
+                done
+            fi
+        fi
+
+    done
+
+}
+
+# run the completion cmd
+function run_completion() {
+    PODMAN="$PODMAN_COMPLETION" run_podman "$@"
+}
+
+# check for the given ShellCompDirective (always last line)
+function _check_completion_end() {
+    is "${lines[-1]}" "Completion ended with directive: ShellCompDirective$1" "Completion has wrong ShellCompDirective set"
+}
+
+
+@test "podman shell completion test" {
+
+    random_container_name=$(random_string 30)
+    random_pod_name=$(random_string 30)
+    random_image_name=$(random_string 30)
+    random_image_name=${random_image_name,,} # name must be lowercase
+    random_image_tag=$(random_string 5)
+    random_network_name=$(random_string 30)
+    random_volume_name=$(random_string 30)
+
+    # create a container for each state since some commands are only suggesting running container for example
+    run_podman create --name created-$random_container_name $IMAGE
+    run_podman run --name running-$random_container_name -d $IMAGE top
+    run_podman run --name pause-$random_container_name -d $IMAGE top
+    run_podman pause pause-$random_container_name
+    run_podman run --name exited-$random_container_name -d $IMAGE echo exited
+
+    # create pods for each state
+    run_podman pod create --name created-$random_pod_name
+    run_podman pod create --name running-$random_pod_name
+    run_podman run -d --name running-$random_pod_name-con --pod running-$random_pod_name $IMAGE top
+    run_podman pod create --name degraded-$random_pod_name
+    run_podman run -d --name degraded-$random_pod_name-con --pod degraded-$random_pod_name $IMAGE echo degraded
+    run_podman pod create --name exited-$random_pod_name
+    run_podman run -d --name exited-$random_pod_name-con --pod exited-$random_pod_name $IMAGE echo exited
+    run_podman pod stop exited-$random_pod_name
+
+    # create image name (just tag with new names no need to pull)
+    run_podman image tag $IMAGE $random_image_name:$random_image_tag
+    run_podman image list --format '{{.ID}}' --filter reference=$random_image_name
+    random_image_id="${lines[0]}"
+
+    # create network
+    run_podman network create $random_network_name
+
+    # create volume
+    run_podman volume create $random_volume_name
+
+
+    # $PODMAN may be a space-separated string, e.g. if we include a --url.
+    local -a podman_as_array=($PODMAN)
+    # __completeNoDesc must be the first arg if we running the completion cmd
+    PODMAN_COMPLETION="${podman_as_array[0]} __completeNoDesc ${podman_as_array[@]:1}"
+
+    # Called with no args -- start with 'podman --help'. check_shell_completion() will
+    # recurse for any subcommands.
+    check_shell_completion
+
+    # cleanup
+    run_podman volume rm $random_volume_name
+
+    run_podman network rm $random_network_name
+
+    run_podman image untag $IMAGE $random_image_name:$random_image_tag
+
+    for state in created running degraded exited; do
+        run_podman pod rm --force $state-$random_pod_name
+    done
+
+    for state in created running pause exited; do
+        run_podman rm --force $state-$random_container_name
+    done
+
+    # Clean up the pod pause image
+    run_podman image list --format '{{.ID}} {{.Repository}}'
+    while read id name; do
+        if [[ "$name" =~ /pause ]]; then
+            run_podman rmi $id
+        fi
+    done <<<"$output"
+
+}
diff --git a/test/system/helpers.bash b/test/system/helpers.bash
index 2cced10c2..6a7c6cc42 100644
--- a/test/system/helpers.bash
+++ b/test/system/helpers.bash
@@ -521,5 +521,15 @@ function remove_same_dev_warning() {
     output=$(printf '%s\n' "${lines[@]}")
 }
 
+# run 'podman help', parse the output looking for 'Available Commands';
+# return that list.
+function _podman_commands() {
+    dprint "$@"
+    run_podman help "$@" |
+        awk '/^Available Commands:/{ok=1;next}/^Options:/{ok=0}ok { print $1 }' |
+        grep .
+    "$output"
+}
+
 # END   miscellaneous tools
 ###############################################################################