kube play: sd-notify integration

Integrate sd-notify policies into `kube play`.  The policies can be
configured for all contianers via the `io.containers.sdnotify`
annotation or for indidivual containers via the
`io.containers.sdnotify/$name` annotation.

The `kube play` process will wait for all containers to be ready by
waiting for the individual `READY=1` messages which are received via
the `pkg/systemd/notifyproxy` proxy mechanism.

Also update the simple "container" sd-notify test as it did not fully
test the expected behavior which became obvious when adding the new
tests.

Signed-off-by: Valentin Rothberg <vrothberg@redhat.com>

2 files changed, 149 insertions, 9 deletions

diff --git a/test/system/260-sdnotify.bats b/test/system/260-sdnotify.bats
index fe442f57d..6c3ef7f3f 100644
--- a/test/system/260-sdnotify.bats
+++ b/test/system/260-sdnotify.bats
@@ -92,7 +92,7 @@ function _assert_mainpid_is_conmon() {
     cid="$output"
 
     run_podman container inspect $cid --format "{{.Config.SdNotifyMode}} {{.Config.SdNotifySocket}}"
-    is "$output" "ignore $NOTIFY_SOCKET"
+    is "$output" "ignore " "NOTIFY_SOCKET is not set with 'ignore' mode"
 
     run_podman 1 start --attach $cid
     is "$output" "" "\$NOTIFY_SOCKET in container"
@@ -122,6 +122,7 @@ function _assert_mainpid_is_conmon() {
     is "$output" "READY" "\$NOTIFY_SOCKET in container"
 
     # The 'echo's help us debug failed runs
+    wait_for_file $_SOCAT_LOG
     run cat $_SOCAT_LOG
     echo "socat log:"
     echo "$output"
@@ -141,7 +142,7 @@ READY=1" "sdnotify sent MAINPID and READY"
 # These tests can fail in dev. environment because of SELinux.
 # quick fix: chcon -t container_runtime_exec_t ./bin/podman
 @test "sdnotify : container" {
-    skip_if_aarch64 "FIXME: #15074 - fails on aarch64 non-remote"
+    skip_if_aarch64 "FIXME: #15277 sdnotify doesn't work on aarch64"
     # Sigh... we need to pull a humongous image because it has systemd-notify.
     # (IMPORTANT: fedora:32 and above silently removed systemd-notify; this
     # caused CI to hang. That's why we explicitly require fedora:31)
@@ -156,7 +157,7 @@ READY=1" "sdnotify sent MAINPID and READY"
     _start_socat
 
     run_podman run -d --sdnotify=container $_FEDORA \
-               sh -c 'printenv NOTIFY_SOCKET;echo READY;systemd-notify --ready;while ! test -f /stop;do sleep 0.1;done'
+               sh -c 'printenv NOTIFY_SOCKET; echo READY; while ! test -f /stop;do sleep 0.1;done;systemd-notify --ready'
     cid="$output"
     wait_for_ready $cid
 
@@ -166,6 +167,8 @@ READY=1" "sdnotify sent MAINPID and READY"
     run_podman logs $cid
     is "${lines[0]}" "/run/notify/notify.sock" "NOTIFY_SOCKET is passed to container"
 
+    run_podman container inspect $cid --format "{{.State.ConmonPid}}"
+    mainPID="$output"
     # With container, READY=1 isn't necessarily the last message received;
     # just look for it anywhere in received messages
     run cat $_SOCAT_LOG
@@ -173,19 +176,25 @@ READY=1" "sdnotify sent MAINPID and READY"
     echo "socat log:"
     echo "$output"
 
-    is "$output" ".*READY=1" "received READY=1 through notify socket"
-
-    _assert_mainpid_is_conmon "$output"
+    is "$output" "MAINPID=$mainPID" "Container is not ready yet, so we only know the main PID"
 
     # Done. Stop container, clean up.
     run_podman exec $cid touch /stop
     run_podman wait $cid
+
+    wait_for_file $_SOCAT_LOG
+    run cat $_SOCAT_LOG
+    echo "socat log:"
+    echo "$output"
+    is "$output" "MAINPID=$mainPID
+READY=1"
+
     run_podman rm $cid
     run_podman rmi $_FEDORA
     _stop_socat
 }
 
-@test "sdnotify : play kube" {
+@test "sdnotify : play kube - no policies" {
     # Create the YAMl file
     yaml_source="$PODMAN_TMPDIR/test.yaml"
     cat >$yaml_source <<EOF
@@ -214,8 +223,15 @@ EOF
     _start_socat
 
     run_podman play kube --service-container=true $yaml_source
+
+    # Make sure the containers have the correct policy.
+    run_podman container inspect test_pod-test $service_container --format "{{.Config.SdNotifyMode}}"
+    is "$output" "ignore
+ignore"
+
     run_podman container inspect $service_container --format "{{.State.ConmonPid}}"
     mainPID="$output"
+    wait_for_file $_SOCAT_LOG
     # The 'echo's help us debug failed runs
     run cat $_SOCAT_LOG
     echo "socat log:"
@@ -228,9 +244,116 @@ READY=1" "sdnotify sent MAINPID and READY"
 
     # Clean up pod and pause image
     run_podman play kube --down $PODMAN_TMPDIR/test.yaml
-    run_podman version --format "{{.Server.Version}}-{{.Server.Built}}"
-    podman rmi -f localhost/podman-pause:$output
+    run_podman rmi $(pause_image)
 }
 
+@test "sdnotify : play kube - with policies" {
+    skip_if_aarch64 "FIXME: #15277 sdnotify doesn't work on aarch64"
+
+    # Sigh... we need to pull a humongous image because it has systemd-notify.
+    # (IMPORTANT: fedora:32 and above silently removed systemd-notify; this
+    # caused CI to hang. That's why we explicitly require fedora:31)
+    # FIXME: is there a smaller image we could use?
+    local _FEDORA="$PODMAN_TEST_IMAGE_REGISTRY/$PODMAN_TEST_IMAGE_USER/fedora:31"
+    # Pull that image. Retry in case of flakes.
+    run_podman pull $_FEDORA || \
+        run_podman pull $_FEDORA || \
+        run_podman pull $_FEDORA
+
+    # Create the YAMl file
+    yaml_source="$PODMAN_TMPDIR/test.yaml"
+    cat >$yaml_source <<EOF
+apiVersion: v1
+kind: Pod
+metadata:
+  labels:
+    app: test
+  name: test_pod
+  annotations:
+    io.containers.sdnotify:   "container"
+    io.containers.sdnotify/b: "conmon"
+spec:
+  containers:
+  - command:
+    - /bin/sh
+    - -c
+    - 'printenv NOTIFY_SOCKET; echo READY; while ! test -f /stop;do sleep 0.1;done;systemd-notify --ready'
+    image: $_FEDORA
+    name: a
+  - command:
+    - /bin/sh
+    - -c
+    - 'echo READY; top'
+    image: $IMAGE
+    name: b
+EOF
+    container_a="test_pod-a"
+    container_b="test_pod-b"
+
+    # The name of the service container is predictable: the first 12 characters
+    # of the hash of the YAML file followed by the "-service" suffix
+    yaml_sha=$(sha256sum $yaml_source)
+    service_container="${yaml_sha:0:12}-service"
+
+    export NOTIFY_SOCKET=$PODMAN_TMPDIR/conmon.sock
+    _start_socat
+
+    # Run `play kube` in the background as it will wait for all containers to
+    # send the READY=1 message.
+    timeout --foreground -v --kill=10 60 \
+        $PODMAN play kube --service-container=true $yaml_source &>/dev/null &
+
+    # Wait for both containers to be running
+    for i in $(seq 1 20); do
+        run_podman "?" container wait $container_a $container_b --condition="running"
+        if [[ $status == 0 ]]; then
+            break
+        fi
+        sleep 0.5
+        # Just for debugging
+        run_podman ps -a
+    done
+    if [[ $status != 0 ]]; then
+        die "container $container_a and/or $container_b did not start"
+    fi
+
+    # Make sure the containers have the correct policy
+    run_podman container inspect $container_a $container_b $service_container --format "{{.Config.SdNotifyMode}}"
+    is "$output" "container
+conmon
+ignore"
+
+    is "$(< $_SOCAT_LOG)" "" "nothing received on socket"
+
+    # Make sure the container received a "proxy" socket and is not using the
+    # one of `kube play`
+    run_podman container inspect $container_a --format "{{.Config.SdNotifySocket}}"
+    assert "$output" != $NOTIFY_SOCKET
+
+    run_podman logs $container_a
+    is "${lines[0]}" "/run/notify/notify.sock" "NOTIFY_SOCKET is passed to container"
+
+    # Instruct the container to send the READY
+    run_podman exec $container_a /bin/touch /stop
+
+    run_podman container inspect $service_container --format "{{.State.ConmonPid}}"
+    main_pid="$output"
+
+    run_podman container wait $container_a
+    wait_for_file $_SOCAT_LOG
+    # The 'echo's help us debug failed runs
+    run cat $_SOCAT_LOG
+    echo "socat log:"
+    echo "$output"
+
+    is "$output" "MAINPID=$main_pid
+READY=1" "sdnotify sent MAINPID and READY"
+
+    _stop_socat
+
+    # Clean up pod and pause image
+    run_podman play kube --down $yaml_source
+    run_podman rmi $_FEDORA $(pause_image)
+}
 
 # vim: filetype=sh
diff --git a/test/system/helpers.bash b/test/system/helpers.bash
index b821175bb..5ff3fae6d 100644
--- a/test/system/helpers.bash
+++ b/test/system/helpers.bash
@@ -342,6 +342,23 @@ function wait_for_port() {
     die "Timed out waiting for $host:$port"
 }
 
+###################
+#  wait_for_file  #  Returns once file is available on host
+###################
+function wait_for_file() {
+    local file=$1                       # The path to the file
+    local _timeout=${2:-5}              # Optional; default 5 seconds
+
+    # Wait
+    while [ $_timeout -gt 0 ]; do
+        test -e $file && return
+        sleep 1
+        _timeout=$(( $_timeout - 1 ))
+    done
+
+    die "Timed out waiting for $file"
+}
+
 # END   podman helpers
 ###############################################################################
 # BEGIN miscellaneous tools