From 79e21b5b162d3c2d3fb274b20bfe180c15284893 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Mon, 8 Aug 2022 11:44:52 +0200 Subject: kube play: sd-notify integration Integrate sd-notify policies into `kube play`. The policies can be configured for all contianers via the `io.containers.sdnotify` annotation or for indidivual containers via the `io.containers.sdnotify/$name` annotation. The `kube play` process will wait for all containers to be ready by waiting for the individual `READY=1` messages which are received via the `pkg/systemd/notifyproxy` proxy mechanism. Also update the simple "container" sd-notify test as it did not fully test the expected behavior which became obvious when adding the new tests. Signed-off-by: Valentin Rothberg --- test/system/260-sdnotify.bats | 141 +++++++++++++++++++++++++++++++++++++++--- test/system/helpers.bash | 17 +++++ 2 files changed, 149 insertions(+), 9 deletions(-) (limited to 'test/system') diff --git a/test/system/260-sdnotify.bats b/test/system/260-sdnotify.bats index fe442f57d..6c3ef7f3f 100644 --- a/test/system/260-sdnotify.bats +++ b/test/system/260-sdnotify.bats @@ -92,7 +92,7 @@ function _assert_mainpid_is_conmon() { cid="$output" run_podman container inspect $cid --format "{{.Config.SdNotifyMode}} {{.Config.SdNotifySocket}}" - is "$output" "ignore $NOTIFY_SOCKET" + is "$output" "ignore " "NOTIFY_SOCKET is not set with 'ignore' mode" run_podman 1 start --attach $cid is "$output" "" "\$NOTIFY_SOCKET in container" @@ -122,6 +122,7 @@ function _assert_mainpid_is_conmon() { is "$output" "READY" "\$NOTIFY_SOCKET in container" # The 'echo's help us debug failed runs + wait_for_file $_SOCAT_LOG run cat $_SOCAT_LOG echo "socat log:" echo "$output" @@ -141,7 +142,7 @@ READY=1" "sdnotify sent MAINPID and READY" # These tests can fail in dev. environment because of SELinux. # quick fix: chcon -t container_runtime_exec_t ./bin/podman @test "sdnotify : container" { - skip_if_aarch64 "FIXME: #15074 - fails on aarch64 non-remote" + skip_if_aarch64 "FIXME: #15277 sdnotify doesn't work on aarch64" # Sigh... we need to pull a humongous image because it has systemd-notify. # (IMPORTANT: fedora:32 and above silently removed systemd-notify; this # caused CI to hang. That's why we explicitly require fedora:31) @@ -156,7 +157,7 @@ READY=1" "sdnotify sent MAINPID and READY" _start_socat run_podman run -d --sdnotify=container $_FEDORA \ - sh -c 'printenv NOTIFY_SOCKET;echo READY;systemd-notify --ready;while ! test -f /stop;do sleep 0.1;done' + sh -c 'printenv NOTIFY_SOCKET; echo READY; while ! test -f /stop;do sleep 0.1;done;systemd-notify --ready' cid="$output" wait_for_ready $cid @@ -166,6 +167,8 @@ READY=1" "sdnotify sent MAINPID and READY" run_podman logs $cid is "${lines[0]}" "/run/notify/notify.sock" "NOTIFY_SOCKET is passed to container" + run_podman container inspect $cid --format "{{.State.ConmonPid}}" + mainPID="$output" # With container, READY=1 isn't necessarily the last message received; # just look for it anywhere in received messages run cat $_SOCAT_LOG @@ -173,19 +176,25 @@ READY=1" "sdnotify sent MAINPID and READY" echo "socat log:" echo "$output" - is "$output" ".*READY=1" "received READY=1 through notify socket" - - _assert_mainpid_is_conmon "$output" + is "$output" "MAINPID=$mainPID" "Container is not ready yet, so we only know the main PID" # Done. Stop container, clean up. run_podman exec $cid touch /stop run_podman wait $cid + + wait_for_file $_SOCAT_LOG + run cat $_SOCAT_LOG + echo "socat log:" + echo "$output" + is "$output" "MAINPID=$mainPID +READY=1" + run_podman rm $cid run_podman rmi $_FEDORA _stop_socat } -@test "sdnotify : play kube" { +@test "sdnotify : play kube - no policies" { # Create the YAMl file yaml_source="$PODMAN_TMPDIR/test.yaml" cat >$yaml_source <$yaml_source </dev/null & + + # Wait for both containers to be running + for i in $(seq 1 20); do + run_podman "?" container wait $container_a $container_b --condition="running" + if [[ $status == 0 ]]; then + break + fi + sleep 0.5 + # Just for debugging + run_podman ps -a + done + if [[ $status != 0 ]]; then + die "container $container_a and/or $container_b did not start" + fi + + # Make sure the containers have the correct policy + run_podman container inspect $container_a $container_b $service_container --format "{{.Config.SdNotifyMode}}" + is "$output" "container +conmon +ignore" + + is "$(< $_SOCAT_LOG)" "" "nothing received on socket" + + # Make sure the container received a "proxy" socket and is not using the + # one of `kube play` + run_podman container inspect $container_a --format "{{.Config.SdNotifySocket}}" + assert "$output" != $NOTIFY_SOCKET + + run_podman logs $container_a + is "${lines[0]}" "/run/notify/notify.sock" "NOTIFY_SOCKET is passed to container" + + # Instruct the container to send the READY + run_podman exec $container_a /bin/touch /stop + + run_podman container inspect $service_container --format "{{.State.ConmonPid}}" + main_pid="$output" + + run_podman container wait $container_a + wait_for_file $_SOCAT_LOG + # The 'echo's help us debug failed runs + run cat $_SOCAT_LOG + echo "socat log:" + echo "$output" + + is "$output" "MAINPID=$main_pid +READY=1" "sdnotify sent MAINPID and READY" + + _stop_socat + + # Clean up pod and pause image + run_podman play kube --down $yaml_source + run_podman rmi $_FEDORA $(pause_image) +} # vim: filetype=sh diff --git a/test/system/helpers.bash b/test/system/helpers.bash index b821175bb..5ff3fae6d 100644 --- a/test/system/helpers.bash +++ b/test/system/helpers.bash @@ -342,6 +342,23 @@ function wait_for_port() { die "Timed out waiting for $host:$port" } +################### +# wait_for_file # Returns once file is available on host +################### +function wait_for_file() { + local file=$1 # The path to the file + local _timeout=${2:-5} # Optional; default 5 seconds + + # Wait + while [ $_timeout -gt 0 ]; do + test -e $file && return + sleep 1 + _timeout=$(( $_timeout - 1 )) + done + + die "Timed out waiting for $file" +} + # END podman helpers ############################################################################### # BEGIN miscellaneous tools -- cgit v1.2.3-54-g00ecf