From 67a2e7351b0cea00ba455c0c4e52e13898fec203 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Fri, 5 Aug 2022 13:18:57 +0200 Subject: add `pkg/systemd/notifyproxy` Add a new package for proxying notify sockets and waiting for the READY=1 message to appear. May subject to further changes in future commits. Tests make sure that it behaves properly. Signed-off-by: Valentin Rothberg --- pkg/systemd/notifyproxy/notifyproxy.go | 94 +++++++++++++++++++++++++++++ pkg/systemd/notifyproxy/notifyproxy_test.go | 58 ++++++++++++++++++ 2 files changed, 152 insertions(+) create mode 100644 pkg/systemd/notifyproxy/notifyproxy.go create mode 100644 pkg/systemd/notifyproxy/notifyproxy_test.go (limited to 'pkg/systemd') diff --git a/pkg/systemd/notifyproxy/notifyproxy.go b/pkg/systemd/notifyproxy/notifyproxy.go new file mode 100644 index 000000000..b9f5737f7 --- /dev/null +++ b/pkg/systemd/notifyproxy/notifyproxy.go @@ -0,0 +1,94 @@ +package notifyproxy + +import ( + "io/ioutil" + "net" + "os" + "strings" + "syscall" + + "github.com/coreos/go-systemd/v22/daemon" + "github.com/sirupsen/logrus" +) + +// SendMessage sends the specified message to the specified socket. +func SendMessage(socketPath string, message string) error { + socketAddr := &net.UnixAddr{ + Name: socketPath, + Net: "unixgram", + } + conn, err := net.DialUnix(socketAddr.Net, nil, socketAddr) + if err != nil { + return err + } + defer conn.Close() + + _, err = conn.Write([]byte(message)) + return err +} + +// NotifyProxy can be used to proxy notify messages. +type NotifyProxy struct { + connection *net.UnixConn + socketPath string +} + +// New creates a NotifyProxy. The specified temp directory can be left empty. +func New(tmpDir string) (*NotifyProxy, error) { + tempFile, err := ioutil.TempFile(tmpDir, "-podman-notify-proxy.sock") + if err != nil { + return nil, err + } + defer tempFile.Close() + + socketPath := tempFile.Name() + if err := syscall.Unlink(socketPath); err != nil { // Unlink the socket so we can bind it + return nil, err + } + + socketAddr := &net.UnixAddr{ + Name: socketPath, + Net: "unixgram", + } + conn, err := net.ListenUnixgram(socketAddr.Net, socketAddr) + if err != nil { + return nil, err + } + + return &NotifyProxy{connection: conn, socketPath: socketPath}, nil +} + +// SocketPath returns the path of the socket the proxy is listening on. +func (p *NotifyProxy) SocketPath() string { + return p.socketPath +} + +// close closes the listener and removes the socket. +func (p *NotifyProxy) close() error { + defer os.Remove(p.socketPath) + return p.connection.Close() +} + +// WaitAndClose waits until receiving the `READY` notify message and close the +// listener. Note that the this function must only be executed inside a systemd +// service which will kill the process after a given timeout. +func (p *NotifyProxy) WaitAndClose() error { + defer func() { + if err := p.close(); err != nil { + logrus.Errorf("Closing notify proxy: %v", err) + } + }() + + for { + buf := make([]byte, 1024) + num, err := p.connection.Read(buf) + if err != nil { + return err + } + for _, s := range strings.Split(string(buf[:num]), "\n") { + if s == daemon.SdNotifyReady { + return nil + } + } + } +} diff --git a/pkg/systemd/notifyproxy/notifyproxy_test.go b/pkg/systemd/notifyproxy/notifyproxy_test.go new file mode 100644 index 000000000..edad95659 --- /dev/null +++ b/pkg/systemd/notifyproxy/notifyproxy_test.go @@ -0,0 +1,58 @@ +package notifyproxy + +import ( + "testing" + "time" + + "github.com/coreos/go-systemd/v22/daemon" + "github.com/stretchr/testify/require" +) + +// Helper function to send the specified message over the socket of the proxy. +func sendMessage(t *testing.T, proxy *NotifyProxy, message string) { + err := SendMessage(proxy.SocketPath(), message) + require.NoError(t, err) +} + +func TestNotifyProxy(t *testing.T) { + proxy, err := New("") + require.NoError(t, err) + require.FileExists(t, proxy.SocketPath()) + require.NoError(t, proxy.close()) + require.NoFileExists(t, proxy.SocketPath()) +} + +func TestWaitAndClose(t *testing.T) { + proxy, err := New("") + require.NoError(t, err) + require.FileExists(t, proxy.SocketPath()) + + ch := make(chan error) + + go func() { + ch <- proxy.WaitAndClose() + }() + + sendMessage(t, proxy, "foo\n") + time.Sleep(250 * time.Millisecond) + select { + case err := <-ch: + t.Fatalf("Should stil be waiting but received %v", err) + default: + } + + sendMessage(t, proxy, daemon.SdNotifyReady+"\nsomething else") + done := func() bool { + for i := 0; i < 10; i++ { + select { + case err := <-ch: + require.NoError(t, err, "Waiting should succeed") + return true + default: + time.Sleep(time.Duration(i*250) * time.Millisecond) + } + } + return false + }() + require.True(t, done, "READY MESSAGE SHOULD HAVE ARRIVED") +} -- cgit v1.2.3-54-g00ecf From 79e21b5b162d3c2d3fb274b20bfe180c15284893 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Mon, 8 Aug 2022 11:44:52 +0200 Subject: kube play: sd-notify integration Integrate sd-notify policies into `kube play`. The policies can be configured for all contianers via the `io.containers.sdnotify` annotation or for indidivual containers via the `io.containers.sdnotify/$name` annotation. The `kube play` process will wait for all containers to be ready by waiting for the individual `READY=1` messages which are received via the `pkg/systemd/notifyproxy` proxy mechanism. Also update the simple "container" sd-notify test as it did not fully test the expected behavior which became obvious when adding the new tests. Signed-off-by: Valentin Rothberg --- cmd/podman/kube/play.go | 4 + libpod/container_validate.go | 4 + libpod/define/config.go | 7 -- libpod/define/sdnotify.go | 20 +++++ libpod/options.go | 13 +-- pkg/domain/infra/abi/play.go | 79 ++++++++++++++++- pkg/domain/infra/abi/play_utils.go | 16 ++++ pkg/domain/infra/abi/play_utils_test.go | 38 +++++++++ pkg/specgen/container_validate.go | 6 +- pkg/specgen/generate/container_create.go | 8 +- pkg/systemd/notifyproxy/notifyproxy.go | 8 ++ test/system/260-sdnotify.bats | 141 +++++++++++++++++++++++++++++-- test/system/helpers.bash | 17 ++++ 13 files changed, 324 insertions(+), 37 deletions(-) create mode 100644 libpod/define/sdnotify.go create mode 100644 pkg/domain/infra/abi/play_utils.go create mode 100644 pkg/domain/infra/abi/play_utils_test.go (limited to 'pkg/systemd') diff --git a/cmd/podman/kube/play.go b/cmd/podman/kube/play.go index 4811bcf4b..07c4b59b9 100644 --- a/cmd/podman/kube/play.go +++ b/cmd/podman/kube/play.go @@ -182,6 +182,10 @@ func playFlags(cmd *cobra.Command) { } func Play(cmd *cobra.Command, args []string) error { + if playOptions.ServiceContainer && !playOptions.StartCLI { // Sanity check to be future proof + return fmt.Errorf("--service-container does not work with --start=stop") + } + // TLS verification in c/image is controlled via a `types.OptionalBool` // which allows for distinguishing among set-true, set-false, unspecified // which is important to implement a sane way of dealing with defaults of diff --git a/libpod/container_validate.go b/libpod/container_validate.go index e280c60d2..da33f6db7 100644 --- a/libpod/container_validate.go +++ b/libpod/container_validate.go @@ -133,5 +133,9 @@ func (c *Container) validate() error { if len(c.config.InitContainerType) > 0 && len(c.config.Pod) < 1 { return fmt.Errorf("init containers must be created in a pod: %w", define.ErrInvalidArg) } + + if c.config.SdNotifyMode == define.SdNotifyModeIgnore && len(c.config.SdNotifySocket) > 0 { + return fmt.Errorf("cannot set sd-notify socket %q with sd-notify mode %q", c.config.SdNotifySocket, c.config.SdNotifyMode) + } return nil } diff --git a/libpod/define/config.go b/libpod/define/config.go index 0181bd31c..34c1a675d 100644 --- a/libpod/define/config.go +++ b/libpod/define/config.go @@ -81,13 +81,6 @@ const NoLogging = "none" // PassthroughLogging is the string conmon expects when specifying to use the passthrough driver const PassthroughLogging = "passthrough" -// Strings used for --sdnotify option to podman -const ( - SdNotifyModeContainer = "container" - SdNotifyModeConmon = "conmon" - SdNotifyModeIgnore = "ignore" -) - // DefaultRlimitValue is the value set by default for nofile and nproc const RLimitDefaultValue = uint64(1048576) diff --git a/libpod/define/sdnotify.go b/libpod/define/sdnotify.go new file mode 100644 index 000000000..1d548c764 --- /dev/null +++ b/libpod/define/sdnotify.go @@ -0,0 +1,20 @@ +package define + +import "fmt" + +// Strings used for --sdnotify option to podman +const ( + SdNotifyModeContainer = "container" + SdNotifyModeConmon = "conmon" + SdNotifyModeIgnore = "ignore" +) + +// ValidateSdNotifyMode validates the specified mode. +func ValidateSdNotifyMode(mode string) error { + switch mode { + case "", SdNotifyModeContainer, SdNotifyModeConmon, SdNotifyModeIgnore: + return nil + default: + return fmt.Errorf("%w: invalid sdnotify value %q: must be %s, %s or %s", ErrInvalidArg, mode, SdNotifyModeContainer, SdNotifyModeConmon, SdNotifyModeIgnore) + } +} diff --git a/libpod/options.go b/libpod/options.go index 933c9a1c3..43ed1ff78 100644 --- a/libpod/options.go +++ b/libpod/options.go @@ -6,14 +6,12 @@ import ( "net" "os" "path/filepath" - "strings" "syscall" "github.com/containers/buildah/pkg/parse" nettypes "github.com/containers/common/libnetwork/types" "github.com/containers/common/pkg/config" "github.com/containers/common/pkg/secrets" - cutil "github.com/containers/common/pkg/util" "github.com/containers/image/v5/manifest" "github.com/containers/image/v5/types" "github.com/containers/podman/v4/libpod/define" @@ -29,12 +27,6 @@ import ( "github.com/sirupsen/logrus" ) -// Runtime Creation Options -var ( - // SdNotifyModeValues describes the only values that SdNotifyMode can be - SdNotifyModeValues = []string{define.SdNotifyModeContainer, define.SdNotifyModeConmon, define.SdNotifyModeIgnore} -) - // WithStorageConfig uses the given configuration to set up container storage. // If this is not specified, the system default configuration will be used // instead. @@ -631,9 +623,8 @@ func WithSdNotifyMode(mode string) CtrCreateOption { return define.ErrCtrFinalized } - // verify values - if len(mode) > 0 && !cutil.StringInSlice(strings.ToLower(mode), SdNotifyModeValues) { - return fmt.Errorf("--sdnotify values must be one of %q: %w", strings.Join(SdNotifyModeValues, ", "), define.ErrInvalidArg) + if err := define.ValidateSdNotifyMode(mode); err != nil { + return err } ctr.config.SdNotifyMode = mode diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go index 3f2fd5f92..faa89cc26 100644 --- a/pkg/domain/infra/abi/play.go +++ b/pkg/domain/infra/abi/play.go @@ -27,13 +27,19 @@ import ( "github.com/containers/podman/v4/pkg/specgen/generate" "github.com/containers/podman/v4/pkg/specgen/generate/kube" "github.com/containers/podman/v4/pkg/specgenutil" + "github.com/containers/podman/v4/pkg/systemd/notifyproxy" "github.com/containers/podman/v4/pkg/util" + "github.com/coreos/go-systemd/v22/daemon" "github.com/ghodss/yaml" "github.com/opencontainers/go-digest" "github.com/sirupsen/logrus" yamlv3 "gopkg.in/yaml.v3" ) +// sdNotifyAnnotation allows for configuring service-global and +// container-specific sd-notify modes. +const sdNotifyAnnotation = "io.containers.sdnotify" + // createServiceContainer creates a container that can later on // be associated with the pods of a K8s yaml. It will be started along with // the first pod. @@ -73,7 +79,12 @@ func (ic *ContainerEngine) createServiceContainer(ctx context.Context, name stri return nil, fmt.Errorf("creating runtime spec for service container: %w", err) } opts = append(opts, libpod.WithIsService()) - opts = append(opts, libpod.WithSdNotifyMode(define.SdNotifyModeConmon)) + + // Set the sd-notify mode to "ignore". Podman is responsible for + // sending the notify messages when all containers are ready. + // The mode for individual containers or entire pods can be configured + // via the `sdNotifyAnnotation` annotation in the K8s YAML. + opts = append(opts, libpod.WithSdNotifyMode(define.SdNotifyModeIgnore)) // Create a new libpod container based on the spec. ctr, err := ic.Libpod.NewContainer(ctx, runtimeSpec, spec, false, opts...) @@ -96,6 +107,10 @@ func k8sName(content []byte, suffix string) string { } func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options entities.PlayKubeOptions) (_ *entities.PlayKubeReport, finalErr error) { + if options.ServiceContainer && options.Start == types.OptionalBoolFalse { // Sanity check to be future proof + return nil, fmt.Errorf("running a service container requires starting the pod(s)") + } + report := &entities.PlayKubeReport{} validKinds := 0 @@ -121,6 +136,8 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options var configMaps []v1.ConfigMap + ranContainers := false + var serviceContainer *libpod.Container // create pod on each document if it is a pod or deployment // any other kube kind will be skipped for _, document := range documentList { @@ -130,8 +147,7 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options } // TODO: create constants for the various "kinds" of yaml files. - var serviceContainer *libpod.Container - if options.ServiceContainer && (kind == "Pod" || kind == "Deployment") { + if options.ServiceContainer && serviceContainer == nil && (kind == "Pod" || kind == "Deployment") { ctr, err := ic.createServiceContainer(ctx, k8sName(content, "service"), options) if err != nil { return nil, err @@ -178,6 +194,7 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options report.Pods = append(report.Pods, r.Pods...) validKinds++ + ranContainers = true case "Deployment": var deploymentYAML v1apps.Deployment @@ -192,6 +209,7 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options report.Pods = append(report.Pods, r.Pods...) validKinds++ + ranContainers = true case "PersistentVolumeClaim": var pvcYAML v1.PersistentVolumeClaim @@ -239,6 +257,20 @@ func (ic *ContainerEngine) PlayKube(ctx context.Context, body io.Reader, options return nil, fmt.Errorf("YAML document does not contain any supported kube kind") } + if options.ServiceContainer && ranContainers { + // We can consider the service to be up and running now. + // Send the sd-notify messages pointing systemd to the + // service container. + data, err := serviceContainer.Inspect(false) + if err != nil { + return nil, err + } + message := fmt.Sprintf("MAINPID=%d\n%s", data.State.ConmonPid, daemon.SdNotifyReady) + if err := notifyproxy.SendMessage("", message); err != nil { + return nil, err + } + } + return report, nil } @@ -280,6 +312,11 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY report entities.PlayKubeReport ) + mainSdNotifyMode, err := getSdNotifyMode(annotations, "") + if err != nil { + return nil, err + } + // Create the secret manager before hand secretsManager, err := ic.Libpod.SecretsManager() if err != nil { @@ -562,6 +599,9 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY initContainers = append(initContainers, ctr) } + + var sdNotifyProxies []*notifyproxy.NotifyProxy // containers' sd-notify proxies + for _, container := range podYAML.Spec.Containers { // Error out if the same name is used for more than one container if _, ok := ctrNames[container.Name]; ok { @@ -606,7 +646,31 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY if err != nil { return nil, err } - opts = append(opts, libpod.WithSdNotifyMode(define.SdNotifyModeIgnore)) + + sdNotifyMode := mainSdNotifyMode + ctrNotifyMode, err := getSdNotifyMode(annotations, container.Name) + if err != nil { + return nil, err + } + if ctrNotifyMode != "" { + sdNotifyMode = ctrNotifyMode + } + if sdNotifyMode == "" { // Default to "ignore" + sdNotifyMode = define.SdNotifyModeIgnore + } + + opts = append(opts, libpod.WithSdNotifyMode(sdNotifyMode)) + + // Create a notify proxy for the container. + if sdNotifyMode != "" && sdNotifyMode != define.SdNotifyModeIgnore { + proxy, err := notifyproxy.New("") + if err != nil { + return nil, err + } + sdNotifyProxies = append(sdNotifyProxies, proxy) + opts = append(opts, libpod.WithSdNotifySocket(proxy.SocketPath())) + } + ctr, err := generate.ExecuteCreate(ctx, ic.Libpod, rtSpec, spec, false, opts...) if err != nil { return nil, err @@ -624,6 +688,13 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY playKubePod.ContainerErrors = append(playKubePod.ContainerErrors, fmt.Errorf("error starting container %s: %w", id, err).Error()) fmt.Println(playKubePod.ContainerErrors) } + + // Wait for each proxy to receive a READY message. + for _, proxy := range sdNotifyProxies { + if err := proxy.WaitAndClose(); err != nil { + return nil, err + } + } } playKubePod.ID = pod.ID() diff --git a/pkg/domain/infra/abi/play_utils.go b/pkg/domain/infra/abi/play_utils.go new file mode 100644 index 000000000..482a158e6 --- /dev/null +++ b/pkg/domain/infra/abi/play_utils.go @@ -0,0 +1,16 @@ +package abi + +import "github.com/containers/podman/v4/libpod/define" + +// getSdNotifyMode returns the `sdNotifyAnnotation/$name` for the specified +// name. If name is empty, it'll only look for `sdNotifyAnnotation`. +func getSdNotifyMode(annotations map[string]string, name string) (string, error) { + var mode string + switch len(name) { + case 0: + mode = annotations[sdNotifyAnnotation] + default: + mode = annotations[sdNotifyAnnotation+"/"+name] + } + return mode, define.ValidateSdNotifyMode(mode) +} diff --git a/pkg/domain/infra/abi/play_utils_test.go b/pkg/domain/infra/abi/play_utils_test.go new file mode 100644 index 000000000..80a9fe543 --- /dev/null +++ b/pkg/domain/infra/abi/play_utils_test.go @@ -0,0 +1,38 @@ +package abi + +import ( + "testing" + + "github.com/containers/podman/v4/libpod/define" + "github.com/stretchr/testify/require" +) + +func TestGetSdNotifyMode(t *testing.T) { + tests := []struct { + key, value, name, result string + mustError bool + }{ + {sdNotifyAnnotation, define.SdNotifyModeConmon, "", define.SdNotifyModeConmon, false}, + {sdNotifyAnnotation + "/container-a", define.SdNotifyModeContainer, "container-a", define.SdNotifyModeContainer, false}, + {sdNotifyAnnotation + "/container-b", define.SdNotifyModeIgnore, "container-b", define.SdNotifyModeIgnore, false}, + {sdNotifyAnnotation + "/container-c", "", "container-c", "", false}, + {sdNotifyAnnotation + "-/wrong-key", "xxx", "wrong-key", "", false}, + {sdNotifyAnnotation + "/container-error", "invalid", "container-error", "", true}, + } + + annotations := make(map[string]string) + // Populate the annotations + for _, test := range tests { + annotations[test.key] = test.value + } + // Run the tests + for _, test := range tests { + result, err := getSdNotifyMode(annotations, test.name) + if test.mustError { + require.Error(t, err, "%v", test) + continue + } + require.NoError(t, err, "%v", test) + require.Equal(t, test.result, result, "%v", test) + } +} diff --git a/pkg/specgen/container_validate.go b/pkg/specgen/container_validate.go index 63d94b6b3..064245602 100644 --- a/pkg/specgen/container_validate.go +++ b/pkg/specgen/container_validate.go @@ -67,9 +67,9 @@ func (s *SpecGenerator) Validate() error { if len(s.ContainerBasicConfig.Systemd) > 0 && !util.StringInSlice(strings.ToLower(s.ContainerBasicConfig.Systemd), SystemDValues) { return fmt.Errorf("--systemd values must be one of %q: %w", strings.Join(SystemDValues, ", "), ErrInvalidSpecConfig) } - // sdnotify values must be container, conmon, or ignore - if len(s.ContainerBasicConfig.SdNotifyMode) > 0 && !util.StringInSlice(strings.ToLower(s.ContainerBasicConfig.SdNotifyMode), SdNotifyModeValues) { - return fmt.Errorf("--sdnotify values must be one of %q: %w", strings.Join(SdNotifyModeValues, ", "), ErrInvalidSpecConfig) + + if err := define.ValidateSdNotifyMode(s.ContainerBasicConfig.SdNotifyMode); err != nil { + return err } // diff --git a/pkg/specgen/generate/container_create.go b/pkg/specgen/generate/container_create.go index f4c67e534..e9cec2873 100644 --- a/pkg/specgen/generate/container_create.go +++ b/pkg/specgen/generate/container_create.go @@ -353,9 +353,11 @@ func createContainerOptions(rt *libpod.Runtime, s *specgen.SpecGenerator, pod *l } if len(s.SdNotifyMode) > 0 { options = append(options, libpod.WithSdNotifyMode(s.SdNotifyMode)) - } - if notify, ok := os.LookupEnv("NOTIFY_SOCKET"); ok { - options = append(options, libpod.WithSdNotifySocket(notify)) + if s.SdNotifyMode != define.SdNotifyModeIgnore { + if notify, ok := os.LookupEnv("NOTIFY_SOCKET"); ok { + options = append(options, libpod.WithSdNotifySocket(notify)) + } + } } if pod != nil { diff --git a/pkg/systemd/notifyproxy/notifyproxy.go b/pkg/systemd/notifyproxy/notifyproxy.go index b9f5737f7..9e6eb4cf0 100644 --- a/pkg/systemd/notifyproxy/notifyproxy.go +++ b/pkg/systemd/notifyproxy/notifyproxy.go @@ -12,7 +12,15 @@ import ( ) // SendMessage sends the specified message to the specified socket. +// No message is sent if no socketPath is provided and the NOTIFY_SOCKET +// variable is not set either. func SendMessage(socketPath string, message string) error { + if socketPath == "" { + socketPath, _ = os.LookupEnv("NOTIFY_SOCKET") + if socketPath == "" { + return nil + } + } socketAddr := &net.UnixAddr{ Name: socketPath, Net: "unixgram", diff --git a/test/system/260-sdnotify.bats b/test/system/260-sdnotify.bats index fe442f57d..6c3ef7f3f 100644 --- a/test/system/260-sdnotify.bats +++ b/test/system/260-sdnotify.bats @@ -92,7 +92,7 @@ function _assert_mainpid_is_conmon() { cid="$output" run_podman container inspect $cid --format "{{.Config.SdNotifyMode}} {{.Config.SdNotifySocket}}" - is "$output" "ignore $NOTIFY_SOCKET" + is "$output" "ignore " "NOTIFY_SOCKET is not set with 'ignore' mode" run_podman 1 start --attach $cid is "$output" "" "\$NOTIFY_SOCKET in container" @@ -122,6 +122,7 @@ function _assert_mainpid_is_conmon() { is "$output" "READY" "\$NOTIFY_SOCKET in container" # The 'echo's help us debug failed runs + wait_for_file $_SOCAT_LOG run cat $_SOCAT_LOG echo "socat log:" echo "$output" @@ -141,7 +142,7 @@ READY=1" "sdnotify sent MAINPID and READY" # These tests can fail in dev. environment because of SELinux. # quick fix: chcon -t container_runtime_exec_t ./bin/podman @test "sdnotify : container" { - skip_if_aarch64 "FIXME: #15074 - fails on aarch64 non-remote" + skip_if_aarch64 "FIXME: #15277 sdnotify doesn't work on aarch64" # Sigh... we need to pull a humongous image because it has systemd-notify. # (IMPORTANT: fedora:32 and above silently removed systemd-notify; this # caused CI to hang. That's why we explicitly require fedora:31) @@ -156,7 +157,7 @@ READY=1" "sdnotify sent MAINPID and READY" _start_socat run_podman run -d --sdnotify=container $_FEDORA \ - sh -c 'printenv NOTIFY_SOCKET;echo READY;systemd-notify --ready;while ! test -f /stop;do sleep 0.1;done' + sh -c 'printenv NOTIFY_SOCKET; echo READY; while ! test -f /stop;do sleep 0.1;done;systemd-notify --ready' cid="$output" wait_for_ready $cid @@ -166,6 +167,8 @@ READY=1" "sdnotify sent MAINPID and READY" run_podman logs $cid is "${lines[0]}" "/run/notify/notify.sock" "NOTIFY_SOCKET is passed to container" + run_podman container inspect $cid --format "{{.State.ConmonPid}}" + mainPID="$output" # With container, READY=1 isn't necessarily the last message received; # just look for it anywhere in received messages run cat $_SOCAT_LOG @@ -173,19 +176,25 @@ READY=1" "sdnotify sent MAINPID and READY" echo "socat log:" echo "$output" - is "$output" ".*READY=1" "received READY=1 through notify socket" - - _assert_mainpid_is_conmon "$output" + is "$output" "MAINPID=$mainPID" "Container is not ready yet, so we only know the main PID" # Done. Stop container, clean up. run_podman exec $cid touch /stop run_podman wait $cid + + wait_for_file $_SOCAT_LOG + run cat $_SOCAT_LOG + echo "socat log:" + echo "$output" + is "$output" "MAINPID=$mainPID +READY=1" + run_podman rm $cid run_podman rmi $_FEDORA _stop_socat } -@test "sdnotify : play kube" { +@test "sdnotify : play kube - no policies" { # Create the YAMl file yaml_source="$PODMAN_TMPDIR/test.yaml" cat >$yaml_source <$yaml_source </dev/null & + + # Wait for both containers to be running + for i in $(seq 1 20); do + run_podman "?" container wait $container_a $container_b --condition="running" + if [[ $status == 0 ]]; then + break + fi + sleep 0.5 + # Just for debugging + run_podman ps -a + done + if [[ $status != 0 ]]; then + die "container $container_a and/or $container_b did not start" + fi + + # Make sure the containers have the correct policy + run_podman container inspect $container_a $container_b $service_container --format "{{.Config.SdNotifyMode}}" + is "$output" "container +conmon +ignore" + + is "$(< $_SOCAT_LOG)" "" "nothing received on socket" + + # Make sure the container received a "proxy" socket and is not using the + # one of `kube play` + run_podman container inspect $container_a --format "{{.Config.SdNotifySocket}}" + assert "$output" != $NOTIFY_SOCKET + + run_podman logs $container_a + is "${lines[0]}" "/run/notify/notify.sock" "NOTIFY_SOCKET is passed to container" + + # Instruct the container to send the READY + run_podman exec $container_a /bin/touch /stop + + run_podman container inspect $service_container --format "{{.State.ConmonPid}}" + main_pid="$output" + + run_podman container wait $container_a + wait_for_file $_SOCAT_LOG + # The 'echo's help us debug failed runs + run cat $_SOCAT_LOG + echo "socat log:" + echo "$output" + + is "$output" "MAINPID=$main_pid +READY=1" "sdnotify sent MAINPID and READY" + + _stop_socat + + # Clean up pod and pause image + run_podman play kube --down $yaml_source + run_podman rmi $_FEDORA $(pause_image) +} # vim: filetype=sh diff --git a/test/system/helpers.bash b/test/system/helpers.bash index b821175bb..5ff3fae6d 100644 --- a/test/system/helpers.bash +++ b/test/system/helpers.bash @@ -342,6 +342,23 @@ function wait_for_port() { die "Timed out waiting for $host:$port" } +################### +# wait_for_file # Returns once file is available on host +################### +function wait_for_file() { + local file=$1 # The path to the file + local _timeout=${2:-5} # Optional; default 5 seconds + + # Wait + while [ $_timeout -gt 0 ]; do + test -e $file && return + sleep 1 + _timeout=$(( $_timeout - 1 )) + done + + die "Timed out waiting for $file" +} + # END podman helpers ############################################################################### # BEGIN miscellaneous tools -- cgit v1.2.3-54-g00ecf