Show Health Status events

Previously, health status events were not being generated at all. Both
the API and `podman events` will generate health_status events.

```
{"status":"health_status","id":"ae498ac3aa6c63db8b69a37583a6eae1a9cefbdbdbeeadcf8e1d66d745f0df63","from":"localhost/healthcheck-demo:latest","Type":"container","Action":"health_status","Actor":{"ID":"ae498ac3aa6c63db8b69a37583a6eae1a9cefbdbdbeeadcf8e1d66d745f0df63","Attributes":{"containerExitCode":"0","image":"localhost/healthcheck-demo:latest","io.buildah.version":"1.26.1","maintainer":"NGINX Docker Maintainers \u003cdocker-maint@nginx.com\u003e","name":"healthcheck-demo"}},"scope":"local","time":1656082205,"timeNano":1656082205882271276,"HealthStatus":"healthy"}
```
```
2022-06-24 11:06:04.886238493 -0400 EDT container health_status ae498ac3aa6c63db8b69a37583a6eae1a9cefbdbdbeeadcf8e1d66d745f0df63 (image=localhost/healthcheck-demo:latest, name=healthcheck-demo, health_status=healthy, io.buildah.version=1.26.1, maintainer=NGINX Docker Maintainers <docker-maint@nginx.com>)
```

Signed-off-by: Jake Correnti <jcorrenti13@gmail.com>

9 files changed, 81 insertions, 10 deletions

diff --git a/libpod/container_exec.go b/libpod/container_exec.go
index 1e8fce4da..6ec79fb0a 100644
--- a/libpod/container_exec.go
+++ b/libpod/container_exec.go
@@ -277,9 +277,13 @@ func (c *Container) ExecStart(sessionID string) error {
 	return c.save()
 }
 
+func (c *Container) ExecStartAndAttach(sessionID string, streams *define.AttachStreams, newSize *define.TerminalSize) error {
+	return c.execStartAndAttach(sessionID, streams, newSize, false)
+}
+
 // ExecStartAndAttach starts and attaches to an exec session in a container.
 // newSize resizes the tty to this size before the process is started, must be nil if the exec session has no tty
-func (c *Container) ExecStartAndAttach(sessionID string, streams *define.AttachStreams, newSize *define.TerminalSize) error {
+func (c *Container) execStartAndAttach(sessionID string, streams *define.AttachStreams, newSize *define.TerminalSize, isHealthcheck bool) error {
 	if !c.batched {
 		c.lock.Lock()
 		defer c.lock.Unlock()
@@ -315,7 +319,12 @@ func (c *Container) ExecStartAndAttach(sessionID string, streams *define.AttachS
 		return err
 	}
 
-	c.newContainerEvent(events.Exec)
+	if isHealthcheck {
+		c.newContainerEvent(events.HealthStatus)
+	} else {
+		c.newContainerEvent(events.Exec)
+	}
+
 	logrus.Debugf("Successfully started exec session %s in container %s", session.ID(), c.ID())
 
 	var lastErr error
@@ -743,10 +752,14 @@ func (c *Container) ExecResize(sessionID string, newSize define.TerminalSize) er
 	return c.ociRuntime.ExecAttachResize(c, sessionID, newSize)
 }
 
+func (c *Container) Exec(config *ExecConfig, streams *define.AttachStreams, resize <-chan define.TerminalSize) (int, error) {
+	return c.exec(config, streams, resize, false)
+}
+
 // Exec emulates the old Libpod exec API, providing a single call to create,
 // run, and remove an exec session. Returns exit code and error. Exit code is
 // not guaranteed to be set sanely if error is not nil.
-func (c *Container) Exec(config *ExecConfig, streams *define.AttachStreams, resize <-chan define.TerminalSize) (int, error) {
+func (c *Container) exec(config *ExecConfig, streams *define.AttachStreams, resize <-chan define.TerminalSize, isHealthcheck bool) (int, error) {
 	sessionID, err := c.ExecCreate(config)
 	if err != nil {
 		return -1, err
@@ -780,7 +793,7 @@ func (c *Container) Exec(config *ExecConfig, streams *define.AttachStreams, resi
 		}()
 	}
 
-	if err := c.ExecStartAndAttach(sessionID, streams, size); err != nil {
+	if err := c.execStartAndAttach(sessionID, streams, size, isHealthcheck); err != nil {
 		return -1, err
 	}
 
diff --git a/libpod/events.go b/libpod/events.go
index f09d8402a..a7783cf5a 100644
--- a/libpod/events.go
+++ b/libpod/events.go
@@ -33,6 +33,16 @@ func (c *Container) newContainerEvent(status events.Status) {
 		Attributes: c.Labels(),
 	}
 
+	// if the current event is a HealthStatus event, we need to get the current
+	// status of the container to pass to the event
+	if status == events.HealthStatus {
+		containerHealthStatus, err := c.healthCheckStatus()
+		if err != nil {
+			e.HealthStatus = fmt.Sprintf("%v", err)
+		}
+		e.HealthStatus = containerHealthStatus
+	}
+
 	if err := c.runtime.eventer.Write(e); err != nil {
 		logrus.Errorf("Unable to write pod event: %q", err)
 	}
diff --git a/libpod/events/config.go b/libpod/events/config.go
index 2e7016136..a678baa2d 100644
--- a/libpod/events/config.go
+++ b/libpod/events/config.go
@@ -40,6 +40,8 @@ type Event struct {
 	Time time.Time
 	// Type of event that occurred
 	Type Type
+	// Health status of the current container
+	HealthStatus string `json:"health_status,omitempty"`
 
 	Details
 }
@@ -141,6 +143,8 @@ const (
 	Exited Status = "died"
 	// Export ...
 	Export Status = "export"
+	// HealthStatus ...
+	HealthStatus Status = "health_status"
 	// History ...
 	History Status = "history"
 	// Import ...
diff --git a/libpod/events/events.go b/libpod/events/events.go
index a30e0f1ca..a8001ab95 100644
--- a/libpod/events/events.go
+++ b/libpod/events/events.go
@@ -76,7 +76,7 @@ func (e *Event) ToHumanReadable(truncate bool) string {
 	}
 	switch e.Type {
 	case Container, Pod:
-		humanFormat = fmt.Sprintf("%s %s %s %s (image=%s, name=%s", e.Time, e.Type, e.Status, id, e.Image, e.Name)
+		humanFormat = fmt.Sprintf("%s %s %s %s (image=%s, name=%s, health_status=%s", e.Time, e.Type, e.Status, id, e.Image, e.Name, e.HealthStatus)
 		// check if the container has labels and add it to the output
 		if len(e.Attributes) > 0 {
 			for k, v := range e.Attributes {
@@ -168,6 +168,8 @@ func StringToStatus(name string) (Status, error) {
 		return Exited, nil
 	case Export.String():
 		return Export, nil
+	case HealthStatus.String():
+		return HealthStatus, nil
 	case History.String():
 		return History, nil
 	case Import.String():
diff --git a/libpod/events/journal_linux.go b/libpod/events/journal_linux.go
index 866042a4c..5db903ea0 100644
--- a/libpod/events/journal_linux.go
+++ b/libpod/events/journal_linux.go
@@ -58,6 +58,7 @@ func (e EventJournalD) Write(ee Event) error {
 			}
 			m["PODMAN_LABELS"] = string(b)
 		}
+		m["PODMAN_HEALTH_STATUS"] = ee.HealthStatus
 	case Network:
 		m["PODMAN_ID"] = ee.ID
 		m["PODMAN_NETWORK_NAME"] = ee.Network
@@ -214,6 +215,7 @@ func newEventFromJournalEntry(entry *sdjournal.JournalEntry) (*Event, error) { /
 				newEvent.Details = Details{Attributes: labels}
 			}
 		}
+		newEvent.HealthStatus = entry.Fields["PODMAN_HEALTH_STATUS"]
 	case Network:
 		newEvent.ID = entry.Fields["PODMAN_ID"]
 		newEvent.Network = entry.Fields["PODMAN_NETWORK_NAME"]
diff --git a/libpod/healthcheck.go b/libpod/healthcheck.go
index 40af9aec3..40315813c 100644
--- a/libpod/healthcheck.go
+++ b/libpod/healthcheck.go
@@ -90,7 +90,7 @@ func (c *Container) runHealthCheck() (define.HealthCheckStatus, error) {
 	hcResult := define.HealthCheckSuccess
 	config := new(ExecConfig)
 	config.Command = newCommand
-	exitCode, hcErr := c.Exec(config, streams, nil)
+	exitCode, hcErr := c.exec(config, streams, nil, true)
 	if hcErr != nil {
 		errCause := errors.Cause(hcErr)
 		hcResult = define.HealthCheckFailure
@@ -232,18 +232,27 @@ func (c *Container) getHealthCheckLog() (define.HealthCheckResults, error) {
 
 // HealthCheckStatus returns the current state of a container with a healthcheck
 func (c *Container) HealthCheckStatus() (string, error) {
+	c.lock.Lock()
+	defer c.lock.Unlock()
+	return c.healthCheckStatus()
+}
+
+// Internal function to return the current state of a container with a healthcheck.
+// This function does not lock the container.
+func (c *Container) healthCheckStatus() (string, error) {
 	if !c.HasHealthCheck() {
 		return "", errors.Errorf("container %s has no defined healthcheck", c.ID())
 	}
-	c.lock.Lock()
-	defer c.lock.Unlock()
+
 	if err := c.syncContainer(); err != nil {
 		return "", err
 	}
+
 	results, err := c.getHealthCheckLog()
 	if err != nil {
 		return "", errors.Wrapf(err, "unable to get healthcheck log for %s", c.ID())
 	}
+
 	return results.Status, nil
 }
 
diff --git a/pkg/domain/entities/events.go b/pkg/domain/entities/events.go
index d8ba0f1d3..de218b285 100644
--- a/pkg/domain/entities/events.go
+++ b/pkg/domain/entities/events.go
@@ -14,6 +14,7 @@ type Event struct {
 	// TODO: it would be nice to have full control over the types at some
 	// point and fork such Docker types.
 	dockerEvents.Message
+	HealthStatus string
 }
 
 // ConvertToLibpodEvent converts an entities event to a libpod one.
@@ -44,6 +45,7 @@ func ConvertToLibpodEvent(e Event) *libpodEvents.Event {
 		Status:            status,
 		Time:              time.Unix(0, e.TimeNano),
 		Type:              t,
+		HealthStatus:      e.HealthStatus,
 		Details: libpodEvents.Details{
 			Attributes: details,
 		},
@@ -59,7 +61,7 @@ func ConvertToEntitiesEvent(e libpodEvents.Event) *Event {
 	attributes["image"] = e.Image
 	attributes["name"] = e.Name
 	attributes["containerExitCode"] = strconv.Itoa(e.ContainerExitCode)
-	return &Event{dockerEvents.Message{
+	message := dockerEvents.Message{
 		// Compatibility with clients that still look for deprecated API elements
 		Status: e.Status.String(),
 		ID:     e.ID,
@@ -73,5 +75,9 @@ func ConvertToEntitiesEvent(e libpodEvents.Event) *Event {
 		Scope:    "local",
 		Time:     e.Time.Unix(),
 		TimeNano: e.Time.UnixNano(),
-	}}
+	}
+	return &Event{
+		message,
+		e.HealthStatus,
+	}
 }
diff --git a/test/apiv2/27-containersEvents.at b/test/apiv2/27-containersEvents.at
index a86f2e353..e0a66e0ac 100644
--- a/test/apiv2/27-containersEvents.at
+++ b/test/apiv2/27-containersEvents.at
@@ -18,6 +18,10 @@ t GET "libpod/events?stream=false&since=$START"  200  \
   'select(.status | contains("died")).Action=died' \
   'select(.status | contains("died")).Actor.Attributes.containerExitCode=1'
 
+t GET "libpod/events?stream=false&since=$START" 200 \
+    'select(.status | contains("start")).Action=start' \
+    'select(.status | contains("start")).HealthStatus='\
+
 # compat api, uses status=die (#12643)
 t GET "events?stream=false&since=$START"  200  \
   'select(.status | contains("start")).Action=start' \
diff --git a/test/e2e/events_test.go b/test/e2e/events_test.go
index 725118ab0..528fa143d 100644
--- a/test/e2e/events_test.go
+++ b/test/e2e/events_test.go
@@ -216,4 +216,25 @@ var _ = Describe("Podman events", func() {
 		Expect(result.OutputToString()).To(ContainSubstring("create"))
 	})
 
+	It("podman events health_status generated", func() {
+		session := podmanTest.Podman([]string{"run", "--name", "test-hc", "-dt", "--health-cmd", "echo working", "busybox"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		for i := 0; i < 5; i++ {
+			hc := podmanTest.Podman([]string{"healthcheck", "run", "test-hc"})
+			hc.WaitWithDefaultTimeout()
+			exitCode := hc.ExitCode()
+			if exitCode == 0 || i == 4 {
+				break
+			}
+			time.Sleep(1 * time.Second)
+		}
+
+		result := podmanTest.Podman([]string{"events", "--stream=false", "--filter", "event=health_status"})
+		result.WaitWithDefaultTimeout()
+		Expect(result).Should(Exit(0))
+		Expect(len(result.OutputToStringArray())).To(BeNumerically(">=", 1), "Number of health_status events")
+	})
+
 })