Add support for retry count with --restart flag

The on-failure restart option supports restarting only a given
number of times. To do this, we need one additional field in the
DB to track restart count (which conveniently fills a field in
Inspect we weren't populating), plus some plumbing logic.

Signed-off-by: Matthew Heon <matthew.heon@pm.me>

4 files changed, 37 insertions, 20 deletions

diff --git a/libpod/container.go b/libpod/container.go
index 4236f2456..769c4f69c 100644
--- a/libpod/container.go
+++ b/libpod/container.go
@@ -185,6 +185,10 @@ type ContainerState struct {
 	// RestartPolicyMatch indicates whether the conditions for restart
 	// policy have been met.
 	RestartPolicyMatch bool `json:"restartPolicyMatch,omitempty"`
+	// RestartCount is how many times the container was restarted by its
+	// restart policy. This is NOT incremented by normal container restarts
+	// (only by restart policy).
+	RestartCount uint `json:"restartCount,omitempty"`
 
 	// ExtensionStageHooks holds hooks which will be executed by libpod
 	// and not delegated to the OCI runtime.
diff --git a/libpod/container_api.go b/libpod/container_api.go
index 8b3dd4186..7132a82c2 100644
--- a/libpod/container_api.go
+++ b/libpod/container_api.go
@@ -57,11 +57,11 @@ func (c *Container) Init(ctx context.Context) (err error) {
 
 	if c.state.State == ContainerStateStopped {
 		// Reinitialize the container
-		return c.reinit(ctx)
+		return c.reinit(ctx, false)
 	}
 
 	// Initialize the container for the first time
-	return c.init(ctx)
+	return c.init(ctx, false)
 }
 
 // Start starts a container.
@@ -605,13 +605,13 @@ func (c *Container) Cleanup(ctx context.Context) error {
 	// restart the container.
 	// However, perform a full validation of restart policy first.
 	if c.state.RestartPolicyMatch {
-		// if restart policy is on-error and exit code is 0, we do
-		// nothing.
-		// TODO: if restart retries is set, handle that here.
-		if c.config.RestartRetries != 0 {
-			return errors.Wrapf(ErrNotImplemented, "restart retries not yet implemented")
+		if c.config.RestartPolicy == "on-failure" && c.state.ExitCode != 0 {
+			logrus.Debugf("Container %s restart policy trigger: on retry %d (of %d)",
+				c.ID(), c.state.RestartCount, c.config.RestartRetries)
 		}
-		if (c.config.RestartPolicy == "on-error" && c.state.ExitCode == 0) || c.config.RestartPolicy == "always" {
+		if (c.config.RestartPolicy == "on-failure" && c.state.ExitCode != 0 &&
+			(c.config.RestartRetries > 0 && c.state.RestartCount < c.config.RestartRetries)) ||
+			c.config.RestartPolicy == "always" {
 			// The container stopped. We need to restart it.
 			return c.handleRestartPolicy(ctx)
 		}
@@ -780,7 +780,7 @@ func (c *Container) Refresh(ctx context.Context) error {
 		if err := c.prepare(); err != nil {
 			return err
 		}
-		if err := c.init(ctx); err != nil {
+		if err := c.init(ctx, false); err != nil {
 			return err
 		}
 	}
diff --git a/libpod/container_inspect.go b/libpod/container_inspect.go
index aa3a07888..a7369bfdd 100644
--- a/libpod/container_inspect.go
+++ b/libpod/container_inspect.go
@@ -95,6 +95,7 @@ func (c *Container) getContainerInspectData(size bool, driverData *inspect.Data)
 		LogPath:         config.LogPath,
 		ConmonPidFile:   config.ConmonPidFile,
 		Name:            config.Name,
+		RestartCount:    int32(runtimeInfo.RestartCount),
 		Driver:          driverData.Name,
 		MountLabel:      config.MountLabel,
 		ProcessLabel:    config.ProcessLabel,
diff --git a/libpod/container_internal.go b/libpod/container_internal.go
index f72d86c84..b1b7a090f 100644
--- a/libpod/container_internal.go
+++ b/libpod/container_internal.go
@@ -221,6 +221,13 @@ func (c *Container) handleRestartPolicy(ctx context.Context) (err error) {
 		return err
 	}
 
+	// Increment restart count
+	c.state.RestartCount = c.state.RestartCount + 1
+	logrus.Debugf("Container %s now on retry %d", c.ID(), c.state.RestartCount)
+	if err := c.save(); err != nil {
+		return err
+	}
+
 	defer func() {
 		if err != nil {
 			if err2 := c.cleanup(ctx); err2 != nil {
@@ -234,13 +241,13 @@ func (c *Container) handleRestartPolicy(ctx context.Context) (err error) {
 
 	if c.state.State == ContainerStateStopped {
 		// Reinitialize the container if we need to
-		if err := c.reinit(ctx); err != nil {
+		if err := c.reinit(ctx, true); err != nil {
 			return err
 		}
 	} else if c.state.State == ContainerStateConfigured ||
 		c.state.State == ContainerStateExited {
 		// Initialize the container
-		if err := c.init(ctx); err != nil {
+		if err := c.init(ctx, true); err != nil {
 			return err
 		}
 	}
@@ -423,6 +430,7 @@ func resetState(state *ContainerState) error {
 	state.BindMounts = make(map[string]string)
 	state.StoppedByUser = false
 	state.RestartPolicyMatch = false
+	state.RestartCount = 0
 
 	return nil
 }
@@ -616,13 +624,13 @@ func (c *Container) prepareToStart(ctx context.Context, recursive bool) (err err
 
 	if c.state.State == ContainerStateStopped {
 		// Reinitialize the container if we need to
-		if err := c.reinit(ctx); err != nil {
+		if err := c.reinit(ctx, false); err != nil {
 			return err
 		}
 	} else if c.state.State == ContainerStateConfigured ||
 		c.state.State == ContainerStateExited {
 		// Or initialize it if necessary
-		if err := c.init(ctx); err != nil {
+		if err := c.init(ctx, false); err != nil {
 			return err
 		}
 	}
@@ -810,7 +818,7 @@ func (c *Container) completeNetworkSetup() error {
 }
 
 // Initialize a container, creating it in the runtime
-func (c *Container) init(ctx context.Context) error {
+func (c *Container) init(ctx context.Context, retainRetries bool) error {
 	span, _ := opentracing.StartSpanFromContext(ctx, "init")
 	span.SetTag("struct", "container")
 	defer span.Finish()
@@ -839,6 +847,10 @@ func (c *Container) init(ctx context.Context) error {
 	c.state.StoppedByUser = false
 	c.state.RestartPolicyMatch = false
 
+	if !retainRetries {
+		c.state.RestartCount = 0
+	}
+
 	if err := c.save(); err != nil {
 		return err
 	}
@@ -900,7 +912,7 @@ func (c *Container) cleanupRuntime(ctx context.Context) error {
 // Should only be done on ContainerStateStopped containers.
 // Not necessary for ContainerStateExited - the container has already been
 // removed from the runtime, so init() can proceed freely.
-func (c *Container) reinit(ctx context.Context) error {
+func (c *Container) reinit(ctx context.Context, retainRetries bool) error {
 	span, _ := opentracing.StartSpanFromContext(ctx, "reinit")
 	span.SetTag("struct", "container")
 	defer span.Finish()
@@ -912,7 +924,7 @@ func (c *Container) reinit(ctx context.Context) error {
 	}
 
 	// Initialize the container again
-	return c.init(ctx)
+	return c.init(ctx, retainRetries)
 }
 
 // Initialize (if necessary) and start a container
@@ -950,12 +962,12 @@ func (c *Container) initAndStart(ctx context.Context) (err error) {
 	if c.state.State == ContainerStateStopped {
 		logrus.Debugf("Recreating container %s in OCI runtime", c.ID())
 
-		if err := c.reinit(ctx); err != nil {
+		if err := c.reinit(ctx, false); err != nil {
 			return err
 		}
 	} else if c.state.State == ContainerStateConfigured ||
 		c.state.State == ContainerStateExited {
-		if err := c.init(ctx); err != nil {
+		if err := c.init(ctx, false); err != nil {
 			return err
 		}
 	}
@@ -1058,13 +1070,13 @@ func (c *Container) restartWithTimeout(ctx context.Context, timeout uint) (err e
 
 	if c.state.State == ContainerStateStopped {
 		// Reinitialize the container if we need to
-		if err := c.reinit(ctx); err != nil {
+		if err := c.reinit(ctx, false); err != nil {
 			return err
 		}
 	} else if c.state.State == ContainerStateConfigured ||
 		c.state.State == ContainerStateExited {
 		// Initialize the container
-		if err := c.init(ctx); err != nil {
+		if err := c.init(ctx, false); err != nil {
 			return err
 		}
 	}