libpod: Execute poststop hooks locally

Instead of delegating to the runtime, since some runtimes do not seem
to handle these reliably [1].

[1]: https://github.com/projectatomic/libpod/issues/730#issuecomment-392959938

Signed-off-by: W. Trevor King <wking@tremily.us>

Closes: #864
Approved by: rhatdan

11 files changed, 177 insertions, 29 deletions

diff --git a/cmd/podman/rm.go b/cmd/podman/rm.go
index d6af6db07..9fc65cb4d 100644
--- a/cmd/podman/rm.go
+++ b/cmd/podman/rm.go
@@ -37,6 +37,7 @@ var (
 
 // saveCmd saves the image to either docker-archive or oci
 func rmCmd(c *cli.Context) error {
+	ctx := getContext()
 	if err := validateFlags(c, rmFlags); err != nil {
 		return err
 	}
@@ -81,7 +82,7 @@ func rmCmd(c *cli.Context) error {
 		}
 	}
 	for _, container := range delContainers {
-		err = runtime.RemoveContainer(container, c.Bool("force"))
+		err = runtime.RemoveContainer(ctx, container, c.Bool("force"))
 		if err != nil {
 			if lastError != nil {
 				fmt.Fprintln(os.Stderr, lastError)
diff --git a/cmd/podman/rmi.go b/cmd/podman/rmi.go
index dcf46a8e5..97c5b38cf 100644
--- a/cmd/podman/rmi.go
+++ b/cmd/podman/rmi.go
@@ -35,6 +35,7 @@ var (
 )
 
 func rmiCmd(c *cli.Context) error {
+	ctx := getContext()
 	if err := validateFlags(c, rmiFlags); err != nil {
 		return err
 	}
@@ -76,7 +77,7 @@ func rmiCmd(c *cli.Context) error {
 		return errors.Errorf("no valid images to delete")
 	}
 	for _, img := range imagesToDelete {
-		msg, err := runtime.RemoveImage(img, c.Bool("force"))
+		msg, err := runtime.RemoveImage(ctx, img, c.Bool("force"))
 		if err != nil {
 			if errors.Cause(err) == storage.ErrImageUsedByContainer {
 				fmt.Printf("A container associated with containers/storage, i.e. via Buildah, CRI-O, etc., may be associated with this image: %-12.12s\n", img.ID())
diff --git a/cmd/podman/run.go b/cmd/podman/run.go
index f12a241fa..d6755df5e 100644
--- a/cmd/podman/run.go
+++ b/cmd/podman/run.go
@@ -225,7 +225,7 @@ func runCmd(c *cli.Context) error {
 	}
 
 	if createConfig.Rm {
-		return runtime.RemoveContainer(ctr, true)
+		return runtime.RemoveContainer(ctx, ctr, true)
 	}
 
 	if err := ctr.Cleanup(); err != nil {
diff --git a/libpod/container.go b/libpod/container.go
index 1e3b923c0..85b6fa3dd 100644
--- a/libpod/container.go
+++ b/libpod/container.go
@@ -166,6 +166,10 @@ type containerState struct {
 	// UserNSRoot is the directory used as root for the container when using
 	// user namespaces.
 	UserNSRoot string `json:"userNSRoot,omitempty"`
+
+	// ExtensionStageHooks holds hooks which will be executed by libpod
+	// and not delegated to the OCI runtime.
+	ExtensionStageHooks map[string][]spec.Hook `json:"extensionStageHooks,omitempty"`
 }
 
 // ExecSession contains information on an active exec session
diff --git a/libpod/container_internal.go b/libpod/container_internal.go
index 258ee6ffd..84724e013 100644
--- a/libpod/container_internal.go
+++ b/libpod/container_internal.go
@@ -1,6 +1,7 @@
 package libpod
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -27,6 +28,7 @@ import (
 	crioAnnotations "github.com/projectatomic/libpod/pkg/annotations"
 	"github.com/projectatomic/libpod/pkg/chrootuser"
 	"github.com/projectatomic/libpod/pkg/hooks"
+	"github.com/projectatomic/libpod/pkg/hooks/exec"
 	"github.com/projectatomic/libpod/pkg/secrets"
 	"github.com/projectatomic/libpod/pkg/util"
 	"github.com/sirupsen/logrus"
@@ -548,10 +550,10 @@ func (c *Container) reinit(ctx context.Context) error {
 		return err
 	}
 
-	// Delete the container in the runtime
-	if err := c.runtime.ociRuntime.deleteContainer(c); err != nil {
-		return errors.Wrapf(err, "error removing container %s from runtime", c.ID())
+	if err := c.delete(ctx); err != nil {
+		return err
 	}
+
 	// Our state is now Configured, as we've removed ourself from
 	// the runtime
 	// Set and save now to make sure that, if the init() below fails
@@ -850,6 +852,62 @@ func (c *Container) cleanup() error {
 	return lastError
 }
 
+// delete deletes the container and runs any configured poststop
+// hooks.
+func (c *Container) delete(ctx context.Context) (err error) {
+	if err := c.runtime.ociRuntime.deleteContainer(c); err != nil {
+		return errors.Wrapf(err, "error removing container %s from runtime", c.ID())
+	}
+
+	if err := c.postDeleteHooks(ctx); err != nil {
+		return errors.Wrapf(err, "container %s poststop hooks", c.ID())
+	}
+
+	return nil
+}
+
+// postDeleteHooks runs the poststop hooks (if any) as specified by
+// the OCI Runtime Specification (which requires them to run
+// post-delete, despite the stage name).
+func (c *Container) postDeleteHooks(ctx context.Context) (err error) {
+	if c.state.ExtensionStageHooks != nil {
+		extensionHooks, ok := c.state.ExtensionStageHooks["poststop"]
+		if ok {
+			state, err := json.Marshal(spec.State{
+				Version:     spec.Version,
+				ID:          c.ID(),
+				Status:      "stopped",
+				Bundle:      c.bundlePath(),
+				Annotations: c.config.Spec.Annotations,
+			})
+			if err != nil {
+				return err
+			}
+			for i, hook := range extensionHooks {
+				logrus.Debugf("container %s: invoke poststop hook %d", c.ID(), i)
+				var stderr, stdout bytes.Buffer
+				hookErr, err := exec.Run(ctx, &hook, state, &stdout, &stderr, exec.DefaultPostKillTimeout)
+				if err != nil {
+					logrus.Warnf("container %s: poststop hook %d: %v", c.ID(), i, err)
+					if hookErr != err {
+						logrus.Debugf("container %s: poststop hook %d (hook error): %v", c.ID(), i, hookErr)
+					}
+					stdoutString := stdout.String()
+					if stdoutString != "" {
+						logrus.Debugf("container %s: poststop hook %d: stdout:\n%s", c.ID(), i, stdoutString)
+					}
+					stderrString := stderr.String()
+					if stderrString != "" {
+						logrus.Debugf("container %s: poststop hook %d: stderr:\n%s", c.ID(), i, stderrString)
+					}
+				}
+			}
+		}
+	}
+
+	return nil
+}
+
 // Make standard bind mounts to include in the container
 func (c *Container) makeBindMounts() error {
 	if err := os.Chown(c.state.RunDir, c.RootUID(), c.RootGID()); err != nil {
@@ -1092,7 +1150,8 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 		}
 	}
 
-	if err := c.setupOCIHooks(ctx, &g); err != nil {
+	var err error
+	if c.state.ExtensionStageHooks, err = c.setupOCIHooks(ctx, &g); err != nil {
 		return nil, errors.Wrapf(err, "error setting up OCI Hooks")
 	}
 	// Bind builtin image volumes
@@ -1313,9 +1372,9 @@ func (c *Container) saveSpec(spec *spec.Spec) error {
 	return nil
 }
 
-func (c *Container) setupOCIHooks(ctx context.Context, g *generate.Generator) error {
+func (c *Container) setupOCIHooks(ctx context.Context, g *generate.Generator) (extensionStageHooks map[string][]spec.Hook, err error) {
 	if c.runtime.config.HooksDir == "" {
-		return nil
+		return nil, nil
 	}
 
 	var locale string
@@ -1341,19 +1400,18 @@ func (c *Container) setupOCIHooks(ctx context.Context, g *generate.Generator) er
 		logrus.Warnf("failed to parse language %q: %s", langString, err)
 		lang, err = language.Parse("und-u-va-posix")
 		if err != nil {
-			return err
+			return nil, err
 		}
 	}
 
-	manager, err := hooks.New(ctx, []string{c.runtime.config.HooksDir}, []string{}, lang)
+	manager, err := hooks.New(ctx, []string{c.runtime.config.HooksDir}, []string{"poststop"}, lang)
 	if err != nil {
 		if c.runtime.config.HooksDirNotExistFatal || !os.IsNotExist(err) {
-			return err
+			return nil, err
 		}
 		logrus.Warnf("failed to load hooks: {}", err)
-		return nil
+		return nil, nil
 	}
 
-	_, err = manager.Hooks(g.Spec(), c.Spec().Annotations, len(c.config.UserVolumes) > 0)
-	return err
+	return manager.Hooks(g.Spec(), c.Spec().Annotations, len(c.config.UserVolumes) > 0)
 }
diff --git a/libpod/container_internal_test.go b/libpod/container_internal_test.go
new file mode 100644
index 000000000..78891a672
--- /dev/null
+++ b/libpod/container_internal_test.go
@@ -0,0 +1,80 @@
+package libpod
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"runtime"
+	"testing"
+
+	rspec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/stretchr/testify/assert"
+)
+
+// hookPath is the path to an example hook executable.
+var hookPath string
+
+func TestPostDeleteHooks(t *testing.T) {
+	ctx := context.Background()
+	dir, err := ioutil.TempDir("", "libpod_test_")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer os.RemoveAll(dir)
+
+	statePath := filepath.Join(dir, "state")
+	copyPath := filepath.Join(dir, "copy")
+	c := Container{
+		config: &ContainerConfig{
+			ID: "123abc",
+			Spec: &rspec.Spec{
+				Annotations: map[string]string{
+					"a": "b",
+				},
+			},
+			StaticDir: dir, // not the bundle, but good enough for this test
+		},
+		state: &containerState{
+			ExtensionStageHooks: map[string][]rspec.Hook{
+				"poststop": {
+					rspec.Hook{
+						Path: hookPath,
+						Args: []string{"sh", "-c", fmt.Sprintf("cat >%s", statePath)},
+					},
+					rspec.Hook{
+						Path: "/does/not/exist",
+					},
+					rspec.Hook{
+						Path: hookPath,
+						Args: []string{"sh", "-c", fmt.Sprintf("cp %s %s", statePath, copyPath)},
+					},
+				},
+			},
+		},
+	}
+	err = c.postDeleteHooks(ctx)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	stateRegexp := "{\"ociVersion\":\"1\\.0\\.0\",\"id\":\"123abc\",\"status\":\"stopped\",\"bundle\":\"/tmp/libpod_test_[0-9]*\",\"annotations\":{\"a\":\"b\"}}"
+	for _, path := range []string{statePath, copyPath} {
+		t.Run(path, func(t *testing.T) {
+			content, err := ioutil.ReadFile(path)
+			if err != nil {
+				t.Fatal(err)
+			}
+			assert.Regexp(t, stateRegexp, string(content))
+		})
+	}
+}
+
+func init() {
+	if runtime.GOOS != "windows" {
+		hookPath = "/bin/sh"
+	} else {
+		panic("we need a reliable executable path on Windows")
+	}
+}
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
index c6973ff2a..bde9db764 100644
--- a/libpod/runtime_ctr.go
+++ b/libpod/runtime_ctr.go
@@ -154,16 +154,16 @@ func (r *Runtime) NewContainer(ctx context.Context, rSpec *spec.Spec, options ..
 // RemoveContainer removes the given container
 // If force is specified, the container will be stopped first
 // Otherwise, RemoveContainer will return an error if the container is running
-func (r *Runtime) RemoveContainer(c *Container, force bool) error {
+func (r *Runtime) RemoveContainer(ctx context.Context, c *Container, force bool) error {
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
-	return r.removeContainer(c, force)
+	return r.removeContainer(ctx, c, force)
 }
 
 // Internal function to remove a container
 // Locks the container, but does not lock the runtime
-func (r *Runtime) removeContainer(c *Container, force bool) error {
+func (r *Runtime) removeContainer(ctx context.Context, c *Container, force bool) error {
 	if !c.valid {
 		// Container probably already removed
 		// Or was never in the runtime to begin with
@@ -263,8 +263,9 @@ func (r *Runtime) removeContainer(c *Container, force bool) error {
 	// Only do this if we're not ContainerStateConfigured - if we are,
 	// we haven't been created in the runtime yet
 	if c.state.State != ContainerStateConfigured {
-		if err := r.ociRuntime.deleteContainer(c); err != nil {
-			return errors.Wrapf(err, "error removing container %s from OCI runtime", c.ID())
+		err = c.delete(ctx)
+		if err != nil {
+			return err
 		}
 	}
 
diff --git a/libpod/runtime_img.go b/libpod/runtime_img.go
index 2392c41d4..8b0c08cd3 100644
--- a/libpod/runtime_img.go
+++ b/libpod/runtime_img.go
@@ -75,7 +75,7 @@ type CopyOptions struct {
 
 // RemoveImage deletes an image from local storage
 // Images being used by running containers can only be removed if force=true
-func (r *Runtime) RemoveImage(image *image.Image, force bool) (string, error) {
+func (r *Runtime) RemoveImage(ctx context.Context, image *image.Image, force bool) (string, error) {
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
@@ -97,7 +97,7 @@ func (r *Runtime) RemoveImage(image *image.Image, force bool) (string, error) {
 	if len(imageCtrs) > 0 && len(image.Names()) <= 1 {
 		if force {
 			for _, ctr := range imageCtrs {
-				if err := r.removeContainer(ctr, true); err != nil {
+				if err := r.removeContainer(ctx, ctr, true); err != nil {
 					return "", errors.Wrapf(err, "error removing image %s: container %s using image could not be removed", image.ID(), ctr.ID())
 				}
 			}
diff --git a/libpod/runtime_pod.go b/libpod/runtime_pod.go
index 3872a4b67..067a110fa 100644
--- a/libpod/runtime_pod.go
+++ b/libpod/runtime_pod.go
@@ -1,6 +1,7 @@
 package libpod
 
 import (
+	"context"
 	"path"
 	"path/filepath"
 	"strings"
@@ -91,7 +92,7 @@ func (r *Runtime) NewPod(options ...PodCreateOption) (*Pod, error) {
 // If force is specified with removeCtrs, all containers will be stopped before
 // being removed
 // Otherwise, the pod will not be removed if any containers are running
-func (r *Runtime) RemovePod(p *Pod, removeCtrs, force bool) error {
+func (r *Runtime) RemovePod(ctx context.Context, p *Pod, removeCtrs, force bool) error {
 	r.lock.Lock()
 	defer r.lock.Unlock()
 
@@ -206,11 +207,10 @@ func (r *Runtime) RemovePod(p *Pod, removeCtrs, force bool) error {
 		// Delete the container from runtime (only if we are not
 		// ContainerStateConfigured)
 		if ctr.state.State != ContainerStateConfigured {
-			if err := r.ociRuntime.deleteContainer(ctr); err != nil {
-				return errors.Wrapf(err, "error removing container %s from runtime", ctr.ID())
+			if err := ctr.delete(ctx); err != nil {
+				return err
 			}
 		}
-
 	}
 
 	// Remove containers from the state
diff --git a/pkg/varlinkapi/containers.go b/pkg/varlinkapi/containers.go
index 2b84151a9..71e5c31d0 100644
--- a/pkg/varlinkapi/containers.go
+++ b/pkg/varlinkapi/containers.go
@@ -409,6 +409,7 @@ func (i *LibpodAPI) WaitContainer(call ioprojectatomicpodman.VarlinkCall, name s
 
 // RemoveContainer ...
 func (i *LibpodAPI) RemoveContainer(call ioprojectatomicpodman.VarlinkCall, name string, force bool) error {
+	ctx := getContext()
 	runtime, err := libpodruntime.GetRuntime(i.Cli)
 	if err != nil {
 		return call.ReplyRuntimeError(err.Error())
@@ -417,7 +418,7 @@ func (i *LibpodAPI) RemoveContainer(call ioprojectatomicpodman.VarlinkCall, name
 	if err != nil {
 		return call.ReplyContainerNotFound(name)
 	}
-	if err := runtime.RemoveContainer(ctr, force); err != nil {
+	if err := runtime.RemoveContainer(ctx, ctr, force); err != nil {
 		return call.ReplyErrorOccurred(err.Error())
 	}
 	return call.ReplyRemoveContainer(ctr.ID())
@@ -426,6 +427,7 @@ func (i *LibpodAPI) RemoveContainer(call ioprojectatomicpodman.VarlinkCall, name
 
 // DeleteStoppedContainers ...
 func (i *LibpodAPI) DeleteStoppedContainers(call ioprojectatomicpodman.VarlinkCall) error {
+	ctx := getContext()
 	var deletedContainers []string
 	runtime, err := libpodruntime.GetRuntime(i.Cli)
 	if err != nil {
@@ -441,7 +443,7 @@ func (i *LibpodAPI) DeleteStoppedContainers(call ioprojectatomicpodman.VarlinkCa
 			return call.ReplyErrorOccurred(err.Error())
 		}
 		if state != libpod.ContainerStateRunning {
-			if err := runtime.RemoveContainer(ctr, false); err != nil {
+			if err := runtime.RemoveContainer(ctx, ctr, false); err != nil {
 				return call.ReplyErrorOccurred(err.Error())
 			}
 			deletedContainers = append(deletedContainers, ctr.ID())
diff --git a/pkg/varlinkapi/images.go b/pkg/varlinkapi/images.go
index 551eb781c..d80c7043a 100644
--- a/pkg/varlinkapi/images.go
+++ b/pkg/varlinkapi/images.go
@@ -360,6 +360,7 @@ func (i *LibpodAPI) TagImage(call ioprojectatomicpodman.VarlinkCall, name, tag s
 // RemoveImage accepts a image name or ID as a string and force bool to determine if it should
 // remove the image even if being used by stopped containers
 func (i *LibpodAPI) RemoveImage(call ioprojectatomicpodman.VarlinkCall, name string, force bool) error {
+	ctx := getContext()
 	runtime, err := libpodruntime.GetRuntime(i.Cli)
 	if err != nil {
 		return call.ReplyRuntimeError(err.Error())
@@ -368,7 +369,7 @@ func (i *LibpodAPI) RemoveImage(call ioprojectatomicpodman.VarlinkCall, name str
 	if err != nil {
 		return call.ReplyImageNotFound(name)
 	}
-	imageID, err := runtime.RemoveImage(newImage, force)
+	imageID, err := runtime.RemoveImage(ctx, newImage, force)
 	if err != nil {
 		return call.ReplyErrorOccurred(err.Error())
 	}