Add --storage flag to 'podman rm' (local only)

This flag switches to removing containers directly from c/storage
and is mostly used to remove orphan containers.

It's a superior solution to our former one, which attempted
removal from storage under certain circumstances and could, under
some conditions, not trigger.

Also contains the beginning of support for storage in `ps` but
wiring that in is going to be a much bigger pain.

Fixes #3329.

Signed-off-by: Matthew Heon <matthew.heon@pm.me>

7 files changed, 146 insertions, 19 deletions

diff --git a/cmd/podman/cliconfig/config.go b/cmd/podman/cliconfig/config.go
index 545166d05..4a4c839cc 100644
--- a/cmd/podman/cliconfig/config.go
+++ b/cmd/podman/cliconfig/config.go
@@ -439,6 +439,7 @@ type RmValues struct {
 	All     bool
 	Force   bool
 	Latest  bool
+	Storage bool
 	Volumes bool
 }
 
diff --git a/cmd/podman/rm.go b/cmd/podman/rm.go
index 1bf56b782..2710a8194 100644
--- a/cmd/podman/rm.go
+++ b/cmd/podman/rm.go
@@ -42,7 +42,9 @@ func init() {
 	flags.BoolVarP(&rmCommand.All, "all", "a", false, "Remove all containers")
 	flags.BoolVarP(&rmCommand.Force, "force", "f", false, "Force removal of a running container.  The default is false")
 	flags.BoolVarP(&rmCommand.Latest, "latest", "l", false, "Act on the latest container podman is aware of")
+	flags.BoolVar(&rmCommand.Storage, "storage", false, "Remove container from storage library")
 	flags.BoolVarP(&rmCommand.Volumes, "volumes", "v", false, "Remove the volumes associated with the container")
+	markFlagHiddenForRemoteClient("storage", flags)
 	markFlagHiddenForRemoteClient("latest", flags)
 }
 
@@ -54,6 +56,13 @@ func rmCmd(c *cliconfig.RmValues) error {
 	}
 	defer runtime.Shutdown(false)
 
+	// Storage conflicts with --all/--latest/--volumes
+	if c.Storage {
+		if c.All || c.Latest || c.Volumes {
+			return errors.Errorf("--storage conflicts with --volumes, --all, and --latest")
+		}
+	}
+
 	ok, failures, err := runtime.RemoveContainers(getContext(), c)
 	if err != nil {
 		if errors.Cause(err) == libpod.ErrNoSuchCtr {
diff --git a/completions/bash/podman b/completions/bash/podman
index b049f309a..65c6308cc 100644
--- a/completions/bash/podman
+++ b/completions/bash/podman
@@ -2041,6 +2041,7 @@ _podman_rm() {
 	-h
 	--latest
 	-l
+	--storage
 	--volumes
 	-v
     "
diff --git a/docs/podman-rm.1.md b/docs/podman-rm.1.md
index c7ff23bdf..32a8c2943 100644
--- a/docs/podman-rm.1.md
+++ b/docs/podman-rm.1.md
@@ -30,6 +30,13 @@ to run containers such as CRI-O, the last started container could be from either
 
 The latest option is not supported on the remote client.
 
+**--storage**
+
+Remove the container from the storage library only.
+This is only possible with containers that are not present in libpod (cannot be seen by `podman ps`).
+It is used to remove containers from `podman build` and `buildah`, and orphan containers which were only partially removed by `podman rm`.
+The storage option conflicts with the **--all**, **--latest**, and **--volumes** options.
+
 **--volumes**, **-v**
 
 Remove the volumes associated with the container.
diff --git a/libpod/runtime_cstorage.go b/libpod/runtime_cstorage.go
new file mode 100644
index 000000000..569f63322
--- /dev/null
+++ b/libpod/runtime_cstorage.go
@@ -0,0 +1,118 @@
+package libpod
+
+import (
+	"github.com/containers/storage"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// StorageContainer represents a container present in c/storage but not in
+// libpod.
+type StorageContainer struct {
+	ID              string
+	Names           []string
+	PresentInLibpod bool
+}
+
+// ListStorageContainers lists all containers visible to c/storage.
+func (r *Runtime) ListStorageContainers() ([]*StorageContainer, error) {
+	r.lock.RLock()
+	defer r.lock.RUnlock()
+
+	finalCtrs := []*StorageContainer{}
+
+	ctrs, err := r.store.Containers()
+	if err != nil {
+		return nil, err
+	}
+
+	for _, ctr := range ctrs {
+		storageCtr := new(StorageContainer)
+		storageCtr.ID = ctr.ID
+		storageCtr.Names = ctr.Names
+
+		// Look up if container is in state
+		hasCtr, err := r.state.HasContainer(ctr.ID)
+		if err != nil {
+			return nil, errors.Wrapf(err, "error looking up container %s in state", ctr.ID)
+		}
+
+		storageCtr.PresentInLibpod = hasCtr
+
+		finalCtrs = append(finalCtrs, storageCtr)
+	}
+
+	return finalCtrs, nil
+}
+
+// RemoveStorageContainer removes a container from c/storage.
+// The container WILL NOT be removed if it exists in libpod.
+// Accepts ID or full name of container.
+// If force is set, the container will be unmounted first to ensure removal.
+func (r *Runtime) RemoveStorageContainer(idOrName string, force bool) error {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	targetID, err := r.store.Lookup(idOrName)
+	if err != nil {
+		if err == storage.ErrLayerUnknown {
+			return errors.Wrapf(ErrNoSuchCtr, "no container with ID or name %q found", idOrName)
+		}
+		return errors.Wrapf(err, "error looking up container %q", idOrName)
+	}
+
+	// Lookup returns an ID but it's not guaranteed to be a container ID.
+	// So we can still error here.
+	ctr, err := r.store.Container(targetID)
+	if err != nil {
+		if err == storage.ErrContainerUnknown {
+			return errors.Wrapf(ErrNoSuchCtr, "%q does not refer to a container", idOrName)
+		}
+		return errors.Wrapf(err, "error retrieving container %q", idOrName)
+	}
+
+	// Error out if the container exists in libpod
+	exists, err := r.state.HasContainer(ctr.ID)
+	if err != nil {
+		return err
+	}
+	if exists {
+		return errors.Wrapf(ErrCtrExists, "refusing to remove %q as it exists in libpod as container %s", idOrName, ctr.ID)
+	}
+
+	if !force {
+		timesMounted, err := r.store.Mounted(ctr.ID)
+		if err != nil {
+			if err == storage.ErrContainerUnknown {
+				// Container was removed from under us.
+				// It's gone, so don't bother erroring.
+				logrus.Warnf("Storage for container %s already removed", ctr.ID)
+				return nil
+			}
+			return errors.Wrapf(err, "error looking up container %q mounts", idOrName)
+		}
+		if timesMounted > 0 {
+			return errors.Wrapf(ErrCtrStateInvalid, "container %q is mounted and cannot be removed without using force", idOrName)
+		}
+	} else {
+		if _, err := r.store.Unmount(ctr.ID, true); err != nil {
+			if err == storage.ErrContainerUnknown {
+				// Container again gone, no error
+				logrus.Warnf("Storage for container %s already removed", ctr.ID)
+				return nil
+			}
+			return errors.Wrapf(err, "error unmounting container %q", idOrName)
+		}
+	}
+
+	if err := r.store.DeleteContainer(ctr.ID); err != nil {
+		if err == storage.ErrContainerUnknown {
+			// Container again gone, no error
+			logrus.Warnf("Storage for container %s already removed", ctr.ID)
+			return nil
+		}
+		return errors.Wrapf(err, "error removing storage for container %q", idOrName)
+	}
+
+	return nil
+}
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
index cf1f5701d..0871b83a7 100644
--- a/libpod/runtime_ctr.go
+++ b/libpod/runtime_ctr.go
@@ -9,9 +9,7 @@ import (
 	"time"
 
 	"github.com/containers/libpod/libpod/events"
-	"github.com/containers/libpod/libpod/image"
 	"github.com/containers/libpod/pkg/rootless"
-	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/stringid"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
@@ -614,16 +612,3 @@ func (r *Runtime) GetLatestContainer() (*Container, error) {
 	}
 	return ctrs[lastCreatedIndex], nil
 }
-
-// RemoveContainersFromStorage attempt to remove containers from storage that do not exist in libpod database
-func (r *Runtime) RemoveContainersFromStorage(ctrs []string) {
-	for _, i := range ctrs {
-		// if the container does not exist in database, attempt to remove it from storage
-		if _, err := r.LookupContainer(i); err != nil && errors.Cause(err) == image.ErrNoSuchCtr {
-			r.storageService.UnmountContainerImage(i, true)
-			if err := r.storageService.DeleteContainer(i); err != nil && errors.Cause(err) != storage.ErrContainerUnknown {
-				logrus.Errorf("Failed to remove container %q from storage: %s", i, err)
-			}
-		}
-	}
-}
diff --git a/pkg/adapter/containers.go b/pkg/adapter/containers.go
index 29297fbd5..40b1e6b43 100644
--- a/pkg/adapter/containers.go
+++ b/pkg/adapter/containers.go
@@ -190,12 +190,18 @@ func (r *LocalRuntime) RemoveContainers(ctx context.Context, cli *cliconfig.RmVa
 	}
 	logrus.Debugf("Setting maximum rm workers to %d", maxWorkers)
 
+	if cli.Storage {
+		for _, ctr := range cli.InputArgs {
+			if err := r.RemoveStorageContainer(ctr, cli.Force); err != nil {
+				failures[ctr] = err
+			}
+			ok = append(ok, ctr)
+		}
+		return ok, failures, nil
+	}
+
 	ctrs, err := shortcuts.GetContainersByContext(cli.All, cli.Latest, cli.InputArgs, r.Runtime)
 	if err != nil {
-		// Force may be used to remove containers no longer found in the database
-		if cli.Force && len(cli.InputArgs) > 0 && errors.Cause(err) == libpod.ErrNoSuchCtr {
-			r.RemoveContainersFromStorage(cli.InputArgs)
-		}
 		return ok, failures, err
 	}