add podman volume reload to sync volume plugins

Libpod requires that all volumes are stored in the libpod db. Because
volume plugins can be created outside of podman, it will not show all
available plugins. This podman volume reload command allows users to
sync the libpod db with their external volume plugins. All new volumes
from the plugin are also created in the libpod db and when a volume from
the db no longer exists it will be removed if possible.

There are some problems:
- naming conflicts, in this case we only use the first volume we found.
  This is not deterministic.
- race conditions, we have no control over the volume plugins. It is
  possible that the volumes changed while we run this command.

Fixes #14207

Signed-off-by: Paul Holzinger <pholzing@redhat.com>

14 files changed, 268 insertions, 13 deletions

diff --git a/cmd/podman/volumes/reload.go b/cmd/podman/volumes/reload.go
new file mode 100644
index 000000000..d0d76fb88
--- /dev/null
+++ b/cmd/podman/volumes/reload.go
@@ -0,0 +1,52 @@
+package volumes
+
+import (
+	"fmt"
+
+	"github.com/containers/common/pkg/completion"
+	"github.com/containers/podman/v4/cmd/podman/registry"
+	"github.com/containers/podman/v4/cmd/podman/utils"
+	"github.com/containers/podman/v4/cmd/podman/validate"
+	"github.com/spf13/cobra"
+)
+
+var (
+	reloadDescription = `Check all configured volume plugins and update the libpod database with all available volumes.
+
+  Existing volumes are also removed from the database when they are no longer present in the plugin.`
+	reloadCommand = &cobra.Command{
+		Use:               "reload",
+		Args:              validate.NoArgs,
+		Short:             "reload all volumes from volume plugins",
+		Long:              reloadDescription,
+		RunE:              reload,
+		ValidArgsFunction: completion.AutocompleteNone,
+	}
+)
+
+func init() {
+	registry.Commands = append(registry.Commands, registry.CliCommand{
+		Command: reloadCommand,
+		Parent:  volumeCmd,
+	})
+}
+
+func reload(cmd *cobra.Command, args []string) error {
+	report, err := registry.ContainerEngine().VolumeReload(registry.Context())
+	if err != nil {
+		return err
+	}
+	printReload("Added", report.Added)
+	printReload("Removed", report.Removed)
+	errs := (utils.OutputErrors)(report.Errors)
+	return errs.PrintErrors()
+}
+
+func printReload(typ string, values []string) {
+	if len(values) > 0 {
+		fmt.Println(typ + ":")
+		for _, name := range values {
+			fmt.Println(name)
+		}
+	}
+}
diff --git a/docs/source/markdown/podman-volume-reload.1.md b/docs/source/markdown/podman-volume-reload.1.md
new file mode 100644
index 000000000..5b9e9b9ac
--- /dev/null
+++ b/docs/source/markdown/podman-volume-reload.1.md
@@ -0,0 +1,29 @@
+% podman-volume-reload(1)
+
+## NAME
+podman\-volume\-reload - Reload all volumes from volumes plugins
+
+## SYNOPSIS
+**podman volume reload**
+
+## DESCRIPTION
+
+**podman volume reload** checks all configured volume plugins and updates the libpod database with all available volumes.
+Existing volumes are also removed from the database when they are no longer present in the plugin.
+
+This command it is best effort and cannot guarantee a perfect state because plugins can be modified from the outside at any time.
+
+Note: This command is not supported with podman-remote.
+
+## EXAMPLES
+
+```
+$ podman volume reload
+Added:
+vol6
+Removed:
+t3
+```
+
+## SEE ALSO
+**[podman(1)](podman.1.md)**, **[podman-volume(1)](podman-volume.1.md)**
diff --git a/docs/source/markdown/podman-volume.1.md b/docs/source/markdown/podman-volume.1.md
index 476d58591..a437590b3 100644
--- a/docs/source/markdown/podman-volume.1.md
+++ b/docs/source/markdown/podman-volume.1.md
@@ -21,6 +21,7 @@ podman volume is a set of subcommands that manage volumes.
 | ls      | [podman-volume-ls(1)](podman-volume-ls.1.md)           | List all the available volumes.                                                |
 | mount   | [podman-volume-mount(1)](podman-volume-mount.1.md)     | Mount a volume filesystem.                                                     |
 | prune   | [podman-volume-prune(1)](podman-volume-prune.1.md)     | Remove all unused volumes.                                                     |
+| reload  | [podman-volume-reload(1)](podman-volume-reload.1.md)   | Reload all volumes from volumes plugins.                                       |
 | rm      | [podman-volume-rm(1)](podman-volume-rm.1.md)           | Remove one or more volumes.                                                    |
 | unmount | [podman-volume-unmount(1)](podman-volume-unmount.1.md) | Unmount a volume.                                                     |
 
diff --git a/libpod/define/volume_inspect.go b/libpod/define/volume_inspect.go
index fac179176..aaa23b4fc 100644
--- a/libpod/define/volume_inspect.go
+++ b/libpod/define/volume_inspect.go
@@ -57,3 +57,9 @@ type InspectVolumeData struct {
 	// UID/GID.
 	NeedsChown bool `json:"NeedsChown,omitempty"`
 }
+
+type VolumeReload struct {
+	Added   []string
+	Removed []string
+	Errors  []error
+}
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
index a9ae9d1db..459514d47 100644
--- a/libpod/runtime_ctr.go
+++ b/libpod/runtime_ctr.go
@@ -502,7 +502,7 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (_ *Contai
 				volOptions = append(volOptions, parsedOptions...)
 			}
 		}
-		newVol, err := r.newVolume(volOptions...)
+		newVol, err := r.newVolume(false, volOptions...)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error creating named volume %q", vol.Name)
 		}
@@ -805,7 +805,7 @@ func (r *Runtime) removeContainer(ctx context.Context, c *Container, force, remo
 			if !volume.Anonymous() {
 				continue
 			}
-			if err := runtime.removeVolume(ctx, volume, false, timeout); err != nil && errors.Cause(err) != define.ErrNoSuchVolume {
+			if err := runtime.removeVolume(ctx, volume, false, timeout, false); err != nil && errors.Cause(err) != define.ErrNoSuchVolume {
 				if errors.Cause(err) == define.ErrVolumeBeingUsed {
 					// Ignore error, since podman will report original error
 					volumesFrom, _ := c.volumesFrom()
@@ -963,7 +963,7 @@ func (r *Runtime) evictContainer(ctx context.Context, idOrName string, removeVol
 			if !volume.Anonymous() {
 				continue
 			}
-			if err := r.removeVolume(ctx, volume, false, timeout); err != nil && err != define.ErrNoSuchVolume && err != define.ErrVolumeBeingUsed {
+			if err := r.removeVolume(ctx, volume, false, timeout, false); err != nil && err != define.ErrNoSuchVolume && err != define.ErrVolumeBeingUsed {
 				logrus.Errorf("Cleaning up volume (%s): %v", v, err)
 			}
 		}
diff --git a/libpod/runtime_pod_linux.go b/libpod/runtime_pod_linux.go
index dcc3a044f..fc55a2bc2 100644
--- a/libpod/runtime_pod_linux.go
+++ b/libpod/runtime_pod_linux.go
@@ -301,7 +301,7 @@ func (r *Runtime) removePod(ctx context.Context, p *Pod, removeCtrs, force bool,
 		if !volume.Anonymous() {
 			continue
 		}
-		if err := r.removeVolume(ctx, volume, false, timeout); err != nil {
+		if err := r.removeVolume(ctx, volume, false, timeout, false); err != nil {
 			if errors.Cause(err) == define.ErrNoSuchVolume || errors.Cause(err) == define.ErrVolumeRemoved {
 				continue
 			}
diff --git a/libpod/runtime_volume.go b/libpod/runtime_volume.go
index 21bf8aefc..6872db21d 100644
--- a/libpod/runtime_volume.go
+++ b/libpod/runtime_volume.go
@@ -33,7 +33,7 @@ func (r *Runtime) RemoveVolume(ctx context.Context, v *Volume, force bool, timeo
 			return nil
 		}
 	}
-	return r.removeVolume(ctx, v, force, timeout)
+	return r.removeVolume(ctx, v, force, timeout, false)
 }
 
 // GetVolume retrieves a volume given its full name.
diff --git a/libpod/runtime_volume_linux.go b/libpod/runtime_volume_linux.go
index f8788e183..877f3a1fd 100644
--- a/libpod/runtime_volume_linux.go
+++ b/libpod/runtime_volume_linux.go
@@ -5,6 +5,7 @@ package libpod
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -25,11 +26,13 @@ func (r *Runtime) NewVolume(ctx context.Context, options ...VolumeCreateOption)
 	if !r.valid {
 		return nil, define.ErrRuntimeStopped
 	}
-	return r.newVolume(options...)
+	return r.newVolume(false, options...)
 }
 
-// newVolume creates a new empty volume
-func (r *Runtime) newVolume(options ...VolumeCreateOption) (_ *Volume, deferredErr error) {
+// newVolume creates a new empty volume with the given options.
+// The createPluginVolume can be set to true to make it not create the volume in the volume plugin,
+// this is required for the UpdateVolumePlugins() function. If you are not sure set this to false.
+func (r *Runtime) newVolume(noCreatePluginVolume bool, options ...VolumeCreateOption) (_ *Volume, deferredErr error) {
 	volume := newVolume(r)
 	for _, option := range options {
 		if err := option(volume); err != nil {
@@ -83,7 +86,7 @@ func (r *Runtime) newVolume(options ...VolumeCreateOption) (_ *Volume, deferredE
 
 	// Now we get conditional: we either need to make the volume in the
 	// volume plugin, or on disk if not using a plugin.
-	if volume.plugin != nil {
+	if volume.plugin != nil && !noCreatePluginVolume {
 		// We can't chown, or relabel, or similar the path the volume is
 		// using, because it's not managed by us.
 		// TODO: reevaluate this once we actually have volume plugins in
@@ -164,6 +167,85 @@ func (r *Runtime) newVolume(options ...VolumeCreateOption) (_ *Volume, deferredE
 	return volume, nil
 }
 
+// UpdateVolumePlugins reads all volumes from all configured volume plugins and
+// imports them into the libpod db. It also checks if existing libpod volumes
+// are removed in the plugin, in this case we try to remove it from libpod.
+// On errors we continue and try to do as much as possible. all errors are
+// returned as array in the returned struct.
+// This function has many race conditions, it is best effort but cannot guarantee
+// a perfect state since plugins can be modified from the outside at any time.
+func (r *Runtime) UpdateVolumePlugins(ctx context.Context) *define.VolumeReload {
+	var (
+		added            []string
+		removed          []string
+		errs             []error
+		allPluginVolumes = map[string]struct{}{}
+	)
+
+	for driverName, socket := range r.config.Engine.VolumePlugins {
+		driver, err := volplugin.GetVolumePlugin(driverName, socket)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		vols, err := driver.ListVolumes()
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to read volumes from plugin %q: %w", driverName, err))
+			continue
+		}
+		for _, vol := range vols {
+			allPluginVolumes[vol.Name] = struct{}{}
+			if _, err := r.newVolume(true, WithVolumeName(vol.Name), WithVolumeDriver(driverName)); err != nil {
+				// If the volume exists this is not an error, just ignore it and log. It is very likely
+				// that the volume from the plugin was already in our db.
+				if !errors.Is(err, define.ErrVolumeExists) {
+					errs = append(errs, err)
+					continue
+				}
+				logrus.Infof("Volume %q already exists: %v", vol.Name, err)
+				continue
+			}
+			added = append(added, vol.Name)
+		}
+	}
+
+	libpodVolumes, err := r.state.AllVolumes()
+	if err != nil {
+		errs = append(errs, fmt.Errorf("cannot delete dangling plugin volumes: failed to read libpod volumes: %w", err))
+	}
+	for _, vol := range libpodVolumes {
+		if vol.UsesVolumeDriver() {
+			if _, ok := allPluginVolumes[vol.Name()]; !ok {
+				// The volume is no longer in the plugin, lets remove it from the libpod db.
+				if err := r.removeVolume(ctx, vol, false, nil, true); err != nil {
+					if errors.Is(err, define.ErrVolumeBeingUsed) {
+						// Volume is still used by at least one container. This is very bad,
+						// the plugin no longer has this but we still need it.
+						errs = append(errs, fmt.Errorf("volume was removed from the plugin %q but containers still require it: %w", vol.config.Driver, err))
+						continue
+					}
+					if errors.Is(err, define.ErrNoSuchVolume) || errors.Is(err, define.ErrVolumeRemoved) || errors.Is(err, define.ErrMissingPlugin) {
+						// Volume was already removed, no problem just ignore it and continue.
+						continue
+					}
+
+					// some other error
+					errs = append(errs, err)
+					continue
+				}
+				// Volume was successfully removed
+				removed = append(removed, vol.Name())
+			}
+		}
+	}
+
+	return &define.VolumeReload{
+		Added:   added,
+		Removed: removed,
+		Errors:  errs,
+	}
+}
+
 // makeVolumeInPluginIfNotExist makes a volume in the given volume plugin if it
 // does not already exist.
 func makeVolumeInPluginIfNotExist(name string, options map[string]string, plugin *volplugin.VolumePlugin) error {
@@ -197,8 +279,10 @@ func makeVolumeInPluginIfNotExist(name string, options map[string]string, plugin
 	return nil
 }
 
-// removeVolume removes the specified volume from state as well tears down its mountpoint and storage
-func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool, timeout *uint) error {
+// removeVolume removes the specified volume from state as well tears down its mountpoint and storage.
+// ignoreVolumePlugin is used to only remove the volume from the db and not the plugin,
+// this is required when the volume was already removed from the plugin, i.e. in UpdateVolumePlugins().
+func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool, timeout *uint, ignoreVolumePlugin bool) error {
 	if !v.valid {
 		if ok, _ := r.state.HasVolume(v.Name()); !ok {
 			return nil
@@ -263,7 +347,7 @@ func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool, timeo
 	var removalErr error
 
 	// If we use a volume plugin, we need to remove from the plugin.
-	if v.UsesVolumeDriver() {
+	if v.UsesVolumeDriver() && !ignoreVolumePlugin {
 		canRemove := true
 
 		// Do we have a volume driver?
diff --git a/pkg/domain/entities/engine_container.go b/pkg/domain/entities/engine_container.go
index df42876f6..e4eb808b4 100644
--- a/pkg/domain/entities/engine_container.go
+++ b/pkg/domain/entities/engine_container.go
@@ -104,4 +104,5 @@ type ContainerEngine interface {
 	VolumePrune(ctx context.Context, options VolumePruneOptions) ([]*reports.PruneReport, error)
 	VolumeRm(ctx context.Context, namesOrIds []string, opts VolumeRmOptions) ([]*VolumeRmReport, error)
 	VolumeUnmount(ctx context.Context, namesOrIds []string) ([]*VolumeUnmountReport, error)
+	VolumeReload(ctx context.Context) (*VolumeReloadReport, error)
 }
diff --git a/pkg/domain/entities/volumes.go b/pkg/domain/entities/volumes.go
index 556df16c1..9a06b2238 100644
--- a/pkg/domain/entities/volumes.go
+++ b/pkg/domain/entities/volumes.go
@@ -54,6 +54,11 @@ type VolumeListReport struct {
 	VolumeConfigResponse
 }
 
+// VolumeReloadReport describes the response from reload volume plugins
+type VolumeReloadReport struct {
+	define.VolumeReload
+}
+
 /*
  * Docker API compatibility types
  */
diff --git a/pkg/domain/infra/abi/volumes.go b/pkg/domain/infra/abi/volumes.go
index a9c53c140..1186d8e81 100644
--- a/pkg/domain/infra/abi/volumes.go
+++ b/pkg/domain/infra/abi/volumes.go
@@ -211,3 +211,8 @@ func (ic *ContainerEngine) VolumeUnmount(ctx context.Context, nameOrIDs []string
 
 	return reports, nil
 }
+
+func (ic *ContainerEngine) VolumeReload(ctx context.Context) (*entities.VolumeReloadReport, error) {
+	report := ic.Libpod.UpdateVolumePlugins(ctx)
+	return &entities.VolumeReloadReport{VolumeReload: *report}, nil
+}
diff --git a/pkg/domain/infra/tunnel/volumes.go b/pkg/domain/infra/tunnel/volumes.go
index 33e090148..6ec35e836 100644
--- a/pkg/domain/infra/tunnel/volumes.go
+++ b/pkg/domain/infra/tunnel/volumes.go
@@ -108,3 +108,7 @@ func (ic *ContainerEngine) VolumeMount(ctx context.Context, nameOrIDs []string)
 func (ic *ContainerEngine) VolumeUnmount(ctx context.Context, nameOrIDs []string) ([]*entities.VolumeUnmountReport, error) {
 	return nil, errors.New("unmounting volumes is not supported for remote clients")
 }
+
+func (ic *ContainerEngine) VolumeReload(ctx context.Context) (*entities.VolumeReloadReport, error) {
+	return nil, errors.New("volume reload is not supported for remote clients")
+}
diff --git a/test/e2e/config.go b/test/e2e/config.go
index 2ca8e2a15..fbcc9dfff 100644
--- a/test/e2e/config.go
+++ b/test/e2e/config.go
@@ -15,7 +15,7 @@ var (
 	healthcheck       = "quay.io/libpod/alpine_healthcheck:latest"
 	ImageCacheDir     = "/tmp/podman/imagecachedir"
 	fedoraToolbox     = "registry.fedoraproject.org/fedora-toolbox:36"
-	volumeTest        = "quay.io/libpod/volume-plugin-test-img:latest"
+	volumeTest        = "quay.io/libpod/volume-plugin-test-img:20220623"
 
 	// This image has seccomp profiles that blocks all syscalls.
 	// The intention behind blocking all syscalls is to prevent
diff --git a/test/e2e/volume_plugin_test.go b/test/e2e/volume_plugin_test.go
index 4700afdb5..b585f8dd8 100644
--- a/test/e2e/volume_plugin_test.go
+++ b/test/e2e/volume_plugin_test.go
@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 
 	. "github.com/containers/podman/v4/test/utils"
+	"github.com/containers/storage/pkg/stringid"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gexec"
@@ -188,4 +189,71 @@ var _ = Describe("Podman volume plugins", func() {
 		rmAll.WaitWithDefaultTimeout()
 		Expect(rmAll).Should(Exit(0))
 	})
+
+	It("podman volume reload", func() {
+		podmanTest.AddImageToRWStore(volumeTest)
+
+		confFile := filepath.Join(podmanTest.TempDir, "containers.conf")
+		err := os.WriteFile(confFile, []byte(`[engine]
+[engine.volume_plugins]
+testvol5 = "/run/docker/plugins/testvol5.sock"`), 0o644)
+		Expect(err).ToNot(HaveOccurred())
+		os.Setenv("CONTAINERS_CONF", confFile)
+
+		pluginStatePath := filepath.Join(podmanTest.TempDir, "volumes")
+		err = os.Mkdir(pluginStatePath, 0755)
+		Expect(err).ToNot(HaveOccurred())
+
+		// Keep this distinct within tests to avoid multiple tests using the same plugin.
+		pluginName := "testvol5"
+		ctrName := "pluginCtr"
+		plugin := podmanTest.Podman([]string{"run", "--name", ctrName, "--security-opt", "label=disable", "-v", "/run/docker/plugins:/run/docker/plugins",
+			"-v", fmt.Sprintf("%v:%v", pluginStatePath, pluginStatePath), "-d", volumeTest, "--sock-name", pluginName, "--path", pluginStatePath})
+		plugin.WaitWithDefaultTimeout()
+		Expect(plugin).Should(Exit(0))
+
+		localvol := "local-" + stringid.GenerateNonCryptoID()
+		// create local volume
+		session := podmanTest.Podman([]string{"volume", "create", localvol})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(Exit(0))
+
+		vol1 := "vol1-" + stringid.GenerateNonCryptoID()
+		session = podmanTest.Podman([]string{"volume", "create", "--driver", pluginName, vol1})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(Exit(0))
+
+		// now create volume in plugin without podman
+		vol2 := "vol2-" + stringid.GenerateNonCryptoID()
+		plugin = podmanTest.Podman([]string{"exec", ctrName, "/usr/local/bin/testvol", "--sock-name", pluginName, "create", vol2})
+		plugin.WaitWithDefaultTimeout()
+		Expect(plugin).Should(Exit(0))
+
+		session = podmanTest.Podman([]string{"volume", "ls", "-q"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(Exit(0))
+		Expect(session.OutputToStringArray()).To(ContainElements(localvol, vol1))
+		Expect(session.ErrorToString()).To(Equal("")) // make sure no errors are shown
+
+		plugin = podmanTest.Podman([]string{"exec", ctrName, "/usr/local/bin/testvol", "--sock-name", pluginName, "remove", vol1})
+		plugin.WaitWithDefaultTimeout()
+		Expect(plugin).Should(Exit(0))
+
+		// now reload volumes from plugins
+		session = podmanTest.Podman([]string{"volume", "reload"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(Exit(0))
+		Expect(string(session.Out.Contents())).To(Equal(fmt.Sprintf(`Added:
+%s
+Removed:
+%s
+`, vol2, vol1)))
+		Expect(session.ErrorToString()).To(Equal("")) // make sure no errors are shown
+
+		session = podmanTest.Podman([]string{"volume", "ls", "-q"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(Exit(0))
+		Expect(session.OutputToStringArray()).To(ContainElements(localvol, vol2))
+		Expect(session.ErrorToString()).To(Equal("")) // make no errors are shown
+	})
 })