add podman volume reload to sync volume plugins

Libpod requires that all volumes are stored in the libpod db. Because
volume plugins can be created outside of podman, it will not show all
available plugins. This podman volume reload command allows users to
sync the libpod db with their external volume plugins. All new volumes
from the plugin are also created in the libpod db and when a volume from
the db no longer exists it will be removed if possible.

There are some problems:
- naming conflicts, in this case we only use the first volume we found.
  This is not deterministic.
- race conditions, we have no control over the volume plugins. It is
  possible that the volumes changed while we run this command.

Fixes #14207

Signed-off-by: Paul Holzinger <pholzing@redhat.com>

5 files changed, 102 insertions, 12 deletions

diff --git a/libpod/define/volume_inspect.go b/libpod/define/volume_inspect.go
index fac179176..aaa23b4fc 100644
--- a/libpod/define/volume_inspect.go
+++ b/libpod/define/volume_inspect.go
@@ -57,3 +57,9 @@ type InspectVolumeData struct {
 	// UID/GID.
 	NeedsChown bool `json:"NeedsChown,omitempty"`
 }
+
+type VolumeReload struct {
+	Added   []string
+	Removed []string
+	Errors  []error
+}
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
index a9ae9d1db..459514d47 100644
--- a/libpod/runtime_ctr.go
+++ b/libpod/runtime_ctr.go
@@ -502,7 +502,7 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (_ *Contai
 				volOptions = append(volOptions, parsedOptions...)
 			}
 		}
-		newVol, err := r.newVolume(volOptions...)
+		newVol, err := r.newVolume(false, volOptions...)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error creating named volume %q", vol.Name)
 		}
@@ -805,7 +805,7 @@ func (r *Runtime) removeContainer(ctx context.Context, c *Container, force, remo
 			if !volume.Anonymous() {
 				continue
 			}
-			if err := runtime.removeVolume(ctx, volume, false, timeout); err != nil && errors.Cause(err) != define.ErrNoSuchVolume {
+			if err := runtime.removeVolume(ctx, volume, false, timeout, false); err != nil && errors.Cause(err) != define.ErrNoSuchVolume {
 				if errors.Cause(err) == define.ErrVolumeBeingUsed {
 					// Ignore error, since podman will report original error
 					volumesFrom, _ := c.volumesFrom()
@@ -963,7 +963,7 @@ func (r *Runtime) evictContainer(ctx context.Context, idOrName string, removeVol
 			if !volume.Anonymous() {
 				continue
 			}
-			if err := r.removeVolume(ctx, volume, false, timeout); err != nil && err != define.ErrNoSuchVolume && err != define.ErrVolumeBeingUsed {
+			if err := r.removeVolume(ctx, volume, false, timeout, false); err != nil && err != define.ErrNoSuchVolume && err != define.ErrVolumeBeingUsed {
 				logrus.Errorf("Cleaning up volume (%s): %v", v, err)
 			}
 		}
diff --git a/libpod/runtime_pod_linux.go b/libpod/runtime_pod_linux.go
index dcc3a044f..fc55a2bc2 100644
--- a/libpod/runtime_pod_linux.go
+++ b/libpod/runtime_pod_linux.go
@@ -301,7 +301,7 @@ func (r *Runtime) removePod(ctx context.Context, p *Pod, removeCtrs, force bool,
 		if !volume.Anonymous() {
 			continue
 		}
-		if err := r.removeVolume(ctx, volume, false, timeout); err != nil {
+		if err := r.removeVolume(ctx, volume, false, timeout, false); err != nil {
 			if errors.Cause(err) == define.ErrNoSuchVolume || errors.Cause(err) == define.ErrVolumeRemoved {
 				continue
 			}
diff --git a/libpod/runtime_volume.go b/libpod/runtime_volume.go
index 21bf8aefc..6872db21d 100644
--- a/libpod/runtime_volume.go
+++ b/libpod/runtime_volume.go
@@ -33,7 +33,7 @@ func (r *Runtime) RemoveVolume(ctx context.Context, v *Volume, force bool, timeo
 			return nil
 		}
 	}
-	return r.removeVolume(ctx, v, force, timeout)
+	return r.removeVolume(ctx, v, force, timeout, false)
 }
 
 // GetVolume retrieves a volume given its full name.
diff --git a/libpod/runtime_volume_linux.go b/libpod/runtime_volume_linux.go
index f8788e183..877f3a1fd 100644
--- a/libpod/runtime_volume_linux.go
+++ b/libpod/runtime_volume_linux.go
@@ -5,6 +5,7 @@ package libpod
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -25,11 +26,13 @@ func (r *Runtime) NewVolume(ctx context.Context, options ...VolumeCreateOption)
 	if !r.valid {
 		return nil, define.ErrRuntimeStopped
 	}
-	return r.newVolume(options...)
+	return r.newVolume(false, options...)
 }
 
-// newVolume creates a new empty volume
-func (r *Runtime) newVolume(options ...VolumeCreateOption) (_ *Volume, deferredErr error) {
+// newVolume creates a new empty volume with the given options.
+// The createPluginVolume can be set to true to make it not create the volume in the volume plugin,
+// this is required for the UpdateVolumePlugins() function. If you are not sure set this to false.
+func (r *Runtime) newVolume(noCreatePluginVolume bool, options ...VolumeCreateOption) (_ *Volume, deferredErr error) {
 	volume := newVolume(r)
 	for _, option := range options {
 		if err := option(volume); err != nil {
@@ -83,7 +86,7 @@ func (r *Runtime) newVolume(options ...VolumeCreateOption) (_ *Volume, deferredE
 
 	// Now we get conditional: we either need to make the volume in the
 	// volume plugin, or on disk if not using a plugin.
-	if volume.plugin != nil {
+	if volume.plugin != nil && !noCreatePluginVolume {
 		// We can't chown, or relabel, or similar the path the volume is
 		// using, because it's not managed by us.
 		// TODO: reevaluate this once we actually have volume plugins in
@@ -164,6 +167,85 @@ func (r *Runtime) newVolume(options ...VolumeCreateOption) (_ *Volume, deferredE
 	return volume, nil
 }
 
+// UpdateVolumePlugins reads all volumes from all configured volume plugins and
+// imports them into the libpod db. It also checks if existing libpod volumes
+// are removed in the plugin, in this case we try to remove it from libpod.
+// On errors we continue and try to do as much as possible. all errors are
+// returned as array in the returned struct.
+// This function has many race conditions, it is best effort but cannot guarantee
+// a perfect state since plugins can be modified from the outside at any time.
+func (r *Runtime) UpdateVolumePlugins(ctx context.Context) *define.VolumeReload {
+	var (
+		added            []string
+		removed          []string
+		errs             []error
+		allPluginVolumes = map[string]struct{}{}
+	)
+
+	for driverName, socket := range r.config.Engine.VolumePlugins {
+		driver, err := volplugin.GetVolumePlugin(driverName, socket)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		vols, err := driver.ListVolumes()
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to read volumes from plugin %q: %w", driverName, err))
+			continue
+		}
+		for _, vol := range vols {
+			allPluginVolumes[vol.Name] = struct{}{}
+			if _, err := r.newVolume(true, WithVolumeName(vol.Name), WithVolumeDriver(driverName)); err != nil {
+				// If the volume exists this is not an error, just ignore it and log. It is very likely
+				// that the volume from the plugin was already in our db.
+				if !errors.Is(err, define.ErrVolumeExists) {
+					errs = append(errs, err)
+					continue
+				}
+				logrus.Infof("Volume %q already exists: %v", vol.Name, err)
+				continue
+			}
+			added = append(added, vol.Name)
+		}
+	}
+
+	libpodVolumes, err := r.state.AllVolumes()
+	if err != nil {
+		errs = append(errs, fmt.Errorf("cannot delete dangling plugin volumes: failed to read libpod volumes: %w", err))
+	}
+	for _, vol := range libpodVolumes {
+		if vol.UsesVolumeDriver() {
+			if _, ok := allPluginVolumes[vol.Name()]; !ok {
+				// The volume is no longer in the plugin, lets remove it from the libpod db.
+				if err := r.removeVolume(ctx, vol, false, nil, true); err != nil {
+					if errors.Is(err, define.ErrVolumeBeingUsed) {
+						// Volume is still used by at least one container. This is very bad,
+						// the plugin no longer has this but we still need it.
+						errs = append(errs, fmt.Errorf("volume was removed from the plugin %q but containers still require it: %w", vol.config.Driver, err))
+						continue
+					}
+					if errors.Is(err, define.ErrNoSuchVolume) || errors.Is(err, define.ErrVolumeRemoved) || errors.Is(err, define.ErrMissingPlugin) {
+						// Volume was already removed, no problem just ignore it and continue.
+						continue
+					}
+
+					// some other error
+					errs = append(errs, err)
+					continue
+				}
+				// Volume was successfully removed
+				removed = append(removed, vol.Name())
+			}
+		}
+	}
+
+	return &define.VolumeReload{
+		Added:   added,
+		Removed: removed,
+		Errors:  errs,
+	}
+}
+
 // makeVolumeInPluginIfNotExist makes a volume in the given volume plugin if it
 // does not already exist.
 func makeVolumeInPluginIfNotExist(name string, options map[string]string, plugin *volplugin.VolumePlugin) error {
@@ -197,8 +279,10 @@ func makeVolumeInPluginIfNotExist(name string, options map[string]string, plugin
 	return nil
 }
 
-// removeVolume removes the specified volume from state as well tears down its mountpoint and storage
-func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool, timeout *uint) error {
+// removeVolume removes the specified volume from state as well tears down its mountpoint and storage.
+// ignoreVolumePlugin is used to only remove the volume from the db and not the plugin,
+// this is required when the volume was already removed from the plugin, i.e. in UpdateVolumePlugins().
+func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool, timeout *uint, ignoreVolumePlugin bool) error {
 	if !v.valid {
 		if ok, _ := r.state.HasVolume(v.Name()); !ok {
 			return nil
@@ -263,7 +347,7 @@ func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool, timeo
 	var removalErr error
 
 	// If we use a volume plugin, we need to remove from the plugin.
-	if v.UsesVolumeDriver() {
+	if v.UsesVolumeDriver() && !ignoreVolumePlugin {
 		canRemove := true
 
 		// Do we have a volume driver?