Merge pull request #14713 from Luap99/volume-plugin

add podman volume reload to sync volume plugins

6 files changed, 111 insertions, 21 deletions

diff --git a/libpod/define/volume_inspect.go b/libpod/define/volume_inspect.go
index fac179176..aaa23b4fc 100644
--- a/libpod/define/volume_inspect.go
+++ b/libpod/define/volume_inspect.go
@@ -57,3 +57,9 @@ type InspectVolumeData struct {
 	// UID/GID.
 	NeedsChown bool `json:"NeedsChown,omitempty"`
 }
+
+type VolumeReload struct {
+	Added   []string
+	Removed []string
+	Errors  []error
+}
diff --git a/libpod/plugin/volume_api.go b/libpod/plugin/volume_api.go
index f997ccf22..2de7db32c 100644
--- a/libpod/plugin/volume_api.go
+++ b/libpod/plugin/volume_api.go
@@ -197,13 +197,13 @@ func (p *VolumePlugin) verifyReachable() error {
 
 // Send a request to the volume plugin for handling.
 // Callers *MUST* close the response when they are done.
-func (p *VolumePlugin) sendRequest(toJSON interface{}, hasBody bool, endpoint string) (*http.Response, error) {
+func (p *VolumePlugin) sendRequest(toJSON interface{}, endpoint string) (*http.Response, error) {
 	var (
 		reqJSON []byte
 		err     error
 	)
 
-	if hasBody {
+	if toJSON != nil {
 		reqJSON, err = json.Marshal(toJSON)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error marshalling request JSON for volume plugin %s endpoint %s", p.Name, endpoint)
@@ -274,7 +274,7 @@ func (p *VolumePlugin) CreateVolume(req *volume.CreateRequest) error {
 
 	logrus.Infof("Creating volume %s using plugin %s", req.Name, p.Name)
 
-	resp, err := p.sendRequest(req, true, createPath)
+	resp, err := p.sendRequest(req, createPath)
 	if err != nil {
 		return err
 	}
@@ -291,7 +291,7 @@ func (p *VolumePlugin) ListVolumes() ([]*volume.Volume, error) {
 
 	logrus.Infof("Listing volumes using plugin %s", p.Name)
 
-	resp, err := p.sendRequest(nil, false, listPath)
+	resp, err := p.sendRequest(nil, listPath)
 	if err != nil {
 		return nil, err
 	}
@@ -326,7 +326,7 @@ func (p *VolumePlugin) GetVolume(req *volume.GetRequest) (*volume.Volume, error)
 
 	logrus.Infof("Getting volume %s using plugin %s", req.Name, p.Name)
 
-	resp, err := p.sendRequest(req, true, getPath)
+	resp, err := p.sendRequest(req, getPath)
 	if err != nil {
 		return nil, err
 	}
@@ -361,7 +361,7 @@ func (p *VolumePlugin) RemoveVolume(req *volume.RemoveRequest) error {
 
 	logrus.Infof("Removing volume %s using plugin %s", req.Name, p.Name)
 
-	resp, err := p.sendRequest(req, true, removePath)
+	resp, err := p.sendRequest(req, removePath)
 	if err != nil {
 		return err
 	}
@@ -382,7 +382,7 @@ func (p *VolumePlugin) GetVolumePath(req *volume.PathRequest) (string, error) {
 
 	logrus.Infof("Getting volume %s path using plugin %s", req.Name, p.Name)
 
-	resp, err := p.sendRequest(req, true, hostVirtualPath)
+	resp, err := p.sendRequest(req, hostVirtualPath)
 	if err != nil {
 		return "", err
 	}
@@ -419,7 +419,7 @@ func (p *VolumePlugin) MountVolume(req *volume.MountRequest) (string, error) {
 
 	logrus.Infof("Mounting volume %s using plugin %s for container %s", req.Name, p.Name, req.ID)
 
-	resp, err := p.sendRequest(req, true, mountPath)
+	resp, err := p.sendRequest(req, mountPath)
 	if err != nil {
 		return "", err
 	}
@@ -455,7 +455,7 @@ func (p *VolumePlugin) UnmountVolume(req *volume.UnmountRequest) error {
 
 	logrus.Infof("Unmounting volume %s using plugin %s for container %s", req.Name, p.Name, req.ID)
 
-	resp, err := p.sendRequest(req, true, unmountPath)
+	resp, err := p.sendRequest(req, unmountPath)
 	if err != nil {
 		return err
 	}
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
index a9ae9d1db..459514d47 100644
--- a/libpod/runtime_ctr.go
+++ b/libpod/runtime_ctr.go
@@ -502,7 +502,7 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (_ *Contai
 				volOptions = append(volOptions, parsedOptions...)
 			}
 		}
-		newVol, err := r.newVolume(volOptions...)
+		newVol, err := r.newVolume(false, volOptions...)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error creating named volume %q", vol.Name)
 		}
@@ -805,7 +805,7 @@ func (r *Runtime) removeContainer(ctx context.Context, c *Container, force, remo
 			if !volume.Anonymous() {
 				continue
 			}
-			if err := runtime.removeVolume(ctx, volume, false, timeout); err != nil && errors.Cause(err) != define.ErrNoSuchVolume {
+			if err := runtime.removeVolume(ctx, volume, false, timeout, false); err != nil && errors.Cause(err) != define.ErrNoSuchVolume {
 				if errors.Cause(err) == define.ErrVolumeBeingUsed {
 					// Ignore error, since podman will report original error
 					volumesFrom, _ := c.volumesFrom()
@@ -963,7 +963,7 @@ func (r *Runtime) evictContainer(ctx context.Context, idOrName string, removeVol
 			if !volume.Anonymous() {
 				continue
 			}
-			if err := r.removeVolume(ctx, volume, false, timeout); err != nil && err != define.ErrNoSuchVolume && err != define.ErrVolumeBeingUsed {
+			if err := r.removeVolume(ctx, volume, false, timeout, false); err != nil && err != define.ErrNoSuchVolume && err != define.ErrVolumeBeingUsed {
 				logrus.Errorf("Cleaning up volume (%s): %v", v, err)
 			}
 		}
diff --git a/libpod/runtime_pod_linux.go b/libpod/runtime_pod_linux.go
index d75ac2971..00017ca21 100644
--- a/libpod/runtime_pod_linux.go
+++ b/libpod/runtime_pod_linux.go
@@ -315,7 +315,7 @@ func (r *Runtime) removePod(ctx context.Context, p *Pod, removeCtrs, force bool,
 		if !volume.Anonymous() {
 			continue
 		}
-		if err := r.removeVolume(ctx, volume, false, timeout); err != nil {
+		if err := r.removeVolume(ctx, volume, false, timeout, false); err != nil {
 			if errors.Cause(err) == define.ErrNoSuchVolume || errors.Cause(err) == define.ErrVolumeRemoved {
 				continue
 			}
diff --git a/libpod/runtime_volume.go b/libpod/runtime_volume.go
index 21bf8aefc..6872db21d 100644
--- a/libpod/runtime_volume.go
+++ b/libpod/runtime_volume.go
@@ -33,7 +33,7 @@ func (r *Runtime) RemoveVolume(ctx context.Context, v *Volume, force bool, timeo
 			return nil
 		}
 	}
-	return r.removeVolume(ctx, v, force, timeout)
+	return r.removeVolume(ctx, v, force, timeout, false)
 }
 
 // GetVolume retrieves a volume given its full name.
diff --git a/libpod/runtime_volume_linux.go b/libpod/runtime_volume_linux.go
index f8788e183..877f3a1fd 100644
--- a/libpod/runtime_volume_linux.go
+++ b/libpod/runtime_volume_linux.go
@@ -5,6 +5,7 @@ package libpod
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -25,11 +26,13 @@ func (r *Runtime) NewVolume(ctx context.Context, options ...VolumeCreateOption)
 	if !r.valid {
 		return nil, define.ErrRuntimeStopped
 	}
-	return r.newVolume(options...)
+	return r.newVolume(false, options...)
 }
 
-// newVolume creates a new empty volume
-func (r *Runtime) newVolume(options ...VolumeCreateOption) (_ *Volume, deferredErr error) {
+// newVolume creates a new empty volume with the given options.
+// The createPluginVolume can be set to true to make it not create the volume in the volume plugin,
+// this is required for the UpdateVolumePlugins() function. If you are not sure set this to false.
+func (r *Runtime) newVolume(noCreatePluginVolume bool, options ...VolumeCreateOption) (_ *Volume, deferredErr error) {
 	volume := newVolume(r)
 	for _, option := range options {
 		if err := option(volume); err != nil {
@@ -83,7 +86,7 @@ func (r *Runtime) newVolume(options ...VolumeCreateOption) (_ *Volume, deferredE
 
 	// Now we get conditional: we either need to make the volume in the
 	// volume plugin, or on disk if not using a plugin.
-	if volume.plugin != nil {
+	if volume.plugin != nil && !noCreatePluginVolume {
 		// We can't chown, or relabel, or similar the path the volume is
 		// using, because it's not managed by us.
 		// TODO: reevaluate this once we actually have volume plugins in
@@ -164,6 +167,85 @@ func (r *Runtime) newVolume(options ...VolumeCreateOption) (_ *Volume, deferredE
 	return volume, nil
 }
 
+// UpdateVolumePlugins reads all volumes from all configured volume plugins and
+// imports them into the libpod db. It also checks if existing libpod volumes
+// are removed in the plugin, in this case we try to remove it from libpod.
+// On errors we continue and try to do as much as possible. all errors are
+// returned as array in the returned struct.
+// This function has many race conditions, it is best effort but cannot guarantee
+// a perfect state since plugins can be modified from the outside at any time.
+func (r *Runtime) UpdateVolumePlugins(ctx context.Context) *define.VolumeReload {
+	var (
+		added            []string
+		removed          []string
+		errs             []error
+		allPluginVolumes = map[string]struct{}{}
+	)
+
+	for driverName, socket := range r.config.Engine.VolumePlugins {
+		driver, err := volplugin.GetVolumePlugin(driverName, socket)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		vols, err := driver.ListVolumes()
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to read volumes from plugin %q: %w", driverName, err))
+			continue
+		}
+		for _, vol := range vols {
+			allPluginVolumes[vol.Name] = struct{}{}
+			if _, err := r.newVolume(true, WithVolumeName(vol.Name), WithVolumeDriver(driverName)); err != nil {
+				// If the volume exists this is not an error, just ignore it and log. It is very likely
+				// that the volume from the plugin was already in our db.
+				if !errors.Is(err, define.ErrVolumeExists) {
+					errs = append(errs, err)
+					continue
+				}
+				logrus.Infof("Volume %q already exists: %v", vol.Name, err)
+				continue
+			}
+			added = append(added, vol.Name)
+		}
+	}
+
+	libpodVolumes, err := r.state.AllVolumes()
+	if err != nil {
+		errs = append(errs, fmt.Errorf("cannot delete dangling plugin volumes: failed to read libpod volumes: %w", err))
+	}
+	for _, vol := range libpodVolumes {
+		if vol.UsesVolumeDriver() {
+			if _, ok := allPluginVolumes[vol.Name()]; !ok {
+				// The volume is no longer in the plugin, lets remove it from the libpod db.
+				if err := r.removeVolume(ctx, vol, false, nil, true); err != nil {
+					if errors.Is(err, define.ErrVolumeBeingUsed) {
+						// Volume is still used by at least one container. This is very bad,
+						// the plugin no longer has this but we still need it.
+						errs = append(errs, fmt.Errorf("volume was removed from the plugin %q but containers still require it: %w", vol.config.Driver, err))
+						continue
+					}
+					if errors.Is(err, define.ErrNoSuchVolume) || errors.Is(err, define.ErrVolumeRemoved) || errors.Is(err, define.ErrMissingPlugin) {
+						// Volume was already removed, no problem just ignore it and continue.
+						continue
+					}
+
+					// some other error
+					errs = append(errs, err)
+					continue
+				}
+				// Volume was successfully removed
+				removed = append(removed, vol.Name())
+			}
+		}
+	}
+
+	return &define.VolumeReload{
+		Added:   added,
+		Removed: removed,
+		Errors:  errs,
+	}
+}
+
 // makeVolumeInPluginIfNotExist makes a volume in the given volume plugin if it
 // does not already exist.
 func makeVolumeInPluginIfNotExist(name string, options map[string]string, plugin *volplugin.VolumePlugin) error {
@@ -197,8 +279,10 @@ func makeVolumeInPluginIfNotExist(name string, options map[string]string, plugin
 	return nil
 }
 
-// removeVolume removes the specified volume from state as well tears down its mountpoint and storage
-func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool, timeout *uint) error {
+// removeVolume removes the specified volume from state as well tears down its mountpoint and storage.
+// ignoreVolumePlugin is used to only remove the volume from the db and not the plugin,
+// this is required when the volume was already removed from the plugin, i.e. in UpdateVolumePlugins().
+func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool, timeout *uint, ignoreVolumePlugin bool) error {
 	if !v.valid {
 		if ok, _ := r.state.HasVolume(v.Name()); !ok {
 			return nil
@@ -263,7 +347,7 @@ func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool, timeo
 	var removalErr error
 
 	// If we use a volume plugin, we need to remove from the plugin.
-	if v.UsesVolumeDriver() {
+	if v.UsesVolumeDriver() && !ignoreVolumePlugin {
 		canRemove := true
 
 		// Do we have a volume driver?