Merge pull request #14713 from Luap99/volume-plugin

add podman volume reload to sync volume plugins

23 files changed, 417 insertions, 43 deletions

diff --git a/Containerfile-testvol b/Containerfile-testvol
deleted file mode 100644
index 6ff45064b..000000000
--- a/Containerfile-testvol
+++ /dev/null
@@ -1,10 +0,0 @@
-FROM golang:1.15-alpine AS build-img
-COPY ./test/testvol/ /go/src/github.com/containers/podman/cmd/testvol/
-COPY ./vendor /go/src/github.com/containers/podman/vendor/
-WORKDIR /go/src/github.com/containers/podman
-RUN go build -o /testvol ./cmd/testvol
-
-FROM alpine
-COPY --from=build-img /testvol /usr/local/bin
-WORKDIR /
-ENTRYPOINT ["/usr/local/bin/testvol"]
diff --git a/Makefile b/Makefile
index 38ea7a256..3ed522b32 100644
--- a/Makefile
+++ b/Makefile
@@ -225,11 +225,11 @@ test/checkseccomp/checkseccomp: $(wildcard test/checkseccomp/*.go)
 
 .PHONY: test/testvol/testvol
 test/testvol/testvol: $(wildcard test/testvol/*.go)
-	$(GOCMD) build $(BUILDFLAGS) $(GO_LDFLAGS) '$(LDFLAGS_PODMAN)' -o $@ ./test/testvol
+	$(GOCMD) build -o $@ ./test/testvol
 
-.PHONY: volume-plugin-test-image
+.PHONY: volume-plugin-test-img
 volume-plugin-test-img:
-	podman build -t quay.io/libpod/volume-plugin-test-img -f Containerfile-testvol .
+	./bin/podman build --network none -t quay.io/libpod/volume-plugin-test-img:$$(date +%Y%m%d) -f ./test/testvol/Containerfile .
 
 .PHONY: test/goecho/goecho
 test/goecho/goecho: $(wildcard test/goecho/*.go)
diff --git a/cmd/podman/volumes/reload.go b/cmd/podman/volumes/reload.go
new file mode 100644
index 000000000..d0d76fb88
--- /dev/null
+++ b/cmd/podman/volumes/reload.go
@@ -0,0 +1,52 @@
+package volumes
+
+import (
+	"fmt"
+
+	"github.com/containers/common/pkg/completion"
+	"github.com/containers/podman/v4/cmd/podman/registry"
+	"github.com/containers/podman/v4/cmd/podman/utils"
+	"github.com/containers/podman/v4/cmd/podman/validate"
+	"github.com/spf13/cobra"
+)
+
+var (
+	reloadDescription = `Check all configured volume plugins and update the libpod database with all available volumes.
+
+  Existing volumes are also removed from the database when they are no longer present in the plugin.`
+	reloadCommand = &cobra.Command{
+		Use:               "reload",
+		Args:              validate.NoArgs,
+		Short:             "reload all volumes from volume plugins",
+		Long:              reloadDescription,
+		RunE:              reload,
+		ValidArgsFunction: completion.AutocompleteNone,
+	}
+)
+
+func init() {
+	registry.Commands = append(registry.Commands, registry.CliCommand{
+		Command: reloadCommand,
+		Parent:  volumeCmd,
+	})
+}
+
+func reload(cmd *cobra.Command, args []string) error {
+	report, err := registry.ContainerEngine().VolumeReload(registry.Context())
+	if err != nil {
+		return err
+	}
+	printReload("Added", report.Added)
+	printReload("Removed", report.Removed)
+	errs := (utils.OutputErrors)(report.Errors)
+	return errs.PrintErrors()
+}
+
+func printReload(typ string, values []string) {
+	if len(values) > 0 {
+		fmt.Println(typ + ":")
+		for _, name := range values {
+			fmt.Println(name)
+		}
+	}
+}
diff --git a/docs/source/markdown/podman-volume-reload.1.md b/docs/source/markdown/podman-volume-reload.1.md
new file mode 100644
index 000000000..5b9e9b9ac
--- /dev/null
+++ b/docs/source/markdown/podman-volume-reload.1.md
@@ -0,0 +1,29 @@
+% podman-volume-reload(1)
+
+## NAME
+podman\-volume\-reload - Reload all volumes from volumes plugins
+
+## SYNOPSIS
+**podman volume reload**
+
+## DESCRIPTION
+
+**podman volume reload** checks all configured volume plugins and updates the libpod database with all available volumes.
+Existing volumes are also removed from the database when they are no longer present in the plugin.
+
+This command it is best effort and cannot guarantee a perfect state because plugins can be modified from the outside at any time.
+
+Note: This command is not supported with podman-remote.
+
+## EXAMPLES
+
+```
+$ podman volume reload
+Added:
+vol6
+Removed:
+t3
+```
+
+## SEE ALSO
+**[podman(1)](podman.1.md)**, **[podman-volume(1)](podman-volume.1.md)**
diff --git a/docs/source/markdown/podman-volume.1.md b/docs/source/markdown/podman-volume.1.md
index 476d58591..a437590b3 100644
--- a/docs/source/markdown/podman-volume.1.md
+++ b/docs/source/markdown/podman-volume.1.md
@@ -21,6 +21,7 @@ podman volume is a set of subcommands that manage volumes.
 | ls      | [podman-volume-ls(1)](podman-volume-ls.1.md)           | List all the available volumes.                                                |
 | mount   | [podman-volume-mount(1)](podman-volume-mount.1.md)     | Mount a volume filesystem.                                                     |
 | prune   | [podman-volume-prune(1)](podman-volume-prune.1.md)     | Remove all unused volumes.                                                     |
+| reload  | [podman-volume-reload(1)](podman-volume-reload.1.md)   | Reload all volumes from volumes plugins.                                       |
 | rm      | [podman-volume-rm(1)](podman-volume-rm.1.md)           | Remove one or more volumes.                                                    |
 | unmount | [podman-volume-unmount(1)](podman-volume-unmount.1.md) | Unmount a volume.                                                     |
 
diff --git a/libpod/define/volume_inspect.go b/libpod/define/volume_inspect.go
index fac179176..aaa23b4fc 100644
--- a/libpod/define/volume_inspect.go
+++ b/libpod/define/volume_inspect.go
@@ -57,3 +57,9 @@ type InspectVolumeData struct {
 	// UID/GID.
 	NeedsChown bool `json:"NeedsChown,omitempty"`
 }
+
+type VolumeReload struct {
+	Added   []string
+	Removed []string
+	Errors  []error
+}
diff --git a/libpod/plugin/volume_api.go b/libpod/plugin/volume_api.go
index f997ccf22..2de7db32c 100644
--- a/libpod/plugin/volume_api.go
+++ b/libpod/plugin/volume_api.go
@@ -197,13 +197,13 @@ func (p *VolumePlugin) verifyReachable() error {
 
 // Send a request to the volume plugin for handling.
 // Callers *MUST* close the response when they are done.
-func (p *VolumePlugin) sendRequest(toJSON interface{}, hasBody bool, endpoint string) (*http.Response, error) {
+func (p *VolumePlugin) sendRequest(toJSON interface{}, endpoint string) (*http.Response, error) {
 	var (
 		reqJSON []byte
 		err     error
 	)
 
-	if hasBody {
+	if toJSON != nil {
 		reqJSON, err = json.Marshal(toJSON)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error marshalling request JSON for volume plugin %s endpoint %s", p.Name, endpoint)
@@ -274,7 +274,7 @@ func (p *VolumePlugin) CreateVolume(req *volume.CreateRequest) error {
 
 	logrus.Infof("Creating volume %s using plugin %s", req.Name, p.Name)
 
-	resp, err := p.sendRequest(req, true, createPath)
+	resp, err := p.sendRequest(req, createPath)
 	if err != nil {
 		return err
 	}
@@ -291,7 +291,7 @@ func (p *VolumePlugin) ListVolumes() ([]*volume.Volume, error) {
 
 	logrus.Infof("Listing volumes using plugin %s", p.Name)
 
-	resp, err := p.sendRequest(nil, false, listPath)
+	resp, err := p.sendRequest(nil, listPath)
 	if err != nil {
 		return nil, err
 	}
@@ -326,7 +326,7 @@ func (p *VolumePlugin) GetVolume(req *volume.GetRequest) (*volume.Volume, error)
 
 	logrus.Infof("Getting volume %s using plugin %s", req.Name, p.Name)
 
-	resp, err := p.sendRequest(req, true, getPath)
+	resp, err := p.sendRequest(req, getPath)
 	if err != nil {
 		return nil, err
 	}
@@ -361,7 +361,7 @@ func (p *VolumePlugin) RemoveVolume(req *volume.RemoveRequest) error {
 
 	logrus.Infof("Removing volume %s using plugin %s", req.Name, p.Name)
 
-	resp, err := p.sendRequest(req, true, removePath)
+	resp, err := p.sendRequest(req, removePath)
 	if err != nil {
 		return err
 	}
@@ -382,7 +382,7 @@ func (p *VolumePlugin) GetVolumePath(req *volume.PathRequest) (string, error) {
 
 	logrus.Infof("Getting volume %s path using plugin %s", req.Name, p.Name)
 
-	resp, err := p.sendRequest(req, true, hostVirtualPath)
+	resp, err := p.sendRequest(req, hostVirtualPath)
 	if err != nil {
 		return "", err
 	}
@@ -419,7 +419,7 @@ func (p *VolumePlugin) MountVolume(req *volume.MountRequest) (string, error) {
 
 	logrus.Infof("Mounting volume %s using plugin %s for container %s", req.Name, p.Name, req.ID)
 
-	resp, err := p.sendRequest(req, true, mountPath)
+	resp, err := p.sendRequest(req, mountPath)
 	if err != nil {
 		return "", err
 	}
@@ -455,7 +455,7 @@ func (p *VolumePlugin) UnmountVolume(req *volume.UnmountRequest) error {
 
 	logrus.Infof("Unmounting volume %s using plugin %s for container %s", req.Name, p.Name, req.ID)
 
-	resp, err := p.sendRequest(req, true, unmountPath)
+	resp, err := p.sendRequest(req, unmountPath)
 	if err != nil {
 		return err
 	}
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
index a9ae9d1db..459514d47 100644
--- a/libpod/runtime_ctr.go
+++ b/libpod/runtime_ctr.go
@@ -502,7 +502,7 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container) (_ *Contai
 				volOptions = append(volOptions, parsedOptions...)
 			}
 		}
-		newVol, err := r.newVolume(volOptions...)
+		newVol, err := r.newVolume(false, volOptions...)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error creating named volume %q", vol.Name)
 		}
@@ -805,7 +805,7 @@ func (r *Runtime) removeContainer(ctx context.Context, c *Container, force, remo
 			if !volume.Anonymous() {
 				continue
 			}
-			if err := runtime.removeVolume(ctx, volume, false, timeout); err != nil && errors.Cause(err) != define.ErrNoSuchVolume {
+			if err := runtime.removeVolume(ctx, volume, false, timeout, false); err != nil && errors.Cause(err) != define.ErrNoSuchVolume {
 				if errors.Cause(err) == define.ErrVolumeBeingUsed {
 					// Ignore error, since podman will report original error
 					volumesFrom, _ := c.volumesFrom()
@@ -963,7 +963,7 @@ func (r *Runtime) evictContainer(ctx context.Context, idOrName string, removeVol
 			if !volume.Anonymous() {
 				continue
 			}
-			if err := r.removeVolume(ctx, volume, false, timeout); err != nil && err != define.ErrNoSuchVolume && err != define.ErrVolumeBeingUsed {
+			if err := r.removeVolume(ctx, volume, false, timeout, false); err != nil && err != define.ErrNoSuchVolume && err != define.ErrVolumeBeingUsed {
 				logrus.Errorf("Cleaning up volume (%s): %v", v, err)
 			}
 		}
diff --git a/libpod/runtime_pod_linux.go b/libpod/runtime_pod_linux.go
index d75ac2971..00017ca21 100644
--- a/libpod/runtime_pod_linux.go
+++ b/libpod/runtime_pod_linux.go
@@ -315,7 +315,7 @@ func (r *Runtime) removePod(ctx context.Context, p *Pod, removeCtrs, force bool,
 		if !volume.Anonymous() {
 			continue
 		}
-		if err := r.removeVolume(ctx, volume, false, timeout); err != nil {
+		if err := r.removeVolume(ctx, volume, false, timeout, false); err != nil {
 			if errors.Cause(err) == define.ErrNoSuchVolume || errors.Cause(err) == define.ErrVolumeRemoved {
 				continue
 			}
diff --git a/libpod/runtime_volume.go b/libpod/runtime_volume.go
index 21bf8aefc..6872db21d 100644
--- a/libpod/runtime_volume.go
+++ b/libpod/runtime_volume.go
@@ -33,7 +33,7 @@ func (r *Runtime) RemoveVolume(ctx context.Context, v *Volume, force bool, timeo
 			return nil
 		}
 	}
-	return r.removeVolume(ctx, v, force, timeout)
+	return r.removeVolume(ctx, v, force, timeout, false)
 }
 
 // GetVolume retrieves a volume given its full name.
diff --git a/libpod/runtime_volume_linux.go b/libpod/runtime_volume_linux.go
index f8788e183..877f3a1fd 100644
--- a/libpod/runtime_volume_linux.go
+++ b/libpod/runtime_volume_linux.go
@@ -5,6 +5,7 @@ package libpod
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"path/filepath"
 	"strings"
@@ -25,11 +26,13 @@ func (r *Runtime) NewVolume(ctx context.Context, options ...VolumeCreateOption)
 	if !r.valid {
 		return nil, define.ErrRuntimeStopped
 	}
-	return r.newVolume(options...)
+	return r.newVolume(false, options...)
 }
 
-// newVolume creates a new empty volume
-func (r *Runtime) newVolume(options ...VolumeCreateOption) (_ *Volume, deferredErr error) {
+// newVolume creates a new empty volume with the given options.
+// The createPluginVolume can be set to true to make it not create the volume in the volume plugin,
+// this is required for the UpdateVolumePlugins() function. If you are not sure set this to false.
+func (r *Runtime) newVolume(noCreatePluginVolume bool, options ...VolumeCreateOption) (_ *Volume, deferredErr error) {
 	volume := newVolume(r)
 	for _, option := range options {
 		if err := option(volume); err != nil {
@@ -83,7 +86,7 @@ func (r *Runtime) newVolume(options ...VolumeCreateOption) (_ *Volume, deferredE
 
 	// Now we get conditional: we either need to make the volume in the
 	// volume plugin, or on disk if not using a plugin.
-	if volume.plugin != nil {
+	if volume.plugin != nil && !noCreatePluginVolume {
 		// We can't chown, or relabel, or similar the path the volume is
 		// using, because it's not managed by us.
 		// TODO: reevaluate this once we actually have volume plugins in
@@ -164,6 +167,85 @@ func (r *Runtime) newVolume(options ...VolumeCreateOption) (_ *Volume, deferredE
 	return volume, nil
 }
 
+// UpdateVolumePlugins reads all volumes from all configured volume plugins and
+// imports them into the libpod db. It also checks if existing libpod volumes
+// are removed in the plugin, in this case we try to remove it from libpod.
+// On errors we continue and try to do as much as possible. all errors are
+// returned as array in the returned struct.
+// This function has many race conditions, it is best effort but cannot guarantee
+// a perfect state since plugins can be modified from the outside at any time.
+func (r *Runtime) UpdateVolumePlugins(ctx context.Context) *define.VolumeReload {
+	var (
+		added            []string
+		removed          []string
+		errs             []error
+		allPluginVolumes = map[string]struct{}{}
+	)
+
+	for driverName, socket := range r.config.Engine.VolumePlugins {
+		driver, err := volplugin.GetVolumePlugin(driverName, socket)
+		if err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		vols, err := driver.ListVolumes()
+		if err != nil {
+			errs = append(errs, fmt.Errorf("failed to read volumes from plugin %q: %w", driverName, err))
+			continue
+		}
+		for _, vol := range vols {
+			allPluginVolumes[vol.Name] = struct{}{}
+			if _, err := r.newVolume(true, WithVolumeName(vol.Name), WithVolumeDriver(driverName)); err != nil {
+				// If the volume exists this is not an error, just ignore it and log. It is very likely
+				// that the volume from the plugin was already in our db.
+				if !errors.Is(err, define.ErrVolumeExists) {
+					errs = append(errs, err)
+					continue
+				}
+				logrus.Infof("Volume %q already exists: %v", vol.Name, err)
+				continue
+			}
+			added = append(added, vol.Name)
+		}
+	}
+
+	libpodVolumes, err := r.state.AllVolumes()
+	if err != nil {
+		errs = append(errs, fmt.Errorf("cannot delete dangling plugin volumes: failed to read libpod volumes: %w", err))
+	}
+	for _, vol := range libpodVolumes {
+		if vol.UsesVolumeDriver() {
+			if _, ok := allPluginVolumes[vol.Name()]; !ok {
+				// The volume is no longer in the plugin, lets remove it from the libpod db.
+				if err := r.removeVolume(ctx, vol, false, nil, true); err != nil {
+					if errors.Is(err, define.ErrVolumeBeingUsed) {
+						// Volume is still used by at least one container. This is very bad,
+						// the plugin no longer has this but we still need it.
+						errs = append(errs, fmt.Errorf("volume was removed from the plugin %q but containers still require it: %w", vol.config.Driver, err))
+						continue
+					}
+					if errors.Is(err, define.ErrNoSuchVolume) || errors.Is(err, define.ErrVolumeRemoved) || errors.Is(err, define.ErrMissingPlugin) {
+						// Volume was already removed, no problem just ignore it and continue.
+						continue
+					}
+
+					// some other error
+					errs = append(errs, err)
+					continue
+				}
+				// Volume was successfully removed
+				removed = append(removed, vol.Name())
+			}
+		}
+	}
+
+	return &define.VolumeReload{
+		Added:   added,
+		Removed: removed,
+		Errors:  errs,
+	}
+}
+
 // makeVolumeInPluginIfNotExist makes a volume in the given volume plugin if it
 // does not already exist.
 func makeVolumeInPluginIfNotExist(name string, options map[string]string, plugin *volplugin.VolumePlugin) error {
@@ -197,8 +279,10 @@ func makeVolumeInPluginIfNotExist(name string, options map[string]string, plugin
 	return nil
 }
 
-// removeVolume removes the specified volume from state as well tears down its mountpoint and storage
-func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool, timeout *uint) error {
+// removeVolume removes the specified volume from state as well tears down its mountpoint and storage.
+// ignoreVolumePlugin is used to only remove the volume from the db and not the plugin,
+// this is required when the volume was already removed from the plugin, i.e. in UpdateVolumePlugins().
+func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool, timeout *uint, ignoreVolumePlugin bool) error {
 	if !v.valid {
 		if ok, _ := r.state.HasVolume(v.Name()); !ok {
 			return nil
@@ -263,7 +347,7 @@ func (r *Runtime) removeVolume(ctx context.Context, v *Volume, force bool, timeo
 	var removalErr error
 
 	// If we use a volume plugin, we need to remove from the plugin.
-	if v.UsesVolumeDriver() {
+	if v.UsesVolumeDriver() && !ignoreVolumePlugin {
 		canRemove := true
 
 		// Do we have a volume driver?
diff --git a/pkg/domain/entities/engine_container.go b/pkg/domain/entities/engine_container.go
index df42876f6..e4eb808b4 100644
--- a/pkg/domain/entities/engine_container.go
+++ b/pkg/domain/entities/engine_container.go
@@ -104,4 +104,5 @@ type ContainerEngine interface {
 	VolumePrune(ctx context.Context, options VolumePruneOptions) ([]*reports.PruneReport, error)
 	VolumeRm(ctx context.Context, namesOrIds []string, opts VolumeRmOptions) ([]*VolumeRmReport, error)
 	VolumeUnmount(ctx context.Context, namesOrIds []string) ([]*VolumeUnmountReport, error)
+	VolumeReload(ctx context.Context) (*VolumeReloadReport, error)
 }
diff --git a/pkg/domain/entities/volumes.go b/pkg/domain/entities/volumes.go
index 556df16c1..9a06b2238 100644
--- a/pkg/domain/entities/volumes.go
+++ b/pkg/domain/entities/volumes.go
@@ -54,6 +54,11 @@ type VolumeListReport struct {
 	VolumeConfigResponse
 }
 
+// VolumeReloadReport describes the response from reload volume plugins
+type VolumeReloadReport struct {
+	define.VolumeReload
+}
+
 /*
  * Docker API compatibility types
  */
diff --git a/pkg/domain/infra/abi/volumes.go b/pkg/domain/infra/abi/volumes.go
index a9c53c140..1186d8e81 100644
--- a/pkg/domain/infra/abi/volumes.go
+++ b/pkg/domain/infra/abi/volumes.go
@@ -211,3 +211,8 @@ func (ic *ContainerEngine) VolumeUnmount(ctx context.Context, nameOrIDs []string
 
 	return reports, nil
 }
+
+func (ic *ContainerEngine) VolumeReload(ctx context.Context) (*entities.VolumeReloadReport, error) {
+	report := ic.Libpod.UpdateVolumePlugins(ctx)
+	return &entities.VolumeReloadReport{VolumeReload: *report}, nil
+}
diff --git a/pkg/domain/infra/tunnel/volumes.go b/pkg/domain/infra/tunnel/volumes.go
index 33e090148..6ec35e836 100644
--- a/pkg/domain/infra/tunnel/volumes.go
+++ b/pkg/domain/infra/tunnel/volumes.go
@@ -108,3 +108,7 @@ func (ic *ContainerEngine) VolumeMount(ctx context.Context, nameOrIDs []string)
 func (ic *ContainerEngine) VolumeUnmount(ctx context.Context, nameOrIDs []string) ([]*entities.VolumeUnmountReport, error) {
 	return nil, errors.New("unmounting volumes is not supported for remote clients")
 }
+
+func (ic *ContainerEngine) VolumeReload(ctx context.Context) (*entities.VolumeReloadReport, error) {
+	return nil, errors.New("volume reload is not supported for remote clients")
+}
diff --git a/test/e2e/config.go b/test/e2e/config.go
index 2ca8e2a15..fbcc9dfff 100644
--- a/test/e2e/config.go
+++ b/test/e2e/config.go
@@ -15,7 +15,7 @@ var (
 	healthcheck       = "quay.io/libpod/alpine_healthcheck:latest"
 	ImageCacheDir     = "/tmp/podman/imagecachedir"
 	fedoraToolbox     = "registry.fedoraproject.org/fedora-toolbox:36"
-	volumeTest        = "quay.io/libpod/volume-plugin-test-img:latest"
+	volumeTest        = "quay.io/libpod/volume-plugin-test-img:20220623"
 
 	// This image has seccomp profiles that blocks all syscalls.
 	// The intention behind blocking all syscalls is to prevent
diff --git a/test/e2e/volume_plugin_test.go b/test/e2e/volume_plugin_test.go
index 4700afdb5..b585f8dd8 100644
--- a/test/e2e/volume_plugin_test.go
+++ b/test/e2e/volume_plugin_test.go
@@ -6,6 +6,7 @@ import (
 	"path/filepath"
 
 	. "github.com/containers/podman/v4/test/utils"
+	"github.com/containers/storage/pkg/stringid"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 	. "github.com/onsi/gomega/gexec"
@@ -188,4 +189,71 @@ var _ = Describe("Podman volume plugins", func() {
 		rmAll.WaitWithDefaultTimeout()
 		Expect(rmAll).Should(Exit(0))
 	})
+
+	It("podman volume reload", func() {
+		podmanTest.AddImageToRWStore(volumeTest)
+
+		confFile := filepath.Join(podmanTest.TempDir, "containers.conf")
+		err := os.WriteFile(confFile, []byte(`[engine]
+[engine.volume_plugins]
+testvol5 = "/run/docker/plugins/testvol5.sock"`), 0o644)
+		Expect(err).ToNot(HaveOccurred())
+		os.Setenv("CONTAINERS_CONF", confFile)
+
+		pluginStatePath := filepath.Join(podmanTest.TempDir, "volumes")
+		err = os.Mkdir(pluginStatePath, 0755)
+		Expect(err).ToNot(HaveOccurred())
+
+		// Keep this distinct within tests to avoid multiple tests using the same plugin.
+		pluginName := "testvol5"
+		ctrName := "pluginCtr"
+		plugin := podmanTest.Podman([]string{"run", "--name", ctrName, "--security-opt", "label=disable", "-v", "/run/docker/plugins:/run/docker/plugins",
+			"-v", fmt.Sprintf("%v:%v", pluginStatePath, pluginStatePath), "-d", volumeTest, "--sock-name", pluginName, "--path", pluginStatePath})
+		plugin.WaitWithDefaultTimeout()
+		Expect(plugin).Should(Exit(0))
+
+		localvol := "local-" + stringid.GenerateNonCryptoID()
+		// create local volume
+		session := podmanTest.Podman([]string{"volume", "create", localvol})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(Exit(0))
+
+		vol1 := "vol1-" + stringid.GenerateNonCryptoID()
+		session = podmanTest.Podman([]string{"volume", "create", "--driver", pluginName, vol1})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(Exit(0))
+
+		// now create volume in plugin without podman
+		vol2 := "vol2-" + stringid.GenerateNonCryptoID()
+		plugin = podmanTest.Podman([]string{"exec", ctrName, "/usr/local/bin/testvol", "--sock-name", pluginName, "create", vol2})
+		plugin.WaitWithDefaultTimeout()
+		Expect(plugin).Should(Exit(0))
+
+		session = podmanTest.Podman([]string{"volume", "ls", "-q"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(Exit(0))
+		Expect(session.OutputToStringArray()).To(ContainElements(localvol, vol1))
+		Expect(session.ErrorToString()).To(Equal("")) // make sure no errors are shown
+
+		plugin = podmanTest.Podman([]string{"exec", ctrName, "/usr/local/bin/testvol", "--sock-name", pluginName, "remove", vol1})
+		plugin.WaitWithDefaultTimeout()
+		Expect(plugin).Should(Exit(0))
+
+		// now reload volumes from plugins
+		session = podmanTest.Podman([]string{"volume", "reload"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(Exit(0))
+		Expect(string(session.Out.Contents())).To(Equal(fmt.Sprintf(`Added:
+%s
+Removed:
+%s
+`, vol2, vol1)))
+		Expect(session.ErrorToString()).To(Equal("")) // make sure no errors are shown
+
+		session = podmanTest.Podman([]string{"volume", "ls", "-q"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).To(Exit(0))
+		Expect(session.OutputToStringArray()).To(ContainElements(localvol, vol2))
+		Expect(session.ErrorToString()).To(Equal("")) // make no errors are shown
+	})
 })
diff --git a/test/testvol/Containerfile b/test/testvol/Containerfile
new file mode 100644
index 000000000..32448f5a9
--- /dev/null
+++ b/test/testvol/Containerfile
@@ -0,0 +1,9 @@
+FROM docker.io/library/golang:1.18-alpine AS build-img
+COPY ./ /go/src/github.com/containers/podman/
+WORKDIR /go/src/github.com/containers/podman
+RUN GO111MODULE=off go build -o /testvol ./test/testvol
+
+FROM alpine
+COPY --from=build-img /testvol /usr/local/bin
+WORKDIR /
+ENTRYPOINT ["/usr/local/bin/testvol", "serve"]
diff --git a/test/testvol/create.go b/test/testvol/create.go
new file mode 100644
index 000000000..d29300f0b
--- /dev/null
+++ b/test/testvol/create.go
@@ -0,0 +1,26 @@
+package main
+
+import (
+	pluginapi "github.com/docker/go-plugins-helpers/volume"
+	"github.com/spf13/cobra"
+)
+
+var createCmd = &cobra.Command{
+	Use:   "create NAME",
+	Short: "create a volume",
+	Long:  `Create a volume in the volume plugin listening on --sock-name`,
+	Args:  cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return createVol(config.sockName, args[0])
+	},
+}
+
+func createVol(sockName, volName string) error {
+	plugin, err := getPlugin(sockName)
+	if err != nil {
+		return err
+	}
+	createReq := new(pluginapi.CreateRequest)
+	createReq.Name = volName
+	return plugin.CreateVolume(createReq)
+}
diff --git a/test/testvol/list.go b/test/testvol/list.go
new file mode 100644
index 000000000..fea615a70
--- /dev/null
+++ b/test/testvol/list.go
@@ -0,0 +1,32 @@
+package main
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var listCmd = &cobra.Command{
+	Use:   "list",
+	Short: "list all volumes",
+	Long:  `List all volumes from the volume plugin listening on --sock-name`,
+	Args:  cobra.NoArgs,
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return listVol(config.sockName)
+	},
+}
+
+func listVol(sockName string) error {
+	plugin, err := getPlugin(sockName)
+	if err != nil {
+		return err
+	}
+	vols, err := plugin.ListVolumes()
+	if err != nil {
+		return err
+	}
+	for _, vol := range vols {
+		fmt.Println(vol.Name)
+	}
+	return nil
+}
diff --git a/test/testvol/main.go b/test/testvol/main.go
index 30ab365b3..99c6fb694 100644
--- a/test/testvol/main.go
+++ b/test/testvol/main.go
@@ -14,13 +14,20 @@ import (
 )
 
 var rootCmd = &cobra.Command{
-	Use:   "testvol",
-	Short: "testvol - volume plugin for Podman",
+	Use:               "testvol",
+	Short:             "testvol - volume plugin for Podman testing",
+	PersistentPreRunE: before,
+	SilenceUsage:      true,
+}
+
+var serveCmd = &cobra.Command{
+	Use:   "serve",
+	Short: "serve the volume plugin on the unix socket",
 	Long:  `Creates simple directory volumes using the Volume Plugin API for testing volume plugin functionality`,
+	Args:  cobra.NoArgs,
 	RunE: func(cmd *cobra.Command, args []string) error {
 		return startServer(config.sockName)
 	},
-	PersistentPreRunE: before,
 }
 
 // Configuration for the volume plugin
@@ -37,9 +44,12 @@ var config = cliConfig{
 }
 
 func init() {
-	rootCmd.Flags().StringVar(&config.sockName, "sock-name", config.sockName, "Name of unix socket for plugin")
-	rootCmd.Flags().StringVar(&config.path, "path", "", "Path to initialize state and mount points")
+	rootCmd.PersistentFlags().StringVar(&config.sockName, "sock-name", config.sockName, "Name of unix socket for plugin")
 	rootCmd.PersistentFlags().StringVar(&config.logLevel, "log-level", config.logLevel, "Log messages including and over the specified level: debug, info, warn, error, fatal, panic")
+
+	serveCmd.Flags().StringVar(&config.path, "path", "", "Path to initialize state and mount points")
+
+	rootCmd.AddCommand(serveCmd, createCmd, removeCmd, listCmd)
 }
 
 func before(cmd *cobra.Command, args []string) error {
@@ -59,11 +69,8 @@ func before(cmd *cobra.Command, args []string) error {
 
 func main() {
 	if err := rootCmd.Execute(); err != nil {
-		logrus.Errorf("Running volume plugin: %v", err)
 		os.Exit(1)
 	}
-
-	os.Exit(0)
 }
 
 // startServer runs the HTTP server and responds to requests
diff --git a/test/testvol/remove.go b/test/testvol/remove.go
new file mode 100644
index 000000000..2839b0b50
--- /dev/null
+++ b/test/testvol/remove.go
@@ -0,0 +1,26 @@
+package main
+
+import (
+	pluginapi "github.com/docker/go-plugins-helpers/volume"
+	"github.com/spf13/cobra"
+)
+
+var removeCmd = &cobra.Command{
+	Use:   "remove NAME",
+	Short: "remove a volume",
+	Long:  `Remove a volume in the volume plugin listening on --sock-name`,
+	Args:  cobra.ExactArgs(1),
+	RunE: func(cmd *cobra.Command, args []string) error {
+		return removeVol(config.sockName, args[0])
+	},
+}
+
+func removeVol(sockName, volName string) error {
+	plugin, err := getPlugin(sockName)
+	if err != nil {
+		return err
+	}
+	removeReq := new(pluginapi.RemoveRequest)
+	removeReq.Name = volName
+	return plugin.RemoveVolume(removeReq)
+}
diff --git a/test/testvol/util.go b/test/testvol/util.go
new file mode 100644
index 000000000..7a0aeba86
--- /dev/null
+++ b/test/testvol/util.go
@@ -0,0 +1,29 @@
+package main
+
+import (
+	"path/filepath"
+	"strings"
+
+	"github.com/containers/podman/v4/libpod/plugin"
+)
+
+const pluginSockDir = "/run/docker/plugins"
+
+func getSocketPath(pathOrName string) string {
+	if filepath.IsAbs(pathOrName) {
+		return pathOrName
+	}
+
+	// only a name join it with the default path
+	return filepath.Join(pluginSockDir, pathOrName+".sock")
+}
+
+func getPluginName(pathOrName string) string {
+	return strings.TrimSuffix(filepath.Base(pathOrName), ".sock")
+}
+
+func getPlugin(sockNameOrPath string) (*plugin.VolumePlugin, error) {
+	path := getSocketPath(sockNameOrPath)
+	name := getPluginName(sockNameOrPath)
+	return plugin.GetVolumePlugin(name, path)
+}