volume: add support for non-volatile upperdir,workdir for overlay volumes

Often users want their overlayed volumes to be `non-volatile` in nature
that means that same `upper` dir can be re-used by one or more
containers but overall of nature of volumes still have to be `overlay`
so work done is still on a overlay not on the actual volume.

Following PR adds support for more advanced options i.e custom `workdir`
and `upperdir` for overlayed volumes. So that users can re-use `workdir`
and `upperdir` across new containers as well.

Usage
```console

$ podman run -it -v myvol:/data:O,upperdir=/path/persistant/upper,workdir=/path/persistant/work alpine sh

```

Signed-off-by: Aditya R <arajan@redhat.com>

4 files changed, 109 insertions, 3 deletions

diff --git a/docs/source/markdown/podman-run.1.md b/docs/source/markdown/podman-run.1.md
index 241cb6d95..59f36f67d 100644
--- a/docs/source/markdown/podman-run.1.md
+++ b/docs/source/markdown/podman-run.1.md
@@ -1394,6 +1394,10 @@ directory will be the lower, and the container storage directory will be the
 upper. Modifications to the mount point are destroyed when the container
 finishes executing, similar to a tmpfs mount point being unmounted.
 
+  For advanced users overlay option also supports custom non-volatile `upperdir` and `workdir`
+for the overlay mount. Custom `upperdir` and `workdir` can be fully managed by the users themselves
+and `podman` will not remove it on lifecycle completion. Example `:O,upperdir=/some/upper,workdir=/some/work`
+
   Subsequent executions of the container will see the original source directory
 content, any changes from previous container executions no longer exist.
 
diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go
index 84293ccb2..5cc2a78fc 100644
--- a/libpod/container_internal_linux.go
+++ b/libpod/container_internal_linux.go
@@ -391,18 +391,52 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 		}
 
 		overlayFlag := false
+		upperDir := ""
+		workDir := ""
 		for _, o := range namedVol.Options {
 			if o == "O" {
 				overlayFlag = true
 			}
+			if overlayFlag && strings.Contains(o, "upperdir") {
+				splitOpt := strings.SplitN(o, "=", 2)
+				if len(splitOpt) > 1 {
+					upperDir = splitOpt[1]
+					if upperDir == "" {
+						return nil, errors.New("cannot accept empty value for upperdir")
+					}
+				}
+			}
+			if overlayFlag && strings.Contains(o, "workdir") {
+				splitOpt := strings.SplitN(o, "=", 2)
+				if len(splitOpt) > 1 {
+					workDir = splitOpt[1]
+					if workDir == "" {
+						return nil, errors.New("cannot accept empty value for workdir")
+					}
+				}
+			}
 		}
 
 		if overlayFlag {
+			var overlayMount spec.Mount
+			var overlayOpts *overlay.Options
 			contentDir, err := overlay.TempDir(c.config.StaticDir, c.RootUID(), c.RootGID())
 			if err != nil {
 				return nil, err
 			}
-			overlayMount, err := overlay.Mount(contentDir, mountPoint, namedVol.Dest, c.RootUID(), c.RootGID(), c.runtime.store.GraphOptions())
+
+			if (upperDir != "" && workDir == "") || (upperDir == "" && workDir != "") {
+				return nil, errors.Wrapf(err, "must specify both upperdir and workdir")
+			}
+
+			overlayOpts = &overlay.Options{RootUID: c.RootUID(),
+				RootGID:                c.RootGID(),
+				UpperDirOptionFragment: upperDir,
+				WorkDirOptionFragment:  workDir,
+				GraphOpts:              c.runtime.store.GraphOptions(),
+			}
+
+			overlayMount, err = overlay.MountWithOptions(contentDir, mountPoint, namedVol.Dest, overlayOpts)
 			if err != nil {
 				return nil, errors.Wrapf(err, "mounting overlay failed %q", mountPoint)
 			}
diff --git a/pkg/util/mountOpts.go b/pkg/util/mountOpts.go
index 959763dba..f32cf6ea6 100644
--- a/pkg/util/mountOpts.go
+++ b/pkg/util/mountOpts.go
@@ -25,16 +25,30 @@ type defaultMountOptions struct {
 // The sourcePath variable, if not empty, contains a bind mount source.
 func ProcessOptions(options []string, isTmpfs bool, sourcePath string) ([]string, error) {
 	var (
-		foundWrite, foundSize, foundProp, foundMode, foundExec, foundSuid, foundDev, foundCopyUp, foundBind, foundZ, foundU bool
+		foundWrite, foundSize, foundProp, foundMode, foundExec, foundSuid, foundDev, foundCopyUp, foundBind, foundZ, foundU, foundOverlay bool
 	)
 
 	newOptions := make([]string, 0, len(options))
 	for _, opt := range options {
 		// Some options have parameters - size, mode
 		splitOpt := strings.SplitN(opt, "=", 2)
+
+		// add advanced options such as upperdir=/path and workdir=/path, when overlay is specified
+		if foundOverlay {
+			if strings.Contains(opt, "upperdir") {
+				newOptions = append(newOptions, opt)
+				continue
+			}
+			if strings.Contains(opt, "workdir") {
+				newOptions = append(newOptions, opt)
+				continue
+			}
+		}
+
 		switch splitOpt[0] {
-		case "idmap":
 		case "O":
+			foundOverlay = true
+		case "idmap":
 			if len(options) > 1 {
 				return nil, errors.Wrapf(ErrDupeMntOption, "'O' option can not be used with other options")
 			}
diff --git a/test/e2e/run_volume_test.go b/test/e2e/run_volume_test.go
index 00faf8089..d23c5dc14 100644
--- a/test/e2e/run_volume_test.go
+++ b/test/e2e/run_volume_test.go
@@ -260,6 +260,60 @@ var _ = Describe("Podman run with volumes", func() {
 
 	})
 
+	It("podman support overlay on named volume with custom upperdir and workdir", func() {
+		SkipIfRemote("Overlay volumes only work locally")
+		if os.Getenv("container") != "" {
+			Skip("Overlay mounts not supported when running in a container")
+		}
+		if rootless.IsRootless() {
+			if _, err := exec.LookPath("fuse-overlayfs"); err != nil {
+				Skip("Fuse-Overlayfs required for rootless overlay mount test")
+			}
+		}
+
+		// create persistent upperdir on host
+		upperDir := filepath.Join(tempdir, "upper")
+		err := os.Mkdir(upperDir, 0755)
+		Expect(err).To(BeNil(), "mkdir "+upperDir)
+
+		// create persistent workdir on host
+		workDir := filepath.Join(tempdir, "work")
+		err = os.Mkdir(workDir, 0755)
+		Expect(err).To(BeNil(), "mkdir "+workDir)
+
+		overlayOpts := fmt.Sprintf("upperdir=%s,workdir=%s", upperDir, workDir)
+
+		session := podmanTest.Podman([]string{"volume", "create", "myvolume"})
+		session.WaitWithDefaultTimeout()
+		volName := session.OutputToString()
+		Expect(session).Should(Exit(0))
+
+		// create file on actual volume
+		session = podmanTest.Podman([]string{"run", "--volume", volName + ":/data", ALPINE, "sh", "-c", "echo hello >> " + "/data/test"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		// create file on overlay volume
+		session = podmanTest.Podman([]string{"run", "--volume", volName + ":/data:O," + overlayOpts, ALPINE, "sh", "-c", "echo hello >> " + "/data/overlay"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		session = podmanTest.Podman([]string{"run", "--volume", volName + ":/data:O," + overlayOpts, ALPINE, "sh", "-c", "ls /data"})
+		session.WaitWithDefaultTimeout()
+		// must contain `overlay` file since it should be persistent on specified upper and workdir
+		Expect(session.OutputToString()).To(ContainSubstring("overlay"))
+		// this should be there since `test` was written on actual volume not on any overlay
+		Expect(session.OutputToString()).To(ContainSubstring("test"))
+
+		session = podmanTest.Podman([]string{"run", "--volume", volName + ":/data:O", ALPINE, "sh", "-c", "ls /data"})
+		session.WaitWithDefaultTimeout()
+		// must not contain `overlay` file which was on custom upper and workdir since we have not specified any upper or workdir
+		Expect(session.OutputToString()).To(Not(ContainSubstring("overlay")))
+		// this should be there since `test` was written on actual volume not on any overlay
+		Expect(session.OutputToString()).To(ContainSubstring("test"))
+
+	})
+
 	It("podman run with noexec can't exec", func() {
 		session := podmanTest.Podman([]string{"run", "--rm", "-v", "/bin:/hostbin:noexec", ALPINE, "/hostbin/ls", "/"})
 		session.WaitWithDefaultTimeout()