overlay-volumes: add support for non-volatile upperdir,workdir for anonymous volumes

Similar feature was added for named overlay volumes here: https://github.com/containers/podman/pull/12712
Following PR just mimics similar feature for anonymous volumes.

Often users want their anonymous overlayed volumes to be `non-volatile` in nature
that means that same `upper` dir can be re-used by one or more
containers but overall of nature of volumes still have to be overlay
so work done is still on a overlay not on the actual volume.

Following PR adds support for more advanced options i.e custom `workdir`
and `upperdir` for overlayed volumes. So that users can re-use `workdir`
and `upperdir` across new containers as well.

Usage

```console
podman run -it -v /some/path:/data:O,upperdir=/path/persistant/upper,workdir=/path/persistant/work alpine sh
```

Signed-off-by: Aditya R <arajan@redhat.com>

3 files changed, 103 insertions, 24 deletions

diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go
index e19d75deb..0056b8e86 100644
--- a/libpod/container_internal_linux.go
+++ b/libpod/container_internal_linux.go
@@ -388,6 +388,37 @@ func lookupHostUser(name string) (*runcuser.ExecUser, error) {
 	return &execUser, nil
 }
 
+// Internal only function which returns upper and work dir from
+// overlay options.
+func getOverlayUpperAndWorkDir(options []string) (string, string, error) {
+	upperDir := ""
+	workDir := ""
+	for _, o := range options {
+		if strings.HasPrefix(o, "upperdir") {
+			splitOpt := strings.SplitN(o, "=", 2)
+			if len(splitOpt) > 1 {
+				upperDir = splitOpt[1]
+				if upperDir == "" {
+					return "", "", errors.New("cannot accept empty value for upperdir")
+				}
+			}
+		}
+		if strings.HasPrefix(o, "workdir") {
+			splitOpt := strings.SplitN(o, "=", 2)
+			if len(splitOpt) > 1 {
+				workDir = splitOpt[1]
+				if workDir == "" {
+					return "", "", errors.New("cannot accept empty value for workdir")
+				}
+			}
+		}
+	}
+	if (upperDir != "" && workDir == "") || (upperDir == "" && workDir != "") {
+		return "", "", errors.New("must specify both upperdir and workdir")
+	}
+	return upperDir, workDir, nil
+}
+
 // Generate spec for a container
 // Accepts a map of the container's dependencies
 func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
@@ -460,23 +491,9 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 		for _, o := range namedVol.Options {
 			if o == "O" {
 				overlayFlag = true
-			}
-			if overlayFlag && strings.Contains(o, "upperdir") {
-				splitOpt := strings.SplitN(o, "=", 2)
-				if len(splitOpt) > 1 {
-					upperDir = splitOpt[1]
-					if upperDir == "" {
-						return nil, errors.New("cannot accept empty value for upperdir")
-					}
-				}
-			}
-			if overlayFlag && strings.Contains(o, "workdir") {
-				splitOpt := strings.SplitN(o, "=", 2)
-				if len(splitOpt) > 1 {
-					workDir = splitOpt[1]
-					if workDir == "" {
-						return nil, errors.New("cannot accept empty value for workdir")
-					}
+				upperDir, workDir, err = getOverlayUpperAndWorkDir(namedVol.Options)
+				if err != nil {
+					return nil, err
 				}
 			}
 		}
@@ -489,10 +506,6 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 				return nil, err
 			}
 
-			if (upperDir != "" && workDir == "") || (upperDir == "" && workDir != "") {
-				return nil, errors.Wrapf(err, "must specify both upperdir and workdir")
-			}
-
 			overlayOpts = &overlay.Options{RootUID: c.RootUID(),
 				RootGID:                c.RootGID(),
 				UpperDirOptionFragment: upperDir,
@@ -585,11 +598,22 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 
 	// Add overlay volumes
 	for _, overlayVol := range c.config.OverlayVolumes {
+		upperDir, workDir, err := getOverlayUpperAndWorkDir(overlayVol.Options)
+		if err != nil {
+			return nil, err
+		}
 		contentDir, err := overlay.TempDir(c.config.StaticDir, c.RootUID(), c.RootGID())
 		if err != nil {
 			return nil, err
 		}
-		overlayMount, err := overlay.Mount(contentDir, overlayVol.Source, overlayVol.Dest, c.RootUID(), c.RootGID(), c.runtime.store.GraphOptions())
+		overlayOpts := &overlay.Options{RootUID: c.RootUID(),
+			RootGID:                c.RootGID(),
+			UpperDirOptionFragment: upperDir,
+			WorkDirOptionFragment:  workDir,
+			GraphOpts:              c.runtime.store.GraphOptions(),
+		}
+
+		overlayMount, err := overlay.MountWithOptions(contentDir, overlayVol.Source, overlayVol.Dest, overlayOpts)
 		if err != nil {
 			return nil, errors.Wrapf(err, "mounting overlay failed %q", overlayVol.Source)
 		}
diff --git a/pkg/specgen/volumes.go b/pkg/specgen/volumes.go
index b26666df3..a7a1022b0 100644
--- a/pkg/specgen/volumes.go
+++ b/pkg/specgen/volumes.go
@@ -97,6 +97,8 @@ func GenVolumeMounts(volumeFlag []string) (map[string]spec.Mount, map[string]*Na
 			// This is not a named volume
 			overlayFlag := false
 			chownFlag := false
+			upperDirFlag := false
+			workDirFlag := false
 			for _, o := range options {
 				if o == "O" {
 					overlayFlag = true
@@ -105,8 +107,16 @@ func GenVolumeMounts(volumeFlag []string) (map[string]spec.Mount, map[string]*Na
 					if strings.Contains(joinedOpts, "U") {
 						chownFlag = true
 					}
-
-					if len(options) > 2 || (len(options) == 2 && !chownFlag) {
+					if strings.Contains(joinedOpts, "upperdir") {
+						upperDirFlag = true
+					}
+					if strings.Contains(joinedOpts, "workdir") {
+						workDirFlag = true
+					}
+					if (workDirFlag && !upperDirFlag) || (!workDirFlag && upperDirFlag) {
+						return nil, nil, nil, errors.New("must set both `upperdir` and `workdir`")
+					}
+					if len(options) > 2 && !(len(options) == 3 && upperDirFlag && workDirFlag) || (len(options) == 2 && !chownFlag) {
 						return nil, nil, nil, errors.New("can't use 'O' with other options")
 					}
 				}
diff --git a/test/e2e/run_volume_test.go b/test/e2e/run_volume_test.go
index 3bef889b7..1c0480407 100644
--- a/test/e2e/run_volume_test.go
+++ b/test/e2e/run_volume_test.go
@@ -325,6 +325,51 @@ var _ = Describe("Podman run with volumes", func() {
 
 	})
 
+	It("podman support overlay volume with custom upperdir and workdir", func() {
+		SkipIfRemote("Overlay volumes only work locally")
+		if os.Getenv("container") != "" {
+			Skip("Overlay mounts not supported when running in a container")
+		}
+		if rootless.IsRootless() {
+			if _, err := exec.LookPath("fuse-overlayfs"); err != nil {
+				Skip("Fuse-Overlayfs required for rootless overlay mount test")
+			}
+		}
+
+		// Use bindsource instead of named volume
+		bindSource := filepath.Join(tempdir, "bindsource")
+		err := os.Mkdir(bindSource, 0755)
+		Expect(err).To(BeNil(), "mkdir "+bindSource)
+
+		// create persistent upperdir on host
+		upperDir := filepath.Join(tempdir, "upper")
+		err = os.Mkdir(upperDir, 0755)
+		Expect(err).To(BeNil(), "mkdir "+upperDir)
+
+		// create persistent workdir on host
+		workDir := filepath.Join(tempdir, "work")
+		err = os.Mkdir(workDir, 0755)
+		Expect(err).To(BeNil(), "mkdir "+workDir)
+
+		overlayOpts := fmt.Sprintf("upperdir=%s,workdir=%s", upperDir, workDir)
+
+		// create file on overlay volume
+		session := podmanTest.Podman([]string{"run", "--volume", bindSource + ":/data:O," + overlayOpts, ALPINE, "sh", "-c", "echo hello >> " + "/data/overlay"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+
+		session = podmanTest.Podman([]string{"run", "--volume", bindSource + ":/data:O," + overlayOpts, ALPINE, "sh", "-c", "ls /data"})
+		session.WaitWithDefaultTimeout()
+		// must contain `overlay` file since it should be persistent on specified upper and workdir
+		Expect(session.OutputToString()).To(ContainSubstring("overlay"))
+
+		session = podmanTest.Podman([]string{"run", "--volume", bindSource + ":/data:O", ALPINE, "sh", "-c", "ls /data"})
+		session.WaitWithDefaultTimeout()
+		// must not contain `overlay` file which was on custom upper and workdir since we have not specified any upper or workdir
+		Expect(session.OutputToString()).To(Not(ContainSubstring("overlay")))
+
+	})
+
 	It("podman run with noexec can't exec", func() {
 		session := podmanTest.Podman([]string{"run", "--rm", "-v", "/bin:/hostbin:noexec", ALPINE, "/hostbin/ls", "/"})
 		session.WaitWithDefaultTimeout()