Add RequiresMountsFor= to systemd generate

It is rare but possible that storage locations for the graphroot and the
runroot are not mounted at boot time, and therefore might race when
doing container operations.  An example we've seen in the wild is that a
slow tmpfs mount for the runroot would suddenly mount over /run, causing
the container to lose all currently-running data, requiring a system
refresh to get it back.

This patch adds RequiresMountsFor= to the systemd.unit header to ensure
the paths for both the graphroot and runroot are mounted prior to
starting any generated unit files.

Signed-off-by: Robb Manes <rmanes@redhat.com>

6 files changed, 111 insertions, 1 deletions

diff --git a/docs/source/markdown/podman-generate-systemd.1.md b/docs/source/markdown/podman-generate-systemd.1.md
index 00b13669a..43cb30f59 100644
--- a/docs/source/markdown/podman-generate-systemd.1.md
+++ b/docs/source/markdown/podman-generate-systemd.1.md
@@ -61,7 +61,7 @@ Set the systemd unit name separator between the name/id of a container/pod and t
 
 ### Generate and print a systemd unit file for a container
 
-Generate a systemd unit file for a container running nginx with an *always* restart policy and 1-second timeout to stdout.
+Generate a systemd unit file for a container running nginx with an *always* restart policy and 1-second timeout to stdout. Note that the **RequiresMountsFor** option in the **Unit** section ensures that the container storage for both the GraphRoot and the RunRoot are mounted prior to starting the service. For systems with container storage on disks like iSCSI or other remote block protocols, this ensures that Podman is not executed prior to any necessary storage operations coming online.
 
 ```
 $ podman create --name nginx nginx:latest
@@ -73,6 +73,9 @@ $ podman generate systemd --restart-policy=always -t 1 nginx
 [Unit]
 Description=Podman container-de1e3223b1b888bc02d0962dd6cb5855eb00734061013ffdd3479d225abacdc6.service
 Documentation=man:podman-generate-systemd(1)
+Wants=network.target
+After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/container/storage
 
 [Service]
 Restart=always
@@ -101,6 +104,7 @@ Description=Podman container-busy_moser.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/container/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -140,6 +144,9 @@ Description=Podman pod-systemd-pod.service
 Documentation=man:podman-generate-systemd(1)
 Requires=container-amazing_chandrasekhar.service container-jolly_shtern.service
 Before=container-amazing_chandrasekhar.service container-jolly_shtern.service
+Wants=network.target
+After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/container/storage
 
 [Service]
 Restart=on-failure
diff --git a/pkg/systemd/generate/common.go b/pkg/systemd/generate/common.go
index 94a6f4cb5..19d468403 100644
--- a/pkg/systemd/generate/common.go
+++ b/pkg/systemd/generate/common.go
@@ -36,6 +36,7 @@ Description=Podman {{{{.ServiceName}}}}.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor={{{{.GraphRoot}}}} {{{{.RunRoot}}}}
 `
 
 // filterPodFlags removes --pod and --pod-id-file from the specified command.
diff --git a/pkg/systemd/generate/containers.go b/pkg/systemd/generate/containers.go
index 9343a5067..bc13a6116 100644
--- a/pkg/systemd/generate/containers.go
+++ b/pkg/systemd/generate/containers.go
@@ -71,6 +71,12 @@ type containerInfo struct {
 	// If not nil, the container is part of the pod.  We can use the
 	// podInfo to extract the relevant data.
 	Pod *podInfo
+	// Location of the GraphRoot for the container.  Required for ensuring the
+	// volume has finished mounting when coming online at boot.
+	GraphRoot string
+	// Location of the RunRoot for the container.  Required for ensuring the tmpfs
+	// or volume exists and is mounted when coming online at boot.
+	RunRoot string
 }
 
 const containerTemplate = headerTemplate + `
@@ -132,6 +138,21 @@ func generateContainerInfo(ctr *libpod.Container, options entities.GenerateSyste
 
 	nameOrID, serviceName := containerServiceName(ctr, options)
 
+	store := ctr.Runtime().GetStore()
+	if store == nil {
+		return nil, errors.Errorf("could not determine storage store for container")
+	}
+
+	graphRoot := store.GraphRoot()
+	if graphRoot == "" {
+		return nil, errors.Errorf("could not lookup container's graphroot: got empty string")
+	}
+
+	runRoot := store.RunRoot()
+	if runRoot == "" {
+		return nil, errors.Errorf("could not lookup container's runroot: got empty string")
+	}
+
 	info := containerInfo{
 		ServiceName:       serviceName,
 		ContainerNameOrID: nameOrID,
@@ -140,6 +161,8 @@ func generateContainerInfo(ctr *libpod.Container, options entities.GenerateSyste
 		StopTimeout:       timeout,
 		GenerateTimestamp: true,
 		CreateCommand:     createCommand,
+		GraphRoot:         graphRoot,
+		RunRoot:           runRoot,
 	}
 
 	return &info, nil
diff --git a/pkg/systemd/generate/containers_test.go b/pkg/systemd/generate/containers_test.go
index ebbbdb786..1359c1a37 100644
--- a/pkg/systemd/generate/containers_test.go
+++ b/pkg/systemd/generate/containers_test.go
@@ -48,6 +48,7 @@ Description=Podman container-639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -73,6 +74,7 @@ Description=Podman container-foobar.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -96,6 +98,7 @@ Description=Podman container-foobar.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 BindsTo=a.service b.service c.service pod.service
 After=a.service b.service c.service pod.service
 
@@ -121,6 +124,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -145,6 +149,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -169,6 +174,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -193,6 +199,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -217,6 +224,7 @@ Description=Podman container-639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -242,6 +250,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -270,6 +279,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -294,6 +304,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -318,6 +329,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -342,6 +354,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -366,6 +379,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -400,6 +414,8 @@ WantedBy=multi-user.target default.target
 				StopTimeout:       22,
 				PodmanVersion:     "CI",
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodID,
 			false,
@@ -416,6 +432,8 @@ WantedBy=multi-user.target default.target
 				StopTimeout:       22,
 				PodmanVersion:     "CI",
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodIDNoHeaderInfo,
 			false,
@@ -432,6 +450,8 @@ WantedBy=multi-user.target default.target
 				StopTimeout:       10,
 				PodmanVersion:     "CI",
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodName,
 			false,
@@ -449,6 +469,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				BoundToServices:   []string{"pod", "a", "b", "c"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNameBoundTo,
 			false,
@@ -464,6 +486,8 @@ WantedBy=multi-user.target default.target
 				StopTimeout:   10,
 				PodmanVersion: "CI",
 				EnvVariable:   define.EnvVariable,
+				GraphRoot:     "/var/lib/containers/storage",
+				RunRoot:       "/var/run/containers/storage",
 			},
 			"",
 			false,
@@ -481,6 +505,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "container", "run", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN", "foo=arg \"with \" space"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodWithNameAndGeneric,
 			true,
@@ -498,6 +524,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "-d", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodWithExplicitShortDetachParam,
 			true,
@@ -515,6 +543,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "-d", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 				Pod: &podInfo{
 					PodIDFile: "%t/pod-foobar.pod-id-file",
 				},
@@ -535,6 +565,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--detach", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNameNewDetach,
 			true,
@@ -552,6 +584,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodIDNew,
 			true,
@@ -569,6 +603,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--detach=true", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			genGoodNewDetach("--detach=true"),
 			true,
@@ -586,6 +622,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--detach=false", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			genGoodNewDetach("-d"),
 			true,
@@ -603,6 +641,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--name", "test", "-p", "80:80", "--detach=false", "awesome-image:latest", "somecmd", "--detach=false"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNameNewDetachFalseWithCmd,
 			true,
@@ -620,6 +660,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--name", "test", "-p", "80:80", "--detach=false", "--detach=false", "awesome-image:latest", "somecmd", "--detach=false"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNameNewDetachFalseWithCmd,
 			true,
@@ -637,6 +679,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "-dti", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			genGoodNewDetach("-dti"),
 			true,
@@ -654,6 +698,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "-tid", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			genGoodNewDetach("-tid"),
 			true,
@@ -671,6 +717,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "--events-backend", "none", "--runroot", "/root", "run", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNewRootFlags,
 			true,
@@ -688,6 +736,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "container", "create", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodContainerCreate,
 			true,
@@ -705,6 +755,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "create", "--name", "test", "--log-driver=journald", "--log-opt=tag={{.Name}}", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNewWithJournaldTag,
 			true,
@@ -722,6 +774,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "create", "--name", "test", "awesome-image:latest", "sh", "-c", "kill $$ && echo %\\"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNewWithSpecialChars,
 			true,
diff --git a/pkg/systemd/generate/pods.go b/pkg/systemd/generate/pods.go
index f96058d36..a76979ecf 100644
--- a/pkg/systemd/generate/pods.go
+++ b/pkg/systemd/generate/pods.go
@@ -73,6 +73,12 @@ type podInfo struct {
 	ExecStopPost string
 	// Removes autogenerated by Podman and timestamp if set to true
 	GenerateNoHeader bool
+	// Location of the GraphRoot for the pod.  Required for ensuring the
+	// volume has finished mounting when coming online at boot.
+	GraphRoot string
+	// Location of the RunRoot for the pod.  Required for ensuring the tmpfs
+	// or volume exists and is mounted when coming online at boot.
+	RunRoot string
 }
 
 const podTemplate = headerTemplate + `Requires={{{{- range $index, $value := .RequiredServices -}}}}{{{{if $index}}}} {{{{end}}}}{{{{ $value }}}}.service{{{{end}}}}
diff --git a/pkg/systemd/generate/pods_test.go b/pkg/systemd/generate/pods_test.go
index 50c8d4556..559f7365f 100644
--- a/pkg/systemd/generate/pods_test.go
+++ b/pkg/systemd/generate/pods_test.go
@@ -47,6 +47,7 @@ Description=Podman pod-123abc.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 Requires=container-1.service container-2.service
 Before=container-1.service container-2.service
 
@@ -74,6 +75,7 @@ Description=Podman pod-123abc.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 Requires=container-1.service container-2.service
 Before=container-1.service container-2.service
 
@@ -101,6 +103,7 @@ Description=Podman pod-123abc.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 Requires=container-1.service container-2.service
 Before=container-1.service container-2.service
 
@@ -128,6 +131,7 @@ Description=Podman pod-123abc.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 Requires=container-1.service container-2.service
 Before=container-1.service container-2.service
 
@@ -155,6 +159,7 @@ Description=Podman pod-123abc.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 Requires=container-1.service container-2.service
 Before=container-1.service container-2.service
 
@@ -191,6 +196,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      42,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "pod", "create", "--name", "foo", "bar=arg with space"},
 			},
@@ -208,6 +215,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      42,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "pod", "create", "--name", "foo", "bar=arg with space"},
 			},
@@ -225,6 +234,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      42,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "--events-backend", "none", "--runroot", "/root", "pod", "create", "--name", "foo", "bar=arg with space"},
 			},
@@ -242,6 +253,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      10,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "pod", "create", "--name", "foo", "bar=arg with space"},
 			},
@@ -259,6 +272,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      10,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "--events-backend", "none", "--runroot", "/root", "pod", "create", "--name", "foo", "bar=arg with space"},
 			},
@@ -276,6 +291,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      10,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "pod", "create", "--name", "foo", "--replace=false"},
 			},
@@ -293,6 +310,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      10,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "pod", "create", "--name", "foo", "--label", "key={{someval}}"},
 			},