Add RequiresMountsFor= to systemd generate

It is rare but possible that storage locations for the graphroot and the
runroot are not mounted at boot time, and therefore might race when
doing container operations.  An example we've seen in the wild is that a
slow tmpfs mount for the runroot would suddenly mount over /run, causing
the container to lose all currently-running data, requiring a system
refresh to get it back.

This patch adds RequiresMountsFor= to the systemd.unit header to ensure
the paths for both the graphroot and runroot are mounted prior to
starting any generated unit files.

Signed-off-by: Robb Manes <rmanes@redhat.com>

5 files changed, 103 insertions, 0 deletions

diff --git a/pkg/systemd/generate/common.go b/pkg/systemd/generate/common.go
index 94a6f4cb5..19d468403 100644
--- a/pkg/systemd/generate/common.go
+++ b/pkg/systemd/generate/common.go
@@ -36,6 +36,7 @@ Description=Podman {{{{.ServiceName}}}}.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor={{{{.GraphRoot}}}} {{{{.RunRoot}}}}
 `
 
 // filterPodFlags removes --pod and --pod-id-file from the specified command.
diff --git a/pkg/systemd/generate/containers.go b/pkg/systemd/generate/containers.go
index 9343a5067..bc13a6116 100644
--- a/pkg/systemd/generate/containers.go
+++ b/pkg/systemd/generate/containers.go
@@ -71,6 +71,12 @@ type containerInfo struct {
 	// If not nil, the container is part of the pod.  We can use the
 	// podInfo to extract the relevant data.
 	Pod *podInfo
+	// Location of the GraphRoot for the container.  Required for ensuring the
+	// volume has finished mounting when coming online at boot.
+	GraphRoot string
+	// Location of the RunRoot for the container.  Required for ensuring the tmpfs
+	// or volume exists and is mounted when coming online at boot.
+	RunRoot string
 }
 
 const containerTemplate = headerTemplate + `
@@ -132,6 +138,21 @@ func generateContainerInfo(ctr *libpod.Container, options entities.GenerateSyste
 
 	nameOrID, serviceName := containerServiceName(ctr, options)
 
+	store := ctr.Runtime().GetStore()
+	if store == nil {
+		return nil, errors.Errorf("could not determine storage store for container")
+	}
+
+	graphRoot := store.GraphRoot()
+	if graphRoot == "" {
+		return nil, errors.Errorf("could not lookup container's graphroot: got empty string")
+	}
+
+	runRoot := store.RunRoot()
+	if runRoot == "" {
+		return nil, errors.Errorf("could not lookup container's runroot: got empty string")
+	}
+
 	info := containerInfo{
 		ServiceName:       serviceName,
 		ContainerNameOrID: nameOrID,
@@ -140,6 +161,8 @@ func generateContainerInfo(ctr *libpod.Container, options entities.GenerateSyste
 		StopTimeout:       timeout,
 		GenerateTimestamp: true,
 		CreateCommand:     createCommand,
+		GraphRoot:         graphRoot,
+		RunRoot:           runRoot,
 	}
 
 	return &info, nil
diff --git a/pkg/systemd/generate/containers_test.go b/pkg/systemd/generate/containers_test.go
index ebbbdb786..1359c1a37 100644
--- a/pkg/systemd/generate/containers_test.go
+++ b/pkg/systemd/generate/containers_test.go
@@ -48,6 +48,7 @@ Description=Podman container-639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -73,6 +74,7 @@ Description=Podman container-foobar.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -96,6 +98,7 @@ Description=Podman container-foobar.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 BindsTo=a.service b.service c.service pod.service
 After=a.service b.service c.service pod.service
 
@@ -121,6 +124,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -145,6 +149,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -169,6 +174,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -193,6 +199,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -217,6 +224,7 @@ Description=Podman container-639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -242,6 +250,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -270,6 +279,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -294,6 +304,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -318,6 +329,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -342,6 +354,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -366,6 +379,7 @@ Description=Podman jadda-jadda.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 
 [Service]
 Environment=PODMAN_SYSTEMD_UNIT=%n
@@ -400,6 +414,8 @@ WantedBy=multi-user.target default.target
 				StopTimeout:       22,
 				PodmanVersion:     "CI",
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodID,
 			false,
@@ -416,6 +432,8 @@ WantedBy=multi-user.target default.target
 				StopTimeout:       22,
 				PodmanVersion:     "CI",
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodIDNoHeaderInfo,
 			false,
@@ -432,6 +450,8 @@ WantedBy=multi-user.target default.target
 				StopTimeout:       10,
 				PodmanVersion:     "CI",
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodName,
 			false,
@@ -449,6 +469,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				BoundToServices:   []string{"pod", "a", "b", "c"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNameBoundTo,
 			false,
@@ -464,6 +486,8 @@ WantedBy=multi-user.target default.target
 				StopTimeout:   10,
 				PodmanVersion: "CI",
 				EnvVariable:   define.EnvVariable,
+				GraphRoot:     "/var/lib/containers/storage",
+				RunRoot:       "/var/run/containers/storage",
 			},
 			"",
 			false,
@@ -481,6 +505,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "container", "run", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN", "foo=arg \"with \" space"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodWithNameAndGeneric,
 			true,
@@ -498,6 +524,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "-d", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodWithExplicitShortDetachParam,
 			true,
@@ -515,6 +543,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "-d", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 				Pod: &podInfo{
 					PodIDFile: "%t/pod-foobar.pod-id-file",
 				},
@@ -535,6 +565,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--detach", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNameNewDetach,
 			true,
@@ -552,6 +584,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodIDNew,
 			true,
@@ -569,6 +603,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--detach=true", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			genGoodNewDetach("--detach=true"),
 			true,
@@ -586,6 +622,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--detach=false", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			genGoodNewDetach("-d"),
 			true,
@@ -603,6 +641,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--name", "test", "-p", "80:80", "--detach=false", "awesome-image:latest", "somecmd", "--detach=false"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNameNewDetachFalseWithCmd,
 			true,
@@ -620,6 +660,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "--name", "test", "-p", "80:80", "--detach=false", "--detach=false", "awesome-image:latest", "somecmd", "--detach=false"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNameNewDetachFalseWithCmd,
 			true,
@@ -637,6 +679,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "-dti", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			genGoodNewDetach("-dti"),
 			true,
@@ -654,6 +698,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "run", "-tid", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			genGoodNewDetach("-tid"),
 			true,
@@ -671,6 +717,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "--events-backend", "none", "--runroot", "/root", "run", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNewRootFlags,
 			true,
@@ -688,6 +736,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "container", "create", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodContainerCreate,
 			true,
@@ -705,6 +755,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "create", "--name", "test", "--log-driver=journald", "--log-opt=tag={{.Name}}", "awesome-image:latest"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNewWithJournaldTag,
 			true,
@@ -722,6 +774,8 @@ WantedBy=multi-user.target default.target
 				PodmanVersion:     "CI",
 				CreateCommand:     []string{"I'll get stripped", "create", "--name", "test", "awesome-image:latest", "sh", "-c", "kill $$ && echo %\\"},
 				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
 			},
 			goodNewWithSpecialChars,
 			true,
diff --git a/pkg/systemd/generate/pods.go b/pkg/systemd/generate/pods.go
index f96058d36..a76979ecf 100644
--- a/pkg/systemd/generate/pods.go
+++ b/pkg/systemd/generate/pods.go
@@ -73,6 +73,12 @@ type podInfo struct {
 	ExecStopPost string
 	// Removes autogenerated by Podman and timestamp if set to true
 	GenerateNoHeader bool
+	// Location of the GraphRoot for the pod.  Required for ensuring the
+	// volume has finished mounting when coming online at boot.
+	GraphRoot string
+	// Location of the RunRoot for the pod.  Required for ensuring the tmpfs
+	// or volume exists and is mounted when coming online at boot.
+	RunRoot string
 }
 
 const podTemplate = headerTemplate + `Requires={{{{- range $index, $value := .RequiredServices -}}}}{{{{if $index}}}} {{{{end}}}}{{{{ $value }}}}.service{{{{end}}}}
diff --git a/pkg/systemd/generate/pods_test.go b/pkg/systemd/generate/pods_test.go
index 50c8d4556..559f7365f 100644
--- a/pkg/systemd/generate/pods_test.go
+++ b/pkg/systemd/generate/pods_test.go
@@ -47,6 +47,7 @@ Description=Podman pod-123abc.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 Requires=container-1.service container-2.service
 Before=container-1.service container-2.service
 
@@ -74,6 +75,7 @@ Description=Podman pod-123abc.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 Requires=container-1.service container-2.service
 Before=container-1.service container-2.service
 
@@ -101,6 +103,7 @@ Description=Podman pod-123abc.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 Requires=container-1.service container-2.service
 Before=container-1.service container-2.service
 
@@ -128,6 +131,7 @@ Description=Podman pod-123abc.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 Requires=container-1.service container-2.service
 Before=container-1.service container-2.service
 
@@ -155,6 +159,7 @@ Description=Podman pod-123abc.service
 Documentation=man:podman-generate-systemd(1)
 Wants=network.target
 After=network-online.target
+RequiresMountsFor=/var/lib/containers/storage /var/run/containers/storage
 Requires=container-1.service container-2.service
 Before=container-1.service container-2.service
 
@@ -191,6 +196,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      42,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "pod", "create", "--name", "foo", "bar=arg with space"},
 			},
@@ -208,6 +215,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      42,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "pod", "create", "--name", "foo", "bar=arg with space"},
 			},
@@ -225,6 +234,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      42,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "--events-backend", "none", "--runroot", "/root", "pod", "create", "--name", "foo", "bar=arg with space"},
 			},
@@ -242,6 +253,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      10,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "pod", "create", "--name", "foo", "bar=arg with space"},
 			},
@@ -259,6 +272,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      10,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "--events-backend", "none", "--runroot", "/root", "pod", "create", "--name", "foo", "bar=arg with space"},
 			},
@@ -276,6 +291,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      10,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "pod", "create", "--name", "foo", "--replace=false"},
 			},
@@ -293,6 +310,8 @@ WantedBy=multi-user.target default.target
 				PIDFile:          "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
 				StopTimeout:      10,
 				PodmanVersion:    "CI",
+				GraphRoot:        "/var/lib/containers/storage",
+				RunRoot:          "/var/run/containers/storage",
 				RequiredServices: []string{"container-1", "container-2"},
 				CreateCommand:    []string{"podman", "pod", "create", "--name", "foo", "--label", "key={{someval}}"},
 			},