auto-update: simple rollback

Add support for simple rollbacks during `podman auto-update`.  Rollbacks
are enabled by default.  If a systemd unit cannot be restarted after an
update, the previous image will be retagged and the unit will be
restarted a second time.

Add system tests for rollbacks.  Also fix a bug in the restart sequence;
we have to use the channel to actually know whether the restart was
successful or not.

NOTE: To make rollbacks really useful, users must run their containers
with `--sdnotify=container` such that the containers send the ready
message over the (mounted) socket.  This way, restarting the systemd
units during auto update will block until the message has been received
(or a timeout kicked in).

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

3 files changed, 64 insertions, 17 deletions

diff --git a/pkg/systemd/generate/common.go b/pkg/systemd/generate/common.go
index 349805980..45e12014a 100644
--- a/pkg/systemd/generate/common.go
+++ b/pkg/systemd/generate/common.go
@@ -74,8 +74,7 @@ func filterCommonContainerFlags(command []string, argCount int) []string {
 		case s == "--sdnotify", s == "--cgroups":
 			i++
 			continue
-		case strings.HasPrefix(s, "--sdnotify="),
-			strings.HasPrefix(s, "--rm="),
+		case strings.HasPrefix(s, "--rm="),
 			strings.HasPrefix(s, "--cgroups="):
 			continue
 		}
diff --git a/pkg/systemd/generate/containers.go b/pkg/systemd/generate/containers.go
index 083520316..78b81b54b 100644
--- a/pkg/systemd/generate/containers.go
+++ b/pkg/systemd/generate/containers.go
@@ -258,7 +258,6 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst
 		}
 		startCommand = append(startCommand,
 			"run",
-			"--sdnotify=conmon",
 			"--cgroups=no-conmon",
 			"--rm",
 		)
@@ -273,6 +272,7 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst
 		fs.String("name", "", "")
 		fs.Bool("replace", false, "")
 		fs.StringArrayP("env", "e", nil, "")
+		fs.String("sdnotify", "", "")
 		fs.Parse(remainingCmd)
 
 		remainingCmd = filterCommonContainerFlags(remainingCmd, fs.NArg())
@@ -294,6 +294,13 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst
 			return "", err
 		}
 
+		// Default to --sdnotify=conmon unless already set by the
+		// container.
+		hasSdnotifyParam := fs.Lookup("sdnotify").Changed
+		if !hasSdnotifyParam {
+			startCommand = append(startCommand, "--sdnotify=conmon")
+		}
+
 		if !hasDetachParam {
 			// Enforce detaching
 			//
diff --git a/pkg/systemd/generate/containers_test.go b/pkg/systemd/generate/containers_test.go
index 1d24cc4a9..6141950d0 100644
--- a/pkg/systemd/generate/containers_test.go
+++ b/pkg/systemd/generate/containers_test.go
@@ -130,7 +130,29 @@ RequiresMountsFor=/var/run/containers/storage
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 TimeoutStopSec=70
-ExecStart=/usr/bin/podman container run --sdnotify=conmon --cgroups=no-conmon --rm -d --replace --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN "foo=arg \"with \" space"
+ExecStart=/usr/bin/podman container run --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN "foo=arg \"with \" space"
+Type=notify
+NotifyAccess=all
+
+[Install]
+WantedBy=multi-user.target default.target
+`
+
+	goodWithNameAndSdnotify := `# jadda-jadda.service
+# autogenerated by Podman CI
+
+[Unit]
+Description=Podman jadda-jadda.service
+Documentation=man:podman-generate-systemd(1)
+Wants=network-online.target
+After=network-online.target
+RequiresMountsFor=/var/run/containers/storage
+
+[Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
+Restart=always
+TimeoutStopSec=70
+ExecStart=/usr/bin/podman container run --cgroups=no-conmon --rm -d --replace --sdnotify=container --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN "foo=arg \"with \" space"
 Type=notify
 NotifyAccess=all
 
@@ -152,7 +174,7 @@ RequiresMountsFor=/var/run/containers/storage
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm --replace -d --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon --replace -d --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
 Type=notify
 NotifyAccess=all
 
@@ -174,7 +196,7 @@ RequiresMountsFor=/var/run/containers/storage
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm --pod-id-file %t/pod-foobar.pod-id-file --replace -d --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --pod-id-file %t/pod-foobar.pod-id-file --sdnotify=conmon --replace -d --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
 Type=notify
 NotifyAccess=all
 
@@ -196,7 +218,7 @@ RequiresMountsFor=/var/run/containers/storage
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm --replace --detach --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon --replace --detach --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
 Type=notify
 NotifyAccess=all
 
@@ -218,7 +240,7 @@ RequiresMountsFor=/var/run/containers/storage
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm -d awesome-image:latest
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d awesome-image:latest
 Type=notify
 NotifyAccess=all
 
@@ -241,7 +263,7 @@ RequiresMountsFor=/var/run/containers/storage
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 TimeoutStopSec=102
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm ` +
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon ` +
 			detachparam +
 			` awesome-image:latest
 Type=notify
@@ -267,7 +289,7 @@ RequiresMountsFor=/var/run/containers/storage
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 TimeoutStopSec=102
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm -d --replace --name test -p 80:80 awesome-image:latest somecmd --detach=false
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name test -p 80:80 awesome-image:latest somecmd --detach=false
 Type=notify
 NotifyAccess=all
 
@@ -289,7 +311,7 @@ RequiresMountsFor=/var/run/containers/storage
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 TimeoutStopSec=102
-ExecStart=/usr/bin/podman --events-backend none --runroot /root run --sdnotify=conmon --cgroups=no-conmon --rm -d awesome-image:latest
+ExecStart=/usr/bin/podman --events-backend none --runroot /root run --cgroups=no-conmon --rm --sdnotify=conmon -d awesome-image:latest
 Type=notify
 NotifyAccess=all
 
@@ -311,7 +333,7 @@ RequiresMountsFor=/var/run/containers/storage
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 TimeoutStopSec=70
-ExecStart=/usr/bin/podman container run --sdnotify=conmon --cgroups=no-conmon --rm -d awesome-image:latest
+ExecStart=/usr/bin/podman container run --cgroups=no-conmon --rm --sdnotify=conmon -d awesome-image:latest
 Type=notify
 NotifyAccess=all
 
@@ -333,7 +355,7 @@ RequiresMountsFor=/var/run/containers/storage
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm -d --replace --name test --log-driver=journald --log-opt=tag={{.Name}} awesome-image:latest
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name test --log-driver=journald --log-opt=tag={{.Name}} awesome-image:latest
 Type=notify
 NotifyAccess=all
 
@@ -355,7 +377,7 @@ RequiresMountsFor=/var/run/containers/storage
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm -d --replace --name test awesome-image:latest sh -c "kill $$$$ && echo %%\\"
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name test awesome-image:latest sh -c "kill $$$$ && echo %%\\"
 Type=notify
 NotifyAccess=all
 
@@ -377,7 +399,7 @@ RequiresMountsFor=/var/run/containers/storage
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm -d --conmon-pidfile=foo --cidfile=foo awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo alpine
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --conmon-pidfile=foo --cidfile=foo awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo alpine
 Type=notify
 NotifyAccess=all
 
@@ -399,7 +421,7 @@ RequiresMountsFor=/var/run/containers/storage
 Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm --pod-id-file %t/pod-foobar.pod-id-file -d --conmon-pidfile=foo --cidfile=foo awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo --pod-id-file /tmp/pod-foobar.pod-id-file alpine
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --pod-id-file %t/pod-foobar.pod-id-file --sdnotify=conmon -d --conmon-pidfile=foo --cidfile=foo awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo --pod-id-file /tmp/pod-foobar.pod-id-file alpine
 Type=notify
 NotifyAccess=all
 
@@ -422,7 +444,7 @@ Environment=PODMAN_SYSTEMD_UNIT=%n
 Environment=FOO=abc "BAR=my test" USER=%%a
 Restart=always
 TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm -d --env FOO --env=BAR --env=MYENV=2 -e USER awesome-image:latest
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --env FOO --env=BAR --env=MYENV=2 -e USER awesome-image:latest
 Type=notify
 NotifyAccess=all
 
@@ -547,6 +569,25 @@ WantedBy=multi-user.target default.target
 			false,
 			false,
 		},
+		{"good with name and sdnotify",
+			containerInfo{
+				Executable:        "/usr/bin/podman",
+				ServiceName:       "jadda-jadda",
+				ContainerNameOrID: "jadda-jadda",
+				RestartPolicy:     "always",
+				PIDFile:           "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
+				StopTimeout:       10,
+				PodmanVersion:     "CI",
+				CreateCommand:     []string{"I'll get stripped", "container", "run", "--sdnotify=container", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN", "foo=arg \"with \" space"},
+				EnvVariable:       define.EnvVariable,
+				GraphRoot:         "/var/lib/containers/storage",
+				RunRoot:           "/var/run/containers/storage",
+			},
+			goodWithNameAndSdnotify,
+			true,
+			false,
+			false,
+		},
 		{"good with explicit short detach param",
 			containerInfo{
 				Executable:        "/usr/bin/podman",