summaryrefslogtreecommitdiff
path: root/pkg
diff options
context:
space:
mode:
authorValentin Rothberg <rothberg@redhat.com>2021-07-28 15:19:04 +0200
committerValentin Rothberg <rothberg@redhat.com>2021-08-05 15:20:38 +0200
commit30df551bde460f4f37d6dbd373701873fa5353dc (patch)
treea1a48d3b25706006ac391ac271ed21c3a6191c8f /pkg
parent117583c293713f2baa920c4035e820ad59fe6622 (diff)
downloadpodman-30df551bde460f4f37d6dbd373701873fa5353dc.tar.gz
podman-30df551bde460f4f37d6dbd373701873fa5353dc.tar.bz2
podman-30df551bde460f4f37d6dbd373701873fa5353dc.zip
auto-update: simple rollback
Add support for simple rollbacks during `podman auto-update`. Rollbacks are enabled by default. If a systemd unit cannot be restarted after an update, the previous image will be retagged and the unit will be restarted a second time. Add system tests for rollbacks. Also fix a bug in the restart sequence; we have to use the channel to actually know whether the restart was successful or not. NOTE: To make rollbacks really useful, users must run their containers with `--sdnotify=container` such that the containers send the ready message over the (mounted) socket. This way, restarting the systemd units during auto update will block until the message has been received (or a timeout kicked in). Signed-off-by: Valentin Rothberg <rothberg@redhat.com>
Diffstat (limited to 'pkg')
-rw-r--r--pkg/autoupdate/autoupdate.go80
-rw-r--r--pkg/domain/entities/auto-update.go3
-rw-r--r--pkg/systemd/generate/common.go3
-rw-r--r--pkg/systemd/generate/containers.go9
-rw-r--r--pkg/systemd/generate/containers_test.go69
5 files changed, 126 insertions, 38 deletions
diff --git a/pkg/autoupdate/autoupdate.go b/pkg/autoupdate/autoupdate.go
index c51e2cd03..b1ebbfa8e 100644
--- a/pkg/autoupdate/autoupdate.go
+++ b/pkg/autoupdate/autoupdate.go
@@ -88,7 +88,7 @@ func ValidateImageReference(imageName string) error {
} else if err != nil {
repo, err := reference.Parse(imageName)
if err != nil {
- return errors.Wrap(err, "error enforcing fully-qualified docker transport reference for auto updates")
+ return errors.Wrap(err, "enforcing fully-qualified docker transport reference for auto updates")
}
if _, ok := repo.(reference.NamedTagged); !ok {
return errors.Errorf("auto updates require fully-qualified image references (no tag): %q", imageName)
@@ -181,13 +181,13 @@ func autoUpdateRegistry(ctx context.Context, image *libimage.Image, ctr *libpod.
cid := ctr.ID()
rawImageName := ctr.RawImageName()
if rawImageName == "" {
- return nil, errors.Errorf("error registry auto-updating container %q: raw-image name is empty", cid)
+ return nil, errors.Errorf("registry auto-updating container %q: raw-image name is empty", cid)
}
labels := ctr.Labels()
unit, exists := labels[systemdDefine.EnvVariable]
if !exists {
- return nil, errors.Errorf("error auto-updating container %q: no %s label found", ctr.ID(), systemdDefine.EnvVariable)
+ return nil, errors.Errorf("auto-updating container %q: no %s label found", ctr.ID(), systemdDefine.EnvVariable)
}
report := &entities.AutoUpdateReport{
@@ -201,7 +201,7 @@ func autoUpdateRegistry(ctx context.Context, image *libimage.Image, ctr *libpod.
if _, updated := updatedRawImages[rawImageName]; updated {
logrus.Infof("Auto-updating container %q using registry image %q", cid, rawImageName)
- if err := restartSystemdUnit(ctr, unit, conn); err != nil {
+ if err := restartSystemdUnit(ctx, ctr, unit, conn); err != nil {
return report, err
}
report.Updated = "true"
@@ -211,7 +211,7 @@ func autoUpdateRegistry(ctx context.Context, image *libimage.Image, ctr *libpod.
authfile := getAuthfilePath(ctr, options)
needsUpdate, err := newerRemoteImageAvailable(ctx, runtime, image, rawImageName, authfile)
if err != nil {
- return report, errors.Wrapf(err, "error registry auto-updating container %q: image check for %q failed", cid, rawImageName)
+ return report, errors.Wrapf(err, "registry auto-updating container %q: image check for %q failed", cid, rawImageName)
}
if !needsUpdate {
@@ -225,16 +225,30 @@ func autoUpdateRegistry(ctx context.Context, image *libimage.Image, ctr *libpod.
}
if _, err := updateImage(ctx, runtime, rawImageName, options); err != nil {
- return report, errors.Wrapf(err, "error registry auto-updating container %q: image update for %q failed", cid, rawImageName)
+ return report, errors.Wrapf(err, "registry auto-updating container %q: image update for %q failed", cid, rawImageName)
}
updatedRawImages[rawImageName] = true
logrus.Infof("Auto-updating container %q using registry image %q", cid, rawImageName)
- if err := restartSystemdUnit(ctr, unit, conn); err != nil {
- return report, err
+ updateErr := restartSystemdUnit(ctx, ctr, unit, conn)
+ if updateErr == nil {
+ report.Updated = "true"
+ return report, nil
+ }
+
+ if !options.Rollback {
+ return report, updateErr
+ }
+
+ // To fallback, simply retag the old image and restart the service.
+ if err := image.Tag(rawImageName); err != nil {
+ return report, errors.Wrap(err, "falling back to previous image")
+ }
+ if err := restartSystemdUnit(ctx, ctr, unit, conn); err != nil {
+ return report, errors.Wrap(err, "restarting unit with old image during fallback")
}
- report.Updated = "true"
+ report.Updated = "rolled back"
return report, nil
}
@@ -243,13 +257,13 @@ func autoUpdateLocally(ctx context.Context, image *libimage.Image, ctr *libpod.C
cid := ctr.ID()
rawImageName := ctr.RawImageName()
if rawImageName == "" {
- return nil, errors.Errorf("error locally auto-updating container %q: raw-image name is empty", cid)
+ return nil, errors.Errorf("locally auto-updating container %q: raw-image name is empty", cid)
}
labels := ctr.Labels()
unit, exists := labels[systemdDefine.EnvVariable]
if !exists {
- return nil, errors.Errorf("error auto-updating container %q: no %s label found", ctr.ID(), systemdDefine.EnvVariable)
+ return nil, errors.Errorf("auto-updating container %q: no %s label found", ctr.ID(), systemdDefine.EnvVariable)
}
report := &entities.AutoUpdateReport{
@@ -263,7 +277,7 @@ func autoUpdateLocally(ctx context.Context, image *libimage.Image, ctr *libpod.C
needsUpdate, err := newerLocalImageAvailable(runtime, image, rawImageName)
if err != nil {
- return report, errors.Wrapf(err, "error locally auto-updating container %q: image check for %q failed", cid, rawImageName)
+ return report, errors.Wrapf(err, "locally auto-updating container %q: image check for %q failed", cid, rawImageName)
}
if !needsUpdate {
@@ -277,23 +291,47 @@ func autoUpdateLocally(ctx context.Context, image *libimage.Image, ctr *libpod.C
}
logrus.Infof("Auto-updating container %q using local image %q", cid, rawImageName)
- if err := restartSystemdUnit(ctr, unit, conn); err != nil {
- return report, err
+ updateErr := restartSystemdUnit(ctx, ctr, unit, conn)
+ if updateErr == nil {
+ report.Updated = "true"
+ return report, nil
}
- report.Updated = "true"
+ if !options.Rollback {
+ return report, updateErr
+ }
+
+ // To fallback, simply retag the old image and restart the service.
+ if err := image.Tag(rawImageName); err != nil {
+ return report, errors.Wrap(err, "falling back to previous image")
+ }
+ if err := restartSystemdUnit(ctx, ctr, unit, conn); err != nil {
+ return report, errors.Wrap(err, "restarting unit with old image during fallback")
+ }
+
+ report.Updated = "rolled back"
return report, nil
}
// restartSystemdUnit restarts the systemd unit the container is running in.
-func restartSystemdUnit(ctr *libpod.Container, unit string, conn *dbus.Conn) error {
- _, err := conn.RestartUnit(unit, "replace", nil)
- if err != nil {
- return errors.Wrapf(err, "error auto-updating container %q: restarting systemd unit %q failed", ctr.ID(), unit)
+func restartSystemdUnit(ctx context.Context, ctr *libpod.Container, unit string, conn *dbus.Conn) error {
+ restartChan := make(chan string)
+ if _, err := conn.RestartUnitContext(ctx, unit, "replace", restartChan); err != nil {
+ return errors.Wrapf(err, "auto-updating container %q: restarting systemd unit %q failed", ctr.ID(), unit)
}
- logrus.Infof("Successfully restarted systemd unit %q of container %q", unit, ctr.ID())
- return nil
+ // Wait for the restart to finish and actually check if it was
+ // successful or not.
+ result := <-restartChan
+
+ switch result {
+ case "done":
+ logrus.Infof("Successfully restarted systemd unit %q of container %q", unit, ctr.ID())
+ return nil
+
+ default:
+ return errors.Errorf("auto-updating container %q: restarting systemd unit %q failed: expected %q but received %q", ctr.ID(), unit, "done", result)
+ }
}
// imageContainersMap generates a map[image ID] -> [containers using the image]
diff --git a/pkg/domain/entities/auto-update.go b/pkg/domain/entities/auto-update.go
index eed617bf8..5ea2cdf15 100644
--- a/pkg/domain/entities/auto-update.go
+++ b/pkg/domain/entities/auto-update.go
@@ -8,6 +8,9 @@ type AutoUpdateOptions struct {
// pending, it will be indicated in the Updated field of
// AutoUpdateReport.
DryRun bool
+ // If restarting the service with the new image failed, restart it
+ // another time with the previous image.
+ Rollback bool
}
// AutoUpdateReport contains the results from running auto-update.
diff --git a/pkg/systemd/generate/common.go b/pkg/systemd/generate/common.go
index 349805980..45e12014a 100644
--- a/pkg/systemd/generate/common.go
+++ b/pkg/systemd/generate/common.go
@@ -74,8 +74,7 @@ func filterCommonContainerFlags(command []string, argCount int) []string {
case s == "--sdnotify", s == "--cgroups":
i++
continue
- case strings.HasPrefix(s, "--sdnotify="),
- strings.HasPrefix(s, "--rm="),
+ case strings.HasPrefix(s, "--rm="),
strings.HasPrefix(s, "--cgroups="):
continue
}
diff --git a/pkg/systemd/generate/containers.go b/pkg/systemd/generate/containers.go
index 083520316..78b81b54b 100644
--- a/pkg/systemd/generate/containers.go
+++ b/pkg/systemd/generate/containers.go
@@ -258,7 +258,6 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst
}
startCommand = append(startCommand,
"run",
- "--sdnotify=conmon",
"--cgroups=no-conmon",
"--rm",
)
@@ -273,6 +272,7 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst
fs.String("name", "", "")
fs.Bool("replace", false, "")
fs.StringArrayP("env", "e", nil, "")
+ fs.String("sdnotify", "", "")
fs.Parse(remainingCmd)
remainingCmd = filterCommonContainerFlags(remainingCmd, fs.NArg())
@@ -294,6 +294,13 @@ func executeContainerTemplate(info *containerInfo, options entities.GenerateSyst
return "", err
}
+ // Default to --sdnotify=conmon unless already set by the
+ // container.
+ hasSdnotifyParam := fs.Lookup("sdnotify").Changed
+ if !hasSdnotifyParam {
+ startCommand = append(startCommand, "--sdnotify=conmon")
+ }
+
if !hasDetachParam {
// Enforce detaching
//
diff --git a/pkg/systemd/generate/containers_test.go b/pkg/systemd/generate/containers_test.go
index 1d24cc4a9..6141950d0 100644
--- a/pkg/systemd/generate/containers_test.go
+++ b/pkg/systemd/generate/containers_test.go
@@ -130,7 +130,29 @@ RequiresMountsFor=/var/run/containers/storage
Environment=PODMAN_SYSTEMD_UNIT=%n
Restart=always
TimeoutStopSec=70
-ExecStart=/usr/bin/podman container run --sdnotify=conmon --cgroups=no-conmon --rm -d --replace --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN "foo=arg \"with \" space"
+ExecStart=/usr/bin/podman container run --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN "foo=arg \"with \" space"
+Type=notify
+NotifyAccess=all
+
+[Install]
+WantedBy=multi-user.target default.target
+`
+
+ goodWithNameAndSdnotify := `# jadda-jadda.service
+# autogenerated by Podman CI
+
+[Unit]
+Description=Podman jadda-jadda.service
+Documentation=man:podman-generate-systemd(1)
+Wants=network-online.target
+After=network-online.target
+RequiresMountsFor=/var/run/containers/storage
+
+[Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
+Restart=always
+TimeoutStopSec=70
+ExecStart=/usr/bin/podman container run --cgroups=no-conmon --rm -d --replace --sdnotify=container --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN "foo=arg \"with \" space"
Type=notify
NotifyAccess=all
@@ -152,7 +174,7 @@ RequiresMountsFor=/var/run/containers/storage
Environment=PODMAN_SYSTEMD_UNIT=%n
Restart=always
TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm --replace -d --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon --replace -d --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
Type=notify
NotifyAccess=all
@@ -174,7 +196,7 @@ RequiresMountsFor=/var/run/containers/storage
Environment=PODMAN_SYSTEMD_UNIT=%n
Restart=always
TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm --pod-id-file %t/pod-foobar.pod-id-file --replace -d --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --pod-id-file %t/pod-foobar.pod-id-file --sdnotify=conmon --replace -d --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
Type=notify
NotifyAccess=all
@@ -196,7 +218,7 @@ RequiresMountsFor=/var/run/containers/storage
Environment=PODMAN_SYSTEMD_UNIT=%n
Restart=always
TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm --replace --detach --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon --replace --detach --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
Type=notify
NotifyAccess=all
@@ -218,7 +240,7 @@ RequiresMountsFor=/var/run/containers/storage
Environment=PODMAN_SYSTEMD_UNIT=%n
Restart=always
TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm -d awesome-image:latest
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d awesome-image:latest
Type=notify
NotifyAccess=all
@@ -241,7 +263,7 @@ RequiresMountsFor=/var/run/containers/storage
Environment=PODMAN_SYSTEMD_UNIT=%n
Restart=always
TimeoutStopSec=102
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm ` +
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon ` +
detachparam +
` awesome-image:latest
Type=notify
@@ -267,7 +289,7 @@ RequiresMountsFor=/var/run/containers/storage
Environment=PODMAN_SYSTEMD_UNIT=%n
Restart=always
TimeoutStopSec=102
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm -d --replace --name test -p 80:80 awesome-image:latest somecmd --detach=false
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name test -p 80:80 awesome-image:latest somecmd --detach=false
Type=notify
NotifyAccess=all
@@ -289,7 +311,7 @@ RequiresMountsFor=/var/run/containers/storage
Environment=PODMAN_SYSTEMD_UNIT=%n
Restart=always
TimeoutStopSec=102
-ExecStart=/usr/bin/podman --events-backend none --runroot /root run --sdnotify=conmon --cgroups=no-conmon --rm -d awesome-image:latest
+ExecStart=/usr/bin/podman --events-backend none --runroot /root run --cgroups=no-conmon --rm --sdnotify=conmon -d awesome-image:latest
Type=notify
NotifyAccess=all
@@ -311,7 +333,7 @@ RequiresMountsFor=/var/run/containers/storage
Environment=PODMAN_SYSTEMD_UNIT=%n
Restart=always
TimeoutStopSec=70
-ExecStart=/usr/bin/podman container run --sdnotify=conmon --cgroups=no-conmon --rm -d awesome-image:latest
+ExecStart=/usr/bin/podman container run --cgroups=no-conmon --rm --sdnotify=conmon -d awesome-image:latest
Type=notify
NotifyAccess=all
@@ -333,7 +355,7 @@ RequiresMountsFor=/var/run/containers/storage
Environment=PODMAN_SYSTEMD_UNIT=%n
Restart=always
TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm -d --replace --name test --log-driver=journald --log-opt=tag={{.Name}} awesome-image:latest
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name test --log-driver=journald --log-opt=tag={{.Name}} awesome-image:latest
Type=notify
NotifyAccess=all
@@ -355,7 +377,7 @@ RequiresMountsFor=/var/run/containers/storage
Environment=PODMAN_SYSTEMD_UNIT=%n
Restart=always
TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm -d --replace --name test awesome-image:latest sh -c "kill $$$$ && echo %%\\"
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --replace --name test awesome-image:latest sh -c "kill $$$$ && echo %%\\"
Type=notify
NotifyAccess=all
@@ -377,7 +399,7 @@ RequiresMountsFor=/var/run/containers/storage
Environment=PODMAN_SYSTEMD_UNIT=%n
Restart=always
TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm -d --conmon-pidfile=foo --cidfile=foo awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo alpine
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --conmon-pidfile=foo --cidfile=foo awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo alpine
Type=notify
NotifyAccess=all
@@ -399,7 +421,7 @@ RequiresMountsFor=/var/run/containers/storage
Environment=PODMAN_SYSTEMD_UNIT=%n
Restart=always
TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm --pod-id-file %t/pod-foobar.pod-id-file -d --conmon-pidfile=foo --cidfile=foo awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo --pod-id-file /tmp/pod-foobar.pod-id-file alpine
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --pod-id-file %t/pod-foobar.pod-id-file --sdnotify=conmon -d --conmon-pidfile=foo --cidfile=foo awesome-image:latest podman run --cgroups=foo --conmon-pidfile=foo --cidfile=foo --pod-id-file /tmp/pod-foobar.pod-id-file alpine
Type=notify
NotifyAccess=all
@@ -422,7 +444,7 @@ Environment=PODMAN_SYSTEMD_UNIT=%n
Environment=FOO=abc "BAR=my test" USER=%%a
Restart=always
TimeoutStopSec=70
-ExecStart=/usr/bin/podman run --sdnotify=conmon --cgroups=no-conmon --rm -d --env FOO --env=BAR --env=MYENV=2 -e USER awesome-image:latest
+ExecStart=/usr/bin/podman run --cgroups=no-conmon --rm --sdnotify=conmon -d --env FOO --env=BAR --env=MYENV=2 -e USER awesome-image:latest
Type=notify
NotifyAccess=all
@@ -547,6 +569,25 @@ WantedBy=multi-user.target default.target
false,
false,
},
+ {"good with name and sdnotify",
+ containerInfo{
+ Executable: "/usr/bin/podman",
+ ServiceName: "jadda-jadda",
+ ContainerNameOrID: "jadda-jadda",
+ RestartPolicy: "always",
+ PIDFile: "/run/containers/storage/overlay-containers/639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401/userdata/conmon.pid",
+ StopTimeout: 10,
+ PodmanVersion: "CI",
+ CreateCommand: []string{"I'll get stripped", "container", "run", "--sdnotify=container", "--name", "jadda-jadda", "--hostname", "hello-world", "awesome-image:latest", "command", "arg1", "...", "argN", "foo=arg \"with \" space"},
+ EnvVariable: define.EnvVariable,
+ GraphRoot: "/var/lib/containers/storage",
+ RunRoot: "/var/run/containers/storage",
+ },
+ goodWithNameAndSdnotify,
+ true,
+ false,
+ false,
+ },
{"good with explicit short detach param",
containerInfo{
Executable: "/usr/bin/podman",