Merge pull request #5480 from vrothberg/auto-updates

auto update containers in systemd units

10 files changed, 422 insertions, 2 deletions

diff --git a/pkg/adapter/autoupdate.go b/pkg/adapter/autoupdate.go
new file mode 100644
index 000000000..01f7a29c5
--- /dev/null
+++ b/pkg/adapter/autoupdate.go
@@ -0,0 +1,11 @@
+// +build !remoteclient
+
+package adapter
+
+import (
+	"github.com/containers/libpod/pkg/autoupdate"
+)
+
+func (r *LocalRuntime) AutoUpdate() ([]string, []error) {
+	return autoupdate.AutoUpdate(r.Runtime)
+}
diff --git a/pkg/adapter/autoupdate_remote.go b/pkg/adapter/autoupdate_remote.go
new file mode 100644
index 000000000..a2a82d0d4
--- /dev/null
+++ b/pkg/adapter/autoupdate_remote.go
@@ -0,0 +1,11 @@
+// +build remoteclient
+
+package adapter
+
+import (
+	"github.com/containers/libpod/libpod/define"
+)
+
+func (r *LocalRuntime) AutoUpdate() ([]string, []error) {
+	return nil, []error{define.ErrNotImplemented}
+}
diff --git a/pkg/autoupdate/autoupdate.go b/pkg/autoupdate/autoupdate.go
new file mode 100644
index 000000000..7c243eb00
--- /dev/null
+++ b/pkg/autoupdate/autoupdate.go
@@ -0,0 +1,280 @@
+package autoupdate
+
+import (
+	"context"
+	"os"
+	"sort"
+
+	"github.com/containers/image/v5/docker"
+	"github.com/containers/image/v5/docker/reference"
+	"github.com/containers/image/v5/manifest"
+	"github.com/containers/image/v5/transports/alltransports"
+	"github.com/containers/libpod/libpod"
+	"github.com/containers/libpod/libpod/define"
+	"github.com/containers/libpod/libpod/image"
+	"github.com/containers/libpod/pkg/systemd"
+	systemdGen "github.com/containers/libpod/pkg/systemd/generate"
+	"github.com/containers/libpod/pkg/util"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+// Label denotes the container/pod label key to specify auto-update policies in
+// container labels.
+const Label = "io.containers.autoupdate"
+
+// Policy represents an auto-update policy.
+type Policy string
+
+const (
+	// PolicyDefault is the default policy denoting no auto updates.
+	PolicyDefault Policy = "disabled"
+	// PolicyNewImage is the policy to update as soon as there's a new image found.
+	PolicyNewImage = "image"
+)
+
+// Map for easy lookups of supported policies.
+var supportedPolicies = map[string]Policy{
+	"":         PolicyDefault,
+	"disabled": PolicyDefault,
+	"image":    PolicyNewImage,
+}
+
+// LookupPolicy looksup the corresponding Policy for the specified
+// string. If none is found, an errors is returned including the list of
+// supported policies.
+//
+// Note that an empty string resolved to PolicyDefault.
+func LookupPolicy(s string) (Policy, error) {
+	policy, exists := supportedPolicies[s]
+	if exists {
+		return policy, nil
+	}
+
+	// Sort the keys first as maps are non-deterministic.
+	keys := []string{}
+	for k := range supportedPolicies {
+		if k != "" {
+			keys = append(keys, k)
+		}
+	}
+	sort.Strings(keys)
+
+	return "", errors.Errorf("invalid auto-update policy %q: valid policies are %+q", s, keys)
+}
+
+// ValidateImageReference checks if the specified imageName is a fully-qualified
+// image reference to the docker transport (without digest).  Such a reference
+// includes a domain, name and tag (e.g., quay.io/podman/stable:latest).  The
+// reference may also be prefixed with "docker://" explicitly indicating that
+// it's a reference to the docker transport.
+func ValidateImageReference(imageName string) error {
+	// Make sure the input image is a docker.
+	imageRef, err := alltransports.ParseImageName(imageName)
+	if err == nil && imageRef.Transport().Name() != docker.Transport.Name() {
+		return errors.Errorf("auto updates require the docker image transport but image is of transport %q", imageRef.Transport().Name())
+	} else if err != nil {
+		repo, err := reference.Parse(imageName)
+		if err != nil {
+			return errors.Wrap(err, "error enforcing fully-qualified docker transport reference for auto updates")
+		}
+		if _, ok := repo.(reference.NamedTagged); !ok {
+			return errors.Errorf("auto updates require fully-qualified image references (no tag): %q", imageName)
+		}
+		if _, ok := repo.(reference.Digested); ok {
+			return errors.Errorf("auto updates require fully-qualified image references without digest: %q", imageName)
+		}
+	}
+	return nil
+}
+
+// AutoUpdate looks up containers with a specified auto-update policy and acts
+// accordingly.  If the policy is set to PolicyNewImage, it checks if the image
+// on the remote registry is different than the local one. If the image digests
+// differ, it pulls the remote image and restarts the systemd unit running the
+// container.
+//
+// It returns a slice of successfully restarted systemd units and a slice of
+// errors encountered during auto update.
+func AutoUpdate(runtime *libpod.Runtime) ([]string, []error) {
+	// Create a map from `image ID -> []*Container`.
+	containerMap, errs := imageContainersMap(runtime)
+	if len(containerMap) == 0 {
+		return nil, errs
+	}
+
+	// Create a map from `image ID -> *image.Image` for image lookups.
+	imagesSlice, err := runtime.ImageRuntime().GetImages()
+	if err != nil {
+		return nil, []error{err}
+	}
+	imageMap := make(map[string]*image.Image)
+	for i := range imagesSlice {
+		imageMap[imagesSlice[i].ID()] = imagesSlice[i]
+	}
+
+	// Connect to DBUS.
+	conn, err := systemd.ConnectToDBUS()
+	if err != nil {
+		logrus.Errorf(err.Error())
+		return nil, []error{err}
+	}
+	defer conn.Close()
+
+	// Update images.
+	containersToRestart := []*libpod.Container{}
+	updatedRawImages := make(map[string]bool)
+	for imageID, containers := range containerMap {
+		image, exists := imageMap[imageID]
+		if !exists {
+			errs = append(errs, errors.Errorf("container image ID %q not found in local storage", imageID))
+			return nil, errs
+		}
+		// Now we have to check if the image of any containers must be updated.
+		// Note that the image ID is NOT enough for this check as a given image
+		// may have multiple tags.
+		for i, ctr := range containers {
+			rawImageName := ctr.RawImageName()
+			if rawImageName == "" {
+				errs = append(errs, errors.Errorf("error auto-updating container %q: raw-image name is empty", ctr.ID()))
+			}
+			needsUpdate, err := newerImageAvailable(runtime, image, rawImageName)
+			if err != nil {
+				errs = append(errs, errors.Wrapf(err, "error auto-updating container %q: image check for %q failed", ctr.ID(), rawImageName))
+				continue
+			}
+			if !needsUpdate {
+				continue
+			}
+			logrus.Infof("Auto-updating container %q using image %q", ctr.ID(), rawImageName)
+			if _, updated := updatedRawImages[rawImageName]; !updated {
+				_, err = updateImage(runtime, rawImageName)
+				if err != nil {
+					errs = append(errs, errors.Wrapf(err, "error auto-updating container %q: image update for %q failed", ctr.ID(), rawImageName))
+					continue
+				}
+				updatedRawImages[rawImageName] = true
+			}
+			containersToRestart = append(containersToRestart, containers[i])
+		}
+	}
+
+	// Restart containers.
+	updatedUnits := []string{}
+	for _, ctr := range containersToRestart {
+		labels := ctr.Labels()
+		unit, exists := labels[systemdGen.EnvVariable]
+		if !exists {
+			// Shouldn't happen but let's be sure of it.
+			errs = append(errs, errors.Errorf("error auto-updating container %q: no %s label found", ctr.ID(), systemdGen.EnvVariable))
+			continue
+		}
+		_, err := conn.RestartUnit(unit, "replace", nil)
+		if err != nil {
+			errs = append(errs, errors.Wrapf(err, "error auto-updating container %q: restarting systemd unit %q failed", ctr.ID(), unit))
+			continue
+		}
+		logrus.Infof("Successfully restarted systemd unit %q", unit)
+		updatedUnits = append(updatedUnits, unit)
+	}
+
+	return updatedUnits, errs
+}
+
+// imageContainersMap generates a map[image ID] -> [containers using the image]
+// of all containers with a valid auto-update policy.
+func imageContainersMap(runtime *libpod.Runtime) (map[string][]*libpod.Container, []error) {
+	allContainers, err := runtime.GetAllContainers()
+	if err != nil {
+		return nil, []error{err}
+	}
+
+	errors := []error{}
+	imageMap := make(map[string][]*libpod.Container)
+	for i, ctr := range allContainers {
+		state, err := ctr.State()
+		if err != nil {
+			errors = append(errors, err)
+			continue
+		}
+		// Only update running containers.
+		if state != define.ContainerStateRunning {
+			continue
+		}
+		// Only update containers with the specific label/policy set.
+		labels := ctr.Labels()
+		if value, exists := labels[Label]; exists {
+			policy, err := LookupPolicy(value)
+			if err != nil {
+				errors = append(errors, err)
+				continue
+			}
+			if policy != PolicyNewImage {
+				continue
+			}
+		}
+		// Now we know that `ctr` is configured for auto updates.
+		id, _ := ctr.Image()
+		imageMap[id] = append(imageMap[id], allContainers[i])
+	}
+
+	return imageMap, errors
+}
+
+// newerImageAvailable returns true if there corresponding image on the remote
+// registry is newer.
+func newerImageAvailable(runtime *libpod.Runtime, img *image.Image, origName string) (bool, error) {
+	remoteRef, err := docker.ParseReference("//" + origName)
+	if err != nil {
+		return false, err
+	}
+
+	remoteImg, err := remoteRef.NewImage(context.Background(), runtime.SystemContext())
+	if err != nil {
+		return false, err
+	}
+
+	rawManifest, _, err := remoteImg.Manifest(context.Background())
+	if err != nil {
+		return false, err
+	}
+
+	remoteDigest, err := manifest.Digest(rawManifest)
+	if err != nil {
+		return false, err
+	}
+
+	return img.Digest().String() != remoteDigest.String(), nil
+}
+
+// updateImage pulls the specified image.
+func updateImage(runtime *libpod.Runtime, name string) (*image.Image, error) {
+	sys := runtime.SystemContext()
+	registryOpts := image.DockerRegistryOptions{}
+	signaturePolicyPath := ""
+	authFilePath := ""
+
+	if sys != nil {
+		registryOpts.OSChoice = sys.OSChoice
+		registryOpts.ArchitectureChoice = sys.OSChoice
+		registryOpts.DockerCertPath = sys.DockerCertPath
+
+		signaturePolicyPath = sys.SignaturePolicyPath
+		authFilePath = sys.AuthFilePath
+	}
+
+	newImage, err := runtime.ImageRuntime().New(context.Background(),
+		docker.Transport.Name()+"://"+name,
+		signaturePolicyPath,
+		authFilePath,
+		os.Stderr,
+		&registryOpts,
+		image.SigningOptions{},
+		nil,
+		util.PullImageAlways,
+	)
+	if err != nil {
+		return nil, err
+	}
+	return newImage, nil
+}
diff --git a/pkg/autoupdate/autoupdate_test.go b/pkg/autoupdate/autoupdate_test.go
new file mode 100644
index 000000000..7a5da5bb0
--- /dev/null
+++ b/pkg/autoupdate/autoupdate_test.go
@@ -0,0 +1,50 @@
+package autoupdate
+
+import (
+	"testing"
+)
+
+func TestValidateImageReference(t *testing.T) {
+	tests := []struct {
+		input string
+		valid bool
+	}{
+		{ // Fully-qualified reference
+			input: "quay.io/foo/bar:tag",
+			valid: true,
+		},
+		{ // Fully-qualified reference in transport notation
+			input: "docker://quay.io/foo/bar:tag",
+			valid: true,
+		},
+		{ // Fully-qualified reference but with digest
+			input: "quay.io/foo/bar@sha256:c9b1b535fdd91a9855fb7f82348177e5f019329a58c53c47272962dd60f71fc9",
+			valid: false,
+		},
+		{ // Reference with missing tag
+			input: "quay.io/foo/bar",
+			valid: false,
+		},
+		{ // Short name
+			input: "alpine",
+			valid: false,
+		},
+		{ // Short name with repo
+			input: "library/alpine",
+			valid: false,
+		},
+		{ // Wrong transport
+			input: "docker-archive:/some/path.tar",
+			valid: false,
+		},
+	}
+
+	for _, test := range tests {
+		err := ValidateImageReference(test.input)
+		if test.valid && err != nil {
+			t.Fatalf("parsing %q should have succeeded: %v", test.input, err)
+		} else if !test.valid && err == nil {
+			t.Fatalf("parsing %q should have failed", test.input)
+		}
+	}
+}
diff --git a/pkg/spec/createconfig.go b/pkg/spec/createconfig.go
index 9b2255d61..12dfed8c3 100644
--- a/pkg/spec/createconfig.go
+++ b/pkg/spec/createconfig.go
@@ -144,6 +144,7 @@ type CreateConfig struct {
 	InitPath          string //init-path
 	Image             string
 	ImageID           string
+	RawImageName      string
 	BuiltinImgVolumes map[string]struct{} // volumes defined in the image config
 	ImageVolumeType   string              // how to handle the image volume, either bind, tmpfs, or ignore
 	Interactive       bool                //interactive
@@ -348,7 +349,7 @@ func (c *CreateConfig) getContainerCreateOptions(runtime *libpod.Runtime, pod *l
 	options = append(options, nsOpts...)
 
 	// Gather up the options for NewContainer which consist of With... funcs
-	options = append(options, libpod.WithRootFSFromImage(c.ImageID, c.Image))
+	options = append(options, libpod.WithRootFSFromImage(c.ImageID, c.Image, c.RawImageName))
 	options = append(options, libpod.WithConmonPidFile(c.ConmonPidFile))
 	options = append(options, libpod.WithLabels(c.Labels))
 	options = append(options, libpod.WithShmSize(c.Resources.ShmSize))
diff --git a/pkg/specgen/create.go b/pkg/specgen/create.go
index 99a99083b..aefbe7405 100644
--- a/pkg/specgen/create.go
+++ b/pkg/specgen/create.go
@@ -36,7 +36,7 @@ func (s *SpecGenerator) MakeContainer(rt *libpod.Runtime) (*libpod.Container, er
 		return nil, err
 	}
 
-	options = append(options, libpod.WithRootFSFromImage(newImage.ID(), s.Image))
+	options = append(options, libpod.WithRootFSFromImage(newImage.ID(), s.Image, s.RawImageName))
 
 	runtimeSpec, err := s.toOCISpec(rt, newImage)
 	if err != nil {
diff --git a/pkg/specgen/specgen.go b/pkg/specgen/specgen.go
index e1dfe4dc5..7a430652a 100644
--- a/pkg/specgen/specgen.go
+++ b/pkg/specgen/specgen.go
@@ -143,6 +143,10 @@ type ContainerStorageConfig struct {
 	// Conflicts with Rootfs.
 	// At least one of Image or Rootfs must be specified.
 	Image string `json:"image"`
+	// RawImageName is the unprocessed and not-normalized user-specified image
+	// name.  One use case for having this data at hand are auto-updates where
+	// the _exact_ user input is needed in order to look-up the correct image.
+	RawImageName string `json:"raw_image_name,omitempty"`
 	// Rootfs is the path to a directory that will be used as the
 	// container's root filesystem. No modification will be made to the
 	// directory, it will be directly mounted into the container as root.
diff --git a/pkg/systemd/dbus.go b/pkg/systemd/dbus.go
new file mode 100644
index 000000000..df24667a1
--- /dev/null
+++ b/pkg/systemd/dbus.go
@@ -0,0 +1,47 @@
+package systemd
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strconv"
+
+	"github.com/containers/libpod/pkg/rootless"
+	"github.com/coreos/go-systemd/v22/dbus"
+	godbus "github.com/godbus/dbus/v5"
+)
+
+func dbusAuthRootlessConnection(createBus func(opts ...godbus.ConnOption) (*godbus.Conn, error)) (*godbus.Conn, error) {
+	conn, err := createBus()
+	if err != nil {
+		return nil, err
+	}
+
+	methods := []godbus.Auth{godbus.AuthExternal(strconv.Itoa(rootless.GetRootlessUID()))}
+
+	err = conn.Auth(methods)
+	if err != nil {
+		conn.Close()
+		return nil, err
+	}
+
+	return conn, nil
+}
+
+func newRootlessConnection() (*dbus.Conn, error) {
+	return dbus.NewConnection(func() (*godbus.Conn, error) {
+		return dbusAuthRootlessConnection(func(opts ...godbus.ConnOption) (*godbus.Conn, error) {
+			path := filepath.Join(os.Getenv("XDG_RUNTIME_DIR"), "systemd/private")
+			return godbus.Dial(fmt.Sprintf("unix:path=%s", path))
+		})
+	})
+}
+
+// ConnectToDBUS returns a DBUS connection.  It works both as root and non-root
+// users.
+func ConnectToDBUS() (*dbus.Conn, error) {
+	if rootless.IsRootless() {
+		return newRootlessConnection()
+	}
+	return dbus.NewSystemdConnection()
+}
diff --git a/pkg/systemd/generate/systemdgen.go b/pkg/systemd/generate/systemdgen.go
index 4cd7745c0..eb15d4927 100644
--- a/pkg/systemd/generate/systemdgen.go
+++ b/pkg/systemd/generate/systemdgen.go
@@ -16,6 +16,10 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
+// EnvVariable "PODMAN_SYSTEMD_UNIT" is set in all generated systemd units and
+// is set to the unit's (unique) name.
+const EnvVariable = "PODMAN_SYSTEMD_UNIT"
+
 // ContainerInfo contains data required for generating a container's systemd
 // unit file.
 type ContainerInfo struct {
@@ -57,6 +61,8 @@ type ContainerInfo struct {
 	// RunCommand is a post-processed variant of CreateCommand and used for
 	// the ExecStart field in generic unit files.
 	RunCommand string
+	// EnvVariable is generate.EnvVariable and must not be set.
+	EnvVariable string
 }
 
 var restartPolicies = []string{"no", "on-success", "on-failure", "on-abnormal", "on-watchdog", "on-abort", "always"}
@@ -94,6 +100,7 @@ Before={{- range $index, $value := .RequiredServices -}}{{if $index}} {{end}}{{
 {{- end}}
 
 [Service]
+Environment={{.EnvVariable}}=%n
 Restart={{.RestartPolicy}}
 {{- if .New}}
 ExecStartPre=/usr/bin/rm -f %t/%n-pid %t/%n-cid
@@ -138,6 +145,8 @@ func CreateContainerSystemdUnit(info *ContainerInfo, opts Options) (string, erro
 		info.Executable = executable
 	}
 
+	info.EnvVariable = EnvVariable
+
 	// Assemble the ExecStart command when creating a new container.
 	//
 	// Note that we cannot catch all corner cases here such that users
diff --git a/pkg/systemd/generate/systemdgen_test.go b/pkg/systemd/generate/systemdgen_test.go
index bbdccdcf8..3269405a6 100644
--- a/pkg/systemd/generate/systemdgen_test.go
+++ b/pkg/systemd/generate/systemdgen_test.go
@@ -44,6 +44,7 @@ Wants=network.target
 After=network-online.target
 
 [Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 ExecStart=/usr/bin/podman start 639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401
 ExecStop=/usr/bin/podman stop -t 10 639c53578af4d84b8800b4635fa4e680ee80fd67e0e6a2d4eea48d1e3230f401
@@ -64,6 +65,7 @@ Wants=network.target
 After=network-online.target
 
 [Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 ExecStart=/usr/bin/podman start foobar
 ExecStop=/usr/bin/podman stop -t 10 foobar
@@ -88,6 +90,7 @@ BindsTo=a.service b.service c.service pod.service
 After=a.service b.service c.service pod.service
 
 [Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 ExecStart=/usr/bin/podman start foobar
 ExecStop=/usr/bin/podman stop -t 10 foobar
@@ -110,6 +113,7 @@ Requires=container-1.service container-2.service
 Before=container-1.service container-2.service
 
 [Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 ExecStart=/usr/bin/podman start jadda-jadda-infra
 ExecStop=/usr/bin/podman stop -t 10 jadda-jadda-infra
@@ -130,6 +134,7 @@ Wants=network.target
 After=network-online.target
 
 [Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 ExecStartPre=/usr/bin/rm -f %t/%n-pid %t/%n-cid
 ExecStart=/usr/bin/podman run --conmon-pidfile %t/%n-pid --cidfile %t/%n-cid --cgroups=no-conmon -d --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
@@ -152,6 +157,7 @@ Wants=network.target
 After=network-online.target
 
 [Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 ExecStartPre=/usr/bin/rm -f %t/%n-pid %t/%n-cid
 ExecStart=/usr/bin/podman run --conmon-pidfile %t/%n-pid --cidfile %t/%n-cid --cgroups=no-conmon --detach --name jadda-jadda --hostname hello-world awesome-image:latest command arg1 ... argN
@@ -174,6 +180,7 @@ Wants=network.target
 After=network-online.target
 
 [Service]
+Environment=PODMAN_SYSTEMD_UNIT=%n
 Restart=always
 ExecStartPre=/usr/bin/rm -f %t/%n-pid %t/%n-cid
 ExecStart=/usr/bin/podman run --conmon-pidfile %t/%n-pid --cidfile %t/%n-cid --cgroups=no-conmon -d awesome-image:latest