From bdfc4df1f20b18c785c67b79369b9011303889cc Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Mon, 29 Aug 2022 13:42:21 +0200 Subject: pkg/autoupdate: allow updating multiple tasks per unit Refactor the auto-update backend to allow for updating multiple tasks/containers per unit. This commit is merely doing the plumbing. The actual integration comes in a following commit. [NO NEW TESTS NEEDED] as behavior should not change and existing tests are expected to continue to pass. Signed-off-by: Valentin Rothberg --- pkg/autoupdate/autoupdate.go | 147 +++++++++++++++++++++++++------------------ 1 file changed, 86 insertions(+), 61 deletions(-) (limited to 'pkg') diff --git a/pkg/autoupdate/autoupdate.go b/pkg/autoupdate/autoupdate.go index 297d6640e..17cea6719 100644 --- a/pkg/autoupdate/autoupdate.go +++ b/pkg/autoupdate/autoupdate.go @@ -153,18 +153,19 @@ func AutoUpdate(ctx context.Context, runtime *libpod.Runtime, options entities.A } // Find auto-update tasks and assemble them by unit. - errors := auto.assembleTasks(ctx) + allErrors := auto.assembleTasks(ctx) // Nothing to do. if len(auto.unitToTasks) == 0 { - return nil, errors + return nil, allErrors } // Connect to DBUS. conn, err := systemd.ConnectToDBUS() if err != nil { logrus.Errorf(err.Error()) - return nil, []error{err} + allErrors = append(allErrors, err) + return nil, allErrors } defer conn.Close() auto.conn = conn @@ -174,72 +175,96 @@ func AutoUpdate(ctx context.Context, runtime *libpod.Runtime, options entities.A // Update all images/container according to their auto-update policy. var allReports []*entities.AutoUpdateReport for unit, tasks := range auto.unitToTasks { - // Sanity check: we'll support that in the future. - if len(tasks) != 1 { - errors = append(errors, fmt.Errorf("only 1 task per unit supported but unit %s has %d", unit, len(tasks))) - return nil, errors + unitErrors := auto.updateUnit(ctx, unit, tasks) + allErrors = append(allErrors, unitErrors...) + for _, task := range tasks { + allReports = append(allReports, task.report()) } + } - for _, task := range tasks { - err := func() error { - // Transition from state to state. Will be - // split into multiple loops in the future to - // support more than one container/task per - // unit. - updateAvailable, err := task.updateAvailable(ctx) - if err != nil { - task.status = statusFailed - return fmt.Errorf("checking image updates for container %s: %w", task.container.ID(), err) - } - - if !updateAvailable { - task.status = statusNotUpdated - return nil - } - - if options.DryRun { - task.status = statusPending - return nil - } - - if err := task.update(ctx); err != nil { - task.status = statusFailed - return fmt.Errorf("updating image for container %s: %w", task.container.ID(), err) - } - - updateError := auto.restartSystemdUnit(ctx, unit) - if updateError == nil { - task.status = statusUpdated - return nil - } - - if !options.Rollback { - task.status = statusFailed - return fmt.Errorf("restarting unit %s for container %s: %w", task.unit, task.container.ID(), err) - } - - if err := task.rollbackImage(); err != nil { - task.status = statusFailed - return fmt.Errorf("rolling back image for container %s: %w", task.container.ID(), err) - } - - if err := auto.restartSystemdUnit(ctx, unit); err != nil { - task.status = statusFailed - return fmt.Errorf("restarting unit %s for container %s during rollback: %w", task.unit, task.container.ID(), err) - } - - task.status = statusRolledBack - return nil - }() + return allReports, allErrors +} + +// updateUnit auto updates the tasks in the specified systemd unit. +func (u *updater) updateUnit(ctx context.Context, unit string, tasks []*task) []error { + var errors []error + // Sanity check: we'll support that in the future. + if len(tasks) != 1 { + errors = append(errors, fmt.Errorf("only 1 task per unit supported but unit %s has %d", unit, len(tasks))) + return errors + } + tasksUpdated := false + for _, task := range tasks { + err := func() error { // Use an anonymous function to avoid spaghetti continue's + updateAvailable, err := task.updateAvailable(ctx) if err != nil { - errors = append(errors, err) + task.status = statusFailed + return fmt.Errorf("checking image updates for container %s: %w", task.container.ID(), err) } - allReports = append(allReports, task.report()) + + if !updateAvailable { + task.status = statusNotUpdated + return nil + } + + if u.options.DryRun { + task.status = statusPending + return nil + } + + if err := task.update(ctx); err != nil { + task.status = statusFailed + return fmt.Errorf("updating image for container %s: %w", task.container.ID(), err) + } + + tasksUpdated = true + return nil + }() + + if err != nil { + errors = append(errors, err) + } + } + + // If no task has been updated, we can jump directly to the next unit. + if !tasksUpdated { + return errors + } + + updateError := u.restartSystemdUnit(ctx, unit) + for _, task := range tasks { + if updateError == nil { + task.status = statusUpdated + } else { + task.status = statusFailed + } + } + + // Jump to the next unit on successful update or if rollbacks are disabled. + if updateError == nil || !u.options.Rollback { + return errors + } + + // The update has failed and rollbacks are enabled. + for _, task := range tasks { + if err := task.rollbackImage(); err != nil { + err = fmt.Errorf("rolling back image for container %s in unit %s: %w", task.container.ID(), unit, err) + errors = append(errors, err) } } - return allReports, errors + if err := u.restartSystemdUnit(ctx, unit); err != nil { + err = fmt.Errorf("restarting unit %s during rollback: %w", unit, err) + errors = append(errors, err) + return errors + } + + for _, task := range tasks { + task.status = statusRolledBack + } + + return errors } // report creates an auto-update report for the task. -- cgit v1.2.3-54-g00ecf From 274d34a25a3ed7b69a6e4caec07e845157048c96 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Tue, 30 Aug 2022 11:17:25 +0200 Subject: kube play: support auto updates and rollbacks Add auto-update support to `podman kube play`. Auto-update policies can be configured for: * the entire pod via the `io.containers.autoupdate` annotation * a specific container via the `io.containers.autoupdate/$name` annotation To make use of rollbacks, the `io.containers.sdnotify` policy should be set to `container` such that the workload running _inside_ the container can send the READY message via the NOTIFY_SOCKET once ready. For further details on auto updates and rollbacks, please refer to the specific article [1]. Since auto updates and rollbacks bases on Podman's systemd integration, the k8s YAML must be executed in the `podman-kube@` systemd template. For further details on how to run k8s YAML in systemd via Podman, please refer to the specific article [2]. An examplary k8s YAML may look as follows: ```YAML apiVersion: v1 kind: Pod metadata: annotations: io.containers.autoupdate: "local" io.containers.autoupdate/b: "registry" labels: app: test name: test_pod spec: containers: - command: - top image: alpine name: a - command: - top image: alpine name: b ``` [1] https://www.redhat.com/sysadmin/podman-auto-updates-rollbacks [2] https://www.redhat.com/sysadmin/kubernetes-workloads-podman-systemd Signed-off-by: Valentin Rothberg --- docs/source/markdown/podman-auto-update.1.md.in | 12 +++ pkg/autoupdate/autoupdate.go | 21 ++++-- pkg/domain/infra/abi/play.go | 35 +++++---- pkg/specgen/generate/kube/kube.go | 8 ++ pkg/systemd/notifyproxy/notifyproxy.go | 65 +++++++++++++++-- pkg/systemd/notifyproxy/notifyproxy_test.go | 2 +- test/system/250-systemd.bats | 29 ++++---- test/system/255-auto-update.bats | 97 ++++++++++++++++++++++++- test/system/helpers.systemd.bash | 14 ++++ 9 files changed, 240 insertions(+), 43 deletions(-) (limited to 'pkg') diff --git a/docs/source/markdown/podman-auto-update.1.md.in b/docs/source/markdown/podman-auto-update.1.md.in index bc92d6165..b724f1157 100644 --- a/docs/source/markdown/podman-auto-update.1.md.in +++ b/docs/source/markdown/podman-auto-update.1.md.in @@ -29,6 +29,18 @@ This data is then being used in the auto-update sequence to instruct systemd (vi Note that **podman auto-update** relies on systemd. The systemd units are expected to be generated with **[podman-generate-systemd --new](podman-generate-systemd.1.md#--new)**, or similar units that create new containers in order to run the updated images. Systemd units that start and stop a container cannot run a new image. +### Auto Updates and Kubernetes YAML + +Podman supports auto updates for Kubernetes workloads. As mentioned above, `podman auto-update` requires the containers to be running systemd. Podman ships with a systemd template that can be instantiated with a Kubernetes YAML file, see podman-generate-systemd(1). + +To enable auto updates for containers running in a Kubernetes workload, set the following Podman-specific annotations in the YAML: + * `io.containers.autoupdate: "registry|local"` to apply the auto-update policy to all containers + * `io.containers.autoupdate/$container: "registry|local"` to apply the auto-update policy to `$container` only + * `io.containers.sdnotify: "conmon|container"` to apply the sdnotify policy to all containers + * `io.containers.sdnotify/$container: "conmon|container"` to apply the sdnotify policy to `$container` only + +By default, the autoupdate policy is set to "disabled", the sdnotify policy is set to "conmon". + ### Systemd Unit and Timer Podman ships with a `podman-auto-update.service` systemd unit. This unit is triggered daily at midnight by the `podman-auto-update.timer` systemd timer. The timer can be altered for custom time-based updates if desired. The unit can further be invoked by other systemd units (e.g., via the dependency tree) or manually via **systemctl start podman-auto-update.service**. diff --git a/pkg/autoupdate/autoupdate.go b/pkg/autoupdate/autoupdate.go index 17cea6719..9cf77d135 100644 --- a/pkg/autoupdate/autoupdate.go +++ b/pkg/autoupdate/autoupdate.go @@ -188,13 +188,8 @@ func AutoUpdate(ctx context.Context, runtime *libpod.Runtime, options entities.A // updateUnit auto updates the tasks in the specified systemd unit. func (u *updater) updateUnit(ctx context.Context, unit string, tasks []*task) []error { var errors []error - // Sanity check: we'll support that in the future. - if len(tasks) != 1 { - errors = append(errors, fmt.Errorf("only 1 task per unit supported but unit %s has %d", unit, len(tasks))) - return errors - } - tasksUpdated := false + for _, task := range tasks { err := func() error { // Use an anonymous function to avoid spaghetti continue's updateAvailable, err := task.updateAvailable(ctx) @@ -255,6 +250,9 @@ func (u *updater) updateUnit(ctx context.Context, unit string, tasks []*task) [] } if err := u.restartSystemdUnit(ctx, unit); err != nil { + for _, task := range tasks { + task.status = statusFailed + } err = fmt.Errorf("restarting unit %s during rollback: %w", unit, err) errors = append(errors, err) return errors @@ -283,7 +281,16 @@ func (t *task) report() *entities.AutoUpdateReport { func (t *task) updateAvailable(ctx context.Context) (bool, error) { switch t.policy { case PolicyRegistryImage: - return t.registryUpdateAvailable(ctx) + // Errors checking for updates only should not be fatal. + // Especially on Edge systems, connection may be limited or + // there may just be a temporary downtime of the registry. + // But make sure to leave some breadcrumbs in the debug logs + // such that potential issues _can_ be analyzed if needed. + available, err := t.registryUpdateAvailable(ctx) + if err != nil { + logrus.Debugf("Error checking updates for image %s: %v (ignoring error)", t.rawImageName, err) + } + return available, nil case PolicyLocalImage: return t.localUpdateAvailable() default: diff --git a/pkg/domain/infra/abi/play.go b/pkg/domain/infra/abi/play.go index 6ea20a4f2..12786afcd 100644 --- a/pkg/domain/infra/abi/play.go +++ b/pkg/domain/infra/abi/play.go @@ -661,9 +661,10 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY opts = append(opts, libpod.WithSdNotifyMode(sdNotifyMode)) + var proxy *notifyproxy.NotifyProxy // Create a notify proxy for the container. if sdNotifyMode != "" && sdNotifyMode != define.SdNotifyModeIgnore { - proxy, err := notifyproxy.New("") + proxy, err = notifyproxy.New("") if err != nil { return nil, err } @@ -675,6 +676,9 @@ func (ic *ContainerEngine) playKubePod(ctx context.Context, podName string, podY if err != nil { return nil, err } + if proxy != nil { + proxy.AddContainer(ctr) + } containers = append(containers, ctr) } @@ -774,21 +778,26 @@ func (ic *ContainerEngine) getImageAndLabelInfo(ctx context.Context, cwd string, } // Handle kube annotations - for k, v := range annotations { - switch k { - // Auto update annotation without container name will apply to - // all containers within the pod - case autoupdate.Label, autoupdate.AuthfileLabel: - labels[k] = v - // Auto update annotation with container name will apply only - // to the specified container - case fmt.Sprintf("%s/%s", autoupdate.Label, container.Name), - fmt.Sprintf("%s/%s", autoupdate.AuthfileLabel, container.Name): - prefixAndCtr := strings.Split(k, "/") - labels[prefixAndCtr[0]] = v + setLabel := func(label string) { + var result string + ctrSpecific := fmt.Sprintf("%s/%s", label, container.Name) + for k, v := range annotations { + switch k { + case label: + result = v + case ctrSpecific: + labels[label] = v + return + } + } + if result != "" { + labels[label] = result } } + setLabel(autoupdate.Label) + setLabel(autoupdate.AuthfileLabel) + return pulledImage, labels, nil } diff --git a/pkg/specgen/generate/kube/kube.go b/pkg/specgen/generate/kube/kube.go index 5862d3f1c..9fd0adecf 100644 --- a/pkg/specgen/generate/kube/kube.go +++ b/pkg/specgen/generate/kube/kube.go @@ -7,6 +7,7 @@ import ( "fmt" "math" "net" + "os" "regexp" "runtime" "strconv" @@ -26,6 +27,7 @@ import ( "github.com/containers/podman/v4/pkg/k8s.io/apimachinery/pkg/api/resource" "github.com/containers/podman/v4/pkg/specgen" "github.com/containers/podman/v4/pkg/specgen/generate" + systemdDefine "github.com/containers/podman/v4/pkg/systemd/define" "github.com/containers/podman/v4/pkg/util" "github.com/docker/docker/pkg/system" "github.com/docker/go-units" @@ -445,6 +447,12 @@ func ToSpecGen(ctx context.Context, opts *CtrSpecGenOptions) (*specgen.SpecGener } } + // Make sure the container runs in a systemd unit which is + // stored as a label at container creation. + if unit := os.Getenv(systemdDefine.EnvVariable); unit != "" { + s.Labels[systemdDefine.EnvVariable] = unit + } + return s, nil } diff --git a/pkg/systemd/notifyproxy/notifyproxy.go b/pkg/systemd/notifyproxy/notifyproxy.go index 9e6eb4cf0..1bfab9ca0 100644 --- a/pkg/systemd/notifyproxy/notifyproxy.go +++ b/pkg/systemd/notifyproxy/notifyproxy.go @@ -1,12 +1,17 @@ package notifyproxy import ( + "errors" + "fmt" + "io" "io/ioutil" "net" "os" "strings" "syscall" + "time" + "github.com/containers/podman/v4/libpod/define" "github.com/coreos/go-systemd/v22/daemon" "github.com/sirupsen/logrus" ) @@ -39,6 +44,7 @@ func SendMessage(socketPath string, message string) error { type NotifyProxy struct { connection *net.UnixConn socketPath string + container Container // optional } // New creates a NotifyProxy. The specified temp directory can be left empty. @@ -77,9 +83,26 @@ func (p *NotifyProxy) close() error { return p.connection.Close() } +// AddContainer associates a container with the proxy. +func (p *NotifyProxy) AddContainer(container Container) { + p.container = container +} + +// ErrNoReadyMessage is returned when we are waiting for the READY message of a +// container that is not in the running state anymore. +var ErrNoReadyMessage = errors.New("container stopped running before READY message was received") + +// Container avoids a circular dependency among this package and libpod. +type Container interface { + State() (define.ContainerStatus, error) + ID() string +} + // WaitAndClose waits until receiving the `READY` notify message and close the // listener. Note that the this function must only be executed inside a systemd // service which will kill the process after a given timeout. +// If the (optional) container stopped running before the `READY` is received, +// the waiting gets canceled and ErrNoReadyMessage is returned. func (p *NotifyProxy) WaitAndClose() error { defer func() { if err := p.close(); err != nil { @@ -87,16 +110,48 @@ func (p *NotifyProxy) WaitAndClose() error { } }() + const bufferSize = 1024 + sBuilder := strings.Builder{} for { - buf := make([]byte, 1024) - num, err := p.connection.Read(buf) - if err != nil { + // Set a read deadline of one second such that we achieve a + // non-blocking read and can check if the container has already + // stopped running; in that case no READY message will be send + // and we're done. + if err := p.connection.SetReadDeadline(time.Now().Add(time.Second)); err != nil { return err } - for _, s := range strings.Split(string(buf[:num]), "\n") { - if s == daemon.SdNotifyReady { + + for { + buffer := make([]byte, bufferSize) + num, err := p.connection.Read(buffer) + if err != nil { + if !errors.Is(err, os.ErrDeadlineExceeded) && !errors.Is(err, io.EOF) { + return err + } + } + sBuilder.Write(buffer[:num]) + if num != bufferSize || buffer[num-1] == '\n' { + break + } + } + + for _, line := range strings.Split(sBuilder.String(), "\n") { + if line == daemon.SdNotifyReady { return nil } } + sBuilder.Reset() + + if p.container == nil { + continue + } + + state, err := p.container.State() + if err != nil { + return err + } + if state != define.ContainerStateRunning { + return fmt.Errorf("%w: %s", ErrNoReadyMessage, p.container.ID()) + } } } diff --git a/pkg/systemd/notifyproxy/notifyproxy_test.go b/pkg/systemd/notifyproxy/notifyproxy_test.go index ce63fc9cd..066046cb8 100644 --- a/pkg/systemd/notifyproxy/notifyproxy_test.go +++ b/pkg/systemd/notifyproxy/notifyproxy_test.go @@ -41,7 +41,7 @@ func TestWaitAndClose(t *testing.T) { default: } - sendMessage(t, proxy, daemon.SdNotifyReady+"\nsomething else") + sendMessage(t, proxy, daemon.SdNotifyReady+"\nsomething else\n") done := func() bool { for i := 0; i < 10; i++ { select { diff --git a/test/system/250-systemd.bats b/test/system/250-systemd.bats index 8f4471f91..dd1a0f05a 100644 --- a/test/system/250-systemd.bats +++ b/test/system/250-systemd.bats @@ -301,24 +301,16 @@ LISTEN_FDNAMES=listen_fdnames" | sort) } @test "podman-kube@.service template" { - # If running from a podman source directory, build and use the source - # version of the play-kube-@ unit file - unit_name="podman-kube@.service" - unit_file="contrib/systemd/system/${unit_name}" - if [[ -e ${unit_file}.in ]]; then - echo "# [Building & using $unit_name from source]" >&3 - # Force regenerating unit file (existing one may have /usr/bin path) - rm -f $unit_file - BINDIR=$(dirname $PODMAN) make $unit_file - cp $unit_file $UNIT_DIR/$unit_name - fi - + install_kube_template # Create the YAMl file yaml_source="$PODMAN_TMPDIR/test.yaml" cat >$yaml_source <$dockerfile1 <> /runme +RUN chmod +x /runme +EOF + + dockerfile2=$PODMAN_TMPDIR/Dockerfile.2 + cat >$dockerfile2 <> /runme +RUN chmod +x /runme +EOF + local_image=localhost/image:$(random_string 10) + + # Generate a healthy image that will run correctly. + run_podman build -t $local_image -f $dockerfile1 + run_podman image inspect --format "{{.ID}}" $local_image + oldID="$output" + + # Create the YAMl file + yaml_source="$PODMAN_TMPDIR/test.yaml" + cat >$yaml_source <&3 + # Force regenerating unit file (existing one may have /usr/bin path) + rm -f $unit_file + BINDIR=$(dirname $PODMAN) make $unit_file + cp $unit_file $UNIT_DIR/$unit_name + fi +} -- cgit v1.2.3-54-g00ecf