From 2e3611d61f6fa3a9195e601027628a696aa77923 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Wed, 20 Oct 2021 13:15:04 +0200 Subject: overlay root fs: create mount on runtime dir Make sure to create the mounts for containers with an overlay root FS in the runtime dir (e.g., /run/user/1000/...) to guarantee that we can actually overlay mount on the specific path which is not the case for the graph root. [NO NEW TESTS NEEDED] since it is not a user-facing change. Signed-off-by: Valentin Rothberg --- libpod/container_internal.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/libpod/container_internal.go b/libpod/container_internal.go index 994ffeec7..747fe6ceb 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -1511,8 +1511,8 @@ func (c *Container) mountStorage() (_ string, deferredErr error) { mountPoint := c.config.Rootfs // Check if overlay has to be created on top of Rootfs if c.config.RootfsOverlay { - overlayDest := c.runtime.store.GraphRoot() - contentDir, err := overlay.GenerateStructure(c.runtime.store.GraphRoot(), c.ID(), "rootfs", c.RootUID(), c.RootGID()) + overlayDest := c.runtime.RunRoot() + contentDir, err := overlay.GenerateStructure(overlayDest, c.ID(), "rootfs", c.RootUID(), c.RootGID()) if err != nil { return "", errors.Wrapf(err, "rootfs-overlay: failed to create TempDir in the %s directory", overlayDest) } @@ -1737,11 +1737,11 @@ func (c *Container) cleanupStorage() error { // umount rootfs overlay if it was created if c.config.RootfsOverlay { - overlayBasePath := filepath.Dir(c.config.StaticDir) - overlayBasePath = filepath.Join(overlayBasePath, "rootfs") + overlayBasePath := filepath.Dir(c.state.Mountpoint) if err := overlay.Unmount(overlayBasePath); err != nil { - // If the container can't remove content report the error - logrus.Errorf("Failed to cleanup overlay mounts for %s: %v", c.ID(), err) + if cleanupErr != nil { + logrus.Errorf("Failed to cleanup overlay mounts for %s: %v", c.ID(), err) + } cleanupErr = err } } -- cgit v1.2.3-54-g00ecf From e86549b18dd413fe3d4939f84ad9c1e21afd225c Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Wed, 20 Oct 2021 17:56:13 +0200 Subject: cirrus: containers: mount directory in /var/tmp to /tmp Mount a directory from /var/tmp to /tmp to make sure that /tmp is not on an overlay mount. This should make overlay mounts possible in the containerized tests which we're currently skipping. Signed-off-by: Valentin Rothberg --- contrib/cirrus/runner.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh index 22a66dd08..8ef2a6e64 100755 --- a/contrib/cirrus/runner.sh +++ b/contrib/cirrus/runner.sh @@ -117,6 +117,7 @@ exec_container() { set -x # shellcheck disable=SC2154 exec podman run --rm --privileged --net=host --cgroupns=host \ + -v `mktemp -d -p /var/tmp`:/tmp:Z \ -v /dev/fuse:/dev/fuse \ -v "$GOPATH:$GOPATH:Z" \ --workdir "$GOSRC" \ -- cgit v1.2.3-54-g00ecf From 9d2b8d2791c23b83b6155b046099a83483860c56 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Wed, 13 Oct 2021 10:11:38 +0200 Subject: add kubernetes pause Add the k8s pause binary to `pause/pause.c` and do the plumbing in the Makefile to install it in $libexec/podman/pause/pause. It is intended to replace the k8s pause image and hence the need for network connectivity when creating pods. [NO NEW TESTS NEEDED] since it will be tested in a following commit. Signed-off-by: Valentin Rothberg --- Makefile | 23 ++++++++++++--- contrib/spec/podman.spec.in | 1 + pause/pause.c | 69 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 89 insertions(+), 4 deletions(-) create mode 100644 pause/pause.c diff --git a/Makefile b/Makefile index 3b70c4f4f..00afd1dfd 100644 --- a/Makefile +++ b/Makefile @@ -185,6 +185,10 @@ ifdef HOMEBREW_PREFIX endif endif +# For building pause/pause.c +GCC ?= gcc +PAUSE_CFLAGS = -Os -static -Wall -Werror -DVERSION=v$(RELEASE_VERSION) + ### ### Primary entry-point targets ### @@ -196,7 +200,7 @@ default: all all: binaries docs .PHONY: binaries -binaries: podman podman-remote rootlessport ## Build podman, podman-remote and rootlessport binaries +binaries: podman podman-remote rootlessport pause # Extract text following double-# for targets, as their description for # the `help` target. Otherwise These simple-substitutions are resolved @@ -374,6 +378,12 @@ bin/rootlessport: .gopathok $(SOURCES) go.mod go.sum .PHONY: rootlessport rootlessport: bin/rootlessport +bin/pause: pause/pause.c + $(GCC) $(PAUSE_CFLAGS) pause/pause.c -o bin/pause + +.PHONY: pause +pause: bin/pause + ### ### Secondary binary-build targets ### @@ -733,7 +743,7 @@ install.remote-nobuild: install.remote: podman-remote install.remote-nobuild .PHONY: install.bin-nobuild -install.bin-nobuild: +install.bin-nobuild: install.pause install ${SELINUXOPT} -d -m 755 $(DESTDIR)$(BINDIR) install ${SELINUXOPT} -m 755 bin/podman $(DESTDIR)$(BINDIR)/podman test -z "${SELINUXOPT}" || chcon --verbose --reference=$(DESTDIR)$(BINDIR)/podman bin/podman @@ -787,8 +797,10 @@ install.docker-docs-nobuild: .PHONY: install.docker-docs install.docker-docs: docker-docs install.docker-docs-nobuild -.PHONY: install.docker-full -install.docker-full: install.docker install.docker-docs +.PHONY: install.pause +install.pause: pause + install ${SELINUXOPT} -m 755 -d $(DESTDIR)$(LIBEXECPODMAN)/pause + install ${SELINUXOPT} -m 755 bin/pause $(DESTDIR)$(LIBEXECPODMAN)/pause/pause .PHONY: install.systemd ifneq (,$(findstring systemd,$(BUILDTAGS))) @@ -819,6 +831,9 @@ else install.systemd: endif +.PHONY: install.pause +install.pause: pause + .PHONY: install.tools install.tools: .install.goimports .install.gitvalidation .install.md2man .install.ginkgo .install.golangci-lint .install.bats ## Install needed tools diff --git a/contrib/spec/podman.spec.in b/contrib/spec/podman.spec.in index 295a953ef..2db8f6e67 100644 --- a/contrib/spec/podman.spec.in +++ b/contrib/spec/podman.spec.in @@ -528,6 +528,7 @@ export GOPATH=%{buildroot}/%{gopath}:$(pwd)/vendor:%{gopath} %{_usr}/lib/tmpfiles.d/podman.conf %dir %{_libexecdir}/%{name} %{_libexecdir}/%{name}/rootlessport +%{_libexecdir}/%{name}/pause/pause %if 0%{?with_devel} %files -n libpod-devel -f devel.file-list diff --git a/pause/pause.c b/pause/pause.c new file mode 100644 index 000000000..1e363bd7a --- /dev/null +++ b/pause/pause.c @@ -0,0 +1,69 @@ +/* +Copyright 2016 The Kubernetes Authors. +Copyright 2021 The Podman Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +#include +#include +#include +#include +#include +#include +#include + +#define STRINGIFY(x) #x +#define VERSION_STRING(x) STRINGIFY(x) + +#ifndef VERSION +#define VERSION HEAD +#endif + +static void sigdown(int signo) { + psignal(signo, "Shutting down, got signal"); + exit(0); +} + +static void sigreap(int signo) { + while (waitpid(-1, NULL, WNOHANG) > 0) + ; +} + +int main(int argc, char **argv) { + int i; + for (i = 1; i < argc; ++i) { + if (!strcasecmp(argv[i], "-v")) { + printf("pause.c %s\n", VERSION_STRING(VERSION)); + return 0; + } + } + + if (getpid() != 1) + /* Not an error because pause sees use outside of infra containers. */ + fprintf(stderr, "Warning: pause should be the first process\n"); + + if (sigaction(SIGINT, &(struct sigaction){.sa_handler = sigdown}, NULL) < 0) + return 1; + if (sigaction(SIGTERM, &(struct sigaction){.sa_handler = sigdown}, NULL) < 0) + return 2; + if (sigaction(SIGCHLD, &(struct sigaction){.sa_handler = sigreap, + .sa_flags = SA_NOCLDSTOP}, + NULL) < 0) + return 3; + + for (;;) + pause(); + fprintf(stderr, "Error: infinite loop terminated\n"); + return 42; +} -- cgit v1.2.3-54-g00ecf From 75f478c08b82a9d6a99628bee5698a85610f863c Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Wed, 13 Oct 2021 16:33:38 +0200 Subject: pod create: remove need for pause image So far, the infra containers of pods required pulling down an image rendering pods not usable in disconnected environments. Instead, build an image locally which uses local pause binary. Fixes: #10354 Signed-off-by: Valentin Rothberg --- cmd/podman/containers/create.go | 9 +-- cmd/podman/pods/create.go | 10 --- cmd/podman/system/version.go | 2 +- docs/source/markdown/podman-pod-create.1.md | 2 +- libpod/kube.go | 15 ++++- pkg/api/handlers/libpod/pods.go | 17 ----- pkg/specgen/generate/pod_create.go | 97 ++++++++++++++++++++++++++++- test/e2e/play_kube_test.go | 19 ------ test/system/200-pod.bats | 30 ++++++--- 9 files changed, 135 insertions(+), 66 deletions(-) diff --git a/cmd/podman/containers/create.go b/cmd/podman/containers/create.go index bfeeb7ebe..3be426ae2 100644 --- a/cmd/podman/containers/create.go +++ b/cmd/podman/containers/create.go @@ -372,15 +372,10 @@ func createPodIfNecessary(s *specgen.SpecGenerator, netOpts *entities.NetOptions } infraOpts := entities.ContainerCreateOptions{ImageVolume: "bind", Net: netOpts, Quiet: true} - rawImageName := config.DefaultInfraImage - name, err := PullImage(rawImageName, infraOpts) - if err != nil { - fmt.Println(err) - } - imageName := name + imageName := config.DefaultInfraImage podGen.InfraImage = imageName podGen.InfraContainerSpec = specgen.NewSpecGenerator(imageName, false) - podGen.InfraContainerSpec.RawImageName = rawImageName + podGen.InfraContainerSpec.RawImageName = imageName podGen.InfraContainerSpec.NetworkOptions = podGen.NetworkOptions err = specgenutil.FillOutSpecGen(podGen.InfraContainerSpec, &infraOpts, []string{}) if err != nil { diff --git a/cmd/podman/pods/create.go b/cmd/podman/pods/create.go index 7c2c72171..0a0d43b53 100644 --- a/cmd/podman/pods/create.go +++ b/cmd/podman/pods/create.go @@ -242,16 +242,6 @@ func create(cmd *cobra.Command, args []string) error { } if createOptions.Infra { rawImageName = img - if !infraOptions.RootFS { - curr := infraOptions.Quiet - infraOptions.Quiet = true - name, err := containers.PullImage(imageName, infraOptions) - if err != nil { - fmt.Println(err) - } - imageName = name - infraOptions.Quiet = curr - } podSpec.InfraImage = imageName if infraOptions.Entrypoint != nil { createOptions.InfraCommand = infraOptions.Entrypoint diff --git a/cmd/podman/system/version.go b/cmd/podman/system/version.go index 4502b156c..3ddd19bae 100644 --- a/cmd/podman/system/version.go +++ b/cmd/podman/system/version.go @@ -67,7 +67,7 @@ func version(cmd *cobra.Command, args []string) error { } if err := tmpl.Execute(w, versions); err != nil { // On Failure, assume user is using older version of podman version --format and check client - row = strings.Replace(row, ".Server.", ".", 1) + row = strings.ReplaceAll(row, ".Server.", ".") tmpl, err := report.NewTemplate("version 1.0.0").Parse(row) if err != nil { return err diff --git a/docs/source/markdown/podman-pod-create.1.md b/docs/source/markdown/podman-pod-create.1.md index 08ac19b8b..061727559 100644 --- a/docs/source/markdown/podman-pod-create.1.md +++ b/docs/source/markdown/podman-pod-create.1.md @@ -112,7 +112,7 @@ The command that will be run to start the infra container. Default: "/pause". #### **--infra-image**=*image* -The image that will be created for the infra container. Default: "k8s.gcr.io/pause:3.1". +The custom image that will be used for the infra container. Unless specified, Podman builds a custom local image which does not require pulling down an image. #### **--infra-name**=*name* diff --git a/libpod/kube.go b/libpod/kube.go index 02ac8ed98..ad5acea78 100644 --- a/libpod/kube.go +++ b/libpod/kube.go @@ -11,6 +11,7 @@ import ( "strings" "time" + "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/libpod/network/types" "github.com/containers/podman/v3/pkg/env" @@ -468,11 +469,23 @@ func containerToV1Container(ctx context.Context, c *Container) (v1.Container, [] kubeContainer.Name = removeUnderscores(c.Name()) _, image := c.Image() + + // The infra container may have been created with an overlay root FS + // instead of an infra image. If so, set the imageto the default K8s + // pause one and make sure it's in the storage by pulling it down if + // missing. + if image == "" && c.IsInfra() { + image = config.DefaultInfraImage + if _, err := c.runtime.libimageRuntime.Pull(ctx, image, config.PullPolicyMissing, nil); err != nil { + return kubeContainer, nil, nil, nil, err + } + } + kubeContainer.Image = image kubeContainer.Stdin = c.Stdin() img, _, err := c.runtime.libimageRuntime.LookupImage(image, nil) if err != nil { - return kubeContainer, kubeVolumes, nil, annotations, err + return kubeContainer, kubeVolumes, nil, annotations, fmt.Errorf("looking up image %q of container %q: %w", image, c.ID(), err) } imgData, err := img.Inspect(ctx, nil) if err != nil { diff --git a/pkg/api/handlers/libpod/pods.go b/pkg/api/handlers/libpod/pods.go index 77d026550..1e64de0ee 100644 --- a/pkg/api/handlers/libpod/pods.go +++ b/pkg/api/handlers/libpod/pods.go @@ -1,15 +1,12 @@ package libpod import ( - "context" "encoding/json" "fmt" "net/http" "strings" - "github.com/containers/common/libimage" "github.com/containers/common/pkg/config" - "github.com/containers/image/v5/transports/alltransports" "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/api/handlers" @@ -67,20 +64,6 @@ func PodCreate(w http.ResponseWriter, r *http.Request) { imageName = config.DefaultInfraImage rawImageName = config.DefaultInfraImage } - curr := infraOptions.Quiet - infraOptions.Quiet = true - pullOptions := &libimage.PullOptions{} - pulledImages, err := runtime.LibimageRuntime().Pull(context.Background(), imageName, config.PullPolicyMissing, pullOptions) - if err != nil { - utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "could not pull image")) - return - } - if _, err := alltransports.ParseImageName(imageName); err == nil { - if len(pulledImages) != 0 { - imageName = pulledImages[0].ID() - } - } - infraOptions.Quiet = curr psg.InfraImage = imageName psg.InfraContainerSpec.Image = imageName psg.InfraContainerSpec.RawImageName = rawImageName diff --git a/pkg/specgen/generate/pod_create.go b/pkg/specgen/generate/pod_create.go index a4027eae7..88ebc7ae3 100644 --- a/pkg/specgen/generate/pod_create.go +++ b/pkg/specgen/generate/pod_create.go @@ -2,8 +2,12 @@ package generate import ( "context" + "fmt" + "io/ioutil" "net" + "os" + buildahDefine "github.com/containers/buildah/define" "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/libpod/define" @@ -14,10 +18,102 @@ import ( "github.com/sirupsen/logrus" ) +func buildPauseImage(rt *libpod.Runtime, rtConfig *config.Config) (string, error) { + version, err := define.GetVersion() + if err != nil { + return "", err + } + imageName := fmt.Sprintf("localhost/podman-pause:%s-%d", version.Version, version.Built) + + // First check if the image has already been built. + if _, _, err := rt.LibimageRuntime().LookupImage(imageName, nil); err == nil { + return imageName, nil + } + + // NOTE: Having the pause binary in its own directory keeps the door + // open for replacing the image building with using an overlay root FS. + // The latter turned out to be complex and error prone (see #11956) but + // we may be able to come up with a proper solution at a later point in + // time. + pausePath, err := rtConfig.FindHelperBinary("pause/pause", false) + if err != nil { + return "", fmt.Errorf("finding pause binary: %w", err) + } + + buildContent := fmt.Sprintf(`FROM scratch +COPY %s /pause +ENTRYPOINT ["/pause"]`, pausePath) + + tmpF, err := ioutil.TempFile("", "pause.containerfile") + if err != nil { + return "", err + } + if _, err := tmpF.WriteString(buildContent); err != nil { + return "", err + } + if err := tmpF.Close(); err != nil { + return "", err + } + defer os.Remove(tmpF.Name()) + + buildOptions := buildahDefine.BuildOptions{ + CommonBuildOpts: &buildahDefine.CommonBuildOptions{}, + Output: imageName, + Quiet: true, + IIDFile: "/dev/null", // prevents Buildah from writing the ID on stdout + } + if _, _, err := rt.Build(context.Background(), buildOptions, tmpF.Name()); err != nil { + return "", err + } + + return imageName, nil +} + +func pullOrBuildInfraImage(p *entities.PodSpec, rt *libpod.Runtime) error { + if p.PodSpecGen.NoInfra { + return nil + } + + rtConfig, err := rt.GetConfigNoCopy() + if err != nil { + return err + } + + // NOTE: we need pull down the infra image if it was explicitly set by + // the user (or containers.conf) to the non-default one. + imageName := p.PodSpecGen.InfraImage + if imageName == "" { + imageName = rtConfig.Engine.InfraImage + } + + if imageName != config.DefaultInfraImage { + _, err := rt.LibimageRuntime().Pull(context.Background(), imageName, config.PullPolicyMissing, nil) + if err != nil { + return err + } + } else { + name, err := buildPauseImage(rt, rtConfig) + if err != nil { + return fmt.Errorf("building local pause image: %w", err) + } + imageName = name + } + + p.PodSpecGen.InfraImage = imageName + p.PodSpecGen.InfraContainerSpec.RawImageName = imageName + + return nil +} + func MakePod(p *entities.PodSpec, rt *libpod.Runtime) (*libpod.Pod, error) { if err := p.PodSpecGen.Validate(); err != nil { return nil, err } + + if err := pullOrBuildInfraImage(p, rt); err != nil { + return nil, err + } + if !p.PodSpecGen.NoInfra && p.PodSpecGen.InfraContainerSpec != nil { var err error p.PodSpecGen.InfraContainerSpec, err = MapSpec(&p.PodSpecGen) @@ -35,7 +131,6 @@ func MakePod(p *entities.PodSpec, rt *libpod.Runtime) (*libpod.Pod, error) { return nil, err } if !p.PodSpecGen.NoInfra && p.PodSpecGen.InfraContainerSpec != nil { - p.PodSpecGen.InfraContainerSpec.ContainerCreateCommand = []string{} // we do NOT want os.Args as the command, will display the pod create cmd if p.PodSpecGen.InfraContainerSpec.Name == "" { p.PodSpecGen.InfraContainerSpec.Name = pod.ID()[:12] + "-infra" } diff --git a/test/e2e/play_kube_test.go b/test/e2e/play_kube_test.go index d5b913743..b0b927445 100644 --- a/test/e2e/play_kube_test.go +++ b/test/e2e/play_kube_test.go @@ -11,7 +11,6 @@ import ( "text/template" "time" - "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/util" . "github.com/containers/podman/v3/test/utils" @@ -1120,24 +1119,6 @@ var _ = Describe("Podman play kube", func() { Expect(label).To(ContainSubstring("unconfined_u:system_r:spc_t:s0")) }) - It("podman play kube should use default infra_image", func() { - err := writeYaml(checkInfraImagePodYaml, kubeYaml) - Expect(err).To(BeNil()) - - kube := podmanTest.Podman([]string{"play", "kube", kubeYaml}) - kube.WaitWithDefaultTimeout() - Expect(kube).Should(Exit(0)) - - podInspect := podmanTest.Podman([]string{"inspect", "check-infra-image", "--format", "{{ .InfraContainerID }}"}) - podInspect.WaitWithDefaultTimeout() - infraContainerID := podInspect.OutputToString() - - conInspect := podmanTest.Podman([]string{"inspect", infraContainerID, "--format", "{{ .ImageName }}"}) - conInspect.WaitWithDefaultTimeout() - infraContainerImage := conInspect.OutputToString() - Expect(infraContainerImage).To(Equal(config.DefaultInfraImage)) - }) - It("podman play kube --no-host", func() { err := writeYaml(checkInfraImagePodYaml, kubeYaml) Expect(err).To(BeNil()) diff --git a/test/system/200-pod.bats b/test/system/200-pod.bats index 86f3610ab..09a419914 100644 --- a/test/system/200-pod.bats +++ b/test/system/200-pod.bats @@ -60,6 +60,10 @@ function teardown() { run_podman pod rm -f -t 0 $podid } +function rm_podman_pause_image() { + run_podman version --format "{{.Server.Version}}-{{.Server.Built}}" + run_podman rmi -f "localhost/podman-pause:$output" +} @test "podman pod - communicating between pods" { podname=pod$(random_string) @@ -100,19 +104,14 @@ function teardown() { # Clean up. First the nc -l container... run_podman rm $cid1 - # ...then, from pause container, find the image ID of the pause image... - run_podman pod inspect --format '{{(index .Containers 0).ID}}' $podname - pause_cid="$output" - run_podman container inspect --format '{{.Image}}' $pause_cid - pause_iid="$output" - # ...then rm the pod, then rmi the pause image so we don't leave strays. run_podman pod rm $podname - run_podman rmi $pause_iid # Pod no longer exists run_podman 1 pod exists $podid run_podman 1 pod exists $podname + + rm_podman_pause_image } @test "podman pod - communicating via /dev/shm " { @@ -133,6 +132,10 @@ function teardown() { # Pod no longer exists run_podman 1 pod exists $podid run_podman 1 pod exists $podname + + # Pause image hasn't been pulled + run_podman 1 image exists k8s.gcr.io/pause:3.5 + rm_podman_pause_image } # Random byte @@ -303,16 +306,25 @@ EOF run_podman rm $cid run_podman pod rm -t 0 -f mypod run_podman rmi $infra_image - } @test "podman pod create should fail when infra-name is already in use" { local infra_name="infra_container_$(random_string 10 | tr A-Z a-z)" - run_podman pod create --infra-name "$infra_name" + local pod_name="$(random_string 10 | tr A-Z a-z)" + + # Note that the internal pause image is built even when --infra-image is + # set to the K8s one. + run_podman pod create --name $pod_name --infra-name "$infra_name" --infra-image "k8s.gcr.io/pause:3.5" run_podman '?' pod create --infra-name "$infra_name" if [ $status -eq 0 ]; then die "Podman should fail when user try to create two pods with the same infra-name value" fi + run_podman pod rm -f $pod_name + run_podman images -a + + # Pause image hasn't been pulled + run_podman 1 image exists k8s.gcr.io/pause:3.5 + rm_podman_pause_image } # vim: filetype=sh -- cgit v1.2.3-54-g00ecf