From bb589bec24ede0ca7bc981a5a285fb66d7242655 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Thu, 13 May 2021 08:45:29 -0400 Subject: Fix problem copying files when container is in host pid namespace When attempting to copy files into and out of running containers within the host pidnamespace, the code was attempting to join the host pidns again, and getting an error. This was causing the podman cp command to fail. Since we are already in the host pid namespace, we should not be attempting to join. This PR adds a check to see if the container is in NOT host pid namespace, and only then attempts to join. Fixes: https://github.com/containers/podman/issues/9985 Signed-off-by: Daniel J Walsh --- test/system/065-cp.bats | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) (limited to 'test/system') diff --git a/test/system/065-cp.bats b/test/system/065-cp.bats index 679cdc209..24ac8118e 100644 --- a/test/system/065-cp.bats +++ b/test/system/065-cp.bats @@ -130,6 +130,22 @@ load helpers } +@test "podman cp file from/to host while --pid=host" { + if is_rootless && ! is_cgroupsv2; then + skip "'podman cp --pid=host' (rootless) only works with cgroups v2" + fi + + srcdir=$PODMAN_TMPDIR/cp-pid-equals-host + mkdir -p $srcdir + touch $srcdir/hostfile + + run_podman run --pid=host -d --name cpcontainer $IMAGE sleep infinity + run_podman cp $srcdir/hostfile cpcontainer:/tmp/hostfile + run_podman cp cpcontainer:/tmp/hostfile $srcdir/hostfile1 + run_podman kill cpcontainer + run_podman rm -f cpcontainer +} + @test "podman cp file from container to host" { srcdir=$PODMAN_TMPDIR/cp-test-file-ctr-to-host mkdir -p $srcdir -- cgit v1.2.3-54-g00ecf From efa15b96c5d11796cdfbf2fdbe10664ec91b4c29 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Wed, 5 May 2021 15:43:50 +0200 Subject: add libimage events libimage now supports events which `libpod.Runtime` now uses for image events. Signed-off-by: Valentin Rothberg Signed-off-by: Matthew Heon --- libpod/runtime.go | 90 +++++++++++++++++++++++++++++++++++++++------ test/system/090-events.bats | 36 ++++++++++++++++++ 2 files changed, 114 insertions(+), 12 deletions(-) (limited to 'test/system') diff --git a/libpod/runtime.go b/libpod/runtime.go index 80fe92b54..2cf2b3f79 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -13,6 +13,7 @@ import ( "strings" "sync" "syscall" + "time" "github.com/containers/common/libimage" "github.com/containers/common/pkg/config" @@ -67,17 +68,18 @@ type Runtime struct { storageConfig storage.StoreOptions storageSet storageSet - state State - store storage.Store - storageService *storageService - imageContext *types.SystemContext - defaultOCIRuntime OCIRuntime - ociRuntimes map[string]OCIRuntime - runtimeFlags []string - netPlugin ocicni.CNIPlugin - conmonPath string - libimageRuntime *libimage.Runtime - lockManager lock.Manager + state State + store storage.Store + storageService *storageService + imageContext *types.SystemContext + defaultOCIRuntime OCIRuntime + ociRuntimes map[string]OCIRuntime + runtimeFlags []string + netPlugin ocicni.CNIPlugin + conmonPath string + libimageRuntime *libimage.Runtime + libimageEventsShutdown chan bool + lockManager lock.Manager // doRenumber indicates that the runtime should perform a lock renumber // during initialization. @@ -212,6 +214,8 @@ func newRuntimeFromConfig(ctx context.Context, conf *config.Config, options ...R return nil, err } + runtime.libimageEventsShutdown = make(chan bool) + return runtime, nil } @@ -677,6 +681,62 @@ func (r *Runtime) GetConfig() (*config.Config, error) { return config, nil } +// libimageEventsMap translates a libimage event type to a libpod event status. +var libimageEventsMap = map[libimage.EventType]events.Status{ + libimage.EventTypeImagePull: events.Pull, + libimage.EventTypeImagePush: events.Push, + libimage.EventTypeImageRemove: events.Remove, + libimage.EventTypeImageLoad: events.LoadFromArchive, + libimage.EventTypeImageSave: events.Save, + libimage.EventTypeImageTag: events.Tag, + libimage.EventTypeImageUntag: events.Untag, + libimage.EventTypeImageMount: events.Mount, + libimage.EventTypeImageUnmount: events.Unmount, +} + +// libimageEvents spawns a goroutine in the background which is listenting for +// events on the libimage.Runtime. The gourtine will be cleaned up implicitly +// when the main() exists. +func (r *Runtime) libimageEvents() { + toLibpodEventStatus := func(e *libimage.Event) events.Status { + status, found := libimageEventsMap[e.Type] + if !found { + return "Unknown" + } + return status + } + + go func() { + eventChannel := r.libimageRuntime.EventChannel() + + for { + // Make sure to read and write all events before + // checking if we're about to shutdown. + for len(eventChannel) > 0 { + libimageEvent := <-eventChannel + e := events.Event{ + ID: libimageEvent.ID, + Name: libimageEvent.Name, + Status: toLibpodEventStatus(libimageEvent), + Time: libimageEvent.Time, + Type: events.Image, + } + if err := r.eventer.Write(e); err != nil { + logrus.Errorf("unable to write image event: %q", err) + } + } + + select { + case <-r.libimageEventsShutdown: + return + + default: + time.Sleep(100 * time.Millisecond) + } + } + }() +} + // DeferredShutdown shuts down the runtime without exposing any // errors. This is only meant to be used when the runtime is being // shutdown within a defer statement; else use Shutdown @@ -716,7 +776,11 @@ func (r *Runtime) Shutdown(force bool) error { // If no store was requested, it can be nil and there is no need to // attempt to shut it down if r.store != nil { - if _, err := r.store.Shutdown(force); err != nil { + // Wait for the events to be written. + r.libimageEventsShutdown <- true + + // Note that the libimage runtime shuts down the store. + if err := r.libimageRuntime.Shutdown(force); err != nil { lastError = errors.Wrapf(err, "error shutting down container storage") } } @@ -842,6 +906,8 @@ func (r *Runtime) configureStore() error { return err } r.libimageRuntime = libimageRuntime + // Run the libimage events routine. + r.libimageEvents() return nil } diff --git a/test/system/090-events.bats b/test/system/090-events.bats index 8a9db41fa..19bee5506 100644 --- a/test/system/090-events.bats +++ b/test/system/090-events.bats @@ -25,3 +25,39 @@ load helpers run_podman events --filter type=container --filter container=$cname --filter event=start --stream=false is "$output" "$expect" "filtering just by label" } + +@test "image events" { + skip_if_remote "FIXME: remove events on podman-remote seem to be broken" + pushedDir=$PODMAN_TMPDIR/dir + mkdir -p $pushedDir + + tarball=$PODMAN_TMPDIR/ball.tar + + run_podman image inspect --format "{{.ID}}" $IMAGE + imageID="$output" + + t0=$(date --iso-8601=seconds) + tag=registry.com/$(random_string 10 | tr A-Z a-z) + + # Force using the file backend since the journal backend is eating events + # (see containers/podman/pull/10219#issuecomment-842325032). + run_podman --events-backend=file push $IMAGE dir:$pushedDir + run_podman --events-backend=file save $IMAGE -o $tarball + run_podman --events-backend=file load -i $tarball + run_podman --events-backend=file pull docker-archive:$tarball + run_podman --events-backend=file tag $IMAGE $tag + run_podman --events-backend=file untag $IMAGE $tag + run_podman --events-backend=file tag $IMAGE $tag + run_podman --events-backend=file rmi $tag + + run_podman --events-backend=file events --stream=false --filter type=image --since $t0 + is "$output" ".*image push $imageID dir:$pushedDir +.*image save $imageID $tarball +.*image loadfromarchive *$tarball +.*image pull *docker-archive:$tarball +.*image tag $imageID $tag +.*image untag $imageID $tag:latest +.*image tag $imageID $tag +.*image remove $imageID $tag.*" \ + "podman events" +} -- cgit v1.2.3-54-g00ecf From ba884865c58d83d7a020bfb7fe90a182ebbe1405 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Thu, 20 May 2021 16:09:40 -0400 Subject: Handle hard links in remote builds Fixes: https://github.com/containers/podman/issues/9893 Signed-off-by: Daniel J Walsh --- Makefile | 3 +++ pkg/bindings/images/build.go | 44 ++++++++++++++++++++++++++---------- pkg/bindings/images/build_unix.go | 16 +++++++++++++ pkg/bindings/images/build_windows.go | 9 ++++++++ test/system/070-build.bats | 20 ++++++++++++++++ 5 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 pkg/bindings/images/build_unix.go create mode 100644 pkg/bindings/images/build_windows.go (limited to 'test/system') diff --git a/Makefile b/Makefile index 25644dffd..a0b12e7de 100644 --- a/Makefile +++ b/Makefile @@ -382,6 +382,9 @@ bin/podman.cross.%: .gopathok .PHONY: local-cross local-cross: $(CROSS_BUILD_TARGETS) ## Cross compile podman binary for multiple architectures +.PHONY: cross +cross: local-cross + # Update nix/nixpkgs.json its latest stable commit .PHONY: nixpkgs nixpkgs: diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go index f5e7c0c98..b56afbceb 100644 --- a/pkg/bindings/images/build.go +++ b/pkg/bindings/images/build.go @@ -28,6 +28,11 @@ import ( "github.com/sirupsen/logrus" ) +type devino struct { + Dev uint64 + Ino uint64 +} + var ( iidRegex = regexp.MustCompile(`^[0-9a-f]{12}`) ) @@ -402,7 +407,7 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) { defer pw.Close() defer gw.Close() defer tw.Close() - + seen := make(map[devino]string) for _, src := range sources { s, err := filepath.Abs(src) if err != nil { @@ -431,25 +436,40 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) { } if info.Mode().IsRegular() { // add file item - f, lerr := os.Open(path) - if lerr != nil { - return lerr + di, isHardLink := checkHardLink(info) + if err != nil { + return err } - hdr, lerr := tar.FileInfoHeader(info, name) - if lerr != nil { - f.Close() - return lerr + hdr, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + orig, ok := seen[di] + if ok { + hdr.Typeflag = tar.TypeLink + hdr.Linkname = orig + hdr.Size = 0 + + return tw.WriteHeader(hdr) + } + f, err := os.Open(path) + if err != nil { + return err } + hdr.Name = name - if lerr := tw.WriteHeader(hdr); lerr != nil { + if err := tw.WriteHeader(hdr); err != nil { f.Close() - return lerr + return err } - _, cerr := io.Copy(tw, f) + _, err = io.Copy(tw, f) f.Close() - return cerr + if err == nil && isHardLink { + seen[di] = name + } + return err } else if info.Mode().IsDir() { // add folders hdr, lerr := tar.FileInfoHeader(info, name) if lerr != nil { diff --git a/pkg/bindings/images/build_unix.go b/pkg/bindings/images/build_unix.go new file mode 100644 index 000000000..0afb1deb6 --- /dev/null +++ b/pkg/bindings/images/build_unix.go @@ -0,0 +1,16 @@ +// +build !windows + +package images + +import ( + "os" + "syscall" +) + +func checkHardLink(fi os.FileInfo) (devino, bool) { + st := fi.Sys().(*syscall.Stat_t) + return devino{ + Dev: uint64(st.Dev), + Ino: uint64(st.Ino), + }, st.Nlink > 1 +} diff --git a/pkg/bindings/images/build_windows.go b/pkg/bindings/images/build_windows.go new file mode 100644 index 000000000..bd71d1bf0 --- /dev/null +++ b/pkg/bindings/images/build_windows.go @@ -0,0 +1,9 @@ +package images + +import ( + "os" +) + +func checkHardLink(fi os.FileInfo) (devino, bool) { + return devino{}, false +} diff --git a/test/system/070-build.bats b/test/system/070-build.bats index d2d56c051..9e1559013 100644 --- a/test/system/070-build.bats +++ b/test/system/070-build.bats @@ -766,6 +766,26 @@ EOF is "$output" ".*/tmp/bogus: no such file or directory" } +@test "podman build COPY hardlinks " { + tmpdir=$PODMAN_TMPDIR/build-test + mkdir -p $tmpdir + + dockerfile=$tmpdir/Dockerfile + cat >$dockerfile < Date: Fri, 21 May 2021 16:02:13 -0400 Subject: Improve OCI Runtime error ErrOCIRuntimeNotFound error is misleading. Try to make it more understandable to the user that the OCI Runtime IE crun or runc is not missing, but the command they attempted to run within the container is missing. [NO TESTS NEEDED] Regular tests should handle this. Fixes: https://github.com/containers/podman/issues/10432 Signed-off-by: Daniel J Walsh --- libpod/define/errors.go | 2 +- test/system/030-run.bats | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) (limited to 'test/system') diff --git a/libpod/define/errors.go b/libpod/define/errors.go index 81bf5f69c..9fd210eed 100644 --- a/libpod/define/errors.go +++ b/libpod/define/errors.go @@ -152,7 +152,7 @@ var ( // ErrOCIRuntimeNotFound indicates the OCI runtime attempted to invoke a command // that was not found - ErrOCIRuntimeNotFound = errors.New("OCI not found") + ErrOCIRuntimeNotFound = errors.New("OCI runtime attempted to invoke a command that was not found") // ErrOCIRuntimeUnavailable indicates that the OCI runtime associated to a container // could not be found in the configuration diff --git a/test/system/030-run.bats b/test/system/030-run.bats index 9a136ff13..ccae703e5 100644 --- a/test/system/030-run.bats +++ b/test/system/030-run.bats @@ -14,7 +14,7 @@ load helpers # ...but check the configured runtime engine, and switch to crun as needed run_podman info --format '{{ .Host.OCIRuntime.Path }}' if expr "$output" : ".*/crun"; then - err_no_such_cmd="Error: executable file.* not found in \$PATH: No such file or directory: OCI not found" + err_no_such_cmd="Error: executable file.* not found in \$PATH: No such file or directory: OCI runtime attempted to invoke a command that was not found" err_no_exec_dir="Error: open executable: Operation not permitted: OCI permission denied" fi -- cgit v1.2.3-54-g00ecf From 934f36df5e94dd976e73f4a5cd99f092ecba052a Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Mon, 24 May 2021 15:41:51 -0400 Subject: Handle hard links in different directories Fixes: https://github.com/containers/podman/issues/10444 Signed-off-by: Daniel J Walsh --- pkg/bindings/images/build.go | 2 +- test/system/070-build.bats | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) (limited to 'test/system') diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go index b56afbceb..346d55c47 100644 --- a/pkg/bindings/images/build.go +++ b/pkg/bindings/images/build.go @@ -450,7 +450,7 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) { hdr.Typeflag = tar.TypeLink hdr.Linkname = orig hdr.Size = 0 - + hdr.Name = name return tw.WriteHeader(hdr) } f, err := os.Open(path) diff --git a/test/system/070-build.bats b/test/system/070-build.bats index 9e1559013..0f3f3fa7f 100644 --- a/test/system/070-build.bats +++ b/test/system/070-build.bats @@ -768,19 +768,27 @@ EOF @test "podman build COPY hardlinks " { tmpdir=$PODMAN_TMPDIR/build-test - mkdir -p $tmpdir + subdir=$tmpdir/subdir + subsubdir=$subdir/subsubdir + mkdir -p $subsubdir dockerfile=$tmpdir/Dockerfile cat >$dockerfile < Date: Tue, 25 May 2021 10:43:49 -0400 Subject: Fix race on podman start --all Make sure all containers exit after start There is a race condition in that container could still be running when we attempt to remove them. Signed-off-by: Daniel J Walsh --- test/system/045-start.bats | 2 ++ 1 file changed, 2 insertions(+) (limited to 'test/system') diff --git a/test/system/045-start.bats b/test/system/045-start.bats index 542f9d1c2..3e0118dba 100644 --- a/test/system/045-start.bats +++ b/test/system/045-start.bats @@ -25,6 +25,8 @@ load helpers die "podman start --all restarted a running container" fi + run_podman wait $cid_none_implicit $cid_none_explicit $cid_on_failure + run_podman rm $cid_none_implicit $cid_none_explicit $cid_on_failure run_podman stop -t 1 $cid_always run_podman rm $cid_always -- cgit v1.2.3-54-g00ecf