From 18e917cdc361dc7f1badbce89b69e80a522062fe Mon Sep 17 00:00:00 2001 From: Jhon Honce Date: Mon, 10 May 2021 10:22:33 -0700 Subject: Downgrade API service routing table logging [NO TESTS NEEDED] * Log the routing table output at Trace vs. Debug level. Reduce noise in debugging output. * Tweak SDNotify message to report Warn when it fails. Previously failures were silent. Signed-off-by: Jhon Honce --- pkg/api/server/server.go | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'pkg') diff --git a/pkg/api/server/server.go b/pkg/api/server/server.go index e23448fd1..972541bc6 100644 --- a/pkg/api/server/server.go +++ b/pkg/api/server/server.go @@ -136,7 +136,7 @@ func newServer(runtime *libpod.Runtime, duration time.Duration, listener *net.Li } } - if logrus.IsLevelEnabled(logrus.DebugLevel) { + if logrus.IsLevelEnabled(logrus.TraceLevel) { router.Walk(func(route *mux.Route, r *mux.Router, ancestors []*mux.Route) error { // nolint path, err := route.GetPathTemplate() if err != nil { @@ -146,7 +146,7 @@ func newServer(runtime *libpod.Runtime, duration time.Duration, listener *net.Li if err != nil { methods = []string{""} } - logrus.Debugf("Methods: %6s Path: %s", strings.Join(methods, ", "), path) + logrus.Tracef("Methods: %6s Path: %s", strings.Join(methods, ", "), path) return nil }) } @@ -162,13 +162,12 @@ func setupSystemd() { if len(os.Getenv("NOTIFY_SOCKET")) == 0 { return } - payload := fmt.Sprintf("MAINPID=%d", os.Getpid()) - payload += "\n" + payload := fmt.Sprintf("MAINPID=%d\n", os.Getpid()) payload += daemon.SdNotifyReady if sent, err := daemon.SdNotify(true, payload); err != nil { logrus.Errorf("Error notifying systemd of Conmon PID: %s", err.Error()) - } else if sent { - logrus.Debugf("Notify sent successfully") + } else if !sent { + logrus.Warn("SDNotify not sent successfully") } if err := os.Unsetenv("INVOCATION_ID"); err != nil { -- cgit v1.2.3-54-g00ecf From ac8b7ddd8dc2b3d4ac2da6b4ddd08b900eb02353 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Fri, 21 May 2021 08:00:47 -0400 Subject: Drop container does not exist on removal to debugf We have race conditions where a container can be removed by two different processes when running podman --remove rm. It can be cleaned up in the API or by the conmon executing podman container cleanup. When we fail to remove a container that does not exists we should not be printing errors or warnings, we should just debug the fact. [NO TESTS NEEDED] Since this is a race condition it is difficult to test. Signed-off-by: Daniel J Walsh --- pkg/domain/infra/abi/containers.go | 2 +- pkg/domain/infra/tunnel/containers.go | 13 ++++++++----- 2 files changed, 9 insertions(+), 6 deletions(-) (limited to 'pkg') diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index d0a2b1bae..237a43441 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -275,7 +275,7 @@ func (ic *ContainerEngine) ContainerRm(ctx context.Context, namesOrIds []string, case nil: // remove container names that we successfully deleted reports = append(reports, &report) - case define.ErrNoSuchCtr: + case define.ErrNoSuchCtr, define.ErrCtrExists: // There is still a potential this is a libpod container tmpNames = append(tmpNames, ctr) default: diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go index 3830835cc..74ced300a 100644 --- a/pkg/domain/infra/tunnel/containers.go +++ b/pkg/domain/infra/tunnel/containers.go @@ -21,6 +21,7 @@ import ( "github.com/containers/podman/v3/pkg/errorhandling" "github.com/containers/podman/v3/pkg/specgen" "github.com/containers/podman/v3/pkg/util" + "github.com/containers/storage/types" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -580,7 +581,7 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri if err := containers.Remove(ic.ClientCtx, ctr.ID, removeOptions); err != nil { if errorhandling.Contains(err, define.ErrNoSuchCtr) || errorhandling.Contains(err, define.ErrCtrRemoved) { - logrus.Warnf("Container %s does not exist: %v", ctr.ID, err) + logrus.Debugf("Container %s does not exist: %v", ctr.ID, err) } else { logrus.Errorf("Error removing container %s: %v", ctr.ID, err) } @@ -613,8 +614,9 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri rmOptions := new(containers.RemoveOptions).WithForce(false).WithVolumes(true) if err := containers.Remove(ic.ClientCtx, ctr.ID, rmOptions); err != nil { if errorhandling.Contains(err, define.ErrNoSuchCtr) || - errorhandling.Contains(err, define.ErrCtrRemoved) { - logrus.Warnf("Container %s does not exist: %v", ctr.ID, err) + errorhandling.Contains(err, define.ErrCtrRemoved) || + errorhandling.Contains(err, types.ErrLayerUnknown) { + logrus.Debugf("Container %s does not exist: %v", ctr.ID, err) } else { logrus.Errorf("Error removing container %s: %v", ctr.ID, err) } @@ -691,8 +693,9 @@ func (ic *ContainerEngine) ContainerRun(ctx context.Context, opts entities.Conta if !shouldRestart { if err := containers.Remove(ic.ClientCtx, con.ID, new(containers.RemoveOptions).WithForce(false).WithVolumes(true)); err != nil { if errorhandling.Contains(err, define.ErrNoSuchCtr) || - errorhandling.Contains(err, define.ErrCtrRemoved) { - logrus.Warnf("Container %s does not exist: %v", con.ID, err) + errorhandling.Contains(err, define.ErrCtrRemoved) || + errorhandling.Contains(err, types.ErrLayerUnknown) { + logrus.Debugf("Container %s does not exist: %v", con.ID, err) } else { logrus.Errorf("Error removing container %s: %v", con.ID, err) } -- cgit v1.2.3-54-g00ecf From c53638e9f61e61b1344cbf090c9766a1891c8a44 Mon Sep 17 00:00:00 2001 From: flouthoc Date: Wed, 19 May 2021 01:19:20 +0530 Subject: Podman info add support for status of cgroup controllers Signed-off-by: flouthoc --- docs/source/markdown/podman-info.1.md | 13 +++++ libpod/define/info.go | 31 +++++----- libpod/info.go | 44 ++++++++------ pkg/cgroups/cgroups.go | 106 +++++++++++++++++++++++++++++++--- test/e2e/info_test.go | 10 ++++ 5 files changed, 163 insertions(+), 41 deletions(-) (limited to 'pkg') diff --git a/docs/source/markdown/podman-info.1.md b/docs/source/markdown/podman-info.1.md index 4af51d3eb..227fbd92d 100644 --- a/docs/source/markdown/podman-info.1.md +++ b/docs/source/markdown/podman-info.1.md @@ -32,6 +32,12 @@ $ podman info host: arch: amd64 buildahVersion: 1.19.0-dev + cgroupControllers: + - cpuset + - cpu + - io + - memory + - pids cgroupManager: systemd cgroupVersion: v2 conmon: @@ -145,6 +151,13 @@ Run podman info with JSON formatted response: "buildahVersion": "1.19.0-dev", "cgroupManager": "systemd", "cgroupVersion": "v2", + "cgroupControllers": [ + "cpuset", + "cpu", + "io", + "memory", + "pids" + ], "conmon": { "package": "conmon-2.0.22-2.fc33.x86_64", "path": "/usr/bin/conmon", diff --git a/libpod/define/info.go b/libpod/define/info.go index c9d6877c0..de709be74 100644 --- a/libpod/define/info.go +++ b/libpod/define/info.go @@ -23,21 +23,22 @@ type SecurityInfo struct { // HostInfo describes the libpod host type HostInfo struct { - Arch string `json:"arch"` - BuildahVersion string `json:"buildahVersion"` - CgroupManager string `json:"cgroupManager"` - CGroupsVersion string `json:"cgroupVersion"` - Conmon *ConmonInfo `json:"conmon"` - CPUs int `json:"cpus"` - Distribution DistributionInfo `json:"distribution"` - EventLogger string `json:"eventLogger"` - Hostname string `json:"hostname"` - IDMappings IDMappings `json:"idMappings,omitempty"` - Kernel string `json:"kernel"` - MemFree int64 `json:"memFree"` - MemTotal int64 `json:"memTotal"` - OCIRuntime *OCIRuntimeInfo `json:"ociRuntime"` - OS string `json:"os"` + Arch string `json:"arch"` + BuildahVersion string `json:"buildahVersion"` + CgroupManager string `json:"cgroupManager"` + CGroupsVersion string `json:"cgroupVersion"` + CgroupControllers []string `json:"cgroupControllers"` + Conmon *ConmonInfo `json:"conmon"` + CPUs int `json:"cpus"` + Distribution DistributionInfo `json:"distribution"` + EventLogger string `json:"eventLogger"` + Hostname string `json:"hostname"` + IDMappings IDMappings `json:"idMappings,omitempty"` + Kernel string `json:"kernel"` + MemFree int64 `json:"memFree"` + MemTotal int64 `json:"memTotal"` + OCIRuntime *OCIRuntimeInfo `json:"ociRuntime"` + OS string `json:"os"` // RemoteSocket returns the UNIX domain socket the Podman service is listening on RemoteSocket *RemoteSocket `json:"remoteSocket,omitempty"` RuntimeInfo map[string]interface{} `json:"runtimeInfo,omitempty"` diff --git a/libpod/info.go b/libpod/info.go index 7a28a4cf7..461e39a48 100644 --- a/libpod/info.go +++ b/libpod/info.go @@ -93,20 +93,33 @@ func (r *Runtime) hostInfo() (*define.HostInfo, error) { return nil, errors.Wrapf(err, "error getting Seccomp profile path") } + // CGroups version + unified, err := cgroups.IsCgroup2UnifiedMode() + if err != nil { + return nil, errors.Wrapf(err, "error reading cgroups mode") + } + + // Get Map of all available controllers + availableControllers, err := cgroups.GetAvailableControllers(nil, unified) + if err != nil { + return nil, errors.Wrapf(err, "error getting available cgroup controllers") + } + info := define.HostInfo{ - Arch: runtime.GOARCH, - BuildahVersion: buildah.Version, - CgroupManager: r.config.Engine.CgroupManager, - Linkmode: linkmode.Linkmode(), - CPUs: runtime.NumCPU(), - Distribution: hostDistributionInfo, - EventLogger: r.eventer.String(), - Hostname: host, - IDMappings: define.IDMappings{}, - Kernel: kv, - MemFree: mi.MemFree, - MemTotal: mi.MemTotal, - OS: runtime.GOOS, + Arch: runtime.GOARCH, + BuildahVersion: buildah.Version, + CgroupManager: r.config.Engine.CgroupManager, + CgroupControllers: availableControllers, + Linkmode: linkmode.Linkmode(), + CPUs: runtime.NumCPU(), + Distribution: hostDistributionInfo, + EventLogger: r.eventer.String(), + Hostname: host, + IDMappings: define.IDMappings{}, + Kernel: kv, + MemFree: mi.MemFree, + MemTotal: mi.MemTotal, + OS: runtime.GOOS, Security: define.SecurityInfo{ AppArmorEnabled: apparmor.IsEnabled(), DefaultCapabilities: strings.Join(r.config.Containers.DefaultCapabilities, ","), @@ -120,11 +133,6 @@ func (r *Runtime) hostInfo() (*define.HostInfo, error) { SwapTotal: mi.SwapTotal, } - // CGroups version - unified, err := cgroups.IsCgroup2UnifiedMode() - if err != nil { - return nil, errors.Wrapf(err, "error reading cgroups mode") - } cgroupVersion := "v1" if unified { cgroupVersion = "v2" diff --git a/pkg/cgroups/cgroups.go b/pkg/cgroups/cgroups.go index aefb5183b..911edeb5b 100644 --- a/pkg/cgroups/cgroups.go +++ b/pkg/cgroups/cgroups.go @@ -128,28 +128,118 @@ func init() { // getAvailableControllers get the available controllers func getAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]controller, error) { if cgroup2 { - return nil, fmt.Errorf("getAvailableControllers not implemented yet for cgroup v2") + controllers := []controller{} + subtreeControl := cgroupRoot + "/cgroup.subtree_control" + // rootless cgroupv2: check available controllers for current user ,systemd or servicescope will inherit + if rootless.IsRootless() { + userSlice, err := getCgroupPathForCurrentProcess() + if err != nil { + return controllers, err + } + //userSlice already contains '/' so not adding here + basePath := cgroupRoot + userSlice + subtreeControl = fmt.Sprintf("%s/cgroup.subtree_control", basePath) + } + subtreeControlBytes, err := ioutil.ReadFile(subtreeControl) + if err != nil { + return nil, errors.Wrapf(err, "failed while reading controllers for cgroup v2 from %q", subtreeControl) + } + for _, controllerName := range strings.Fields(string(subtreeControlBytes)) { + c := controller{ + name: controllerName, + symlink: false, + } + controllers = append(controllers, c) + } + return controllers, nil } - infos, err := ioutil.ReadDir(cgroupRoot) - if err != nil { - return nil, err - } + subsystems, _ := cgroupV1GetAllSubsystems() controllers := []controller{} - for _, i := range infos { - name := i.Name() + // cgroupv1 and rootless: No subsystem is available: delegation is unsafe. + if rootless.IsRootless() { + return controllers, nil + } + + for _, name := range subsystems { if _, found := exclude[name]; found { continue } + isSymLink := false + fileInfo, err := os.Stat(cgroupRoot + "/" + name) + if err != nil { + isSymLink = !fileInfo.IsDir() + } c := controller{ name: name, - symlink: !i.IsDir(), + symlink: isSymLink, } controllers = append(controllers, c) } + return controllers, nil } +// GetAvailableControllers get string:bool map of all the available controllers +func GetAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]string, error) { + availableControllers, err := getAvailableControllers(exclude, cgroup2) + if err != nil { + return nil, err + } + controllerList := []string{} + for _, controller := range availableControllers { + controllerList = append(controllerList, controller.name) + } + + return controllerList, nil +} + +func cgroupV1GetAllSubsystems() ([]string, error) { + f, err := os.Open("/proc/cgroups") + if err != nil { + return nil, err + } + defer f.Close() + + subsystems := []string{} + + s := bufio.NewScanner(f) + for s.Scan() { + text := s.Text() + if text[0] != '#' { + parts := strings.Fields(text) + if len(parts) >= 4 && parts[3] != "0" { + subsystems = append(subsystems, parts[0]) + } + } + } + if err := s.Err(); err != nil { + return nil, err + } + return subsystems, nil +} + +func getCgroupPathForCurrentProcess() (string, error) { + path := fmt.Sprintf("/proc/%d/cgroup", os.Getpid()) + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + cgroupPath := "" + s := bufio.NewScanner(f) + for s.Scan() { + text := s.Text() + procEntries := strings.SplitN(text, "::", 2) + cgroupPath = procEntries[1] + } + if err := s.Err(); err != nil { + return cgroupPath, err + } + return cgroupPath, nil +} + // getCgroupv1Path is a helper function to get the cgroup v1 path func (c *CgroupControl) getCgroupv1Path(name string) string { return filepath.Join(cgroupRoot, name, c.path) diff --git a/test/e2e/info_test.go b/test/e2e/info_test.go index 60136bcc2..f5b70d6bf 100644 --- a/test/e2e/info_test.go +++ b/test/e2e/info_test.go @@ -135,4 +135,14 @@ var _ = Describe("Podman Info", func() { Expect(session.OutputToString()).To(ContainSubstring("false")) } }) + + It("Podman info must contain cgroupControllers with ReleventControllers", func() { + SkipIfRootless("Hard to tell which controllers are going to be enabled for rootless") + SkipIfRootlessCgroupsV1("Disable cgroups not supported on cgroupv1 for rootless users") + session := podmanTest.Podman([]string{"info", "--format", "{{.Host.CgroupControllers}}"}) + session.WaitWithDefaultTimeout() + Expect(session).To(Exit(0)) + Expect(session.OutputToString()).To(ContainSubstring("memory")) + Expect(session.OutputToString()).To(ContainSubstring("pids")) + }) }) -- cgit v1.2.3-54-g00ecf From ba884865c58d83d7a020bfb7fe90a182ebbe1405 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Thu, 20 May 2021 16:09:40 -0400 Subject: Handle hard links in remote builds Fixes: https://github.com/containers/podman/issues/9893 Signed-off-by: Daniel J Walsh --- Makefile | 3 +++ pkg/bindings/images/build.go | 44 ++++++++++++++++++++++++++---------- pkg/bindings/images/build_unix.go | 16 +++++++++++++ pkg/bindings/images/build_windows.go | 9 ++++++++ test/system/070-build.bats | 20 ++++++++++++++++ 5 files changed, 80 insertions(+), 12 deletions(-) create mode 100644 pkg/bindings/images/build_unix.go create mode 100644 pkg/bindings/images/build_windows.go (limited to 'pkg') diff --git a/Makefile b/Makefile index 25644dffd..a0b12e7de 100644 --- a/Makefile +++ b/Makefile @@ -382,6 +382,9 @@ bin/podman.cross.%: .gopathok .PHONY: local-cross local-cross: $(CROSS_BUILD_TARGETS) ## Cross compile podman binary for multiple architectures +.PHONY: cross +cross: local-cross + # Update nix/nixpkgs.json its latest stable commit .PHONY: nixpkgs nixpkgs: diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go index f5e7c0c98..b56afbceb 100644 --- a/pkg/bindings/images/build.go +++ b/pkg/bindings/images/build.go @@ -28,6 +28,11 @@ import ( "github.com/sirupsen/logrus" ) +type devino struct { + Dev uint64 + Ino uint64 +} + var ( iidRegex = regexp.MustCompile(`^[0-9a-f]{12}`) ) @@ -402,7 +407,7 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) { defer pw.Close() defer gw.Close() defer tw.Close() - + seen := make(map[devino]string) for _, src := range sources { s, err := filepath.Abs(src) if err != nil { @@ -431,25 +436,40 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) { } if info.Mode().IsRegular() { // add file item - f, lerr := os.Open(path) - if lerr != nil { - return lerr + di, isHardLink := checkHardLink(info) + if err != nil { + return err } - hdr, lerr := tar.FileInfoHeader(info, name) - if lerr != nil { - f.Close() - return lerr + hdr, err := tar.FileInfoHeader(info, "") + if err != nil { + return err + } + orig, ok := seen[di] + if ok { + hdr.Typeflag = tar.TypeLink + hdr.Linkname = orig + hdr.Size = 0 + + return tw.WriteHeader(hdr) + } + f, err := os.Open(path) + if err != nil { + return err } + hdr.Name = name - if lerr := tw.WriteHeader(hdr); lerr != nil { + if err := tw.WriteHeader(hdr); err != nil { f.Close() - return lerr + return err } - _, cerr := io.Copy(tw, f) + _, err = io.Copy(tw, f) f.Close() - return cerr + if err == nil && isHardLink { + seen[di] = name + } + return err } else if info.Mode().IsDir() { // add folders hdr, lerr := tar.FileInfoHeader(info, name) if lerr != nil { diff --git a/pkg/bindings/images/build_unix.go b/pkg/bindings/images/build_unix.go new file mode 100644 index 000000000..0afb1deb6 --- /dev/null +++ b/pkg/bindings/images/build_unix.go @@ -0,0 +1,16 @@ +// +build !windows + +package images + +import ( + "os" + "syscall" +) + +func checkHardLink(fi os.FileInfo) (devino, bool) { + st := fi.Sys().(*syscall.Stat_t) + return devino{ + Dev: uint64(st.Dev), + Ino: uint64(st.Ino), + }, st.Nlink > 1 +} diff --git a/pkg/bindings/images/build_windows.go b/pkg/bindings/images/build_windows.go new file mode 100644 index 000000000..bd71d1bf0 --- /dev/null +++ b/pkg/bindings/images/build_windows.go @@ -0,0 +1,9 @@ +package images + +import ( + "os" +) + +func checkHardLink(fi os.FileInfo) (devino, bool) { + return devino{}, false +} diff --git a/test/system/070-build.bats b/test/system/070-build.bats index d2d56c051..9e1559013 100644 --- a/test/system/070-build.bats +++ b/test/system/070-build.bats @@ -766,6 +766,26 @@ EOF is "$output" ".*/tmp/bogus: no such file or directory" } +@test "podman build COPY hardlinks " { + tmpdir=$PODMAN_TMPDIR/build-test + mkdir -p $tmpdir + + dockerfile=$tmpdir/Dockerfile + cat >$dockerfile < Date: Mon, 24 May 2021 15:41:51 -0400 Subject: Handle hard links in different directories Fixes: https://github.com/containers/podman/issues/10444 Signed-off-by: Daniel J Walsh --- pkg/bindings/images/build.go | 2 +- test/system/070-build.bats | 14 +++++++++++--- 2 files changed, 12 insertions(+), 4 deletions(-) (limited to 'pkg') diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go index b56afbceb..346d55c47 100644 --- a/pkg/bindings/images/build.go +++ b/pkg/bindings/images/build.go @@ -450,7 +450,7 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) { hdr.Typeflag = tar.TypeLink hdr.Linkname = orig hdr.Size = 0 - + hdr.Name = name return tw.WriteHeader(hdr) } f, err := os.Open(path) diff --git a/test/system/070-build.bats b/test/system/070-build.bats index 9e1559013..0f3f3fa7f 100644 --- a/test/system/070-build.bats +++ b/test/system/070-build.bats @@ -768,19 +768,27 @@ EOF @test "podman build COPY hardlinks " { tmpdir=$PODMAN_TMPDIR/build-test - mkdir -p $tmpdir + subdir=$tmpdir/subdir + subsubdir=$subdir/subsubdir + mkdir -p $subsubdir dockerfile=$tmpdir/Dockerfile cat >$dockerfile <