From 2130d185395b897798dcb1c17bbaf147dfb4da3c Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Mon, 6 Dec 2021 06:46:27 -0500 Subject: Update vendor or containers/common moving pkg/cgroups there [NO NEW TESTS NEEDED] This is just moving pkg/cgroups out so existing tests should be fine. Signed-off-by: Daniel J Walsh --- cmd/podman/common/create_opts.go | 2 +- cmd/podman/containers/pause.go | 2 +- cmd/podman/containers/unpause.go | 2 +- libpod/container_internal.go | 2 +- libpod/container_internal_linux.go | 2 +- libpod/info.go | 2 +- libpod/oci_conmon_linux.go | 2 +- libpod/pod_api.go | 2 +- libpod/runtime.go | 2 +- libpod/runtime_ctr.go | 2 +- libpod/runtime_pod_linux.go | 2 +- libpod/stats.go | 2 +- libpod/util_linux.go | 2 +- pkg/api/handlers/compat/containers_stats.go | 2 +- pkg/api/handlers/libpod/containers_stats.go | 2 +- pkg/cgroups/blkio.go | 149 ----- pkg/cgroups/cgroups.go | 673 -------------------- pkg/cgroups/cgroups_supported.go | 90 --- pkg/cgroups/cgroups_test.go | 32 - pkg/cgroups/cgroups_unsupported.go | 14 - pkg/cgroups/cpu.go | 160 ----- pkg/cgroups/cpuset.go | 85 --- pkg/cgroups/memory.go | 66 -- pkg/cgroups/pids.go | 66 -- pkg/cgroups/systemd.go | 79 --- pkg/domain/infra/abi/containers.go | 2 +- pkg/domain/infra/abi/pods_stats.go | 2 +- pkg/domain/infra/abi/system.go | 2 +- pkg/domain/infra/runtime_libpod.go | 2 +- pkg/specgen/generate/oci.go | 2 +- pkg/specgen/generate/validate.go | 2 +- pkg/specgen/namespaces.go | 2 +- test/e2e/common_test.go | 2 +- test/e2e/run_test.go | 2 +- utils/utils.go | 2 +- utils/utils_supported.go | 2 +- .../containers/common/pkg/cgroups/blkio.go | 149 +++++ .../containers/common/pkg/cgroups/cgroups.go | 674 +++++++++++++++++++++ .../common/pkg/cgroups/cgroups_supported.go | 90 +++ .../common/pkg/cgroups/cgroups_unsupported.go | 14 + .../containers/common/pkg/cgroups/cpu.go | 160 +++++ .../containers/common/pkg/cgroups/cpuset.go | 85 +++ .../containers/common/pkg/cgroups/memory.go | 66 ++ .../containers/common/pkg/cgroups/pids.go | 66 ++ .../containers/common/pkg/cgroups/systemd.go | 80 +++ vendor/modules.txt | 1 + 46 files changed, 1411 insertions(+), 1440 deletions(-) delete mode 100644 pkg/cgroups/blkio.go delete mode 100644 pkg/cgroups/cgroups.go delete mode 100644 pkg/cgroups/cgroups_supported.go delete mode 100644 pkg/cgroups/cgroups_test.go delete mode 100644 pkg/cgroups/cgroups_unsupported.go delete mode 100644 pkg/cgroups/cpu.go delete mode 100644 pkg/cgroups/cpuset.go delete mode 100644 pkg/cgroups/memory.go delete mode 100644 pkg/cgroups/pids.go delete mode 100644 pkg/cgroups/systemd.go create mode 100644 vendor/github.com/containers/common/pkg/cgroups/blkio.go create mode 100644 vendor/github.com/containers/common/pkg/cgroups/cgroups.go create mode 100644 vendor/github.com/containers/common/pkg/cgroups/cgroups_supported.go create mode 100644 vendor/github.com/containers/common/pkg/cgroups/cgroups_unsupported.go create mode 100644 vendor/github.com/containers/common/pkg/cgroups/cpu.go create mode 100644 vendor/github.com/containers/common/pkg/cgroups/cpuset.go create mode 100644 vendor/github.com/containers/common/pkg/cgroups/memory.go create mode 100644 vendor/github.com/containers/common/pkg/cgroups/pids.go create mode 100644 vendor/github.com/containers/common/pkg/cgroups/systemd.go diff --git a/cmd/podman/common/create_opts.go b/cmd/podman/common/create_opts.go index abb55f8c7..7d6471fd4 100644 --- a/cmd/podman/common/create_opts.go +++ b/cmd/podman/common/create_opts.go @@ -8,11 +8,11 @@ import ( "strconv" "strings" + "github.com/containers/common/pkg/cgroups" "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/cmd/podman/registry" "github.com/containers/podman/v3/libpod/network/types" "github.com/containers/podman/v3/pkg/api/handlers" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/rootless" "github.com/containers/podman/v3/pkg/specgen" diff --git a/cmd/podman/containers/pause.go b/cmd/podman/containers/pause.go index 8e1b69d71..1e37bab55 100644 --- a/cmd/podman/containers/pause.go +++ b/cmd/podman/containers/pause.go @@ -4,10 +4,10 @@ import ( "context" "fmt" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/cmd/podman/common" "github.com/containers/podman/v3/cmd/podman/registry" "github.com/containers/podman/v3/cmd/podman/utils" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/rootless" "github.com/pkg/errors" diff --git a/cmd/podman/containers/unpause.go b/cmd/podman/containers/unpause.go index e152f70e1..cf37b6369 100644 --- a/cmd/podman/containers/unpause.go +++ b/cmd/podman/containers/unpause.go @@ -4,10 +4,10 @@ import ( "context" "fmt" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/cmd/podman/common" "github.com/containers/podman/v3/cmd/podman/registry" "github.com/containers/podman/v3/cmd/podman/utils" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/rootless" "github.com/pkg/errors" diff --git a/libpod/container_internal.go b/libpod/container_internal.go index 871c6787a..d8187c609 100644 --- a/libpod/container_internal.go +++ b/libpod/container_internal.go @@ -17,10 +17,10 @@ import ( "github.com/containers/buildah/copier" "github.com/containers/buildah/pkg/overlay" butil "github.com/containers/buildah/util" + "github.com/containers/common/pkg/cgroups" "github.com/containers/common/pkg/chown" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/libpod/events" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/ctime" "github.com/containers/podman/v3/pkg/hooks" "github.com/containers/podman/v3/pkg/hooks/exec" diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index 956460c32..f741b589a 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -28,6 +28,7 @@ import ( "github.com/containers/buildah/pkg/overlay" butil "github.com/containers/buildah/util" "github.com/containers/common/pkg/apparmor" + "github.com/containers/common/pkg/cgroups" "github.com/containers/common/pkg/chown" "github.com/containers/common/pkg/config" "github.com/containers/common/pkg/subscriptions" @@ -36,7 +37,6 @@ import ( "github.com/containers/podman/v3/libpod/events" "github.com/containers/podman/v3/libpod/network/types" "github.com/containers/podman/v3/pkg/annotations" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/checkpoint/crutils" "github.com/containers/podman/v3/pkg/criu" "github.com/containers/podman/v3/pkg/lookup" diff --git a/libpod/info.go b/libpod/info.go index a1db5763a..354364ccc 100644 --- a/libpod/info.go +++ b/libpod/info.go @@ -14,11 +14,11 @@ import ( "github.com/containers/buildah" "github.com/containers/common/pkg/apparmor" + "github.com/containers/common/pkg/cgroups" "github.com/containers/common/pkg/seccomp" "github.com/containers/image/v5/pkg/sysregistriesv2" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/libpod/linkmode" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/rootless" "github.com/containers/storage" "github.com/containers/storage/pkg/system" diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go index a83f166a3..5446a8f8a 100644 --- a/libpod/oci_conmon_linux.go +++ b/libpod/oci_conmon_linux.go @@ -22,11 +22,11 @@ import ( "text/template" "time" + "github.com/containers/common/pkg/cgroups" "github.com/containers/common/pkg/config" conmonConfig "github.com/containers/conmon/runner/config" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/libpod/logs" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/checkpoint/crutils" "github.com/containers/podman/v3/pkg/errorhandling" "github.com/containers/podman/v3/pkg/rootless" diff --git a/libpod/pod_api.go b/libpod/pod_api.go index feb8ff250..80ecb690a 100644 --- a/libpod/pod_api.go +++ b/libpod/pod_api.go @@ -3,9 +3,9 @@ package libpod import ( "context" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/libpod/events" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/parallel" "github.com/containers/podman/v3/pkg/rootless" "github.com/pkg/errors" diff --git a/libpod/runtime.go b/libpod/runtime.go index 1a22cd09a..9794b3605 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -19,6 +19,7 @@ import ( "github.com/containers/buildah/pkg/parse" "github.com/containers/common/libimage" + "github.com/containers/common/pkg/cgroups" "github.com/containers/common/pkg/config" "github.com/containers/common/pkg/secrets" "github.com/containers/image/v5/pkg/sysregistriesv2" @@ -32,7 +33,6 @@ import ( nettypes "github.com/containers/podman/v3/libpod/network/types" "github.com/containers/podman/v3/libpod/plugin" "github.com/containers/podman/v3/libpod/shutdown" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/rootless" "github.com/containers/podman/v3/pkg/systemd" "github.com/containers/podman/v3/pkg/util" diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index 05f22c1fe..6c46eb747 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -9,11 +9,11 @@ import ( "time" "github.com/containers/buildah" + "github.com/containers/common/pkg/cgroups" "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/libpod/events" "github.com/containers/podman/v3/libpod/shutdown" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/domain/entities/reports" "github.com/containers/podman/v3/pkg/rootless" "github.com/containers/podman/v3/pkg/specgen" diff --git a/libpod/runtime_pod_linux.go b/libpod/runtime_pod_linux.go index 15050ef48..ee59cd8c3 100644 --- a/libpod/runtime_pod_linux.go +++ b/libpod/runtime_pod_linux.go @@ -9,10 +9,10 @@ import ( "path/filepath" "strings" + "github.com/containers/common/pkg/cgroups" "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/libpod/events" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/rootless" "github.com/containers/podman/v3/pkg/specgen" spec "github.com/opencontainers/runtime-spec/specs-go" diff --git a/libpod/stats.go b/libpod/stats.go index cc1250e83..879ae7207 100644 --- a/libpod/stats.go +++ b/libpod/stats.go @@ -8,8 +8,8 @@ import ( "syscall" "time" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/libpod/define" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/pkg/errors" ) diff --git a/libpod/util_linux.go b/libpod/util_linux.go index e2ea97185..ef871ef3d 100644 --- a/libpod/util_linux.go +++ b/libpod/util_linux.go @@ -7,8 +7,8 @@ import ( "strings" "syscall" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/libpod/define" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/rootless" "github.com/opencontainers/selinux/go-selinux/label" "github.com/pkg/errors" diff --git a/pkg/api/handlers/compat/containers_stats.go b/pkg/api/handlers/compat/containers_stats.go index 0051e7235..a92fe9fe0 100644 --- a/pkg/api/handlers/compat/containers_stats.go +++ b/pkg/api/handlers/compat/containers_stats.go @@ -5,11 +5,11 @@ import ( "net/http" "time" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/pkg/api/handlers/utils" api "github.com/containers/podman/v3/pkg/api/types" - "github.com/containers/podman/v3/pkg/cgroups" docker "github.com/docker/docker/api/types" "github.com/gorilla/schema" "github.com/pkg/errors" diff --git a/pkg/api/handlers/libpod/containers_stats.go b/pkg/api/handlers/libpod/containers_stats.go index 084f1252d..016214aa2 100644 --- a/pkg/api/handlers/libpod/containers_stats.go +++ b/pkg/api/handlers/libpod/containers_stats.go @@ -4,10 +4,10 @@ import ( "encoding/json" "net/http" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/pkg/api/handlers/utils" api "github.com/containers/podman/v3/pkg/api/types" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/domain/infra/abi" "github.com/containers/podman/v3/pkg/rootless" diff --git a/pkg/cgroups/blkio.go b/pkg/cgroups/blkio.go deleted file mode 100644 index bacd4eb93..000000000 --- a/pkg/cgroups/blkio.go +++ /dev/null @@ -1,149 +0,0 @@ -package cgroups - -import ( - "bufio" - "fmt" - "os" - "path/filepath" - "strconv" - "strings" - - spec "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" -) - -type blkioHandler struct { -} - -func getBlkioHandler() *blkioHandler { - return &blkioHandler{} -} - -// Apply set the specified constraints -func (c *blkioHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { - if res.BlockIO == nil { - return nil - } - return fmt.Errorf("blkio apply function not implemented yet") -} - -// Create the cgroup -func (c *blkioHandler) Create(ctr *CgroupControl) (bool, error) { - if ctr.cgroup2 { - return false, nil - } - return ctr.createCgroupDirectory(Blkio) -} - -// Destroy the cgroup -func (c *blkioHandler) Destroy(ctr *CgroupControl) error { - return rmDirRecursively(ctr.getCgroupv1Path(Blkio)) -} - -// Stat fills a metrics structure with usage stats for the controller -func (c *blkioHandler) Stat(ctr *CgroupControl, m *Metrics) error { - var ioServiceBytesRecursive []BlkIOEntry - - if ctr.cgroup2 { - // more details on the io.stat file format:X https://facebookmicrosites.github.io/cgroup2/docs/io-controller.html - values, err := readCgroup2MapFile(ctr, "io.stat") - if err != nil { - return err - } - for k, v := range values { - d := strings.Split(k, ":") - if len(d) != 2 { - continue - } - minor, err := strconv.ParseUint(d[0], 10, 0) - if err != nil { - return err - } - major, err := strconv.ParseUint(d[1], 10, 0) - if err != nil { - return err - } - - for _, item := range v { - d := strings.Split(item, "=") - if len(d) != 2 { - continue - } - op := d[0] - - // Accommodate the cgroup v1 naming - switch op { - case "rbytes": - op = "read" - case "wbytes": - op = "write" - } - - value, err := strconv.ParseUint(d[1], 10, 0) - if err != nil { - return err - } - - entry := BlkIOEntry{ - Op: op, - Major: major, - Minor: minor, - Value: value, - } - ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry) - } - } - } else { - BlkioRoot := ctr.getCgroupv1Path(Blkio) - - p := filepath.Join(BlkioRoot, "blkio.throttle.io_service_bytes_recursive") - f, err := os.Open(p) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return errors.Wrapf(err, "open %s", p) - } - defer f.Close() - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - parts := strings.Fields(line) - if len(parts) < 3 { - continue - } - d := strings.Split(parts[0], ":") - if len(d) != 2 { - continue - } - minor, err := strconv.ParseUint(d[0], 10, 0) - if err != nil { - return err - } - major, err := strconv.ParseUint(d[1], 10, 0) - if err != nil { - return err - } - - op := parts[1] - - value, err := strconv.ParseUint(parts[2], 10, 0) - if err != nil { - return err - } - entry := BlkIOEntry{ - Op: op, - Major: major, - Minor: minor, - Value: value, - } - ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry) - } - if err := scanner.Err(); err != nil { - return errors.Wrapf(err, "parse %s", p) - } - } - m.Blkio = BlkioMetrics{IoServiceBytesRecursive: ioServiceBytesRecursive} - return nil -} diff --git a/pkg/cgroups/cgroups.go b/pkg/cgroups/cgroups.go deleted file mode 100644 index d0c090012..000000000 --- a/pkg/cgroups/cgroups.go +++ /dev/null @@ -1,673 +0,0 @@ -package cgroups - -import ( - "bufio" - "bytes" - "fmt" - "io/ioutil" - "math" - "os" - "path/filepath" - "strconv" - "strings" - - "github.com/containers/podman/v3/pkg/rootless" - systemdDbus "github.com/coreos/go-systemd/v22/dbus" - "github.com/godbus/dbus/v5" - spec "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" - "github.com/sirupsen/logrus" -) - -var ( - // ErrCgroupDeleted means the cgroup was deleted - ErrCgroupDeleted = errors.New("cgroup deleted") - // ErrCgroupV1Rootless means the cgroup v1 were attempted to be used in rootless environment - ErrCgroupV1Rootless = errors.New("no support for CGroups V1 in rootless environments") - ErrStatCgroup = errors.New("no cgroup available for gathering user statistics") -) - -// CgroupControl controls a cgroup hierarchy -type CgroupControl struct { - cgroup2 bool - path string - systemd bool - // List of additional cgroup subsystems joined that - // do not have a custom handler. - additionalControllers []controller -} - -// CPUUsage keeps stats for the CPU usage (unit: nanoseconds) -type CPUUsage struct { - Kernel uint64 - Total uint64 - PerCPU []uint64 -} - -// MemoryUsage keeps stats for the memory usage -type MemoryUsage struct { - Usage uint64 - Limit uint64 -} - -// CPUMetrics keeps stats for the CPU usage -type CPUMetrics struct { - Usage CPUUsage -} - -// BlkIOEntry describes an entry in the blkio stats -type BlkIOEntry struct { - Op string - Major uint64 - Minor uint64 - Value uint64 -} - -// BlkioMetrics keeps usage stats for the blkio cgroup controller -type BlkioMetrics struct { - IoServiceBytesRecursive []BlkIOEntry -} - -// MemoryMetrics keeps usage stats for the memory cgroup controller -type MemoryMetrics struct { - Usage MemoryUsage -} - -// PidsMetrics keeps usage stats for the pids cgroup controller -type PidsMetrics struct { - Current uint64 -} - -// Metrics keeps usage stats for the cgroup controllers -type Metrics struct { - CPU CPUMetrics - Blkio BlkioMetrics - Memory MemoryMetrics - Pids PidsMetrics -} - -type controller struct { - name string - symlink bool -} - -type controllerHandler interface { - Create(*CgroupControl) (bool, error) - Apply(*CgroupControl, *spec.LinuxResources) error - Destroy(*CgroupControl) error - Stat(*CgroupControl, *Metrics) error -} - -const ( - cgroupRoot = "/sys/fs/cgroup" - // CPU is the cpu controller - CPU = "cpu" - // CPUAcct is the cpuacct controller - CPUAcct = "cpuacct" - // CPUset is the cpuset controller - CPUset = "cpuset" - // Memory is the memory controller - Memory = "memory" - // Pids is the pids controller - Pids = "pids" - // Blkio is the blkio controller - Blkio = "blkio" -) - -var handlers map[string]controllerHandler - -func init() { - handlers = make(map[string]controllerHandler) - handlers[CPU] = getCPUHandler() - handlers[CPUset] = getCpusetHandler() - handlers[Memory] = getMemoryHandler() - handlers[Pids] = getPidsHandler() - handlers[Blkio] = getBlkioHandler() -} - -// getAvailableControllers get the available controllers -func getAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]controller, error) { - if cgroup2 { - controllers := []controller{} - controllersFile := cgroupRoot + "/cgroup.controllers" - // rootless cgroupv2: check available controllers for current user, systemd or servicescope will inherit - if rootless.IsRootless() { - userSlice, err := getCgroupPathForCurrentProcess() - if err != nil { - return controllers, err - } - //userSlice already contains '/' so not adding here - basePath := cgroupRoot + userSlice - controllersFile = fmt.Sprintf("%s/cgroup.controllers", basePath) - } - controllersFileBytes, err := ioutil.ReadFile(controllersFile) - if err != nil { - return nil, errors.Wrapf(err, "failed while reading controllers for cgroup v2 from %q", controllersFile) - } - for _, controllerName := range strings.Fields(string(controllersFileBytes)) { - c := controller{ - name: controllerName, - symlink: false, - } - controllers = append(controllers, c) - } - return controllers, nil - } - - subsystems, _ := cgroupV1GetAllSubsystems() - controllers := []controller{} - // cgroupv1 and rootless: No subsystem is available: delegation is unsafe. - if rootless.IsRootless() { - return controllers, nil - } - - for _, name := range subsystems { - if _, found := exclude[name]; found { - continue - } - fileInfo, err := os.Stat(cgroupRoot + "/" + name) - if err != nil { - continue - } - c := controller{ - name: name, - symlink: !fileInfo.IsDir(), - } - controllers = append(controllers, c) - } - - return controllers, nil -} - -// GetAvailableControllers get string:bool map of all the available controllers -func GetAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]string, error) { - availableControllers, err := getAvailableControllers(exclude, cgroup2) - if err != nil { - return nil, err - } - controllerList := []string{} - for _, controller := range availableControllers { - controllerList = append(controllerList, controller.name) - } - - return controllerList, nil -} - -func cgroupV1GetAllSubsystems() ([]string, error) { - f, err := os.Open("/proc/cgroups") - if err != nil { - return nil, err - } - defer f.Close() - - subsystems := []string{} - - s := bufio.NewScanner(f) - for s.Scan() { - text := s.Text() - if text[0] != '#' { - parts := strings.Fields(text) - if len(parts) >= 4 && parts[3] != "0" { - subsystems = append(subsystems, parts[0]) - } - } - } - if err := s.Err(); err != nil { - return nil, err - } - return subsystems, nil -} - -func getCgroupPathForCurrentProcess() (string, error) { - path := fmt.Sprintf("/proc/%d/cgroup", os.Getpid()) - f, err := os.Open(path) - if err != nil { - return "", err - } - defer f.Close() - - cgroupPath := "" - s := bufio.NewScanner(f) - for s.Scan() { - text := s.Text() - procEntries := strings.SplitN(text, "::", 2) - // set process cgroupPath only if entry is valid - if len(procEntries) > 1 { - cgroupPath = procEntries[1] - } - } - if err := s.Err(); err != nil { - return cgroupPath, err - } - return cgroupPath, nil -} - -// getCgroupv1Path is a helper function to get the cgroup v1 path -func (c *CgroupControl) getCgroupv1Path(name string) string { - return filepath.Join(cgroupRoot, name, c.path) -} - -// createCgroupv2Path creates the cgroupv2 path and enables all the available controllers -func createCgroupv2Path(path string) (deferredError error) { - if !strings.HasPrefix(path, cgroupRoot+"/") { - return fmt.Errorf("invalid cgroup path %s", path) - } - content, err := ioutil.ReadFile(cgroupRoot + "/cgroup.controllers") - if err != nil { - return err - } - ctrs := bytes.Fields(content) - res := append([]byte("+"), bytes.Join(ctrs, []byte(" +"))...) - - current := "/sys/fs" - elements := strings.Split(path, "/") - for i, e := range elements[3:] { - current = filepath.Join(current, e) - if i > 0 { - if err := os.Mkdir(current, 0755); err != nil { - if !os.IsExist(err) { - return err - } - } else { - // If the directory was created, be sure it is not left around on errors. - defer func() { - if deferredError != nil { - os.Remove(current) - } - }() - } - } - // We enable the controllers for all the path components except the last one. It is not allowed to add - // PIDs if there are already enabled controllers. - if i < len(elements[3:])-1 { - if err := ioutil.WriteFile(filepath.Join(current, "cgroup.subtree_control"), res, 0755); err != nil { - return err - } - } - } - return nil -} - -// initialize initializes the specified hierarchy -func (c *CgroupControl) initialize() (err error) { - createdSoFar := map[string]controllerHandler{} - defer func() { - if err != nil { - for name, ctr := range createdSoFar { - if err := ctr.Destroy(c); err != nil { - logrus.Warningf("error cleaning up controller %s for %s", name, c.path) - } - } - } - }() - if c.cgroup2 { - if err := createCgroupv2Path(filepath.Join(cgroupRoot, c.path)); err != nil { - return errors.Wrapf(err, "error creating cgroup path %s", c.path) - } - } - for name, handler := range handlers { - created, err := handler.Create(c) - if err != nil { - return err - } - if created { - createdSoFar[name] = handler - } - } - - if !c.cgroup2 { - // We won't need to do this for cgroup v2 - for _, ctr := range c.additionalControllers { - if ctr.symlink { - continue - } - path := c.getCgroupv1Path(ctr.name) - if err := os.MkdirAll(path, 0755); err != nil { - return errors.Wrapf(err, "error creating cgroup path for %s", ctr.name) - } - } - } - - return nil -} - -func (c *CgroupControl) createCgroupDirectory(controller string) (bool, error) { - cPath := c.getCgroupv1Path(controller) - _, err := os.Stat(cPath) - if err == nil { - return false, nil - } - - if !os.IsNotExist(err) { - return false, err - } - - if err := os.MkdirAll(cPath, 0755); err != nil { - return false, errors.Wrapf(err, "error creating cgroup for %s", controller) - } - return true, nil -} - -func readFileAsUint64(path string) (uint64, error) { - data, err := ioutil.ReadFile(path) - if err != nil { - return 0, err - } - v := cleanString(string(data)) - if v == "max" { - return math.MaxUint64, nil - } - ret, err := strconv.ParseUint(v, 10, 64) - if err != nil { - return ret, errors.Wrapf(err, "parse %s from %s", v, path) - } - return ret, nil -} - -// New creates a new cgroup control -func New(path string, resources *spec.LinuxResources) (*CgroupControl, error) { - cgroup2, err := IsCgroup2UnifiedMode() - if err != nil { - return nil, err - } - control := &CgroupControl{ - cgroup2: cgroup2, - path: path, - } - - if !cgroup2 { - controllers, err := getAvailableControllers(handlers, false) - if err != nil { - return nil, err - } - control.additionalControllers = controllers - } - - if err := control.initialize(); err != nil { - return nil, err - } - - return control, nil -} - -// NewSystemd creates a new cgroup control -func NewSystemd(path string) (*CgroupControl, error) { - cgroup2, err := IsCgroup2UnifiedMode() - if err != nil { - return nil, err - } - control := &CgroupControl{ - cgroup2: cgroup2, - path: path, - systemd: true, - } - return control, nil -} - -// Load loads an existing cgroup control -func Load(path string) (*CgroupControl, error) { - cgroup2, err := IsCgroup2UnifiedMode() - if err != nil { - return nil, err - } - control := &CgroupControl{ - cgroup2: cgroup2, - path: path, - systemd: false, - } - if !cgroup2 { - controllers, err := getAvailableControllers(handlers, false) - if err != nil { - return nil, err - } - control.additionalControllers = controllers - } - if !cgroup2 { - oneExists := false - // check that the cgroup exists at least under one controller - for name := range handlers { - p := control.getCgroupv1Path(name) - if _, err := os.Stat(p); err == nil { - oneExists = true - break - } - } - - // if there is no controller at all, raise an error - if !oneExists { - if rootless.IsRootless() { - return nil, ErrCgroupV1Rootless - } - // compatible with the error code - // used by containerd/cgroups - return nil, ErrCgroupDeleted - } - } - return control, nil -} - -// CreateSystemdUnit creates the systemd cgroup -func (c *CgroupControl) CreateSystemdUnit(path string) error { - if !c.systemd { - return fmt.Errorf("the cgroup controller is not using systemd") - } - - conn, err := systemdDbus.New() - if err != nil { - return err - } - defer conn.Close() - - return systemdCreate(path, conn) -} - -// GetUserConnection returns a user connection to D-BUS -func GetUserConnection(uid int) (*systemdDbus.Conn, error) { - return systemdDbus.NewConnection(func() (*dbus.Conn, error) { - return dbusAuthConnection(uid, dbus.SessionBusPrivateNoAutoStartup) - }) -} - -// CreateSystemdUserUnit creates the systemd cgroup for the specified user -func (c *CgroupControl) CreateSystemdUserUnit(path string, uid int) error { - if !c.systemd { - return fmt.Errorf("the cgroup controller is not using systemd") - } - - conn, err := GetUserConnection(uid) - if err != nil { - return err - } - defer conn.Close() - - return systemdCreate(path, conn) -} - -func dbusAuthConnection(uid int, createBus func(opts ...dbus.ConnOption) (*dbus.Conn, error)) (*dbus.Conn, error) { - conn, err := createBus() - if err != nil { - return nil, err - } - - methods := []dbus.Auth{dbus.AuthExternal(strconv.Itoa(uid))} - - err = conn.Auth(methods) - if err != nil { - conn.Close() - return nil, err - } - if err := conn.Hello(); err != nil { - return nil, err - } - - return conn, nil -} - -// Delete cleans a cgroup -func (c *CgroupControl) Delete() error { - return c.DeleteByPath(c.path) -} - -// rmDirRecursively delete recursively a cgroup directory. -// It differs from os.RemoveAll as it doesn't attempt to unlink files. -// On cgroupfs we are allowed only to rmdir empty directories. -func rmDirRecursively(path string) error { - if err := os.Remove(path); err == nil || os.IsNotExist(err) { - return nil - } - entries, err := ioutil.ReadDir(path) - if err != nil { - return err - } - for _, i := range entries { - if i.IsDir() { - if err := rmDirRecursively(filepath.Join(path, i.Name())); err != nil { - return err - } - } - } - if err := os.Remove(path); err != nil { - if !os.IsNotExist(err) { - return errors.Wrapf(err, "remove %s", path) - } - } - return nil -} - -// DeleteByPathConn deletes the specified cgroup path using the specified -// dbus connection if needed. -func (c *CgroupControl) DeleteByPathConn(path string, conn *systemdDbus.Conn) error { - if c.systemd { - return systemdDestroyConn(path, conn) - } - if c.cgroup2 { - return rmDirRecursively(filepath.Join(cgroupRoot, c.path)) - } - var lastError error - for _, h := range handlers { - if err := h.Destroy(c); err != nil { - lastError = err - } - } - - for _, ctr := range c.additionalControllers { - if ctr.symlink { - continue - } - p := c.getCgroupv1Path(ctr.name) - if err := rmDirRecursively(p); err != nil { - lastError = errors.Wrapf(err, "remove %s", p) - } - } - return lastError -} - -// DeleteByPath deletes the specified cgroup path -func (c *CgroupControl) DeleteByPath(path string) error { - if c.systemd { - conn, err := systemdDbus.New() - if err != nil { - return err - } - defer conn.Close() - return c.DeleteByPathConn(path, conn) - } - return c.DeleteByPathConn(path, nil) -} - -// Update updates the cgroups -func (c *CgroupControl) Update(resources *spec.LinuxResources) error { - for _, h := range handlers { - if err := h.Apply(c, resources); err != nil { - return err - } - } - return nil -} - -// AddPid moves the specified pid to the cgroup -func (c *CgroupControl) AddPid(pid int) error { - pidString := []byte(fmt.Sprintf("%d\n", pid)) - - if c.cgroup2 { - p := filepath.Join(cgroupRoot, c.path, "cgroup.procs") - if err := ioutil.WriteFile(p, pidString, 0644); err != nil { - return errors.Wrapf(err, "write %s", p) - } - return nil - } - - names := make([]string, 0, len(handlers)) - for n := range handlers { - names = append(names, n) - } - - for _, c := range c.additionalControllers { - if !c.symlink { - names = append(names, c.name) - } - } - - for _, n := range names { - // If we aren't using cgroup2, we won't write correctly to unified hierarchy - if !c.cgroup2 && n == "unified" { - continue - } - p := filepath.Join(c.getCgroupv1Path(n), "tasks") - if err := ioutil.WriteFile(p, pidString, 0644); err != nil { - return errors.Wrapf(err, "write %s", p) - } - } - return nil -} - -// Stat returns usage statistics for the cgroup -func (c *CgroupControl) Stat() (*Metrics, error) { - m := Metrics{} - found := false - for _, h := range handlers { - if err := h.Stat(c, &m); err != nil { - if !os.IsNotExist(errors.Cause(err)) { - return nil, err - } - logrus.Warningf("Failed to retrieve cgroup stats: %v", err) - continue - } - found = true - } - if !found { - return nil, ErrStatCgroup - } - return &m, nil -} - -func readCgroup2MapPath(path string) (map[string][]string, error) { - ret := map[string][]string{} - f, err := os.Open(path) - if err != nil { - if os.IsNotExist(err) { - return ret, nil - } - return nil, errors.Wrapf(err, "open file %s", path) - } - defer f.Close() - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - parts := strings.Fields(line) - if len(parts) < 2 { - continue - } - ret[parts[0]] = parts[1:] - } - if err := scanner.Err(); err != nil { - return nil, errors.Wrapf(err, "parsing file %s", path) - } - return ret, nil -} - -func readCgroup2MapFile(ctr *CgroupControl, name string) (map[string][]string, error) { - p := filepath.Join(cgroupRoot, ctr.path, name) - - return readCgroup2MapPath(p) -} diff --git a/pkg/cgroups/cgroups_supported.go b/pkg/cgroups/cgroups_supported.go deleted file mode 100644 index fe17db7f7..000000000 --- a/pkg/cgroups/cgroups_supported.go +++ /dev/null @@ -1,90 +0,0 @@ -// +build linux - -package cgroups - -import ( - "bufio" - "fmt" - "os" - "path/filepath" - "strings" - "sync" - "syscall" - - "github.com/pkg/errors" - "golang.org/x/sys/unix" -) - -var ( - isUnifiedOnce sync.Once - isUnified bool - isUnifiedErr error -) - -// IsCgroup2UnifiedMode returns whether we are running in cgroup 2 cgroup2 mode. -func IsCgroup2UnifiedMode() (bool, error) { - isUnifiedOnce.Do(func() { - var st syscall.Statfs_t - if err := syscall.Statfs("/sys/fs/cgroup", &st); err != nil { - isUnified, isUnifiedErr = false, err - } else { - isUnified, isUnifiedErr = st.Type == unix.CGROUP2_SUPER_MAGIC, nil - } - }) - return isUnified, isUnifiedErr -} - -// UserOwnsCurrentSystemdCgroup checks whether the current EUID owns the -// current cgroup. -func UserOwnsCurrentSystemdCgroup() (bool, error) { - uid := os.Geteuid() - - cgroup2, err := IsCgroup2UnifiedMode() - if err != nil { - return false, err - } - - f, err := os.Open("/proc/self/cgroup") - if err != nil { - return false, err - } - defer f.Close() - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - parts := strings.SplitN(line, ":", 3) - - if len(parts) < 3 { - continue - } - - var cgroupPath string - - if cgroup2 { - cgroupPath = filepath.Join(cgroupRoot, parts[2]) - } else { - if parts[1] != "name=systemd" { - continue - } - cgroupPath = filepath.Join(cgroupRoot, "systemd", parts[2]) - } - - st, err := os.Stat(cgroupPath) - if err != nil { - return false, err - } - s := st.Sys() - if s == nil { - return false, fmt.Errorf("error stat cgroup path %s", cgroupPath) - } - - if int(s.(*syscall.Stat_t).Uid) != uid { - return false, nil - } - } - if err := scanner.Err(); err != nil { - return false, errors.Wrapf(err, "parsing file /proc/self/cgroup") - } - return true, nil -} diff --git a/pkg/cgroups/cgroups_test.go b/pkg/cgroups/cgroups_test.go deleted file mode 100644 index 22d5efb76..000000000 --- a/pkg/cgroups/cgroups_test.go +++ /dev/null @@ -1,32 +0,0 @@ -package cgroups - -import ( - "testing" - - "github.com/containers/podman/v3/pkg/rootless" - spec "github.com/opencontainers/runtime-spec/specs-go" -) - -func TestCreated(t *testing.T) { - // tests only works in rootless mode - if rootless.IsRootless() { - return - } - - var resources spec.LinuxResources - cgr, err := New("machine.slice", &resources) - if err != nil { - t.Error(err) - } - if err := cgr.Delete(); err != nil { - t.Error(err) - } - - cgr, err = NewSystemd("machine.slice") - if err != nil { - t.Error(err) - } - if err := cgr.Delete(); err != nil { - t.Error(err) - } -} diff --git a/pkg/cgroups/cgroups_unsupported.go b/pkg/cgroups/cgroups_unsupported.go deleted file mode 100644 index cd140fbf3..000000000 --- a/pkg/cgroups/cgroups_unsupported.go +++ /dev/null @@ -1,14 +0,0 @@ -// +build !linux - -package cgroups - -// IsCgroup2UnifiedMode returns whether we are running in cgroup 2 cgroup2 mode. -func IsCgroup2UnifiedMode() (bool, error) { - return false, nil -} - -// UserOwnsCurrentSystemdCgroup checks whether the current EUID owns the -// current cgroup. -func UserOwnsCurrentSystemdCgroup() (bool, error) { - return false, nil -} diff --git a/pkg/cgroups/cpu.go b/pkg/cgroups/cpu.go deleted file mode 100644 index 23539757d..000000000 --- a/pkg/cgroups/cpu.go +++ /dev/null @@ -1,160 +0,0 @@ -package cgroups - -import ( - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strconv" - "strings" - - spec "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" -) - -type cpuHandler struct { -} - -func getCPUHandler() *cpuHandler { - return &cpuHandler{} -} - -func cleanString(s string) string { - return strings.Trim(s, "\n") -} - -func readAcct(ctr *CgroupControl, name string) (uint64, error) { - p := filepath.Join(ctr.getCgroupv1Path(CPUAcct), name) - return readFileAsUint64(p) -} - -func readAcctList(ctr *CgroupControl, name string) ([]uint64, error) { - p := filepath.Join(ctr.getCgroupv1Path(CPUAcct), name) - data, err := ioutil.ReadFile(p) - if err != nil { - return nil, errors.Wrapf(err, "reading %s", p) - } - r := []uint64{} - for _, s := range strings.Split(string(data), " ") { - s = cleanString(s) - if s == "" { - break - } - v, err := strconv.ParseUint(s, 10, 64) - if err != nil { - return nil, errors.Wrapf(err, "parsing %s", s) - } - r = append(r, v) - } - return r, nil -} - -// Apply set the specified constraints -func (c *cpuHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { - if res.CPU == nil { - return nil - } - return fmt.Errorf("cpu apply not implemented yet") -} - -// Create the cgroup -func (c *cpuHandler) Create(ctr *CgroupControl) (bool, error) { - if ctr.cgroup2 { - return false, nil - } - return ctr.createCgroupDirectory(CPU) -} - -// Destroy the cgroup -func (c *cpuHandler) Destroy(ctr *CgroupControl) error { - return rmDirRecursively(ctr.getCgroupv1Path(CPU)) -} - -// Stat fills a metrics structure with usage stats for the controller -func (c *cpuHandler) Stat(ctr *CgroupControl, m *Metrics) error { - var err error - usage := CPUUsage{} - if ctr.cgroup2 { - values, err := readCgroup2MapFile(ctr, "cpu.stat") - if err != nil { - return err - } - if val, found := values["usage_usec"]; found { - usage.Total, err = strconv.ParseUint(cleanString(val[0]), 10, 64) - if err != nil { - return err - } - usage.Kernel *= 1000 - } - if val, found := values["system_usec"]; found { - usage.Kernel, err = strconv.ParseUint(cleanString(val[0]), 10, 64) - if err != nil { - return err - } - usage.Total *= 1000 - } - // FIXME: How to read usage.PerCPU? - } else { - usage.Total, err = readAcct(ctr, "cpuacct.usage") - if err != nil { - if !os.IsNotExist(errors.Cause(err)) { - return err - } - usage.Total = 0 - } - usage.Kernel, err = readAcct(ctr, "cpuacct.usage_sys") - if err != nil { - if !os.IsNotExist(errors.Cause(err)) { - return err - } - usage.Kernel = 0 - } - usage.PerCPU, err = readAcctList(ctr, "cpuacct.usage_percpu") - if err != nil { - if !os.IsNotExist(errors.Cause(err)) { - return err - } - usage.PerCPU = nil - } - } - m.CPU = CPUMetrics{Usage: usage} - return nil -} - -// GetSystemCPUUsage returns the system usage for all the cgroups -func GetSystemCPUUsage() (uint64, error) { - cgroupv2, err := IsCgroup2UnifiedMode() - if err != nil { - return 0, err - } - if !cgroupv2 { - p := filepath.Join(cgroupRoot, CPUAcct, "cpuacct.usage") - return readFileAsUint64(p) - } - - files, err := ioutil.ReadDir(cgroupRoot) - if err != nil { - return 0, err - } - var total uint64 - for _, file := range files { - if !file.IsDir() { - continue - } - p := filepath.Join(cgroupRoot, file.Name(), "cpu.stat") - - values, err := readCgroup2MapPath(p) - if err != nil { - return 0, err - } - - if val, found := values["usage_usec"]; found { - v, err := strconv.ParseUint(cleanString(val[0]), 10, 64) - if err != nil { - return 0, err - } - total += v * 1000 - } - } - return total, nil -} diff --git a/pkg/cgroups/cpuset.go b/pkg/cgroups/cpuset.go deleted file mode 100644 index 46d0484f2..000000000 --- a/pkg/cgroups/cpuset.go +++ /dev/null @@ -1,85 +0,0 @@ -package cgroups - -import ( - "fmt" - "io/ioutil" - "path/filepath" - "strings" - - spec "github.com/opencontainers/runtime-spec/specs-go" - "github.com/pkg/errors" -) - -type cpusetHandler struct { -} - -func cpusetCopyFileFromParent(dir, file string, cgroupv2 bool) ([]byte, error) { - if dir == cgroupRoot { - return nil, fmt.Errorf("could not find parent to initialize cpuset %s", file) - } - path := filepath.Join(dir, file) - parentPath := path - if cgroupv2 { - parentPath = fmt.Sprintf("%s.effective", parentPath) - } - data, err := ioutil.ReadFile(parentPath) - if err != nil { - return nil, errors.Wrapf(err, "open %s", path) - } - if len(strings.Trim(string(data), "\n")) != 0 { - return data, nil - } - data, err = cpusetCopyFileFromParent(filepath.Dir(dir), file, cgroupv2) - if err != nil { - return nil, err - } - if err := ioutil.WriteFile(path, data, 0644); err != nil { - return nil, errors.Wrapf(err, "write %s", path) - } - return data, nil -} - -func cpusetCopyFromParent(path string, cgroupv2 bool) error { - for _, file := range []string{"cpuset.cpus", "cpuset.mems"} { - if _, err := cpusetCopyFileFromParent(path, file, cgroupv2); err != nil { - return err - } - } - return nil -} - -func getCpusetHandler() *cpusetHandler { - return &cpusetHandler{} -} - -// Apply set the specified constraints -func (c *cpusetHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { - if res.CPU == nil { - return nil - } - return fmt.Errorf("cpuset apply not implemented yet") -} - -// Create the cgroup -func (c *cpusetHandler) Create(ctr *CgroupControl) (bool, error) { - if ctr.cgroup2 { - path := filepath.Join(cgroupRoot, ctr.path) - return true, cpusetCopyFromParent(path, true) - } - - created, err := ctr.createCgroupDirectory(CPUset) - if !created || err != nil { - return created, err - } - return true, cpusetCopyFromParent(ctr.getCgroupv1Path(CPUset), false) -} - -// Destroy the cgroup -func (c *cpusetHandler) Destroy(ctr *CgroupControl) error { - return rmDirRecursively(ctr.getCgroupv1Path(CPUset)) -} - -// Stat fills a metrics structure with usage stats for the controller -func (c *cpusetHandler) Stat(ctr *CgroupControl, m *Metrics) error { - return nil -} diff --git a/pkg/cgroups/memory.go b/pkg/cgroups/memory.go deleted file mode 100644 index b3991f7e3..000000000 --- a/pkg/cgroups/memory.go +++ /dev/null @@ -1,66 +0,0 @@ -package cgroups - -import ( - "fmt" - "path/filepath" - - spec "github.com/opencontainers/runtime-spec/specs-go" -) - -type memHandler struct { -} - -func getMemoryHandler() *memHandler { - return &memHandler{} -} - -// Apply set the specified constraints -func (c *memHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { - if res.Memory == nil { - return nil - } - return fmt.Errorf("memory apply not implemented yet") -} - -// Create the cgroup -func (c *memHandler) Create(ctr *CgroupControl) (bool, error) { - if ctr.cgroup2 { - return false, nil - } - return ctr.createCgroupDirectory(Memory) -} - -// Destroy the cgroup -func (c *memHandler) Destroy(ctr *CgroupControl) error { - return rmDirRecursively(ctr.getCgroupv1Path(Memory)) -} - -// Stat fills a metrics structure with usage stats for the controller -func (c *memHandler) Stat(ctr *CgroupControl, m *Metrics) error { - var err error - usage := MemoryUsage{} - - var memoryRoot string - filenames := map[string]string{} - - if ctr.cgroup2 { - memoryRoot = filepath.Join(cgroupRoot, ctr.path) - filenames["usage"] = "memory.current" - filenames["limit"] = "memory.max" - } else { - memoryRoot = ctr.getCgroupv1Path(Memory) - filenames["usage"] = "memory.usage_in_bytes" - filenames["limit"] = "memory.limit_in_bytes" - } - usage.Usage, err = readFileAsUint64(filepath.Join(memoryRoot, filenames["usage"])) - if err != nil { - return err - } - usage.Limit, err = readFileAsUint64(filepath.Join(memoryRoot, filenames["limit"])) - if err != nil { - return err - } - - m.Memory = MemoryMetrics{Usage: usage} - return nil -} diff --git a/pkg/cgroups/pids.go b/pkg/cgroups/pids.go deleted file mode 100644 index b2bfebe4d..000000000 --- a/pkg/cgroups/pids.go +++ /dev/null @@ -1,66 +0,0 @@ -package cgroups - -import ( - "fmt" - "io/ioutil" - "path/filepath" - - spec "github.com/opencontainers/runtime-spec/specs-go" -) - -type pidHandler struct { -} - -func getPidsHandler() *pidHandler { - return &pidHandler{} -} - -// Apply set the specified constraints -func (c *pidHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { - if res.Pids == nil { - return nil - } - var PIDRoot string - - if ctr.cgroup2 { - PIDRoot = filepath.Join(cgroupRoot, ctr.path) - } else { - PIDRoot = ctr.getCgroupv1Path(Pids) - } - - p := filepath.Join(PIDRoot, "pids.max") - return ioutil.WriteFile(p, []byte(fmt.Sprintf("%d\n", res.Pids.Limit)), 0644) -} - -// Create the cgroup -func (c *pidHandler) Create(ctr *CgroupControl) (bool, error) { - return ctr.createCgroupDirectory(Pids) -} - -// Destroy the cgroup -func (c *pidHandler) Destroy(ctr *CgroupControl) error { - return rmDirRecursively(ctr.getCgroupv1Path(Pids)) -} - -// Stat fills a metrics structure with usage stats for the controller -func (c *pidHandler) Stat(ctr *CgroupControl, m *Metrics) error { - if ctr.path == "" { - // nothing we can do to retrieve the pids.current path - return nil - } - - var PIDRoot string - if ctr.cgroup2 { - PIDRoot = filepath.Join(cgroupRoot, ctr.path) - } else { - PIDRoot = ctr.getCgroupv1Path(Pids) - } - - current, err := readFileAsUint64(filepath.Join(PIDRoot, "pids.current")) - if err != nil { - return err - } - - m.Pids = PidsMetrics{Current: current} - return nil -} diff --git a/pkg/cgroups/systemd.go b/pkg/cgroups/systemd.go deleted file mode 100644 index f26988c5a..000000000 --- a/pkg/cgroups/systemd.go +++ /dev/null @@ -1,79 +0,0 @@ -package cgroups - -import ( - "fmt" - "path/filepath" - "strings" - - systemdDbus "github.com/coreos/go-systemd/v22/dbus" - "github.com/godbus/dbus/v5" -) - -func systemdCreate(path string, c *systemdDbus.Conn) error { - slice, name := filepath.Split(path) - slice = strings.TrimSuffix(slice, "/") - - var lastError error - for i := 0; i < 2; i++ { - properties := []systemdDbus.Property{ - systemdDbus.PropDescription(fmt.Sprintf("cgroup %s", name)), - systemdDbus.PropWants(slice), - } - pMap := map[string]bool{ - "DefaultDependencies": false, - "MemoryAccounting": true, - "CPUAccounting": true, - "BlockIOAccounting": true, - } - if i == 0 { - pMap["Delegate"] = true - } - for k, v := range pMap { - p := systemdDbus.Property{ - Name: k, - Value: dbus.MakeVariant(v), - } - properties = append(properties, p) - } - - ch := make(chan string) - _, err := c.StartTransientUnit(name, "replace", properties, ch) - if err != nil { - lastError = err - continue - } - <-ch - return nil - } - return lastError -} - -/* - systemdDestroyConn is copied from containerd/cgroups/systemd.go file, that - has the following license: - - Copyright The containerd Authors. - - Licensed under the Apache License, Version 2.0 (the "License"); - you may not use this file except in compliance with the License. - You may obtain a copy of the License at - - https://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. -*/ -func systemdDestroyConn(path string, c *systemdDbus.Conn) error { - name := filepath.Base(path) - - ch := make(chan string) - _, err := c.StopUnit(name, "replace", ch) - if err != nil { - return err - } - <-ch - return nil -} diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index 631eb3a43..d1af4a479 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -10,13 +10,13 @@ import ( "time" "github.com/containers/buildah" + "github.com/containers/common/pkg/cgroups" "github.com/containers/common/pkg/config" "github.com/containers/image/v5/manifest" "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/libpod/define" "github.com/containers/podman/v3/libpod/events" "github.com/containers/podman/v3/libpod/logs" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/checkpoint" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/domain/entities/reports" diff --git a/pkg/domain/infra/abi/pods_stats.go b/pkg/domain/infra/abi/pods_stats.go index abb15bd1f..70f953fd4 100644 --- a/pkg/domain/infra/abi/pods_stats.go +++ b/pkg/domain/infra/abi/pods_stats.go @@ -4,8 +4,8 @@ import ( "context" "fmt" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/libpod" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/rootless" "github.com/containers/podman/v3/utils" diff --git a/pkg/domain/infra/abi/system.go b/pkg/domain/infra/abi/system.go index e6c9d850b..02b4181f5 100644 --- a/pkg/domain/infra/abi/system.go +++ b/pkg/domain/infra/abi/system.go @@ -8,9 +8,9 @@ import ( "os/exec" "path/filepath" + "github.com/containers/common/pkg/cgroups" "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/libpod/define" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/domain/entities/reports" "github.com/containers/podman/v3/pkg/rootless" diff --git a/pkg/domain/infra/runtime_libpod.go b/pkg/domain/infra/runtime_libpod.go index 90eb6abeb..b307d04b3 100644 --- a/pkg/domain/infra/runtime_libpod.go +++ b/pkg/domain/infra/runtime_libpod.go @@ -9,9 +9,9 @@ import ( "os/signal" "sync" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/cmd/podman/utils" "github.com/containers/podman/v3/libpod" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/namespaces" "github.com/containers/podman/v3/pkg/rootless" diff --git a/pkg/specgen/generate/oci.go b/pkg/specgen/generate/oci.go index df5788099..9f8807915 100644 --- a/pkg/specgen/generate/oci.go +++ b/pkg/specgen/generate/oci.go @@ -6,10 +6,10 @@ import ( "strings" "github.com/containers/common/libimage" + "github.com/containers/common/pkg/cgroups" "github.com/containers/common/pkg/config" "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/libpod/define" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/rootless" "github.com/containers/podman/v3/pkg/specgen" spec "github.com/opencontainers/runtime-spec/specs-go" diff --git a/pkg/specgen/generate/validate.go b/pkg/specgen/generate/validate.go index b0d84825e..a44bf9979 100644 --- a/pkg/specgen/generate/validate.go +++ b/pkg/specgen/generate/validate.go @@ -4,8 +4,8 @@ import ( "os" "path/filepath" + "github.com/containers/common/pkg/cgroups" "github.com/containers/common/pkg/sysinfo" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/specgen" "github.com/containers/podman/v3/utils" "github.com/pkg/errors" diff --git a/pkg/specgen/namespaces.go b/pkg/specgen/namespaces.go index 2f4c48811..bb5385ef1 100644 --- a/pkg/specgen/namespaces.go +++ b/pkg/specgen/namespaces.go @@ -5,7 +5,7 @@ import ( "os" "strings" - "github.com/containers/podman/v3/pkg/cgroups" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/pkg/rootless" "github.com/containers/podman/v3/pkg/util" "github.com/containers/storage" diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go index a411a860b..6e1a62b99 100644 --- a/test/e2e/common_test.go +++ b/test/e2e/common_test.go @@ -15,8 +15,8 @@ import ( "testing" "time" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/libpod/define" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/podman/v3/pkg/inspect" "github.com/containers/podman/v3/pkg/rootless" . "github.com/containers/podman/v3/test/utils" diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go index f063c79e9..8db23080e 100644 --- a/test/e2e/run_test.go +++ b/test/e2e/run_test.go @@ -12,7 +12,7 @@ import ( "syscall" "time" - "github.com/containers/podman/v3/pkg/cgroups" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/pkg/rootless" . "github.com/containers/podman/v3/test/utils" "github.com/containers/storage/pkg/stringid" diff --git a/utils/utils.go b/utils/utils.go index 80eed7536..095370a08 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -12,8 +12,8 @@ import ( "strings" "sync" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/libpod/define" - "github.com/containers/podman/v3/pkg/cgroups" "github.com/containers/storage/pkg/archive" "github.com/godbus/dbus/v5" "github.com/pkg/errors" diff --git a/utils/utils_supported.go b/utils/utils_supported.go index 0f0c9a9ba..be200509b 100644 --- a/utils/utils_supported.go +++ b/utils/utils_supported.go @@ -11,7 +11,7 @@ import ( "path/filepath" "strings" - "github.com/containers/podman/v3/pkg/cgroups" + "github.com/containers/common/pkg/cgroups" "github.com/containers/podman/v3/pkg/rootless" systemdDbus "github.com/coreos/go-systemd/v22/dbus" "github.com/godbus/dbus/v5" diff --git a/vendor/github.com/containers/common/pkg/cgroups/blkio.go b/vendor/github.com/containers/common/pkg/cgroups/blkio.go new file mode 100644 index 000000000..bacd4eb93 --- /dev/null +++ b/vendor/github.com/containers/common/pkg/cgroups/blkio.go @@ -0,0 +1,149 @@ +package cgroups + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" +) + +type blkioHandler struct { +} + +func getBlkioHandler() *blkioHandler { + return &blkioHandler{} +} + +// Apply set the specified constraints +func (c *blkioHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { + if res.BlockIO == nil { + return nil + } + return fmt.Errorf("blkio apply function not implemented yet") +} + +// Create the cgroup +func (c *blkioHandler) Create(ctr *CgroupControl) (bool, error) { + if ctr.cgroup2 { + return false, nil + } + return ctr.createCgroupDirectory(Blkio) +} + +// Destroy the cgroup +func (c *blkioHandler) Destroy(ctr *CgroupControl) error { + return rmDirRecursively(ctr.getCgroupv1Path(Blkio)) +} + +// Stat fills a metrics structure with usage stats for the controller +func (c *blkioHandler) Stat(ctr *CgroupControl, m *Metrics) error { + var ioServiceBytesRecursive []BlkIOEntry + + if ctr.cgroup2 { + // more details on the io.stat file format:X https://facebookmicrosites.github.io/cgroup2/docs/io-controller.html + values, err := readCgroup2MapFile(ctr, "io.stat") + if err != nil { + return err + } + for k, v := range values { + d := strings.Split(k, ":") + if len(d) != 2 { + continue + } + minor, err := strconv.ParseUint(d[0], 10, 0) + if err != nil { + return err + } + major, err := strconv.ParseUint(d[1], 10, 0) + if err != nil { + return err + } + + for _, item := range v { + d := strings.Split(item, "=") + if len(d) != 2 { + continue + } + op := d[0] + + // Accommodate the cgroup v1 naming + switch op { + case "rbytes": + op = "read" + case "wbytes": + op = "write" + } + + value, err := strconv.ParseUint(d[1], 10, 0) + if err != nil { + return err + } + + entry := BlkIOEntry{ + Op: op, + Major: major, + Minor: minor, + Value: value, + } + ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry) + } + } + } else { + BlkioRoot := ctr.getCgroupv1Path(Blkio) + + p := filepath.Join(BlkioRoot, "blkio.throttle.io_service_bytes_recursive") + f, err := os.Open(p) + if err != nil { + if os.IsNotExist(err) { + return nil + } + return errors.Wrapf(err, "open %s", p) + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(line) + if len(parts) < 3 { + continue + } + d := strings.Split(parts[0], ":") + if len(d) != 2 { + continue + } + minor, err := strconv.ParseUint(d[0], 10, 0) + if err != nil { + return err + } + major, err := strconv.ParseUint(d[1], 10, 0) + if err != nil { + return err + } + + op := parts[1] + + value, err := strconv.ParseUint(parts[2], 10, 0) + if err != nil { + return err + } + entry := BlkIOEntry{ + Op: op, + Major: major, + Minor: minor, + Value: value, + } + ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry) + } + if err := scanner.Err(); err != nil { + return errors.Wrapf(err, "parse %s", p) + } + } + m.Blkio = BlkioMetrics{IoServiceBytesRecursive: ioServiceBytesRecursive} + return nil +} diff --git a/vendor/github.com/containers/common/pkg/cgroups/cgroups.go b/vendor/github.com/containers/common/pkg/cgroups/cgroups.go new file mode 100644 index 000000000..d0bcd8bfd --- /dev/null +++ b/vendor/github.com/containers/common/pkg/cgroups/cgroups.go @@ -0,0 +1,674 @@ +package cgroups + +import ( + "bufio" + "bytes" + "context" + "fmt" + "io/ioutil" + "math" + "os" + "path/filepath" + "strconv" + "strings" + + "github.com/containers/storage/pkg/unshare" + systemdDbus "github.com/coreos/go-systemd/v22/dbus" + "github.com/godbus/dbus/v5" + spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" +) + +var ( + // ErrCgroupDeleted means the cgroup was deleted + ErrCgroupDeleted = errors.New("cgroup deleted") + // ErrCgroupV1Rootless means the cgroup v1 were attempted to be used in rootless environment + ErrCgroupV1Rootless = errors.New("no support for CGroups V1 in rootless environments") + ErrStatCgroup = errors.New("no cgroup available for gathering user statistics") +) + +// CgroupControl controls a cgroup hierarchy +type CgroupControl struct { + cgroup2 bool + path string + systemd bool + // List of additional cgroup subsystems joined that + // do not have a custom handler. + additionalControllers []controller +} + +// CPUUsage keeps stats for the CPU usage (unit: nanoseconds) +type CPUUsage struct { + Kernel uint64 + Total uint64 + PerCPU []uint64 +} + +// MemoryUsage keeps stats for the memory usage +type MemoryUsage struct { + Usage uint64 + Limit uint64 +} + +// CPUMetrics keeps stats for the CPU usage +type CPUMetrics struct { + Usage CPUUsage +} + +// BlkIOEntry describes an entry in the blkio stats +type BlkIOEntry struct { + Op string + Major uint64 + Minor uint64 + Value uint64 +} + +// BlkioMetrics keeps usage stats for the blkio cgroup controller +type BlkioMetrics struct { + IoServiceBytesRecursive []BlkIOEntry +} + +// MemoryMetrics keeps usage stats for the memory cgroup controller +type MemoryMetrics struct { + Usage MemoryUsage +} + +// PidsMetrics keeps usage stats for the pids cgroup controller +type PidsMetrics struct { + Current uint64 +} + +// Metrics keeps usage stats for the cgroup controllers +type Metrics struct { + CPU CPUMetrics + Blkio BlkioMetrics + Memory MemoryMetrics + Pids PidsMetrics +} + +type controller struct { + name string + symlink bool +} + +type controllerHandler interface { + Create(*CgroupControl) (bool, error) + Apply(*CgroupControl, *spec.LinuxResources) error + Destroy(*CgroupControl) error + Stat(*CgroupControl, *Metrics) error +} + +const ( + cgroupRoot = "/sys/fs/cgroup" + // CPU is the cpu controller + CPU = "cpu" + // CPUAcct is the cpuacct controller + CPUAcct = "cpuacct" + // CPUset is the cpuset controller + CPUset = "cpuset" + // Memory is the memory controller + Memory = "memory" + // Pids is the pids controller + Pids = "pids" + // Blkio is the blkio controller + Blkio = "blkio" +) + +var handlers map[string]controllerHandler + +func init() { + handlers = make(map[string]controllerHandler) + handlers[CPU] = getCPUHandler() + handlers[CPUset] = getCpusetHandler() + handlers[Memory] = getMemoryHandler() + handlers[Pids] = getPidsHandler() + handlers[Blkio] = getBlkioHandler() +} + +// getAvailableControllers get the available controllers +func getAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]controller, error) { + if cgroup2 { + controllers := []controller{} + controllersFile := cgroupRoot + "/cgroup.controllers" + // rootless cgroupv2: check available controllers for current user, systemd or servicescope will inherit + if unshare.IsRootless() { + userSlice, err := getCgroupPathForCurrentProcess() + if err != nil { + return controllers, err + } + // userSlice already contains '/' so not adding here + basePath := cgroupRoot + userSlice + controllersFile = fmt.Sprintf("%s/cgroup.controllers", basePath) + } + controllersFileBytes, err := ioutil.ReadFile(controllersFile) + if err != nil { + return nil, errors.Wrapf(err, "failed while reading controllers for cgroup v2 from %q", controllersFile) + } + for _, controllerName := range strings.Fields(string(controllersFileBytes)) { + c := controller{ + name: controllerName, + symlink: false, + } + controllers = append(controllers, c) + } + return controllers, nil + } + + subsystems, _ := cgroupV1GetAllSubsystems() + controllers := []controller{} + // cgroupv1 and rootless: No subsystem is available: delegation is unsafe. + if unshare.IsRootless() { + return controllers, nil + } + + for _, name := range subsystems { + if _, found := exclude[name]; found { + continue + } + fileInfo, err := os.Stat(cgroupRoot + "/" + name) + if err != nil { + continue + } + c := controller{ + name: name, + symlink: !fileInfo.IsDir(), + } + controllers = append(controllers, c) + } + + return controllers, nil +} + +// GetAvailableControllers get string:bool map of all the available controllers +func GetAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]string, error) { + availableControllers, err := getAvailableControllers(exclude, cgroup2) + if err != nil { + return nil, err + } + controllerList := []string{} + for _, controller := range availableControllers { + controllerList = append(controllerList, controller.name) + } + + return controllerList, nil +} + +func cgroupV1GetAllSubsystems() ([]string, error) { + f, err := os.Open("/proc/cgroups") + if err != nil { + return nil, err + } + defer f.Close() + + subsystems := []string{} + + s := bufio.NewScanner(f) + for s.Scan() { + text := s.Text() + if text[0] != '#' { + parts := strings.Fields(text) + if len(parts) >= 4 && parts[3] != "0" { + subsystems = append(subsystems, parts[0]) + } + } + } + if err := s.Err(); err != nil { + return nil, err + } + return subsystems, nil +} + +func getCgroupPathForCurrentProcess() (string, error) { + path := fmt.Sprintf("/proc/%d/cgroup", os.Getpid()) + f, err := os.Open(path) + if err != nil { + return "", err + } + defer f.Close() + + cgroupPath := "" + s := bufio.NewScanner(f) + for s.Scan() { + text := s.Text() + procEntries := strings.SplitN(text, "::", 2) + // set process cgroupPath only if entry is valid + if len(procEntries) > 1 { + cgroupPath = procEntries[1] + } + } + if err := s.Err(); err != nil { + return cgroupPath, err + } + return cgroupPath, nil +} + +// getCgroupv1Path is a helper function to get the cgroup v1 path +func (c *CgroupControl) getCgroupv1Path(name string) string { + return filepath.Join(cgroupRoot, name, c.path) +} + +// createCgroupv2Path creates the cgroupv2 path and enables all the available controllers +func createCgroupv2Path(path string) (deferredError error) { + if !strings.HasPrefix(path, cgroupRoot+"/") { + return fmt.Errorf("invalid cgroup path %s", path) + } + content, err := ioutil.ReadFile(cgroupRoot + "/cgroup.controllers") + if err != nil { + return err + } + ctrs := bytes.Fields(content) + res := append([]byte("+"), bytes.Join(ctrs, []byte(" +"))...) + + current := "/sys/fs" + elements := strings.Split(path, "/") + for i, e := range elements[3:] { + current = filepath.Join(current, e) + if i > 0 { + if err := os.Mkdir(current, 0755); err != nil { + if !os.IsExist(err) { + return err + } + } else { + // If the directory was created, be sure it is not left around on errors. + defer func() { + if deferredError != nil { + os.Remove(current) + } + }() + } + } + // We enable the controllers for all the path components except the last one. It is not allowed to add + // PIDs if there are already enabled controllers. + if i < len(elements[3:])-1 { + if err := ioutil.WriteFile(filepath.Join(current, "cgroup.subtree_control"), res, 0755); err != nil { + return err + } + } + } + return nil +} + +// initialize initializes the specified hierarchy +func (c *CgroupControl) initialize() (err error) { + createdSoFar := map[string]controllerHandler{} + defer func() { + if err != nil { + for name, ctr := range createdSoFar { + if err := ctr.Destroy(c); err != nil { + logrus.Warningf("error cleaning up controller %s for %s", name, c.path) + } + } + } + }() + if c.cgroup2 { + if err := createCgroupv2Path(filepath.Join(cgroupRoot, c.path)); err != nil { + return errors.Wrapf(err, "error creating cgroup path %s", c.path) + } + } + for name, handler := range handlers { + created, err := handler.Create(c) + if err != nil { + return err + } + if created { + createdSoFar[name] = handler + } + } + + if !c.cgroup2 { + // We won't need to do this for cgroup v2 + for _, ctr := range c.additionalControllers { + if ctr.symlink { + continue + } + path := c.getCgroupv1Path(ctr.name) + if err := os.MkdirAll(path, 0755); err != nil { + return errors.Wrapf(err, "error creating cgroup path for %s", ctr.name) + } + } + } + + return nil +} + +func (c *CgroupControl) createCgroupDirectory(controller string) (bool, error) { + cPath := c.getCgroupv1Path(controller) + _, err := os.Stat(cPath) + if err == nil { + return false, nil + } + + if !os.IsNotExist(err) { + return false, err + } + + if err := os.MkdirAll(cPath, 0755); err != nil { + return false, errors.Wrapf(err, "error creating cgroup for %s", controller) + } + return true, nil +} + +func readFileAsUint64(path string) (uint64, error) { + data, err := ioutil.ReadFile(path) + if err != nil { + return 0, err + } + v := cleanString(string(data)) + if v == "max" { + return math.MaxUint64, nil + } + ret, err := strconv.ParseUint(v, 10, 64) + if err != nil { + return ret, errors.Wrapf(err, "parse %s from %s", v, path) + } + return ret, nil +} + +// New creates a new cgroup control +func New(path string, resources *spec.LinuxResources) (*CgroupControl, error) { + cgroup2, err := IsCgroup2UnifiedMode() + if err != nil { + return nil, err + } + control := &CgroupControl{ + cgroup2: cgroup2, + path: path, + } + + if !cgroup2 { + controllers, err := getAvailableControllers(handlers, false) + if err != nil { + return nil, err + } + control.additionalControllers = controllers + } + + if err := control.initialize(); err != nil { + return nil, err + } + + return control, nil +} + +// NewSystemd creates a new cgroup control +func NewSystemd(path string) (*CgroupControl, error) { + cgroup2, err := IsCgroup2UnifiedMode() + if err != nil { + return nil, err + } + control := &CgroupControl{ + cgroup2: cgroup2, + path: path, + systemd: true, + } + return control, nil +} + +// Load loads an existing cgroup control +func Load(path string) (*CgroupControl, error) { + cgroup2, err := IsCgroup2UnifiedMode() + if err != nil { + return nil, err + } + control := &CgroupControl{ + cgroup2: cgroup2, + path: path, + systemd: false, + } + if !cgroup2 { + controllers, err := getAvailableControllers(handlers, false) + if err != nil { + return nil, err + } + control.additionalControllers = controllers + } + if !cgroup2 { + oneExists := false + // check that the cgroup exists at least under one controller + for name := range handlers { + p := control.getCgroupv1Path(name) + if _, err := os.Stat(p); err == nil { + oneExists = true + break + } + } + + // if there is no controller at all, raise an error + if !oneExists { + if unshare.IsRootless() { + return nil, ErrCgroupV1Rootless + } + // compatible with the error code + // used by containerd/cgroups + return nil, ErrCgroupDeleted + } + } + return control, nil +} + +// CreateSystemdUnit creates the systemd cgroup +func (c *CgroupControl) CreateSystemdUnit(path string) error { + if !c.systemd { + return fmt.Errorf("the cgroup controller is not using systemd") + } + + conn, err := systemdDbus.NewWithContext(context.TODO()) + if err != nil { + return err + } + defer conn.Close() + + return systemdCreate(path, conn) +} + +// GetUserConnection returns an user connection to D-BUS +func GetUserConnection(uid int) (*systemdDbus.Conn, error) { + return systemdDbus.NewConnection(func() (*dbus.Conn, error) { + return dbusAuthConnection(uid, dbus.SessionBusPrivate) + }) +} + +// CreateSystemdUserUnit creates the systemd cgroup for the specified user +func (c *CgroupControl) CreateSystemdUserUnit(path string, uid int) error { + if !c.systemd { + return fmt.Errorf("the cgroup controller is not using systemd") + } + + conn, err := GetUserConnection(uid) + if err != nil { + return err + } + defer conn.Close() + + return systemdCreate(path, conn) +} + +func dbusAuthConnection(uid int, createBus func(opts ...dbus.ConnOption) (*dbus.Conn, error)) (*dbus.Conn, error) { + conn, err := createBus() + if err != nil { + return nil, err + } + + methods := []dbus.Auth{dbus.AuthExternal(strconv.Itoa(uid))} + + err = conn.Auth(methods) + if err != nil { + conn.Close() + return nil, err + } + if err := conn.Hello(); err != nil { + return nil, err + } + + return conn, nil +} + +// Delete cleans a cgroup +func (c *CgroupControl) Delete() error { + return c.DeleteByPath(c.path) +} + +// rmDirRecursively delete recursively a cgroup directory. +// It differs from os.RemoveAll as it doesn't attempt to unlink files. +// On cgroupfs we are allowed only to rmdir empty directories. +func rmDirRecursively(path string) error { + if err := os.Remove(path); err == nil || os.IsNotExist(err) { + return nil + } + entries, err := ioutil.ReadDir(path) + if err != nil { + return err + } + for _, i := range entries { + if i.IsDir() { + if err := rmDirRecursively(filepath.Join(path, i.Name())); err != nil { + return err + } + } + } + if err := os.Remove(path); err != nil { + if !os.IsNotExist(err) { + return errors.Wrapf(err, "remove %s", path) + } + } + return nil +} + +// DeleteByPathConn deletes the specified cgroup path using the specified +// dbus connection if needed. +func (c *CgroupControl) DeleteByPathConn(path string, conn *systemdDbus.Conn) error { + if c.systemd { + return systemdDestroyConn(path, conn) + } + if c.cgroup2 { + return rmDirRecursively(filepath.Join(cgroupRoot, c.path)) + } + var lastError error + for _, h := range handlers { + if err := h.Destroy(c); err != nil { + lastError = err + } + } + + for _, ctr := range c.additionalControllers { + if ctr.symlink { + continue + } + p := c.getCgroupv1Path(ctr.name) + if err := rmDirRecursively(p); err != nil { + lastError = errors.Wrapf(err, "remove %s", p) + } + } + return lastError +} + +// DeleteByPath deletes the specified cgroup path +func (c *CgroupControl) DeleteByPath(path string) error { + if c.systemd { + conn, err := systemdDbus.NewWithContext(context.TODO()) + if err != nil { + return err + } + defer conn.Close() + return c.DeleteByPathConn(path, conn) + } + return c.DeleteByPathConn(path, nil) +} + +// Update updates the cgroups +func (c *CgroupControl) Update(resources *spec.LinuxResources) error { + for _, h := range handlers { + if err := h.Apply(c, resources); err != nil { + return err + } + } + return nil +} + +// AddPid moves the specified pid to the cgroup +func (c *CgroupControl) AddPid(pid int) error { + pidString := []byte(fmt.Sprintf("%d\n", pid)) + + if c.cgroup2 { + p := filepath.Join(cgroupRoot, c.path, "cgroup.procs") + if err := ioutil.WriteFile(p, pidString, 0644); err != nil { + return errors.Wrapf(err, "write %s", p) + } + return nil + } + + names := make([]string, 0, len(handlers)) + for n := range handlers { + names = append(names, n) + } + + for _, c := range c.additionalControllers { + if !c.symlink { + names = append(names, c.name) + } + } + + for _, n := range names { + // If we aren't using cgroup2, we won't write correctly to unified hierarchy + if !c.cgroup2 && n == "unified" { + continue + } + p := filepath.Join(c.getCgroupv1Path(n), "tasks") + if err := ioutil.WriteFile(p, pidString, 0644); err != nil { + return errors.Wrapf(err, "write %s", p) + } + } + return nil +} + +// Stat returns usage statistics for the cgroup +func (c *CgroupControl) Stat() (*Metrics, error) { + m := Metrics{} + found := false + for _, h := range handlers { + if err := h.Stat(c, &m); err != nil { + if !os.IsNotExist(errors.Cause(err)) { + return nil, err + } + logrus.Warningf("Failed to retrieve cgroup stats: %v", err) + continue + } + found = true + } + if !found { + return nil, ErrStatCgroup + } + return &m, nil +} + +func readCgroup2MapPath(path string) (map[string][]string, error) { + ret := map[string][]string{} + f, err := os.Open(path) + if err != nil { + if os.IsNotExist(err) { + return ret, nil + } + return nil, errors.Wrapf(err, "open file %s", path) + } + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(line) + if len(parts) < 2 { + continue + } + ret[parts[0]] = parts[1:] + } + if err := scanner.Err(); err != nil { + return nil, errors.Wrapf(err, "parsing file %s", path) + } + return ret, nil +} + +func readCgroup2MapFile(ctr *CgroupControl, name string) (map[string][]string, error) { + p := filepath.Join(cgroupRoot, ctr.path, name) + + return readCgroup2MapPath(p) +} diff --git a/vendor/github.com/containers/common/pkg/cgroups/cgroups_supported.go b/vendor/github.com/containers/common/pkg/cgroups/cgroups_supported.go new file mode 100644 index 000000000..fe17db7f7 --- /dev/null +++ b/vendor/github.com/containers/common/pkg/cgroups/cgroups_supported.go @@ -0,0 +1,90 @@ +// +build linux + +package cgroups + +import ( + "bufio" + "fmt" + "os" + "path/filepath" + "strings" + "sync" + "syscall" + + "github.com/pkg/errors" + "golang.org/x/sys/unix" +) + +var ( + isUnifiedOnce sync.Once + isUnified bool + isUnifiedErr error +) + +// IsCgroup2UnifiedMode returns whether we are running in cgroup 2 cgroup2 mode. +func IsCgroup2UnifiedMode() (bool, error) { + isUnifiedOnce.Do(func() { + var st syscall.Statfs_t + if err := syscall.Statfs("/sys/fs/cgroup", &st); err != nil { + isUnified, isUnifiedErr = false, err + } else { + isUnified, isUnifiedErr = st.Type == unix.CGROUP2_SUPER_MAGIC, nil + } + }) + return isUnified, isUnifiedErr +} + +// UserOwnsCurrentSystemdCgroup checks whether the current EUID owns the +// current cgroup. +func UserOwnsCurrentSystemdCgroup() (bool, error) { + uid := os.Geteuid() + + cgroup2, err := IsCgroup2UnifiedMode() + if err != nil { + return false, err + } + + f, err := os.Open("/proc/self/cgroup") + if err != nil { + return false, err + } + defer f.Close() + + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + parts := strings.SplitN(line, ":", 3) + + if len(parts) < 3 { + continue + } + + var cgroupPath string + + if cgroup2 { + cgroupPath = filepath.Join(cgroupRoot, parts[2]) + } else { + if parts[1] != "name=systemd" { + continue + } + cgroupPath = filepath.Join(cgroupRoot, "systemd", parts[2]) + } + + st, err := os.Stat(cgroupPath) + if err != nil { + return false, err + } + s := st.Sys() + if s == nil { + return false, fmt.Errorf("error stat cgroup path %s", cgroupPath) + } + + if int(s.(*syscall.Stat_t).Uid) != uid { + return false, nil + } + } + if err := scanner.Err(); err != nil { + return false, errors.Wrapf(err, "parsing file /proc/self/cgroup") + } + return true, nil +} diff --git a/vendor/github.com/containers/common/pkg/cgroups/cgroups_unsupported.go b/vendor/github.com/containers/common/pkg/cgroups/cgroups_unsupported.go new file mode 100644 index 000000000..cd140fbf3 --- /dev/null +++ b/vendor/github.com/containers/common/pkg/cgroups/cgroups_unsupported.go @@ -0,0 +1,14 @@ +// +build !linux + +package cgroups + +// IsCgroup2UnifiedMode returns whether we are running in cgroup 2 cgroup2 mode. +func IsCgroup2UnifiedMode() (bool, error) { + return false, nil +} + +// UserOwnsCurrentSystemdCgroup checks whether the current EUID owns the +// current cgroup. +func UserOwnsCurrentSystemdCgroup() (bool, error) { + return false, nil +} diff --git a/vendor/github.com/containers/common/pkg/cgroups/cpu.go b/vendor/github.com/containers/common/pkg/cgroups/cpu.go new file mode 100644 index 000000000..23539757d --- /dev/null +++ b/vendor/github.com/containers/common/pkg/cgroups/cpu.go @@ -0,0 +1,160 @@ +package cgroups + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" +) + +type cpuHandler struct { +} + +func getCPUHandler() *cpuHandler { + return &cpuHandler{} +} + +func cleanString(s string) string { + return strings.Trim(s, "\n") +} + +func readAcct(ctr *CgroupControl, name string) (uint64, error) { + p := filepath.Join(ctr.getCgroupv1Path(CPUAcct), name) + return readFileAsUint64(p) +} + +func readAcctList(ctr *CgroupControl, name string) ([]uint64, error) { + p := filepath.Join(ctr.getCgroupv1Path(CPUAcct), name) + data, err := ioutil.ReadFile(p) + if err != nil { + return nil, errors.Wrapf(err, "reading %s", p) + } + r := []uint64{} + for _, s := range strings.Split(string(data), " ") { + s = cleanString(s) + if s == "" { + break + } + v, err := strconv.ParseUint(s, 10, 64) + if err != nil { + return nil, errors.Wrapf(err, "parsing %s", s) + } + r = append(r, v) + } + return r, nil +} + +// Apply set the specified constraints +func (c *cpuHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { + if res.CPU == nil { + return nil + } + return fmt.Errorf("cpu apply not implemented yet") +} + +// Create the cgroup +func (c *cpuHandler) Create(ctr *CgroupControl) (bool, error) { + if ctr.cgroup2 { + return false, nil + } + return ctr.createCgroupDirectory(CPU) +} + +// Destroy the cgroup +func (c *cpuHandler) Destroy(ctr *CgroupControl) error { + return rmDirRecursively(ctr.getCgroupv1Path(CPU)) +} + +// Stat fills a metrics structure with usage stats for the controller +func (c *cpuHandler) Stat(ctr *CgroupControl, m *Metrics) error { + var err error + usage := CPUUsage{} + if ctr.cgroup2 { + values, err := readCgroup2MapFile(ctr, "cpu.stat") + if err != nil { + return err + } + if val, found := values["usage_usec"]; found { + usage.Total, err = strconv.ParseUint(cleanString(val[0]), 10, 64) + if err != nil { + return err + } + usage.Kernel *= 1000 + } + if val, found := values["system_usec"]; found { + usage.Kernel, err = strconv.ParseUint(cleanString(val[0]), 10, 64) + if err != nil { + return err + } + usage.Total *= 1000 + } + // FIXME: How to read usage.PerCPU? + } else { + usage.Total, err = readAcct(ctr, "cpuacct.usage") + if err != nil { + if !os.IsNotExist(errors.Cause(err)) { + return err + } + usage.Total = 0 + } + usage.Kernel, err = readAcct(ctr, "cpuacct.usage_sys") + if err != nil { + if !os.IsNotExist(errors.Cause(err)) { + return err + } + usage.Kernel = 0 + } + usage.PerCPU, err = readAcctList(ctr, "cpuacct.usage_percpu") + if err != nil { + if !os.IsNotExist(errors.Cause(err)) { + return err + } + usage.PerCPU = nil + } + } + m.CPU = CPUMetrics{Usage: usage} + return nil +} + +// GetSystemCPUUsage returns the system usage for all the cgroups +func GetSystemCPUUsage() (uint64, error) { + cgroupv2, err := IsCgroup2UnifiedMode() + if err != nil { + return 0, err + } + if !cgroupv2 { + p := filepath.Join(cgroupRoot, CPUAcct, "cpuacct.usage") + return readFileAsUint64(p) + } + + files, err := ioutil.ReadDir(cgroupRoot) + if err != nil { + return 0, err + } + var total uint64 + for _, file := range files { + if !file.IsDir() { + continue + } + p := filepath.Join(cgroupRoot, file.Name(), "cpu.stat") + + values, err := readCgroup2MapPath(p) + if err != nil { + return 0, err + } + + if val, found := values["usage_usec"]; found { + v, err := strconv.ParseUint(cleanString(val[0]), 10, 64) + if err != nil { + return 0, err + } + total += v * 1000 + } + } + return total, nil +} diff --git a/vendor/github.com/containers/common/pkg/cgroups/cpuset.go b/vendor/github.com/containers/common/pkg/cgroups/cpuset.go new file mode 100644 index 000000000..22ac0a079 --- /dev/null +++ b/vendor/github.com/containers/common/pkg/cgroups/cpuset.go @@ -0,0 +1,85 @@ +package cgroups + +import ( + "fmt" + "io/ioutil" + "path/filepath" + "strings" + + spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" +) + +type cpusetHandler struct { +} + +func cpusetCopyFileFromParent(dir, file string, cgroupv2 bool) ([]byte, error) { + if dir == cgroupRoot { + return nil, fmt.Errorf("could not find parent to initialize cpuset %s", file) + } + path := filepath.Join(dir, file) + parentPath := path + if cgroupv2 { + parentPath = fmt.Sprintf("%s.effective", parentPath) + } + data, err := ioutil.ReadFile(parentPath) + if err != nil { + return nil, errors.Wrapf(err, "open %s", path) + } + if strings.Trim(string(data), "\n") != "" { + return data, nil + } + data, err = cpusetCopyFileFromParent(filepath.Dir(dir), file, cgroupv2) + if err != nil { + return nil, err + } + if err := ioutil.WriteFile(path, data, 0644); err != nil { + return nil, errors.Wrapf(err, "write %s", path) + } + return data, nil +} + +func cpusetCopyFromParent(path string, cgroupv2 bool) error { + for _, file := range []string{"cpuset.cpus", "cpuset.mems"} { + if _, err := cpusetCopyFileFromParent(path, file, cgroupv2); err != nil { + return err + } + } + return nil +} + +func getCpusetHandler() *cpusetHandler { + return &cpusetHandler{} +} + +// Apply set the specified constraints +func (c *cpusetHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { + if res.CPU == nil { + return nil + } + return fmt.Errorf("cpuset apply not implemented yet") +} + +// Create the cgroup +func (c *cpusetHandler) Create(ctr *CgroupControl) (bool, error) { + if ctr.cgroup2 { + path := filepath.Join(cgroupRoot, ctr.path) + return true, cpusetCopyFromParent(path, true) + } + + created, err := ctr.createCgroupDirectory(CPUset) + if !created || err != nil { + return created, err + } + return true, cpusetCopyFromParent(ctr.getCgroupv1Path(CPUset), false) +} + +// Destroy the cgroup +func (c *cpusetHandler) Destroy(ctr *CgroupControl) error { + return rmDirRecursively(ctr.getCgroupv1Path(CPUset)) +} + +// Stat fills a metrics structure with usage stats for the controller +func (c *cpusetHandler) Stat(ctr *CgroupControl, m *Metrics) error { + return nil +} diff --git a/vendor/github.com/containers/common/pkg/cgroups/memory.go b/vendor/github.com/containers/common/pkg/cgroups/memory.go new file mode 100644 index 000000000..b3991f7e3 --- /dev/null +++ b/vendor/github.com/containers/common/pkg/cgroups/memory.go @@ -0,0 +1,66 @@ +package cgroups + +import ( + "fmt" + "path/filepath" + + spec "github.com/opencontainers/runtime-spec/specs-go" +) + +type memHandler struct { +} + +func getMemoryHandler() *memHandler { + return &memHandler{} +} + +// Apply set the specified constraints +func (c *memHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { + if res.Memory == nil { + return nil + } + return fmt.Errorf("memory apply not implemented yet") +} + +// Create the cgroup +func (c *memHandler) Create(ctr *CgroupControl) (bool, error) { + if ctr.cgroup2 { + return false, nil + } + return ctr.createCgroupDirectory(Memory) +} + +// Destroy the cgroup +func (c *memHandler) Destroy(ctr *CgroupControl) error { + return rmDirRecursively(ctr.getCgroupv1Path(Memory)) +} + +// Stat fills a metrics structure with usage stats for the controller +func (c *memHandler) Stat(ctr *CgroupControl, m *Metrics) error { + var err error + usage := MemoryUsage{} + + var memoryRoot string + filenames := map[string]string{} + + if ctr.cgroup2 { + memoryRoot = filepath.Join(cgroupRoot, ctr.path) + filenames["usage"] = "memory.current" + filenames["limit"] = "memory.max" + } else { + memoryRoot = ctr.getCgroupv1Path(Memory) + filenames["usage"] = "memory.usage_in_bytes" + filenames["limit"] = "memory.limit_in_bytes" + } + usage.Usage, err = readFileAsUint64(filepath.Join(memoryRoot, filenames["usage"])) + if err != nil { + return err + } + usage.Limit, err = readFileAsUint64(filepath.Join(memoryRoot, filenames["limit"])) + if err != nil { + return err + } + + m.Memory = MemoryMetrics{Usage: usage} + return nil +} diff --git a/vendor/github.com/containers/common/pkg/cgroups/pids.go b/vendor/github.com/containers/common/pkg/cgroups/pids.go new file mode 100644 index 000000000..b2bfebe4d --- /dev/null +++ b/vendor/github.com/containers/common/pkg/cgroups/pids.go @@ -0,0 +1,66 @@ +package cgroups + +import ( + "fmt" + "io/ioutil" + "path/filepath" + + spec "github.com/opencontainers/runtime-spec/specs-go" +) + +type pidHandler struct { +} + +func getPidsHandler() *pidHandler { + return &pidHandler{} +} + +// Apply set the specified constraints +func (c *pidHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { + if res.Pids == nil { + return nil + } + var PIDRoot string + + if ctr.cgroup2 { + PIDRoot = filepath.Join(cgroupRoot, ctr.path) + } else { + PIDRoot = ctr.getCgroupv1Path(Pids) + } + + p := filepath.Join(PIDRoot, "pids.max") + return ioutil.WriteFile(p, []byte(fmt.Sprintf("%d\n", res.Pids.Limit)), 0644) +} + +// Create the cgroup +func (c *pidHandler) Create(ctr *CgroupControl) (bool, error) { + return ctr.createCgroupDirectory(Pids) +} + +// Destroy the cgroup +func (c *pidHandler) Destroy(ctr *CgroupControl) error { + return rmDirRecursively(ctr.getCgroupv1Path(Pids)) +} + +// Stat fills a metrics structure with usage stats for the controller +func (c *pidHandler) Stat(ctr *CgroupControl, m *Metrics) error { + if ctr.path == "" { + // nothing we can do to retrieve the pids.current path + return nil + } + + var PIDRoot string + if ctr.cgroup2 { + PIDRoot = filepath.Join(cgroupRoot, ctr.path) + } else { + PIDRoot = ctr.getCgroupv1Path(Pids) + } + + current, err := readFileAsUint64(filepath.Join(PIDRoot, "pids.current")) + if err != nil { + return err + } + + m.Pids = PidsMetrics{Current: current} + return nil +} diff --git a/vendor/github.com/containers/common/pkg/cgroups/systemd.go b/vendor/github.com/containers/common/pkg/cgroups/systemd.go new file mode 100644 index 000000000..92065a2d7 --- /dev/null +++ b/vendor/github.com/containers/common/pkg/cgroups/systemd.go @@ -0,0 +1,80 @@ +package cgroups + +import ( + "context" + "fmt" + "path/filepath" + "strings" + + systemdDbus "github.com/coreos/go-systemd/v22/dbus" + "github.com/godbus/dbus/v5" +) + +func systemdCreate(path string, c *systemdDbus.Conn) error { + slice, name := filepath.Split(path) + slice = strings.TrimSuffix(slice, "/") + + var lastError error + for i := 0; i < 2; i++ { + properties := []systemdDbus.Property{ + systemdDbus.PropDescription(fmt.Sprintf("cgroup %s", name)), + systemdDbus.PropWants(slice), + } + pMap := map[string]bool{ + "DefaultDependencies": false, + "MemoryAccounting": true, + "CPUAccounting": true, + "BlockIOAccounting": true, + } + if i == 0 { + pMap["Delegate"] = true + } + for k, v := range pMap { + p := systemdDbus.Property{ + Name: k, + Value: dbus.MakeVariant(v), + } + properties = append(properties, p) + } + + ch := make(chan string) + _, err := c.StartTransientUnitContext(context.TODO(), name, "replace", properties, ch) + if err != nil { + lastError = err + continue + } + <-ch + return nil + } + return lastError +} + +/* + systemdDestroyConn is copied from containerd/cgroups/systemd.go file, that + has the following license: + + Copyright The containerd Authors. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +func systemdDestroyConn(path string, c *systemdDbus.Conn) error { + name := filepath.Base(path) + + ch := make(chan string) + _, err := c.StopUnitContext(context.TODO(), name, "replace", ch) + if err != nil { + return err + } + <-ch + return nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 076f93039..b5cc6083c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -114,6 +114,7 @@ github.com/containers/common/pkg/apparmor github.com/containers/common/pkg/apparmor/internal/supported github.com/containers/common/pkg/auth github.com/containers/common/pkg/capabilities +github.com/containers/common/pkg/cgroups github.com/containers/common/pkg/cgroupv2 github.com/containers/common/pkg/chown github.com/containers/common/pkg/completion -- cgit v1.2.3-54-g00ecf