From fa1869381301797295dece0aec12435fc902295b Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Wed, 19 Jun 2019 11:09:54 +0200 Subject: pkg: new package cgroups provide a package for managing cgroups. This is not supposed to be a complete implementation with all the features supported by cgroups, but it is a minimal implementation designed around what libpod needs and it is currently using. For example, it is currently possible to Apply only the pids limit, as it is used by libpod for stopping containers, any other Apply will just fail. The main goal here is to have a minimal library where we have full control, so we can start playing with cgroup v2. When the need arises, we can add more features. Signed-off-by: Giuseppe Scrivano --- pkg/cgroups/cpu.go | 97 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 97 insertions(+) create mode 100644 pkg/cgroups/cpu.go (limited to 'pkg/cgroups/cpu.go') diff --git a/pkg/cgroups/cpu.go b/pkg/cgroups/cpu.go new file mode 100644 index 000000000..d27f1257f --- /dev/null +++ b/pkg/cgroups/cpu.go @@ -0,0 +1,97 @@ +package cgroups + +import ( + "fmt" + "io/ioutil" + "os" + "path/filepath" + "strconv" + "strings" + + spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" +) + +type cpuHandler struct { +} + +func getCPUHandler() *cpuHandler { + return &cpuHandler{} +} + +func cleanString(s string) string { + return strings.Trim(s, "\n") +} + +func readAcct(ctr *CgroupControl, name string) (uint64, error) { + p := filepath.Join(ctr.getCgroupv1Path(CPUAcct), name) + return readFileAsUint64(p) +} + +func readAcctList(ctr *CgroupControl, name string) ([]uint64, error) { + var r []uint64 + + p := filepath.Join(ctr.getCgroupv1Path(CPUAcct), name) + data, err := ioutil.ReadFile(p) + if err != nil { + return nil, errors.Wrapf(err, "reading %s", p) + } + for _, s := range strings.Split(string(data), " ") { + s = cleanString(s) + if s == "" { + break + } + v, err := strconv.ParseUint(s, 10, 0) + if err != nil { + return nil, errors.Wrapf(err, "parsing %s", s) + } + r = append(r, v) + } + return r, nil +} + +// Apply set the specified constraints +func (c *cpuHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { + if res.CPU == nil { + return nil + } + return fmt.Errorf("function not implemented yet") +} + +// Create the cgroup +func (c *cpuHandler) Create(ctr *CgroupControl) (bool, error) { + if ctr.cgroup2 { + return false, fmt.Errorf("function not implemented yet") + } + return ctr.createCgroupDirectory(CPU) +} + +// Destroy the cgroup +func (c *cpuHandler) Destroy(ctr *CgroupControl) error { + return os.Remove(ctr.getCgroupv1Path(CPU)) +} + +// Stat fills a metrics structure with usage stats for the controller +func (c *cpuHandler) Stat(ctr *CgroupControl, m *Metrics) error { + if ctr.cgroup2 { + return fmt.Errorf("function not implemented yet") + } + + var err error + usage := CPUUsage{} + + usage.Total, err = readAcct(ctr, "cpuacct.usage") + if err != nil { + return err + } + usage.Kernel, err = readAcct(ctr, "cpuacct.usage_sys") + if err != nil { + return err + } + usage.PerCPU, err = readAcctList(ctr, "cpuacct.usage_percpu") + if err != nil { + return err + } + m.CPU = CPUMetrics{Usage: usage} + return nil +} -- cgit v1.2.3-54-g00ecf From 1778bfa5fef337158537a77344300e1a755a7ffe Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Wed, 19 Jun 2019 13:07:23 +0200 Subject: pkg, cgroups: add initial support for cgroup v2 This is an initial implementation of cgroup v2 support for pkg/cgroups. It currently works with crun, with this patch: https://github.com/giuseppe/crun/pull/49). It adds the pieces for: - set PID limit to 1 - retrieve stats so that "podman stats" work. the only missing part is the support for reading per CPU stats (that is cpuacct.usage_percpu on cgroup v1), so for now it always returns an empty result. Signed-off-by: Giuseppe Scrivano --- pkg/cgroups/blkio.go | 134 +++++++++++++++++++++++++++++++++---------------- pkg/cgroups/cgroups.go | 64 ++++++++++++++++++----- pkg/cgroups/cpu.go | 51 ++++++++++++------- pkg/cgroups/cpuset.go | 9 ++-- pkg/cgroups/memory.go | 25 +++++---- pkg/cgroups/pids.go | 16 +++--- 6 files changed, 207 insertions(+), 92 deletions(-) (limited to 'pkg/cgroups/cpu.go') diff --git a/pkg/cgroups/blkio.go b/pkg/cgroups/blkio.go index 8eb54abec..ca9107d97 100644 --- a/pkg/cgroups/blkio.go +++ b/pkg/cgroups/blkio.go @@ -9,6 +9,7 @@ import ( "strings" spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" ) type blkioHandler struct { @@ -29,7 +30,7 @@ func (c *blkioHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error // Create the cgroup func (c *blkioHandler) Create(ctr *CgroupControl) (bool, error) { if ctr.cgroup2 { - return false, fmt.Errorf("function not implemented yet") + return false, fmt.Errorf("io create not implemented for cgroup v2") } return ctr.createCgroupDirectory(Blkio) } @@ -44,57 +45,104 @@ func (c *blkioHandler) Stat(ctr *CgroupControl, m *Metrics) error { var ioServiceBytesRecursive []BlkIOEntry if ctr.cgroup2 { - return fmt.Errorf("function not implemented yet") - } - - BlkioRoot := ctr.getCgroupv1Path(Blkio) - - p := filepath.Join(BlkioRoot, "blkio.throttle.io_service_bytes_recursive") - f, err := os.Open(p) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - defer f.Close() - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - parts := strings.Fields(line) - if len(parts) < 3 { - continue - } - d := strings.Split(parts[0], ":") - if len(d) != 2 { - continue - } - minor, err := strconv.ParseUint(d[0], 10, 0) + // more details on the io.stat file format:X https://facebookmicrosites.github.io/cgroup2/docs/io-controller.html + values, err := readCgroup2MapFile(ctr, "io.stat") if err != nil { return err } - major, err := strconv.ParseUint(d[1], 10, 0) + for k, v := range values { + d := strings.Split(k, ":") + if len(d) != 2 { + continue + } + minor, err := strconv.ParseUint(d[0], 10, 0) + if err != nil { + return err + } + major, err := strconv.ParseUint(d[1], 10, 0) + if err != nil { + return err + } + + for _, item := range v { + d := strings.Split(item, "=") + if len(d) != 2 { + continue + } + op := d[0] + + // Accommodate the cgroup v1 naming + switch op { + case "rbytes": + op = "read" + case "wbytes": + op = "write" + } + + value, err := strconv.ParseUint(d[1], 10, 0) + if err != nil { + return err + } + + entry := BlkIOEntry{ + Op: op, + Major: major, + Minor: minor, + Value: value, + } + ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry) + } + } + } else { + BlkioRoot := ctr.getCgroupv1Path(Blkio) + + p := filepath.Join(BlkioRoot, "blkio.throttle.io_service_bytes_recursive") + f, err := os.Open(p) if err != nil { - return err + if os.IsNotExist(err) { + return nil + } + return errors.Wrapf(err, "open %s", p) } + defer f.Close() - op := parts[1] + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(line) + if len(parts) < 3 { + continue + } + d := strings.Split(parts[0], ":") + if len(d) != 2 { + continue + } + minor, err := strconv.ParseUint(d[0], 10, 0) + if err != nil { + return err + } + major, err := strconv.ParseUint(d[1], 10, 0) + if err != nil { + return err + } - value, err := strconv.ParseUint(parts[2], 10, 0) - if err != nil { - return err + op := parts[1] + + value, err := strconv.ParseUint(parts[2], 10, 0) + if err != nil { + return err + } + entry := BlkIOEntry{ + Op: op, + Major: major, + Minor: minor, + Value: value, + } + ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry) } - entry := BlkIOEntry{ - Op: op, - Major: major, - Minor: minor, - Value: value, + if err := scanner.Err(); err != nil { + return errors.Wrapf(err, "parse %s", p) } - ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry) - } - if err := scanner.Err(); err != nil { - return err } m.Blkio = BlkioMetrics{IoServiceBytesRecursive: ioServiceBytesRecursive} return nil diff --git a/pkg/cgroups/cgroups.go b/pkg/cgroups/cgroups.go index 24dce71bd..426bda559 100644 --- a/pkg/cgroups/cgroups.go +++ b/pkg/cgroups/cgroups.go @@ -1,11 +1,14 @@ package cgroups import ( + "bufio" "fmt" "io/ioutil" + "math" "os" "path/filepath" "strconv" + "strings" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -119,12 +122,12 @@ func init() { // getAvailableControllers get the available controllers func getAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]controller, error) { if cgroup2 { - return nil, fmt.Errorf("function not implemented yet") + return nil, fmt.Errorf("getAvailableControllers not implemented yet for cgroup v2") } infos, err := ioutil.ReadDir(cgroupRoot) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "read directory %s", cgroupRoot) } var controllers []controller for _, i := range infos { @@ -204,9 +207,17 @@ func (c *CgroupControl) createCgroupDirectory(controller string) (bool, error) { func readFileAsUint64(path string) (uint64, error) { data, err := ioutil.ReadFile(path) if err != nil { - return 0, err + return 0, errors.Wrapf(err, "open %s", path) + } + v := cleanString(string(data)) + if v == "max" { + return math.MaxUint64, nil + } + ret, err := strconv.ParseUint(v, 10, 0) + if err != nil { + return ret, errors.Wrapf(err, "parse %s from %s", v, path) } - return strconv.ParseUint(cleanString(string(data)), 10, 0) + return ret, nil } func (c *CgroupControl) writePidToTasks(pid int, name string) error { @@ -307,8 +318,9 @@ func (c *CgroupControl) DeleteByPath(path string) error { } for _, ctr := range c.additionalControllers { - if err := os.Remove(c.getCgroupv1Path(ctr.name)); err != nil { - lastError = err + p := c.getCgroupv1Path(ctr.name) + if err := os.Remove(p); err != nil { + lastError = errors.Wrapf(err, "remove %s", p) } } return lastError @@ -326,10 +338,15 @@ func (c *CgroupControl) Update(resources *spec.LinuxResources) error { // AddPid moves the specified pid to the cgroup func (c *CgroupControl) AddPid(pid int) error { + pidString := []byte(fmt.Sprintf("%d\n", pid)) + if c.cgroup2 { - return fmt.Errorf("function not implemented yet") + p := filepath.Join(cgroupRoot, c.path, "tasks") + if err := ioutil.WriteFile(p, pidString, 0644); err != nil { + return errors.Wrapf(err, "write %s", p) + } + return nil } - pidString := []byte(fmt.Sprintf("%d\n", pid)) var names []string for n := range handlers { @@ -345,7 +362,7 @@ func (c *CgroupControl) AddPid(pid int) error { for _, n := range names { p := filepath.Join(c.getCgroupv1Path(n), "tasks") if err := ioutil.WriteFile(p, pidString, 0644); err != nil { - return err + return errors.Wrapf(err, "write %s", p) } } return nil @@ -353,9 +370,6 @@ func (c *CgroupControl) AddPid(pid int) error { // Stat returns usage statistics for the cgroup func (c *CgroupControl) Stat() (*Metrics, error) { - if c.cgroup2 { - return nil, fmt.Errorf("function not implemented yet") - } m := Metrics{} for _, h := range handlers { if err := h.Stat(c, &m); err != nil { @@ -364,3 +378,29 @@ func (c *CgroupControl) Stat() (*Metrics, error) { } return &m, nil } + +func readCgroup2MapFile(ctr *CgroupControl, name string) (map[string][]string, error) { + ret := map[string][]string{} + p := filepath.Join(cgroupRoot, ctr.path, name) + f, err := os.Open(p) + if err != nil { + if os.IsNotExist(err) { + return ret, nil + } + return nil, errors.Wrapf(err, "open file %s", p) + } + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(line) + if len(parts) < 2 { + continue + } + ret[parts[0]] = parts[1:] + } + if err := scanner.Err(); err != nil { + return nil, errors.Wrapf(err, "parsing file %s", p) + } + return ret, nil +} diff --git a/pkg/cgroups/cpu.go b/pkg/cgroups/cpu.go index d27f1257f..3f969fd3c 100644 --- a/pkg/cgroups/cpu.go +++ b/pkg/cgroups/cpu.go @@ -55,13 +55,13 @@ func (c *cpuHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { if res.CPU == nil { return nil } - return fmt.Errorf("function not implemented yet") + return fmt.Errorf("cpu apply not implemented yet") } // Create the cgroup func (c *cpuHandler) Create(ctr *CgroupControl) (bool, error) { if ctr.cgroup2 { - return false, fmt.Errorf("function not implemented yet") + return false, fmt.Errorf("cpu create not implemented for cgroup v2") } return ctr.createCgroupDirectory(CPU) } @@ -73,24 +73,39 @@ func (c *cpuHandler) Destroy(ctr *CgroupControl) error { // Stat fills a metrics structure with usage stats for the controller func (c *cpuHandler) Stat(ctr *CgroupControl, m *Metrics) error { - if ctr.cgroup2 { - return fmt.Errorf("function not implemented yet") - } - var err error usage := CPUUsage{} - - usage.Total, err = readAcct(ctr, "cpuacct.usage") - if err != nil { - return err - } - usage.Kernel, err = readAcct(ctr, "cpuacct.usage_sys") - if err != nil { - return err - } - usage.PerCPU, err = readAcctList(ctr, "cpuacct.usage_percpu") - if err != nil { - return err + if ctr.cgroup2 { + values, err := readCgroup2MapFile(ctr, "cpu.stat") + if err != nil { + return err + } + if val, found := values["usage_usec"]; found { + usage.Kernel, err = strconv.ParseUint(cleanString(val[0]), 10, 0) + if err != nil { + return err + } + } + if val, found := values["system_usec"]; found { + usage.Total, err = strconv.ParseUint(cleanString(val[0]), 10, 0) + if err != nil { + return err + } + } + // FIXME: How to read usage.PerCPU? + } else { + usage.Total, err = readAcct(ctr, "cpuacct.usage") + if err != nil { + return err + } + usage.Kernel, err = readAcct(ctr, "cpuacct.usage_sys") + if err != nil { + return err + } + usage.PerCPU, err = readAcctList(ctr, "cpuacct.usage_percpu") + if err != nil { + return err + } } m.CPU = CPUMetrics{Usage: usage} return nil diff --git a/pkg/cgroups/cpuset.go b/pkg/cgroups/cpuset.go index 15c649e46..9aef493c9 100644 --- a/pkg/cgroups/cpuset.go +++ b/pkg/cgroups/cpuset.go @@ -8,6 +8,7 @@ import ( "strings" spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" ) type cpusetHandler struct { @@ -20,7 +21,7 @@ func cpusetCopyFileFromParent(dir, file string) ([]byte, error) { path := filepath.Join(dir, file) data, err := ioutil.ReadFile(path) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "open %s", path) } if len(strings.Trim(string(data), "\n")) != 0 { return data, nil @@ -30,7 +31,7 @@ func cpusetCopyFileFromParent(dir, file string) ([]byte, error) { return nil, err } if err := ioutil.WriteFile(path, data, 0644); err != nil { - return nil, err + return nil, errors.Wrapf(err, "write %s", path) } return data, nil } @@ -53,13 +54,13 @@ func (c *cpusetHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) erro if res.CPU == nil { return nil } - return fmt.Errorf("function not implemented yet") + return fmt.Errorf("cpuset apply not implemented yet") } // Create the cgroup func (c *cpusetHandler) Create(ctr *CgroupControl) (bool, error) { if ctr.cgroup2 { - return false, fmt.Errorf("function not implemented yet") + return false, fmt.Errorf("cpuset create not implemented for cgroup v2") } created, err := ctr.createCgroupDirectory(CPUset) diff --git a/pkg/cgroups/memory.go b/pkg/cgroups/memory.go index 2224e4f1e..0505eac40 100644 --- a/pkg/cgroups/memory.go +++ b/pkg/cgroups/memory.go @@ -20,13 +20,13 @@ func (c *memHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { if res.Memory == nil { return nil } - return fmt.Errorf("function not implemented yet") + return fmt.Errorf("memory apply not implemented yet") } // Create the cgroup func (c *memHandler) Create(ctr *CgroupControl) (bool, error) { if ctr.cgroup2 { - return false, fmt.Errorf("function not implemented yet") + return false, fmt.Errorf("memory create not implemented for cgroup v2") } return ctr.createCgroupDirectory(Memory) } @@ -38,19 +38,26 @@ func (c *memHandler) Destroy(ctr *CgroupControl) error { // Stat fills a metrics structure with usage stats for the controller func (c *memHandler) Stat(ctr *CgroupControl, m *Metrics) error { - if ctr.cgroup2 { - return fmt.Errorf("function not implemented yet") - } + var err error usage := MemoryUsage{} - memoryRoot := ctr.getCgroupv1Path(Memory) + var memoryRoot string + filenames := map[string]string{} - var err error - usage.Usage, err = readFileAsUint64(filepath.Join(memoryRoot, "memory.usage_in_bytes")) + if ctr.cgroup2 { + memoryRoot = filepath.Join(cgroupRoot, ctr.path) + filenames["usage"] = "memory.current" + filenames["limit"] = "memory.max" + } else { + memoryRoot = ctr.getCgroupv1Path(Memory) + filenames["usage"] = "memory.usage_in_bytes" + filenames["limit"] = "memory.limit_in_bytes" + } + usage.Usage, err = readFileAsUint64(filepath.Join(memoryRoot, filenames["usage"])) if err != nil { return err } - usage.Limit, err = readFileAsUint64(filepath.Join(memoryRoot, "memory.limit_in_bytes")) + usage.Limit, err = readFileAsUint64(filepath.Join(memoryRoot, filenames["limit"])) if err != nil { return err } diff --git a/pkg/cgroups/pids.go b/pkg/cgroups/pids.go index 342b25d85..c90dc1c02 100644 --- a/pkg/cgroups/pids.go +++ b/pkg/cgroups/pids.go @@ -21,18 +21,22 @@ func (c *pidHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { if res.Pids == nil { return nil } + var PIDRoot string + if ctr.cgroup2 { - return fmt.Errorf("function not implemented yet") + PIDRoot = filepath.Join(cgroupRoot, ctr.path) + } else { + PIDRoot = ctr.getCgroupv1Path(Pids) } - p := filepath.Join(ctr.getCgroupv1Path(Pids), "pids.max") + p := filepath.Join(PIDRoot, "pids.max") return ioutil.WriteFile(p, []byte(fmt.Sprintf("%d\n", res.Pids.Limit)), 0644) } // Create the cgroup func (c *pidHandler) Create(ctr *CgroupControl) (bool, error) { if ctr.cgroup2 { - return false, fmt.Errorf("function not implemented yet") + return false, fmt.Errorf("pid create not implemented for cgroup v2") } return ctr.createCgroupDirectory(Pids) } @@ -47,11 +51,11 @@ func (c *pidHandler) Stat(ctr *CgroupControl, m *Metrics) error { var PIDRoot string if ctr.cgroup2 { - return fmt.Errorf("function not implemented yet") + PIDRoot = filepath.Join(cgroupRoot, ctr.path) + } else { + PIDRoot = ctr.getCgroupv1Path(Pids) } - PIDRoot = ctr.getCgroupv1Path(Pids) - current, err := readFileAsUint64(filepath.Join(PIDRoot, "pids.current")) if err != nil { return err -- cgit v1.2.3-54-g00ecf