From 1778bfa5fef337158537a77344300e1a755a7ffe Mon Sep 17 00:00:00 2001 From: Giuseppe Scrivano Date: Wed, 19 Jun 2019 13:07:23 +0200 Subject: pkg, cgroups: add initial support for cgroup v2 This is an initial implementation of cgroup v2 support for pkg/cgroups. It currently works with crun, with this patch: https://github.com/giuseppe/crun/pull/49). It adds the pieces for: - set PID limit to 1 - retrieve stats so that "podman stats" work. the only missing part is the support for reading per CPU stats (that is cpuacct.usage_percpu on cgroup v1), so for now it always returns an empty result. Signed-off-by: Giuseppe Scrivano --- pkg/cgroups/blkio.go | 134 +++++++++++++++++++++++++++++++++---------------- pkg/cgroups/cgroups.go | 64 ++++++++++++++++++----- pkg/cgroups/cpu.go | 51 ++++++++++++------- pkg/cgroups/cpuset.go | 9 ++-- pkg/cgroups/memory.go | 25 +++++---- pkg/cgroups/pids.go | 16 +++--- 6 files changed, 207 insertions(+), 92 deletions(-) diff --git a/pkg/cgroups/blkio.go b/pkg/cgroups/blkio.go index 8eb54abec..ca9107d97 100644 --- a/pkg/cgroups/blkio.go +++ b/pkg/cgroups/blkio.go @@ -9,6 +9,7 @@ import ( "strings" spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" ) type blkioHandler struct { @@ -29,7 +30,7 @@ func (c *blkioHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error // Create the cgroup func (c *blkioHandler) Create(ctr *CgroupControl) (bool, error) { if ctr.cgroup2 { - return false, fmt.Errorf("function not implemented yet") + return false, fmt.Errorf("io create not implemented for cgroup v2") } return ctr.createCgroupDirectory(Blkio) } @@ -44,57 +45,104 @@ func (c *blkioHandler) Stat(ctr *CgroupControl, m *Metrics) error { var ioServiceBytesRecursive []BlkIOEntry if ctr.cgroup2 { - return fmt.Errorf("function not implemented yet") - } - - BlkioRoot := ctr.getCgroupv1Path(Blkio) - - p := filepath.Join(BlkioRoot, "blkio.throttle.io_service_bytes_recursive") - f, err := os.Open(p) - if err != nil { - if os.IsNotExist(err) { - return nil - } - return err - } - defer f.Close() - - scanner := bufio.NewScanner(f) - for scanner.Scan() { - line := scanner.Text() - parts := strings.Fields(line) - if len(parts) < 3 { - continue - } - d := strings.Split(parts[0], ":") - if len(d) != 2 { - continue - } - minor, err := strconv.ParseUint(d[0], 10, 0) + // more details on the io.stat file format:X https://facebookmicrosites.github.io/cgroup2/docs/io-controller.html + values, err := readCgroup2MapFile(ctr, "io.stat") if err != nil { return err } - major, err := strconv.ParseUint(d[1], 10, 0) + for k, v := range values { + d := strings.Split(k, ":") + if len(d) != 2 { + continue + } + minor, err := strconv.ParseUint(d[0], 10, 0) + if err != nil { + return err + } + major, err := strconv.ParseUint(d[1], 10, 0) + if err != nil { + return err + } + + for _, item := range v { + d := strings.Split(item, "=") + if len(d) != 2 { + continue + } + op := d[0] + + // Accommodate the cgroup v1 naming + switch op { + case "rbytes": + op = "read" + case "wbytes": + op = "write" + } + + value, err := strconv.ParseUint(d[1], 10, 0) + if err != nil { + return err + } + + entry := BlkIOEntry{ + Op: op, + Major: major, + Minor: minor, + Value: value, + } + ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry) + } + } + } else { + BlkioRoot := ctr.getCgroupv1Path(Blkio) + + p := filepath.Join(BlkioRoot, "blkio.throttle.io_service_bytes_recursive") + f, err := os.Open(p) if err != nil { - return err + if os.IsNotExist(err) { + return nil + } + return errors.Wrapf(err, "open %s", p) } + defer f.Close() - op := parts[1] + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(line) + if len(parts) < 3 { + continue + } + d := strings.Split(parts[0], ":") + if len(d) != 2 { + continue + } + minor, err := strconv.ParseUint(d[0], 10, 0) + if err != nil { + return err + } + major, err := strconv.ParseUint(d[1], 10, 0) + if err != nil { + return err + } - value, err := strconv.ParseUint(parts[2], 10, 0) - if err != nil { - return err + op := parts[1] + + value, err := strconv.ParseUint(parts[2], 10, 0) + if err != nil { + return err + } + entry := BlkIOEntry{ + Op: op, + Major: major, + Minor: minor, + Value: value, + } + ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry) } - entry := BlkIOEntry{ - Op: op, - Major: major, - Minor: minor, - Value: value, + if err := scanner.Err(); err != nil { + return errors.Wrapf(err, "parse %s", p) } - ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry) - } - if err := scanner.Err(); err != nil { - return err } m.Blkio = BlkioMetrics{IoServiceBytesRecursive: ioServiceBytesRecursive} return nil diff --git a/pkg/cgroups/cgroups.go b/pkg/cgroups/cgroups.go index 24dce71bd..426bda559 100644 --- a/pkg/cgroups/cgroups.go +++ b/pkg/cgroups/cgroups.go @@ -1,11 +1,14 @@ package cgroups import ( + "bufio" "fmt" "io/ioutil" + "math" "os" "path/filepath" "strconv" + "strings" spec "github.com/opencontainers/runtime-spec/specs-go" "github.com/pkg/errors" @@ -119,12 +122,12 @@ func init() { // getAvailableControllers get the available controllers func getAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]controller, error) { if cgroup2 { - return nil, fmt.Errorf("function not implemented yet") + return nil, fmt.Errorf("getAvailableControllers not implemented yet for cgroup v2") } infos, err := ioutil.ReadDir(cgroupRoot) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "read directory %s", cgroupRoot) } var controllers []controller for _, i := range infos { @@ -204,9 +207,17 @@ func (c *CgroupControl) createCgroupDirectory(controller string) (bool, error) { func readFileAsUint64(path string) (uint64, error) { data, err := ioutil.ReadFile(path) if err != nil { - return 0, err + return 0, errors.Wrapf(err, "open %s", path) + } + v := cleanString(string(data)) + if v == "max" { + return math.MaxUint64, nil + } + ret, err := strconv.ParseUint(v, 10, 0) + if err != nil { + return ret, errors.Wrapf(err, "parse %s from %s", v, path) } - return strconv.ParseUint(cleanString(string(data)), 10, 0) + return ret, nil } func (c *CgroupControl) writePidToTasks(pid int, name string) error { @@ -307,8 +318,9 @@ func (c *CgroupControl) DeleteByPath(path string) error { } for _, ctr := range c.additionalControllers { - if err := os.Remove(c.getCgroupv1Path(ctr.name)); err != nil { - lastError = err + p := c.getCgroupv1Path(ctr.name) + if err := os.Remove(p); err != nil { + lastError = errors.Wrapf(err, "remove %s", p) } } return lastError @@ -326,10 +338,15 @@ func (c *CgroupControl) Update(resources *spec.LinuxResources) error { // AddPid moves the specified pid to the cgroup func (c *CgroupControl) AddPid(pid int) error { + pidString := []byte(fmt.Sprintf("%d\n", pid)) + if c.cgroup2 { - return fmt.Errorf("function not implemented yet") + p := filepath.Join(cgroupRoot, c.path, "tasks") + if err := ioutil.WriteFile(p, pidString, 0644); err != nil { + return errors.Wrapf(err, "write %s", p) + } + return nil } - pidString := []byte(fmt.Sprintf("%d\n", pid)) var names []string for n := range handlers { @@ -345,7 +362,7 @@ func (c *CgroupControl) AddPid(pid int) error { for _, n := range names { p := filepath.Join(c.getCgroupv1Path(n), "tasks") if err := ioutil.WriteFile(p, pidString, 0644); err != nil { - return err + return errors.Wrapf(err, "write %s", p) } } return nil @@ -353,9 +370,6 @@ func (c *CgroupControl) AddPid(pid int) error { // Stat returns usage statistics for the cgroup func (c *CgroupControl) Stat() (*Metrics, error) { - if c.cgroup2 { - return nil, fmt.Errorf("function not implemented yet") - } m := Metrics{} for _, h := range handlers { if err := h.Stat(c, &m); err != nil { @@ -364,3 +378,29 @@ func (c *CgroupControl) Stat() (*Metrics, error) { } return &m, nil } + +func readCgroup2MapFile(ctr *CgroupControl, name string) (map[string][]string, error) { + ret := map[string][]string{} + p := filepath.Join(cgroupRoot, ctr.path, name) + f, err := os.Open(p) + if err != nil { + if os.IsNotExist(err) { + return ret, nil + } + return nil, errors.Wrapf(err, "open file %s", p) + } + defer f.Close() + scanner := bufio.NewScanner(f) + for scanner.Scan() { + line := scanner.Text() + parts := strings.Fields(line) + if len(parts) < 2 { + continue + } + ret[parts[0]] = parts[1:] + } + if err := scanner.Err(); err != nil { + return nil, errors.Wrapf(err, "parsing file %s", p) + } + return ret, nil +} diff --git a/pkg/cgroups/cpu.go b/pkg/cgroups/cpu.go index d27f1257f..3f969fd3c 100644 --- a/pkg/cgroups/cpu.go +++ b/pkg/cgroups/cpu.go @@ -55,13 +55,13 @@ func (c *cpuHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { if res.CPU == nil { return nil } - return fmt.Errorf("function not implemented yet") + return fmt.Errorf("cpu apply not implemented yet") } // Create the cgroup func (c *cpuHandler) Create(ctr *CgroupControl) (bool, error) { if ctr.cgroup2 { - return false, fmt.Errorf("function not implemented yet") + return false, fmt.Errorf("cpu create not implemented for cgroup v2") } return ctr.createCgroupDirectory(CPU) } @@ -73,24 +73,39 @@ func (c *cpuHandler) Destroy(ctr *CgroupControl) error { // Stat fills a metrics structure with usage stats for the controller func (c *cpuHandler) Stat(ctr *CgroupControl, m *Metrics) error { - if ctr.cgroup2 { - return fmt.Errorf("function not implemented yet") - } - var err error usage := CPUUsage{} - - usage.Total, err = readAcct(ctr, "cpuacct.usage") - if err != nil { - return err - } - usage.Kernel, err = readAcct(ctr, "cpuacct.usage_sys") - if err != nil { - return err - } - usage.PerCPU, err = readAcctList(ctr, "cpuacct.usage_percpu") - if err != nil { - return err + if ctr.cgroup2 { + values, err := readCgroup2MapFile(ctr, "cpu.stat") + if err != nil { + return err + } + if val, found := values["usage_usec"]; found { + usage.Kernel, err = strconv.ParseUint(cleanString(val[0]), 10, 0) + if err != nil { + return err + } + } + if val, found := values["system_usec"]; found { + usage.Total, err = strconv.ParseUint(cleanString(val[0]), 10, 0) + if err != nil { + return err + } + } + // FIXME: How to read usage.PerCPU? + } else { + usage.Total, err = readAcct(ctr, "cpuacct.usage") + if err != nil { + return err + } + usage.Kernel, err = readAcct(ctr, "cpuacct.usage_sys") + if err != nil { + return err + } + usage.PerCPU, err = readAcctList(ctr, "cpuacct.usage_percpu") + if err != nil { + return err + } } m.CPU = CPUMetrics{Usage: usage} return nil diff --git a/pkg/cgroups/cpuset.go b/pkg/cgroups/cpuset.go index 15c649e46..9aef493c9 100644 --- a/pkg/cgroups/cpuset.go +++ b/pkg/cgroups/cpuset.go @@ -8,6 +8,7 @@ import ( "strings" spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" ) type cpusetHandler struct { @@ -20,7 +21,7 @@ func cpusetCopyFileFromParent(dir, file string) ([]byte, error) { path := filepath.Join(dir, file) data, err := ioutil.ReadFile(path) if err != nil { - return nil, err + return nil, errors.Wrapf(err, "open %s", path) } if len(strings.Trim(string(data), "\n")) != 0 { return data, nil @@ -30,7 +31,7 @@ func cpusetCopyFileFromParent(dir, file string) ([]byte, error) { return nil, err } if err := ioutil.WriteFile(path, data, 0644); err != nil { - return nil, err + return nil, errors.Wrapf(err, "write %s", path) } return data, nil } @@ -53,13 +54,13 @@ func (c *cpusetHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) erro if res.CPU == nil { return nil } - return fmt.Errorf("function not implemented yet") + return fmt.Errorf("cpuset apply not implemented yet") } // Create the cgroup func (c *cpusetHandler) Create(ctr *CgroupControl) (bool, error) { if ctr.cgroup2 { - return false, fmt.Errorf("function not implemented yet") + return false, fmt.Errorf("cpuset create not implemented for cgroup v2") } created, err := ctr.createCgroupDirectory(CPUset) diff --git a/pkg/cgroups/memory.go b/pkg/cgroups/memory.go index 2224e4f1e..0505eac40 100644 --- a/pkg/cgroups/memory.go +++ b/pkg/cgroups/memory.go @@ -20,13 +20,13 @@ func (c *memHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { if res.Memory == nil { return nil } - return fmt.Errorf("function not implemented yet") + return fmt.Errorf("memory apply not implemented yet") } // Create the cgroup func (c *memHandler) Create(ctr *CgroupControl) (bool, error) { if ctr.cgroup2 { - return false, fmt.Errorf("function not implemented yet") + return false, fmt.Errorf("memory create not implemented for cgroup v2") } return ctr.createCgroupDirectory(Memory) } @@ -38,19 +38,26 @@ func (c *memHandler) Destroy(ctr *CgroupControl) error { // Stat fills a metrics structure with usage stats for the controller func (c *memHandler) Stat(ctr *CgroupControl, m *Metrics) error { - if ctr.cgroup2 { - return fmt.Errorf("function not implemented yet") - } + var err error usage := MemoryUsage{} - memoryRoot := ctr.getCgroupv1Path(Memory) + var memoryRoot string + filenames := map[string]string{} - var err error - usage.Usage, err = readFileAsUint64(filepath.Join(memoryRoot, "memory.usage_in_bytes")) + if ctr.cgroup2 { + memoryRoot = filepath.Join(cgroupRoot, ctr.path) + filenames["usage"] = "memory.current" + filenames["limit"] = "memory.max" + } else { + memoryRoot = ctr.getCgroupv1Path(Memory) + filenames["usage"] = "memory.usage_in_bytes" + filenames["limit"] = "memory.limit_in_bytes" + } + usage.Usage, err = readFileAsUint64(filepath.Join(memoryRoot, filenames["usage"])) if err != nil { return err } - usage.Limit, err = readFileAsUint64(filepath.Join(memoryRoot, "memory.limit_in_bytes")) + usage.Limit, err = readFileAsUint64(filepath.Join(memoryRoot, filenames["limit"])) if err != nil { return err } diff --git a/pkg/cgroups/pids.go b/pkg/cgroups/pids.go index 342b25d85..c90dc1c02 100644 --- a/pkg/cgroups/pids.go +++ b/pkg/cgroups/pids.go @@ -21,18 +21,22 @@ func (c *pidHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error { if res.Pids == nil { return nil } + var PIDRoot string + if ctr.cgroup2 { - return fmt.Errorf("function not implemented yet") + PIDRoot = filepath.Join(cgroupRoot, ctr.path) + } else { + PIDRoot = ctr.getCgroupv1Path(Pids) } - p := filepath.Join(ctr.getCgroupv1Path(Pids), "pids.max") + p := filepath.Join(PIDRoot, "pids.max") return ioutil.WriteFile(p, []byte(fmt.Sprintf("%d\n", res.Pids.Limit)), 0644) } // Create the cgroup func (c *pidHandler) Create(ctr *CgroupControl) (bool, error) { if ctr.cgroup2 { - return false, fmt.Errorf("function not implemented yet") + return false, fmt.Errorf("pid create not implemented for cgroup v2") } return ctr.createCgroupDirectory(Pids) } @@ -47,11 +51,11 @@ func (c *pidHandler) Stat(ctr *CgroupControl, m *Metrics) error { var PIDRoot string if ctr.cgroup2 { - return fmt.Errorf("function not implemented yet") + PIDRoot = filepath.Join(cgroupRoot, ctr.path) + } else { + PIDRoot = ctr.getCgroupv1Path(Pids) } - PIDRoot = ctr.getCgroupv1Path(Pids) - current, err := readFileAsUint64(filepath.Join(PIDRoot, "pids.current")) if err != nil { return err -- cgit v1.2.3-54-g00ecf