pkg, cgroups: add initial support for cgroup v2

This is an initial implementation of cgroup v2 support for
pkg/cgroups.  It currently works with crun, with this patch:
https://github.com/giuseppe/crun/pull/49).

It adds the pieces for:

- set PID limit to 1
- retrieve stats so that "podman stats" work.

the only missing part is the support for reading per
CPU stats (that is cpuacct.usage_percpu on cgroup v1), so for now it
always returns an empty result.

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

6 files changed, 207 insertions, 92 deletions

diff --git a/pkg/cgroups/blkio.go b/pkg/cgroups/blkio.go
index 8eb54abec..ca9107d97 100644
--- a/pkg/cgroups/blkio.go
+++ b/pkg/cgroups/blkio.go
@@ -9,6 +9,7 @@ import (
 	"strings"
 
 	spec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
 )
 
 type blkioHandler struct {
@@ -29,7 +30,7 @@ func (c *blkioHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error
 // Create the cgroup
 func (c *blkioHandler) Create(ctr *CgroupControl) (bool, error) {
 	if ctr.cgroup2 {
-		return false, fmt.Errorf("function not implemented yet")
+		return false, fmt.Errorf("io create not implemented for cgroup v2")
 	}
 	return ctr.createCgroupDirectory(Blkio)
 }
@@ -44,57 +45,104 @@ func (c *blkioHandler) Stat(ctr *CgroupControl, m *Metrics) error {
 	var ioServiceBytesRecursive []BlkIOEntry
 
 	if ctr.cgroup2 {
-		return fmt.Errorf("function not implemented yet")
-	}
-
-	BlkioRoot := ctr.getCgroupv1Path(Blkio)
-
-	p := filepath.Join(BlkioRoot, "blkio.throttle.io_service_bytes_recursive")
-	f, err := os.Open(p)
-	if err != nil {
-		if os.IsNotExist(err) {
-			return nil
-		}
-		return err
-	}
-	defer f.Close()
-
-	scanner := bufio.NewScanner(f)
-	for scanner.Scan() {
-		line := scanner.Text()
-		parts := strings.Fields(line)
-		if len(parts) < 3 {
-			continue
-		}
-		d := strings.Split(parts[0], ":")
-		if len(d) != 2 {
-			continue
-		}
-		minor, err := strconv.ParseUint(d[0], 10, 0)
+		// more details on the io.stat file format:X https://facebookmicrosites.github.io/cgroup2/docs/io-controller.html
+		values, err := readCgroup2MapFile(ctr, "io.stat")
 		if err != nil {
 			return err
 		}
-		major, err := strconv.ParseUint(d[1], 10, 0)
+		for k, v := range values {
+			d := strings.Split(k, ":")
+			if len(d) != 2 {
+				continue
+			}
+			minor, err := strconv.ParseUint(d[0], 10, 0)
+			if err != nil {
+				return err
+			}
+			major, err := strconv.ParseUint(d[1], 10, 0)
+			if err != nil {
+				return err
+			}
+
+			for _, item := range v {
+				d := strings.Split(item, "=")
+				if len(d) != 2 {
+					continue
+				}
+				op := d[0]
+
+				// Accommodate the cgroup v1 naming
+				switch op {
+				case "rbytes":
+					op = "read"
+				case "wbytes":
+					op = "write"
+				}
+
+				value, err := strconv.ParseUint(d[1], 10, 0)
+				if err != nil {
+					return err
+				}
+
+				entry := BlkIOEntry{
+					Op:    op,
+					Major: major,
+					Minor: minor,
+					Value: value,
+				}
+				ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry)
+			}
+		}
+	} else {
+		BlkioRoot := ctr.getCgroupv1Path(Blkio)
+
+		p := filepath.Join(BlkioRoot, "blkio.throttle.io_service_bytes_recursive")
+		f, err := os.Open(p)
 		if err != nil {
-			return err
+			if os.IsNotExist(err) {
+				return nil
+			}
+			return errors.Wrapf(err, "open %s", p)
 		}
+		defer f.Close()
 
-		op := parts[1]
+		scanner := bufio.NewScanner(f)
+		for scanner.Scan() {
+			line := scanner.Text()
+			parts := strings.Fields(line)
+			if len(parts) < 3 {
+				continue
+			}
+			d := strings.Split(parts[0], ":")
+			if len(d) != 2 {
+				continue
+			}
+			minor, err := strconv.ParseUint(d[0], 10, 0)
+			if err != nil {
+				return err
+			}
+			major, err := strconv.ParseUint(d[1], 10, 0)
+			if err != nil {
+				return err
+			}
 
-		value, err := strconv.ParseUint(parts[2], 10, 0)
-		if err != nil {
-			return err
+			op := parts[1]
+
+			value, err := strconv.ParseUint(parts[2], 10, 0)
+			if err != nil {
+				return err
+			}
+			entry := BlkIOEntry{
+				Op:    op,
+				Major: major,
+				Minor: minor,
+				Value: value,
+			}
+			ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry)
 		}
-		entry := BlkIOEntry{
-			Op:    op,
-			Major: major,
-			Minor: minor,
-			Value: value,
+		if err := scanner.Err(); err != nil {
+			return errors.Wrapf(err, "parse %s", p)
 		}
-		ioServiceBytesRecursive = append(ioServiceBytesRecursive, entry)
-	}
-	if err := scanner.Err(); err != nil {
-		return err
 	}
 	m.Blkio = BlkioMetrics{IoServiceBytesRecursive: ioServiceBytesRecursive}
 	return nil
diff --git a/pkg/cgroups/cgroups.go b/pkg/cgroups/cgroups.go
index 24dce71bd..426bda559 100644
--- a/pkg/cgroups/cgroups.go
+++ b/pkg/cgroups/cgroups.go
@@ -1,11 +1,14 @@
 package cgroups
 
 import (
+	"bufio"
 	"fmt"
 	"io/ioutil"
+	"math"
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 
 	spec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
@@ -119,12 +122,12 @@ func init() {
 // getAvailableControllers get the available controllers
 func getAvailableControllers(exclude map[string]controllerHandler, cgroup2 bool) ([]controller, error) {
 	if cgroup2 {
-		return nil, fmt.Errorf("function not implemented yet")
+		return nil, fmt.Errorf("getAvailableControllers not implemented yet for cgroup v2")
 	}
 
 	infos, err := ioutil.ReadDir(cgroupRoot)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "read directory %s", cgroupRoot)
 	}
 	var controllers []controller
 	for _, i := range infos {
@@ -204,9 +207,17 @@ func (c *CgroupControl) createCgroupDirectory(controller string) (bool, error) {
 func readFileAsUint64(path string) (uint64, error) {
 	data, err := ioutil.ReadFile(path)
 	if err != nil {
-		return 0, err
+		return 0, errors.Wrapf(err, "open %s", path)
+	}
+	v := cleanString(string(data))
+	if v == "max" {
+		return math.MaxUint64, nil
+	}
+	ret, err := strconv.ParseUint(v, 10, 0)
+	if err != nil {
+		return ret, errors.Wrapf(err, "parse %s from %s", v, path)
 	}
-	return strconv.ParseUint(cleanString(string(data)), 10, 0)
+	return ret, nil
 }
 
 func (c *CgroupControl) writePidToTasks(pid int, name string) error {
@@ -307,8 +318,9 @@ func (c *CgroupControl) DeleteByPath(path string) error {
 	}
 
 	for _, ctr := range c.additionalControllers {
-		if err := os.Remove(c.getCgroupv1Path(ctr.name)); err != nil {
-			lastError = err
+		p := c.getCgroupv1Path(ctr.name)
+		if err := os.Remove(p); err != nil {
+			lastError = errors.Wrapf(err, "remove %s", p)
 		}
 	}
 	return lastError
@@ -326,10 +338,15 @@ func (c *CgroupControl) Update(resources *spec.LinuxResources) error {
 
 // AddPid moves the specified pid to the cgroup
 func (c *CgroupControl) AddPid(pid int) error {
+	pidString := []byte(fmt.Sprintf("%d\n", pid))
+
 	if c.cgroup2 {
-		return fmt.Errorf("function not implemented yet")
+		p := filepath.Join(cgroupRoot, c.path, "tasks")
+		if err := ioutil.WriteFile(p, pidString, 0644); err != nil {
+			return errors.Wrapf(err, "write %s", p)
+		}
+		return nil
 	}
-	pidString := []byte(fmt.Sprintf("%d\n", pid))
 
 	var names []string
 	for n := range handlers {
@@ -345,7 +362,7 @@ func (c *CgroupControl) AddPid(pid int) error {
 	for _, n := range names {
 		p := filepath.Join(c.getCgroupv1Path(n), "tasks")
 		if err := ioutil.WriteFile(p, pidString, 0644); err != nil {
-			return err
+			return errors.Wrapf(err, "write %s", p)
 		}
 	}
 	return nil
@@ -353,9 +370,6 @@ func (c *CgroupControl) AddPid(pid int) error {
 
 // Stat returns usage statistics for the cgroup
 func (c *CgroupControl) Stat() (*Metrics, error) {
-	if c.cgroup2 {
-		return nil, fmt.Errorf("function not implemented yet")
-	}
 	m := Metrics{}
 	for _, h := range handlers {
 		if err := h.Stat(c, &m); err != nil {
@@ -364,3 +378,29 @@ func (c *CgroupControl) Stat() (*Metrics, error) {
 	}
 	return &m, nil
 }
+
+func readCgroup2MapFile(ctr *CgroupControl, name string) (map[string][]string, error) {
+	ret := map[string][]string{}
+	p := filepath.Join(cgroupRoot, ctr.path, name)
+	f, err := os.Open(p)
+	if err != nil {
+		if os.IsNotExist(err) {
+			return ret, nil
+		}
+		return nil, errors.Wrapf(err, "open file %s", p)
+	}
+	defer f.Close()
+	scanner := bufio.NewScanner(f)
+	for scanner.Scan() {
+		line := scanner.Text()
+		parts := strings.Fields(line)
+		if len(parts) < 2 {
+			continue
+		}
+		ret[parts[0]] = parts[1:]
+	}
+	if err := scanner.Err(); err != nil {
+		return nil, errors.Wrapf(err, "parsing file %s", p)
+	}
+	return ret, nil
+}
diff --git a/pkg/cgroups/cpu.go b/pkg/cgroups/cpu.go
index d27f1257f..3f969fd3c 100644
--- a/pkg/cgroups/cpu.go
+++ b/pkg/cgroups/cpu.go
@@ -55,13 +55,13 @@ func (c *cpuHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error {
 	if res.CPU == nil {
 		return nil
 	}
-	return fmt.Errorf("function not implemented yet")
+	return fmt.Errorf("cpu apply not implemented yet")
 }
 
 // Create the cgroup
 func (c *cpuHandler) Create(ctr *CgroupControl) (bool, error) {
 	if ctr.cgroup2 {
-		return false, fmt.Errorf("function not implemented yet")
+		return false, fmt.Errorf("cpu create not implemented for cgroup v2")
 	}
 	return ctr.createCgroupDirectory(CPU)
 }
@@ -73,24 +73,39 @@ func (c *cpuHandler) Destroy(ctr *CgroupControl) error {
 
 // Stat fills a metrics structure with usage stats for the controller
 func (c *cpuHandler) Stat(ctr *CgroupControl, m *Metrics) error {
-	if ctr.cgroup2 {
-		return fmt.Errorf("function not implemented yet")
-	}
-
 	var err error
 	usage := CPUUsage{}
-
-	usage.Total, err = readAcct(ctr, "cpuacct.usage")
-	if err != nil {
-		return err
-	}
-	usage.Kernel, err = readAcct(ctr, "cpuacct.usage_sys")
-	if err != nil {
-		return err
-	}
-	usage.PerCPU, err = readAcctList(ctr, "cpuacct.usage_percpu")
-	if err != nil {
-		return err
+	if ctr.cgroup2 {
+		values, err := readCgroup2MapFile(ctr, "cpu.stat")
+		if err != nil {
+			return err
+		}
+		if val, found := values["usage_usec"]; found {
+			usage.Kernel, err = strconv.ParseUint(cleanString(val[0]), 10, 0)
+			if err != nil {
+				return err
+			}
+		}
+		if val, found := values["system_usec"]; found {
+			usage.Total, err = strconv.ParseUint(cleanString(val[0]), 10, 0)
+			if err != nil {
+				return err
+			}
+		}
+		// FIXME: How to read usage.PerCPU?
+	} else {
+		usage.Total, err = readAcct(ctr, "cpuacct.usage")
+		if err != nil {
+			return err
+		}
+		usage.Kernel, err = readAcct(ctr, "cpuacct.usage_sys")
+		if err != nil {
+			return err
+		}
+		usage.PerCPU, err = readAcctList(ctr, "cpuacct.usage_percpu")
+		if err != nil {
+			return err
+		}
 	}
 	m.CPU = CPUMetrics{Usage: usage}
 	return nil
diff --git a/pkg/cgroups/cpuset.go b/pkg/cgroups/cpuset.go
index 15c649e46..9aef493c9 100644
--- a/pkg/cgroups/cpuset.go
+++ b/pkg/cgroups/cpuset.go
@@ -8,6 +8,7 @@ import (
 	"strings"
 
 	spec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
 )
 
 type cpusetHandler struct {
@@ -20,7 +21,7 @@ func cpusetCopyFileFromParent(dir, file string) ([]byte, error) {
 	path := filepath.Join(dir, file)
 	data, err := ioutil.ReadFile(path)
 	if err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "open %s", path)
 	}
 	if len(strings.Trim(string(data), "\n")) != 0 {
 		return data, nil
@@ -30,7 +31,7 @@ func cpusetCopyFileFromParent(dir, file string) ([]byte, error) {
 		return nil, err
 	}
 	if err := ioutil.WriteFile(path, data, 0644); err != nil {
-		return nil, err
+		return nil, errors.Wrapf(err, "write %s", path)
 	}
 	return data, nil
 }
@@ -53,13 +54,13 @@ func (c *cpusetHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) erro
 	if res.CPU == nil {
 		return nil
 	}
-	return fmt.Errorf("function not implemented yet")
+	return fmt.Errorf("cpuset apply not implemented yet")
 }
 
 // Create the cgroup
 func (c *cpusetHandler) Create(ctr *CgroupControl) (bool, error) {
 	if ctr.cgroup2 {
-		return false, fmt.Errorf("function not implemented yet")
+		return false, fmt.Errorf("cpuset create not implemented for cgroup v2")
 	}
 
 	created, err := ctr.createCgroupDirectory(CPUset)
diff --git a/pkg/cgroups/memory.go b/pkg/cgroups/memory.go
index 2224e4f1e..0505eac40 100644
--- a/pkg/cgroups/memory.go
+++ b/pkg/cgroups/memory.go
@@ -20,13 +20,13 @@ func (c *memHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error {
 	if res.Memory == nil {
 		return nil
 	}
-	return fmt.Errorf("function not implemented yet")
+	return fmt.Errorf("memory apply not implemented yet")
 }
 
 // Create the cgroup
 func (c *memHandler) Create(ctr *CgroupControl) (bool, error) {
 	if ctr.cgroup2 {
-		return false, fmt.Errorf("function not implemented yet")
+		return false, fmt.Errorf("memory create not implemented for cgroup v2")
 	}
 	return ctr.createCgroupDirectory(Memory)
 }
@@ -38,19 +38,26 @@ func (c *memHandler) Destroy(ctr *CgroupControl) error {
 
 // Stat fills a metrics structure with usage stats for the controller
 func (c *memHandler) Stat(ctr *CgroupControl, m *Metrics) error {
-	if ctr.cgroup2 {
-		return fmt.Errorf("function not implemented yet")
-	}
+	var err error
 	usage := MemoryUsage{}
 
-	memoryRoot := ctr.getCgroupv1Path(Memory)
+	var memoryRoot string
+	filenames := map[string]string{}
 
-	var err error
-	usage.Usage, err = readFileAsUint64(filepath.Join(memoryRoot, "memory.usage_in_bytes"))
+	if ctr.cgroup2 {
+		memoryRoot = filepath.Join(cgroupRoot, ctr.path)
+		filenames["usage"] = "memory.current"
+		filenames["limit"] = "memory.max"
+	} else {
+		memoryRoot = ctr.getCgroupv1Path(Memory)
+		filenames["usage"] = "memory.usage_in_bytes"
+		filenames["limit"] = "memory.limit_in_bytes"
+	}
+	usage.Usage, err = readFileAsUint64(filepath.Join(memoryRoot, filenames["usage"]))
 	if err != nil {
 		return err
 	}
-	usage.Limit, err = readFileAsUint64(filepath.Join(memoryRoot, "memory.limit_in_bytes"))
+	usage.Limit, err = readFileAsUint64(filepath.Join(memoryRoot, filenames["limit"]))
 	if err != nil {
 		return err
 	}
diff --git a/pkg/cgroups/pids.go b/pkg/cgroups/pids.go
index 342b25d85..c90dc1c02 100644
--- a/pkg/cgroups/pids.go
+++ b/pkg/cgroups/pids.go
@@ -21,18 +21,22 @@ func (c *pidHandler) Apply(ctr *CgroupControl, res *spec.LinuxResources) error {
 	if res.Pids == nil {
 		return nil
 	}
+	var PIDRoot string
+
 	if ctr.cgroup2 {
-		return fmt.Errorf("function not implemented yet")
+		PIDRoot = filepath.Join(cgroupRoot, ctr.path)
+	} else {
+		PIDRoot = ctr.getCgroupv1Path(Pids)
 	}
 
-	p := filepath.Join(ctr.getCgroupv1Path(Pids), "pids.max")
+	p := filepath.Join(PIDRoot, "pids.max")
 	return ioutil.WriteFile(p, []byte(fmt.Sprintf("%d\n", res.Pids.Limit)), 0644)
 }
 
 // Create the cgroup
 func (c *pidHandler) Create(ctr *CgroupControl) (bool, error) {
 	if ctr.cgroup2 {
-		return false, fmt.Errorf("function not implemented yet")
+		return false, fmt.Errorf("pid create not implemented for cgroup v2")
 	}
 	return ctr.createCgroupDirectory(Pids)
 }
@@ -47,11 +51,11 @@ func (c *pidHandler) Stat(ctr *CgroupControl, m *Metrics) error {
 	var PIDRoot string
 
 	if ctr.cgroup2 {
-		return fmt.Errorf("function not implemented yet")
+		PIDRoot = filepath.Join(cgroupRoot, ctr.path)
+	} else {
+		PIDRoot = ctr.getCgroupv1Path(Pids)
 	}
 
-	PIDRoot = ctr.getCgroupv1Path(Pids)
-
 	current, err := readFileAsUint64(filepath.Join(PIDRoot, "pids.current"))
 	if err != nil {
 		return err