Merge pull request #3374 from giuseppe/cgroups

cgroups: add initial support for cgroups v2

8 files changed, 46 insertions, 52 deletions

diff --git a/libpod/container.go b/libpod/container.go
index 0c1315843..d05baa7e0 100644
--- a/libpod/container.go
+++ b/libpod/container.go
@@ -14,6 +14,7 @@ import (
 	"github.com/containers/libpod/libpod/define"
 	"github.com/containers/libpod/libpod/lock"
 	"github.com/containers/libpod/pkg/namespaces"
+	"github.com/containers/libpod/pkg/rootless"
 	"github.com/containers/storage"
 	"github.com/cri-o/ocicni/pkg/ocicni"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
@@ -52,6 +53,10 @@ const CgroupfsDefaultCgroupParent = "/libpod_parent"
 // manager in libpod
 const SystemdDefaultCgroupParent = "machine.slice"
 
+// SystemdDefaultRootlessCgroupParent is the cgroup parent for the systemd cgroup
+// manager in libpod when running as rootless
+const SystemdDefaultRootlessCgroupParent = "user.slice"
+
 // JournaldLogging is the string conmon expects to specify journald logging
 const JournaldLogging = "journald"
 
@@ -1109,6 +1114,10 @@ func (c *Container) CGroupPath() (string, error) {
 	case CgroupfsCgroupsManager:
 		return filepath.Join(c.config.CgroupParent, fmt.Sprintf("libpod-%s", c.ID())), nil
 	case SystemdCgroupsManager:
+		if rootless.IsRootless() {
+			uid := rootless.GetRootlessUID()
+			return filepath.Join(c.config.CgroupParent, fmt.Sprintf("user-%d.slice/user@%d.service/user.slice", uid, uid), createUnitName("libpod", c.ID())), nil
+		}
 		return filepath.Join(c.config.CgroupParent, createUnitName("libpod", c.ID())), nil
 	default:
 		return "", errors.Wrapf(define.ErrInvalidArg, "unsupported CGroup manager %s in use", c.runtime.config.CgroupManager)
diff --git a/libpod/container_internal.go b/libpod/container_internal.go
index 2687bcdd1..fcd6a990a 100644
--- a/libpod/container_internal.go
+++ b/libpod/container_internal.go
@@ -1461,13 +1461,6 @@ func (c *Container) unmount(force bool) error {
 	return nil
 }
 
-// getExcludedCGroups returns a string slice of cgroups we want to exclude
-// because runc or other components are unaware of them.
-func getExcludedCGroups() (excludes []string) {
-	excludes = []string{"rdma"}
-	return
-}
-
 // this should be from chrootarchive.
 func (c *Container) copyWithTarFromImage(src, dest string) error {
 	mountpoint, err := c.mount()
diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go
index 1ea858886..e93e0cad8 100644
--- a/libpod/container_internal_linux.go
+++ b/libpod/container_internal_linux.go
@@ -23,6 +23,7 @@ import (
 	"github.com/containers/libpod/libpod/define"
 	crioAnnotations "github.com/containers/libpod/pkg/annotations"
 	"github.com/containers/libpod/pkg/apparmor"
+	"github.com/containers/libpod/pkg/cgroups"
 	"github.com/containers/libpod/pkg/criu"
 	"github.com/containers/libpod/pkg/lookup"
 	"github.com/containers/libpod/pkg/resolvconf"
@@ -350,7 +351,11 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 		g.AddProcessEnv("container", "libpod")
 	}
 
-	if rootless.IsRootless() {
+	unified, err := cgroups.IsCgroup2UnifiedMode()
+	if err != nil {
+		return nil, err
+	}
+	if rootless.IsRootless() && !unified {
 		g.SetLinuxCgroupsPath("")
 	} else if c.runtime.config.CgroupManager == SystemdCgroupsManager {
 		// When runc is set to use Systemd as a cgroup manager, it
diff --git a/libpod/oci_linux.go b/libpod/oci_linux.go
index 07bc4e5f3..7d9f47ae2 100644
--- a/libpod/oci_linux.go
+++ b/libpod/oci_linux.go
@@ -15,8 +15,8 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/containerd/cgroups"
 	"github.com/containers/libpod/libpod/define"
+	"github.com/containers/libpod/pkg/cgroups"
 	"github.com/containers/libpod/pkg/rootless"
 	"github.com/containers/libpod/pkg/util"
 	"github.com/containers/libpod/utils"
@@ -49,13 +49,13 @@ func (r *OCIRuntime) moveConmonToCgroup(ctr *Container, cgroupParent string, cmd
 			}
 		} else {
 			cgroupPath := filepath.Join(ctr.config.CgroupParent, "conmon")
-			control, err := cgroups.New(cgroups.V1, cgroups.StaticPath(cgroupPath), &spec.LinuxResources{})
+			control, err := cgroups.New(cgroupPath, &spec.LinuxResources{})
 			if err != nil {
 				logrus.Warnf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err)
 			} else {
 				// we need to remove this defer and delete the cgroup once conmon exits
 				// maybe need a conmon monitor?
-				if err := control.Add(cgroups.Process{Pid: cmd.Process.Pid}); err != nil {
+				if err := control.AddPid(cmd.Process.Pid); err != nil {
 					logrus.Warnf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err)
 				}
 			}
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
index bd1d8a198..79e18dcd1 100644
--- a/libpod/runtime_ctr.go
+++ b/libpod/runtime_ctr.go
@@ -191,6 +191,8 @@ func (r *Runtime) setupContainer(ctx context.Context, ctr *Container, restore bo
 					return nil, errors.Wrapf(err, "error retrieving pod %s cgroup", pod.ID())
 				}
 				ctr.config.CgroupParent = podCgroup
+			} else if rootless.IsRootless() {
+				ctr.config.CgroupParent = SystemdDefaultRootlessCgroupParent
 			} else {
 				ctr.config.CgroupParent = SystemdDefaultCgroupParent
 			}
diff --git a/libpod/runtime_pod_linux.go b/libpod/runtime_pod_linux.go
index d1d303ad5..e9ce130da 100644
--- a/libpod/runtime_pod_linux.go
+++ b/libpod/runtime_pod_linux.go
@@ -9,9 +9,10 @@ import (
 	"path/filepath"
 	"strings"
 
-	"github.com/containerd/cgroups"
 	"github.com/containers/libpod/libpod/define"
 	"github.com/containers/libpod/libpod/events"
+	"github.com/containers/libpod/pkg/cgroups"
+	"github.com/containers/libpod/pkg/rootless"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -78,7 +79,11 @@ func (r *Runtime) NewPod(ctx context.Context, options ...PodCreateOption) (*Pod,
 		}
 	case SystemdCgroupsManager:
 		if pod.config.CgroupParent == "" {
-			pod.config.CgroupParent = SystemdDefaultCgroupParent
+			if rootless.IsRootless() {
+				pod.config.CgroupParent = SystemdDefaultRootlessCgroupParent
+			} else {
+				pod.config.CgroupParent = SystemdDefaultCgroupParent
+			}
 		} else if len(pod.config.CgroupParent) < 6 || !strings.HasSuffix(path.Base(pod.config.CgroupParent), ".slice") {
 			return nil, errors.Wrapf(define.ErrInvalidArg, "did not receive systemd slice as cgroup parent when using systemd to manage cgroups")
 		}
@@ -183,9 +188,8 @@ func (r *Runtime) removePod(ctx context.Context, p *Pod, removeCtrs, force bool)
 	// would prevent removing the CGroups.
 	if p.runtime.config.CgroupManager == CgroupfsCgroupsManager {
 		// Get the conmon CGroup
-		v1CGroups := GetV1CGroups(getExcludedCGroups())
 		conmonCgroupPath := filepath.Join(p.state.CgroupPath, "conmon")
-		conmonCgroup, err := cgroups.Load(v1CGroups, cgroups.StaticPath(conmonCgroupPath))
+		conmonCgroup, err := cgroups.Load(conmonCgroupPath)
 		if err != nil && err != cgroups.ErrCgroupDeleted {
 			if removalErr == nil {
 				removalErr = errors.Wrapf(err, "error retrieving pod %s conmon cgroup %s", p.ID(), conmonCgroupPath)
@@ -250,9 +254,8 @@ func (r *Runtime) removePod(ctx context.Context, p *Pod, removeCtrs, force bool)
 			// Make sure the conmon cgroup is deleted first
 			// Since the pod is almost gone, don't bother failing
 			// hard - instead, just log errors.
-			v1CGroups := GetV1CGroups(getExcludedCGroups())
 			conmonCgroupPath := filepath.Join(p.state.CgroupPath, "conmon")
-			conmonCgroup, err := cgroups.Load(v1CGroups, cgroups.StaticPath(conmonCgroupPath))
+			conmonCgroup, err := cgroups.Load(conmonCgroupPath)
 			if err != nil && err != cgroups.ErrCgroupDeleted {
 				if removalErr == nil {
 					removalErr = errors.Wrapf(err, "error retrieving pod %s conmon cgroup", p.ID())
@@ -269,7 +272,7 @@ func (r *Runtime) removePod(ctx context.Context, p *Pod, removeCtrs, force bool)
 					}
 				}
 			}
-			cgroup, err := cgroups.Load(v1CGroups, cgroups.StaticPath(p.state.CgroupPath))
+			cgroup, err := cgroups.Load(p.state.CgroupPath)
 			if err != nil && err != cgroups.ErrCgroupDeleted {
 				if removalErr == nil {
 					removalErr = errors.Wrapf(err, "error retrieving pod %s cgroup", p.ID())
diff --git a/libpod/stats.go b/libpod/stats.go
index 926a1a511..e003f145b 100644
--- a/libpod/stats.go
+++ b/libpod/stats.go
@@ -7,8 +7,8 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/containerd/cgroups"
 	"github.com/containers/libpod/libpod/define"
+	"github.com/containers/libpod/pkg/cgroups"
 	"github.com/pkg/errors"
 )
 
@@ -34,14 +34,13 @@ func (c *Container) GetContainerStats(previousStats *ContainerStats) (*Container
 	if err != nil {
 		return nil, err
 	}
-	v1CGroups := GetV1CGroups(getExcludedCGroups())
-	cgroup, err := cgroups.Load(v1CGroups, cgroups.StaticPath(cgroupPath))
+	cgroup, err := cgroups.Load(cgroupPath)
 	if err != nil {
 		return stats, errors.Wrapf(err, "unable to load cgroup at %s", cgroupPath)
 	}
 
 	// Ubuntu does not have swap memory in cgroups because swap is often not enabled.
-	cgroupStats, err := cgroup.Stat(cgroups.IgnoreNotExist)
+	cgroupStats, err := cgroup.Stat()
 	if err != nil {
 		return stats, errors.Wrapf(err, "unable to obtain cgroup stats")
 	}
diff --git a/libpod/util_linux.go b/libpod/util_linux.go
index 8752eba20..78cbc75a7 100644
--- a/libpod/util_linux.go
+++ b/libpod/util_linux.go
@@ -6,10 +6,9 @@ import (
 	"fmt"
 	"strings"
 
-	"github.com/containerd/cgroups"
 	"github.com/containers/libpod/libpod/define"
-	"github.com/containers/libpod/pkg/util"
-	spec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/containers/libpod/pkg/cgroups"
+	"github.com/containers/libpod/pkg/rootless"
 	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -35,24 +34,31 @@ func systemdSliceFromPath(parent, name string) (string, error) {
 	return cgroupPath, nil
 }
 
+func getDefaultSystemdCgroup() string {
+	if rootless.IsRootless() {
+		return SystemdDefaultRootlessCgroupParent
+	}
+	return SystemdDefaultCgroupParent
+}
+
 // makeSystemdCgroup creates a systemd CGroup at the given location.
 func makeSystemdCgroup(path string) error {
-	controller, err := cgroups.NewSystemd(SystemdDefaultCgroupParent)
+	controller, err := cgroups.NewSystemd(getDefaultSystemdCgroup())
 	if err != nil {
 		return err
 	}
 
-	return controller.Create(path, &spec.LinuxResources{})
+	return controller.CreateSystemdUnit(path)
 }
 
 // deleteSystemdCgroup deletes the systemd cgroup at the given location
 func deleteSystemdCgroup(path string) error {
-	controller, err := cgroups.NewSystemd(SystemdDefaultCgroupParent)
+	controller, err := cgroups.NewSystemd(getDefaultSystemdCgroup())
 	if err != nil {
 		return err
 	}
 
-	return controller.Delete(path)
+	return controller.DeleteByPath(path)
 }
 
 // assembleSystemdCgroupName creates a systemd cgroup path given a base and
@@ -71,29 +77,6 @@ func assembleSystemdCgroupName(baseSlice, newSlice string) (string, error) {
 	return final, nil
 }
 
-// GetV1CGroups gets the V1 cgroup subsystems and then "filters"
-// out any subsystems that are provided by the caller.  Passing nil
-// for excludes will return the subsystems unfiltered.
-//func GetV1CGroups(excludes []string) ([]cgroups.Subsystem, error) {
-func GetV1CGroups(excludes []string) cgroups.Hierarchy {
-	return func() ([]cgroups.Subsystem, error) {
-		var filtered []cgroups.Subsystem
-
-		subSystem, err := cgroups.V1()
-		if err != nil {
-			return nil, err
-		}
-		for _, s := range subSystem {
-			// If the name of the subsystem is not in the list of excludes, then
-			// add it as a keeper.
-			if !util.StringInSlice(string(s.Name()), excludes) {
-				filtered = append(filtered, s)
-			}
-		}
-		return filtered, nil
-	}
-}
-
 // LabelVolumePath takes a mount path for a volume and gives it an
 // selinux label of either shared or not
 func LabelVolumePath(path string, shared bool) error {