Merge pull request #3796 from giuseppe/enable-cgroupsv2-crun

cirrus: enable cgroups v2 tests with crun

5 files changed, 218 insertions, 34 deletions

diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go
index b6dd1ecd1..4e9881d59 100644
--- a/test/e2e/common_test.go
+++ b/test/e2e/common_test.go
@@ -538,7 +538,7 @@ func (p *PodmanTestIntegration) RunHealthCheck(cid string) error {
 			return nil
 		}
 		// Restart container if it's not running
-		ps := p.Podman([]string{"ps", "--no-trunc", "--q", "--filter", fmt.Sprintf("id=%s", cid)})
+		ps := p.Podman([]string{"ps", "--no-trunc", "--quiet", "--filter", fmt.Sprintf("id=%s", cid)})
 		ps.WaitWithDefaultTimeout()
 		if ps.ExitCode() == 0 {
 			if !strings.Contains(ps.OutputToString(), cid) {
diff --git a/test/e2e/pause_test.go b/test/e2e/pause_test.go
index 455f60937..c61131078 100644
--- a/test/e2e/pause_test.go
+++ b/test/e2e/pause_test.go
@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/containers/libpod/pkg/cgroups"
 	. "github.com/containers/libpod/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -25,6 +26,17 @@ var _ = Describe("Podman pause", func() {
 		if err != nil {
 			os.Exit(1)
 		}
+
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		if cgroupsv2 {
+			_, err := os.Stat("/sys/fs/cgroup/cgroup.freeze")
+			if err != nil {
+				Skip("freezer controller not available on the current kernel")
+			}
+		}
+
 		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.Setup()
 		podmanTest.SeedImages()
diff --git a/test/e2e/run_cpu_test.go b/test/e2e/run_cpu_test.go
index 87f89b1dd..4be9da3d2 100644
--- a/test/e2e/run_cpu_test.go
+++ b/test/e2e/run_cpu_test.go
@@ -3,8 +3,10 @@
 package integration
 
 import (
+	"io/ioutil"
 	"os"
 
+	"github.com/containers/libpod/pkg/cgroups"
 	. "github.com/containers/libpod/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -22,6 +24,16 @@ var _ = Describe("Podman run cpu", func() {
 		if err != nil {
 			os.Exit(1)
 		}
+
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		if cgroupsv2 {
+			if err := ioutil.WriteFile("/sys/fs/cgroup/cgroup.subtree_control", []byte("+cpuset"), 0644); err != nil {
+				Skip("cpuset controller not available on the current kernel")
+			}
+		}
+
 		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.Setup()
 		podmanTest.SeedImages()
@@ -36,44 +48,96 @@ var _ = Describe("Podman run cpu", func() {
 
 	It("podman run cpu-period", func() {
 		SkipIfRootless()
-		result := podmanTest.Podman([]string{"run", "--rm", "--cpu-period=5000", ALPINE, "cat", "/sys/fs/cgroup/cpu/cpu.cfs_period_us"})
+
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		var result *PodmanSessionIntegration
+		if cgroupsv2 {
+			result = podmanTest.Podman([]string{"run", "--rm", "--cpu-period=5000", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/cpu.max"})
+		} else {
+			result = podmanTest.Podman([]string{"run", "--rm", "--cpu-period=5000", ALPINE, "cat", "/sys/fs/cgroup/cpu/cpu.cfs_period_us"})
+		}
 		result.WaitWithDefaultTimeout()
 		Expect(result.ExitCode()).To(Equal(0))
-		Expect(result.OutputToString()).To(Equal("5000"))
+		Expect(result.LineInOutputContains("5000"))
 	})
 
 	It("podman run cpu-quota", func() {
 		SkipIfRootless()
-		result := podmanTest.Podman([]string{"run", "--rm", "--cpu-quota=5000", ALPINE, "cat", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"})
+
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		var result *PodmanSessionIntegration
+
+		if cgroupsv2 {
+			result = podmanTest.Podman([]string{"run", "--rm", "--cpu-quota=5000", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/cpu.max"})
+		} else {
+			result = podmanTest.Podman([]string{"run", "--rm", "--cpu-quota=5000", ALPINE, "cat", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"})
+		}
 		result.WaitWithDefaultTimeout()
 		Expect(result.ExitCode()).To(Equal(0))
-		Expect(result.OutputToString()).To(Equal("5000"))
+		Expect(result.LineInOutputContains("5000"))
 	})
 
 	It("podman run cpus", func() {
 		SkipIfRootless()
-		result := podmanTest.Podman([]string{"run", "--rm", "--cpus=0.5", ALPINE, "cat", "/sys/fs/cgroup/cpu/cpu.cfs_period_us"})
-		result.WaitWithDefaultTimeout()
-		Expect(result.ExitCode()).To(Equal(0))
-		Expect(result.OutputToString()).To(Equal("100000"))
 
-		result = podmanTest.Podman([]string{"run", "--rm", "--cpus=0.5", ALPINE, "cat", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"})
-		result.WaitWithDefaultTimeout()
-		Expect(result.ExitCode()).To(Equal(0))
-		Expect(result.OutputToString()).To(Equal("50000"))
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		if cgroupsv2 {
+			result := podmanTest.Podman([]string{"run", "--rm", "--cpu-quota=5000", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/cpu.max"})
+			result.WaitWithDefaultTimeout()
+			Expect(result.ExitCode()).To(Equal(0))
+			Expect(result.OutputToString()).To(Equal("5000 100000"))
+		} else {
+			result := podmanTest.Podman([]string{"run", "--rm", "--cpus=0.5", ALPINE, "cat", "/sys/fs/cgroup/cpu/cpu.cfs_period_us"})
+			result.WaitWithDefaultTimeout()
+			Expect(result.ExitCode()).To(Equal(0))
+			Expect(result.OutputToString()).To(Equal("100000"))
+
+			result = podmanTest.Podman([]string{"run", "--rm", "--cpus=0.5", ALPINE, "cat", "/sys/fs/cgroup/cpu/cpu.cfs_quota_us"})
+			result.WaitWithDefaultTimeout()
+			Expect(result.ExitCode()).To(Equal(0))
+			Expect(result.OutputToString()).To(Equal("50000"))
+		}
 	})
 
 	It("podman run cpu-shares", func() {
 		SkipIfRootless()
-		result := podmanTest.Podman([]string{"run", "--rm", "--cpu-shares=2", ALPINE, "cat", "/sys/fs/cgroup/cpu/cpu.shares"})
-		result.WaitWithDefaultTimeout()
-		Expect(result.ExitCode()).To(Equal(0))
-		Expect(result.OutputToString()).To(Equal("2"))
+
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		if cgroupsv2 {
+			// [2-262144] is mapped to [1-10000]
+			result := podmanTest.Podman([]string{"run", "--rm", "--cpu-shares=262144", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/cpu.weight"})
+			result.WaitWithDefaultTimeout()
+			Expect(result.ExitCode()).To(Equal(0))
+			Expect(result.OutputToString()).To(Equal("10000"))
+		} else {
+			result := podmanTest.Podman([]string{"run", "--rm", "--cpu-shares=2", ALPINE, "cat", "/sys/fs/cgroup/cpu/cpu.shares"})
+			result.WaitWithDefaultTimeout()
+			Expect(result.ExitCode()).To(Equal(0))
+			Expect(result.OutputToString()).To(Equal("2"))
+		}
 	})
 
 	It("podman run cpuset-cpus", func() {
 		SkipIfRootless()
-		result := podmanTest.Podman([]string{"run", "--rm", "--cpuset-cpus=0", ALPINE, "cat", "/sys/fs/cgroup/cpuset/cpuset.cpus"})
+
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		var result *PodmanSessionIntegration
+
+		if cgroupsv2 {
+			result = podmanTest.Podman([]string{"run", "--rm", "--cpuset-cpus=0", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/cpuset.cpus.effective"})
+		} else {
+			result = podmanTest.Podman([]string{"run", "--rm", "--cpuset-cpus=0", ALPINE, "cat", "/sys/fs/cgroup/cpuset/cpuset.cpus"})
+		}
 		result.WaitWithDefaultTimeout()
 		Expect(result.ExitCode()).To(Equal(0))
 		Expect(result.OutputToString()).To(Equal("0"))
@@ -81,7 +145,17 @@ var _ = Describe("Podman run cpu", func() {
 
 	It("podman run cpuset-mems", func() {
 		SkipIfRootless()
-		result := podmanTest.Podman([]string{"run", "--rm", "--cpuset-mems=0", ALPINE, "cat", "/sys/fs/cgroup/cpuset/cpuset.mems"})
+
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		var result *PodmanSessionIntegration
+
+		if cgroupsv2 {
+			result = podmanTest.Podman([]string{"run", "--rm", "--cpuset-mems=0", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/cpuset.mems.effective"})
+		} else {
+			result = podmanTest.Podman([]string{"run", "--rm", "--cpuset-mems=0", ALPINE, "cat", "/sys/fs/cgroup/cpuset/cpuset.mems"})
+		}
 		result.WaitWithDefaultTimeout()
 		Expect(result.ExitCode()).To(Equal(0))
 		Expect(result.OutputToString()).To(Equal("0"))
diff --git a/test/e2e/run_memory_test.go b/test/e2e/run_memory_test.go
index 8fe90c8d8..a45735a8a 100644
--- a/test/e2e/run_memory_test.go
+++ b/test/e2e/run_memory_test.go
@@ -5,6 +5,7 @@ package integration
 import (
 	"os"
 
+	"github.com/containers/libpod/pkg/cgroups"
 	. "github.com/containers/libpod/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
@@ -36,7 +37,16 @@ var _ = Describe("Podman run memory", func() {
 	})
 
 	It("podman run memory test", func() {
-		session := podmanTest.Podman([]string{"run", "--memory=40m", ALPINE, "cat", "/sys/fs/cgroup/memory/memory.limit_in_bytes"})
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		var session *PodmanSessionIntegration
+
+		if cgroupsv2 {
+			session = podmanTest.Podman([]string{"run", "--memory=40m", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/memory.max"})
+		} else {
+			session = podmanTest.Podman([]string{"run", "--memory=40m", ALPINE, "cat", "/sys/fs/cgroup/memory/memory.limit_in_bytes"})
+		}
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 		Expect(session.OutputToString()).To(Equal("41943040"))
@@ -46,13 +56,31 @@ var _ = Describe("Podman run memory", func() {
 		if podmanTest.Host.Distribution == "ubuntu" {
 			Skip("Unable to perform test on Ubuntu distributions due to memory management")
 		}
-		session := podmanTest.Podman([]string{"run", "--memory-reservation=40m", ALPINE, "cat", "/sys/fs/cgroup/memory/memory.soft_limit_in_bytes"})
+
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		var session *PodmanSessionIntegration
+
+		if cgroupsv2 {
+			session = podmanTest.Podman([]string{"run", "--memory-reservation=40m", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/memory.high"})
+		} else {
+			session = podmanTest.Podman([]string{"run", "--memory-reservation=40m", ALPINE, "cat", "/sys/fs/cgroup/memory/memory.soft_limit_in_bytes"})
+		}
+
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 		Expect(session.OutputToString()).To(Equal("41943040"))
 	})
 
 	It("podman run memory-swappiness test", func() {
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		if cgroupsv2 {
+			Skip("Memory swappiness not supported on cgroups v2")
+		}
+
 		session := podmanTest.Podman([]string{"run", "--memory-swappiness=15", ALPINE, "cat", "/sys/fs/cgroup/memory/memory.swappiness"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
@@ -60,6 +88,12 @@ var _ = Describe("Podman run memory", func() {
 	})
 
 	It("podman run kernel-memory test", func() {
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		if cgroupsv2 {
+			Skip("Kernel memory not supported on cgroups v2")
+		}
 		session := podmanTest.Podman([]string{"run", "--kernel-memory=40m", ALPINE, "cat", "/sys/fs/cgroup/memory/memory.kmem.limit_in_bytes"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go
index 1420a8403..ce2044a72 100644
--- a/test/e2e/run_test.go
+++ b/test/e2e/run_test.go
@@ -13,6 +13,7 @@ import (
 	"syscall"
 	"time"
 
+	"github.com/containers/libpod/pkg/cgroups"
 	. "github.com/containers/libpod/test/utils"
 	"github.com/containers/storage/pkg/stringid"
 	"github.com/mrunalp/fileutils"
@@ -263,9 +264,15 @@ var _ = Describe("Podman run", func() {
 		Expect(session.ExitCode()).To(Equal(0))
 		Expect(session.OutputToString()).To(ContainSubstring("1024"))
 
-		session = podmanTest.Podman([]string{"run", "--rm", "--oom-kill-disable=true", fedoraMinimal, "echo", "memory-hog"})
-		session.WaitWithDefaultTimeout()
-		Expect(session.ExitCode()).To(Equal(0))
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		if !cgroupsv2 {
+			// --oom-kill-disable not supported on cgroups v2.
+			session = podmanTest.Podman([]string{"run", "--rm", "--oom-kill-disable=true", fedoraMinimal, "echo", "memory-hog"})
+			session.WaitWithDefaultTimeout()
+			Expect(session.ExitCode()).To(Equal(0))
+		}
 
 		session = podmanTest.Podman([]string{"run", "--rm", "--oom-score-adj=100", fedoraMinimal, "cat", "/proc/self/oom_score_adj"})
 		session.WaitWithDefaultTimeout()
@@ -310,18 +317,43 @@ var _ = Describe("Podman run", func() {
 
 	It("podman run blkio-weight test", func() {
 		SkipIfRootless()
-		if _, err := os.Stat("/sys/fs/cgroup/blkio/blkio.weight"); os.IsNotExist(err) {
-			Skip("Kernel does not support blkio.weight")
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		if !cgroupsv2 {
+			if _, err := os.Stat("/sys/fs/cgroup/blkio/blkio.weight"); os.IsNotExist(err) {
+				Skip("Kernel does not support blkio.weight")
+			}
+		}
+
+		if cgroupsv2 {
+			// convert linearly from [10-1000] to [1-10000]
+			session := podmanTest.Podman([]string{"run", "--rm", "--blkio-weight=15", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.bfq.weight"})
+			session.WaitWithDefaultTimeout()
+			Expect(session.ExitCode()).To(Equal(0))
+			Expect(session.OutputToString()).To(ContainSubstring("51"))
+		} else {
+			session := podmanTest.Podman([]string{"run", "--rm", "--blkio-weight=15", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.weight"})
+			session.WaitWithDefaultTimeout()
+			Expect(session.ExitCode()).To(Equal(0))
+			Expect(session.OutputToString()).To(ContainSubstring("15"))
 		}
-		session := podmanTest.Podman([]string{"run", "--rm", "--blkio-weight=15", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.weight"})
-		session.WaitWithDefaultTimeout()
-		Expect(session.ExitCode()).To(Equal(0))
-		Expect(session.OutputToString()).To(ContainSubstring("15"))
 	})
 
 	It("podman run device-read-bps test", func() {
 		SkipIfRootless()
-		session := podmanTest.Podman([]string{"run", "--rm", "--device-read-bps=/dev/zero:1mb", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.read_bps_device"})
+
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		var session *PodmanSessionIntegration
+
+		if cgroupsv2 {
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-read-bps=/dev/zero:1mb", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.max"})
+		} else {
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-read-bps=/dev/zero:1mb", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.read_bps_device"})
+		}
+
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 		Expect(session.OutputToString()).To(ContainSubstring("1048576"))
@@ -329,7 +361,17 @@ var _ = Describe("Podman run", func() {
 
 	It("podman run device-write-bps test", func() {
 		SkipIfRootless()
-		session := podmanTest.Podman([]string{"run", "--rm", "--device-write-bps=/dev/zero:1mb", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.write_bps_device"})
+
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		var session *PodmanSessionIntegration
+
+		if cgroupsv2 {
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-write-bps=/dev/zero:1mb", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.max"})
+		} else {
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-write-bps=/dev/zero:1mb", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.write_bps_device"})
+		}
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 		Expect(session.OutputToString()).To(ContainSubstring("1048576"))
@@ -337,7 +379,18 @@ var _ = Describe("Podman run", func() {
 
 	It("podman run device-read-iops test", func() {
 		SkipIfRootless()
-		session := podmanTest.Podman([]string{"run", "--rm", "--device-read-iops=/dev/zero:100", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.read_iops_device"})
+
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		var session *PodmanSessionIntegration
+
+		if cgroupsv2 {
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-read-iops=/dev/zero:100", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.max"})
+		} else {
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-read-iops=/dev/zero:100", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.read_iops_device"})
+		}
+
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 		Expect(session.OutputToString()).To(ContainSubstring("100"))
@@ -345,7 +398,18 @@ var _ = Describe("Podman run", func() {
 
 	It("podman run device-write-iops test", func() {
 		SkipIfRootless()
-		session := podmanTest.Podman([]string{"run", "--rm", "--device-write-iops=/dev/zero:100", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.write_iops_device"})
+
+		cgroupsv2, err := cgroups.IsCgroup2UnifiedMode()
+		Expect(err).To(BeNil())
+
+		var session *PodmanSessionIntegration
+
+		if cgroupsv2 {
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-write-iops=/dev/zero:100", ALPINE, "sh", "-c", "cat /sys/fs/cgroup/$(sed -e 's|0::||' < /proc/self/cgroup)/io.max"})
+		} else {
+			session = podmanTest.Podman([]string{"run", "--rm", "--device-write-iops=/dev/zero:100", ALPINE, "cat", "/sys/fs/cgroup/blkio/blkio.throttle.write_iops_device"})
+		}
+
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 		Expect(session.OutputToString()).To(ContainSubstring("100"))