resource limits for pods

added the following flags and handling for podman pod create

--memory-swap
--cpuset-mems
--device-read-bps
--device-write-bps
--blkio-weight
--blkio-weight-device
--cpu-shares

given the new backend for systemd in c/common, all of these can now be exposed to pod create.
most of the heavy lifting (nearly all) is done within c/common. However, some rewiring needed to be done here
as well!

Signed-off-by: Charlie Doern <cdoern@redhat.com>

5 files changed, 135 insertions, 66 deletions

diff --git a/pkg/specgen/generate/container.go b/pkg/specgen/generate/container.go
index 2248c9235..9bb7caace 100644
--- a/pkg/specgen/generate/container.go
+++ b/pkg/specgen/generate/container.go
@@ -302,60 +302,6 @@ func CompleteSpec(ctx context.Context, r *libpod.Runtime, s *specgen.SpecGenerat
 	return warnings, nil
 }
 
-// FinishThrottleDevices takes the temporary representation of the throttle
-// devices in the specgen and looks up the major and major minors. it then
-// sets the throttle devices proper in the specgen
-func FinishThrottleDevices(s *specgen.SpecGenerator) error {
-	if bps := s.ThrottleReadBpsDevice; len(bps) > 0 {
-		for k, v := range bps {
-			statT := unix.Stat_t{}
-			if err := unix.Stat(k, &statT); err != nil {
-				return err
-			}
-			v.Major = (int64(unix.Major(uint64(statT.Rdev)))) //nolint: unconvert
-			v.Minor = (int64(unix.Minor(uint64(statT.Rdev)))) //nolint: unconvert
-			if s.ResourceLimits.BlockIO == nil {
-				s.ResourceLimits.BlockIO = new(spec.LinuxBlockIO)
-			}
-			s.ResourceLimits.BlockIO.ThrottleReadBpsDevice = append(s.ResourceLimits.BlockIO.ThrottleReadBpsDevice, v)
-		}
-	}
-	if bps := s.ThrottleWriteBpsDevice; len(bps) > 0 {
-		for k, v := range bps {
-			statT := unix.Stat_t{}
-			if err := unix.Stat(k, &statT); err != nil {
-				return err
-			}
-			v.Major = (int64(unix.Major(uint64(statT.Rdev)))) //nolint: unconvert
-			v.Minor = (int64(unix.Minor(uint64(statT.Rdev)))) //nolint: unconvert
-			s.ResourceLimits.BlockIO.ThrottleWriteBpsDevice = append(s.ResourceLimits.BlockIO.ThrottleWriteBpsDevice, v)
-		}
-	}
-	if iops := s.ThrottleReadIOPSDevice; len(iops) > 0 {
-		for k, v := range iops {
-			statT := unix.Stat_t{}
-			if err := unix.Stat(k, &statT); err != nil {
-				return err
-			}
-			v.Major = (int64(unix.Major(uint64(statT.Rdev)))) //nolint: unconvert
-			v.Minor = (int64(unix.Minor(uint64(statT.Rdev)))) //nolint: unconvert
-			s.ResourceLimits.BlockIO.ThrottleReadIOPSDevice = append(s.ResourceLimits.BlockIO.ThrottleReadIOPSDevice, v)
-		}
-	}
-	if iops := s.ThrottleWriteIOPSDevice; len(iops) > 0 {
-		for k, v := range iops {
-			statT := unix.Stat_t{}
-			if err := unix.Stat(k, &statT); err != nil {
-				return err
-			}
-			v.Major = (int64(unix.Major(uint64(statT.Rdev)))) //nolint: unconvert
-			v.Minor = (int64(unix.Minor(uint64(statT.Rdev)))) //nolint: unconvert
-			s.ResourceLimits.BlockIO.ThrottleWriteIOPSDevice = append(s.ResourceLimits.BlockIO.ThrottleWriteIOPSDevice, v)
-		}
-	}
-	return nil
-}
-
 // ConfigToSpec takes a completed container config and converts it back into a specgenerator for purposes of cloning an existing container
 func ConfigToSpec(rt *libpod.Runtime, specg *specgen.SpecGenerator, contaierID string) (*libpod.Container, *libpod.InfraInherit, error) {
 	c, err := rt.LookupContainer(contaierID)
@@ -540,3 +486,63 @@ func mapSecurityConfig(c *libpod.ContainerConfig, s *specgen.SpecGenerator) {
 	s.Groups = c.Groups
 	s.HostUsers = c.HostUsers
 }
+
+// FinishThrottleDevices takes the temporary representation of the throttle
+// devices in the specgen and looks up the major and major minors. it then
+// sets the throttle devices proper in the specgen
+func FinishThrottleDevices(s *specgen.SpecGenerator) error {
+	if s.ResourceLimits == nil {
+		s.ResourceLimits = &spec.LinuxResources{}
+	}
+	if s.ResourceLimits.BlockIO == nil {
+		s.ResourceLimits.BlockIO = &spec.LinuxBlockIO{}
+	}
+	if bps := s.ThrottleReadBpsDevice; len(bps) > 0 {
+		for k, v := range bps {
+			statT := unix.Stat_t{}
+			if err := unix.Stat(k, &statT); err != nil {
+				return fmt.Errorf("could not parse throttle device at %s: %w", k, err)
+			}
+			v.Major = (int64(unix.Major(uint64(statT.Rdev)))) //nolint: unconvert
+			v.Minor = (int64(unix.Minor(uint64(statT.Rdev)))) //nolint: unconvert
+			if s.ResourceLimits.BlockIO == nil {
+				s.ResourceLimits.BlockIO = new(spec.LinuxBlockIO)
+			}
+			s.ResourceLimits.BlockIO.ThrottleReadBpsDevice = append(s.ResourceLimits.BlockIO.ThrottleReadBpsDevice, v)
+		}
+	}
+	if bps := s.ThrottleWriteBpsDevice; len(bps) > 0 {
+		for k, v := range bps {
+			statT := unix.Stat_t{}
+			if err := unix.Stat(k, &statT); err != nil {
+				return fmt.Errorf("could not parse throttle device at %s: %w", k, err)
+			}
+			v.Major = (int64(unix.Major(uint64(statT.Rdev)))) //nolint: unconvert
+			v.Minor = (int64(unix.Minor(uint64(statT.Rdev)))) //nolint: unconvert
+			s.ResourceLimits.BlockIO.ThrottleWriteBpsDevice = append(s.ResourceLimits.BlockIO.ThrottleWriteBpsDevice, v)
+		}
+	}
+	if iops := s.ThrottleReadIOPSDevice; len(iops) > 0 {
+		for k, v := range iops {
+			statT := unix.Stat_t{}
+			if err := unix.Stat(k, &statT); err != nil {
+				return fmt.Errorf("could not parse throttle device at %s: %w", k, err)
+			}
+			v.Major = (int64(unix.Major(uint64(statT.Rdev)))) //nolint: unconvert
+			v.Minor = (int64(unix.Minor(uint64(statT.Rdev)))) //nolint: unconvert
+			s.ResourceLimits.BlockIO.ThrottleReadIOPSDevice = append(s.ResourceLimits.BlockIO.ThrottleReadIOPSDevice, v)
+		}
+	}
+	if iops := s.ThrottleWriteIOPSDevice; len(iops) > 0 {
+		for k, v := range iops {
+			statT := unix.Stat_t{}
+			if err := unix.Stat(k, &statT); err != nil {
+				return fmt.Errorf("could not parse throttle device at %s: %w", k, err)
+			}
+			v.Major = (int64(unix.Major(uint64(statT.Rdev)))) //nolint: unconvert
+			v.Minor = (int64(unix.Minor(uint64(statT.Rdev)))) //nolint: unconvert
+			s.ResourceLimits.BlockIO.ThrottleWriteIOPSDevice = append(s.ResourceLimits.BlockIO.ThrottleWriteIOPSDevice, v)
+		}
+	}
+	return nil
+}
diff --git a/pkg/specgen/generate/container_create.go b/pkg/specgen/generate/container_create.go
index 51d290bb4..389900820 100644
--- a/pkg/specgen/generate/container_create.go
+++ b/pkg/specgen/generate/container_create.go
@@ -55,9 +55,6 @@ func MakeContainer(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGener
 		}
 	}
 
-	if err := FinishThrottleDevices(s); err != nil {
-		return nil, nil, nil, err
-	}
 	// Set defaults for unset namespaces
 	if s.PidNS.IsDefault() {
 		defaultNS, err := GetDefaultNamespaceMode("pid", rtc, pod)
diff --git a/pkg/specgen/generate/oci.go b/pkg/specgen/generate/oci.go
index bb5f2d0ec..f59fe1011 100644
--- a/pkg/specgen/generate/oci.go
+++ b/pkg/specgen/generate/oci.go
@@ -309,6 +309,17 @@ func SpecGenToOCI(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Runt
 		}
 		g.Config.Linux.Resources = s.ResourceLimits
 	}
+
+	weightDevices, err := WeightDevices(s.WeightDevice)
+	if err != nil {
+		return nil, err
+	}
+	if len(weightDevices) > 0 {
+		for _, dev := range weightDevices {
+			g.AddLinuxResourcesBlockIOWeightDevice(dev.Major, dev.Minor, *dev.Weight)
+		}
+	}
+
 	// Devices
 	// set the default rule at the beginning of device configuration
 	if !inUserNS && !s.Privileged {
@@ -345,14 +356,6 @@ func SpecGenToOCI(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Runt
 		}
 	}
 
-	for k, v := range s.WeightDevice {
-		statT := unix.Stat_t{}
-		if err := unix.Stat(k, &statT); err != nil {
-			return nil, fmt.Errorf("failed to inspect '%s' in --blkio-weight-device: %w", k, err)
-		}
-		g.AddLinuxResourcesBlockIOWeightDevice((int64(unix.Major(uint64(statT.Rdev)))), (int64(unix.Minor(uint64(statT.Rdev)))), *v.Weight) //nolint: unconvert
-	}
-
 	BlockAccessToKernelFilesystems(s.Privileged, s.PidNS.IsHost(), s.Mask, s.Unmask, &g)
 
 	g.ClearProcessEnv()
@@ -413,3 +416,19 @@ func SpecGenToOCI(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Runt
 
 	return configSpec, nil
 }
+
+func WeightDevices(wtDevices map[string]spec.LinuxWeightDevice) ([]spec.LinuxWeightDevice, error) {
+	devs := []spec.LinuxWeightDevice{}
+	for k, v := range wtDevices {
+		statT := unix.Stat_t{}
+		if err := unix.Stat(k, &statT); err != nil {
+			return nil, fmt.Errorf("failed to inspect '%s' in --blkio-weight-device: %w", k, err)
+		}
+		dev := new(spec.LinuxWeightDevice)
+		dev.Major = (int64(unix.Major(uint64(statT.Rdev)))) //nolint: unconvert
+		dev.Minor = (int64(unix.Minor(uint64(statT.Rdev)))) //nolint: unconvert
+		dev.Weight = v.Weight
+		devs = append(devs, *dev)
+	}
+	return devs, nil
+}
diff --git a/pkg/specgen/generate/pod_create.go b/pkg/specgen/generate/pod_create.go
index 212d613fe..4e6362c9b 100644
--- a/pkg/specgen/generate/pod_create.go
+++ b/pkg/specgen/generate/pod_create.go
@@ -13,6 +13,7 @@ import (
 	"github.com/containers/podman/v4/pkg/domain/entities"
 	"github.com/containers/podman/v4/pkg/specgen"
 	"github.com/containers/podman/v4/pkg/specgenutil"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/sirupsen/logrus"
 )
 
@@ -21,6 +22,10 @@ func MakePod(p *entities.PodSpec, rt *libpod.Runtime) (*libpod.Pod, error) {
 		return nil, err
 	}
 
+	if p.PodSpecGen.ResourceLimits == nil {
+		p.PodSpecGen.ResourceLimits = &specs.LinuxResources{}
+	}
+
 	if !p.PodSpecGen.NoInfra {
 		imageName, err := PullOrBuildInfraImage(rt, p.PodSpecGen.InfraImage)
 		if err != nil {
@@ -38,10 +43,33 @@ func MakePod(p *entities.PodSpec, rt *libpod.Runtime) (*libpod.Pod, error) {
 		}
 	}
 
+	if !p.PodSpecGen.NoInfra {
+		err := FinishThrottleDevices(p.PodSpecGen.InfraContainerSpec)
+		if err != nil {
+			return nil, err
+		}
+		if p.PodSpecGen.InfraContainerSpec.ResourceLimits.BlockIO != nil {
+			p.PodSpecGen.ResourceLimits.BlockIO = p.PodSpecGen.InfraContainerSpec.ResourceLimits.BlockIO
+		}
+
+		weightDevices, err := WeightDevices(p.PodSpecGen.InfraContainerSpec.WeightDevice)
+		if err != nil {
+			return nil, err
+		}
+
+		if p.PodSpecGen.ResourceLimits != nil && len(weightDevices) > 0 {
+			if p.PodSpecGen.ResourceLimits.BlockIO == nil {
+				p.PodSpecGen.ResourceLimits.BlockIO = &specs.LinuxBlockIO{}
+			}
+			p.PodSpecGen.ResourceLimits.BlockIO.WeightDevice = weightDevices
+		}
+	}
+
 	options, err := createPodOptions(&p.PodSpecGen)
 	if err != nil {
 		return nil, err
 	}
+
 	pod, err := rt.NewPod(context.Background(), p.PodSpecGen, options...)
 	if err != nil {
 		return nil, err
@@ -55,6 +83,11 @@ func MakePod(p *entities.PodSpec, rt *libpod.Runtime) (*libpod.Pod, error) {
 			return nil, err
 		}
 		p.PodSpecGen.InfraContainerSpec.User = "" // infraSpec user will get incorrectly assigned via the container creation process, overwrite here
+		// infra's resource limits are used as a parsing tool,
+		// we do not want infra to get these resources in its cgroup
+		// make sure of that here.
+		p.PodSpecGen.InfraContainerSpec.ResourceLimits = nil
+		p.PodSpecGen.InfraContainerSpec.WeightDevice = nil
 		rtSpec, spec, opts, err := MakeContainer(context.Background(), rt, p.PodSpecGen.InfraContainerSpec, false, nil)
 		if err != nil {
 			return nil, err
@@ -122,6 +155,10 @@ func createPodOptions(p *specgen.PodSpecGenerator) ([]libpod.PodCreateOption, er
 		options = append(options, libpod.WithPodHostname(p.Hostname))
 	}
 
+	if p.ResourceLimits != nil {
+		options = append(options, libpod.WithPodResources(*p.ResourceLimits))
+	}
+
 	options = append(options, libpod.WithPodExitPolicy(p.ExitPolicy))
 
 	return options, nil
diff --git a/pkg/specgenutil/specgen.go b/pkg/specgenutil/specgen.go
index 9a7d50947..4ab019b5b 100644
--- a/pkg/specgenutil/specgen.go
+++ b/pkg/specgenutil/specgen.go
@@ -74,6 +74,12 @@ func getCPULimits(c *entities.ContainerCreateOptions) *specs.LinuxCPU {
 func getIOLimits(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions) (*specs.LinuxBlockIO, error) {
 	var err error
 	io := &specs.LinuxBlockIO{}
+	if s.ResourceLimits == nil {
+		s.ResourceLimits = &specs.LinuxResources{}
+	}
+	if s.ResourceLimits.BlockIO == nil {
+		s.ResourceLimits.BlockIO = &specs.LinuxBlockIO{}
+	}
 	hasLimits := false
 	if b := c.BlkIOWeight; len(b) > 0 {
 		u, err := strconv.ParseUint(b, 10, 16)
@@ -82,6 +88,7 @@ func getIOLimits(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions) (
 		}
 		nu := uint16(u)
 		io.Weight = &nu
+		s.ResourceLimits.BlockIO.Weight = &nu
 		hasLimits = true
 	}
 
@@ -96,6 +103,7 @@ func getIOLimits(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions) (
 		if s.ThrottleReadBpsDevice, err = parseThrottleBPSDevices(bps); err != nil {
 			return nil, err
 		}
+
 		hasLimits = true
 	}
 
@@ -123,6 +131,8 @@ func getIOLimits(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions) (
 	if !hasLimits {
 		return nil, nil
 	}
+	io = s.ResourceLimits.BlockIO
+
 	return io, nil
 }
 
@@ -509,7 +519,7 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions
 			return err
 		}
 	}
-	if s.ResourceLimits.BlockIO == nil || (len(c.BlkIOWeight) != 0 || len(c.BlkIOWeightDevice) != 0) {
+	if s.ResourceLimits.BlockIO == nil || (len(c.BlkIOWeight) != 0 || len(c.BlkIOWeightDevice) != 0 || len(c.DeviceReadBPs) != 0 || len(c.DeviceWriteBPs) != 0) {
 		s.ResourceLimits.BlockIO, err = getIOLimits(s, c)
 		if err != nil {
 			return err