resource limits for pods

added the following flags and handling for podman pod create

--memory-swap
--cpuset-mems
--device-read-bps
--device-write-bps
--blkio-weight
--blkio-weight-device
--cpu-shares

given the new backend for systemd in c/common, all of these can now be exposed to pod create.
most of the heavy lifting (nearly all) is done within c/common. However, some rewiring needed to be done here
as well!

Signed-off-by: Charlie Doern <cdoern@redhat.com>

5 files changed, 128 insertions, 80 deletions

diff --git a/libpod/define/pod_inspect.go b/libpod/define/pod_inspect.go
index 2afef48c4..d56074882 100644
--- a/libpod/define/pod_inspect.go
+++ b/libpod/define/pod_inspect.go
@@ -57,20 +57,32 @@ type InspectPodData struct {
 	CPUPeriod uint64 `json:"cpu_period,omitempty"`
 	// CPUQuota contains the CPU quota of the pod
 	CPUQuota int64 `json:"cpu_quota,omitempty"`
+	// CPUShares contains the cpu shares for the pod
+	CPUShares uint64 `json:"cpu_shares,omitempty"`
 	// CPUSetCPUs contains linux specific CPU data for the pod
 	CPUSetCPUs string `json:"cpuset_cpus,omitempty"`
+	// CPUSetMems contains linux specific CPU data for the pod
+	CPUSetMems string `json:"cpuset_mems,omitempty"`
 	// Mounts contains volume related information for the pod
 	Mounts []InspectMount `json:"mounts,omitempty"`
 	// Devices contains the specified host devices
 	Devices []InspectDevice `json:"devices,omitempty"`
 	// BlkioDeviceReadBps contains the Read/Access limit for the pod's devices
 	BlkioDeviceReadBps []InspectBlkioThrottleDevice `json:"device_read_bps,omitempty"`
+	// BlkioDeviceReadBps contains the Read/Access limit for the pod's devices
+	BlkioDeviceWriteBps []InspectBlkioThrottleDevice `json:"device_write_bps,omitempty"`
 	// VolumesFrom contains the containers that the pod inherits mounts from
 	VolumesFrom []string `json:"volumes_from,omitempty"`
 	// SecurityOpt contains the specified security labels and related SELinux information
 	SecurityOpts []string `json:"security_opt,omitempty"`
 	// MemoryLimit contains the specified cgroup memory limit for the pod
 	MemoryLimit uint64 `json:"memory_limit,omitempty"`
+	// MemorySwap contains the specified memory swap limit for the pod
+	MemorySwap uint64 `json:"memory_swap,omitempty"`
+	// BlkioWeight contains the blkio weight limit for the pod
+	BlkioWeight uint64 `json:"blkio_weight,omitempty"`
+	// BlkioWeightDevice contains the blkio weight device limits for the pod
+	BlkioWeightDevice []InspectBlkioWeightDevice `json:"blkio_weight_device,omitempty"`
 }
 
 // InspectPodInfraConfig contains the configuration of the pod's infra
diff --git a/libpod/options.go b/libpod/options.go
index f03980017..b31cb4ab2 100644
--- a/libpod/options.go
+++ b/libpod/options.go
@@ -2145,6 +2145,18 @@ func WithServiceContainer(id string) PodCreateOption {
 	}
 }
 
+// WithPodResources sets resource limits to be applied to the pod's cgroup
+// these will be inherited by all containers unless overridden.
+func WithPodResources(resources specs.LinuxResources) PodCreateOption {
+	return func(pod *Pod) error {
+		if pod.valid {
+			return define.ErrPodFinalized
+		}
+		pod.config.ResourceLimits = resources
+		return nil
+	}
+}
+
 // WithVolatile sets the volatile flag for the container storage.
 // The option can potentially cause data loss when used on a container that must survive a machine reboot.
 func WithVolatile() CtrCreateOption {
diff --git a/libpod/pod.go b/libpod/pod.go
index e059c9416..89f6d6cfe 100644
--- a/libpod/pod.go
+++ b/libpod/pod.go
@@ -83,6 +83,9 @@ type PodConfig struct {
 
 	// ID of the pod's lock
 	LockID uint32 `json:"lockID"`
+
+	// ResourceLimits hold the pod level resource limits
+	ResourceLimits specs.LinuxResources
 }
 
 // podState represents a pod's state
@@ -116,18 +119,7 @@ func (p *Pod) ResourceLim() *specs.LinuxResources {
 	empty := &specs.LinuxResources{
 		CPU: &specs.LinuxCPU{},
 	}
-	infra, err := p.runtime.GetContainer(p.state.InfraContainerID)
-	if err != nil {
-		return empty
-	}
-	conf := infra.config.Spec
-	if err != nil {
-		return empty
-	}
-	if conf.Linux == nil || conf.Linux.Resources == nil {
-		return empty
-	}
-	if err = JSONDeepCopy(conf.Linux.Resources, resCopy); err != nil {
+	if err := JSONDeepCopy(p.config.ResourceLimits, resCopy); err != nil {
 		return nil
 	}
 	if resCopy.CPU != nil {
@@ -139,51 +131,91 @@ func (p *Pod) ResourceLim() *specs.LinuxResources {
 
 // CPUPeriod returns the pod CPU period
 func (p *Pod) CPUPeriod() uint64 {
-	if p.state.InfraContainerID == "" {
+	resLim := p.ResourceLim()
+	if resLim.CPU == nil || resLim.CPU.Period == nil {
 		return 0
 	}
-	infra, err := p.runtime.GetContainer(p.state.InfraContainerID)
-	if err != nil {
+	return *resLim.CPU.Period
+}
+
+// CPUQuota returns the pod CPU quota
+func (p *Pod) CPUQuota() int64 {
+	resLim := p.ResourceLim()
+	if resLim.CPU == nil || resLim.CPU.Quota == nil {
 		return 0
 	}
-	conf := infra.config.Spec
-	if conf != nil && conf.Linux != nil && conf.Linux.Resources != nil && conf.Linux.Resources.CPU != nil && conf.Linux.Resources.CPU.Period != nil {
-		return *conf.Linux.Resources.CPU.Period
+	return *resLim.CPU.Quota
+}
+
+// MemoryLimit returns the pod Memory Limit
+func (p *Pod) MemoryLimit() uint64 {
+	resLim := p.ResourceLim()
+	if resLim.Memory == nil || resLim.Memory.Limit == nil {
+		return 0
 	}
-	return 0
+	return uint64(*resLim.Memory.Limit)
 }
 
-// CPUQuota returns the pod CPU quota
-func (p *Pod) CPUQuota() int64 {
-	if p.state.InfraContainerID == "" {
+// MemorySwap returns the pod Memory swap limit
+func (p *Pod) MemorySwap() uint64 {
+	resLim := p.ResourceLim()
+	if resLim.Memory == nil || resLim.Memory.Swap == nil {
 		return 0
 	}
-	infra, err := p.runtime.GetContainer(p.state.InfraContainerID)
-	if err != nil {
+	return uint64(*resLim.Memory.Swap)
+}
+
+// BlkioWeight returns the pod blkio weight
+func (p *Pod) BlkioWeight() uint64 {
+	resLim := p.ResourceLim()
+	if resLim.BlockIO == nil || resLim.BlockIO.Weight == nil {
 		return 0
 	}
-	conf := infra.config.Spec
-	if conf != nil && conf.Linux != nil && conf.Linux.Resources != nil && conf.Linux.Resources.CPU != nil && conf.Linux.Resources.CPU.Quota != nil {
-		return *conf.Linux.Resources.CPU.Quota
+	return uint64(*resLim.BlockIO.Weight)
+}
+
+// CPUSetMems returns the pod CPUSet memory nodes
+func (p *Pod) CPUSetMems() string {
+	resLim := p.ResourceLim()
+	if resLim.CPU == nil {
+		return ""
 	}
-	return 0
+	return resLim.CPU.Mems
 }
 
-// MemoryLimit returns the pod Memory Limit
-func (p *Pod) MemoryLimit() uint64 {
-	if p.state.InfraContainerID == "" {
+// CPUShares returns the pod cpu shares
+func (p *Pod) CPUShares() uint64 {
+	resLim := p.ResourceLim()
+	if resLim.CPU == nil || resLim.CPU.Shares == nil {
 		return 0
 	}
-	infra, err := p.runtime.GetContainer(p.state.InfraContainerID)
+	return *resLim.CPU.Shares
+}
+
+// BlkiThrottleReadBps returns the pod  throttle devices
+func (p *Pod) BlkiThrottleReadBps() []define.InspectBlkioThrottleDevice {
+	resLim := p.ResourceLim()
+	if resLim.BlockIO == nil || resLim.BlockIO.ThrottleReadBpsDevice == nil {
+		return []define.InspectBlkioThrottleDevice{}
+	}
+	devs, err := blkioDeviceThrottle(nil, resLim.BlockIO.ThrottleReadBpsDevice)
 	if err != nil {
-		return 0
+		return []define.InspectBlkioThrottleDevice{}
 	}
-	conf := infra.config.Spec
-	if conf != nil && conf.Linux != nil && conf.Linux.Resources != nil && conf.Linux.Resources.Memory != nil && conf.Linux.Resources.Memory.Limit != nil {
-		val := *conf.Linux.Resources.Memory.Limit
-		return uint64(val)
+	return devs
+}
+
+// BlkiThrottleWriteBps returns the pod  throttle devices
+func (p *Pod) BlkiThrottleWriteBps() []define.InspectBlkioThrottleDevice {
+	resLim := p.ResourceLim()
+	if resLim.BlockIO == nil || resLim.BlockIO.ThrottleWriteBpsDevice == nil {
+		return []define.InspectBlkioThrottleDevice{}
+	}
+	devs, err := blkioDeviceThrottle(nil, resLim.BlockIO.ThrottleWriteBpsDevice)
+	if err != nil {
+		return []define.InspectBlkioThrottleDevice{}
 	}
-	return 0
+	return devs
 }
 
 // NetworkMode returns the Network mode given by the user ex: pod, private...
diff --git a/libpod/pod_api.go b/libpod/pod_api.go
index c1d54d55e..29964ae95 100644
--- a/libpod/pod_api.go
+++ b/libpod/pod_api.go
@@ -659,7 +659,6 @@ func (p *Pod) Inspect() (*define.InspectPodData, error) {
 	var infraConfig *define.InspectPodInfraConfig
 	var inspectMounts []define.InspectMount
 	var devices []define.InspectDevice
-	var deviceLimits []define.InspectBlkioThrottleDevice
 	var infraSecurity []string
 	if p.state.InfraContainerID != "" {
 		infra, err := p.runtime.GetContainer(p.state.InfraContainerID)
@@ -683,18 +682,6 @@ func (p *Pod) Inspect() (*define.InspectPodData, error) {
 		if err != nil {
 			return nil, err
 		}
-		var nodes map[string]string
-		devices, err = infra.GetDevices(false, *infra.config.Spec, nodes)
-		if err != nil {
-			return nil, err
-		}
-		spec := infra.config.Spec
-		if spec.Linux != nil && spec.Linux.Resources != nil && spec.Linux.Resources.BlockIO != nil {
-			deviceLimits, err = blkioDeviceThrottle(nodes, spec.Linux.Resources.BlockIO.ThrottleReadBpsDevice)
-			if err != nil {
-				return nil, err
-			}
-		}
 
 		if len(infra.config.ContainerNetworkConfig.DNSServer) > 0 {
 			infraConfig.DNSServer = make([]string, 0, len(infra.config.ContainerNetworkConfig.DNSServer))
@@ -731,33 +718,38 @@ func (p *Pod) Inspect() (*define.InspectPodData, error) {
 	}
 
 	inspectData := define.InspectPodData{
-		ID:                 p.ID(),
-		Name:               p.Name(),
-		Namespace:          p.Namespace(),
-		Created:            p.CreatedTime(),
-		CreateCommand:      p.config.CreateCommand,
-		ExitPolicy:         string(p.config.ExitPolicy),
-		State:              podState,
-		Hostname:           p.config.Hostname,
-		Labels:             p.Labels(),
-		CreateCgroup:       p.config.UsePodCgroup,
-		CgroupParent:       p.CgroupParent(),
-		CgroupPath:         p.state.CgroupPath,
-		CreateInfra:        infraConfig != nil,
-		InfraContainerID:   p.state.InfraContainerID,
-		InfraConfig:        infraConfig,
-		SharedNamespaces:   sharesNS,
-		NumContainers:      uint(len(containers)),
-		Containers:         ctrs,
-		CPUSetCPUs:         p.ResourceLim().CPU.Cpus,
-		CPUPeriod:          p.CPUPeriod(),
-		CPUQuota:           p.CPUQuota(),
-		MemoryLimit:        p.MemoryLimit(),
-		Mounts:             inspectMounts,
-		Devices:            devices,
-		BlkioDeviceReadBps: deviceLimits,
-		VolumesFrom:        p.VolumesFrom(),
-		SecurityOpts:       infraSecurity,
+		ID:                  p.ID(),
+		Name:                p.Name(),
+		Namespace:           p.Namespace(),
+		Created:             p.CreatedTime(),
+		CreateCommand:       p.config.CreateCommand,
+		ExitPolicy:          string(p.config.ExitPolicy),
+		State:               podState,
+		Hostname:            p.config.Hostname,
+		Labels:              p.Labels(),
+		CreateCgroup:        p.config.UsePodCgroup,
+		CgroupParent:        p.CgroupParent(),
+		CgroupPath:          p.state.CgroupPath,
+		CreateInfra:         infraConfig != nil,
+		InfraContainerID:    p.state.InfraContainerID,
+		InfraConfig:         infraConfig,
+		SharedNamespaces:    sharesNS,
+		NumContainers:       uint(len(containers)),
+		Containers:          ctrs,
+		CPUSetCPUs:          p.ResourceLim().CPU.Cpus,
+		CPUPeriod:           p.CPUPeriod(),
+		CPUQuota:            p.CPUQuota(),
+		MemoryLimit:         p.MemoryLimit(),
+		Mounts:              inspectMounts,
+		Devices:             devices,
+		BlkioDeviceReadBps:  p.BlkiThrottleReadBps(),
+		VolumesFrom:         p.VolumesFrom(),
+		SecurityOpts:        infraSecurity,
+		MemorySwap:          p.MemorySwap(),
+		BlkioWeight:         p.BlkioWeight(),
+		CPUSetMems:          p.CPUSetMems(),
+		BlkioDeviceWriteBps: p.BlkiThrottleWriteBps(),
+		CPUShares:           p.CPUShares(),
 	}
 
 	return &inspectData, nil
diff --git a/libpod/runtime_pod_linux.go b/libpod/runtime_pod_linux.go
index 75ff24e41..3bb22ec26 100644
--- a/libpod/runtime_pod_linux.go
+++ b/libpod/runtime_pod_linux.go
@@ -80,7 +80,7 @@ func (r *Runtime) NewPod(ctx context.Context, p specgen.PodSpecGenerator, option
 						p.InfraContainerSpec.CgroupParent = pod.state.CgroupPath
 						// cgroupfs + rootless = permission denied when creating the cgroup.
 						if !rootless.IsRootless() {
-							res, err := GetLimits(p.InfraContainerSpec.ResourceLimits)
+							res, err := GetLimits(p.ResourceLimits)
 							if err != nil {
 								return nil, err
 							}
@@ -113,7 +113,7 @@ func (r *Runtime) NewPod(ctx context.Context, p specgen.PodSpecGenerator, option
 			// If we are set to use pod cgroups, set the cgroup parent that
 			// all containers in the pod will share
 			if pod.config.UsePodCgroup {
-				cgroupPath, err := systemdSliceFromPath(pod.config.CgroupParent, fmt.Sprintf("libpod_pod_%s", pod.ID()), p.InfraContainerSpec.ResourceLimits)
+				cgroupPath, err := systemdSliceFromPath(pod.config.CgroupParent, fmt.Sprintf("libpod_pod_%s", pod.ID()), p.ResourceLimits)
 				if err != nil {
 					return nil, fmt.Errorf("unable to create pod cgroup for pod %s: %w", pod.ID(), err)
 				}