Pod Device Support

added support for pod devices. The device gets added to the infra container and
recreated in all containers that join the pod.

This required a new container config item to keep track of the original device passed in by the user before
the path was parsed into the container device.

Signed-off-by: cdoern <cdoern@redhat.com>

16 files changed, 138 insertions, 42 deletions

diff --git a/cmd/podman/common/create.go b/cmd/podman/common/create.go
index a65e90fab..a969e17e9 100644
--- a/cmd/podman/common/create.go
+++ b/cmd/podman/common/create.go
@@ -156,14 +156,6 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions,
 		)
 		_ = cmd.RegisterFlagCompletionFunc(cpusetMemsFlagName, completion.AutocompleteNone)
 
-		deviceFlagName := "device"
-		createFlags.StringSliceVar(
-			&cf.Devices,
-			deviceFlagName, devices(),
-			"Add a host device to the container",
-		)
-		_ = cmd.RegisterFlagCompletionFunc(deviceFlagName, completion.AutocompleteDefault)
-
 		deviceCgroupRuleFlagName := "device-cgroup-rule"
 		createFlags.StringSliceVar(
 			&cf.DeviceCGroupRule,
@@ -877,4 +869,11 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions,
 		volumeDesciption,
 	)
 	_ = cmd.RegisterFlagCompletionFunc(volumeFlagName, AutocompleteVolumeFlag)
+	deviceFlagName := "device"
+	createFlags.StringSliceVar(
+		&cf.Devices,
+		deviceFlagName, devices(),
+		"Add a host device to the container",
+	)
+	_ = cmd.RegisterFlagCompletionFunc(deviceFlagName, completion.AutocompleteDefault)
 }
diff --git a/docs/source/markdown/podman-pod-create.1.md b/docs/source/markdown/podman-pod-create.1.md
index b4c3fc2eb..a1540672c 100644
--- a/docs/source/markdown/podman-pod-create.1.md
+++ b/docs/source/markdown/podman-pod-create.1.md
@@ -38,6 +38,22 @@ Examples of the List Format:
 0-4,9           # bits 0, 1, 2, 3, 4, and 9 set
 0-2,7,12-14     # bits 0, 1, 2, 7, 12, 13, and 14 set
 
+#### **--device**=_host-device_[**:**_container-device_][**:**_permissions_]
+
+Add a host device to the pod. Optional *permissions* parameter
+can be used to specify device permissions It is a combination of
+**r** for read, **w** for write, and **m** for **mknod**(2).
+
+Example: **--device=/dev/sdc:/dev/xvdc:rwm**.
+
+Note: if _host_device_ is a symbolic link then it will be resolved first.
+The pod will only store the major and minor numbers of the host device.
+
+Note: the pod implements devices by storing the initial configuration passed by the user and recreating the device on each container added to the pod.
+
+Podman may load kernel modules required for using the specified
+device. The devices that Podman will load modules for when necessary are:
+/dev/fuse.
 
 #### **--dns**=*ipaddr*
 
diff --git a/libpod/container.go b/libpod/container.go
index cf727926c..7d602326e 100644
--- a/libpod/container.go
+++ b/libpod/container.go
@@ -278,6 +278,11 @@ func (c *Container) Config() *ContainerConfig {
 	return returnConfig
 }
 
+// DeviceHostSrc returns the user supplied device to be passed down in the pod
+func (c *Container) DeviceHostSrc() []spec.LinuxDevice {
+	return c.config.DeviceHostSrc
+}
+
 // Runtime returns the container's Runtime.
 func (c *Container) Runtime() *Runtime {
 	return c.runtime
diff --git a/libpod/container_config.go b/libpod/container_config.go
index 0374c25fe..54d102a71 100644
--- a/libpod/container_config.go
+++ b/libpod/container_config.go
@@ -381,6 +381,8 @@ type ContainerMiscConfig struct {
 	PidFile string `json:"pid_file,omitempty"`
 	// CDIDevices contains devices that use the CDI
 	CDIDevices []string `json:"cdiDevices,omitempty"`
+	// DeviceHostSrc contains the original source on the host
+	DeviceHostSrc []spec.LinuxDevice `json:"device_host_src,omitempty"`
 	// EnvSecrets are secrets that are set as environment variables
 	EnvSecrets map[string]*secrets.Secret `json:"secret_env,omitempty"`
 	// InitContainerType specifies if the container is an initcontainer
diff --git a/libpod/container_inspect.go b/libpod/container_inspect.go
index 530160b2d..e65c86cef 100644
--- a/libpod/container_inspect.go
+++ b/libpod/container_inspect.go
@@ -819,27 +819,10 @@ func (c *Container) generateInspectContainerHostConfig(ctrSpec *spec.Spec, named
 	// Devices
 	// Do not include if privileged - assumed that all devices will be
 	// included.
-	hostConfig.Devices = []define.InspectDevice{}
-	if ctrSpec.Linux != nil && !hostConfig.Privileged {
-		for _, dev := range ctrSpec.Linux.Devices {
-			key := fmt.Sprintf("%d:%d", dev.Major, dev.Minor)
-			if deviceNodes == nil {
-				nodes, err := util.FindDeviceNodes()
-				if err != nil {
-					return nil, err
-				}
-				deviceNodes = nodes
-			}
-			path, ok := deviceNodes[key]
-			if !ok {
-				logrus.Warnf("Could not locate device %s on host", key)
-				continue
-			}
-			newDev := define.InspectDevice{}
-			newDev.PathOnHost = path
-			newDev.PathInContainer = dev.Path
-			hostConfig.Devices = append(hostConfig.Devices, newDev)
-		}
+	var err error
+	hostConfig.Devices, err = c.GetDevices(*&hostConfig.Privileged, *ctrSpec, deviceNodes)
+	if err != nil {
+		return nil, err
 	}
 
 	// Ulimits
@@ -885,3 +868,29 @@ func (c *Container) inHostPidNS() (bool, error) {
 	}
 	return true, nil
 }
+
+func (c *Container) GetDevices(priv bool, ctrSpec spec.Spec, deviceNodes map[string]string) ([]define.InspectDevice, error) {
+	devices := []define.InspectDevice{}
+	if ctrSpec.Linux != nil && !priv {
+		for _, dev := range ctrSpec.Linux.Devices {
+			key := fmt.Sprintf("%d:%d", dev.Major, dev.Minor)
+			if deviceNodes == nil {
+				nodes, err := util.FindDeviceNodes()
+				if err != nil {
+					return nil, err
+				}
+				deviceNodes = nodes
+			}
+			path, ok := deviceNodes[key]
+			if !ok {
+				logrus.Warnf("Could not locate device %s on host", key)
+				continue
+			}
+			newDev := define.InspectDevice{}
+			newDev.PathOnHost = path
+			newDev.PathInContainer = dev.Path
+			devices = append(devices, newDev)
+		}
+	}
+	return devices, nil
+}
diff --git a/libpod/define/pod_inspect.go b/libpod/define/pod_inspect.go
index b7a6e76b5..e78d97850 100644
--- a/libpod/define/pod_inspect.go
+++ b/libpod/define/pod_inspect.go
@@ -59,6 +59,8 @@ type InspectPodData struct {
 	CPUSetCPUs string `json:"cpuset_cpus,omitempty"`
 	// Mounts contains volume related information for the pod
 	Mounts []InspectMount `json:"mounts,omitempty"`
+	// Devices contains the specified host devices
+	Devices []InspectDevice `json:"devices,omitempty"`
 }
 
 // InspectPodInfraConfig contains the configuration of the pod's infra
diff --git a/libpod/options.go b/libpod/options.go
index 3f6ccf1cb..a80f51c6a 100644
--- a/libpod/options.go
+++ b/libpod/options.go
@@ -21,6 +21,7 @@ import (
 	"github.com/containers/podman/v3/pkg/util"
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/idtools"
+	"github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -1809,6 +1810,17 @@ func WithInitCtrType(containerType string) CtrCreateOption {
 	}
 }
 
+// WithHostDevice adds the original host src to the config
+func WithHostDevice(dev []specs.LinuxDevice) CtrCreateOption {
+	return func(ctr *Container) error {
+		if ctr.valid {
+			return define.ErrCtrFinalized
+		}
+		ctr.config.DeviceHostSrc = dev
+		return nil
+	}
+}
+
 // Pod Creation Options
 
 // WithPodCreateCommand adds the full command plus arguments of the current
diff --git a/libpod/pod_api.go b/libpod/pod_api.go
index 4e0acf950..ff818edc2 100644
--- a/libpod/pod_api.go
+++ b/libpod/pod_api.go
@@ -583,6 +583,7 @@ func (p *Pod) Inspect() (*define.InspectPodData, error) {
 	// container.
 	var infraConfig *define.InspectPodInfraConfig
 	var inspectMounts []define.InspectMount
+	var devices []define.InspectDevice
 	if p.state.InfraContainerID != "" {
 		infra, err := p.runtime.GetContainer(p.state.InfraContainerID)
 		if err != nil {
@@ -604,6 +605,12 @@ func (p *Pod) Inspect() (*define.InspectPodData, error) {
 			return nil, err
 		}
 
+		var nodes map[string]string
+		devices, err = infra.GetDevices(false, *infra.config.Spec, nodes)
+		if err != nil {
+			return nil, err
+		}
+
 		if len(infra.Config().ContainerNetworkConfig.DNSServer) > 0 {
 			infraConfig.DNSServer = make([]string, 0, len(infra.Config().ContainerNetworkConfig.DNSServer))
 			for _, entry := range infra.Config().ContainerNetworkConfig.DNSServer {
@@ -652,6 +659,7 @@ func (p *Pod) Inspect() (*define.InspectPodData, error) {
 		CPUPeriod:        p.CPUPeriod(),
 		CPUQuota:         p.CPUQuota(),
 		Mounts:           inspectMounts,
+		Devices:          devices,
 	}
 
 	return &inspectData, nil
diff --git a/pkg/api/handlers/libpod/pods.go b/pkg/api/handlers/libpod/pods.go
index 1f03e121e..9f46ecc52 100644
--- a/pkg/api/handlers/libpod/pods.go
+++ b/pkg/api/handlers/libpod/pods.go
@@ -41,8 +41,8 @@ func PodCreate(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 	if !psg.NoInfra {
-		infraOptions := &entities.ContainerCreateOptions{ImageVolume: "bind", IsInfra: true, Net: &entities.NetOptions{}} // options for pulling the image and FillOutSpec
-		err = specgenutil.FillOutSpecGen(psg.InfraContainerSpec, infraOptions, []string{})                                // necessary for default values in many cases (userns, idmappings)
+		infraOptions := &entities.ContainerCreateOptions{ImageVolume: "bind", IsInfra: true, Net: &entities.NetOptions{}, Devices: psg.Devices} // options for pulling the image and FillOutSpec
+		err = specgenutil.FillOutSpecGen(psg.InfraContainerSpec, infraOptions, []string{})                                                      // necessary for default values in many cases (userns, idmappings)
 		if err != nil {
 			utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "error filling out specgen"))
 			return
diff --git a/pkg/domain/entities/pods.go b/pkg/domain/entities/pods.go
index 88bd3c6ce..f0c88d77e 100644
--- a/pkg/domain/entities/pods.go
+++ b/pkg/domain/entities/pods.go
@@ -118,6 +118,7 @@ type PodSpec struct {
 type PodCreateOptions struct {
 	CGroupParent       string            `json:"cgroup_parent,omitempty"`
 	CreateCommand      []string          `json:"create_command,omitempty"`
+	Devices            []string          `json:"devices,omitempty"`
 	Hostname           string            `json:"hostname,omitempty"`
 	Infra              bool              `json:"infra,omitempty"`
 	InfraImage         string            `json:"infra_image,omitempty"`
@@ -164,7 +165,7 @@ type ContainerCreateOptions struct {
 	CPUS              float64 `json:"cpus,omitempty"`
 	CPUSetCPUs        string  `json:"cpuset_cpus,omitempty"`
 	CPUSetMems        string
-	Devices           []string
+	Devices           []string `json:"devices,omitempty"`
 	DeviceCGroupRule  []string
 	DeviceReadBPs     []string
 	DeviceReadIOPs    []string
@@ -295,6 +296,7 @@ func ToPodSpecGen(s specgen.PodSpecGenerator, p *PodCreateOptions) (*specgen.Pod
 	s.Pid = out
 	s.Hostname = p.Hostname
 	s.Labels = p.Labels
+	s.Devices = p.Devices
 	s.NoInfra = !p.Infra
 	if p.InfraCommand != nil && len(*p.InfraCommand) > 0 {
 		s.InfraCommand = strings.Split(*p.InfraCommand, " ")
diff --git a/pkg/specgen/generate/config_linux.go b/pkg/specgen/generate/config_linux.go
index 6b9e9c4bf..2d1e2b288 100644
--- a/pkg/specgen/generate/config_linux.go
+++ b/pkg/specgen/generate/config_linux.go
@@ -132,7 +132,6 @@ func DevicesFromPath(g *generate.Generator, devicePath string) error {
 		}
 		return nil
 	}
-
 	return addDevice(g, strings.Join(append([]string{resolvedDevicePath}, devs[1:]...), ":"))
 }
 
diff --git a/pkg/specgen/generate/container_create.go b/pkg/specgen/generate/container_create.go
index 91230338e..fefa9b4a9 100644
--- a/pkg/specgen/generate/container_create.go
+++ b/pkg/specgen/generate/container_create.go
@@ -30,24 +30,27 @@ func MakeContainer(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGener
 
 	// If joining a pod, retrieve the pod for use, and its infra container
 	var pod *libpod.Pod
-	var cont *libpod.Container
-	var config *libpod.ContainerConfig
+	var infraConfig *libpod.ContainerConfig
 	if s.Pod != "" {
 		pod, err = rt.LookupPod(s.Pod)
 		if err != nil {
 			return nil, nil, nil, errors.Wrapf(err, "error retrieving pod %s", s.Pod)
 		}
 		if pod.HasInfraContainer() {
-			cont, err = pod.InfraContainer()
+			infra, err := pod.InfraContainer()
 			if err != nil {
 				return nil, nil, nil, err
 			}
-			config = cont.Config()
+			infraConfig = infra.Config()
 		}
 	}
 
-	if config != nil && (len(config.NamedVolumes) > 0 || len(config.UserVolumes) > 0 || len(config.ImageVolumes) > 0 || len(config.OverlayVolumes) > 0) {
-		s.VolumesFrom = append(s.VolumesFrom, config.ID)
+	if infraConfig != nil && (len(infraConfig.NamedVolumes) > 0 || len(infraConfig.UserVolumes) > 0 || len(infraConfig.ImageVolumes) > 0 || len(infraConfig.OverlayVolumes) > 0) {
+		s.VolumesFrom = append(s.VolumesFrom, infraConfig.ID)
+	}
+
+	if infraConfig != nil && len(infraConfig.Spec.Linux.Devices) > 0 {
+		s.DevicesFrom = append(s.DevicesFrom, infraConfig.ID)
 	}
 	// Set defaults for unset namespaces
 	if s.PidNS.IsDefault() {
@@ -166,6 +169,16 @@ func MakeContainer(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGener
 		logrus.Debugf("setting container name %s", s.Name)
 		options = append(options, libpod.WithName(s.Name))
 	}
+	if len(s.DevicesFrom) > 0 {
+		for _, dev := range s.DevicesFrom {
+			ctr, err := rt.GetContainer(dev)
+			if err != nil {
+				return nil, nil, nil, err
+			}
+			devices := ctr.DeviceHostSrc()
+			s.Devices = append(s.Devices, devices...)
+		}
+	}
 	if len(s.Devices) > 0 {
 		opts = extractCDIDevices(s)
 		options = append(options, opts...)
@@ -174,6 +187,9 @@ func MakeContainer(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGener
 	if err != nil {
 		return nil, nil, nil, err
 	}
+	if len(s.HostDeviceList) > 0 {
+		options = append(options, libpod.WithHostDevice(s.HostDeviceList))
+	}
 	return runtimeSpec, s, options, err
 }
 func ExecuteCreate(ctx context.Context, rt *libpod.Runtime, runtimeSpec *spec.Spec, s *specgen.SpecGenerator, infra bool, options ...libpod.CtrCreateOption) (*libpod.Container, error) {
diff --git a/pkg/specgen/generate/oci.go b/pkg/specgen/generate/oci.go
index 80c7f112f..55010f716 100644
--- a/pkg/specgen/generate/oci.go
+++ b/pkg/specgen/generate/oci.go
@@ -301,8 +301,8 @@ func SpecGenToOCI(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Runt
 	g.AddProcessEnv("container", "podman")
 
 	g.Config.Linux.Resources = s.ResourceLimits
-
 	// Devices
+
 	if s.Privileged {
 		// If privileged, we need to add all the host devices to the
 		// spec.  We do not add the user provided ones because we are
@@ -313,17 +313,18 @@ func SpecGenToOCI(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Runt
 	} else {
 		// add default devices from containers.conf
 		for _, device := range rtc.Containers.Devices {
-			if err := DevicesFromPath(&g, device); err != nil {
+			if err = DevicesFromPath(&g, device); err != nil {
 				return nil, err
 			}
 		}
 		// add default devices specified by caller
 		for _, device := range s.Devices {
-			if err := DevicesFromPath(&g, device.Path); err != nil {
+			if err = DevicesFromPath(&g, device.Path); err != nil {
 				return nil, err
 			}
 		}
 	}
+	s.HostDeviceList = s.Devices
 
 	for _, dev := range s.DeviceCGroupRule {
 		g.AddLinuxResourcesDevice(true, dev.Type, dev.Major, dev.Minor, dev.Access)
diff --git a/pkg/specgen/podspecgen.go b/pkg/specgen/podspecgen.go
index 5f72fc47d..83fa9426c 100644
--- a/pkg/specgen/podspecgen.go
+++ b/pkg/specgen/podspecgen.go
@@ -88,6 +88,8 @@ type PodBasicConfig struct {
 	// Image volumes bind-mount a container-image mount into the pod's infra container.
 	// Optional.
 	ImageVolumes []*ImageVolume `json:"image_volumes,omitempty"`
+	// Devices contains user specified Devices to be added to the Pod
+	Devices []string `json:"pod_devices,omitempty"`
 }
 
 // PodNetworkConfig contains networking configuration for a pod.
diff --git a/pkg/specgen/specgen.go b/pkg/specgen/specgen.go
index e0609c5bc..7aa27487a 100644
--- a/pkg/specgen/specgen.go
+++ b/pkg/specgen/specgen.go
@@ -254,6 +254,10 @@ type ContainerStorageConfig struct {
 	// DeviceCGroupRule are device cgroup rules that allow containers
 	// to use additional types of devices.
 	DeviceCGroupRule []spec.LinuxDeviceCgroup `json:"device_cgroup_rule,omitempty"`
+	// DevicesFrom is a way to ensure your container inherits device specific information from another container
+	DevicesFrom []string `json:"devices_from,omitempty"`
+	// HostDeviceList is used to recreate the mounted device on inherited containers
+	HostDeviceList []spec.LinuxDevice `json:"host_device_list,omitempty"`
 	// IpcNS is the container's IPC namespace.
 	// Default is private.
 	// Conflicts with ShmSize if not set to private.
diff --git a/test/e2e/pod_create_test.go b/test/e2e/pod_create_test.go
index 7d40d36dd..76a05fa0f 100644
--- a/test/e2e/pod_create_test.go
+++ b/test/e2e/pod_create_test.go
@@ -881,6 +881,25 @@ ENTRYPOINT ["sleep","99999"]
 		ctr3 := podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/tmp1/test"})
 		ctr3.WaitWithDefaultTimeout()
 		Expect(ctr3.OutputToString()).To(ContainSubstring("hello"))
+	})
+
+	It("podman pod create --device", func() {
+		SkipIfRootless("Cannot create devices in /dev in rootless mode")
+		Expect(os.MkdirAll("/dev/foodevdir", os.ModePerm)).To(BeNil())
+		defer os.RemoveAll("/dev/foodevdir")
+
+		mknod := SystemExec("mknod", []string{"/dev/foodevdir/null", "c", "1", "3"})
+		mknod.WaitWithDefaultTimeout()
+		Expect(mknod).Should(Exit(0))
+
+		podName := "testPod"
+		session := podmanTest.Podman([]string{"pod", "create", "--device", "/dev/foodevdir:/dev/bar", "--name", podName})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		session = podmanTest.Podman([]string{"run", "-q", "--pod", podName, ALPINE, "stat", "-c%t:%T", "/dev/bar/null"})
+		session.WaitWithDefaultTimeout()
+		Expect(session).Should(Exit(0))
+		Expect(session.OutputToString()).To(Equal("1:3"))
 
 	})