From 8fac34b8ff05314fe6996567af9336cf034b2d03 Mon Sep 17 00:00:00 2001 From: cdoern Date: Sun, 5 Sep 2021 23:22:17 -0400 Subject: Pod Device Support added support for pod devices. The device gets added to the infra container and recreated in all containers that join the pod. This required a new container config item to keep track of the original device passed in by the user before the path was parsed into the container device. Signed-off-by: cdoern --- cmd/podman/common/create.go | 15 ++++----- docs/source/markdown/podman-pod-create.1.md | 16 +++++++++ libpod/container.go | 5 +++ libpod/container_config.go | 2 ++ libpod/container_inspect.go | 51 +++++++++++++++++------------ libpod/define/pod_inspect.go | 2 ++ libpod/options.go | 12 +++++++ libpod/pod_api.go | 8 +++++ pkg/api/handlers/libpod/pods.go | 4 +-- pkg/domain/entities/pods.go | 4 ++- pkg/specgen/generate/config_linux.go | 1 - pkg/specgen/generate/container_create.go | 28 ++++++++++++---- pkg/specgen/generate/oci.go | 7 ++-- pkg/specgen/podspecgen.go | 2 ++ pkg/specgen/specgen.go | 4 +++ test/e2e/pod_create_test.go | 19 +++++++++++ 16 files changed, 138 insertions(+), 42 deletions(-) diff --git a/cmd/podman/common/create.go b/cmd/podman/common/create.go index a65e90fab..a969e17e9 100644 --- a/cmd/podman/common/create.go +++ b/cmd/podman/common/create.go @@ -156,14 +156,6 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions, ) _ = cmd.RegisterFlagCompletionFunc(cpusetMemsFlagName, completion.AutocompleteNone) - deviceFlagName := "device" - createFlags.StringSliceVar( - &cf.Devices, - deviceFlagName, devices(), - "Add a host device to the container", - ) - _ = cmd.RegisterFlagCompletionFunc(deviceFlagName, completion.AutocompleteDefault) - deviceCgroupRuleFlagName := "device-cgroup-rule" createFlags.StringSliceVar( &cf.DeviceCGroupRule, @@ -877,4 +869,11 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions, volumeDesciption, ) _ = cmd.RegisterFlagCompletionFunc(volumeFlagName, AutocompleteVolumeFlag) + deviceFlagName := "device" + createFlags.StringSliceVar( + &cf.Devices, + deviceFlagName, devices(), + "Add a host device to the container", + ) + _ = cmd.RegisterFlagCompletionFunc(deviceFlagName, completion.AutocompleteDefault) } diff --git a/docs/source/markdown/podman-pod-create.1.md b/docs/source/markdown/podman-pod-create.1.md index b4c3fc2eb..a1540672c 100644 --- a/docs/source/markdown/podman-pod-create.1.md +++ b/docs/source/markdown/podman-pod-create.1.md @@ -38,6 +38,22 @@ Examples of the List Format: 0-4,9 # bits 0, 1, 2, 3, 4, and 9 set 0-2,7,12-14 # bits 0, 1, 2, 7, 12, 13, and 14 set +#### **--device**=_host-device_[**:**_container-device_][**:**_permissions_] + +Add a host device to the pod. Optional *permissions* parameter +can be used to specify device permissions It is a combination of +**r** for read, **w** for write, and **m** for **mknod**(2). + +Example: **--device=/dev/sdc:/dev/xvdc:rwm**. + +Note: if _host_device_ is a symbolic link then it will be resolved first. +The pod will only store the major and minor numbers of the host device. + +Note: the pod implements devices by storing the initial configuration passed by the user and recreating the device on each container added to the pod. + +Podman may load kernel modules required for using the specified +device. The devices that Podman will load modules for when necessary are: +/dev/fuse. #### **--dns**=*ipaddr* diff --git a/libpod/container.go b/libpod/container.go index cf727926c..7d602326e 100644 --- a/libpod/container.go +++ b/libpod/container.go @@ -278,6 +278,11 @@ func (c *Container) Config() *ContainerConfig { return returnConfig } +// DeviceHostSrc returns the user supplied device to be passed down in the pod +func (c *Container) DeviceHostSrc() []spec.LinuxDevice { + return c.config.DeviceHostSrc +} + // Runtime returns the container's Runtime. func (c *Container) Runtime() *Runtime { return c.runtime diff --git a/libpod/container_config.go b/libpod/container_config.go index 0374c25fe..54d102a71 100644 --- a/libpod/container_config.go +++ b/libpod/container_config.go @@ -381,6 +381,8 @@ type ContainerMiscConfig struct { PidFile string `json:"pid_file,omitempty"` // CDIDevices contains devices that use the CDI CDIDevices []string `json:"cdiDevices,omitempty"` + // DeviceHostSrc contains the original source on the host + DeviceHostSrc []spec.LinuxDevice `json:"device_host_src,omitempty"` // EnvSecrets are secrets that are set as environment variables EnvSecrets map[string]*secrets.Secret `json:"secret_env,omitempty"` // InitContainerType specifies if the container is an initcontainer diff --git a/libpod/container_inspect.go b/libpod/container_inspect.go index 530160b2d..e65c86cef 100644 --- a/libpod/container_inspect.go +++ b/libpod/container_inspect.go @@ -819,27 +819,10 @@ func (c *Container) generateInspectContainerHostConfig(ctrSpec *spec.Spec, named // Devices // Do not include if privileged - assumed that all devices will be // included. - hostConfig.Devices = []define.InspectDevice{} - if ctrSpec.Linux != nil && !hostConfig.Privileged { - for _, dev := range ctrSpec.Linux.Devices { - key := fmt.Sprintf("%d:%d", dev.Major, dev.Minor) - if deviceNodes == nil { - nodes, err := util.FindDeviceNodes() - if err != nil { - return nil, err - } - deviceNodes = nodes - } - path, ok := deviceNodes[key] - if !ok { - logrus.Warnf("Could not locate device %s on host", key) - continue - } - newDev := define.InspectDevice{} - newDev.PathOnHost = path - newDev.PathInContainer = dev.Path - hostConfig.Devices = append(hostConfig.Devices, newDev) - } + var err error + hostConfig.Devices, err = c.GetDevices(*&hostConfig.Privileged, *ctrSpec, deviceNodes) + if err != nil { + return nil, err } // Ulimits @@ -885,3 +868,29 @@ func (c *Container) inHostPidNS() (bool, error) { } return true, nil } + +func (c *Container) GetDevices(priv bool, ctrSpec spec.Spec, deviceNodes map[string]string) ([]define.InspectDevice, error) { + devices := []define.InspectDevice{} + if ctrSpec.Linux != nil && !priv { + for _, dev := range ctrSpec.Linux.Devices { + key := fmt.Sprintf("%d:%d", dev.Major, dev.Minor) + if deviceNodes == nil { + nodes, err := util.FindDeviceNodes() + if err != nil { + return nil, err + } + deviceNodes = nodes + } + path, ok := deviceNodes[key] + if !ok { + logrus.Warnf("Could not locate device %s on host", key) + continue + } + newDev := define.InspectDevice{} + newDev.PathOnHost = path + newDev.PathInContainer = dev.Path + devices = append(devices, newDev) + } + } + return devices, nil +} diff --git a/libpod/define/pod_inspect.go b/libpod/define/pod_inspect.go index b7a6e76b5..e78d97850 100644 --- a/libpod/define/pod_inspect.go +++ b/libpod/define/pod_inspect.go @@ -59,6 +59,8 @@ type InspectPodData struct { CPUSetCPUs string `json:"cpuset_cpus,omitempty"` // Mounts contains volume related information for the pod Mounts []InspectMount `json:"mounts,omitempty"` + // Devices contains the specified host devices + Devices []InspectDevice `json:"devices,omitempty"` } // InspectPodInfraConfig contains the configuration of the pod's infra diff --git a/libpod/options.go b/libpod/options.go index 3f6ccf1cb..a80f51c6a 100644 --- a/libpod/options.go +++ b/libpod/options.go @@ -21,6 +21,7 @@ import ( "github.com/containers/podman/v3/pkg/util" "github.com/containers/storage" "github.com/containers/storage/pkg/idtools" + "github.com/opencontainers/runtime-spec/specs-go" "github.com/opencontainers/runtime-tools/generate" "github.com/pkg/errors" "github.com/sirupsen/logrus" @@ -1809,6 +1810,17 @@ func WithInitCtrType(containerType string) CtrCreateOption { } } +// WithHostDevice adds the original host src to the config +func WithHostDevice(dev []specs.LinuxDevice) CtrCreateOption { + return func(ctr *Container) error { + if ctr.valid { + return define.ErrCtrFinalized + } + ctr.config.DeviceHostSrc = dev + return nil + } +} + // Pod Creation Options // WithPodCreateCommand adds the full command plus arguments of the current diff --git a/libpod/pod_api.go b/libpod/pod_api.go index 4e0acf950..ff818edc2 100644 --- a/libpod/pod_api.go +++ b/libpod/pod_api.go @@ -583,6 +583,7 @@ func (p *Pod) Inspect() (*define.InspectPodData, error) { // container. var infraConfig *define.InspectPodInfraConfig var inspectMounts []define.InspectMount + var devices []define.InspectDevice if p.state.InfraContainerID != "" { infra, err := p.runtime.GetContainer(p.state.InfraContainerID) if err != nil { @@ -604,6 +605,12 @@ func (p *Pod) Inspect() (*define.InspectPodData, error) { return nil, err } + var nodes map[string]string + devices, err = infra.GetDevices(false, *infra.config.Spec, nodes) + if err != nil { + return nil, err + } + if len(infra.Config().ContainerNetworkConfig.DNSServer) > 0 { infraConfig.DNSServer = make([]string, 0, len(infra.Config().ContainerNetworkConfig.DNSServer)) for _, entry := range infra.Config().ContainerNetworkConfig.DNSServer { @@ -652,6 +659,7 @@ func (p *Pod) Inspect() (*define.InspectPodData, error) { CPUPeriod: p.CPUPeriod(), CPUQuota: p.CPUQuota(), Mounts: inspectMounts, + Devices: devices, } return &inspectData, nil diff --git a/pkg/api/handlers/libpod/pods.go b/pkg/api/handlers/libpod/pods.go index 1f03e121e..9f46ecc52 100644 --- a/pkg/api/handlers/libpod/pods.go +++ b/pkg/api/handlers/libpod/pods.go @@ -41,8 +41,8 @@ func PodCreate(w http.ResponseWriter, r *http.Request) { return } if !psg.NoInfra { - infraOptions := &entities.ContainerCreateOptions{ImageVolume: "bind", IsInfra: true, Net: &entities.NetOptions{}} // options for pulling the image and FillOutSpec - err = specgenutil.FillOutSpecGen(psg.InfraContainerSpec, infraOptions, []string{}) // necessary for default values in many cases (userns, idmappings) + infraOptions := &entities.ContainerCreateOptions{ImageVolume: "bind", IsInfra: true, Net: &entities.NetOptions{}, Devices: psg.Devices} // options for pulling the image and FillOutSpec + err = specgenutil.FillOutSpecGen(psg.InfraContainerSpec, infraOptions, []string{}) // necessary for default values in many cases (userns, idmappings) if err != nil { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "error filling out specgen")) return diff --git a/pkg/domain/entities/pods.go b/pkg/domain/entities/pods.go index 88bd3c6ce..f0c88d77e 100644 --- a/pkg/domain/entities/pods.go +++ b/pkg/domain/entities/pods.go @@ -118,6 +118,7 @@ type PodSpec struct { type PodCreateOptions struct { CGroupParent string `json:"cgroup_parent,omitempty"` CreateCommand []string `json:"create_command,omitempty"` + Devices []string `json:"devices,omitempty"` Hostname string `json:"hostname,omitempty"` Infra bool `json:"infra,omitempty"` InfraImage string `json:"infra_image,omitempty"` @@ -164,7 +165,7 @@ type ContainerCreateOptions struct { CPUS float64 `json:"cpus,omitempty"` CPUSetCPUs string `json:"cpuset_cpus,omitempty"` CPUSetMems string - Devices []string + Devices []string `json:"devices,omitempty"` DeviceCGroupRule []string DeviceReadBPs []string DeviceReadIOPs []string @@ -295,6 +296,7 @@ func ToPodSpecGen(s specgen.PodSpecGenerator, p *PodCreateOptions) (*specgen.Pod s.Pid = out s.Hostname = p.Hostname s.Labels = p.Labels + s.Devices = p.Devices s.NoInfra = !p.Infra if p.InfraCommand != nil && len(*p.InfraCommand) > 0 { s.InfraCommand = strings.Split(*p.InfraCommand, " ") diff --git a/pkg/specgen/generate/config_linux.go b/pkg/specgen/generate/config_linux.go index 6b9e9c4bf..2d1e2b288 100644 --- a/pkg/specgen/generate/config_linux.go +++ b/pkg/specgen/generate/config_linux.go @@ -132,7 +132,6 @@ func DevicesFromPath(g *generate.Generator, devicePath string) error { } return nil } - return addDevice(g, strings.Join(append([]string{resolvedDevicePath}, devs[1:]...), ":")) } diff --git a/pkg/specgen/generate/container_create.go b/pkg/specgen/generate/container_create.go index 91230338e..fefa9b4a9 100644 --- a/pkg/specgen/generate/container_create.go +++ b/pkg/specgen/generate/container_create.go @@ -30,24 +30,27 @@ func MakeContainer(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGener // If joining a pod, retrieve the pod for use, and its infra container var pod *libpod.Pod - var cont *libpod.Container - var config *libpod.ContainerConfig + var infraConfig *libpod.ContainerConfig if s.Pod != "" { pod, err = rt.LookupPod(s.Pod) if err != nil { return nil, nil, nil, errors.Wrapf(err, "error retrieving pod %s", s.Pod) } if pod.HasInfraContainer() { - cont, err = pod.InfraContainer() + infra, err := pod.InfraContainer() if err != nil { return nil, nil, nil, err } - config = cont.Config() + infraConfig = infra.Config() } } - if config != nil && (len(config.NamedVolumes) > 0 || len(config.UserVolumes) > 0 || len(config.ImageVolumes) > 0 || len(config.OverlayVolumes) > 0) { - s.VolumesFrom = append(s.VolumesFrom, config.ID) + if infraConfig != nil && (len(infraConfig.NamedVolumes) > 0 || len(infraConfig.UserVolumes) > 0 || len(infraConfig.ImageVolumes) > 0 || len(infraConfig.OverlayVolumes) > 0) { + s.VolumesFrom = append(s.VolumesFrom, infraConfig.ID) + } + + if infraConfig != nil && len(infraConfig.Spec.Linux.Devices) > 0 { + s.DevicesFrom = append(s.DevicesFrom, infraConfig.ID) } // Set defaults for unset namespaces if s.PidNS.IsDefault() { @@ -166,6 +169,16 @@ func MakeContainer(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGener logrus.Debugf("setting container name %s", s.Name) options = append(options, libpod.WithName(s.Name)) } + if len(s.DevicesFrom) > 0 { + for _, dev := range s.DevicesFrom { + ctr, err := rt.GetContainer(dev) + if err != nil { + return nil, nil, nil, err + } + devices := ctr.DeviceHostSrc() + s.Devices = append(s.Devices, devices...) + } + } if len(s.Devices) > 0 { opts = extractCDIDevices(s) options = append(options, opts...) @@ -174,6 +187,9 @@ func MakeContainer(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGener if err != nil { return nil, nil, nil, err } + if len(s.HostDeviceList) > 0 { + options = append(options, libpod.WithHostDevice(s.HostDeviceList)) + } return runtimeSpec, s, options, err } func ExecuteCreate(ctx context.Context, rt *libpod.Runtime, runtimeSpec *spec.Spec, s *specgen.SpecGenerator, infra bool, options ...libpod.CtrCreateOption) (*libpod.Container, error) { diff --git a/pkg/specgen/generate/oci.go b/pkg/specgen/generate/oci.go index 80c7f112f..55010f716 100644 --- a/pkg/specgen/generate/oci.go +++ b/pkg/specgen/generate/oci.go @@ -301,8 +301,8 @@ func SpecGenToOCI(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Runt g.AddProcessEnv("container", "podman") g.Config.Linux.Resources = s.ResourceLimits - // Devices + if s.Privileged { // If privileged, we need to add all the host devices to the // spec. We do not add the user provided ones because we are @@ -313,17 +313,18 @@ func SpecGenToOCI(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.Runt } else { // add default devices from containers.conf for _, device := range rtc.Containers.Devices { - if err := DevicesFromPath(&g, device); err != nil { + if err = DevicesFromPath(&g, device); err != nil { return nil, err } } // add default devices specified by caller for _, device := range s.Devices { - if err := DevicesFromPath(&g, device.Path); err != nil { + if err = DevicesFromPath(&g, device.Path); err != nil { return nil, err } } } + s.HostDeviceList = s.Devices for _, dev := range s.DeviceCGroupRule { g.AddLinuxResourcesDevice(true, dev.Type, dev.Major, dev.Minor, dev.Access) diff --git a/pkg/specgen/podspecgen.go b/pkg/specgen/podspecgen.go index 5f72fc47d..83fa9426c 100644 --- a/pkg/specgen/podspecgen.go +++ b/pkg/specgen/podspecgen.go @@ -88,6 +88,8 @@ type PodBasicConfig struct { // Image volumes bind-mount a container-image mount into the pod's infra container. // Optional. ImageVolumes []*ImageVolume `json:"image_volumes,omitempty"` + // Devices contains user specified Devices to be added to the Pod + Devices []string `json:"pod_devices,omitempty"` } // PodNetworkConfig contains networking configuration for a pod. diff --git a/pkg/specgen/specgen.go b/pkg/specgen/specgen.go index e0609c5bc..7aa27487a 100644 --- a/pkg/specgen/specgen.go +++ b/pkg/specgen/specgen.go @@ -254,6 +254,10 @@ type ContainerStorageConfig struct { // DeviceCGroupRule are device cgroup rules that allow containers // to use additional types of devices. DeviceCGroupRule []spec.LinuxDeviceCgroup `json:"device_cgroup_rule,omitempty"` + // DevicesFrom is a way to ensure your container inherits device specific information from another container + DevicesFrom []string `json:"devices_from,omitempty"` + // HostDeviceList is used to recreate the mounted device on inherited containers + HostDeviceList []spec.LinuxDevice `json:"host_device_list,omitempty"` // IpcNS is the container's IPC namespace. // Default is private. // Conflicts with ShmSize if not set to private. diff --git a/test/e2e/pod_create_test.go b/test/e2e/pod_create_test.go index 7d40d36dd..76a05fa0f 100644 --- a/test/e2e/pod_create_test.go +++ b/test/e2e/pod_create_test.go @@ -881,6 +881,25 @@ ENTRYPOINT ["sleep","99999"] ctr3 := podmanTest.Podman([]string{"run", "--pod", podName, ALPINE, "cat", "/tmp1/test"}) ctr3.WaitWithDefaultTimeout() Expect(ctr3.OutputToString()).To(ContainSubstring("hello")) + }) + + It("podman pod create --device", func() { + SkipIfRootless("Cannot create devices in /dev in rootless mode") + Expect(os.MkdirAll("/dev/foodevdir", os.ModePerm)).To(BeNil()) + defer os.RemoveAll("/dev/foodevdir") + + mknod := SystemExec("mknod", []string{"/dev/foodevdir/null", "c", "1", "3"}) + mknod.WaitWithDefaultTimeout() + Expect(mknod).Should(Exit(0)) + + podName := "testPod" + session := podmanTest.Podman([]string{"pod", "create", "--device", "/dev/foodevdir:/dev/bar", "--name", podName}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + session = podmanTest.Podman([]string{"run", "-q", "--pod", podName, ALPINE, "stat", "-c%t:%T", "/dev/bar/null"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + Expect(session.OutputToString()).To(Equal("1:3")) }) -- cgit v1.2.3-54-g00ecf