From 46b7c5bc63dec21d1b2a68a36ad84298f9f176f2 Mon Sep 17 00:00:00 2001 From: Evan Lezar Date: Wed, 23 Feb 2022 08:02:44 +0200 Subject: Bump CDI go dependency to v0.3.0 This updates the CDI dependency to the v0.3.0 tagged version instead of relying on a pseudo version. This also addresses the fact that cgroups are not set correctly for devices using the previous dependency. Signed-off-by: Evan Lezar --- go.mod | 2 +- go.sum | 4 +- .../pkg/cdi/annotations.go | 139 ++++++++++++++++++++ .../container-device-interface/pkg/cdi/cache.go | 16 ++- .../pkg/cdi/container-edits.go | 143 ++++++++++++++++++++- .../container-device-interface/pkg/cdi/device.go | 10 +- .../container-device-interface/pkg/cdi/spec.go | 25 +++- .../container-device-interface/specs-go/config.go | 8 +- vendor/modules.txt | 2 +- 9 files changed, 323 insertions(+), 26 deletions(-) create mode 100644 vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/annotations.go diff --git a/go.mod b/go.mod index 451fe72d8..fa16d88b5 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/buger/goterm v1.0.4 github.com/checkpoint-restore/checkpointctl v0.0.0-20211204171957-54b4ebfdb681 github.com/checkpoint-restore/go-criu/v5 v5.3.0 - github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220111162300-46367ec063fd + github.com/container-orchestrated-devices/container-device-interface v0.3.0 github.com/containernetworking/cni v1.0.1 github.com/containernetworking/plugins v1.0.1 github.com/containers/buildah v1.24.2 diff --git a/go.sum b/go.sum index 45eff47e2..cc7fe897b 100644 --- a/go.sum +++ b/go.sum @@ -242,8 +242,8 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/cockroachdb/datadriven v0.0.0-20200714090401-bf6692d28da5/go.mod h1:h6jFvWxBdQXxjopDMZyH2UVceIRfR84bdzbkoKrsWNo= github.com/cockroachdb/errors v1.2.4/go.mod h1:rQD95gz6FARkaKkQXUksEje/d9a6wBJoCr5oaCLELYA= github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220111162300-46367ec063fd h1:Pzh64A349jzW89R73gwkxWoPvpkd8rz2X+KLUaMmBRY= -github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220111162300-46367ec063fd/go.mod h1:YqXyXS/oVW3ix0IHVXitKBq3RZoCF8ccm5RPmRBraUI= +github.com/container-orchestrated-devices/container-device-interface v0.3.0 h1:tM2zdVYZY8getsFaTc7Z+v+UqDXhk5alchOHVEADes0= +github.com/container-orchestrated-devices/container-device-interface v0.3.0/go.mod h1:LGs3yHVe1wZn2XsWl4AxywYQ3NRZ6osTEZozCHQCRSM= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/annotations.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/annotations.go new file mode 100644 index 000000000..1055c7df8 --- /dev/null +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/annotations.go @@ -0,0 +1,139 @@ +/* + Copyright © 2021-2022 The CDI Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cdi + +import ( + "strings" + + "github.com/pkg/errors" +) + +const ( + // AnnotationPrefix is the prefix for CDI container annotation keys. + AnnotationPrefix = "cdi.k8s.io/" +) + +// UpdateAnnotations updates annotations with a plugin-specific CDI device +// injection request for the given devices. Upon any error a non-nil error +// is returned and annotations are left intact. By convention plugin should +// be in the format of "vendor.device-type". +func UpdateAnnotations(annotations map[string]string, plugin string, deviceID string, devices []string) (map[string]string, error) { + key, err := AnnotationKey(plugin, deviceID) + if err != nil { + return annotations, errors.Wrap(err, "CDI annotation failed") + } + if _, ok := annotations[key]; ok { + return annotations, errors.Errorf("CDI annotation failed, key %q used", key) + } + value, err := AnnotationValue(devices) + if err != nil { + return annotations, errors.Wrap(err, "CDI annotation failed") + } + + if annotations == nil { + annotations = make(map[string]string) + } + annotations[key] = value + + return annotations, nil +} + +// ParseAnnotations parses annotations for CDI device injection requests. +// The keys and devices from all such requests are collected into slices +// which are returned as the result. All devices are expected to be fully +// qualified CDI device names. If any device fails this check empty slices +// are returned along with a non-nil error. The annotations are expected +// to be formatted by, or in a compatible fashion to UpdateAnnotations(). +func ParseAnnotations(annotations map[string]string) ([]string, []string, error) { + var ( + keys []string + devices []string + ) + + for key, value := range annotations { + if !strings.HasPrefix(key, AnnotationPrefix) { + continue + } + for _, d := range strings.Split(value, ",") { + if !IsQualifiedName(d) { + return nil, nil, errors.Errorf("invalid CDI device name %q", d) + } + devices = append(devices, d) + } + keys = append(keys, key) + } + + return keys, devices, nil +} + +// AnnotationKey returns a unique annotation key for an device allocation +// by a K8s device plugin. pluginName should be in the format of +// "vendor.device-type". deviceID is the ID of the device the plugin is +// allocating. It is used to make sure that the generated key is unique +// even if multiple allocations by a single plugin needs to be annotated. +func AnnotationKey(pluginName, deviceID string) (string, error) { + const maxNameLen = 63 + + if pluginName == "" { + return "", errors.New("invalid plugin name, empty") + } + if deviceID == "" { + return "", errors.New("invalid deviceID, empty") + } + + name := pluginName + "_" + strings.ReplaceAll(deviceID, "/", "_") + + if len(name) > maxNameLen { + return "", errors.Errorf("invalid plugin+deviceID %q, too long", name) + } + + if c := rune(name[0]); !isAlphaNumeric(c) { + return "", errors.Errorf("invalid name %q, first '%c' should be alphanumeric", + name, c) + } + if len(name) > 2 { + for _, c := range name[1 : len(name)-1] { + switch { + case isAlphaNumeric(c): + case c == '_' || c == '-' || c == '.': + default: + return "", errors.Errorf("invalid name %q, invalid charcter '%c'", + name, c) + } + } + } + if c := rune(name[len(name)-1]); !isAlphaNumeric(c) { + return "", errors.Errorf("invalid name %q, last '%c' should be alphanumeric", + name, c) + } + + return AnnotationPrefix + name, nil +} + +// AnnotationValue returns an annotation value for the given devices. +func AnnotationValue(devices []string) (string, error) { + value, sep := "", "" + for _, d := range devices { + if _, _, _, err := ParseQualifiedName(d); err != nil { + return "", err + } + value += sep + d + sep = "," + } + + return value, nil +} diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/cache.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/cache.go index c4befc83e..5c3f803f4 100644 --- a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/cache.go +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/cache.go @@ -153,10 +153,7 @@ func (c *Cache) Refresh() error { // returns any unresolvable devices and an error if injection fails for // any of the devices. func (c *Cache) InjectDevices(ociSpec *oci.Spec, devices ...string) ([]string, error) { - var ( - unresolved []string - specs = map[*Spec]struct{}{} - ) + var unresolved []string if ociSpec == nil { return devices, errors.Errorf("can't inject devices, nil OCI Spec") @@ -165,6 +162,9 @@ func (c *Cache) InjectDevices(ociSpec *oci.Spec, devices ...string) ([]string, e c.Lock() defer c.Unlock() + edits := &ContainerEdits{} + specs := map[*Spec]struct{}{} + for _, device := range devices { d := c.devices[device] if d == nil { @@ -173,9 +173,9 @@ func (c *Cache) InjectDevices(ociSpec *oci.Spec, devices ...string) ([]string, e } if _, ok := specs[d.GetSpec()]; !ok { specs[d.GetSpec()] = struct{}{} - d.GetSpec().ApplyEdits(ociSpec) + edits.Append(d.GetSpec().edits()) } - d.ApplyEdits(ociSpec) + edits.Append(d.edits()) } if unresolved != nil { @@ -183,6 +183,10 @@ func (c *Cache) InjectDevices(ociSpec *oci.Spec, devices ...string) ([]string, e strings.Join(devices, ", ")) } + if err := edits.Apply(ociSpec); err != nil { + return nil, errors.Wrap(err, "failed to inject devices") + } + return nil, nil } diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/container-edits.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/container-edits.go index 767cba57b..80d88b118 100644 --- a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/container-edits.go +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/container-edits.go @@ -17,6 +17,9 @@ package cdi import ( + "os" + "path/filepath" + "sort" "strings" "github.com/pkg/errors" @@ -24,6 +27,8 @@ import ( "github.com/container-orchestrated-devices/container-device-interface/specs-go" oci "github.com/opencontainers/runtime-spec/specs-go" ocigen "github.com/opencontainers/runtime-tools/generate" + + runc "github.com/opencontainers/runc/libcontainer/devices" ) const ( @@ -78,12 +83,44 @@ func (e *ContainerEdits) Apply(spec *oci.Spec) error { if len(e.Env) > 0 { specgen.AddMultipleProcessEnv(e.Env) } + for _, d := range e.DeviceNodes { - specgen.AddDevice(d.ToOCI()) + dev := d.ToOCI() + if err := fillMissingInfo(&dev); err != nil { + return err + } + + if dev.UID == nil && spec.Process != nil { + if uid := spec.Process.User.UID; uid > 0 { + dev.UID = &uid + } + } + if dev.GID == nil && spec.Process != nil { + if gid := spec.Process.User.GID; gid > 0 { + dev.GID = &gid + } + } + + specgen.RemoveDevice(dev.Path) + specgen.AddDevice(dev) + + if dev.Type == "b" || dev.Type == "c" { + access := d.Permissions + if access == "" { + access = "rwm" + } + specgen.AddLinuxResourcesDevice(true, dev.Type, &dev.Major, &dev.Minor, access) + } } - for _, m := range e.Mounts { - specgen.AddMount(m.ToOCI()) + + if len(e.Mounts) > 0 { + for _, m := range e.Mounts { + specgen.RemoveMount(m.ContainerPath) + specgen.AddMount(m.ToOCI()) + } + sortMounts(&specgen) } + for _, h := range e.Hooks { switch h.HookName { case PrestartHook: @@ -138,6 +175,27 @@ func (e *ContainerEdits) Validate() error { return nil } +// Append other edits into this one. If called with a nil receiver, +// allocates and returns newly allocated edits. +func (e *ContainerEdits) Append(o *ContainerEdits) *ContainerEdits { + if o == nil || o.ContainerEdits == nil { + return e + } + if e == nil { + e = &ContainerEdits{} + } + if e.ContainerEdits == nil { + e.ContainerEdits = &specs.ContainerEdits{} + } + + e.Env = append(e.Env, o.Env...) + e.DeviceNodes = append(e.DeviceNodes, o.DeviceNodes...) + e.Hooks = append(e.Hooks, o.Hooks...) + e.Mounts = append(e.Mounts, o.Mounts...) + + return e +} + // isEmpty returns true if these edits are empty. This is valid in a // global Spec context but invalid in a Device context. func (e *ContainerEdits) isEmpty() bool { @@ -164,10 +222,18 @@ type DeviceNode struct { // Validate a CDI Spec DeviceNode. func (d *DeviceNode) Validate() error { + validTypes := map[string]struct{}{ + "": {}, + "b": {}, + "c": {}, + "u": {}, + "p": {}, + } + if d.Path == "" { return errors.New("invalid (empty) device path") } - if d.Type != "" && d.Type != "b" && d.Type != "c" { + if _, ok := validTypes[d.Type]; !ok { return errors.Errorf("device %q: invalid type %q", d.Path, d.Type) } for _, bit := range d.Permissions { @@ -220,3 +286,72 @@ func ensureOCIHooks(spec *oci.Spec) { spec.Hooks = &oci.Hooks{} } } + +// fillMissingInfo fills in missing mandatory attributes from the host device. +func fillMissingInfo(dev *oci.LinuxDevice) error { + if dev.Type != "" && (dev.Major != 0 || dev.Type == "p") { + return nil + } + hostDev, err := runc.DeviceFromPath(dev.Path, "rwm") + if err != nil { + return errors.Wrapf(err, "failed to stat CDI host device %q", dev.Path) + } + + if dev.Type == "" { + dev.Type = string(hostDev.Type) + } else { + if dev.Type != string(hostDev.Type) { + return errors.Errorf("CDI device %q, host type mismatch (%s, %s)", + dev.Path, dev.Type, string(hostDev.Type)) + } + } + if dev.Major == 0 && dev.Type != "p" { + dev.Major = hostDev.Major + dev.Minor = hostDev.Minor + } + + return nil +} + +// sortMounts sorts the mounts in the given OCI Spec. +func sortMounts(specgen *ocigen.Generator) { + mounts := specgen.Mounts() + specgen.ClearMounts() + sort.Sort(orderedMounts(mounts)) + specgen.Config.Mounts = mounts +} + +// orderedMounts defines how to sort an OCI Spec Mount slice. +// This is the almost the same implementation sa used by CRI-O and Docker, +// with a minor tweak for stable sorting order (easier to test): +// https://github.com/moby/moby/blob/17.05.x/daemon/volumes.go#L26 +type orderedMounts []oci.Mount + +// Len returns the number of mounts. Used in sorting. +func (m orderedMounts) Len() int { + return len(m) +} + +// Less returns true if the number of parts (a/b/c would be 3 parts) in the +// mount indexed by parameter 1 is less than that of the mount indexed by +// parameter 2. Used in sorting. +func (m orderedMounts) Less(i, j int) bool { + ip, jp := m.parts(i), m.parts(j) + if ip < jp { + return true + } + if jp < ip { + return false + } + return m[i].Destination < m[j].Destination +} + +// Swap swaps two items in an array of mounts. Used in sorting +func (m orderedMounts) Swap(i, j int) { + m[i], m[j] = m[j], m[i] +} + +// parts returns the number of parts in the destination of a mount. Used in sorting. +func (m orderedMounts) parts(i int) int { + return strings.Count(filepath.Clean(m[i].Destination), string(os.PathSeparator)) +} diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/device.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/device.go index 84b71429c..0bb1f531b 100644 --- a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/device.go +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/device.go @@ -54,8 +54,12 @@ func (d *Device) GetQualifiedName() string { // ApplyEdits applies the device-speific container edits to an OCI Spec. func (d *Device) ApplyEdits(ociSpec *oci.Spec) error { - e := ContainerEdits{&d.ContainerEdits} - return e.Apply(ociSpec) + return d.edits().Apply(ociSpec) +} + +// edits returns the applicable container edits for this spec. +func (d *Device) edits() *ContainerEdits { + return &ContainerEdits{&d.ContainerEdits} } // Validate the device. @@ -63,7 +67,7 @@ func (d *Device) validate() error { if err := ValidateDeviceName(d.Name); err != nil { return err } - edits := ContainerEdits{&d.ContainerEdits} + edits := d.edits() if edits.isEmpty() { return errors.Errorf("invalid device, empty device edits") } diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec.go index 20f188498..adebc101f 100644 --- a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec.go +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec.go @@ -33,6 +33,7 @@ var ( validSpecVersions = map[string]struct{}{ "0.1.0": {}, "0.2.0": {}, + "0.3.0": {}, } ) @@ -120,14 +121,18 @@ func (s *Spec) GetPriority() int { // ApplyEdits applies the Spec's global-scope container edits to an OCI Spec. func (s *Spec) ApplyEdits(ociSpec *oci.Spec) error { - e := ContainerEdits{&s.ContainerEdits} - return e.Apply(ociSpec) + return s.edits().Apply(ociSpec) +} + +// edits returns the applicable global container edits for this spec. +func (s *Spec) edits() *ContainerEdits { + return &ContainerEdits{&s.ContainerEdits} } // Validate the Spec. func (s *Spec) validate() (map[string]*Device, error) { - if _, ok := validSpecVersions[s.Version]; !ok { - return nil, errors.Errorf("invalid version %q", s.Version) + if err := validateVersion(s.Version); err != nil { + return nil, err } if err := ValidateVendorName(s.vendor); err != nil { return nil, err @@ -135,8 +140,7 @@ func (s *Spec) validate() (map[string]*Device, error) { if err := ValidateClassName(s.class); err != nil { return nil, err } - edits := &ContainerEdits{&s.ContainerEdits} - if err := edits.Validate(); err != nil { + if err := s.edits().Validate(); err != nil { return nil, err } @@ -155,6 +159,15 @@ func (s *Spec) validate() (map[string]*Device, error) { return devices, nil } +// validateVersion checks whether the specified spec version is supported. +func validateVersion(version string) error { + if _, ok := validSpecVersions[version]; !ok { + return errors.Errorf("invalid version %q", version) + } + + return nil +} + // Parse raw CDI Spec file data. func parseSpec(data []byte) (*cdi.Spec, error) { raw := &cdi.Spec{} diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/config.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/config.go index 20914e5b6..090e30e43 100644 --- a/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/config.go +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/config.go @@ -2,11 +2,13 @@ package specs import "os" +// CurrentVersion is the current version of the Spec. +const CurrentVersion = "0.3.0" + // Spec is the base configuration for CDI type Spec struct { - Version string `json:"cdiVersion"` - Kind string `json:"kind"` - ContainerRuntime []string `json:"containerRuntime,omitempty"` + Version string `json:"cdiVersion"` + Kind string `json:"kind"` Devices []Device `json:"devices"` ContainerEdits ContainerEdits `json:"containerEdits,omitempty"` diff --git a/vendor/modules.txt b/vendor/modules.txt index 8d58ab5a5..32d0152e0 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -57,7 +57,7 @@ github.com/checkpoint-restore/go-criu/v5/rpc github.com/checkpoint-restore/go-criu/v5/stats # github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/chzyer/readline -# github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220111162300-46367ec063fd +# github.com/container-orchestrated-devices/container-device-interface v0.3.0 ## explicit github.com/container-orchestrated-devices/container-device-interface/pkg/cdi github.com/container-orchestrated-devices/container-device-interface/specs-go -- cgit v1.2.3-54-g00ecf