diff options
67 files changed, 3709 insertions, 670 deletions
diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md index b8ba48819..6ad306f0f 100644 --- a/.github/ISSUE_TEMPLATE.md +++ b/.github/ISSUE_TEMPLATE.md @@ -12,7 +12,7 @@ documented on the top of Podman's [README.md](../README.md). If they differ, pl update your version of Podman to the latest possible and retry your command before creating an issue. -Also, there is a running list of known issues in the [Podman Troubleshooting Guide](https://github.com/containers/podman/blob/master/troubleshooting.md), +Also, there is a running list of known issues in the [Podman Troubleshooting Guide](https://github.com/containers/podman/blob/main/troubleshooting.md), please reference that page before opening a new issue. If you are filing a bug against `podman build`, please instead file a bug @@ -67,7 +67,7 @@ Briefly describe the problem you are having in a few paragraphs. (paste your output here) ``` -**Have you tested with the latest version of Podman and have you checked the Podman Troubleshooting Guide? (https://github.com/containers/podman/blob/master/troubleshooting.md)** +**Have you tested with the latest version of Podman and have you checked the Podman Troubleshooting Guide? (https://github.com/containers/podman/blob/main/troubleshooting.md)** Yes/No diff --git a/cmd/podman/manifest/add.go b/cmd/podman/manifest/add.go index 9d219601c..d09533967 100644 --- a/cmd/podman/manifest/add.go +++ b/cmd/podman/manifest/add.go @@ -26,14 +26,14 @@ type manifestAddOptsWrapper struct { var ( manifestAddOpts = manifestAddOptsWrapper{} addCmd = &cobra.Command{ - Use: "add [options] LIST LIST", + Use: "add [options] LIST IMAGE [IMAGE...]", Short: "Add images to a manifest list or image index", Long: "Adds an image to a manifest list or image index.", RunE: add, + Args: cobra.MinimumNArgs(2), ValidArgsFunction: common.AutocompleteImages, Example: `podman manifest add mylist:v1.11 image:v1.11-amd64 podman manifest add mylist:v1.11 transport:imageName`, - Args: cobra.ExactArgs(2), } ) @@ -93,10 +93,6 @@ func add(cmd *cobra.Command, args []string) error { return err } - // FIXME: (@vrothberg) this interface confuses me a lot. Why are they - // not two arguments? - manifestAddOpts.Images = []string{args[1], args[0]} - if manifestAddOpts.CredentialsCLI != "" { creds, err := util.ParseRegistryCreds(manifestAddOpts.CredentialsCLI) if err != nil { @@ -114,10 +110,10 @@ func add(cmd *cobra.Command, args []string) error { manifestAddOpts.SkipTLSVerify = types.NewOptionalBool(!manifestAddOpts.TLSVerifyCLI) } - listID, err := registry.ImageEngine().ManifestAdd(context.Background(), manifestAddOpts.ManifestAddOptions) + listID, err := registry.ImageEngine().ManifestAdd(context.Background(), args[0], args[1:], manifestAddOpts.ManifestAddOptions) if err != nil { return err } - fmt.Printf("%s\n", listID) + fmt.Println(listID) return nil } diff --git a/cmd/podman/manifest/annotate.go b/cmd/podman/manifest/annotate.go index d806ce9e6..a032a1fe5 100644 --- a/cmd/podman/manifest/annotate.go +++ b/cmd/podman/manifest/annotate.go @@ -1,14 +1,12 @@ package manifest import ( - "context" "fmt" "github.com/containers/common/pkg/completion" "github.com/containers/podman/v3/cmd/podman/common" "github.com/containers/podman/v3/cmd/podman/registry" "github.com/containers/podman/v3/pkg/domain/entities" - "github.com/pkg/errors" "github.com/spf13/cobra" ) @@ -20,8 +18,8 @@ var ( Short: "Add or update information about an entry in a manifest list or image index", Long: "Adds or updates information about an entry in a manifest list or image index.", RunE: annotate, - Example: `podman manifest annotate --annotation left=right mylist:v1.11 image:v1.11-amd64`, Args: cobra.ExactArgs(2), + Example: `podman manifest annotate --annotation left=right mylist:v1.11 image:v1.11-amd64`, ValidArgsFunction: common.AutocompleteImages, } ) @@ -63,18 +61,10 @@ func init() { } func annotate(cmd *cobra.Command, args []string) error { - listImageSpec := args[0] - instanceSpec := args[1] - if listImageSpec == "" { - return errors.Errorf(`invalid image name "%s"`, listImageSpec) - } - if instanceSpec == "" { - return errors.Errorf(`invalid image digest "%s"`, instanceSpec) - } - updatedListID, err := registry.ImageEngine().ManifestAnnotate(context.Background(), args, manifestAnnotateOpts) + id, err := registry.ImageEngine().ManifestAnnotate(registry.Context(), args[0], args[1], manifestAnnotateOpts) if err != nil { return err } - fmt.Printf("%s\n", updatedListID) + fmt.Println(id) return nil } diff --git a/cmd/podman/manifest/create.go b/cmd/podman/manifest/create.go index 95c9f89b0..acef53045 100644 --- a/cmd/podman/manifest/create.go +++ b/cmd/podman/manifest/create.go @@ -1,7 +1,6 @@ package manifest import ( - "context" "fmt" "github.com/containers/podman/v3/cmd/podman/common" @@ -36,7 +35,7 @@ func init() { } func create(cmd *cobra.Command, args []string) error { - imageID, err := registry.ImageEngine().ManifestCreate(context.Background(), args[:1], args[1:], manifestCreateOpts) + imageID, err := registry.ImageEngine().ManifestCreate(registry.Context(), args[0], args[1:], manifestCreateOpts) if err != nil { return err } diff --git a/cmd/podman/manifest/inspect.go b/cmd/podman/manifest/inspect.go index d444f9066..ac8ee3dc4 100644 --- a/cmd/podman/manifest/inspect.go +++ b/cmd/podman/manifest/inspect.go @@ -1,7 +1,6 @@ package manifest import ( - "context" "fmt" "github.com/containers/podman/v3/cmd/podman/common" @@ -29,10 +28,10 @@ func init() { } func inspect(cmd *cobra.Command, args []string) error { - buf, err := registry.ImageEngine().ManifestInspect(context.Background(), args[0]) + buf, err := registry.ImageEngine().ManifestInspect(registry.Context(), args[0]) if err != nil { return err } - fmt.Printf("%s\n", buf) + fmt.Println(string(buf)) return nil } diff --git a/cmd/podman/manifest/remove.go b/cmd/podman/manifest/remove.go index c44c0991e..4716af201 100644 --- a/cmd/podman/manifest/remove.go +++ b/cmd/podman/manifest/remove.go @@ -1,7 +1,6 @@ package manifest import ( - "context" "fmt" "github.com/containers/podman/v3/cmd/podman/common" @@ -16,9 +15,9 @@ var ( Short: "Remove an entry from a manifest list or image index", Long: "Removes an image from a manifest list or image index.", RunE: remove, + Args: cobra.ExactArgs(2), ValidArgsFunction: common.AutocompleteImages, Example: `podman manifest remove mylist:v1.11 sha256:15352d97781ffdf357bf3459c037be3efac4133dc9070c2dce7eca7c05c3e736`, - Args: cobra.ExactArgs(2), } ) @@ -30,18 +29,10 @@ func init() { } func remove(cmd *cobra.Command, args []string) error { - listImageSpec := args[0] - instanceSpec := args[1] - if listImageSpec == "" { - return errors.Errorf(`invalid image name "%s"`, listImageSpec) - } - if instanceSpec == "" { - return errors.Errorf(`invalid image digest "%s"`, instanceSpec) - } - updatedListID, err := registry.ImageEngine().ManifestRemove(context.Background(), args) + updatedListID, err := registry.ImageEngine().ManifestRemoveDigest(registry.Context(), args[0], args[1]) if err != nil { - return errors.Wrapf(err, "error removing from manifest list %s", listImageSpec) + return errors.Wrapf(err, "error removing from manifest list %s", args[0]) } - fmt.Printf("%s\n", updatedListID) + fmt.Println(updatedListID) return nil } diff --git a/cmd/podman/manifest/rm.go b/cmd/podman/manifest/rm.go index 5e78197ed..b4a52653b 100644 --- a/cmd/podman/manifest/rm.go +++ b/cmd/podman/manifest/rm.go @@ -12,13 +12,13 @@ import ( var ( rmCmd = &cobra.Command{ - Use: "rm LIST", + Use: "rm LIST [LIST...]", Short: "Remove manifest list or image index from local storage", Long: "Remove manifest list or image index from local storage.", RunE: rm, + Args: cobra.MinimumNArgs(1), ValidArgsFunction: common.AutocompleteImages, Example: `podman manifest rm mylist:v1.11`, - Args: cobra.ExactArgs(1), } ) diff --git a/docs/source/markdown/podman-run.1.md b/docs/source/markdown/podman-run.1.md index 80652fcdf..f77d0418b 100644 --- a/docs/source/markdown/podman-run.1.md +++ b/docs/source/markdown/podman-run.1.md @@ -377,7 +377,7 @@ You need to specify multi option commands in the form of a json string. Set environment variables. -This option allows arbitrary environment variables that are available for the process to be launched inside of the container. If an environment variable is specified without a value, Podman will check the host environment for a value and set the variable only if it is set on the host. If an environment variable ending in __*__ is specified, Podman will search the host environment for variables starting with the prefix and will add those variables to the container. If an environment variable with a trailing ***** is specified, then a value must be supplied. +This option allows arbitrary environment variables that are available for the process to be launched inside of the container. If an environment variable is specified without a value, Podman will check the host environment for a value and set the variable only if it is set on the host. If an environment variable ending in __*__ is specified, Podman will search the host environment for variables starting with the prefix and will add those variables to the container. If an environment variable with a trailing __*__ is specified, then a value must be supplied. See [**Environment**](#environment) note below for precedence and examples. @@ -8,7 +8,7 @@ require ( github.com/buger/goterm v0.0.0-20181115115552-c206103e1f37 github.com/checkpoint-restore/checkpointctl v0.0.0-20211204171957-54b4ebfdb681 github.com/checkpoint-restore/go-criu/v5 v5.3.0 - github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9 + github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220111162300-46367ec063fd github.com/containernetworking/cni v1.0.1 github.com/containernetworking/plugins v1.0.1 github.com/containers/buildah v1.23.1-0.20220112160421-d744ebc4b1d5 @@ -69,7 +69,6 @@ require ( golang.org/x/sys v0.0.0-20211214234402-4825e8c3871d golang.org/x/text v0.3.7 google.golang.org/protobuf v1.27.1 - gopkg.in/fsnotify.v1 v1.4.7 // indirect gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b k8s.io/api v0.22.4 k8s.io/apimachinery v0.22.4 @@ -217,8 +217,8 @@ github.com/cncf/xds/go v0.0.0-20211001041855-01bcc9b48dfe/go.mod h1:eXthEFrGJvWH github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20211130200136-a8f946100490/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9 h1:Kn0s9/APRtr5dk/83aXj97WX0+PYnJK9BO8g0Xclm0I= -github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9/go.mod h1:eQt66kIaJpUhCrjCtBFQGQxGLbAUl0OuuwjTH16ON4s= +github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220111162300-46367ec063fd h1:Pzh64A349jzW89R73gwkxWoPvpkd8rz2X+KLUaMmBRY= +github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220111162300-46367ec063fd/go.mod h1:YqXyXS/oVW3ix0IHVXitKBq3RZoCF8ccm5RPmRBraUI= github.com/containerd/aufs v0.0.0-20200908144142-dab0cbea06f4/go.mod h1:nukgQABAEopAHvB6j7cnP5zJ+/3aVcE7hCYqvIwAHyE= github.com/containerd/aufs v0.0.0-20201003224125-76a6863f2989/go.mod h1:AkGGQs9NM2vtYHaUen+NljV0/baGCAPELGm2q9ZXpWU= github.com/containerd/aufs v0.0.0-20210316121734-20793ff83c97/go.mod h1:kL5kd6KM5TzQjR79jljyi4olc1Vrx6XBlcyj3gNv2PU= @@ -992,6 +992,7 @@ github.com/opencontainers/runtime-spec v1.0.3-0.20200929063507-e6143ca7d51d/go.m github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opencontainers/runtime-tools v0.0.0-20181011054405-1d69bd0f9c39/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= +github.com/opencontainers/runtime-tools v0.0.0-20190417131837-cd1349b7c47e/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/runtime-tools v0.9.0/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= github.com/opencontainers/runtime-tools v0.9.1-0.20211020193359-09d837bf40a7 h1:6JHkPc2wUOsj2XBpYzyvmCL5Y/fA3TFaomYv/Iggt1g= github.com/opencontainers/runtime-tools v0.9.1-0.20211020193359-09d837bf40a7/go.mod h1:r3f7wjNzSs2extwzU3Y+6pKfobzPh+kKFJ3ofN+3nfs= @@ -1999,5 +2000,6 @@ sigs.k8s.io/structured-merge-diff/v4 v4.0.3/go.mod h1:bJZC9H9iH24zzfZ/41RGcq60oK sigs.k8s.io/structured-merge-diff/v4 v4.1.2 h1:Hr/htKFmJEbtMgS/UD0N+gtgctAqz81t3nu+sPzynno= sigs.k8s.io/structured-merge-diff/v4 v4.1.2/go.mod h1:j/nl6xW8vLS49O8YvXW1ocPhZawJtm+Yrr7PPRQ0Vg4= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= -sigs.k8s.io/yaml v1.2.0 h1:kr/MCeFWJWTwyaHoR9c8EjH9OumOmoF9YGiZd7lFm/Q= sigs.k8s.io/yaml v1.2.0/go.mod h1:yfXDCHCao9+ENCvLSE62v9VSji2MKu5jeNfTrofGhJc= +sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= +sigs.k8s.io/yaml v1.3.0/go.mod h1:GeOyir5tyXNByN85N/dRIT9es5UQNerPYEKK56eTBm8= diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go index 28cc2c621..508a8a8cf 100644 --- a/libpod/container_internal_linux.go +++ b/libpod/container_internal_linux.go @@ -22,7 +22,7 @@ import ( metadata "github.com/checkpoint-restore/checkpointctl/lib" "github.com/checkpoint-restore/go-criu/v5/stats" - cdi "github.com/container-orchestrated-devices/container-device-interface/pkg" + cdi "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" "github.com/containernetworking/plugins/pkg/ns" "github.com/containers/buildah/pkg/chrootuser" "github.com/containers/buildah/pkg/overlay" @@ -744,7 +744,12 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) { // Warning: CDI may alter g.Config in place. if len(c.config.CDIDevices) > 0 { - if err = cdi.UpdateOCISpecForDevices(g.Config, c.config.CDIDevices); err != nil { + registry := cdi.GetRegistry() + if errs := registry.GetErrors(); len(errs) > 0 { + logrus.Debugf("The following errors were triggered when creating the CDI registry: %v", errs) + } + _, err := registry.InjectDevices(g.Config, c.config.CDIDevices...) + if err != nil { return nil, errors.Wrapf(err, "error setting up CDI devices") } } diff --git a/pkg/api/handlers/libpod/manifests.go b/pkg/api/handlers/libpod/manifests.go index eb0b6827f..ef0839d1f 100644 --- a/pkg/api/handlers/libpod/manifests.go +++ b/pkg/api/handlers/libpod/manifests.go @@ -3,7 +3,11 @@ package libpod import ( "context" "encoding/json" + "fmt" + "io/ioutil" "net/http" + "net/url" + "strings" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/manifest" @@ -15,6 +19,8 @@ import ( "github.com/containers/podman/v3/pkg/auth" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/containers/podman/v3/pkg/domain/infra/abi" + "github.com/containers/podman/v3/pkg/errorhandling" + "github.com/gorilla/mux" "github.com/gorilla/schema" "github.com/opencontainers/go-digest" "github.com/pkg/errors" @@ -24,40 +30,93 @@ func ManifestCreate(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) query := struct { - Name []string `schema:"name"` - Image []string `schema:"image"` - All bool `schema:"all"` + Name string `schema:"name"` + Images []string `schema:"images"` + All bool `schema:"all"` }{ // Add defaults here once needed. } + + // Support 3.x API calls, alias image to images + if image, ok := r.URL.Query()["image"]; ok { + query.Images = image + } + if err := decoder.Decode(&query, r.URL.Query()); err != nil { utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } - // TODO: (jhonce) When c/image is refactored the roadmap calls for this check to be pushed into that library. - for _, n := range query.Name { - if _, err := reference.ParseNormalizedNamed(n); err != nil { + // Support 4.x API calls, map query parameter to path + if name, ok := mux.Vars(r)["name"]; ok { + n, err := url.QueryUnescape(name) + if err != nil { utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, - errors.Wrapf(err, "invalid image name %s", n)) + errors.Wrapf(err, "failed to parse name parameter %q", name)) return } + query.Name = n + } + + if _, err := reference.ParseNormalizedNamed(query.Name); err != nil { + utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + errors.Wrapf(err, "invalid image name %s", query.Name)) + return } imageEngine := abi.ImageEngine{Libpod: runtime} createOptions := entities.ManifestCreateOptions{All: query.All} - manID, err := imageEngine.ManifestCreate(r.Context(), query.Name, query.Image, createOptions) + manID, err := imageEngine.ManifestCreate(r.Context(), query.Name, query.Images, createOptions) + if err != nil { + utils.InternalServerError(w, err) + return + } + + status := http.StatusOK + if _, err := utils.SupportedVersion(r, "< 4.0.0"); err == utils.ErrVersionNotSupported { + status = http.StatusCreated + } + + buffer, err := ioutil.ReadAll(r.Body) + if err != nil { + utils.InternalServerError(w, err) + return + } + + // Treat \r\n as empty body + if len(buffer) < 3 { + utils.WriteResponse(w, status, handlers.IDResponse{ID: manID}) + return + } + + body := new(entities.ManifestModifyOptions) + if err := json.Unmarshal(buffer, body); err != nil { + utils.InternalServerError(w, errors.Wrap(err, "Decode()")) + return + } + + // gather all images for manifest list + var images []string + if len(query.Images) > 0 { + images = append(query.Images) + } + if len(body.Images) > 0 { + images = append(body.Images) + } + + id, err := imageEngine.ManifestAdd(r.Context(), query.Name, images, body.ManifestAddOptions) if err != nil { utils.InternalServerError(w, err) return } - utils.WriteResponse(w, http.StatusOK, handlers.IDResponse{ID: manID}) + + utils.WriteResponse(w, status, handlers.IDResponse{ID: id}) } -// ExistsManifest check if a manifest list exists -func ExistsManifest(w http.ResponseWriter, r *http.Request) { +// ManifestExists return true if manifest list exists. +func ManifestExists(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) name := utils.GetName(r) @@ -94,10 +153,18 @@ func ManifestInspect(w http.ResponseWriter, r *http.Request) { utils.WriteResponse(w, http.StatusOK, schema2List) } +// ManifestAdd remove digest from manifest list +// +// Deprecated: As of 4.0.0 use ManifestModify instead func ManifestAdd(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) - var addOptions entities.ManifestAddOptions - if err := json.NewDecoder(r.Body).Decode(&addOptions); err != nil { + + // Wrapper to support 3.x with 4.x libpod + query := struct { + entities.ManifestAddOptions + Images []string + }{} + if err := json.NewDecoder(r.Body).Decode(&query); err != nil { utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "Decode()")) return } @@ -108,15 +175,8 @@ func ManifestAdd(w http.ResponseWriter, r *http.Request) { return } - // FIXME: we really need to clean up the manifest API. Swagger states - // the arguments were strings not string slices. The use of string - // slices, mixing lists and images is incredibly confusing. - if len(addOptions.Images) == 1 { - addOptions.Images = append(addOptions.Images, name) - } - imageEngine := abi.ImageEngine{Libpod: runtime} - newID, err := imageEngine.ManifestAdd(r.Context(), addOptions) + newID, err := imageEngine.ManifestAdd(r.Context(), name, query.Images, query.ManifestAddOptions) if err != nil { utils.InternalServerError(w, err) return @@ -124,7 +184,10 @@ func ManifestAdd(w http.ResponseWriter, r *http.Request) { utils.WriteResponse(w, http.StatusOK, handlers.IDResponse{ID: newID}) } -func ManifestRemove(w http.ResponseWriter, r *http.Request) { +// ManifestRemoveDigest remove digest from manifest list +// +// Deprecated: As of 4.0.0 use ManifestModify instead +func ManifestRemoveDigest(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) query := struct { @@ -155,7 +218,10 @@ func ManifestRemove(w http.ResponseWriter, r *http.Request) { utils.WriteResponse(w, http.StatusOK, handlers.IDResponse{ID: manifestList.ID()}) } -func ManifestPush(w http.ResponseWriter, r *http.Request) { +// ManifestPushV3 push image to registry +// +// Deprecated: As of 4.0.0 use ManifestPush instead +func ManifestPushV3(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) query := struct { @@ -207,3 +273,156 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) { } utils.WriteResponse(w, http.StatusOK, digest) } + +// ManifestPush push image to registry +// +// As of 4.0.0 +func ManifestPush(w http.ResponseWriter, r *http.Request) { + runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) + decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) + + query := struct { + All bool `schema:"all"` + TLSVerify bool `schema:"tlsVerify"` + }{ + // Add defaults here once needed. + } + if err := decoder.Decode(&query, r.URL.Query()); err != nil { + utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) + return + } + + destination := utils.GetVar(r, "destination") + if err := utils.IsRegistryReference(destination); err != nil { + utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, err) + return + } + + authconf, authfile, err := auth.GetCredentials(r) + if err != nil { + utils.Error(w, "failed to retrieve repository credentials", http.StatusBadRequest, errors.Wrapf(err, "failed to parse registry header for %s", r.URL.String())) + return + } + defer auth.RemoveAuthfile(authfile) + var username, password string + if authconf != nil { + username = authconf.Username + password = authconf.Password + } + options := entities.ImagePushOptions{ + Authfile: authfile, + Username: username, + Password: password, + All: query.All, + } + if sys := runtime.SystemContext(); sys != nil { + options.CertDir = sys.DockerCertPath + } + if _, found := r.URL.Query()["tlsVerify"]; found { + options.SkipTLSVerify = types.NewOptionalBool(!query.TLSVerify) + } + + imageEngine := abi.ImageEngine{Libpod: runtime} + source := utils.GetName(r) + digest, err := imageEngine.ManifestPush(context.Background(), source, destination, options) + if err != nil { + utils.Error(w, "Something went wrong.", http.StatusBadRequest, errors.Wrapf(err, "error pushing image %q", destination)) + return + } + utils.WriteResponse(w, http.StatusOK, handlers.IDResponse{ID: digest}) +} + +// ManifestModify efficiently updates the named manifest list +func ManifestModify(w http.ResponseWriter, r *http.Request) { + runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) + imageEngine := abi.ImageEngine{Libpod: runtime} + + body := new(entities.ManifestModifyOptions) + if err := json.NewDecoder(r.Body).Decode(body); err != nil { + utils.Error(w, "Something went wrong.", http.StatusInternalServerError, errors.Wrap(err, "Decode()")) + return + } + + name := utils.GetName(r) + if _, err := runtime.LibimageRuntime().LookupManifestList(name); err != nil { + utils.Error(w, "Something went wrong.", http.StatusNotFound, err) + return + } + + var report entities.ManifestModifyReport + switch { + case strings.EqualFold("update", body.Operation): + id, err := imageEngine.ManifestAdd(r.Context(), name, body.Images, body.ManifestAddOptions) + if err != nil { + report.Errors = append(report.Errors, err) + break + } + report = entities.ManifestModifyReport{ + ID: id, + Images: body.Images, + } + case strings.EqualFold("remove", body.Operation): + for _, image := range body.Images { + id, err := imageEngine.ManifestRemoveDigest(r.Context(), name, image) + if err != nil { + report.Errors = append(report.Errors, err) + continue + } + report.ID = id + report.Images = append(report.Images, image) + } + case strings.EqualFold("annotate", body.Operation): + options := entities.ManifestAnnotateOptions{ + Annotation: body.Annotation, + Arch: body.Arch, + Features: body.Features, + OS: body.OS, + OSFeatures: body.OSFeatures, + OSVersion: body.OSVersion, + Variant: body.Variant, + } + for _, image := range body.Images { + id, err := imageEngine.ManifestAnnotate(r.Context(), name, image, options) + if err != nil { + report.Errors = append(report.Errors, err) + continue + } + report.ID = id + report.Images = append(report.Images, image) + } + default: + utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, + fmt.Errorf("illegal operation %q for %q", body.Operation, r.URL.String())) + return + } + + statusCode := http.StatusOK + switch { + case len(report.Errors) > 0 && len(report.Images) > 0: + statusCode = http.StatusConflict + case len(report.Errors) > 0: + statusCode = http.StatusInternalServerError + } + utils.WriteResponse(w, statusCode, report) +} + +// ManifestDelete removes a manifest list from storage +func ManifestDelete(w http.ResponseWriter, r *http.Request) { + runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) + imageEngine := abi.ImageEngine{Libpod: runtime} + + name := utils.GetName(r) + if _, err := runtime.LibimageRuntime().LookupManifestList(name); err != nil { + utils.Error(w, http.StatusText(http.StatusNotFound), http.StatusNotFound, err) + return + } + + results, errs := imageEngine.ManifestRm(r.Context(), []string{name}) + errsString := errorhandling.ErrorsToStrings(errs) + report := handlers.LibpodImagesRemoveReport{ + ImageRemoveReport: *results, + Errors: errsString, + } + utils.WriteResponse(w, http.StatusOK, report) +} diff --git a/pkg/api/handlers/utils/handler.go b/pkg/api/handlers/utils/handler.go index 96b7a957c..ee83755a1 100644 --- a/pkg/api/handlers/utils/handler.go +++ b/pkg/api/handlers/utils/handler.go @@ -174,7 +174,7 @@ func FilterMapToString(filters map[string][]string) (string, error) { return string(f), nil } -func getVar(r *http.Request, k string) string { +func GetVar(r *http.Request, k string) string { val := mux.Vars(r)[k] safeVal, err := url.PathUnescape(val) if err != nil { @@ -186,5 +186,5 @@ func getVar(r *http.Request, k string) string { // GetName extracts the name from the mux func GetName(r *http.Request) string { - return getVar(r, "name") + return GetVar(r, "name") } diff --git a/pkg/api/server/register_manifest.go b/pkg/api/server/register_manifest.go index 010d8a79e..8cd3d8b22 100644 --- a/pkg/api/server/register_manifest.go +++ b/pkg/api/server/register_manifest.go @@ -8,7 +8,9 @@ import ( ) func (s *APIServer) registerManifestHandlers(r *mux.Router) error { - // swagger:operation POST /libpod/manifests/create manifests ManifestCreateLibpod + v3 := r.PathPrefix("/v{version:[0-3][0-9A-Za-z.-]*}/libpod/manifests").Subrouter() + v4 := r.PathPrefix("/v{version:[4-9][0-9A-Za-z.-]*}/libpod/manifests").Subrouter() + // swagger:operation POST /libpod/manifests manifests ManifestCreateLibpod // --- // summary: Create // description: Create a manifest list @@ -18,18 +20,30 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // - in: query // name: name // type: string - // description: manifest list name + // description: manifest list or index name to create // required: true // - in: query - // name: image + // name: images // type: string - // description: name of the image + // required: true + // description: | + // One or more names of an image or a manifest list. Repeat parameter as needed. + // + // Support for multiple images, as of version 4.0.0 + // Alias of `image` is support for compatibility with < 4.0.0 + // Response status code is 200 with < 4.0.0 for compatibility // - in: query // name: all // type: boolean // description: add all contents if given list + // - in: body + // name: options + // description: options for new manifest + // required: false + // schema: + // $ref: "#/definitions/ManifestModifyOptions" // responses: - // 200: + // 201: // schema: // $ref: "#/definitions/IDResponse" // 400: @@ -38,17 +52,21 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // $ref: "#/responses/NoSuchImage" // 500: // $ref: "#/responses/InternalError" - r.Handle(VersionedPath("/libpod/manifests/create"), s.APIHandler(libpod.ManifestCreate)).Methods(http.MethodPost) + v3.Handle("/create", s.APIHandler(libpod.ManifestCreate)).Methods(http.MethodPost) + v4.Handle("/{name:.*}", s.APIHandler(libpod.ManifestCreate)).Methods(http.MethodPost) // swagger:operation GET /libpod/manifests/{name}/exists manifests ManifestExistsLibpod // --- // summary: Exists - // description: Check if manifest list exists + // description: | + // Check if manifest list exists + // + // Note: There is no contract that the manifest list will exist for a follow-on operation // parameters: // - in: path // name: name // type: string // required: true - // description: the name of the manifest list + // description: the name or ID of the manifest list // produces: // - application/json // responses: @@ -58,11 +76,12 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // $ref: '#/responses/NoSuchManifest' // 500: // $ref: '#/responses/InternalError' - r.Handle(VersionedPath("/libpod/manifests/{name}/exists"), s.APIHandler(libpod.ExistsManifest)).Methods(http.MethodGet) + v3.Handle("/{name:.*}/exists", s.APIHandler(libpod.ManifestExists)).Methods(http.MethodGet) + v4.Handle("/{name:.*}/exists", s.APIHandler(libpod.ManifestExists)).Methods(http.MethodGet) // swagger:operation GET /libpod/manifests/{name}/json manifests ManifestInspectLibpod // --- // summary: Inspect - // description: Display a manifest list + // description: Display attributes of given manifest list // produces: // - application/json // parameters: @@ -70,7 +89,7 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // name: name // type: string // required: true - // description: the name or ID of the manifest + // description: the name or ID of the manifest list // responses: // 200: // $ref: "#/responses/InspectManifest" @@ -78,11 +97,53 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // $ref: "#/responses/NoSuchManifest" // 500: // $ref: "#/responses/InternalError" - r.Handle(VersionedPath("/libpod/manifests/{name:.*}/json"), s.APIHandler(libpod.ManifestInspect)).Methods(http.MethodGet) + v3.Handle("/{name:.*}/json", s.APIHandler(libpod.ManifestInspect)).Methods(http.MethodGet) + v4.Handle("/{name:.*}/json", s.APIHandler(libpod.ManifestInspect)).Methods(http.MethodGet) + // swagger:operation PUT /libpod/manifests/{name} manifests ManifestModifyLibpod + // --- + // summary: Modify manifest list + // description: | + // Add/Remove an image(s) to a manifest list + // + // Note: operations are not atomic when multiple Images are provided. + // + // As of v4.0.0 + // produces: + // - application/json + // parameters: + // - in: path + // name: name + // type: string + // required: true + // description: the name or ID of the manifest + // - in: body + // name: options + // description: options for mutating a manifest + // required: true + // schema: + // $ref: "#/definitions/ManifestModifyOptions" + // responses: + // 200: + // schema: + // $ref: "#/definitions/ManifestModifyReport" + // 404: + // $ref: "#/responses/NoSuchManifest" + // 400: + // $ref: "#/responses/BadParamError" + // 409: + // description: Operation had partial success, both Images and Errors may have members + // schema: + // $ref: "#/definitions/ManifestModifyReport" + // 500: + // $ref: "#/responses/InternalError" + v4.Handle("/{name:.*}", s.APIHandler(libpod.ManifestModify)).Methods(http.MethodPut) // swagger:operation POST /libpod/manifests/{name}/add manifests ManifestAddLibpod // --- // summary: Add image - // description: Add an image to a manifest list + // description: | + // Add an image to a manifest list + // + // Deprecated: As of 4.0.0 use ManifestModifyLibpod instead // produces: // - application/json // parameters: @@ -95,7 +156,7 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // name: options // description: options for creating a manifest // schema: - // $ref: "#/definitions/ManifestAddOpts" + // $ref: "#/definitions/ManifestAddOptions" // responses: // 200: // schema: @@ -106,11 +167,14 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // $ref: "#/responses/BadParamError" // 500: // $ref: "#/responses/InternalError" - r.Handle(VersionedPath("/libpod/manifests/{name:.*}/add"), s.APIHandler(libpod.ManifestAdd)).Methods(http.MethodPost) - // swagger:operation DELETE /libpod/manifests/{name} manifests ManifestDeleteLibpod + v3.Handle("/{name:.*}/add", s.APIHandler(libpod.ManifestAdd)).Methods(http.MethodPost) + // swagger:operation DELETE /libpod/manifests/{name} manifests ManifestDeleteV3Libpod // --- - // summary: Remove - // description: Remove an image from a manifest list + // summary: Remove image from a manifest list + // description: | + // Remove an image from a manifest list + // + // Deprecated: As of 4.0.0 use ManifestModifyLibpod instead // produces: // - application/json // parameters: @@ -133,11 +197,37 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // $ref: "#/responses/NoSuchManifest" // 500: // $ref: "#/responses/InternalError" - r.Handle(VersionedPath("/libpod/manifests/{name:.*}"), s.APIHandler(libpod.ManifestRemove)).Methods(http.MethodDelete) - // swagger:operation POST /libpod/manifests/{name}/push manifests ManifestPushLibpod + v3.Handle("/{name:.*}", s.APIHandler(libpod.ManifestRemoveDigest)).Methods(http.MethodDelete) + // swagger:operation DELETE /libpod/manifests/{name} manifests ManifestDeleteLibpod // --- - // summary: Push - // description: Push a manifest list or image index to a registry + // summary: Delete manifest list + // description: | + // Delete named manifest list + // + // As of v4.0.0 + // produces: + // - application/json + // parameters: + // - in: path + // name: name + // type: string + // required: true + // description: The name or ID of the list to be deleted + // responses: + // 200: + // $ref: "#/responses/DocsLibpodImagesRemoveResponse" + // 404: + // $ref: "#/responses/NoSuchManifest" + // 500: + // $ref: "#/responses/InternalError" + v4.Handle("/{name:.*}", s.APIHandler(libpod.ManifestDelete)).Methods(http.MethodDelete) + // swagger:operation POST /libpod/manifests/{name}/push manifests ManifestPushV3Libpod + // --- + // summary: Push manifest to registry + // description: | + // Push a manifest list or image index to a registry + // + // Deprecated: As of 4.0.0 use ManifestPushLibpod instead // produces: // - application/json // parameters: @@ -165,6 +255,47 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // $ref: "#/responses/NoSuchManifest" // 500: // $ref: "#/responses/InternalError" - r.Handle(VersionedPath("/libpod/manifests/{name}/push"), s.APIHandler(libpod.ManifestPush)).Methods(http.MethodPost) + v3.Handle("/{name}/push", s.APIHandler(libpod.ManifestPushV3)).Methods(http.MethodPost) + // swagger:operation POST /libpod/manifests/{name}/registry/{destination} manifests ManifestPushLibpod + // --- + // summary: Push manifest list to registry + // description: | + // Push a manifest list or image index to the named registry + // + // As of v4.0.0 + // produces: + // - application/json + // parameters: + // - in: path + // name: name + // type: string + // required: true + // description: the name or ID of the manifest list + // - in: path + // name: destination + // type: string + // required: true + // description: the registry for the manifest list + // - in: query + // name: all + // description: push all images + // type: boolean + // default: false + // - in: query + // name: tlsVerify + // type: boolean + // default: false + // description: skip TLS verification for registries + // responses: + // 200: + // schema: + // $ref: "#/definitions/IDResponse" + // 400: + // $ref: "#/responses/BadParamError" + // 404: + // $ref: "#/responses/NoSuchManifest" + // 500: + // $ref: "#/responses/InternalError" + v4.Handle("/{name:.*}/registry/{destination:.*}", s.APIHandler(libpod.ManifestPush)).Methods(http.MethodPost) return nil } diff --git a/pkg/api/server/register_swagger.go b/pkg/api/server/register_swagger.go index dca1df14b..48af7713f 100644 --- a/pkg/api/server/register_swagger.go +++ b/pkg/api/server/register_swagger.go @@ -7,8 +7,8 @@ import ( "github.com/gorilla/mux" ) -// RegisterSwaggerHandlers maps the swagger endpoint for the server -func (s *APIServer) RegisterSwaggerHandlers(r *mux.Router) error { +// registerSwaggerHandlers maps the swagger endpoint for the server +func (s *APIServer) registerSwaggerHandlers(r *mux.Router) error { // This handler does _*NOT*_ provide an UI rather just a swagger spec that an UI could render r.HandleFunc(VersionedPath("/libpod/swagger"), s.APIHandler(libpod.ServeSwagger)).Methods(http.MethodGet) return nil diff --git a/pkg/api/server/server.go b/pkg/api/server/server.go index 8c5c7aeeb..65b7e2474 100644 --- a/pkg/api/server/server.go +++ b/pkg/api/server/server.go @@ -151,7 +151,7 @@ func newServer(runtime *libpod.Runtime, listener *net.Listener, opts entities.Se server.registerPluginsHandlers, server.registerPodsHandlers, server.registerSecretHandlers, - server.RegisterSwaggerHandlers, + server.registerSwaggerHandlers, server.registerSwarmHandlers, server.registerSystemHandlers, server.registerVersionHandlers, diff --git a/pkg/auth/auth.go b/pkg/auth/auth.go index f423c011d..419225007 100644 --- a/pkg/auth/auth.go +++ b/pkg/auth/auth.go @@ -140,7 +140,7 @@ func getAuthCredentials(headers []string) (*types.DockerAuthConfig, map[string]t // MakeXRegistryConfigHeader returns a map with the "X-Registry-Config" header set, which can // conveniently be used in the http stack. -func MakeXRegistryConfigHeader(sys *types.SystemContext, username, password string) (map[string]string, error) { +func MakeXRegistryConfigHeader(sys *types.SystemContext, username, password string) (http.Header, error) { if sys == nil { sys = &types.SystemContext{} } @@ -163,18 +163,18 @@ func MakeXRegistryConfigHeader(sys *types.SystemContext, username, password stri if err != nil { return nil, err } - return map[string]string{xRegistryConfigHeader: content}, nil + return http.Header{xRegistryConfigHeader: []string{content}}, nil } // MakeXRegistryAuthHeader returns a map with the "X-Registry-Auth" header set, which can // conveniently be used in the http stack. -func MakeXRegistryAuthHeader(sys *types.SystemContext, username, password string) (map[string]string, error) { +func MakeXRegistryAuthHeader(sys *types.SystemContext, username, password string) (http.Header, error) { if username != "" { content, err := encodeSingleAuthConfig(types.DockerAuthConfig{Username: username, Password: password}) if err != nil { return nil, err } - return map[string]string{xRegistryAuthHeader: content}, nil + return http.Header{xRegistryAuthHeader: []string{content}}, nil } if sys == nil { @@ -188,7 +188,7 @@ func MakeXRegistryAuthHeader(sys *types.SystemContext, username, password string if err != nil { return nil, err } - return map[string]string{xRegistryAuthHeader: content}, nil + return http.Header{xRegistryAuthHeader: []string{content}}, nil } // RemoveAuthfile is a convenience function that is meant to be called in a diff --git a/pkg/auth/auth_test.go b/pkg/auth/auth_test.go index f7e6e4ef6..2c79f0b7c 100644 --- a/pkg/auth/auth_test.go +++ b/pkg/auth/auth_test.go @@ -85,8 +85,8 @@ func TestMakeXRegistryConfigHeaderGetCredentialsRoundtrip(t *testing.T) { require.NoError(t, err) req, err := http.NewRequest(http.MethodPost, "/", nil) require.NoError(t, err, tc.name) - for k, v := range headers { - req.Header.Set(k, v) + for _, v := range headers.Values(xRegistryConfigHeader) { + req.Header.Add(xRegistryConfigHeader, v) } override, resPath, err := GetCredentials(req) @@ -137,8 +137,8 @@ func TestMakeXRegistryAuthHeaderGetCredentialsRoundtrip(t *testing.T) { require.NoError(t, err) req, err := http.NewRequest(http.MethodPost, "/", nil) require.NoError(t, err, tc.name) - for k, v := range headers { - req.Header.Set(k, v) + for _, v := range headers.Values(xRegistryAuthHeader) { + req.Header.Set(xRegistryAuthHeader, v) } override, resPath, err := GetCredentials(req) @@ -219,7 +219,7 @@ func TestMakeXRegistryConfigHeader(t *testing.T) { require.Len(t, res, 1, tc.name) header, ok := res[xRegistryConfigHeader] require.True(t, ok, tc.name) - decodedHeader, err := base64.URLEncoding.DecodeString(header) + decodedHeader, err := base64.URLEncoding.DecodeString(header[0]) require.NoError(t, err, tc.name) // Don't test for a specific JSON representation, just for the expected contents. expected := map[string]interface{}{} @@ -282,7 +282,7 @@ func TestMakeXRegistryAuthHeader(t *testing.T) { require.Len(t, res, 1, tc.name) header, ok := res[xRegistryAuthHeader] require.True(t, ok, tc.name) - decodedHeader, err := base64.URLEncoding.DecodeString(header) + decodedHeader, err := base64.URLEncoding.DecodeString(header[0]) require.NoError(t, err, tc.name) // Don't test for a specific JSON representation, just for the expected contents. expected := map[string]interface{}{} diff --git a/pkg/bindings/connection.go b/pkg/bindings/connection.go index b2e949f67..332aa97c8 100644 --- a/pkg/bindings/connection.go +++ b/pkg/bindings/connection.go @@ -15,7 +15,6 @@ import ( "github.com/blang/semver" "github.com/containers/podman/v3/pkg/terminal" "github.com/containers/podman/v3/version" - jsoniter "github.com/json-iterator/go" "github.com/pkg/errors" "github.com/sirupsen/logrus" "golang.org/x/crypto/ssh" @@ -35,16 +34,24 @@ type Connection struct { type valueKey string const ( - clientKey = valueKey("Client") + clientKey = valueKey("Client") + versionKey = valueKey("ServiceVersion") ) // GetClient from context build by NewConnection() func GetClient(ctx context.Context) (*Connection, error) { - c, ok := ctx.Value(clientKey).(*Connection) - if !ok { - return nil, errors.Errorf("ClientKey not set in context") + if c, ok := ctx.Value(clientKey).(*Connection); ok { + return c, nil } - return c, nil + return nil, errors.Errorf("%s not set in context", clientKey) +} + +// ServiceVersion from context build by NewConnection() +func ServiceVersion(ctx context.Context) *semver.Version { + if v, ok := ctx.Value(versionKey).(*semver.Version); ok { + return v + } + return new(semver.Version) } // JoinURL elements with '/' @@ -52,6 +59,7 @@ func JoinURL(elements ...string) string { return "/" + strings.Join(elements, "/") } +// NewConnection creates a new service connection without an identity func NewConnection(ctx context.Context, uri string) (context.Context, error) { return NewConnectionWithIdentity(ctx, uri, "") } @@ -116,9 +124,11 @@ func NewConnectionWithIdentity(ctx context.Context, uri string, identity string) } ctx = context.WithValue(ctx, clientKey, &connection) - if err := pingNewConnection(ctx); err != nil { + serviceVersion, err := pingNewConnection(ctx) + if err != nil { return nil, errors.Wrap(err, "unable to connect to Podman socket") } + ctx = context.WithValue(ctx, versionKey, serviceVersion) return ctx, nil } @@ -139,15 +149,15 @@ func tcpClient(_url *url.URL) Connection { // pingNewConnection pings to make sure the RESTFUL service is up // and running. it should only be used when initializing a connection -func pingNewConnection(ctx context.Context) error { +func pingNewConnection(ctx context.Context) (*semver.Version, error) { client, err := GetClient(ctx) if err != nil { - return err + return nil, err } // the ping endpoint sits at / in this case response, err := client.DoRequest(ctx, nil, http.MethodGet, "/_ping", nil, nil) if err != nil { - return err + return nil, err } defer response.Body.Close() @@ -155,23 +165,23 @@ func pingNewConnection(ctx context.Context) error { versionHdr := response.Header.Get("Libpod-API-Version") if versionHdr == "" { logrus.Info("Service did not provide Libpod-API-Version Header") - return nil + return new(semver.Version), nil } versionSrv, err := semver.ParseTolerant(versionHdr) if err != nil { - return err + return nil, err } switch version.APIVersion[version.Libpod][version.MinimalAPI].Compare(versionSrv) { case -1, 0: // Server's job when Client version is equal or older - return nil + return &versionSrv, nil case 1: - return errors.Errorf("server API version is too old. Client %q server %q", + return nil, errors.Errorf("server API version is too old. Client %q server %q", version.APIVersion[version.Libpod][version.MinimalAPI].String(), versionSrv.String()) } } - return errors.Errorf("ping response was %d", response.StatusCode) + return nil, errors.Errorf("ping response was %d", response.StatusCode) } func sshClient(_url *url.URL, secure bool, passPhrase string, identity string) (Connection, error) { @@ -306,26 +316,29 @@ func unixClient(_url *url.URL) Connection { } // DoRequest assembles the http request and returns the response -func (c *Connection) DoRequest(ctx context.Context, httpBody io.Reader, httpMethod, endpoint string, queryParams url.Values, header map[string]string, pathValues ...string) (*APIResponse, error) { +func (c *Connection) DoRequest(ctx context.Context, httpBody io.Reader, httpMethod, endpoint string, queryParams url.Values, headers http.Header, pathValues ...string) (*APIResponse, error) { var ( err error response *http.Response ) - params := make([]interface{}, len(pathValues)+3) + params := make([]interface{}, len(pathValues)+1) + + if v := headers.Values("API-Version"); len(v) > 0 { + params[0] = v[0] + } else { + // Including the semver suffices breaks older services... so do not include them + v := version.APIVersion[version.Libpod][version.CurrentAPI] + params[0] = fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch) + } - // Including the semver suffices breaks older services... so do not include them - v := version.APIVersion[version.Libpod][version.CurrentAPI] - params[0] = v.Major - params[1] = v.Minor - params[2] = v.Patch for i, pv := range pathValues { // url.URL lacks the semantics for escaping embedded path parameters... so we manually // escape each one and assume the caller included the correct formatting in "endpoint" - params[i+3] = url.PathEscape(pv) + params[i+1] = url.PathEscape(pv) } - uri := fmt.Sprintf("http://d/v%d.%d.%d/libpod"+endpoint, params...) + uri := fmt.Sprintf("http://d/v%s/libpod"+endpoint, params...) logrus.Debugf("DoRequest Method: %s URI: %v", httpMethod, uri) req, err := http.NewRequestWithContext(ctx, httpMethod, uri, httpBody) @@ -335,9 +348,17 @@ func (c *Connection) DoRequest(ctx context.Context, httpBody io.Reader, httpMeth if len(queryParams) > 0 { req.URL.RawQuery = queryParams.Encode() } - for key, val := range header { - req.Header.Set(key, val) + + for key, val := range headers { + if key == "API-Version" { + continue + } + + for _, v := range val { + req.Header.Add(key, v) + } } + // Give the Do three chances in the case of a comm/service hiccup for i := 1; i <= 3; i++ { response, err = c.Client.Do(req) // nolint @@ -349,7 +370,7 @@ func (c *Connection) DoRequest(ctx context.Context, httpBody io.Reader, httpMeth return &APIResponse{response, req}, err } -// Get raw Transport.DialContext from client +// GetDialer returns raw Transport.DialContext from client func (c *Connection) GetDialer(ctx context.Context) (net.Conn, error) { client := c.Client transport := client.Transport.(*http.Transport) @@ -360,16 +381,6 @@ func (c *Connection) GetDialer(ctx context.Context) (net.Conn, error) { return nil, errors.New("Unable to get dial context") } -// FiltersToString converts our typical filter format of a -// map[string][]string to a query/html safe string. -func FiltersToString(filters map[string][]string) (string, error) { - lowerCaseKeys := make(map[string][]string) - for k, v := range filters { - lowerCaseKeys[strings.ToLower(k)] = v - } - return jsoniter.MarshalToString(lowerCaseKeys) -} - // IsInformational returns true if the response code is 1xx func (h *APIResponse) IsInformational() bool { return h.Response.StatusCode/100 == 1 diff --git a/pkg/bindings/containers/attach.go b/pkg/bindings/containers/attach.go index baa3f182e..c02265cd8 100644 --- a/pkg/bindings/containers/attach.go +++ b/pkg/bindings/containers/attach.go @@ -108,9 +108,9 @@ func Attach(ctx context.Context, nameOrID string, stdin io.Reader, stdout io.Wri }() } - headers := make(map[string]string) - headers["Connection"] = "Upgrade" - headers["Upgrade"] = "tcp" + headers := make(http.Header) + headers.Add("Connection", "Upgrade") + headers.Add("Upgrade", "tcp") var socket net.Conn socketSet := false @@ -157,7 +157,7 @@ func Attach(ctx context.Context, nameOrID string, stdin io.Reader, stdout io.Wri } stdoutChan := make(chan error) - stdinChan := make(chan error, 1) //stdin channel should not block + stdinChan := make(chan error, 1) // stdin channel should not block if isSet.stdin { go func() { diff --git a/pkg/bindings/errors.go b/pkg/bindings/errors.go index ec837b39c..be184b916 100644 --- a/pkg/bindings/errors.go +++ b/pkg/bindings/errors.go @@ -25,7 +25,7 @@ func (h APIResponse) Process(unmarshalInto interface{}) error { return h.ProcessWithError(unmarshalInto, &errorhandling.ErrorModel{}) } -// Process drains the response body, and processes the HTTP status code +// ProcessWithError drains the response body, and processes the HTTP status code // Note: Closing the response.Body is left to the caller func (h APIResponse) ProcessWithError(unmarshalInto interface{}, unmarshalErrorInto interface{}) error { data, err := ioutil.ReadAll(h.Response.Body) diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go index 6b5159f52..9880c73e4 100644 --- a/pkg/bindings/images/build.go +++ b/pkg/bindings/images/build.go @@ -300,7 +300,7 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO } var ( - headers map[string]string + headers http.Header err error ) if options.SystemContext != nil && options.SystemContext.DockerAuthConfig != nil { @@ -421,7 +421,7 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO return nil, err } - //add tmp file to context dir + // add tmp file to context dir tarContent = append(tarContent, tmpSecretFile.Name()) modifiedSrc := fmt.Sprintf("src=%s", filepath.Base(tmpSecretFile.Name())) @@ -634,7 +634,7 @@ func nTar(excludes []string, sources ...string) (io.ReadCloser, error) { if lerr := tw.WriteHeader(hdr); lerr != nil { return lerr } - } //skip other than file,folder and symlinks + } // skip other than file,folder and symlinks return nil }) merr = multierror.Append(merr, err) diff --git a/pkg/bindings/internal/util/util.go b/pkg/bindings/internal/util/util.go index bcf6959f2..f8f99d6c1 100644 --- a/pkg/bindings/internal/util/util.go +++ b/pkg/bindings/internal/util/util.go @@ -104,3 +104,11 @@ func ToParams(o interface{}) (url.Values, error) { } return params, nil } + +func MapToArrayString(data map[string]string) []string { + l := make([]string, 0) + for k, v := range data { + l = append(l, k+"="+v) + } + return l +} diff --git a/pkg/bindings/manifests/manifests.go b/pkg/bindings/manifests/manifests.go index af74eb406..50e324efa 100644 --- a/pkg/bindings/manifests/manifests.go +++ b/pkg/bindings/manifests/manifests.go @@ -3,15 +3,18 @@ package manifests import ( "context" "errors" + "fmt" "net/http" "net/url" "strconv" "strings" + "github.com/blang/semver" "github.com/containers/image/v5/manifest" "github.com/containers/podman/v3/pkg/api/handlers" "github.com/containers/podman/v3/pkg/bindings" "github.com/containers/podman/v3/pkg/bindings/images" + "github.com/containers/podman/v3/version" jsoniter "github.com/json-iterator/go" ) @@ -19,7 +22,7 @@ import ( // the new manifest can also be specified. The all boolean specifies to add all entries // of a list if the name provided is a manifest list. The ID of the new manifest list // is returned as a string. -func Create(ctx context.Context, names, images []string, options *CreateOptions) (string, error) { +func Create(ctx context.Context, name string, images []string, options *CreateOptions) (string, error) { var idr handlers.IDResponse if options == nil { options = new(CreateOptions) @@ -28,21 +31,19 @@ func Create(ctx context.Context, names, images []string, options *CreateOptions) if err != nil { return "", err } - if len(names) < 1 { + if len(name) < 1 { return "", errors.New("creating a manifest requires at least one name argument") } params, err := options.ToParams() if err != nil { return "", err } - for _, name := range names { - params.Add("name", name) - } + for _, i := range images { - params.Add("image", i) + params.Add("images", i) } - response, err := conn.DoRequest(ctx, nil, http.MethodPost, "/manifests/create", params, nil) + response, err := conn.DoRequest(ctx, nil, http.MethodPost, "/manifests/%s", params, nil, name) if err != nil { return "", err } @@ -67,70 +68,96 @@ func Exists(ctx context.Context, name string, options *ExistsOptions) (bool, err } // Inspect returns a manifest list for a given name. -func Inspect(ctx context.Context, name string, options *InspectOptions) (*manifest.Schema2List, error) { - var list manifest.Schema2List - if options == nil { - options = new(InspectOptions) - } - _ = options +func Inspect(ctx context.Context, name string, _ *InspectOptions) (*manifest.Schema2List, error) { conn, err := bindings.GetClient(ctx) if err != nil { return nil, err } + response, err := conn.DoRequest(ctx, nil, http.MethodGet, "/manifests/%s/json", nil, nil, name) if err != nil { return nil, err } defer response.Body.Close() + var list manifest.Schema2List return &list, response.Process(&list) } // Add adds a manifest to a given manifest list. Additional options for the manifest // can also be specified. The ID of the new manifest list is returned as a string func Add(ctx context.Context, name string, options *AddOptions) (string, error) { - var idr handlers.IDResponse if options == nil { options = new(AddOptions) } + + if bindings.ServiceVersion(ctx).GTE(semver.MustParse("4.0.0")) { + optionsv4 := ModifyOptions{ + All: options.All, + Annotations: options.Annotation, + Arch: options.Arch, + Features: options.Features, + Images: options.Images, + OS: options.OS, + OSFeatures: nil, + OSVersion: options.OSVersion, + Variant: options.Variant, + } + optionsv4.WithOperation("update") + return Modify(ctx, name, options.Images, &optionsv4) + } + + // API Version < 4.0.0 conn, err := bindings.GetClient(ctx) if err != nil { return "", err } - optionsString, err := jsoniter.MarshalToString(options) + opts, err := jsoniter.MarshalToString(options) if err != nil { return "", err } - stringReader := strings.NewReader(optionsString) - response, err := conn.DoRequest(ctx, stringReader, http.MethodPost, "/manifests/%s/add", nil, nil, name) + reader := strings.NewReader(opts) + + headers := make(http.Header) + v := version.APIVersion[version.Libpod][version.MinimalAPI] + headers.Add("API-Version", + fmt.Sprintf("%d.%d.%d", v.Major, v.Minor, v.Patch)) + response, err := conn.DoRequest(ctx, reader, http.MethodPost, "/manifests/%s/add", nil, headers, name) if err != nil { return "", err } defer response.Body.Close() + var idr handlers.IDResponse return idr.ID, response.Process(&idr) } // Remove deletes a manifest entry from a manifest list. Both name and the digest to be // removed are mandatory inputs. The ID of the new manifest list is returned as a string. -func Remove(ctx context.Context, name, digest string, options *RemoveOptions) (string, error) { - var idr handlers.IDResponse - if options == nil { - options = new(RemoveOptions) +func Remove(ctx context.Context, name, digest string, _ *RemoveOptions) (string, error) { + if bindings.ServiceVersion(ctx).GTE(semver.MustParse("4.0.0")) { + optionsv4 := new(ModifyOptions).WithOperation("remove") + return Modify(ctx, name, []string{digest}, optionsv4) } - _ = options + + // API Version < 4.0.0 conn, err := bindings.GetClient(ctx) if err != nil { return "", err } + + headers := http.Header{} + headers.Add("API-Version", "3.4.0") + params := url.Values{} params.Set("digest", digest) - response, err := conn.DoRequest(ctx, nil, http.MethodDelete, "/manifests/%s", params, nil, name) + response, err := conn.DoRequest(ctx, nil, http.MethodDelete, "/manifests/%s", params, headers, name) if err != nil { return "", err } defer response.Body.Close() + var idr handlers.IDResponse return idr.ID, response.Process(&idr) } @@ -151,19 +178,26 @@ func Push(ctx context.Context, name, destination string, options *images.PushOpt if err != nil { return "", err } + params, err := options.ToParams() if err != nil { return "", err } // SkipTLSVerify is special. We need to delete the param added by - // toparams and change the key and flip the bool + // ToParams() and change the key and flip the bool if options.SkipTLSVerify != nil { params.Del("SkipTLSVerify") params.Set("tlsVerify", strconv.FormatBool(!options.GetSkipTLSVerify())) } - params.Set("image", name) - params.Set("destination", destination) - response, err := conn.DoRequest(ctx, nil, http.MethodPost, "/manifests/%s/push", params, nil, name) + + var response *bindings.APIResponse + if bindings.ServiceVersion(ctx).GTE(semver.MustParse("4.0.0")) { + response, err = conn.DoRequest(ctx, nil, http.MethodPost, "/manifests/%s/registry/%s", params, nil, name, destination) + } else { + params.Set("image", name) + params.Set("destination", destination) + response, err = conn.DoRequest(ctx, nil, http.MethodPost, "/manifests/%s/push", params, nil, name) + } if err != nil { return "", err } @@ -172,25 +206,37 @@ func Push(ctx context.Context, name, destination string, options *images.PushOpt return idr.ID, err } -// There is NO annotate endpoint. this binding could never work -// Annotate updates the image configuration of a given manifest list -//func Annotate(ctx context.Context, name, digest string, options image.ManifestAnnotateOpts) (string, error) { -// var idr handlers.IDResponse -// conn, err := bindings.GetClient(ctx) -// if err != nil { -// return "", err -// } -// params := url.Values{} -// params.Set("digest", digest) -// optionsString, err := jsoniter.MarshalToString(options) -// if err != nil { -// return "", err -// } -// stringReader := strings.NewReader(optionsString) -// response, err := conn.DoRequest(ctx, stringReader, http.MethodPost, "/manifests/%s/annotate", params, name) -// if err != nil { -// return "", err -// } -// defer response.Body.Close() -// return idr.ID, response.Process(&idr) -//} +// Modify modifies the given manifest list using options and the optional list of images +func Modify(ctx context.Context, name string, images []string, options *ModifyOptions) (string, error) { + if options == nil || *options.Operation == "" { + return "", errors.New(`the field ModifyOptions.Operation must be set to either "update" or "remove"`) + } + options.WithImages(images) + + conn, err := bindings.GetClient(ctx) + if err != nil { + return "", err + } + opts, err := jsoniter.MarshalToString(options) + if err != nil { + return "", err + } + reader := strings.NewReader(opts) + + response, err := conn.DoRequest(ctx, reader, http.MethodPut, "/manifests/%s", nil, nil, name) + if err != nil { + return "", err + } + defer response.Body.Close() + + var idr handlers.IDResponse + return idr.ID, response.Process(&idr) +} + +// Annotate modifies the given manifest list using options and the optional list of images +// +// As of 4.0.0 +func Annotate(ctx context.Context, name string, images []string, options *ModifyOptions) (string, error) { + options.WithOperation("annotate") + return Modify(ctx, name, images, options) +} diff --git a/pkg/bindings/manifests/types.go b/pkg/bindings/manifests/types.go index fde90a865..5ff28ee30 100644 --- a/pkg/bindings/manifests/types.go +++ b/pkg/bindings/manifests/types.go @@ -18,7 +18,7 @@ type ExistsOptions struct { } //go:generate go run ../generator/generator.go AddOptions -// AddOptions are optional options for adding manifests +// AddOptions are optional options for adding manifest lists type AddOptions struct { All *bool Annotation map[string]string @@ -31,6 +31,24 @@ type AddOptions struct { } //go:generate go run ../generator/generator.go RemoveOptions -// RemoveOptions are optional options for removing manifests +// RemoveOptions are optional options for removing manifest lists type RemoveOptions struct { } + +//go:generate go run ../generator/generator.go ModifyOptions +// ModifyOptions are optional options for modifying manifest lists +type ModifyOptions struct { + // Operation values are "update", "remove" and "annotate". This allows the service to + // efficiently perform each update on a manifest list. + Operation *string + All *bool // All when true, operate on all images in a manifest list that may be included in Images + Annotations map[string]string // Annotations to add to manifest list + Arch *string // Arch overrides the architecture for the image + Features []string // Feature list for the image + Images []string // Images is an optional list of images to add/remove to/from manifest list depending on operation + OS *string // OS overrides the operating system for the image + OSFeatures []string // OS features for the image + OSVersion *string // OSVersion overrides the operating system for the image + Variant *string // Variant overrides the operating system variant for the image + +} diff --git a/pkg/bindings/manifests/types_modify_options.go b/pkg/bindings/manifests/types_modify_options.go new file mode 100644 index 000000000..ee5d94dbf --- /dev/null +++ b/pkg/bindings/manifests/types_modify_options.go @@ -0,0 +1,168 @@ +// Code generated by go generate; DO NOT EDIT. +package manifests + +import ( + "net/url" + + "github.com/containers/podman/v3/pkg/bindings/internal/util" +) + +// Changed returns true if named field has been set +func (o *ModifyOptions) Changed(fieldName string) bool { + return util.Changed(o, fieldName) +} + +// ToParams formats struct fields to be passed to API service +func (o *ModifyOptions) ToParams() (url.Values, error) { + return util.ToParams(o) +} + +// WithOperation set field Operation to given value +func (o *ModifyOptions) WithOperation(value string) *ModifyOptions { + o.Operation = &value + return o +} + +// GetOperation returns value of field Operation +func (o *ModifyOptions) GetOperation() string { + if o.Operation == nil { + var z string + return z + } + return *o.Operation +} + +// WithAll set all when true, operate on all images in a manifest list that may be included in Images +func (o *ModifyOptions) WithAll(value bool) *ModifyOptions { + o.All = &value + return o +} + +// GetAll returns value of all when true, operate on all images in a manifest list that may be included in Images +func (o *ModifyOptions) GetAll() bool { + if o.All == nil { + var z bool + return z + } + return *o.All +} + +// WithAnnotations set annotations to add to manifest list +func (o *ModifyOptions) WithAnnotations(value map[string]string) *ModifyOptions { + o.Annotations = value + return o +} + +// GetAnnotations returns value of annotations to add to manifest list +func (o *ModifyOptions) GetAnnotations() map[string]string { + if o.Annotations == nil { + var z map[string]string + return z + } + return o.Annotations +} + +// WithArch set arch overrides the architecture for the image +func (o *ModifyOptions) WithArch(value string) *ModifyOptions { + o.Arch = &value + return o +} + +// GetArch returns value of arch overrides the architecture for the image +func (o *ModifyOptions) GetArch() string { + if o.Arch == nil { + var z string + return z + } + return *o.Arch +} + +// WithFeatures set feature list for the image +func (o *ModifyOptions) WithFeatures(value []string) *ModifyOptions { + o.Features = value + return o +} + +// GetFeatures returns value of feature list for the image +func (o *ModifyOptions) GetFeatures() []string { + if o.Features == nil { + var z []string + return z + } + return o.Features +} + +// WithImages set images is an optional list of images to add/remove to/from manifest list depending on operation +func (o *ModifyOptions) WithImages(value []string) *ModifyOptions { + o.Images = value + return o +} + +// GetImages returns value of images is an optional list of images to add/remove to/from manifest list depending on operation +func (o *ModifyOptions) GetImages() []string { + if o.Images == nil { + var z []string + return z + } + return o.Images +} + +// WithOS set oS overrides the operating system for the image +func (o *ModifyOptions) WithOS(value string) *ModifyOptions { + o.OS = &value + return o +} + +// GetOS returns value of oS overrides the operating system for the image +func (o *ModifyOptions) GetOS() string { + if o.OS == nil { + var z string + return z + } + return *o.OS +} + +// WithOSFeatures set oS features for the image +func (o *ModifyOptions) WithOSFeatures(value []string) *ModifyOptions { + o.OSFeatures = value + return o +} + +// GetOSFeatures returns value of oS features for the image +func (o *ModifyOptions) GetOSFeatures() []string { + if o.OSFeatures == nil { + var z []string + return z + } + return o.OSFeatures +} + +// WithOSVersion set oSVersion overrides the operating system for the image +func (o *ModifyOptions) WithOSVersion(value string) *ModifyOptions { + o.OSVersion = &value + return o +} + +// GetOSVersion returns value of oSVersion overrides the operating system for the image +func (o *ModifyOptions) GetOSVersion() string { + if o.OSVersion == nil { + var z string + return z + } + return *o.OSVersion +} + +// WithVariant set variant overrides the operating system variant for the image +func (o *ModifyOptions) WithVariant(value string) *ModifyOptions { + o.Variant = &value + return o +} + +// GetVariant returns value of variant overrides the operating system variant for the image +func (o *ModifyOptions) GetVariant() string { + if o.Variant == nil { + var z string + return z + } + return *o.Variant +} diff --git a/pkg/bindings/test/manifests_test.go b/pkg/bindings/test/manifests_test.go index e65632057..280006d15 100644 --- a/pkg/bindings/test/manifests_test.go +++ b/pkg/bindings/test/manifests_test.go @@ -12,7 +12,7 @@ import ( "github.com/onsi/gomega/gexec" ) -var _ = Describe("Podman containers ", func() { +var _ = Describe("podman manifest", func() { var ( bt *bindingTest s *gexec.Session @@ -24,7 +24,8 @@ var _ = Describe("Podman containers ", func() { s = bt.startAPIService() time.Sleep(1 * time.Second) err := bt.NewConnection() - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) + }) AfterEach(func() { @@ -32,17 +33,19 @@ var _ = Describe("Podman containers ", func() { bt.cleanup() }) - It("create manifest", func() { + It("create", func() { // create manifest list without images - id, err := manifests.Create(bt.conn, []string{"quay.io/libpod/foobar:latest"}, []string{}, nil) - Expect(err).To(BeNil()) + id, err := manifests.Create(bt.conn, "quay.io/libpod/foobar:latest", []string{}, nil) + Expect(err).ToNot(HaveOccurred(), err) list, err := manifests.Inspect(bt.conn, id, nil) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) + Expect(len(list.Manifests)).To(BeZero()) // creating a duplicate should fail as a 500 - _, err = manifests.Create(bt.conn, []string{"quay.io/libpod/foobar:latest"}, []string{}, nil) - Expect(err).ToNot(BeNil()) + _, err = manifests.Create(bt.conn, "quay.io/libpod/foobar:latest", nil, nil) + Expect(err).To(HaveOccurred()) + code, _ := bindings.CheckResponseCode(err) Expect(code).To(BeNumerically("==", http.StatusInternalServerError)) @@ -50,96 +53,113 @@ var _ = Describe("Podman containers ", func() { Expect(len(errs)).To(BeZero()) // create manifest list with images - id, err = manifests.Create(bt.conn, []string{"quay.io/libpod/foobar:latest"}, []string{alpine.name}, nil) - Expect(err).To(BeNil()) + id, err = manifests.Create(bt.conn, "quay.io/libpod/foobar:latest", []string{alpine.name}, nil) + Expect(err).ToNot(HaveOccurred()) + list, err = manifests.Inspect(bt.conn, id, nil) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) + Expect(len(list.Manifests)).To(BeNumerically("==", 1)) }) - It("inspect bogus manifest", func() { + It("inspect", func() { _, err := manifests.Inspect(bt.conn, "larry", nil) - Expect(err).ToNot(BeNil()) + Expect(err).To(HaveOccurred()) + code, _ := bindings.CheckResponseCode(err) Expect(code).To(BeNumerically("==", http.StatusNotFound)) }) - It("add manifest", func() { + It("add", func() { // add to bogus should 404 _, err := manifests.Add(bt.conn, "foobar", nil) - Expect(err).ToNot(BeNil()) + Expect(err).To(HaveOccurred()) + code, _ := bindings.CheckResponseCode(err) - Expect(code).To(BeNumerically("==", http.StatusNotFound)) + Expect(code).To(BeNumerically("==", http.StatusNotFound), err.Error()) + + id, err := manifests.Create(bt.conn, "quay.io/libpod/foobar:latest", []string{}, nil) + Expect(err).ToNot(HaveOccurred()) - id, err := manifests.Create(bt.conn, []string{"quay.io/libpod/foobar:latest"}, []string{}, nil) - Expect(err).To(BeNil()) options := new(manifests.AddOptions).WithImages([]string{alpine.name}) _, err = manifests.Add(bt.conn, id, options) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) + list, err := manifests.Inspect(bt.conn, id, nil) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) + Expect(len(list.Manifests)).To(BeNumerically("==", 1)) // add bogus name to existing list should fail options.WithImages([]string{"larry"}) _, err = manifests.Add(bt.conn, id, options) - Expect(err).ToNot(BeNil()) + Expect(err).To(HaveOccurred()) + code, _ = bindings.CheckResponseCode(err) Expect(code).To(BeNumerically("==", http.StatusInternalServerError)) }) - It("remove manifest", func() { + It("remove digest", func() { // removal on bogus manifest list should be 404 _, err := manifests.Remove(bt.conn, "larry", "1234", nil) - Expect(err).ToNot(BeNil()) + Expect(err).To(HaveOccurred()) + code, _ := bindings.CheckResponseCode(err) Expect(code).To(BeNumerically("==", http.StatusNotFound)) - id, err := manifests.Create(bt.conn, []string{"quay.io/libpod/foobar:latest"}, []string{alpine.name}, nil) - Expect(err).To(BeNil()) + id, err := manifests.Create(bt.conn, "quay.io/libpod/foobar:latest", []string{alpine.name}, nil) + Expect(err).ToNot(HaveOccurred()) + data, err := manifests.Inspect(bt.conn, id, nil) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) + Expect(len(data.Manifests)).To(BeNumerically("==", 1)) // removal on a good manifest list with a bad digest should be 400 _, err = manifests.Remove(bt.conn, id, "!234", nil) - Expect(err).ToNot(BeNil()) + Expect(err).To(HaveOccurred()) + code, _ = bindings.CheckResponseCode(err) Expect(code).To(BeNumerically("==", http.StatusBadRequest)) digest := data.Manifests[0].Digest.String() _, err = manifests.Remove(bt.conn, id, digest, nil) - Expect(err).To(BeNil()) + Expect(err).ToNot(HaveOccurred()) // removal on good manifest with good digest should work data, err = manifests.Inspect(bt.conn, id, nil) - Expect(err).To(BeNil()) - Expect(len(data.Manifests)).To(BeZero()) + Expect(err).ToNot(HaveOccurred()) + + Expect(data.Manifests).Should(BeEmpty()) }) - // There is NO annotate endpoint, this could never work.:w - - //It("annotate manifest", func() { - // id, err := manifests.Create(bt.conn, []string{"quay.io/libpod/foobar:latest"}, []string{}, nil) - // Expect(err).To(BeNil()) - // opts := image.ManifestAddOpts{Images: []string{"docker.io/library/alpine:latest"}} - // - // _, err = manifests.Add(bt.conn, id, opts) - // Expect(err).To(BeNil()) - // data, err := manifests.Inspect(bt.conn, id) - // Expect(err).To(BeNil()) - // Expect(len(data.Manifests)).To(BeNumerically("==", 1)) - // digest := data.Manifests[0].Digest.String() - // annoOpts := image.ManifestAnnotateOpts{OS: "foo"} - // _, err = manifests.Annotate(bt.conn, id, digest, annoOpts) - // Expect(err).To(BeNil()) - // list, err := manifests.Inspect(bt.conn, id) - // Expect(err).To(BeNil()) - // Expect(len(list.Manifests)).To(BeNumerically("==", 1)) - // Expect(list.Manifests[0].Platform.OS).To(Equal("foo")) - //}) + It("annotate", func() { + id, err := manifests.Create(bt.conn, "quay.io/libpod/foobar:latest", []string{}, nil) + Expect(err).ToNot(HaveOccurred()) + + opts := manifests.AddOptions{Images: []string{"quay.io/libpod/alpine:latest"}} + + _, err = manifests.Add(bt.conn, id, &opts) + Expect(err).ToNot(HaveOccurred()) + + data, err := manifests.Inspect(bt.conn, id, nil) + Expect(err).ToNot(HaveOccurred()) + + Expect(len(data.Manifests)).To(BeNumerically("==", 1)) + + digest := data.Manifests[0].Digest.String() + annoOpts := new(manifests.ModifyOptions).WithOS("foo") + _, err = manifests.Annotate(bt.conn, id, []string{digest}, annoOpts) + Expect(err).ToNot(HaveOccurred()) + + list, err := manifests.Inspect(bt.conn, id, nil) + Expect(err).ToNot(HaveOccurred()) + + Expect(len(list.Manifests)).To(BeNumerically("==", 1)) + Expect(list.Manifests[0].Platform.OS).To(Equal("foo")) + }) It("push manifest", func() { - Skip("TODO") + Skip("TODO: implement test for manifest push to registry") }) }) diff --git a/pkg/domain/entities/engine_image.go b/pkg/domain/entities/engine_image.go index bec505163..bf9fcfd27 100644 --- a/pkg/domain/entities/engine_image.go +++ b/pkg/domain/entities/engine_image.go @@ -31,12 +31,12 @@ type ImageEngine interface { Tree(ctx context.Context, nameOrID string, options ImageTreeOptions) (*ImageTreeReport, error) Unmount(ctx context.Context, images []string, options ImageUnmountOptions) ([]*ImageUnmountReport, error) Untag(ctx context.Context, nameOrID string, tags []string, options ImageUntagOptions) error - ManifestCreate(ctx context.Context, names, images []string, opts ManifestCreateOptions) (string, error) + ManifestCreate(ctx context.Context, name string, images []string, opts ManifestCreateOptions) (string, error) ManifestExists(ctx context.Context, name string) (*BoolReport, error) ManifestInspect(ctx context.Context, name string) ([]byte, error) - ManifestAdd(ctx context.Context, opts ManifestAddOptions) (string, error) - ManifestAnnotate(ctx context.Context, names []string, opts ManifestAnnotateOptions) (string, error) - ManifestRemove(ctx context.Context, names []string) (string, error) + ManifestAdd(ctx context.Context, listName string, imageNames []string, opts ManifestAddOptions) (string, error) + ManifestAnnotate(ctx context.Context, names, image string, opts ManifestAnnotateOptions) (string, error) + ManifestRemoveDigest(ctx context.Context, names, image string) (string, error) ManifestRm(ctx context.Context, names []string) (*ImageRemoveReport, []error) ManifestPush(ctx context.Context, name, destination string, imagePushOpts ImagePushOptions) (string, error) Sign(ctx context.Context, names []string, options SignOptions) (*SignReport, error) diff --git a/pkg/domain/entities/images.go b/pkg/domain/entities/images.go index 62e7f67c8..bc41d7844 100644 --- a/pkg/domain/entities/images.go +++ b/pkg/domain/entities/images.go @@ -94,7 +94,7 @@ type ImageRemoveOptions struct { LookupManifest bool } -// ImageRemoveResponse is the response for removing one or more image(s) from storage +// ImageRemoveReport is the response for removing one or more image(s) from storage // and images what was untagged vs actually removed. type ImageRemoveReport struct { // Deleted images. diff --git a/pkg/domain/entities/manifest.go b/pkg/domain/entities/manifest.go index 3f89e4d30..81f3e837b 100644 --- a/pkg/domain/entities/manifest.go +++ b/pkg/domain/entities/manifest.go @@ -2,35 +2,79 @@ package entities import "github.com/containers/image/v5/types" -// TODO: add comments to *all* types and fields. - +// ManifestCreateOptions provides model for creating manifest type ManifestCreateOptions struct { All bool `schema:"all"` } -// swagger:model ManifestAddOpts +// ManifestAddOptions provides model for adding digests to manifest list +// +// swagger:model type ManifestAddOptions struct { - All bool `json:"all" schema:"all"` - Annotation []string `json:"annotation" schema:"annotation"` - Arch string `json:"arch" schema:"arch"` - Authfile string `json:"-" schema:"-"` - CertDir string `json:"-" schema:"-"` - Features []string `json:"features" schema:"features"` - Images []string `json:"images" schema:"images"` - OS string `json:"os" schema:"os"` - OSVersion string `json:"os_version" schema:"os_version"` - Password string `json:"-" schema:"-"` + ManifestAnnotateOptions + // True when operating on a list to include all images + All bool `json:"all" schema:"all"` + // authfile to use when pushing manifest list + Authfile string `json:"-" schema:"-"` + // Home directory for certificates when pushing a manifest list + CertDir string `json:"-" schema:"-"` + // Password to authenticate to registry when pushing manifest list + Password string `json:"-" schema:"-"` + // Should TLS registry certificate be verified? SkipTLSVerify types.OptionalBool `json:"-" schema:"-"` - Username string `json:"-" schema:"-"` - Variant string `json:"variant" schema:"variant"` + // Username to authenticate to registry when pushing manifest list + Username string `json:"-" schema:"-"` + // Images is an optional list of images to add to manifest list + Images []string `json:"images" schema:"images"` } +// ManifestAnnotateOptions provides model for annotating manifest list type ManifestAnnotateOptions struct { - Annotation []string `json:"annotation"` - Arch string `json:"arch" schema:"arch"` - Features []string `json:"features" schema:"features"` - OS string `json:"os" schema:"os"` + // Annotation to add to manifest list + Annotation []string `json:"annotation" schema:"annotation"` + // Arch overrides the architecture for the image + Arch string `json:"arch" schema:"arch"` + // Feature list for the image + Features []string `json:"features" schema:"features"` + // OS overrides the operating system for the image + OS string `json:"os" schema:"os"` + // OS features for the image OSFeatures []string `json:"os_features" schema:"os_features"` - OSVersion string `json:"os_version" schema:"os_version"` - Variant string `json:"variant" schema:"variant"` + // OSVersion overrides the operating system for the image + OSVersion string `json:"os_version" schema:"os_version"` + // Variant for the image + Variant string `json:"variant" schema:"variant"` +} + +// ManifestModifyOptions provides the model for mutating a manifest +// +// swagger 2.0 does not support oneOf for schema validation. +// +// Operation "update" uses all fields. +// Operation "remove" uses fields: Operation and Images +// Operation "annotate" uses fields: Operation and Annotations +// +// swagger:model +type ManifestModifyOptions struct { + Operation string `json:"operation" schema:"operation"` // Valid values: update, remove, annotate + ManifestAddOptions + ManifestRemoveOptions +} + +// ManifestRemoveOptions provides the model for removing digests from a manifest +// +// swagger:model +type ManifestRemoveOptions struct { +} + +// ManifestModifyReport provides the model for removed digests and changed manifest +// +// swagger:model +type ManifestModifyReport struct { + // Manifest List ID + ID string `json:"Id"` + // Images to removed from manifest list, otherwise not provided. + Images []string `json:"images,omitempty" schema:"images"` + // Errors associated with operation + Errors []error `json:"errors,omitempty"` } diff --git a/pkg/domain/infra/abi/manifest.go b/pkg/domain/infra/abi/manifest.go index d1bd5e2e4..d8733130b 100644 --- a/pkg/domain/infra/abi/manifest.go +++ b/pkg/domain/infra/abi/manifest.go @@ -22,13 +22,10 @@ import ( ) // ManifestCreate implements logic for creating manifest lists via ImageEngine -func (ir *ImageEngine) ManifestCreate(ctx context.Context, names []string, images []string, opts entities.ManifestCreateOptions) (string, error) { - // FIXME: change the interface of manifest create `names []string` -> - // `name string`. - if len(names) == 0 { +func (ir *ImageEngine) ManifestCreate(ctx context.Context, name string, images []string, opts entities.ManifestCreateOptions) (string, error) { + if len(name) == 0 { return "", errors.New("no name specified for creating a manifest list") } - name := names[0] manifestList, err := ir.Libpod.LibimageRuntime().CreateManifestList(name) if err != nil { @@ -175,18 +172,12 @@ func (ir *ImageEngine) remoteManifestInspect(ctx context.Context, name string) ( } // ManifestAdd adds images to the manifest list -func (ir *ImageEngine) ManifestAdd(ctx context.Context, opts entities.ManifestAddOptions) (string, error) { - // FIXME: the name options below are *mandatory* arguments and should - // be reflected as such in the signature. - - if len(opts.Images) < 2 { - return "", errors.New("manifest add requires two images") +func (ir *ImageEngine) ManifestAdd(ctx context.Context, name string, images []string, opts entities.ManifestAddOptions) (string, error) { + if len(images) < 1 { + return "", errors.New("manifest add requires at least one image") } - imageName := opts.Images[0] - listName := opts.Images[1] - - manifestList, err := ir.Libpod.LibimageRuntime().LookupManifestList(listName) + manifestList, err := ir.Libpod.LibimageRuntime().LookupManifestList(name) if err != nil { return "", err } @@ -200,53 +191,46 @@ func (ir *ImageEngine) ManifestAdd(ctx context.Context, opts entities.ManifestAd Password: opts.Password, } - instanceDigest, err := manifestList.Add(ctx, imageName, addOptions) - if err != nil { - return "", err - } + for _, image := range images { + instanceDigest, err := manifestList.Add(ctx, image, addOptions) + if err != nil { + return "", err + } - annotateOptions := &libimage.ManifestListAnnotateOptions{ - Architecture: opts.Arch, - Features: opts.Features, - OS: opts.OS, - OSVersion: opts.OSVersion, - Variant: opts.Variant, - } - if len(opts.Annotation) != 0 { - annotations := make(map[string]string) - for _, annotationSpec := range opts.Annotation { - spec := strings.SplitN(annotationSpec, "=", 2) - if len(spec) != 2 { - return "", errors.Errorf("no value given for annotation %q", spec[0]) + annotateOptions := &libimage.ManifestListAnnotateOptions{ + Architecture: opts.Arch, + Features: opts.Features, + OS: opts.OS, + OSVersion: opts.OSVersion, + Variant: opts.Variant, + } + if len(opts.Annotation) != 0 { + annotations := make(map[string]string) + for _, annotationSpec := range opts.Annotation { + spec := strings.SplitN(annotationSpec, "=", 2) + if len(spec) != 2 { + return "", errors.Errorf("no value given for annotation %q", spec[0]) + } + annotations[spec[0]] = spec[1] } - annotations[spec[0]] = spec[1] + annotateOptions.Annotations = annotations } - annotateOptions.Annotations = annotations - } - if err := manifestList.AnnotateInstance(instanceDigest, annotateOptions); err != nil { - return "", err + if err := manifestList.AnnotateInstance(instanceDigest, annotateOptions); err != nil { + return "", err + } } - return manifestList.ID(), nil } // ManifestAnnotate updates an entry of the manifest list -func (ir *ImageEngine) ManifestAnnotate(ctx context.Context, names []string, opts entities.ManifestAnnotateOptions) (string, error) { - // FIXME: the `names` are *mandatory* arguments and should be - // reflected as such in the signature. - - if len(names) < 2 { - return "", errors.New("manifest annotate requires two names") - } - - listName := names[0] - instanceDigest, err := digest.Parse(names[1]) +func (ir *ImageEngine) ManifestAnnotate(ctx context.Context, name, image string, opts entities.ManifestAnnotateOptions) (string, error) { + instanceDigest, err := digest.Parse(image) if err != nil { - return "", errors.Errorf(`invalid image digest "%s": %v`, names[1], err) + return "", errors.Errorf(`invalid image digest "%s": %v`, image, err) } - manifestList, err := ir.Libpod.LibimageRuntime().LookupManifestList(listName) + manifestList, err := ir.Libpod.LibimageRuntime().LookupManifestList(name) if err != nil { return "", err } @@ -277,22 +261,14 @@ func (ir *ImageEngine) ManifestAnnotate(ctx context.Context, names []string, opt return manifestList.ID(), nil } -// ManifestRemove removes specified digest from the specified manifest list -func (ir *ImageEngine) ManifestRemove(ctx context.Context, names []string) (string, error) { - // FIXME: the `names` are *mandatory* arguments and should be - // reflected as such in the signature. - - if len(names) < 2 { - return "", errors.New("manifest remove requires two names") - } - - listName := names[0] - instanceDigest, err := digest.Parse(names[1]) +// ManifestRemoveDigest removes specified digest from the specified manifest list +func (ir *ImageEngine) ManifestRemoveDigest(ctx context.Context, name, image string) (string, error) { + instanceDigest, err := digest.Parse(image) if err != nil { - return "", errors.Errorf(`invalid image digest "%s": %v`, names[1], err) + return "", errors.Errorf(`invalid image digest "%s": %v`, image, err) } - manifestList, err := ir.Libpod.LibimageRuntime().LookupManifestList(listName) + manifestList, err := ir.Libpod.LibimageRuntime().LookupManifestList(name) if err != nil { return "", err } diff --git a/pkg/domain/infra/runtime_abi.go b/pkg/domain/infra/runtime_abi.go index 177e9cff4..ec7de2743 100644 --- a/pkg/domain/infra/runtime_abi.go +++ b/pkg/domain/infra/runtime_abi.go @@ -26,7 +26,7 @@ func NewContainerEngine(facts *entities.PodmanConfig) (entities.ContainerEngine, return nil, fmt.Errorf("runtime mode '%v' is not supported", facts.EngineMode) } -// NewContainerEngine factory provides a libpod runtime for image-related operations +// NewImageEngine factory provides a libpod runtime for image-related operations func NewImageEngine(facts *entities.PodmanConfig) (entities.ImageEngine, error) { switch facts.EngineMode { case entities.ABIMode: diff --git a/pkg/domain/infra/tunnel/manifest.go b/pkg/domain/infra/tunnel/manifest.go index 62634f561..f89c59bc7 100644 --- a/pkg/domain/infra/tunnel/manifest.go +++ b/pkg/domain/infra/tunnel/manifest.go @@ -7,16 +7,16 @@ import ( "strings" "github.com/containers/image/v5/types" - images "github.com/containers/podman/v3/pkg/bindings/images" + "github.com/containers/podman/v3/pkg/bindings/images" "github.com/containers/podman/v3/pkg/bindings/manifests" "github.com/containers/podman/v3/pkg/domain/entities" "github.com/pkg/errors" ) // ManifestCreate implements manifest create via ImageEngine -func (ir *ImageEngine) ManifestCreate(ctx context.Context, names, images []string, opts entities.ManifestCreateOptions) (string, error) { +func (ir *ImageEngine) ManifestCreate(ctx context.Context, name string, images []string, opts entities.ManifestCreateOptions) (string, error) { options := new(manifests.CreateOptions).WithAll(opts.All) - imageID, err := manifests.Create(ir.ClientCtx, names, images, options) + imageID, err := manifests.Create(ir.ClientCtx, name, images, options) if err != nil { return imageID, errors.Wrapf(err, "error creating manifest") } @@ -33,7 +33,7 @@ func (ir *ImageEngine) ManifestExists(ctx context.Context, name string) (*entiti } // ManifestInspect returns contents of manifest list with given name -func (ir *ImageEngine) ManifestInspect(ctx context.Context, name string) ([]byte, error) { +func (ir *ImageEngine) ManifestInspect(_ context.Context, name string) ([]byte, error) { list, err := manifests.Inspect(ir.ClientCtx, name, nil) if err != nil { return nil, errors.Wrapf(err, "error getting content of manifest list or image %s", name) @@ -47,9 +47,9 @@ func (ir *ImageEngine) ManifestInspect(ctx context.Context, name string) ([]byte } // ManifestAdd adds images to the manifest list -func (ir *ImageEngine) ManifestAdd(ctx context.Context, opts entities.ManifestAddOptions) (string, error) { +func (ir *ImageEngine) ManifestAdd(_ context.Context, name string, imageNames []string, opts entities.ManifestAddOptions) (string, error) { options := new(manifests.AddOptions).WithAll(opts.All).WithArch(opts.Arch).WithVariant(opts.Variant) - options.WithFeatures(opts.Features).WithImages(opts.Images).WithOS(opts.OS).WithOSVersion(opts.OSVersion) + options.WithFeatures(opts.Features).WithImages(imageNames).WithOS(opts.OS).WithOSVersion(opts.OSVersion) if len(opts.Annotation) != 0 { annotations := make(map[string]string) for _, annotationSpec := range opts.Annotation { @@ -62,25 +62,25 @@ func (ir *ImageEngine) ManifestAdd(ctx context.Context, opts entities.ManifestAd options.WithAnnotation(annotations) } - listID, err := manifests.Add(ir.ClientCtx, opts.Images[1], options) + id, err := manifests.Add(ir.ClientCtx, name, options) if err != nil { - return listID, errors.Wrapf(err, "error adding to manifest list %s", opts.Images[1]) + return id, errors.Wrapf(err, "error adding to manifest list %s", name) } - return listID, nil + return id, nil } // ManifestAnnotate updates an entry of the manifest list -func (ir *ImageEngine) ManifestAnnotate(ctx context.Context, names []string, opts entities.ManifestAnnotateOptions) (string, error) { +func (ir *ImageEngine) ManifestAnnotate(ctx context.Context, name, images string, opts entities.ManifestAnnotateOptions) (string, error) { return "", errors.New("not implemented") } -// ManifestRemove removes the digest from manifest list -func (ir *ImageEngine) ManifestRemove(ctx context.Context, names []string) (string, error) { - updatedListID, err := manifests.Remove(ir.ClientCtx, names[0], names[1], nil) +// ManifestRemoveDigest removes the digest from manifest list +func (ir *ImageEngine) ManifestRemoveDigest(ctx context.Context, name string, image string) (string, error) { + updatedListID, err := manifests.Remove(ir.ClientCtx, name, image, nil) if err != nil { - return updatedListID, errors.Wrapf(err, "error removing from manifest %s", names[0]) + return updatedListID, errors.Wrapf(err, "error removing from manifest %s", name) } - return fmt.Sprintf("%s :%s\n", updatedListID, names[1]), nil + return fmt.Sprintf("%s :%s\n", updatedListID, image), nil } // ManifestRm removes the specified manifest list from storage diff --git a/pkg/machine/ignition.go b/pkg/machine/ignition.go index ac2cf71cf..09228553c 100644 --- a/pkg/machine/ignition.go +++ b/pkg/machine/ignition.go @@ -1,3 +1,4 @@ +//go:build amd64 || arm64 // +build amd64 arm64 package machine @@ -423,61 +424,67 @@ func getCerts(certsDir string, isDir bool) []File { files []File ) - certs, err := ioutil.ReadDir(certsDir) if isDir { - if err == nil { - for _, cert := range certs { - b, err := ioutil.ReadFile(filepath.Join(certsDir, cert.Name())) + err := filepath.Walk(certsDir, func(path string, info os.FileInfo, err error) error { + if err == nil && !info.IsDir() { + certPath, err := filepath.Rel(certsDir, path) if err != nil { - logrus.Warnf("Unable to read cert file %s", err.Error()) - continue + logrus.Warnf("%s", err) + return nil + } + + file, err := prepareCertFile(filepath.Join(certsDir, certPath), certPath) + if err == nil { + files = append(files, file) } - files = append(files, File{ - Node: Node{ - Group: getNodeGrp("root"), - Path: filepath.Join("/etc/containers/certs.d/", cert.Name()), - User: getNodeUsr("root"), - }, - FileEmbedded1: FileEmbedded1{ - Append: nil, - Contents: Resource{ - Source: encodeDataURLPtr(string(b)), - }, - Mode: intToPtr(0644), - }, - }) } - } else { + + return nil + }) + if err != nil { if !os.IsNotExist(err) { logrus.Warnf("Unable to copy certs via ignition, error while reading certs from %s: %s", certsDir, err.Error()) } } } else { fileName := filepath.Base(certsDir) - b, err := ioutil.ReadFile(certsDir) - if err != nil { - logrus.Warnf("Unable to read cert file %s", err.Error()) - return files + file, err := prepareCertFile(certsDir, fileName) + if err == nil { + files = append(files, file) } - files = append(files, File{ - Node: Node{ - Group: getNodeGrp("root"), - Path: filepath.Join("/etc/containers/certs.d/", fileName), - User: getNodeUsr("root"), - }, - FileEmbedded1: FileEmbedded1{ - Append: nil, - Contents: Resource{ - Source: encodeDataURLPtr(string(b)), - }, - Mode: intToPtr(0644), - }, - }) } return files } +func prepareCertFile(path string, name string) (File, error) { + b, err := ioutil.ReadFile(path) + if err != nil { + logrus.Warnf("Unable to read cert file %s", err.Error()) + return File{}, err + } + + targetPath := filepath.Join("/etc/containers/certs.d", name) + + logrus.Debugf("Copying cert file from '%s' to '%s'.", path, targetPath) + + file := File{ + Node: Node{ + Group: getNodeGrp("root"), + Path: targetPath, + User: getNodeUsr("root"), + }, + FileEmbedded1: FileEmbedded1{ + Append: nil, + Contents: Resource{ + Source: encodeDataURLPtr(string(b)), + }, + Mode: intToPtr(0644), + }, + } + return file, nil +} + func getProxyVariables() string { proxyOpts := "" for _, variable := range config.ProxyEnv { diff --git a/pkg/specgen/generate/container_create.go b/pkg/specgen/generate/container_create.go index 7d792b3b1..dc116d1e6 100644 --- a/pkg/specgen/generate/container_create.go +++ b/pkg/specgen/generate/container_create.go @@ -6,7 +6,7 @@ import ( "path/filepath" "strings" - cdi "github.com/container-orchestrated-devices/container-device-interface/pkg" + cdi "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" "github.com/containers/common/libimage" "github.com/containers/podman/v3/libpod" "github.com/containers/podman/v3/libpod/define" @@ -199,20 +199,21 @@ func ExecuteCreate(ctx context.Context, rt *libpod.Runtime, runtimeSpec *spec.Sp return ctr, rt.PrepareVolumeOnCreateContainer(ctx, ctr) } +// ExtractCDIDevices process the list of Devices in the spec and determines if any of these are CDI devices. +// The CDI devices are added to the list of CtrCreateOptions. +// Note that this may modify the device list associated with the spec, which should then only contain non-CDI devices. func ExtractCDIDevices(s *specgen.SpecGenerator) []libpod.CtrCreateOption { devs := make([]spec.LinuxDevice, 0, len(s.Devices)) var cdiDevs []string var options []libpod.CtrCreateOption for _, device := range s.Devices { - isCDIDevice, err := cdi.HasDevice(device.Path) - if err != nil { - logrus.Debugf("CDI HasDevice Error: %v", err) - } - if err == nil && isCDIDevice { + if isCDIDevice(device.Path) { + logrus.Debugf("Identified CDI device %v", device.Path) cdiDevs = append(cdiDevs, device.Path) continue } + logrus.Debugf("Non-CDI device %v; assuming standard device", device.Path) devs = append(devs, device) } s.Devices = devs @@ -222,6 +223,11 @@ func ExtractCDIDevices(s *specgen.SpecGenerator) []libpod.CtrCreateOption { return options } +// isCDIDevice checks whether the specified device is a CDI device. +func isCDIDevice(device string) bool { + return cdi.IsQualifiedName(device) +} + func createContainerOptions(ctx context.Context, rt *libpod.Runtime, s *specgen.SpecGenerator, pod *libpod.Pod, volumes []*specgen.NamedVolume, overlays []*specgen.OverlayVolume, imageData *libimage.ImageData, command []string, infraVolumes bool, compatibleOptions libpod.InfraInherit) ([]libpod.CtrCreateOption, error) { var options []libpod.CtrCreateOption var err error diff --git a/test/apiv2/15-manifest.at b/test/apiv2/15-manifest.at new file mode 100644 index 000000000..0dd7026fa --- /dev/null +++ b/test/apiv2/15-manifest.at @@ -0,0 +1,19 @@ +# -*- sh -*- +# +# Tests for manifest list endpoints + +t POST /v3.4.0/libpod/manifests/create?name=abc 200 \ + .Id~[0-9a-f]\\{64\\} +id_abc=$(jq -r '.Id' <<<"$output") + +t POST /v4.0.0/libpod/manifests/xyz 201 \ + .Id~[0-9a-f]\\{64\\} +echo xyz $output +id_xyz=$(jq -r '.Id' <<<"$output") + +t GET /v3.4.0/libpod/manifests/$id_abc/exists 204 +t GET /v4.0.0/libpod/manifests/$id_xyz/exists 204 + +# /v3.x cannot delete a manifest list +t DELETE /v4.0.0/libpod/manifests/$id_abc 200 +t DELETE /v4.0.0/libpod/manifests/$id_xyz 200 diff --git a/test/apiv2/test-apiv2 b/test/apiv2/test-apiv2 index 391095539..19e8c12d0 100755 --- a/test/apiv2/test-apiv2 +++ b/test/apiv2/test-apiv2 @@ -364,7 +364,9 @@ function start_service() { echo $WORKDIR # Some tests use shortnames; force registry override to work around # docker.io throttling. - env CONTAINERS_REGISTRIES_CONF=$TESTS_DIR/../registries.conf $PODMAN_BIN \ +# FIXME esm revisit pulling expected images re: shortnames caused tests to fail +# env CONTAINERS_REGISTRIES_CONF=$TESTS_DIR/../registries.conf + $PODMAN_BIN \ --root $WORKDIR/server_root --syslog=true \ system service \ --time 15 \ @@ -497,7 +499,7 @@ function wait_for_port() { ############ function podman() { echo "\$ $PODMAN_BIN $*" >>$WORKDIR/output.log - env CONTAINERS_REGISTRIES_CONF=$TESTS_DIR/../registries.conf \ +# env CONTAINERS_REGISTRIES_CONF=$TESTS_DIR/../registries.conf \ $PODMAN_BIN --root $WORKDIR/server_root "$@" >>$WORKDIR/output.log 2>&1 } diff --git a/test/e2e/libpod_suite_remote_test.go b/test/e2e/libpod_suite_remote_test.go index 4644e3748..53e1fc8f2 100644 --- a/test/e2e/libpod_suite_remote_test.go +++ b/test/e2e/libpod_suite_remote_test.go @@ -80,7 +80,7 @@ func (p *PodmanTestIntegration) StartRemoteService() { args := []string{} if _, found := os.LookupEnv("DEBUG_SERVICE"); found { - args = append(args, "--log-level", "debug") + args = append(args, "--log-level", "trace") } remoteSocket := p.RemoteSocket args = append(args, "system", "service", "--time", "0", remoteSocket) @@ -151,7 +151,7 @@ func (p *PodmanTestIntegration) StopRemoteService() { } } -//MakeOptions assembles all the podman main options +// MakeOptions assembles all the podman main options func getRemoteOptions(p *PodmanTestIntegration, args []string) []string { podmanOptions := strings.Split(fmt.Sprintf("--root %s --runroot %s --runtime %s --conmon %s --cni-config-dir %s --cgroup-manager %s", p.Root, p.RunRoot, p.OCIRuntime, p.ConmonBinary, p.CNIConfigDir, p.CgroupManager), " ") diff --git a/test/e2e/manifest_test.go b/test/e2e/manifest_test.go index 5978214ff..b86d5ecbe 100644 --- a/test/e2e/manifest_test.go +++ b/test/e2e/manifest_test.go @@ -44,19 +44,19 @@ var _ = Describe("Podman manifest", func() { processTestResult(f) }) - It("podman manifest create", func() { + It("create w/o image", func() { session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) }) - It("podman manifest create", func() { + It("create w/ image", func() { session := podmanTest.Podman([]string{"manifest", "create", "foo", imageList}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) }) - It("podman manifest inspect", func() { + It("inspect", func() { session := podmanTest.Podman([]string{"manifest", "inspect", BB}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) @@ -71,29 +71,27 @@ var _ = Describe("Podman manifest", func() { Expect(session).Should(Exit(0)) }) - It("podman manifest add", func() { + It("add w/ inspect", func() { session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - session = podmanTest.Podman([]string{"manifest", "add", "--arch=arm64", "foo", imageListInstance}) - session.WaitWithDefaultTimeout() - Expect(session).Should(Exit(0)) - }) + id := strings.TrimSpace(string(session.Out.Contents())) - It("podman manifest add one", func() { - session := podmanTest.Podman([]string{"manifest", "create", "foo"}) + session = podmanTest.Podman([]string{"manifest", "inspect", id}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) + session = podmanTest.Podman([]string{"manifest", "add", "--arch=arm64", "foo", imageListInstance}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) + session = podmanTest.Podman([]string{"manifest", "inspect", "foo"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) Expect(session.OutputToString()).To(ContainSubstring(imageListARM64InstanceDigest)) }) - It("podman manifest tag", func() { + It("tag", func() { session := podmanTest.Podman([]string{"manifest", "create", "foobar"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) @@ -112,7 +110,7 @@ var _ = Describe("Podman manifest", func() { Expect(session2.OutputToString()).To(Equal(session.OutputToString())) }) - It("podman manifest add --all", func() { + It(" add --all", func() { session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) @@ -122,14 +120,17 @@ var _ = Describe("Podman manifest", func() { session = podmanTest.Podman([]string{"manifest", "inspect", "foo"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(session.OutputToString()).To(ContainSubstring(imageListAMD64InstanceDigest)) - Expect(session.OutputToString()).To(ContainSubstring(imageListARMInstanceDigest)) - Expect(session.OutputToString()).To(ContainSubstring(imageListARM64InstanceDigest)) - Expect(session.OutputToString()).To(ContainSubstring(imageListPPC64LEInstanceDigest)) - Expect(session.OutputToString()).To(ContainSubstring(imageListS390XInstanceDigest)) + Expect(session.OutputToString()).To( + And( + ContainSubstring(imageListAMD64InstanceDigest), + ContainSubstring(imageListARMInstanceDigest), + ContainSubstring(imageListARM64InstanceDigest), + ContainSubstring(imageListPPC64LEInstanceDigest), + ContainSubstring(imageListS390XInstanceDigest), + )) }) - It("podman manifest add --os", func() { + It("add --os", func() { session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) @@ -142,7 +143,7 @@ var _ = Describe("Podman manifest", func() { Expect(session.OutputToString()).To(ContainSubstring(`"os": "bar"`)) }) - It("podman manifest annotate", func() { + It("annotate", func() { SkipIfRemote("Not supporting annotate on remote connections") session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() @@ -159,7 +160,7 @@ var _ = Describe("Podman manifest", func() { Expect(session.OutputToString()).To(ContainSubstring(`"architecture": "bar"`)) }) - It("podman manifest remove", func() { + It("remove digest", func() { session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) @@ -176,14 +177,18 @@ var _ = Describe("Podman manifest", func() { session = podmanTest.Podman([]string{"manifest", "inspect", "foo"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - Expect(session.OutputToString()).To(ContainSubstring(imageListAMD64InstanceDigest)) - Expect(session.OutputToString()).To(ContainSubstring(imageListARMInstanceDigest)) - Expect(session.OutputToString()).To(ContainSubstring(imageListPPC64LEInstanceDigest)) - Expect(session.OutputToString()).To(ContainSubstring(imageListS390XInstanceDigest)) - Expect(session.OutputToString()).To(Not(ContainSubstring(imageListARM64InstanceDigest))) + Expect(session.OutputToString()).To( + And( + ContainSubstring(imageListAMD64InstanceDigest), + ContainSubstring(imageListARMInstanceDigest), + ContainSubstring(imageListPPC64LEInstanceDigest), + ContainSubstring(imageListS390XInstanceDigest), + Not( + ContainSubstring(imageListARM64InstanceDigest)), + )) }) - It("podman manifest remove not-found", func() { + It("remove not-found", func() { session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) @@ -199,7 +204,7 @@ var _ = Describe("Podman manifest", func() { Expect(session).Should(Exit(0)) }) - It("podman manifest push", func() { + It("push --all", func() { SkipIfRemote("manifest push to dir not supported in remote mode") session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() @@ -216,20 +221,24 @@ var _ = Describe("Podman manifest", func() { session = podmanTest.Podman([]string{"manifest", "push", "--all", "foo", "dir:" + dest}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) + files, err := filepath.Glob(dest + string(os.PathSeparator) + "*") - Expect(err).To(BeNil()) + Expect(err).ShouldNot(HaveOccurred()) check := SystemExec("sha256sum", files) check.WaitWithDefaultTimeout() Expect(check).Should(Exit(0)) prefix := "sha256:" - Expect(check.OutputToString()).To(ContainSubstring(strings.TrimPrefix(imageListAMD64InstanceDigest, prefix))) - Expect(check.OutputToString()).To(ContainSubstring(strings.TrimPrefix(imageListARMInstanceDigest, prefix))) - Expect(check.OutputToString()).To(ContainSubstring(strings.TrimPrefix(imageListPPC64LEInstanceDigest, prefix))) - Expect(check.OutputToString()).To(ContainSubstring(strings.TrimPrefix(imageListS390XInstanceDigest, prefix))) - Expect(check.OutputToString()).To(ContainSubstring(strings.TrimPrefix(imageListARM64InstanceDigest, prefix))) + Expect(check.OutputToString()).To( + And( + ContainSubstring(strings.TrimPrefix(imageListAMD64InstanceDigest, prefix)), + ContainSubstring(strings.TrimPrefix(imageListARMInstanceDigest, prefix)), + ContainSubstring(strings.TrimPrefix(imageListPPC64LEInstanceDigest, prefix)), + ContainSubstring(strings.TrimPrefix(imageListS390XInstanceDigest, prefix)), + ContainSubstring(strings.TrimPrefix(imageListARM64InstanceDigest, prefix)), + )) }) - It("podman push --all", func() { + It("push", func() { SkipIfRemote("manifest push to dir not supported in remote mode") session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() @@ -251,15 +260,19 @@ var _ = Describe("Podman manifest", func() { check := SystemExec("sha256sum", files) check.WaitWithDefaultTimeout() Expect(check).Should(Exit(0)) + prefix := "sha256:" - Expect(check.OutputToString()).To(ContainSubstring(strings.TrimPrefix(imageListAMD64InstanceDigest, prefix))) - Expect(check.OutputToString()).To(ContainSubstring(strings.TrimPrefix(imageListARMInstanceDigest, prefix))) - Expect(check.OutputToString()).To(ContainSubstring(strings.TrimPrefix(imageListPPC64LEInstanceDigest, prefix))) - Expect(check.OutputToString()).To(ContainSubstring(strings.TrimPrefix(imageListS390XInstanceDigest, prefix))) - Expect(check.OutputToString()).To(ContainSubstring(strings.TrimPrefix(imageListARM64InstanceDigest, prefix))) + Expect(check.OutputToString()).To( + And( + ContainSubstring(strings.TrimPrefix(imageListAMD64InstanceDigest, prefix)), + ContainSubstring(strings.TrimPrefix(imageListARMInstanceDigest, prefix)), + ContainSubstring(strings.TrimPrefix(imageListPPC64LEInstanceDigest, prefix)), + ContainSubstring(strings.TrimPrefix(imageListS390XInstanceDigest, prefix)), + ContainSubstring(strings.TrimPrefix(imageListARM64InstanceDigest, prefix)), + )) }) - It("podman manifest push --rm", func() { + It("push --rm", func() { SkipIfRemote("remote does not support --rm") session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() @@ -285,7 +298,7 @@ var _ = Describe("Podman manifest", func() { Expect(session).To(ExitWithError()) }) - It("podman manifest exists", func() { + It("exists", func() { manifestList := "manifest-list" session := podmanTest.Podman([]string{"manifest", "create", manifestList}) session.WaitWithDefaultTimeout() @@ -300,7 +313,7 @@ var _ = Describe("Podman manifest", func() { Expect(session).Should(Exit(1)) }) - It("podman manifest rm should not remove referenced images", func() { + It("rm should not remove referenced images", func() { manifestList := "manifestlist" imageName := "quay.io/libpod/busybox" @@ -320,11 +333,9 @@ var _ = Describe("Podman manifest", func() { session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) - //image should still show up - session = podmanTest.Podman([]string{"images"}) + // image should still show up + session = podmanTest.Podman([]string{"image", "exists", imageName}) session.WaitWithDefaultTimeout() - Expect(session.OutputToString()).To(ContainSubstring(imageName)) Expect(session).Should(Exit(0)) }) - }) diff --git a/test/e2e/run_device_test.go b/test/e2e/run_device_test.go index fbf1eb791..4eb03dfff 100644 --- a/test/e2e/run_device_test.go +++ b/test/e2e/run_device_test.go @@ -109,7 +109,7 @@ var _ = Describe("Podman run device", func() { err = cmd.Run() Expect(err).To(BeNil()) - session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "myKmsg", ALPINE, "test", "-c", "/dev/kmsg1"}) + session := podmanTest.Podman([]string{"run", "-q", "--security-opt", "label=disable", "--device", "vendor.com/device=myKmsg", ALPINE, "test", "-c", "/dev/kmsg1"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) }) diff --git a/test/system/030-run.bats b/test/system/030-run.bats index afcda3d3c..2d5ecab39 100644 --- a/test/system/030-run.bats +++ b/test/system/030-run.bats @@ -818,4 +818,11 @@ EOF is "$output" ".*The --kernel-memory flag is no longer supported. This flag is a noop." "warn on use of --kernel-memory" } + +# rhbz#1902979 : podman run fails to update /etc/hosts when --uidmap is provided +@test "podman run update /etc/hosts" { + HOST=$(random_string 25) + run_podman run --uidmap 0:10001:10002 --rm --hostname ${HOST} $IMAGE grep ${HOST} /etc/hosts + is "${lines[0]}" ".*${HOST}.*" +} # vim: filetype=sh diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/cache.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/cache.go new file mode 100644 index 000000000..c4befc83e --- /dev/null +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/cache.go @@ -0,0 +1,275 @@ +/* + Copyright © 2021 The CDI Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cdi + +import ( + "path/filepath" + "sort" + "strings" + "sync" + + "github.com/hashicorp/go-multierror" + oci "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" +) + +// Option is an option to change some aspect of default CDI behavior. +type Option func(*Cache) error + +// Cache stores CDI Specs loaded from Spec directories. +type Cache struct { + sync.Mutex + specDirs []string + specs map[string][]*Spec + devices map[string]*Device + errors map[string][]error +} + +// NewCache creates a new CDI Cache. The cache is populated from a set +// of CDI Spec directories. These can be specified using a WithSpecDirs +// option. The default set of directories is exposed in DefaultSpecDirs. +func NewCache(options ...Option) (*Cache, error) { + c := &Cache{} + + if err := c.Configure(options...); err != nil { + return nil, err + } + if len(c.specDirs) == 0 { + c.Configure(WithSpecDirs(DefaultSpecDirs...)) + } + + return c, c.Refresh() +} + +// Configure applies options to the cache. Updates the cache if options have +// changed. +func (c *Cache) Configure(options ...Option) error { + if len(options) == 0 { + return nil + } + + c.Lock() + defer c.Unlock() + + for _, o := range options { + if err := o(c); err != nil { + return errors.Wrapf(err, "failed to apply cache options") + } + } + + return nil +} + +// Refresh rescans the CDI Spec directories and refreshes the Cache. +func (c *Cache) Refresh() error { + var ( + specs = map[string][]*Spec{} + devices = map[string]*Device{} + conflicts = map[string]struct{}{} + specErrors = map[string][]error{} + result []error + ) + + // collect errors per spec file path and once globally + collectError := func(err error, paths ...string) { + result = append(result, err) + for _, path := range paths { + specErrors[path] = append(specErrors[path], err) + } + } + // resolve conflicts based on device Spec priority (order of precedence) + resolveConflict := func(name string, dev *Device, old *Device) bool { + devSpec, oldSpec := dev.GetSpec(), old.GetSpec() + devPrio, oldPrio := devSpec.GetPriority(), oldSpec.GetPriority() + switch { + case devPrio > oldPrio: + return false + case devPrio == oldPrio: + devPath, oldPath := devSpec.GetPath(), oldSpec.GetPath() + collectError(errors.Errorf("conflicting device %q (specs %q, %q)", + name, devPath, oldPath), devPath, oldPath) + conflicts[name] = struct{}{} + } + return true + } + + _ = scanSpecDirs(c.specDirs, func(path string, priority int, spec *Spec, err error) error { + path = filepath.Clean(path) + if err != nil { + collectError(errors.Wrapf(err, "failed to load CDI Spec"), path) + return nil + } + + vendor := spec.GetVendor() + specs[vendor] = append(specs[vendor], spec) + + for _, dev := range spec.devices { + qualified := dev.GetQualifiedName() + other, ok := devices[qualified] + if ok { + if resolveConflict(qualified, dev, other) { + continue + } + } + devices[qualified] = dev + } + + return nil + }) + + for conflict := range conflicts { + delete(devices, conflict) + } + + c.Lock() + defer c.Unlock() + + c.specs = specs + c.devices = devices + c.errors = specErrors + + if len(result) > 0 { + return multierror.Append(nil, result...) + } + + return nil +} + +// InjectDevices injects the given qualified devices to an OCI Spec. It +// returns any unresolvable devices and an error if injection fails for +// any of the devices. +func (c *Cache) InjectDevices(ociSpec *oci.Spec, devices ...string) ([]string, error) { + var ( + unresolved []string + specs = map[*Spec]struct{}{} + ) + + if ociSpec == nil { + return devices, errors.Errorf("can't inject devices, nil OCI Spec") + } + + c.Lock() + defer c.Unlock() + + for _, device := range devices { + d := c.devices[device] + if d == nil { + unresolved = append(unresolved, device) + continue + } + if _, ok := specs[d.GetSpec()]; !ok { + specs[d.GetSpec()] = struct{}{} + d.GetSpec().ApplyEdits(ociSpec) + } + d.ApplyEdits(ociSpec) + } + + if unresolved != nil { + return unresolved, errors.Errorf("unresolvable CDI devices %s", + strings.Join(devices, ", ")) + } + + return nil, nil +} + +// GetDevice returns the cached device for the given qualified name. +func (c *Cache) GetDevice(device string) *Device { + c.Lock() + defer c.Unlock() + + return c.devices[device] +} + +// ListDevices lists all cached devices by qualified name. +func (c *Cache) ListDevices() []string { + var devices []string + + c.Lock() + defer c.Unlock() + + for name := range c.devices { + devices = append(devices, name) + } + sort.Strings(devices) + + return devices +} + +// ListVendors lists all vendors known to the cache. +func (c *Cache) ListVendors() []string { + var vendors []string + + c.Lock() + defer c.Unlock() + + for vendor := range c.specs { + vendors = append(vendors, vendor) + } + sort.Strings(vendors) + + return vendors +} + +// ListClasses lists all device classes known to the cache. +func (c *Cache) ListClasses() []string { + var ( + cmap = map[string]struct{}{} + classes []string + ) + + c.Lock() + defer c.Unlock() + + for _, specs := range c.specs { + for _, spec := range specs { + cmap[spec.GetClass()] = struct{}{} + } + } + for class := range cmap { + classes = append(classes, class) + } + sort.Strings(classes) + + return classes +} + +// GetVendorSpecs returns all specs for the given vendor. +func (c *Cache) GetVendorSpecs(vendor string) []*Spec { + c.Lock() + defer c.Unlock() + + return c.specs[vendor] +} + +// GetSpecErrors returns all errors encountered for the spec during the +// last cache refresh. +func (c *Cache) GetSpecErrors(spec *Spec) []error { + return c.errors[spec.GetPath()] +} + +// GetErrors returns all errors encountered during the last +// cache refresh. +func (c *Cache) GetErrors() map[string][]error { + return c.errors +} + +// GetSpecDirectories returns the CDI Spec directories currently in use. +func (c *Cache) GetSpecDirectories() []string { + dirs := make([]string, len(c.specDirs)) + copy(dirs, c.specDirs) + return dirs +} diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/container-edits.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/container-edits.go new file mode 100644 index 000000000..767cba57b --- /dev/null +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/container-edits.go @@ -0,0 +1,222 @@ +/* + Copyright © 2021 The CDI Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cdi + +import ( + "strings" + + "github.com/pkg/errors" + + "github.com/container-orchestrated-devices/container-device-interface/specs-go" + oci "github.com/opencontainers/runtime-spec/specs-go" + ocigen "github.com/opencontainers/runtime-tools/generate" +) + +const ( + // PrestartHook is the name of the OCI "prestart" hook. + PrestartHook = "prestart" + // CreateRuntimeHook is the name of the OCI "createRuntime" hook. + CreateRuntimeHook = "createRuntime" + // CreateContainerHook is the name of the OCI "createContainer" hook. + CreateContainerHook = "createContainer" + // StartContainerHook is the name of the OCI "startContainer" hook. + StartContainerHook = "startContainer" + // PoststartHook is the name of the OCI "poststart" hook. + PoststartHook = "poststart" + // PoststopHook is the name of the OCI "poststop" hook. + PoststopHook = "poststop" +) + +var ( + // Names of recognized hooks. + validHookNames = map[string]struct{}{ + PrestartHook: {}, + CreateRuntimeHook: {}, + CreateContainerHook: {}, + StartContainerHook: {}, + PoststartHook: {}, + PoststopHook: {}, + } +) + +// ContainerEdits represent updates to be applied to an OCI Spec. +// These updates can be specific to a CDI device, or they can be +// specific to a CDI Spec. In the former case these edits should +// be applied to all OCI Specs where the corresponding CDI device +// is injected. In the latter case, these edits should be applied +// to all OCI Specs where at least one devices from the CDI Spec +// is injected. +type ContainerEdits struct { + *specs.ContainerEdits +} + +// Apply edits to the given OCI Spec. Updates the OCI Spec in place. +// Returns an error if the update fails. +func (e *ContainerEdits) Apply(spec *oci.Spec) error { + if spec == nil { + return errors.New("can't edit nil OCI Spec") + } + if e == nil || e.ContainerEdits == nil { + return nil + } + + specgen := ocigen.NewFromSpec(spec) + if len(e.Env) > 0 { + specgen.AddMultipleProcessEnv(e.Env) + } + for _, d := range e.DeviceNodes { + specgen.AddDevice(d.ToOCI()) + } + for _, m := range e.Mounts { + specgen.AddMount(m.ToOCI()) + } + for _, h := range e.Hooks { + switch h.HookName { + case PrestartHook: + specgen.AddPreStartHook(h.ToOCI()) + case PoststartHook: + specgen.AddPostStartHook(h.ToOCI()) + case PoststopHook: + specgen.AddPostStopHook(h.ToOCI()) + // TODO: Maybe runtime-tools/generate should be updated with these... + case CreateRuntimeHook: + ensureOCIHooks(spec) + spec.Hooks.CreateRuntime = append(spec.Hooks.CreateRuntime, h.ToOCI()) + case CreateContainerHook: + ensureOCIHooks(spec) + spec.Hooks.CreateContainer = append(spec.Hooks.CreateContainer, h.ToOCI()) + case StartContainerHook: + ensureOCIHooks(spec) + spec.Hooks.StartContainer = append(spec.Hooks.StartContainer, h.ToOCI()) + default: + return errors.Errorf("unknown hook name %q", h.HookName) + } + } + + return nil +} + +// Validate container edits. +func (e *ContainerEdits) Validate() error { + if e == nil || e.ContainerEdits == nil { + return nil + } + + if err := ValidateEnv(e.Env); err != nil { + return errors.Wrap(err, "invalid container edits") + } + for _, d := range e.DeviceNodes { + if err := (&DeviceNode{d}).Validate(); err != nil { + return err + } + } + for _, h := range e.Hooks { + if err := (&Hook{h}).Validate(); err != nil { + return err + } + } + for _, m := range e.Mounts { + if err := (&Mount{m}).Validate(); err != nil { + return err + } + } + + return nil +} + +// isEmpty returns true if these edits are empty. This is valid in a +// global Spec context but invalid in a Device context. +func (e *ContainerEdits) isEmpty() bool { + if e == nil { + return false + } + return len(e.Env)+len(e.DeviceNodes)+len(e.Hooks)+len(e.Mounts) == 0 +} + +// ValidateEnv validates the given environment variables. +func ValidateEnv(env []string) error { + for _, v := range env { + if strings.IndexByte(v, byte('=')) <= 0 { + return errors.Errorf("invalid environment variable %q", v) + } + } + return nil +} + +// DeviceNode is a CDI Spec DeviceNode wrapper, used for validating DeviceNodes. +type DeviceNode struct { + *specs.DeviceNode +} + +// Validate a CDI Spec DeviceNode. +func (d *DeviceNode) Validate() error { + if d.Path == "" { + return errors.New("invalid (empty) device path") + } + if d.Type != "" && d.Type != "b" && d.Type != "c" { + return errors.Errorf("device %q: invalid type %q", d.Path, d.Type) + } + for _, bit := range d.Permissions { + if bit != 'r' && bit != 'w' && bit != 'm' { + return errors.Errorf("device %q: invalid persmissions %q", + d.Path, d.Permissions) + } + } + return nil +} + +// Hook is a CDI Spec Hook wrapper, used for validating hooks. +type Hook struct { + *specs.Hook +} + +// Validate a hook. +func (h *Hook) Validate() error { + if _, ok := validHookNames[h.HookName]; !ok { + return errors.Errorf("invalid hook name %q", h.HookName) + } + if h.Path == "" { + return errors.Errorf("invalid hook %q with empty path", h.HookName) + } + if err := ValidateEnv(h.Env); err != nil { + return errors.Wrapf(err, "invalid hook %q", h.HookName) + } + return nil +} + +// Mount is a CDI Mount wrapper, used for validating mounts. +type Mount struct { + *specs.Mount +} + +// Validate a mount. +func (m *Mount) Validate() error { + if m.HostPath == "" { + return errors.New("invalid mount, empty host path") + } + if m.ContainerPath == "" { + return errors.New("invalid mount, empty container path") + } + return nil +} + +// Ensure OCI Spec hooks are not nil so we can add hooks. +func ensureOCIHooks(spec *oci.Spec) { + if spec.Hooks == nil { + spec.Hooks = &oci.Hooks{} + } +} diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/device.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/device.go new file mode 100644 index 000000000..84b71429c --- /dev/null +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/device.go @@ -0,0 +1,74 @@ +/* + Copyright © 2021 The CDI Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cdi + +import ( + cdi "github.com/container-orchestrated-devices/container-device-interface/specs-go" + oci "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" +) + +// Device represents a CDI device of a Spec. +type Device struct { + *cdi.Device + spec *Spec +} + +// Create a new Device, associate it with the given Spec. +func newDevice(spec *Spec, d cdi.Device) (*Device, error) { + dev := &Device{ + Device: &d, + spec: spec, + } + + if err := dev.validate(); err != nil { + return nil, err + } + + return dev, nil +} + +// GetSpec returns the Spec this device is defined in. +func (d *Device) GetSpec() *Spec { + return d.spec +} + +// GetQualifiedName returns the qualified name for this device. +func (d *Device) GetQualifiedName() string { + return QualifiedName(d.spec.GetVendor(), d.spec.GetClass(), d.Name) +} + +// ApplyEdits applies the device-speific container edits to an OCI Spec. +func (d *Device) ApplyEdits(ociSpec *oci.Spec) error { + e := ContainerEdits{&d.ContainerEdits} + return e.Apply(ociSpec) +} + +// Validate the device. +func (d *Device) validate() error { + if err := ValidateDeviceName(d.Name); err != nil { + return err + } + edits := ContainerEdits{&d.ContainerEdits} + if edits.isEmpty() { + return errors.Errorf("invalid device, empty device edits") + } + if err := edits.Validate(); err != nil { + return errors.Wrapf(err, "invalid device %q", d.Name) + } + return nil +} diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/doc.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/doc.go new file mode 100644 index 000000000..4fcdc44db --- /dev/null +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/doc.go @@ -0,0 +1,130 @@ +// Package cdi has the primary purpose of providing an API for +// interacting with CDI and consuming CDI devices. +// +// For more information about Container Device Interface, please refer to +// https://github.com/container-orchestrated-devices/container-device-interface +// +// Container Device Interface +// +// Container Device Interface, or CDI for short, provides comprehensive +// third party device support for container runtimes. CDI uses vendor +// provided specification files, CDI Specs for short, to describe how a +// container's runtime environment should be modified when one or more +// of the vendor-specific devices is injected into the container. Beyond +// describing the low level platform-specific details of how to gain +// basic access to a device, CDI Specs allow more fine-grained device +// initialization, and the automatic injection of any necessary vendor- +// or device-specific software that might be required for a container +// to use a device or take full advantage of it. +// +// In the CDI device model containers request access to a device using +// fully qualified device names, qualified names for short, consisting of +// a vendor identifier, a device class and a device name or identifier. +// These pieces of information together uniquely identify a device among +// all device vendors, classes and device instances. +// +// This package implements an API for easy consumption of CDI. The API +// implements discovery, loading and caching of CDI Specs and injection +// of CDI devices into containers. This is the most common functionality +// the vast majority of CDI consumers need. The API should be usable both +// by OCI runtime clients and runtime implementations. +// +// CDI Registry +// +// The primary interface to interact with CDI devices is the Registry. It +// is essentially a cache of all Specs and devices discovered in standard +// CDI directories on the host. The registry has two main functionality, +// injecting devices into an OCI Spec and refreshing the cache of CDI +// Specs and devices. +// +// Device Injection +// +// Using the Registry one can inject CDI devices into a container with code +// similar to the following snippet: +// +// import ( +// "fmt" +// "strings" +// +// "github.com/pkg/errors" +// log "github.com/sirupsen/logrus" +// +// "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" +// oci "github.com/opencontainers/runtime-spec/specs-go" +// ) +// +// func injectCDIDevices(spec *oci.Spec, devices []string) error { +// log.Debug("pristine OCI Spec: %s", dumpSpec(spec)) +// +// unresolved, err := cdi.GetRegistry().InjectDevices(spec, devices) +// if err != nil { +// return errors.Wrap(err, "CDI device injection failed") +// } +// +// log.Debug("CDI-updated OCI Spec: %s", dumpSpec(spec)) +// return nil +// } +// +// Cache Refresh +// +// In a runtime implementation one typically wants to make sure the +// CDI Spec cache is up to date before performing device injection. +// A code snippet similar to the following accmplishes that: +// +// import ( +// "fmt" +// "strings" +// +// "github.com/pkg/errors" +// log "github.com/sirupsen/logrus" +// +// "github.com/container-orchestrated-devices/container-device-interface/pkg/cdi" +// oci "github.com/opencontainers/runtime-spec/specs-go" +// ) +// +// func injectCDIDevices(spec *oci.Spec, devices []string) error { +// registry := cdi.GetRegistry() +// +// if err := registry.Refresh(); err != nil { +// // Note: +// // It is up to the implementation to decide whether +// // to abort injection on errors. A failed Refresh() +// // does not necessarily render the registry unusable. +// // For instance, a parse error in a Spec file for +// // vendor A does not have any effect on devices of +// // vendor B... +// log.Warnf("pre-injection Refresh() failed: %v", err) +// } +// +// log.Debug("pristine OCI Spec: %s", dumpSpec(spec)) +// +// unresolved, err := registry.InjectDevices(spec, devices) +// if err != nil { +// return errors.Wrap(err, "CDI device injection failed") +// } +// +// log.Debug("CDI-updated OCI Spec: %s", dumpSpec(spec)) +// return nil +// } +// +// Generated Spec Files, Multiple Directories, Device Precedence +// +// There are systems where the set of available or usable CDI devices +// changes dynamically and this needs to be reflected in the CDI Specs. +// This is done by dynamically regenerating CDI Spec files which are +// affected by these changes. +// +// CDI can collect Spec files from multiple directories. Spec files are +// automatically assigned priorities according to which directory they +// were loaded from. The later a directory occurs in the list of CDI +// directories to scan, the higher priority Spec files loaded from that +// directory are assigned to. When two or more Spec files define the +// same device, conflict is resolved by chosing the definition from the +// Spec file with the highest priority. +// +// The default CDI directory configuration is chosen to encourage +// separating dynamically generated CDI Spec files from static ones. +// The default directories are '/etc/cdi' and '/var/run/cdi'. By putting +// dynamically generated Spec files under '/var/run/cdi', those take +// precedence over static ones in '/etc/cdi'. +package cdi diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/qualified-device.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/qualified-device.go new file mode 100644 index 000000000..54f19143c --- /dev/null +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/qualified-device.go @@ -0,0 +1,203 @@ +/* + Copyright © 2021 The CDI Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cdi + +import ( + "strings" + + "github.com/pkg/errors" +) + +// QualifiedName returns the qualified name for a device. +// The syntax for a qualified device names is +// "<vendor>/<class>=<name>". +// A valid vendor name may contain the following runes: +// 'A'-'Z', 'a'-'z', '0'-'9', '.', '-', '_'. +// A valid class name may contain the following runes: +// 'A'-'Z', 'a'-'z', '0'-'9', '-', '_'. +// A valid device name may containe the following runes: +// 'A'-'Z', 'a'-'z', '0'-'9', '-', '_', '.', ':' +func QualifiedName(vendor, class, name string) string { + return vendor + "/" + class + "=" + name +} + +// IsQualifiedName tests if a device name is qualified. +func IsQualifiedName(device string) bool { + _, _, _, err := ParseQualifiedName(device) + return err == nil +} + +// ParseQualifiedName splits a qualified name into device vendor, class, +// and name. If the device fails to parse as a qualified name, or if any +// of the split components fail to pass syntax validation, vendor and +// class are returned as empty, together with the verbatim input as the +// name and an error describing the reason for failure. +func ParseQualifiedName(device string) (string, string, string, error) { + vendor, class, name := ParseDevice(device) + + if vendor == "" { + return "", "", device, errors.Errorf("unqualified device %q, missing vendor", device) + } + if class == "" { + return "", "", device, errors.Errorf("unqualified device %q, missing class", device) + } + if name == "" { + return "", "", device, errors.Errorf("unqualified device %q, missing device name", device) + } + + if err := ValidateVendorName(vendor); err != nil { + return "", "", device, errors.Wrapf(err, "invalid device %q", device) + } + if err := ValidateClassName(class); err != nil { + return "", "", device, errors.Wrapf(err, "invalid device %q", device) + } + if err := ValidateDeviceName(name); err != nil { + return "", "", device, errors.Wrapf(err, "invalid device %q", device) + } + + return vendor, class, name, nil +} + +// ParseDevice tries to split a device name into vendor, class, and name. +// If this fails, for instance in the case of unqualified device names, +// ParseDevice returns an empty vendor and class together with name set +// to the verbatim input. +func ParseDevice(device string) (string, string, string) { + if device == "" || device[0] == '/' { + return "", "", device + } + + parts := strings.SplitN(device, "=", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", "", device + } + + name := parts[1] + vendor, class := ParseQualifier(parts[0]) + if vendor == "" { + return "", "", device + } + + return vendor, class, name +} + +// ParseQualifier splits a device qualifier into vendor and class. +// The syntax for a device qualifier is +// "<vendor>/<class>" +// If parsing fails, an empty vendor and the class set to the +// verbatim input is returned. +func ParseQualifier(kind string) (string, string) { + parts := strings.SplitN(kind, "/", 2) + if len(parts) != 2 || parts[0] == "" || parts[1] == "" { + return "", kind + } + return parts[0], parts[1] +} + +// ValidateVendorName checks the validity of a vendor name. +// A vendor name may contain the following ASCII characters: +// - upper- and lowercase letters ('A'-'Z', 'a'-'z') +// - digits ('0'-'9') +// - underscore, dash, and dot ('_', '-', and '.') +func ValidateVendorName(vendor string) error { + if vendor == "" { + return errors.Errorf("invalid (empty) vendor name") + } + if !isLetter(rune(vendor[0])) { + return errors.Errorf("invalid vendor %q, should start with letter", vendor) + } + for _, c := range string(vendor[1 : len(vendor)-1]) { + switch { + case isAlphaNumeric(c): + case c == '_' || c == '-' || c == '.': + default: + return errors.Errorf("invalid character '%c' in vendor name %q", + c, vendor) + } + } + if !isAlphaNumeric(rune(vendor[len(vendor)-1])) { + return errors.Errorf("invalid vendor %q, should end with letter", vendor) + } + + return nil +} + +// ValidateClassName checks the validity of class name. +// A class name may contain the following ASCII characters: +// - upper- and lowercase letters ('A'-'Z', 'a'-'z') +// - digits ('0'-'9') +// - underscore and dash ('_', '-') +func ValidateClassName(class string) error { + if class == "" { + return errors.Errorf("invalid (empty) device class") + } + if !isLetter(rune(class[0])) { + return errors.Errorf("invalid class %q, should start with letter", class) + } + for _, c := range string(class[1 : len(class)-1]) { + switch { + case isAlphaNumeric(c): + case c == '_' || c == '-': + default: + return errors.Errorf("invalid character '%c' in device class %q", + c, class) + } + } + if !isAlphaNumeric(rune(class[len(class)-1])) { + return errors.Errorf("invalid class %q, should end with letter", class) + } + return nil +} + +// ValidateDeviceName checks the validity of a device name. +// A device name may contain the following ASCII characters: +// - upper- and lowercase letters ('A'-'Z', 'a'-'z') +// - digits ('0'-'9') +// - underscore, dash, dot, colon ('_', '-', '.', ':') +func ValidateDeviceName(name string) error { + if name == "" { + return errors.Errorf("invalid (empty) device name") + } + if !isLetter(rune(name[0])) { + return errors.Errorf("invalid name %q, should start with letter", name) + } + for _, c := range string(name[1 : len(name)-1]) { + switch { + case isAlphaNumeric(c): + case c == '_' || c == '-' || c == '.' || c == ':': + default: + return errors.Errorf("invalid character '%c' in device name %q", + c, name) + } + } + if !isAlphaNumeric(rune(name[len(name)-1])) { + return errors.Errorf("invalid name %q, should start with letter", name) + } + return nil +} + +func isLetter(c rune) bool { + return ('A' <= c && c <= 'Z') || ('a' <= c && c <= 'z') +} + +func isDigit(c rune) bool { + return '0' <= c && c <= '9' +} + +func isAlphaNumeric(c rune) bool { + return isLetter(c) || isDigit(c) +} diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/registry.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/registry.go new file mode 100644 index 000000000..fa6e0af69 --- /dev/null +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/registry.go @@ -0,0 +1,139 @@ +/* + Copyright © 2021 The CDI Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cdi + +import ( + "sync" + + oci "github.com/opencontainers/runtime-spec/specs-go" +) + +// +// Registry keeps a cache of all CDI Specs installed or generated on +// the host. Registry is the primary interface clients should use to +// interact with CDI. +// +// The most commonly used Registry functions are for refreshing the +// registry and injecting CDI devices into an OCI Spec. +// +type Registry interface { + RegistryResolver + RegistryRefresher + DeviceDB() RegistryDeviceDB + SpecDB() RegistrySpecDB +} + +// RegistryRefresher is the registry interface for refreshing the +// cache of CDI Specs and devices. +// +// Refresh rescans all CDI Spec directories and updates the +// state of the cache to reflect any changes. It returns any +// errors encountered during the refresh. +// +// GetErrors returns all errors encountered for any of the scanned +// Spec files during the last cache refresh. +// +// GetSpecDirectories returns the set up CDI Spec directories +// currently in use. The directories are returned in the scan +// order of Refresh(). +type RegistryRefresher interface { + Refresh() error + GetErrors() map[string][]error + GetSpecDirectories() []string +} + +// RegistryResolver is the registry interface for injecting CDI +// devices into an OCI Spec. +// +// InjectDevices takes an OCI Spec and injects into it a set of +// CDI devices given by qualified name. It returns the names of +// any unresolved devices and an error if injection fails. +type RegistryResolver interface { + InjectDevices(spec *oci.Spec, device ...string) (unresolved []string, err error) +} + +// RegistryDeviceDB is the registry interface for querying devices. +// +// GetDevice returns the CDI device for the given qualified name. If +// the device is not GetDevice returns nil. +// +// ListDevices returns a slice with the names of qualified device +// known. The returned slice is sorted. +type RegistryDeviceDB interface { + GetDevice(device string) *Device + ListDevices() []string +} + +// RegistrySpecDB is the registry interface for querying CDI Specs. +// +// ListVendors returns a slice with all vendors known. The returned +// slice is sorted. +// +// ListClasses returns a slice with all classes known. The returned +// slice is sorted. +// +// GetVendorSpecs returns a slice of all Specs for the vendor. +// +// GetSpecErrors returns any errors for the Spec encountered during +// the last cache refresh. +type RegistrySpecDB interface { + ListVendors() []string + ListClasses() []string + GetVendorSpecs(vendor string) []*Spec + GetSpecErrors(*Spec) []error +} + +type registry struct { + *Cache +} + +var _ Registry = ®istry{} + +var ( + reg *registry + initOnce sync.Once +) + +// GetRegistry returns the CDI registry. If any options are given, those +// are applied to the registry. +func GetRegistry(options ...Option) Registry { + var new bool + initOnce.Do(func() { + reg, _ = getRegistry(options...) + new = true + }) + if !new && len(options) > 0 { + reg.Configure(options...) + reg.Refresh() + } + return reg +} + +// DeviceDB returns the registry interface for querying devices. +func (r *registry) DeviceDB() RegistryDeviceDB { + return r +} + +// SpecDB returns the registry interface for querying Specs. +func (r *registry) SpecDB() RegistrySpecDB { + return r +} + +func getRegistry(options ...Option) (*registry, error) { + c, err := NewCache(options...) + return ®istry{c}, err +} diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec-dirs.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec-dirs.go new file mode 100644 index 000000000..ad017fec7 --- /dev/null +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec-dirs.go @@ -0,0 +1,110 @@ +/* + Copyright © 2021 The CDI Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cdi + +import ( + "os" + "path/filepath" + + "github.com/pkg/errors" +) + +const ( + // DefaultStaticDir is the default directory for static CDI Specs. + DefaultStaticDir = "/etc/cdi" + // DefaultDynamicDir is the default directory for generated CDI Specs + DefaultDynamicDir = "/var/run/cdi" +) + +var ( + // DefaultSpecDirs is the default Spec directory configuration. + // While altering this variable changes the package defaults, + // the preferred way of overriding the default directories is + // to use a WithSpecDirs options. Otherwise the change is only + // effective if it takes place before creating the Registry or + // other Cache instances. + DefaultSpecDirs = []string{DefaultStaticDir, DefaultDynamicDir} + // ErrStopScan can be returned from a ScanSpecFunc to stop the scan. + ErrStopScan = errors.New("stop Spec scan") +) + +// WithSpecDirs returns an option to override the CDI Spec directories. +func WithSpecDirs(dirs ...string) Option { + return func(c *Cache) error { + c.specDirs = make([]string, len(dirs)) + for i, dir := range dirs { + c.specDirs[i] = filepath.Clean(dir) + } + return nil + } +} + +// scanSpecFunc is a function for processing CDI Spec files. +type scanSpecFunc func(string, int, *Spec, error) error + +// ScanSpecDirs scans the given directories looking for CDI Spec files, +// which are all files with a '.json' or '.yaml' suffix. For every Spec +// file discovered, ScanSpecDirs loads a Spec from the file then calls +// the scan function passing it the path to the file, the priority (the +// index of the directory in the slice of directories given), the Spec +// itself, and any error encountered while loading the Spec. +// +// Scanning stops once all files have been processed or when the scan +// function returns an error. The result of ScanSpecDirs is the error +// returned by the scan function, if any. The special error ErrStopScan +// can be used to terminate the scan gracefully without ScanSpecDirs +// returning an error. ScanSpecDirs silently skips any subdirectories. +func scanSpecDirs(dirs []string, scanFn scanSpecFunc) error { + var ( + spec *Spec + err error + ) + + for priority, dir := range dirs { + err = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { + // for initial stat failure Walk calls us with nil info + if info == nil { + return err + } + // first call from Walk is for dir itself, others we skip + if info.IsDir() { + if path == dir { + return nil + } + return filepath.SkipDir + } + + // ignore obviously non-Spec files + if ext := filepath.Ext(path); ext != ".json" && ext != ".yaml" { + return nil + } + + if err != nil { + return scanFn(path, priority, nil, err) + } + + spec, err = ReadSpec(path, priority) + return scanFn(path, priority, spec, err) + }) + + if err != nil && err != ErrStopScan { + return err + } + } + + return nil +} diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec.go new file mode 100644 index 000000000..20f188498 --- /dev/null +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/cdi/spec.go @@ -0,0 +1,172 @@ +/* + Copyright © 2021 The CDI Authors + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package cdi + +import ( + "io/ioutil" + "os" + "path/filepath" + + oci "github.com/opencontainers/runtime-spec/specs-go" + "github.com/pkg/errors" + "sigs.k8s.io/yaml" + + cdi "github.com/container-orchestrated-devices/container-device-interface/specs-go" +) + +var ( + // Valid CDI Spec versions. + validSpecVersions = map[string]struct{}{ + "0.1.0": {}, + "0.2.0": {}, + } +) + +// Spec represents a single CDI Spec. It is usually loaded from a +// file and stored in a cache. The Spec has an associated priority. +// This priority is inherited from the associated priority of the +// CDI Spec directory that contains the CDI Spec file and is used +// to resolve conflicts if multiple CDI Spec files contain entries +// for the same fully qualified device. +type Spec struct { + *cdi.Spec + vendor string + class string + path string + priority int + devices map[string]*Device +} + +// ReadSpec reads the given CDI Spec file. The resulting Spec is +// assigned the given priority. If reading or parsing the Spec +// data fails ReadSpec returns a nil Spec and an error. +func ReadSpec(path string, priority int) (*Spec, error) { + data, err := ioutil.ReadFile(path) + switch { + case os.IsNotExist(err): + return nil, err + case err != nil: + return nil, errors.Wrapf(err, "failed to read CDI Spec %q", path) + } + + raw, err := parseSpec(data) + if err != nil { + return nil, errors.Wrapf(err, "failed to parse CDI Spec %q", path) + } + + return NewSpec(raw, path, priority) +} + +// NewSpec creates a new Spec from the given CDI Spec data. The +// Spec is marked as loaded from the given path with the given +// priority. If Spec data validation fails NewSpec returns a nil +// Spec and an error. +func NewSpec(raw *cdi.Spec, path string, priority int) (*Spec, error) { + var err error + + spec := &Spec{ + Spec: raw, + path: filepath.Clean(path), + priority: priority, + } + + spec.vendor, spec.class = ParseQualifier(spec.Kind) + + if spec.devices, err = spec.validate(); err != nil { + return nil, errors.Wrap(err, "invalid CDI Spec") + } + + return spec, nil +} + +// GetVendor returns the vendor of this Spec. +func (s *Spec) GetVendor() string { + return s.vendor +} + +// GetClass returns the device class of this Spec. +func (s *Spec) GetClass() string { + return s.class +} + +// GetDevice returns the device for the given unqualified name. +func (s *Spec) GetDevice(name string) *Device { + return s.devices[name] +} + +// GetPath returns the filesystem path of this Spec. +func (s *Spec) GetPath() string { + return s.path +} + +// GetPriority returns the priority of this Spec. +func (s *Spec) GetPriority() int { + return s.priority +} + +// ApplyEdits applies the Spec's global-scope container edits to an OCI Spec. +func (s *Spec) ApplyEdits(ociSpec *oci.Spec) error { + e := ContainerEdits{&s.ContainerEdits} + return e.Apply(ociSpec) +} + +// Validate the Spec. +func (s *Spec) validate() (map[string]*Device, error) { + if _, ok := validSpecVersions[s.Version]; !ok { + return nil, errors.Errorf("invalid version %q", s.Version) + } + if err := ValidateVendorName(s.vendor); err != nil { + return nil, err + } + if err := ValidateClassName(s.class); err != nil { + return nil, err + } + edits := &ContainerEdits{&s.ContainerEdits} + if err := edits.Validate(); err != nil { + return nil, err + } + + devices := make(map[string]*Device) + for _, d := range s.Devices { + dev, err := newDevice(s, d) + if err != nil { + return nil, errors.Wrapf(err, "failed add device %q", d.Name) + } + if _, conflict := devices[d.Name]; conflict { + return nil, errors.Errorf("invalid spec, multiple device %q", d.Name) + } + devices[d.Name] = dev + } + + return devices, nil +} + +// Parse raw CDI Spec file data. +func parseSpec(data []byte) (*cdi.Spec, error) { + raw := &cdi.Spec{} + err := yaml.UnmarshalStrict(data, &raw) + if err != nil { + return nil, errors.Wrap(err, "failed to unmarshal CDI Spec") + } + return raw, validateJSONSchema(raw) +} + +// Validate CDI Spec against JSON Schema. +func validateJSONSchema(raw *cdi.Spec) error { + // TODO + return nil +} diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/devices.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/devices.go deleted file mode 100644 index e66fd36c0..000000000 --- a/vendor/github.com/container-orchestrated-devices/container-device-interface/pkg/devices.go +++ /dev/null @@ -1,180 +0,0 @@ -package pkg - -import ( - "encoding/json" - "fmt" - "io/ioutil" - "os" - "path/filepath" - "strings" - - cdispec "github.com/container-orchestrated-devices/container-device-interface/specs-go" - spec "github.com/opencontainers/runtime-spec/specs-go" -) - -const ( - root = "/etc/cdi" -) - -func collectCDISpecs() (map[string]*cdispec.Spec, error) { - var files []string - vendor := make(map[string]*cdispec.Spec) - - err := filepath.Walk(root, func(path string, info os.FileInfo, err error) error { - if info == nil || info.IsDir() { - return nil - } - - if filepath.Ext(path) != ".json" { - return nil - } - - files = append(files, path) - return nil - }) - - if err != nil { - return nil, err - } - - for _, path := range files { - spec, err := loadCDIFile(path) - if err != nil { - continue - } - - if _, ok := vendor[spec.Kind]; ok { - continue - } - - vendor[spec.Kind] = spec - } - - return vendor, nil -} - -// TODO: Validate (e.g: duplicate device names) -func loadCDIFile(path string) (*cdispec.Spec, error) { - file, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - - var spec *cdispec.Spec - err = json.Unmarshal([]byte(file), &spec) - if err != nil { - return nil, err - } - - return spec, nil -} - -/* -* Pattern "vendor.com/device=myDevice" with the vendor being optional - */ -func extractVendor(dev string) (string, string) { - if strings.IndexByte(dev, '=') == -1 { - return "", dev - } - - split := strings.SplitN(dev, "=", 2) - return split[0], split[1] -} - -// GetCDIForDevice returns the CDI specification that matches the device name the user provided. -func GetCDIForDevice(dev string, specs map[string]*cdispec.Spec) (*cdispec.Spec, error) { - vendor, device := extractVendor(dev) - - if vendor != "" { - s, ok := specs[vendor] - if !ok { - return nil, fmt.Errorf("Could not find vendor %q for device %q", vendor, device) - } - - for _, d := range s.Devices { - if d.Name != device { - continue - } - - return s, nil - } - - return nil, fmt.Errorf("Could not find device %q for vendor %q", device, vendor) - } - - var found []*cdispec.Spec - var vendors []string - for vendor, spec := range specs { - - for _, d := range spec.Devices { - if d.Name != device { - continue - } - - found = append(found, spec) - vendors = append(vendors, vendor) - } - } - - if len(found) > 1 { - return nil, fmt.Errorf("%q is ambiguous and currently refers to multiple devices from different vendors: %q", dev, vendors) - } - - if len(found) == 1 { - return found[0], nil - } - - return nil, fmt.Errorf("Could not find device %q", dev) -} - -// HasDevice returns true if a device is a CDI device -// an error may be returned in cases where permissions may be required -func HasDevice(dev string) (bool, error) { - specs, err := collectCDISpecs() - if err != nil { - return false, err - } - - d, err := GetCDIForDevice(dev, specs) - if err != nil { - return false, err - } - - return d != nil, nil -} - -// UpdateOCISpecForDevices updates the given OCI spec based on the requested CDI devices -func UpdateOCISpecForDevices(ociconfig *spec.Spec, devs []string) error { - specs, err := collectCDISpecs() - if err != nil { - return err - } - - return UpdateOCISpecForDevicesWithSpec(ociconfig, devs, specs) -} - -// UpdateOCISpecForDevicesWithLoggerAndSpecs is mainly used for testing -func UpdateOCISpecForDevicesWithSpec(ociconfig *spec.Spec, devs []string, specs map[string]*cdispec.Spec) error { - edits := make(map[string]*cdispec.Spec) - - for _, d := range devs { - spec, err := GetCDIForDevice(d, specs) - if err != nil { - return err - } - - edits[spec.Kind] = spec - err = cdispec.ApplyOCIEditsForDevice(ociconfig, spec, d) - if err != nil { - return err - } - } - - for _, spec := range edits { - if err := cdispec.ApplyOCIEdits(ociconfig, spec); err != nil { - return err - } - } - - return nil -} diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/config.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/config.go index 0223bb703..20914e5b6 100644 --- a/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/config.go +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/config.go @@ -1,20 +1,20 @@ package specs +import "os" + // Spec is the base configuration for CDI type Spec struct { Version string `json:"cdiVersion"` Kind string `json:"kind"` - KindShort []string `json:"kindShort,omitempty"` ContainerRuntime []string `json:"containerRuntime,omitempty"` - Devices []Devices `json:"devices"` + Devices []Device `json:"devices"` ContainerEdits ContainerEdits `json:"containerEdits,omitempty"` } -// Devices is a "Device" a container runtime can add to a container -type Devices struct { +// Device is a "Device" a container runtime can add to a container +type Device struct { Name string `json:"name"` - NameShort []string `json:"nameShort"` ContainerEdits ContainerEdits `json:"containerEdits"` } @@ -28,9 +28,14 @@ type ContainerEdits struct { // DeviceNode represents a device node that needs to be added to the OCI spec. type DeviceNode struct { - HostPath string `json:"hostPath"` - ContainerPath string `json:"containerPath"` - Permissions []string `json:"permissions,omitempty"` + Path string `json:"path"` + Type string `json:"type,omitempty"` + Major int64 `json:"major,omitempty"` + Minor int64 `json:"minor,omitempty"` + FileMode *os.FileMode `json:"fileMode,omitempty"` + Permissions string `json:"permissions,omitempty"` + UID *uint32 `json:"uid,omitempty"` + GID *uint32 `json:"gid,omitempty"` } // Mount represents a mount that needs to be added to the OCI spec. diff --git a/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/oci.go b/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/oci.go index c59cda55d..10bc9fa23 100644 --- a/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/oci.go +++ b/vendor/github.com/container-orchestrated-devices/container-device-interface/specs-go/oci.go @@ -37,20 +37,21 @@ func ApplyEditsToOCISpec(config *spec.Spec, edits *ContainerEdits) error { } if len(edits.Env) > 0 { - if config.Process == nil { config.Process = &spec.Process{} } - config.Process.Env = append(config.Process.Env, edits.Env...) } for _, d := range edits.DeviceNodes { - config.Mounts = append(config.Mounts, toOCIDevice(d)) + if config.Linux == nil { + config.Linux = &spec.Linux{} + } + config.Linux.Devices = append(config.Linux.Devices, d.ToOCI()) } for _, m := range edits.Mounts { - config.Mounts = append(config.Mounts, toOCIMount(m)) + config.Mounts = append(config.Mounts, m.ToOCI()) } for _, h := range edits.Hooks { @@ -59,17 +60,17 @@ func ApplyEditsToOCISpec(config *spec.Spec, edits *ContainerEdits) error { } switch h.HookName { case "prestart": - config.Hooks.Prestart = append(config.Hooks.Prestart, toOCIHook(h)) + config.Hooks.Prestart = append(config.Hooks.Prestart, h.ToOCI()) case "createRuntime": - config.Hooks.CreateRuntime = append(config.Hooks.CreateRuntime, toOCIHook(h)) + config.Hooks.CreateRuntime = append(config.Hooks.CreateRuntime, h.ToOCI()) case "createContainer": - config.Hooks.CreateContainer = append(config.Hooks.CreateContainer, toOCIHook(h)) + config.Hooks.CreateContainer = append(config.Hooks.CreateContainer, h.ToOCI()) case "startContainer": - config.Hooks.StartContainer = append(config.Hooks.StartContainer, toOCIHook(h)) + config.Hooks.StartContainer = append(config.Hooks.StartContainer, h.ToOCI()) case "poststart": - config.Hooks.Poststart = append(config.Hooks.Poststart, toOCIHook(h)) + config.Hooks.Poststart = append(config.Hooks.Poststart, h.ToOCI()) case "poststop": - config.Hooks.Poststop = append(config.Hooks.Poststop, toOCIHook(h)) + config.Hooks.Poststop = append(config.Hooks.Poststop, h.ToOCI()) default: fmt.Printf("CDI: Unknown hook %q\n", h.HookName) } @@ -78,7 +79,8 @@ func ApplyEditsToOCISpec(config *spec.Spec, edits *ContainerEdits) error { return nil } -func toOCIHook(h *Hook) spec.Hook { +// ToOCI returns the opencontainers runtime Spec Hook for this Hook. +func (h *Hook) ToOCI() spec.Hook { return spec.Hook{ Path: h.Path, Args: h.Args, @@ -87,7 +89,8 @@ func toOCIHook(h *Hook) spec.Hook { } } -func toOCIMount(m *Mount) spec.Mount { +// ToOCI returns the opencontainers runtime Spec Mount for this Mount. +func (m *Mount) ToOCI() spec.Mount { return spec.Mount{ Source: m.HostPath, Destination: m.ContainerPath, @@ -95,10 +98,15 @@ func toOCIMount(m *Mount) spec.Mount { } } -func toOCIDevice(d *DeviceNode) spec.Mount { - return spec.Mount{ - Source: d.HostPath, - Destination: d.ContainerPath, - Options: d.Permissions, +// ToOCI returns the opencontainers runtime Spec LinuxDevice for this DeviceNode. +func (d *DeviceNode) ToOCI() spec.LinuxDevice { + return spec.LinuxDevice{ + Path: d.Path, + Type: d.Type, + Major: d.Major, + Minor: d.Minor, + FileMode: d.FileMode, + UID: d.UID, + GID: d.GID, } } diff --git a/vendor/modules.txt b/vendor/modules.txt index bf73708d7..671f37644 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -57,9 +57,9 @@ github.com/checkpoint-restore/go-criu/v5/rpc github.com/checkpoint-restore/go-criu/v5/stats # github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e github.com/chzyer/readline -# github.com/container-orchestrated-devices/container-device-interface v0.0.0-20210325223243-f99e8b6c10b9 +# github.com/container-orchestrated-devices/container-device-interface v0.0.0-20220111162300-46367ec063fd ## explicit -github.com/container-orchestrated-devices/container-device-interface/pkg +github.com/container-orchestrated-devices/container-device-interface/pkg/cdi github.com/container-orchestrated-devices/container-device-interface/specs-go # github.com/containerd/cgroups v1.0.1 github.com/containerd/cgroups/stats/v1 @@ -862,7 +862,6 @@ google.golang.org/protobuf/types/known/anypb google.golang.org/protobuf/types/known/durationpb google.golang.org/protobuf/types/known/timestamppb # gopkg.in/fsnotify.v1 v1.4.7 -## explicit gopkg.in/fsnotify.v1 # gopkg.in/inf.v0 v0.9.1 gopkg.in/inf.v0 @@ -908,4 +907,6 @@ k8s.io/apimachinery/third_party/forked/golang/reflect k8s.io/klog/v2 # sigs.k8s.io/structured-merge-diff/v4 v4.1.2 sigs.k8s.io/structured-merge-diff/v4/value +# sigs.k8s.io/yaml v1.3.0 +sigs.k8s.io/yaml # github.com/onsi/gomega => github.com/onsi/gomega v1.16.0 diff --git a/vendor/sigs.k8s.io/yaml/.gitignore b/vendor/sigs.k8s.io/yaml/.gitignore new file mode 100644 index 000000000..2dc92904e --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/.gitignore @@ -0,0 +1,24 @@ +# OSX leaves these everywhere on SMB shares +._* + +# Eclipse files +.classpath +.project +.settings/** + +# Idea files +.idea/** +.idea/ + +# Emacs save files +*~ + +# Vim-related files +[._]*.s[a-w][a-z] +[._]s[a-w][a-z] +*.un~ +Session.vim +.netrwhist + +# Go test binaries +*.test diff --git a/vendor/sigs.k8s.io/yaml/.travis.yml b/vendor/sigs.k8s.io/yaml/.travis.yml new file mode 100644 index 000000000..54ed8f9cb --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/.travis.yml @@ -0,0 +1,12 @@ +language: go +arch: arm64 +dist: focal +go: 1.15.x +script: + - diff -u <(echo -n) <(gofmt -d *.go) + - diff -u <(echo -n) <(golint $(go list -e ./...) | grep -v YAMLToJSON) + - GO111MODULE=on go vet . + - GO111MODULE=on go test -v -race ./... + - git diff --exit-code +install: + - GO111MODULE=off go get golang.org/x/lint/golint diff --git a/vendor/sigs.k8s.io/yaml/CONTRIBUTING.md b/vendor/sigs.k8s.io/yaml/CONTRIBUTING.md new file mode 100644 index 000000000..de4711513 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/CONTRIBUTING.md @@ -0,0 +1,31 @@ +# Contributing Guidelines + +Welcome to Kubernetes. We are excited about the prospect of you joining our [community](https://github.com/kubernetes/community)! The Kubernetes community abides by the CNCF [code of conduct](code-of-conduct.md). Here is an excerpt: + +_As contributors and maintainers of this project, and in the interest of fostering an open and welcoming community, we pledge to respect all people who contribute through reporting issues, posting feature requests, updating documentation, submitting pull requests or patches, and other activities._ + +## Getting Started + +We have full documentation on how to get started contributing here: + +<!--- +If your repo has certain guidelines for contribution, put them here ahead of the general k8s resources +--> + +- [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests +- [Kubernetes Contributor Guide](http://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](http://git.k8s.io/community/contributors/guide#contributing) +- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet.md) - Common resources for existing developers + +## Mentorship + +- [Mentoring Initiatives](https://git.k8s.io/community/mentoring) - We have a diverse set of mentorship programs available that are always looking for volunteers! + +<!--- +Custom Information - if you're copying this template for the first time you can add custom content here, for example: + +## Contact Information + +- [Slack channel](https://kubernetes.slack.com/messages/kubernetes-users) - Replace `kubernetes-users` with your slack channel string, this will send users directly to your channel. +- [Mailing list](URL) + +--> diff --git a/vendor/sigs.k8s.io/yaml/LICENSE b/vendor/sigs.k8s.io/yaml/LICENSE new file mode 100644 index 000000000..7805d36de --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/LICENSE @@ -0,0 +1,50 @@ +The MIT License (MIT) + +Copyright (c) 2014 Sam Ghods + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. + + +Copyright (c) 2012 The Go Authors. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/vendor/sigs.k8s.io/yaml/OWNERS b/vendor/sigs.k8s.io/yaml/OWNERS new file mode 100644 index 000000000..325b40b07 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/OWNERS @@ -0,0 +1,27 @@ +# See the OWNERS docs at https://go.k8s.io/owners + +approvers: +- dims +- lavalamp +- smarterclayton +- deads2k +- sttts +- liggitt +- caesarxuchao +reviewers: +- dims +- thockin +- lavalamp +- smarterclayton +- wojtek-t +- deads2k +- derekwaynecarr +- caesarxuchao +- mikedanese +- liggitt +- gmarek +- sttts +- ncdc +- tallclair +labels: +- sig/api-machinery diff --git a/vendor/sigs.k8s.io/yaml/README.md b/vendor/sigs.k8s.io/yaml/README.md new file mode 100644 index 000000000..e81cc426b --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/README.md @@ -0,0 +1,123 @@ +# YAML marshaling and unmarshaling support for Go + +[](https://travis-ci.org/kubernetes-sigs/yaml) + +kubernetes-sigs/yaml is a permanent fork of [ghodss/yaml](https://github.com/ghodss/yaml). + +## Introduction + +A wrapper around [go-yaml](https://github.com/go-yaml/yaml) designed to enable a better way of handling YAML when marshaling to and from structs. + +In short, this library first converts YAML to JSON using go-yaml and then uses `json.Marshal` and `json.Unmarshal` to convert to or from the struct. This means that it effectively reuses the JSON struct tags as well as the custom JSON methods `MarshalJSON` and `UnmarshalJSON` unlike go-yaml. For a detailed overview of the rationale behind this method, [see this blog post](http://web.archive.org/web/20190603050330/http://ghodss.com/2014/the-right-way-to-handle-yaml-in-golang/). + +## Compatibility + +This package uses [go-yaml](https://github.com/go-yaml/yaml) and therefore supports [everything go-yaml supports](https://github.com/go-yaml/yaml#compatibility). + +## Caveats + +**Caveat #1:** When using `yaml.Marshal` and `yaml.Unmarshal`, binary data should NOT be preceded with the `!!binary` YAML tag. If you do, go-yaml will convert the binary data from base64 to native binary data, which is not compatible with JSON. You can still use binary in your YAML files though - just store them without the `!!binary` tag and decode the base64 in your code (e.g. in the custom JSON methods `MarshalJSON` and `UnmarshalJSON`). This also has the benefit that your YAML and your JSON binary data will be decoded exactly the same way. As an example: + +``` +BAD: + exampleKey: !!binary gIGC + +GOOD: + exampleKey: gIGC +... and decode the base64 data in your code. +``` + +**Caveat #2:** When using `YAMLToJSON` directly, maps with keys that are maps will result in an error since this is not supported by JSON. This error will occur in `Unmarshal` as well since you can't unmarshal map keys anyways since struct fields can't be keys. + +## Installation and usage + +To install, run: + +``` +$ go get sigs.k8s.io/yaml +``` + +And import using: + +``` +import "sigs.k8s.io/yaml" +``` + +Usage is very similar to the JSON library: + +```go +package main + +import ( + "fmt" + + "sigs.k8s.io/yaml" +) + +type Person struct { + Name string `json:"name"` // Affects YAML field names too. + Age int `json:"age"` +} + +func main() { + // Marshal a Person struct to YAML. + p := Person{"John", 30} + y, err := yaml.Marshal(p) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + fmt.Println(string(y)) + /* Output: + age: 30 + name: John + */ + + // Unmarshal the YAML back into a Person struct. + var p2 Person + err = yaml.Unmarshal(y, &p2) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + fmt.Println(p2) + /* Output: + {John 30} + */ +} +``` + +`yaml.YAMLToJSON` and `yaml.JSONToYAML` methods are also available: + +```go +package main + +import ( + "fmt" + + "sigs.k8s.io/yaml" +) + +func main() { + j := []byte(`{"name": "John", "age": 30}`) + y, err := yaml.JSONToYAML(j) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + fmt.Println(string(y)) + /* Output: + age: 30 + name: John + */ + j2, err := yaml.YAMLToJSON(y) + if err != nil { + fmt.Printf("err: %v\n", err) + return + } + fmt.Println(string(j2)) + /* Output: + {"age":30,"name":"John"} + */ +} +``` diff --git a/vendor/sigs.k8s.io/yaml/RELEASE.md b/vendor/sigs.k8s.io/yaml/RELEASE.md new file mode 100644 index 000000000..6b642464e --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/RELEASE.md @@ -0,0 +1,9 @@ +# Release Process + +The `yaml` Project is released on an as-needed basis. The process is as follows: + +1. An issue is proposing a new release with a changelog since the last release +1. All [OWNERS](OWNERS) must LGTM this release +1. An OWNER runs `git tag -s $VERSION` and inserts the changelog and pushes the tag with `git push $VERSION` +1. The release issue is closed +1. An announcement email is sent to `kubernetes-dev@googlegroups.com` with the subject `[ANNOUNCE] kubernetes-template-project $VERSION is released` diff --git a/vendor/sigs.k8s.io/yaml/SECURITY_CONTACTS b/vendor/sigs.k8s.io/yaml/SECURITY_CONTACTS new file mode 100644 index 000000000..0648a8ebf --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/SECURITY_CONTACTS @@ -0,0 +1,17 @@ +# Defined below are the security contacts for this repo. +# +# They are the contact point for the Product Security Team to reach out +# to for triaging and handling of incoming issues. +# +# The below names agree to abide by the +# [Embargo Policy](https://github.com/kubernetes/sig-release/blob/master/security-release-process-documentation/security-release-process.md#embargo-policy) +# and will be removed and replaced if they violate that agreement. +# +# DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE +# INSTRUCTIONS AT https://kubernetes.io/security/ + +cjcullen +jessfraz +liggitt +philips +tallclair diff --git a/vendor/sigs.k8s.io/yaml/code-of-conduct.md b/vendor/sigs.k8s.io/yaml/code-of-conduct.md new file mode 100644 index 000000000..0d15c00cf --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/code-of-conduct.md @@ -0,0 +1,3 @@ +# Kubernetes Community Code of Conduct + +Please refer to our [Kubernetes Community Code of Conduct](https://git.k8s.io/community/code-of-conduct.md) diff --git a/vendor/sigs.k8s.io/yaml/fields.go b/vendor/sigs.k8s.io/yaml/fields.go new file mode 100644 index 000000000..235b7f2cf --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/fields.go @@ -0,0 +1,502 @@ +// Copyright 2013 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package yaml + +import ( + "bytes" + "encoding" + "encoding/json" + "reflect" + "sort" + "strings" + "sync" + "unicode" + "unicode/utf8" +) + +// indirect walks down v allocating pointers as needed, +// until it gets to a non-pointer. +// if it encounters an Unmarshaler, indirect stops and returns that. +// if decodingNull is true, indirect stops at the last pointer so it can be set to nil. +func indirect(v reflect.Value, decodingNull bool) (json.Unmarshaler, encoding.TextUnmarshaler, reflect.Value) { + // If v is a named type and is addressable, + // start with its address, so that if the type has pointer methods, + // we find them. + if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() { + v = v.Addr() + } + for { + // Load value from interface, but only if the result will be + // usefully addressable. + if v.Kind() == reflect.Interface && !v.IsNil() { + e := v.Elem() + if e.Kind() == reflect.Ptr && !e.IsNil() && (!decodingNull || e.Elem().Kind() == reflect.Ptr) { + v = e + continue + } + } + + if v.Kind() != reflect.Ptr { + break + } + + if v.Elem().Kind() != reflect.Ptr && decodingNull && v.CanSet() { + break + } + if v.IsNil() { + if v.CanSet() { + v.Set(reflect.New(v.Type().Elem())) + } else { + v = reflect.New(v.Type().Elem()) + } + } + if v.Type().NumMethod() > 0 { + if u, ok := v.Interface().(json.Unmarshaler); ok { + return u, nil, reflect.Value{} + } + if u, ok := v.Interface().(encoding.TextUnmarshaler); ok { + return nil, u, reflect.Value{} + } + } + v = v.Elem() + } + return nil, nil, v +} + +// A field represents a single field found in a struct. +type field struct { + name string + nameBytes []byte // []byte(name) + equalFold func(s, t []byte) bool // bytes.EqualFold or equivalent + + tag bool + index []int + typ reflect.Type + omitEmpty bool + quoted bool +} + +func fillField(f field) field { + f.nameBytes = []byte(f.name) + f.equalFold = foldFunc(f.nameBytes) + return f +} + +// byName sorts field by name, breaking ties with depth, +// then breaking ties with "name came from json tag", then +// breaking ties with index sequence. +type byName []field + +func (x byName) Len() int { return len(x) } + +func (x byName) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byName) Less(i, j int) bool { + if x[i].name != x[j].name { + return x[i].name < x[j].name + } + if len(x[i].index) != len(x[j].index) { + return len(x[i].index) < len(x[j].index) + } + if x[i].tag != x[j].tag { + return x[i].tag + } + return byIndex(x).Less(i, j) +} + +// byIndex sorts field by index sequence. +type byIndex []field + +func (x byIndex) Len() int { return len(x) } + +func (x byIndex) Swap(i, j int) { x[i], x[j] = x[j], x[i] } + +func (x byIndex) Less(i, j int) bool { + for k, xik := range x[i].index { + if k >= len(x[j].index) { + return false + } + if xik != x[j].index[k] { + return xik < x[j].index[k] + } + } + return len(x[i].index) < len(x[j].index) +} + +// typeFields returns a list of fields that JSON should recognize for the given type. +// The algorithm is breadth-first search over the set of structs to include - the top struct +// and then any reachable anonymous structs. +func typeFields(t reflect.Type) []field { + // Anonymous fields to explore at the current level and the next. + current := []field{} + next := []field{{typ: t}} + + // Count of queued names for current level and the next. + count := map[reflect.Type]int{} + nextCount := map[reflect.Type]int{} + + // Types already visited at an earlier level. + visited := map[reflect.Type]bool{} + + // Fields found. + var fields []field + + for len(next) > 0 { + current, next = next, current[:0] + count, nextCount = nextCount, map[reflect.Type]int{} + + for _, f := range current { + if visited[f.typ] { + continue + } + visited[f.typ] = true + + // Scan f.typ for fields to include. + for i := 0; i < f.typ.NumField(); i++ { + sf := f.typ.Field(i) + if sf.PkgPath != "" { // unexported + continue + } + tag := sf.Tag.Get("json") + if tag == "-" { + continue + } + name, opts := parseTag(tag) + if !isValidTag(name) { + name = "" + } + index := make([]int, len(f.index)+1) + copy(index, f.index) + index[len(f.index)] = i + + ft := sf.Type + if ft.Name() == "" && ft.Kind() == reflect.Ptr { + // Follow pointer. + ft = ft.Elem() + } + + // Record found field and index sequence. + if name != "" || !sf.Anonymous || ft.Kind() != reflect.Struct { + tagged := name != "" + if name == "" { + name = sf.Name + } + fields = append(fields, fillField(field{ + name: name, + tag: tagged, + index: index, + typ: ft, + omitEmpty: opts.Contains("omitempty"), + quoted: opts.Contains("string"), + })) + if count[f.typ] > 1 { + // If there were multiple instances, add a second, + // so that the annihilation code will see a duplicate. + // It only cares about the distinction between 1 or 2, + // so don't bother generating any more copies. + fields = append(fields, fields[len(fields)-1]) + } + continue + } + + // Record new anonymous struct to explore in next round. + nextCount[ft]++ + if nextCount[ft] == 1 { + next = append(next, fillField(field{name: ft.Name(), index: index, typ: ft})) + } + } + } + } + + sort.Sort(byName(fields)) + + // Delete all fields that are hidden by the Go rules for embedded fields, + // except that fields with JSON tags are promoted. + + // The fields are sorted in primary order of name, secondary order + // of field index length. Loop over names; for each name, delete + // hidden fields by choosing the one dominant field that survives. + out := fields[:0] + for advance, i := 0, 0; i < len(fields); i += advance { + // One iteration per name. + // Find the sequence of fields with the name of this first field. + fi := fields[i] + name := fi.name + for advance = 1; i+advance < len(fields); advance++ { + fj := fields[i+advance] + if fj.name != name { + break + } + } + if advance == 1 { // Only one field with this name + out = append(out, fi) + continue + } + dominant, ok := dominantField(fields[i : i+advance]) + if ok { + out = append(out, dominant) + } + } + + fields = out + sort.Sort(byIndex(fields)) + + return fields +} + +// dominantField looks through the fields, all of which are known to +// have the same name, to find the single field that dominates the +// others using Go's embedding rules, modified by the presence of +// JSON tags. If there are multiple top-level fields, the boolean +// will be false: This condition is an error in Go and we skip all +// the fields. +func dominantField(fields []field) (field, bool) { + // The fields are sorted in increasing index-length order. The winner + // must therefore be one with the shortest index length. Drop all + // longer entries, which is easy: just truncate the slice. + length := len(fields[0].index) + tagged := -1 // Index of first tagged field. + for i, f := range fields { + if len(f.index) > length { + fields = fields[:i] + break + } + if f.tag { + if tagged >= 0 { + // Multiple tagged fields at the same level: conflict. + // Return no field. + return field{}, false + } + tagged = i + } + } + if tagged >= 0 { + return fields[tagged], true + } + // All remaining fields have the same length. If there's more than one, + // we have a conflict (two fields named "X" at the same level) and we + // return no field. + if len(fields) > 1 { + return field{}, false + } + return fields[0], true +} + +var fieldCache struct { + sync.RWMutex + m map[reflect.Type][]field +} + +// cachedTypeFields is like typeFields but uses a cache to avoid repeated work. +func cachedTypeFields(t reflect.Type) []field { + fieldCache.RLock() + f := fieldCache.m[t] + fieldCache.RUnlock() + if f != nil { + return f + } + + // Compute fields without lock. + // Might duplicate effort but won't hold other computations back. + f = typeFields(t) + if f == nil { + f = []field{} + } + + fieldCache.Lock() + if fieldCache.m == nil { + fieldCache.m = map[reflect.Type][]field{} + } + fieldCache.m[t] = f + fieldCache.Unlock() + return f +} + +func isValidTag(s string) bool { + if s == "" { + return false + } + for _, c := range s { + switch { + case strings.ContainsRune("!#$%&()*+-./:<=>?@[]^_{|}~ ", c): + // Backslash and quote chars are reserved, but + // otherwise any punctuation chars are allowed + // in a tag name. + default: + if !unicode.IsLetter(c) && !unicode.IsDigit(c) { + return false + } + } + } + return true +} + +const ( + caseMask = ^byte(0x20) // Mask to ignore case in ASCII. + kelvin = '\u212a' + smallLongEss = '\u017f' +) + +// foldFunc returns one of four different case folding equivalence +// functions, from most general (and slow) to fastest: +// +// 1) bytes.EqualFold, if the key s contains any non-ASCII UTF-8 +// 2) equalFoldRight, if s contains special folding ASCII ('k', 'K', 's', 'S') +// 3) asciiEqualFold, no special, but includes non-letters (including _) +// 4) simpleLetterEqualFold, no specials, no non-letters. +// +// The letters S and K are special because they map to 3 runes, not just 2: +// * S maps to s and to U+017F 'ſ' Latin small letter long s +// * k maps to K and to U+212A 'K' Kelvin sign +// See http://play.golang.org/p/tTxjOc0OGo +// +// The returned function is specialized for matching against s and +// should only be given s. It's not curried for performance reasons. +func foldFunc(s []byte) func(s, t []byte) bool { + nonLetter := false + special := false // special letter + for _, b := range s { + if b >= utf8.RuneSelf { + return bytes.EqualFold + } + upper := b & caseMask + if upper < 'A' || upper > 'Z' { + nonLetter = true + } else if upper == 'K' || upper == 'S' { + // See above for why these letters are special. + special = true + } + } + if special { + return equalFoldRight + } + if nonLetter { + return asciiEqualFold + } + return simpleLetterEqualFold +} + +// equalFoldRight is a specialization of bytes.EqualFold when s is +// known to be all ASCII (including punctuation), but contains an 's', +// 'S', 'k', or 'K', requiring a Unicode fold on the bytes in t. +// See comments on foldFunc. +func equalFoldRight(s, t []byte) bool { + for _, sb := range s { + if len(t) == 0 { + return false + } + tb := t[0] + if tb < utf8.RuneSelf { + if sb != tb { + sbUpper := sb & caseMask + if 'A' <= sbUpper && sbUpper <= 'Z' { + if sbUpper != tb&caseMask { + return false + } + } else { + return false + } + } + t = t[1:] + continue + } + // sb is ASCII and t is not. t must be either kelvin + // sign or long s; sb must be s, S, k, or K. + tr, size := utf8.DecodeRune(t) + switch sb { + case 's', 'S': + if tr != smallLongEss { + return false + } + case 'k', 'K': + if tr != kelvin { + return false + } + default: + return false + } + t = t[size:] + + } + if len(t) > 0 { + return false + } + return true +} + +// asciiEqualFold is a specialization of bytes.EqualFold for use when +// s is all ASCII (but may contain non-letters) and contains no +// special-folding letters. +// See comments on foldFunc. +func asciiEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, sb := range s { + tb := t[i] + if sb == tb { + continue + } + if ('a' <= sb && sb <= 'z') || ('A' <= sb && sb <= 'Z') { + if sb&caseMask != tb&caseMask { + return false + } + } else { + return false + } + } + return true +} + +// simpleLetterEqualFold is a specialization of bytes.EqualFold for +// use when s is all ASCII letters (no underscores, etc) and also +// doesn't contain 'k', 'K', 's', or 'S'. +// See comments on foldFunc. +func simpleLetterEqualFold(s, t []byte) bool { + if len(s) != len(t) { + return false + } + for i, b := range s { + if b&caseMask != t[i]&caseMask { + return false + } + } + return true +} + +// tagOptions is the string following a comma in a struct field's "json" +// tag, or the empty string. It does not include the leading comma. +type tagOptions string + +// parseTag splits a struct field's json tag into its name and +// comma-separated options. +func parseTag(tag string) (string, tagOptions) { + if idx := strings.Index(tag, ","); idx != -1 { + return tag[:idx], tagOptions(tag[idx+1:]) + } + return tag, tagOptions("") +} + +// Contains reports whether a comma-separated list of options +// contains a particular substr flag. substr must be surrounded by a +// string boundary or commas. +func (o tagOptions) Contains(optionName string) bool { + if len(o) == 0 { + return false + } + s := string(o) + for s != "" { + var next string + i := strings.Index(s, ",") + if i >= 0 { + s, next = s[:i], s[i+1:] + } + if s == optionName { + return true + } + s = next + } + return false +} diff --git a/vendor/sigs.k8s.io/yaml/go.mod b/vendor/sigs.k8s.io/yaml/go.mod new file mode 100644 index 000000000..818bbb519 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/go.mod @@ -0,0 +1,8 @@ +module sigs.k8s.io/yaml + +go 1.12 + +require ( + github.com/davecgh/go-spew v1.1.1 + gopkg.in/yaml.v2 v2.4.0 +) diff --git a/vendor/sigs.k8s.io/yaml/go.sum b/vendor/sigs.k8s.io/yaml/go.sum new file mode 100644 index 000000000..b7b8cbb10 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/go.sum @@ -0,0 +1,6 @@ +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405 h1:yhCVgyC4o1eVCa2tZl7eS0r+SDo693bJlVdllGtEeKM= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= diff --git a/vendor/sigs.k8s.io/yaml/yaml.go b/vendor/sigs.k8s.io/yaml/yaml.go new file mode 100644 index 000000000..efbc535d4 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/yaml.go @@ -0,0 +1,380 @@ +package yaml + +import ( + "bytes" + "encoding/json" + "fmt" + "io" + "reflect" + "strconv" + + "gopkg.in/yaml.v2" +) + +// Marshal marshals the object into JSON then converts JSON to YAML and returns the +// YAML. +func Marshal(o interface{}) ([]byte, error) { + j, err := json.Marshal(o) + if err != nil { + return nil, fmt.Errorf("error marshaling into JSON: %v", err) + } + + y, err := JSONToYAML(j) + if err != nil { + return nil, fmt.Errorf("error converting JSON to YAML: %v", err) + } + + return y, nil +} + +// JSONOpt is a decoding option for decoding from JSON format. +type JSONOpt func(*json.Decoder) *json.Decoder + +// Unmarshal converts YAML to JSON then uses JSON to unmarshal into an object, +// optionally configuring the behavior of the JSON unmarshal. +func Unmarshal(y []byte, o interface{}, opts ...JSONOpt) error { + return yamlUnmarshal(y, o, false, opts...) +} + +// UnmarshalStrict strictly converts YAML to JSON then uses JSON to unmarshal +// into an object, optionally configuring the behavior of the JSON unmarshal. +func UnmarshalStrict(y []byte, o interface{}, opts ...JSONOpt) error { + return yamlUnmarshal(y, o, true, append(opts, DisallowUnknownFields)...) +} + +// yamlUnmarshal unmarshals the given YAML byte stream into the given interface, +// optionally performing the unmarshalling strictly +func yamlUnmarshal(y []byte, o interface{}, strict bool, opts ...JSONOpt) error { + vo := reflect.ValueOf(o) + unmarshalFn := yaml.Unmarshal + if strict { + unmarshalFn = yaml.UnmarshalStrict + } + j, err := yamlToJSON(y, &vo, unmarshalFn) + if err != nil { + return fmt.Errorf("error converting YAML to JSON: %v", err) + } + + err = jsonUnmarshal(bytes.NewReader(j), o, opts...) + if err != nil { + return fmt.Errorf("error unmarshaling JSON: %v", err) + } + + return nil +} + +// jsonUnmarshal unmarshals the JSON byte stream from the given reader into the +// object, optionally applying decoder options prior to decoding. We are not +// using json.Unmarshal directly as we want the chance to pass in non-default +// options. +func jsonUnmarshal(r io.Reader, o interface{}, opts ...JSONOpt) error { + d := json.NewDecoder(r) + for _, opt := range opts { + d = opt(d) + } + if err := d.Decode(&o); err != nil { + return fmt.Errorf("while decoding JSON: %v", err) + } + return nil +} + +// JSONToYAML Converts JSON to YAML. +func JSONToYAML(j []byte) ([]byte, error) { + // Convert the JSON to an object. + var jsonObj interface{} + // We are using yaml.Unmarshal here (instead of json.Unmarshal) because the + // Go JSON library doesn't try to pick the right number type (int, float, + // etc.) when unmarshalling to interface{}, it just picks float64 + // universally. go-yaml does go through the effort of picking the right + // number type, so we can preserve number type throughout this process. + err := yaml.Unmarshal(j, &jsonObj) + if err != nil { + return nil, err + } + + // Marshal this object into YAML. + return yaml.Marshal(jsonObj) +} + +// YAMLToJSON converts YAML to JSON. Since JSON is a subset of YAML, +// passing JSON through this method should be a no-op. +// +// Things YAML can do that are not supported by JSON: +// * In YAML you can have binary and null keys in your maps. These are invalid +// in JSON. (int and float keys are converted to strings.) +// * Binary data in YAML with the !!binary tag is not supported. If you want to +// use binary data with this library, encode the data as base64 as usual but do +// not use the !!binary tag in your YAML. This will ensure the original base64 +// encoded data makes it all the way through to the JSON. +// +// For strict decoding of YAML, use YAMLToJSONStrict. +func YAMLToJSON(y []byte) ([]byte, error) { + return yamlToJSON(y, nil, yaml.Unmarshal) +} + +// YAMLToJSONStrict is like YAMLToJSON but enables strict YAML decoding, +// returning an error on any duplicate field names. +func YAMLToJSONStrict(y []byte) ([]byte, error) { + return yamlToJSON(y, nil, yaml.UnmarshalStrict) +} + +func yamlToJSON(y []byte, jsonTarget *reflect.Value, yamlUnmarshal func([]byte, interface{}) error) ([]byte, error) { + // Convert the YAML to an object. + var yamlObj interface{} + err := yamlUnmarshal(y, &yamlObj) + if err != nil { + return nil, err + } + + // YAML objects are not completely compatible with JSON objects (e.g. you + // can have non-string keys in YAML). So, convert the YAML-compatible object + // to a JSON-compatible object, failing with an error if irrecoverable + // incompatibilties happen along the way. + jsonObj, err := convertToJSONableObject(yamlObj, jsonTarget) + if err != nil { + return nil, err + } + + // Convert this object to JSON and return the data. + return json.Marshal(jsonObj) +} + +func convertToJSONableObject(yamlObj interface{}, jsonTarget *reflect.Value) (interface{}, error) { + var err error + + // Resolve jsonTarget to a concrete value (i.e. not a pointer or an + // interface). We pass decodingNull as false because we're not actually + // decoding into the value, we're just checking if the ultimate target is a + // string. + if jsonTarget != nil { + ju, tu, pv := indirect(*jsonTarget, false) + // We have a JSON or Text Umarshaler at this level, so we can't be trying + // to decode into a string. + if ju != nil || tu != nil { + jsonTarget = nil + } else { + jsonTarget = &pv + } + } + + // If yamlObj is a number or a boolean, check if jsonTarget is a string - + // if so, coerce. Else return normal. + // If yamlObj is a map or array, find the field that each key is + // unmarshaling to, and when you recurse pass the reflect.Value for that + // field back into this function. + switch typedYAMLObj := yamlObj.(type) { + case map[interface{}]interface{}: + // JSON does not support arbitrary keys in a map, so we must convert + // these keys to strings. + // + // From my reading of go-yaml v2 (specifically the resolve function), + // keys can only have the types string, int, int64, float64, binary + // (unsupported), or null (unsupported). + strMap := make(map[string]interface{}) + for k, v := range typedYAMLObj { + // Resolve the key to a string first. + var keyString string + switch typedKey := k.(type) { + case string: + keyString = typedKey + case int: + keyString = strconv.Itoa(typedKey) + case int64: + // go-yaml will only return an int64 as a key if the system + // architecture is 32-bit and the key's value is between 32-bit + // and 64-bit. Otherwise the key type will simply be int. + keyString = strconv.FormatInt(typedKey, 10) + case float64: + // Stolen from go-yaml to use the same conversion to string as + // the go-yaml library uses to convert float to string when + // Marshaling. + s := strconv.FormatFloat(typedKey, 'g', -1, 32) + switch s { + case "+Inf": + s = ".inf" + case "-Inf": + s = "-.inf" + case "NaN": + s = ".nan" + } + keyString = s + case bool: + if typedKey { + keyString = "true" + } else { + keyString = "false" + } + default: + return nil, fmt.Errorf("Unsupported map key of type: %s, key: %+#v, value: %+#v", + reflect.TypeOf(k), k, v) + } + + // jsonTarget should be a struct or a map. If it's a struct, find + // the field it's going to map to and pass its reflect.Value. If + // it's a map, find the element type of the map and pass the + // reflect.Value created from that type. If it's neither, just pass + // nil - JSON conversion will error for us if it's a real issue. + if jsonTarget != nil { + t := *jsonTarget + if t.Kind() == reflect.Struct { + keyBytes := []byte(keyString) + // Find the field that the JSON library would use. + var f *field + fields := cachedTypeFields(t.Type()) + for i := range fields { + ff := &fields[i] + if bytes.Equal(ff.nameBytes, keyBytes) { + f = ff + break + } + // Do case-insensitive comparison. + if f == nil && ff.equalFold(ff.nameBytes, keyBytes) { + f = ff + } + } + if f != nil { + // Find the reflect.Value of the most preferential + // struct field. + jtf := t.Field(f.index[0]) + strMap[keyString], err = convertToJSONableObject(v, &jtf) + if err != nil { + return nil, err + } + continue + } + } else if t.Kind() == reflect.Map { + // Create a zero value of the map's element type to use as + // the JSON target. + jtv := reflect.Zero(t.Type().Elem()) + strMap[keyString], err = convertToJSONableObject(v, &jtv) + if err != nil { + return nil, err + } + continue + } + } + strMap[keyString], err = convertToJSONableObject(v, nil) + if err != nil { + return nil, err + } + } + return strMap, nil + case []interface{}: + // We need to recurse into arrays in case there are any + // map[interface{}]interface{}'s inside and to convert any + // numbers to strings. + + // If jsonTarget is a slice (which it really should be), find the + // thing it's going to map to. If it's not a slice, just pass nil + // - JSON conversion will error for us if it's a real issue. + var jsonSliceElemValue *reflect.Value + if jsonTarget != nil { + t := *jsonTarget + if t.Kind() == reflect.Slice { + // By default slices point to nil, but we need a reflect.Value + // pointing to a value of the slice type, so we create one here. + ev := reflect.Indirect(reflect.New(t.Type().Elem())) + jsonSliceElemValue = &ev + } + } + + // Make and use a new array. + arr := make([]interface{}, len(typedYAMLObj)) + for i, v := range typedYAMLObj { + arr[i], err = convertToJSONableObject(v, jsonSliceElemValue) + if err != nil { + return nil, err + } + } + return arr, nil + default: + // If the target type is a string and the YAML type is a number, + // convert the YAML type to a string. + if jsonTarget != nil && (*jsonTarget).Kind() == reflect.String { + // Based on my reading of go-yaml, it may return int, int64, + // float64, or uint64. + var s string + switch typedVal := typedYAMLObj.(type) { + case int: + s = strconv.FormatInt(int64(typedVal), 10) + case int64: + s = strconv.FormatInt(typedVal, 10) + case float64: + s = strconv.FormatFloat(typedVal, 'g', -1, 32) + case uint64: + s = strconv.FormatUint(typedVal, 10) + case bool: + if typedVal { + s = "true" + } else { + s = "false" + } + } + if len(s) > 0 { + yamlObj = interface{}(s) + } + } + return yamlObj, nil + } +} + +// JSONObjectToYAMLObject converts an in-memory JSON object into a YAML in-memory MapSlice, +// without going through a byte representation. A nil or empty map[string]interface{} input is +// converted to an empty map, i.e. yaml.MapSlice(nil). +// +// interface{} slices stay interface{} slices. map[string]interface{} becomes yaml.MapSlice. +// +// int64 and float64 are down casted following the logic of github.com/go-yaml/yaml: +// - float64s are down-casted as far as possible without data-loss to int, int64, uint64. +// - int64s are down-casted to int if possible without data-loss. +// +// Big int/int64/uint64 do not lose precision as in the json-yaml roundtripping case. +// +// string, bool and any other types are unchanged. +func JSONObjectToYAMLObject(j map[string]interface{}) yaml.MapSlice { + if len(j) == 0 { + return nil + } + ret := make(yaml.MapSlice, 0, len(j)) + for k, v := range j { + ret = append(ret, yaml.MapItem{Key: k, Value: jsonToYAMLValue(v)}) + } + return ret +} + +func jsonToYAMLValue(j interface{}) interface{} { + switch j := j.(type) { + case map[string]interface{}: + if j == nil { + return interface{}(nil) + } + return JSONObjectToYAMLObject(j) + case []interface{}: + if j == nil { + return interface{}(nil) + } + ret := make([]interface{}, len(j)) + for i := range j { + ret[i] = jsonToYAMLValue(j[i]) + } + return ret + case float64: + // replicate the logic in https://github.com/go-yaml/yaml/blob/51d6538a90f86fe93ac480b35f37b2be17fef232/resolve.go#L151 + if i64 := int64(j); j == float64(i64) { + if i := int(i64); i64 == int64(i) { + return i + } + return i64 + } + if ui64 := uint64(j); j == float64(ui64) { + return ui64 + } + return j + case int64: + if i := int(j); j == int64(i) { + return i + } + return j + } + return j +} diff --git a/vendor/sigs.k8s.io/yaml/yaml_go110.go b/vendor/sigs.k8s.io/yaml/yaml_go110.go new file mode 100644 index 000000000..ab3e06a22 --- /dev/null +++ b/vendor/sigs.k8s.io/yaml/yaml_go110.go @@ -0,0 +1,14 @@ +// This file contains changes that are only compatible with go 1.10 and onwards. + +// +build go1.10 + +package yaml + +import "encoding/json" + +// DisallowUnknownFields configures the JSON decoder to error out if unknown +// fields come along, instead of dropping them by default. +func DisallowUnknownFields(d *json.Decoder) *json.Decoder { + d.DisallowUnknownFields() + return d +} |