From cf51c7ed9f955390a0e417f208046e0b8fbadb26 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Fri, 15 Jan 2021 03:49:42 -0500 Subject: Allow podman push to push manifest lists When doing a podman images, manifests lists look just like images, so it is logical that users would assume that they can just podman push them to a registry. The problem is we throw out weird errors when this happens and users need to somehow figure out this is a manifest list rather then an image, and frankly the user will not understand the difference. This PR will make podman push just do the right thing, by failing over and attempting to push the manifest if it fails to push the image. Fix up handling of manifest push Protocol should bring back a digest string, which can either be printed or stored in a file. We should not reimplement the manifest push setup code in the tunnel code but take advantage of the api path, to make sure remote and local work the same way. Signed-off-by: Daniel J Walsh --- pkg/api/handlers/libpod/images.go | 49 +++++-------- pkg/api/handlers/libpod/manifests.go | 61 +++++++++------- pkg/bindings/images/types.go | 2 + pkg/bindings/images/types_push_options.go | 16 +++++ pkg/bindings/manifests/manifests.go | 13 +++- pkg/bindings/manifests/types.go | 6 -- pkg/bindings/manifests/types_push_options.go | 104 --------------------------- pkg/domain/entities/engine_image.go | 2 +- pkg/domain/entities/images.go | 4 ++ pkg/domain/entities/manifest.go | 8 --- pkg/domain/infra/abi/images.go | 40 +++-------- pkg/domain/infra/abi/manifest.go | 27 ++++--- pkg/domain/infra/tunnel/manifest.go | 22 ++++-- 13 files changed, 125 insertions(+), 229 deletions(-) delete mode 100644 pkg/bindings/manifests/types_push_options.go (limited to 'pkg') diff --git a/pkg/api/handlers/libpod/images.go b/pkg/api/handlers/libpod/images.go index 5b15527b7..97cd5a65e 100644 --- a/pkg/api/handlers/libpod/images.go +++ b/pkg/api/handlers/libpod/images.go @@ -25,7 +25,6 @@ import ( utils2 "github.com/containers/podman/v2/utils" "github.com/gorilla/schema" "github.com/pkg/errors" - "github.com/sirupsen/logrus" ) // Commit @@ -410,6 +409,8 @@ func PushImage(w http.ResponseWriter, r *http.Request) { query := struct { Destination string `schema:"destination"` TLSVerify bool `schema:"tlsVerify"` + Format string `schema:"format"` + All bool `schema:"all"` }{ // This is where you can override the golang default value for one of fields } @@ -434,45 +435,31 @@ func PushImage(w http.ResponseWriter, r *http.Request) { return } - newImage, err := runtime.ImageRuntime().NewFromLocal(source) - if err != nil { - utils.ImageNotFound(w, source, errors.Wrapf(err, "failed to find image %s", source)) - return - } - - authConf, authfile, key, err := auth.GetCredentials(r) + authconf, authfile, key, err := auth.GetCredentials(r) if err != nil { utils.Error(w, "failed to retrieve repository credentials", http.StatusBadRequest, errors.Wrapf(err, "failed to parse %q header for %s", key, r.URL.String())) return } defer auth.RemoveAuthfile(authfile) - logrus.Errorf("AuthConf: %v", authConf) + var username, password string + if authconf != nil { + username = authconf.Username + password = authconf.Password - dockerRegistryOptions := &image.DockerRegistryOptions{ - DockerRegistryCreds: authConf, } - if sys := runtime.SystemContext(); sys != nil { - dockerRegistryOptions.DockerCertPath = sys.DockerCertPath - dockerRegistryOptions.RegistriesConfPath = sys.SystemRegistriesConfPath + options := entities.ImagePushOptions{ + Authfile: authfile, + Username: username, + Password: password, + Format: query.Format, + All: query.All, } if _, found := r.URL.Query()["tlsVerify"]; found { - dockerRegistryOptions.DockerInsecureSkipTLSVerify = types.NewOptionalBool(!query.TLSVerify) - } - - err = newImage.PushImageToHeuristicDestination( - context.Background(), - destination, - "", // manifest type - authfile, - "", // digest file - "", // signature policy - os.Stderr, - false, // force compression - image.SigningOptions{}, - dockerRegistryOptions, - nil, // additional tags - ) - if err != nil { + options.SkipTLSVerify = types.NewOptionalBool(!query.TLSVerify) + } + + imageEngine := abi.ImageEngine{Libpod: runtime} + if err := imageEngine.Push(context.Background(), source, destination, options); err != nil { utils.Error(w, "Something went wrong.", http.StatusBadRequest, errors.Wrapf(err, "error pushing image %q", destination)) return } diff --git a/pkg/api/handlers/libpod/manifests.go b/pkg/api/handlers/libpod/manifests.go index 2031dd42f..dce861f6f 100644 --- a/pkg/api/handlers/libpod/manifests.go +++ b/pkg/api/handlers/libpod/manifests.go @@ -1,17 +1,18 @@ package libpod import ( + "context" "encoding/json" "net/http" - "github.com/containers/buildah/manifests" - copy2 "github.com/containers/image/v5/copy" "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/transports/alltransports" + "github.com/containers/image/v5/types" "github.com/containers/podman/v2/libpod" "github.com/containers/podman/v2/libpod/image" "github.com/containers/podman/v2/pkg/api/handlers" "github.com/containers/podman/v2/pkg/api/handlers/utils" + "github.com/containers/podman/v2/pkg/auth" + "github.com/containers/podman/v2/pkg/domain/entities" "github.com/containers/podman/v2/pkg/domain/infra/abi" "github.com/gorilla/schema" "github.com/opencontainers/go-digest" @@ -123,15 +124,13 @@ func ManifestRemove(w http.ResponseWriter, r *http.Request) { utils.WriteResponse(w, http.StatusOK, handlers.IDResponse{ID: newID}) } func ManifestPush(w http.ResponseWriter, r *http.Request) { - // FIXME: parameters are missing (tlsVerify, format). - // Also, we should use the ABI function to avoid duplicate code. - // Also, support for XRegistryAuth headers are missing. - runtime := r.Context().Value("runtime").(*libpod.Runtime) decoder := r.Context().Value("decoder").(*schema.Decoder) query := struct { All bool `schema:"all"` Destination string `schema:"destination"` + Format string `schema:"format"` + TLSVerify bool `schema:"tlsVerify"` }{ // Add defaults here once needed. } @@ -140,35 +139,43 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) { errors.Wrapf(err, "failed to parse parameters for %s", r.URL.String())) return } - name := utils.GetName(r) - newImage, err := runtime.ImageRuntime().NewFromLocal(name) - if err != nil { - utils.ImageNotFound(w, name, err) + if _, err := utils.ParseDockerReference(query.Destination); err != nil { + utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, err) return } - dest, err := alltransports.ParseImageName(query.Destination) + + source := utils.GetName(r) + authConf, authfile, key, err := auth.GetCredentials(r) if err != nil { - utils.Error(w, "invalid destination parameter", http.StatusBadRequest, errors.Errorf("invalid destination parameter %q", query.Destination)) + utils.Error(w, "failed to retrieve repository credentials", http.StatusBadRequest, errors.Wrapf(err, "failed to parse %q header for %s", key, r.URL.String())) return } - rtc, err := runtime.GetConfig() - if err != nil { - utils.InternalServerError(w, err) - return + defer auth.RemoveAuthfile(authfile) + var username, password string + if authConf != nil { + username = authConf.Username + password = authConf.Password + } - sc := image.GetSystemContext(rtc.Engine.SignaturePolicyPath, "", false) - opts := manifests.PushOptions{ - Store: runtime.GetStore(), - ImageListSelection: copy2.CopySpecificImages, - SystemContext: sc, + + options := entities.ImagePushOptions{ + Authfile: authfile, + Username: username, + Password: password, + Format: query.Format, + All: query.All, + } + if sys := runtime.SystemContext(); sys != nil { + options.CertDir = sys.DockerCertPath } - if query.All { - opts.ImageListSelection = copy2.CopyAllImages + if _, found := r.URL.Query()["tlsVerify"]; found { + options.SkipTLSVerify = types.NewOptionalBool(!query.TLSVerify) } - newD, err := newImage.PushManifest(dest, opts) + imageEngine := abi.ImageEngine{Libpod: runtime} + digest, err := imageEngine.ManifestPush(context.Background(), source, query.Destination, options) if err != nil { - utils.InternalServerError(w, err) + utils.Error(w, "Something went wrong.", http.StatusBadRequest, errors.Wrapf(err, "error pushing image %q", query.Destination)) return } - utils.WriteResponse(w, http.StatusOK, newD.String()) + utils.WriteResponse(w, http.StatusOK, digest) } diff --git a/pkg/bindings/images/types.go b/pkg/bindings/images/types.go index 3adb4356b..b3799b8c4 100644 --- a/pkg/bindings/images/types.go +++ b/pkg/bindings/images/types.go @@ -99,6 +99,8 @@ type ImportOptions struct { //go:generate go run ../generator/generator.go PushOptions // PushOptions are optional options for importing images type PushOptions struct { + // All indicates whether to push all images related to the image list + All *bool // Authfile is the path to the authentication file. Ignored for remote // calls. Authfile *string diff --git a/pkg/bindings/images/types_push_options.go b/pkg/bindings/images/types_push_options.go index 15210f30b..0c12ce4ac 100644 --- a/pkg/bindings/images/types_push_options.go +++ b/pkg/bindings/images/types_push_options.go @@ -87,6 +87,22 @@ func (o *PushOptions) ToParams() (url.Values, error) { return params, nil } +// WithAll +func (o *PushOptions) WithAll(value bool) *PushOptions { + v := &value + o.All = v + return o +} + +// GetAll +func (o *PushOptions) GetAll() bool { + var all bool + if o.All == nil { + return all + } + return *o.All +} + // WithAuthfile func (o *PushOptions) WithAuthfile(value string) *PushOptions { v := &value diff --git a/pkg/bindings/manifests/manifests.go b/pkg/bindings/manifests/manifests.go index ef4e2a908..b6db64b02 100644 --- a/pkg/bindings/manifests/manifests.go +++ b/pkg/bindings/manifests/manifests.go @@ -5,11 +5,13 @@ import ( "errors" "net/http" "net/url" + "strconv" "strings" "github.com/containers/image/v5/manifest" "github.com/containers/podman/v2/pkg/api/handlers" "github.com/containers/podman/v2/pkg/bindings" + "github.com/containers/podman/v2/pkg/bindings/images" jsoniter "github.com/json-iterator/go" ) @@ -112,12 +114,12 @@ func Remove(ctx context.Context, name, digest string, options *RemoveOptions) (s // Push takes a manifest list and pushes to a destination. If the destination is not specified, // the name will be used instead. If the optional all boolean is specified, all images specified // in the list will be pushed as well. -func Push(ctx context.Context, name, destination string, options *PushOptions) (string, error) { +func Push(ctx context.Context, name, destination string, options *images.PushOptions) (string, error) { var ( idr handlers.IDResponse ) if options == nil { - options = new(PushOptions) + options = new(images.PushOptions) } if len(destination) < 1 { destination = name @@ -130,8 +132,15 @@ func Push(ctx context.Context, name, destination string, options *PushOptions) ( if err != nil { return "", err } + //SkipTLSVerify is special. We need to delete the param added by + //toparams and change the key and flip the bool + if options.SkipTLSVerify != nil { + params.Del("SkipTLSVerify") + params.Set("tlsVerify", strconv.FormatBool(!options.GetSkipTLSVerify())) + } params.Set("image", name) params.Set("destination", destination) + params.Set("format", *options.Format) _, err = conn.DoRequest(nil, http.MethodPost, "/manifests/%s/push", params, nil, name) if err != nil { return "", err diff --git a/pkg/bindings/manifests/types.go b/pkg/bindings/manifests/types.go index f5054d424..7f84d69fc 100644 --- a/pkg/bindings/manifests/types.go +++ b/pkg/bindings/manifests/types.go @@ -28,9 +28,3 @@ type AddOptions struct { // RemoveOptions are optional options for removing manifests type RemoveOptions struct { } - -//go:generate go run ../generator/generator.go PushOptions -// RemoveOptions are optional options for pushing manifests -type PushOptions struct { - All *bool -} diff --git a/pkg/bindings/manifests/types_push_options.go b/pkg/bindings/manifests/types_push_options.go deleted file mode 100644 index 1d689f699..000000000 --- a/pkg/bindings/manifests/types_push_options.go +++ /dev/null @@ -1,104 +0,0 @@ -package manifests - -import ( - "net/url" - "reflect" - "strconv" - "strings" - - jsoniter "github.com/json-iterator/go" - "github.com/pkg/errors" -) - -/* -This file is generated automatically by go generate. Do not edit. -*/ - -// Changed -func (o *PushOptions) Changed(fieldName string) bool { - r := reflect.ValueOf(o) - value := reflect.Indirect(r).FieldByName(fieldName) - return !value.IsNil() -} - -// ToParams -func (o *PushOptions) ToParams() (url.Values, error) { - params := url.Values{} - if o == nil { - return params, nil - } - json := jsoniter.ConfigCompatibleWithStandardLibrary - s := reflect.ValueOf(o) - if reflect.Ptr == s.Kind() { - s = s.Elem() - } - sType := s.Type() - for i := 0; i < s.NumField(); i++ { - fieldName := sType.Field(i).Name - if !o.Changed(fieldName) { - continue - } - fieldName = strings.ToLower(fieldName) - f := s.Field(i) - if reflect.Ptr == f.Kind() { - f = f.Elem() - } - switch f.Kind() { - case reflect.Bool: - params.Set(fieldName, strconv.FormatBool(f.Bool())) - case reflect.String: - params.Set(fieldName, f.String()) - case reflect.Int, reflect.Int64: - // f.Int() is always an int64 - params.Set(fieldName, strconv.FormatInt(f.Int(), 10)) - case reflect.Uint, reflect.Uint64: - // f.Uint() is always an uint64 - params.Set(fieldName, strconv.FormatUint(f.Uint(), 10)) - case reflect.Slice: - typ := reflect.TypeOf(f.Interface()).Elem() - switch typ.Kind() { - case reflect.String: - sl := f.Slice(0, f.Len()) - s, ok := sl.Interface().([]string) - if !ok { - return nil, errors.New("failed to convert to string slice") - } - for _, val := range s { - params.Add(fieldName, val) - } - default: - return nil, errors.Errorf("unknown slice type %s", f.Kind().String()) - } - case reflect.Map: - lowerCaseKeys := make(map[string][]string) - iter := f.MapRange() - for iter.Next() { - lowerCaseKeys[iter.Key().Interface().(string)] = iter.Value().Interface().([]string) - - } - s, err := json.MarshalToString(lowerCaseKeys) - if err != nil { - return nil, err - } - - params.Set(fieldName, s) - } - } - return params, nil -} - -// WithAll -func (o *PushOptions) WithAll(value bool) *PushOptions { - v := &value - o.All = v - return o -} - -// GetAll -func (o *PushOptions) GetAll() bool { - var all bool - if o.All == nil { - return all - } - return *o.All -} diff --git a/pkg/domain/entities/engine_image.go b/pkg/domain/entities/engine_image.go index 26a136f13..935ee6f20 100644 --- a/pkg/domain/entities/engine_image.go +++ b/pkg/domain/entities/engine_image.go @@ -36,6 +36,6 @@ type ImageEngine interface { ManifestAdd(ctx context.Context, opts ManifestAddOptions) (string, error) ManifestAnnotate(ctx context.Context, names []string, opts ManifestAnnotateOptions) (string, error) ManifestRemove(ctx context.Context, names []string) (string, error) - ManifestPush(ctx context.Context, name, destination string, manifestPushOpts ManifestPushOptions) error + ManifestPush(ctx context.Context, name, destination string, imagePushOpts ImagePushOptions) (string, error) Sign(ctx context.Context, names []string, options SignOptions) (*SignReport, error) } diff --git a/pkg/domain/entities/images.go b/pkg/domain/entities/images.go index 0805152c3..78a7d8aa7 100644 --- a/pkg/domain/entities/images.go +++ b/pkg/domain/entities/images.go @@ -165,6 +165,8 @@ type ImagePullReport struct { // ImagePushOptions are the arguments for pushing images. type ImagePushOptions struct { + // All indicates that all images referenced in an manifest list should be pushed + All bool // Authfile is the path to the authentication file. Ignored for remote // calls. Authfile string @@ -189,6 +191,8 @@ type ImagePushOptions struct { // Quiet can be specified to suppress pull progress when pulling. Ignored // for remote calls. Quiet bool + // Rm indicates whether to remove the manifest list if push succeeds + Rm bool // RemoveSignatures, discard any pre-existing signatures in the image. // Ignored for remote calls. RemoveSignatures bool diff --git a/pkg/domain/entities/manifest.go b/pkg/domain/entities/manifest.go index 01180951a..6a645e20b 100644 --- a/pkg/domain/entities/manifest.go +++ b/pkg/domain/entities/manifest.go @@ -33,11 +33,3 @@ type ManifestAnnotateOptions struct { OSVersion string `json:"os_version" schema:"os_version"` Variant string `json:"variant" schema:"variant"` } - -type ManifestPushOptions struct { - Purge, Quiet, All, RemoveSignatures bool - - Authfile, CertDir, Username, Password, DigestFile, Format, SignBy string - - SkipTLSVerify types.OptionalBool -} diff --git a/pkg/domain/infra/abi/images.go b/pkg/domain/infra/abi/images.go index 1c233d9d5..1288ab09b 100644 --- a/pkg/domain/infra/abi/images.go +++ b/pkg/domain/infra/abi/images.go @@ -367,7 +367,7 @@ func (ir *ImageEngine) Push(ctx context.Context, source string, destination stri return err } - return newImage.PushImageToHeuristicDestination( + err = newImage.PushImageToHeuristicDestination( ctx, destination, manifestType, @@ -379,39 +379,15 @@ func (ir *ImageEngine) Push(ctx context.Context, source string, destination stri signOptions, &dockerRegistryOptions, nil) + if err != nil && errors.Cause(err) != storage.ErrImageUnknown { + // Image might be a manifest list so attempt a manifest push + if _, manifestErr := ir.ManifestPush(ctx, source, destination, options); manifestErr == nil { + return nil + } + } + return err } -// func (r *imageRuntime) Delete(ctx context.Context, nameOrID string, opts entities.ImageDeleteOptions) (*entities.ImageDeleteReport, error) { -// image, err := r.libpod.ImageEngine().NewFromLocal(nameOrID) -// if err != nil { -// return nil, err -// } -// -// results, err := r.libpod.RemoveImage(ctx, image, opts.Force) -// if err != nil { -// return nil, err -// } -// -// report := entities.ImageDeleteReport{} -// if err := domainUtils.DeepCopy(&report, results); err != nil { -// return nil, err -// } -// return &report, nil -// } -// -// func (r *imageRuntime) Prune(ctx context.Context, opts entities.ImagePruneOptions) (*entities.ImagePruneReport, error) { -// // TODO: map FilterOptions -// id, err := r.libpod.ImageEngine().PruneImages(ctx, opts.All, []string{}) -// if err != nil { -// return nil, err -// } -// -// // TODO: Determine Size -// report := entities.ImagePruneReport{} -// copy(report.Report.ID, id) -// return &report, nil -// } - func (ir *ImageEngine) Tag(ctx context.Context, nameOrID string, tags []string, options entities.ImageTagOptions) error { newImage, err := ir.Libpod.ImageRuntime().NewFromLocal(nameOrID) if err != nil { diff --git a/pkg/domain/infra/abi/manifest.go b/pkg/domain/infra/abi/manifest.go index a68ed8788..139032ad6 100644 --- a/pkg/domain/infra/abi/manifest.go +++ b/pkg/domain/infra/abi/manifest.go @@ -7,7 +7,6 @@ import ( "context" "encoding/json" "fmt" - "io/ioutil" "os" "strings" @@ -24,9 +23,8 @@ import ( "github.com/containers/podman/v2/pkg/domain/entities" "github.com/opencontainers/go-digest" imgspecv1 "github.com/opencontainers/image-spec/specs-go/v1" - "github.com/sirupsen/logrus" - "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) // ManifestCreate implements logic for creating manifest lists via ImageEngine @@ -243,14 +241,20 @@ func (ir *ImageEngine) ManifestRemove(ctx context.Context, names []string) (stri } // ManifestPush pushes a manifest list or image index to the destination -func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination string, opts entities.ManifestPushOptions) error { +func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination string, opts entities.ImagePushOptions) (string, error) { listImage, err := ir.Libpod.ImageRuntime().NewFromLocal(name) if err != nil { - return errors.Wrapf(err, "error retrieving local image from image name %s", name) + return "", errors.Wrapf(err, "error retrieving local image from image name %s", name) } dest, err := alltransports.ParseImageName(destination) if err != nil { - return err + oldErr := err + // Try adding the images default transport + destination2 := libpodImage.DefaultTransport + destination + dest, err = alltransports.ParseImageName(destination2) + if err != nil { + return "", oldErr + } } var manifestType string @@ -261,7 +265,7 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin case "v2s2", "docker": manifestType = manifest.DockerV2Schema2MediaType default: - return errors.Errorf("unknown format %q. Choose one of the supported formats: 'oci' or 'v2s2'", opts.Format) + return "", errors.Errorf("unknown format %q. Choose one of the supported formats: 'oci' or 'v2s2'", opts.Format) } } @@ -297,13 +301,8 @@ func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination strin options.ReportWriter = os.Stderr } manDigest, err := listImage.PushManifest(dest, options) - if err == nil && opts.Purge { + if err == nil && opts.Rm { _, err = ir.Libpod.GetStore().DeleteImage(listImage.ID(), true) } - if opts.DigestFile != "" { - if err = ioutil.WriteFile(opts.DigestFile, []byte(manDigest.String()), 0644); err != nil { - return buildahUtil.GetFailureCause(err, errors.Wrapf(err, "failed to write digest to file %q", opts.DigestFile)) - } - } - return err + return manDigest.String(), err } diff --git a/pkg/domain/infra/tunnel/manifest.go b/pkg/domain/infra/tunnel/manifest.go index c71349fe0..22ca44165 100644 --- a/pkg/domain/infra/tunnel/manifest.go +++ b/pkg/domain/infra/tunnel/manifest.go @@ -6,6 +6,8 @@ import ( "fmt" "strings" + "github.com/containers/image/v5/types" + images "github.com/containers/podman/v2/pkg/bindings/images" "github.com/containers/podman/v2/pkg/bindings/manifests" "github.com/containers/podman/v2/pkg/domain/entities" "github.com/pkg/errors" @@ -73,8 +75,20 @@ func (ir *ImageEngine) ManifestRemove(ctx context.Context, names []string) (stri } // ManifestPush pushes a manifest list or image index to the destination -func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination string, opts entities.ManifestPushOptions) error { - options := new(manifests.PushOptions).WithAll(opts.All) - _, err := manifests.Push(ir.ClientCtx, name, destination, options) - return err +func (ir *ImageEngine) ManifestPush(ctx context.Context, name, destination string, opts entities.ImagePushOptions) (string, error) { + options := new(images.PushOptions) + options.WithUsername(opts.Username).WithSignaturePolicy(opts.SignaturePolicy).WithQuiet(opts.Quiet) + options.WithPassword(opts.Password).WithCertDir(opts.CertDir).WithAuthfile(opts.Authfile) + options.WithCompress(opts.Compress).WithDigestFile(opts.DigestFile).WithFormat(opts.Format) + options.WithRemoveSignatures(opts.RemoveSignatures).WithSignBy(opts.SignBy) + + if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined { + if s == types.OptionalBoolTrue { + options.WithSkipTLSVerify(true) + } else { + options.WithSkipTLSVerify(false) + } + } + digest, err := manifests.Push(ir.ClientCtx, name, destination, options) + return digest, err } -- cgit v1.2.3-54-g00ecf