From 3bdad6fa2af41be7e783256f3c8a213dc42ca8f6 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Mon, 30 Mar 2020 12:13:50 +0200 Subject: podmanV2: implement pull Implement pulling images for the v2 client. What I _really_ don't like is the fact that we are now having a near identical code clone among `pkg/domain/infra/abi` and `pkg/api/handlers/libpod`. Partly because we don't yet have a higher-level pull function and partly because we have redudancy among `pkg/domain` and `pkg/api`. Pull might be a high outlier but I am concerned already by the potential of introducing more redundancy. I'd love to `infra/abi` and `pkg/abi` to really use the same code in the future. Signed-off-by: Valentin Rothberg --- cmd/podmanV2/images/pull.go | 140 ++++++++++++++++++++++++++++++++++++ pkg/api/handlers/libpod/images.go | 18 +++-- pkg/bindings/images/images.go | 39 ++++++++++ pkg/bindings/test/images_test.go | 21 ++++++ pkg/domain/entities/engine_image.go | 1 + pkg/domain/entities/images.go | 34 +++++++++ pkg/domain/infra/abi/images.go | 100 ++++++++++++++++++++++++++ pkg/domain/infra/tunnel/images.go | 8 +++ 8 files changed, 356 insertions(+), 5 deletions(-) create mode 100644 cmd/podmanV2/images/pull.go diff --git a/cmd/podmanV2/images/pull.go b/cmd/podmanV2/images/pull.go new file mode 100644 index 000000000..c7e325409 --- /dev/null +++ b/cmd/podmanV2/images/pull.go @@ -0,0 +1,140 @@ +package images + +import ( + "fmt" + + buildahcli "github.com/containers/buildah/pkg/cli" + "github.com/containers/image/v5/types" + "github.com/containers/libpod/cmd/podmanV2/registry" + "github.com/containers/libpod/pkg/domain/entities" + "github.com/opentracing/opentracing-go" + "github.com/pkg/errors" + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +// pullOptionsWrapper wraps entities.ImagePullOptions and prevents leaking +// CLI-only fields into the API types. +type pullOptionsWrapper struct { + entities.ImagePullOptions + TLSVerifyCLI bool // CLI only +} + +var ( + pullOptions = pullOptionsWrapper{} + pullDescription = `Pulls an image from a registry and stores it locally. + + An image can be pulled by tag or digest. If a tag is not specified, the image with the 'latest' tag is pulled.` + + // Command: podman pull + pullCmd = &cobra.Command{ + Use: "pull [flags] IMAGE", + Short: "Pull an image from a registry", + Long: pullDescription, + PreRunE: preRunE, + RunE: imagePull, + Example: `podman pull imageName + podman pull fedora:latest`, + } + + // Command: podman image pull + // It's basically a clone of `pullCmd` with the exception of being a + // child of the images command. + imagesPullCmd = &cobra.Command{ + Use: pullCmd.Use, + Short: pullCmd.Short, + Long: pullCmd.Long, + PreRunE: pullCmd.PreRunE, + RunE: pullCmd.RunE, + Example: `podman image pull imageName + podman image pull fedora:latest`, + } +) + +func init() { + // pull + registry.Commands = append(registry.Commands, registry.CliCommand{ + Mode: []entities.EngineMode{entities.ABIMode, entities.TunnelMode}, + Command: pullCmd, + }) + + pullCmd.SetHelpTemplate(registry.HelpTemplate()) + pullCmd.SetUsageTemplate(registry.UsageTemplate()) + + flags := pullCmd.Flags() + pullFlags(flags) + + // images pull + registry.Commands = append(registry.Commands, registry.CliCommand{ + Mode: []entities.EngineMode{entities.ABIMode, entities.TunnelMode}, + Command: imagesPullCmd, + Parent: imageCmd, + }) + + imagesPullCmd.SetHelpTemplate(registry.HelpTemplate()) + imagesPullCmd.SetUsageTemplate(registry.UsageTemplate()) + imagesPullFlags := imagesPullCmd.Flags() + pullFlags(imagesPullFlags) +} + +// pullFlags set the flags for the pull command. +func pullFlags(flags *pflag.FlagSet) { + flags.BoolVar(&pullOptions.AllTags, "all-tags", false, "All tagged images in the repository will be pulled") + flags.StringVar(&pullOptions.Authfile, "authfile", buildahcli.GetDefaultAuthFile(), "Path of the authentication file. Use REGISTRY_AUTH_FILE environment variable to override") + flags.StringVar(&pullOptions.CertDir, "cert-dir", "", "`Pathname` of a directory containing TLS certificates and keys") + flags.StringVar(&pullOptions.Credentials, "creds", "", "`Credentials` (USERNAME:PASSWORD) to use for authenticating to a registry") + flags.StringVar(&pullOptions.OverrideArch, "override-arch", "", "Use `ARCH` instead of the architecture of the machine for choosing images") + flags.StringVar(&pullOptions.OverrideOS, "override-os", "", "Use `OS` instead of the running OS for choosing images") + flags.BoolVarP(&pullOptions.Quiet, "quiet", "q", false, "Suppress output information when pulling images") + flags.StringVar(&pullOptions.SignaturePolicy, "signature-policy", "", "`Pathname` of signature policy file (not usually used)") + flags.BoolVar(&pullOptions.TLSVerifyCLI, "tls-verify", true, "Require HTTPS and verify certificates when contacting registries") + + if registry.IsRemote() { + _ = flags.MarkHidden("authfile") + _ = flags.MarkHidden("cert-dir") + _ = flags.MarkHidden("signature-policy") + _ = flags.MarkHidden("tls-verify") + } +} + +// imagePull is implement the command for pulling images. +func imagePull(cmd *cobra.Command, args []string) error { + // Sanity check input. + if len(args) == 0 { + return errors.Errorf("an image name must be specified") + } + if len(args) > 1 { + return errors.Errorf("too many arguments. Requires exactly 1") + } + + // Start tracing if requested. + if cmd.Flags().Changed("trace") { + span, _ := opentracing.StartSpanFromContext(registry.GetContext(), "pullCmd") + defer span.Finish() + } + + pullOptsAPI := pullOptions.ImagePullOptions + // TLS verification in c/image is controlled via a `types.OptionalBool` + // which allows for distinguishing among set-true, set-false, unspecified + // which is important to implement a sane way of dealing with defaults of + // boolean CLI flags. + if cmd.Flags().Changed("tls-verify") { + pullOptsAPI.TLSVerify = types.NewOptionalBool(pullOptions.TLSVerifyCLI) + } + + // Let's do all the remaining Yoga in the API to prevent us from + // scattering logic across (too) many parts of the code. + pullReport, err := registry.ImageEngine().Pull(registry.GetContext(), args[0], pullOptsAPI) + if err != nil { + return err + } + + if len(pullReport.Images) > 1 { + fmt.Println("Pulled Images:") + } + for _, img := range pullReport.Images { + fmt.Println(img) + } + + return nil +} diff --git a/pkg/api/handlers/libpod/images.go b/pkg/api/handlers/libpod/images.go index 4b24d7d9f..bb54450da 100644 --- a/pkg/api/handlers/libpod/images.go +++ b/pkg/api/handlers/libpod/images.go @@ -303,6 +303,10 @@ func ImagesImport(w http.ResponseWriter, r *http.Request) { utils.WriteResponse(w, http.StatusOK, handlers.LibpodImagesImportReport{ID: importedImage}) } +// ImagesPull is the v2 libpod endpoint for pulling images. Note that the +// mandatory `reference` must be a reference to a registry (i.e., of docker +// transport or be normalized to one). Other transports are rejected as they +// do not make sense in a remote context. func ImagesPull(w http.ResponseWriter, r *http.Request) { runtime := r.Context().Value("runtime").(*libpod.Runtime) decoder := r.Context().Value("decoder").(*schema.Decoder) @@ -328,10 +332,11 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) { return } // Enforce the docker transport. This is just a precaution as some callers - // might accustomed to using the "transport:reference" notation. Using + // might be accustomed to using the "transport:reference" notation. Using // another than the "docker://" transport does not really make sense for a // remote case. For loading tarballs, the load and import endpoints should // be used. + dockerPrefix := fmt.Sprintf("%s://", docker.Transport.Name()) imageRef, err := alltransports.ParseImageName(query.Reference) if err == nil && imageRef.Transport().Name() != docker.Transport.Name() { utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, @@ -339,7 +344,7 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) { return } else if err != nil { origErr := err - imageRef, err = alltransports.ParseImageName(fmt.Sprintf("%s://%s", docker.Transport.Name(), query.Reference)) + imageRef, err = alltransports.ParseImageName(fmt.Sprintf("%s%s", dockerPrefix, query.Reference)) if err != nil { utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, errors.Wrapf(origErr, "reference %q must be a docker reference", query.Reference)) @@ -347,16 +352,19 @@ func ImagesPull(w http.ResponseWriter, r *http.Request) { } } + // Trim the docker-transport prefix. + rawImage := strings.TrimPrefix(query.Reference, dockerPrefix) + // all-tags doesn't work with a tagged reference, so let's check early - namedRef, err := reference.Parse(query.Reference) + namedRef, err := reference.Parse(rawImage) if err != nil { utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, - errors.Wrapf(err, "error parsing reference %q", query.Reference)) + errors.Wrapf(err, "error parsing reference %q", rawImage)) return } if _, isTagged := namedRef.(reference.Tagged); isTagged && query.AllTags { utils.Error(w, http.StatusText(http.StatusBadRequest), http.StatusBadRequest, - errors.Errorf("reference %q must not have a tag for all-tags", query.Reference)) + errors.Errorf("reference %q must not have a tag for all-tags", rawImage)) return } diff --git a/pkg/bindings/images/images.go b/pkg/bindings/images/images.go index 5e3af7a60..5f5c4443f 100644 --- a/pkg/bindings/images/images.go +++ b/pkg/bindings/images/images.go @@ -8,6 +8,7 @@ import ( "net/url" "strconv" + "github.com/containers/image/v5/types" "github.com/containers/libpod/pkg/api/handlers" "github.com/containers/libpod/pkg/bindings" "github.com/containers/libpod/pkg/domain/entities" @@ -229,3 +230,41 @@ func Import(ctx context.Context, changes []string, message, reference, u *string } return id.ID, response.Process(&id) } + +// Pull is the binding for libpod's v2 endpoints for pulling images. Note that +// `rawImage` must be a reference to a registry (i.e., of docker transport or be +// normalized to one). Other transports are rejected as they do not make sense +// in a remote context. +func Pull(ctx context.Context, rawImage string, options entities.ImagePullOptions) ([]string, error) { + conn, err := bindings.GetClient(ctx) + if err != nil { + return nil, err + } + params := url.Values{} + params.Set("reference", rawImage) + params.Set("credentials", options.Credentials) + params.Set("overrideArch", options.OverrideArch) + params.Set("overrideOS", options.OverrideOS) + if options.TLSVerify != types.OptionalBoolUndefined { + val := bool(options.TLSVerify == types.OptionalBoolTrue) + params.Set("tlsVerify", strconv.FormatBool(val)) + } + params.Set("allTags", strconv.FormatBool(options.AllTags)) + + response, err := conn.DoRequest(nil, http.MethodPost, "/images/pull", params) + if err != nil { + return nil, err + } + + reports := []handlers.LibpodImagesPullReport{} + if err := response.Process(&reports); err != nil { + return nil, err + } + + pulledImages := []string{} + for _, r := range reports { + pulledImages = append(pulledImages, r.ID) + } + + return pulledImages, nil +} diff --git a/pkg/bindings/test/images_test.go b/pkg/bindings/test/images_test.go index 13b6086c3..dc01a793b 100644 --- a/pkg/bindings/test/images_test.go +++ b/pkg/bindings/test/images_test.go @@ -9,6 +9,7 @@ import ( "github.com/containers/libpod/pkg/bindings" "github.com/containers/libpod/pkg/bindings/containers" "github.com/containers/libpod/pkg/bindings/images" + "github.com/containers/libpod/pkg/domain/entities" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" @@ -353,4 +354,24 @@ var _ = Describe("Podman images", func() { Expect(results).To(ContainElement("docker.io/library/alpine:latest")) }) + // TODO: we really need to extent to pull tests once we have a more sophisticated CI. + It("Image Pull", func() { + rawImage := "docker.io/library/busybox:latest" + + pulledImages, err := images.Pull(bt.conn, rawImage, entities.ImagePullOptions{}) + Expect(err).To(BeNil()) + Expect(len(pulledImages)).To(Equal(1)) + + exists, err := images.Exists(bt.conn, rawImage) + Expect(err).To(BeNil()) + Expect(exists).To(BeTrue()) + + // Make sure the normalization AND the full-transport reference works. + _, err = images.Pull(bt.conn, "docker://"+rawImage, entities.ImagePullOptions{}) + Expect(err).To(BeNil()) + + // The v2 endpoint only supports the docker transport. Let's see if that's really true. + _, err = images.Pull(bt.conn, "bogus-transport:bogus.com/image:reference", entities.ImagePullOptions{}) + Expect(err).To(Not(BeNil())) + }) }) diff --git a/pkg/domain/entities/engine_image.go b/pkg/domain/entities/engine_image.go index d0c860a04..cdb8d7f0d 100644 --- a/pkg/domain/entities/engine_image.go +++ b/pkg/domain/entities/engine_image.go @@ -10,4 +10,5 @@ type ImageEngine interface { History(ctx context.Context, nameOrId string, opts ImageHistoryOptions) (*ImageHistoryReport, error) List(ctx context.Context, opts ImageListOptions) ([]*ImageSummary, error) Prune(ctx context.Context, opts ImagePruneOptions) (*ImagePruneReport, error) + Pull(ctx context.Context, rawImage string, opts ImagePullOptions) (*ImagePullReport, error) } diff --git a/pkg/domain/entities/images.go b/pkg/domain/entities/images.go index 20af0356f..8e3f49be3 100644 --- a/pkg/domain/entities/images.go +++ b/pkg/domain/entities/images.go @@ -4,6 +4,7 @@ import ( "net/url" "github.com/containers/image/v5/manifest" + "github.com/containers/image/v5/types" docker "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/opencontainers/go-digest" @@ -117,6 +118,39 @@ type ImageInspectOptions struct { Latest bool `json:",omitempty"` } +// ImagePullOptions are the arguments for pulling images. +type ImagePullOptions struct { + // AllTags can be specified to pull all tags of the spiecifed image. Note + // that this only works if the specified image does not include a tag. + AllTags bool + // Authfile is the path to the authentication file. Ignored for remote + // calls. + Authfile string + // CertDir is the path to certificate directories. Ignored for remote + // calls. + CertDir string + // Credentials for authenticating against the registry in the format + // USERNAME:PASSWORD. + Credentials string + // OverrideArch will overwrite the local architecture for image pulls. + OverrideArch string + // OverrideOS will overwrite the local operating system (OS) for image + // pulls. + OverrideOS string + // Quiet can be specified to suppress pull progress when pulling. Ignored + // for remote calls. + Quiet bool + // SignaturePolicy to use when pulling. Ignored for remote calls. + SignaturePolicy string + // TLSVerify to enable/disable HTTPS and certificate verification. + TLSVerify types.OptionalBool +} + +// ImagePullReport is the response from pulling one or more images. +type ImagePullReport struct { + Images []string +} + type ImageListOptions struct { All bool `json:"all" schema:"all"` Filter []string `json:"Filter,omitempty"` diff --git a/pkg/domain/infra/abi/images.go b/pkg/domain/infra/abi/images.go index 44420c1e1..ef2879246 100644 --- a/pkg/domain/infra/abi/images.go +++ b/pkg/domain/infra/abi/images.go @@ -5,11 +5,22 @@ package abi import ( "context" "fmt" + "io" + "os" + "strings" + "github.com/containers/image/v5/docker" + dockerarchive "github.com/containers/image/v5/docker/archive" + "github.com/containers/image/v5/docker/reference" + "github.com/containers/image/v5/transports/alltransports" + "github.com/containers/image/v5/types" + "github.com/containers/libpod/libpod/image" libpodImage "github.com/containers/libpod/libpod/image" "github.com/containers/libpod/pkg/domain/entities" + "github.com/containers/libpod/pkg/util" "github.com/containers/storage" "github.com/pkg/errors" + "github.com/sirupsen/logrus" ) func (ir *ImageEngine) Exists(_ context.Context, nameOrId string) (*entities.BoolReport, error) { @@ -134,6 +145,95 @@ func ToDomainHistoryLayer(layer *libpodImage.History) entities.ImageHistoryLayer return l } +func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, options entities.ImagePullOptions) (*entities.ImagePullReport, error) { + var writer io.Writer + if !options.Quiet { + writer = os.Stderr + } + + dockerPrefix := fmt.Sprintf("%s://", docker.Transport.Name()) + imageRef, err := alltransports.ParseImageName(rawImage) + if err != nil { + imageRef, err = alltransports.ParseImageName(fmt.Sprintf("%s%s", dockerPrefix, rawImage)) + if err != nil { + return nil, errors.Errorf("invalid image reference %q", rawImage) + } + } + + // Special-case for docker-archive which allows multiple tags. + if imageRef.Transport().Name() == dockerarchive.Transport.Name() { + newImage, err := ir.Libpod.ImageRuntime().LoadFromArchiveReference(ctx, imageRef, options.SignaturePolicy, writer) + if err != nil { + return nil, errors.Wrapf(err, "error pulling image %q", rawImage) + } + return &entities.ImagePullReport{Images: []string{newImage[0].ID()}}, nil + } + + var registryCreds *types.DockerAuthConfig + if options.Credentials != "" { + creds, err := util.ParseRegistryCreds(options.Credentials) + if err != nil { + return nil, err + } + registryCreds = creds + } + dockerRegistryOptions := image.DockerRegistryOptions{ + DockerRegistryCreds: registryCreds, + DockerCertPath: options.CertDir, + OSChoice: options.OverrideOS, + ArchitectureChoice: options.OverrideArch, + DockerInsecureSkipTLSVerify: options.TLSVerify, + } + + if !options.AllTags { + newImage, err := ir.Libpod.ImageRuntime().New(ctx, rawImage, options.SignaturePolicy, options.Authfile, writer, &dockerRegistryOptions, image.SigningOptions{}, nil, util.PullImageAlways) + if err != nil { + return nil, errors.Wrapf(err, "error pulling image %q", rawImage) + } + return &entities.ImagePullReport{Images: []string{newImage.ID()}}, nil + } + + // --all-tags requires the docker transport + if imageRef.Transport().Name() != docker.Transport.Name() { + return nil, errors.New("--all-tags requires docker transport") + } + + // Trim the docker-transport prefix. + rawImage = strings.TrimPrefix(rawImage, docker.Transport.Name()) + + // all-tags doesn't work with a tagged reference, so let's check early + namedRef, err := reference.Parse(rawImage) + if err != nil { + return nil, errors.Wrapf(err, "error parsing %q", rawImage) + } + if _, isTagged := namedRef.(reference.Tagged); isTagged { + return nil, errors.New("--all-tags requires a reference without a tag") + + } + + systemContext := image.GetSystemContext("", options.Authfile, false) + tags, err := docker.GetRepositoryTags(ctx, systemContext, imageRef) + if err != nil { + return nil, errors.Wrapf(err, "error getting repository tags") + } + + var foundIDs []string + for _, tag := range tags { + name := rawImage + ":" + tag + newImage, err := ir.Libpod.ImageRuntime().New(ctx, name, options.SignaturePolicy, options.Authfile, writer, &dockerRegistryOptions, image.SigningOptions{}, nil, util.PullImageAlways) + if err != nil { + logrus.Errorf("error pulling image %q", name) + continue + } + foundIDs = append(foundIDs, newImage.ID()) + } + + if len(tags) != len(foundIDs) { + return nil, errors.Errorf("error pulling image %q", rawImage) + } + return &entities.ImagePullReport{Images: foundIDs}, nil +} + // func (r *imageRuntime) Delete(ctx context.Context, nameOrId string, opts entities.ImageDeleteOptions) (*entities.ImageDeleteReport, error) { // image, err := r.libpod.ImageEngine().NewFromLocal(nameOrId) // if err != nil { diff --git a/pkg/domain/infra/tunnel/images.go b/pkg/domain/infra/tunnel/images.go index 6a3adc9ee..7638d908a 100644 --- a/pkg/domain/infra/tunnel/images.go +++ b/pkg/domain/infra/tunnel/images.go @@ -85,3 +85,11 @@ func (ir *ImageEngine) Prune(ctx context.Context, opts entities.ImagePruneOption } return &report, nil } + +func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, options entities.ImagePullOptions) (*entities.ImagePullReport, error) { + pulledImages, err := images.Pull(ir.ClientCxt, rawImage, options) + if err != nil { + return nil, err + } + return &entities.ImagePullReport{Images: pulledImages}, nil +} -- cgit v1.2.3-54-g00ecf