From 4929e37507b530cb1d0e92cfcd252b0abefb4f2f Mon Sep 17 00:00:00 2001 From: baude Date: Mon, 19 Feb 2018 08:40:43 -0600 Subject: Performance enhancement for podman images Previous code was using slow routines to collect some of the information needed to output images. Specifically size was being calculated instead of using the cached, already known size already available. Also, straight- lined several of the code paths. Overall assessment is that these improvements cut the time for images in half. Signed-off-by: baude Closes: #365 Approved by: mheon --- cmd/podman/images.go | 267 +++++++++++++++++++----------------------------- libpod/runtime_img.go | 178 ++++++++++++++++++++++++++++++++ pkg/inspect/inspect.go | 15 +++ test/e2e/images_test.go | 57 +++++++++++ 4 files changed, 357 insertions(+), 160 deletions(-) diff --git a/cmd/podman/images.go b/cmd/podman/images.go index 3b41204f1..2dcd743cf 100644 --- a/cmd/podman/images.go +++ b/cmd/podman/images.go @@ -1,18 +1,15 @@ package main import ( - "fmt" "reflect" "strings" "time" - "github.com/containers/storage" "github.com/docker/go-units" digest "github.com/opencontainers/go-digest" "github.com/pkg/errors" "github.com/projectatomic/libpod/cmd/podman/formats" "github.com/projectatomic/libpod/libpod" - "github.com/projectatomic/libpod/libpod/common" "github.com/projectatomic/libpod/pkg/inspect" "github.com/urfave/cli" ) @@ -31,15 +28,16 @@ type imagesJSONParams struct { Name []string `json:"names"` Digest digest.Digest `json:"digest"` Created time.Time `json:"created"` - Size int64 `json:"size"` + Size *uint64 `json:"size"` } type imagesOptions struct { - quiet bool - noHeading bool - noTrunc bool - digests bool - format string + quiet bool + noHeading bool + noTrunc bool + digests bool + format string + outputformat string } var ( @@ -64,7 +62,7 @@ var ( Name: "format", Usage: "Change the output format to JSON or a Go template", }, - cli.StringFlag{ + cli.StringSliceFlag{ Name: "filter, f", Usage: "filter output based on conditions provided (default [])", }, @@ -92,38 +90,32 @@ func imagesCmd(c *cli.Context) error { return errors.Wrapf(err, "Could not get runtime") } defer runtime.Shutdown(false) - - format := genImagesFormat(c.String("format"), c.Bool("quiet"), c.Bool("noheading"), c.Bool("digests")) - - opts := imagesOptions{ - quiet: c.Bool("quiet"), - noHeading: c.Bool("noheading"), - noTrunc: c.Bool("no-trunc"), - digests: c.Bool("digests"), - format: format, - } - + var filterFuncs []libpod.ImageResultFilter var imageInput string if len(c.Args()) == 1 { imageInput = c.Args().Get(0) } + if len(c.Args()) > 1 { return errors.New("'podman images' requires at most 1 argument") } - params, err := runtime.ParseImageFilter(imageInput, c.String("filter")) - if err != nil { - return errors.Wrapf(err, "error parsing filter") + if len(c.StringSlice("filter")) > 0 || len(strings.TrimSpace(imageInput)) != 0 { + filterFuncs, err = CreateFilterFuncs(runtime, c, imageInput) + if err != nil { + return err + } } - // generate the different filters - labelFilter := generateImagesFilter(params, "label") - beforeImageFilter := generateImagesFilter(params, "before-image") - sinceImageFilter := generateImagesFilter(params, "since-image") - danglingFilter := generateImagesFilter(params, "dangling") - referenceFilter := generateImagesFilter(params, "reference") - imageInputFilter := generateImagesFilter(params, "image-input") + opts := imagesOptions{ + quiet: c.Bool("quiet"), + noHeading: c.Bool("noheading"), + noTrunc: c.Bool("no-trunc"), + digests: c.Bool("digests"), + format: c.String("format"), + } + opts.outputformat = opts.setOutputFormat() /* podman does not implement --all for images @@ -131,29 +123,36 @@ func imagesCmd(c *cli.Context) error { children to the image once built. until buildah supports caching builds, it will not generate these intermediate images. */ - - images, err := runtime.GetImages(params, labelFilter, beforeImageFilter, sinceImageFilter, danglingFilter, referenceFilter, imageInputFilter) + images, err := runtime.GetImageResults() if err != nil { - return errors.Wrapf(err, "could not get list of images matching filter") + return errors.Wrapf(err, "unable to get images") + } + + var filteredImages []inspect.ImageResult + // filter the images + if len(c.StringSlice("filter")) > 0 || len(strings.TrimSpace(imageInput)) != 0 { + filteredImages = libpod.FilterImages(images, filterFuncs) + } else { + filteredImages = images } - return generateImagesOutput(runtime, images, opts) + return generateImagesOutput(runtime, filteredImages, opts) } -func genImagesFormat(format string, quiet, noHeading, digests bool) string { - if format != "" { +func (i imagesOptions) setOutputFormat() string { + if i.format != "" { // "\t" from the command line is not being recognized as a tab // replacing the string "\t" to a tab character if the user passes in "\t" - return strings.Replace(format, `\t`, "\t", -1) + return strings.Replace(i.format, `\t`, "\t", -1) } - if quiet { + if i.quiet { return formats.IDString } - format = "table {{.Repository}}\t{{.Tag}}\t" - if noHeading { + format := "table {{.Repository}}\t{{.Tag}}\t" + if i.noHeading { format = "{{.Repository}}\t{{.Tag}}\t" } - if digests { + if i.digests { format += "{{.Digest}}\t" } format += "{{.ID}}\t{{.Created}}\t{{.Size}}\t" @@ -174,60 +173,22 @@ func imagesToGeneric(templParams []imagesTemplateParams, JSONParams []imagesJSON return } -// generate the header based on the template provided -func (i *imagesTemplateParams) headerMap() map[string]string { - v := reflect.Indirect(reflect.ValueOf(i)) - values := make(map[string]string) - - for i := 0; i < v.NumField(); i++ { - key := v.Type().Field(i).Name - value := key - if value == "ID" { - value = "Image" + value - } - values[key] = strings.ToUpper(splitCamelCase(value)) - } - return values -} - // getImagesTemplateOutput returns the images information to be printed in human readable format -func getImagesTemplateOutput(runtime *libpod.Runtime, images []*storage.Image, opts imagesOptions) (imagesOutput []imagesTemplateParams) { - var ( - lastID string - ) +func getImagesTemplateOutput(runtime *libpod.Runtime, images []inspect.ImageResult, opts imagesOptions) (imagesOutput []imagesTemplateParams) { for _, img := range images { - if opts.quiet && lastID == img.ID { - continue // quiet should not show the same ID multiple times - } createdTime := img.Created imageID := "sha256:" + img.ID if !opts.noTrunc { imageID = shortID(img.ID) } - - repository := "" - tag := "" - if len(img.Names) > 0 { - arr := strings.Split(img.Names[0], ":") - repository = arr[0] - if len(arr) == 2 { - tag = arr[1] - } - } - - imgData, _ := runtime.GetImageInspectInfo(*img) - if imgData != nil { - createdTime = *imgData.Created - } - params := imagesTemplateParams{ - Repository: repository, - Tag: tag, + Repository: img.Repository, + Tag: img.Tag, ID: imageID, - Digest: imgData.Digest, + Digest: img.Digest, Created: units.HumanDuration(time.Since((createdTime))) + " ago", - Size: units.HumanSizeWithPrecision(float64(imgData.Size), 3), + Size: units.HumanSizeWithPrecision(float64(*img.Size), 3), } imagesOutput = append(imagesOutput, params) } @@ -235,21 +196,14 @@ func getImagesTemplateOutput(runtime *libpod.Runtime, images []*storage.Image, o } // getImagesJSONOutput returns the images information in its raw form -func getImagesJSONOutput(runtime *libpod.Runtime, images []*storage.Image) (imagesOutput []imagesJSONParams) { +func getImagesJSONOutput(runtime *libpod.Runtime, images []inspect.ImageResult) (imagesOutput []imagesJSONParams) { for _, img := range images { - createdTime := img.Created - - imgData, _ := runtime.GetImageInspectInfo(*img) - if imgData != nil { - createdTime = *imgData.Created - } - params := imagesJSONParams{ ID: img.ID, - Name: img.Names, - Digest: imgData.Digest, - Created: createdTime, - Size: imgData.Size, + Name: img.RepoTags, + Digest: img.Digest, + Created: img.Created, + Size: img.Size, } imagesOutput = append(imagesOutput, params) } @@ -257,11 +211,11 @@ func getImagesJSONOutput(runtime *libpod.Runtime, images []*storage.Image) (imag } // generateImagesOutput generates the images based on the format provided -func generateImagesOutput(runtime *libpod.Runtime, images []*storage.Image, opts imagesOptions) error { + +func generateImagesOutput(runtime *libpod.Runtime, images []inspect.ImageResult, opts imagesOptions) error { if len(images) == 0 { return nil } - var out formats.Writer switch opts.format { @@ -270,77 +224,70 @@ func generateImagesOutput(runtime *libpod.Runtime, images []*storage.Image, opts out = formats.JSONStructArray{Output: imagesToGeneric([]imagesTemplateParams{}, imagesOutput)} default: imagesOutput := getImagesTemplateOutput(runtime, images, opts) - out = formats.StdoutTemplateArray{Output: imagesToGeneric(imagesOutput, []imagesJSONParams{}), Template: opts.format, Fields: imagesOutput[0].headerMap()} - + out = formats.StdoutTemplateArray{Output: imagesToGeneric(imagesOutput, []imagesJSONParams{}), Template: opts.outputformat, Fields: imagesOutput[0].HeaderMap()} } - return formats.Writer(out).Out() } -// generateImagesFilter returns an ImageFilter based on filterType -// to add more filters, define a new case and write what the ImageFilter function should do -func generateImagesFilter(params *libpod.ImageFilterParams, filterType string) libpod.ImageFilter { - switch filterType { - case "label": - return func(image *storage.Image, info *inspect.ImageData) bool { - if params == nil || params.Label == "" { - return true - } +// HeaderMap produces a generic map of "headers" based on a line +// of output +func (i *imagesTemplateParams) HeaderMap() map[string]string { + v := reflect.Indirect(reflect.ValueOf(i)) + values := make(map[string]string) - pair := strings.SplitN(params.Label, "=", 2) - if val, ok := info.Labels[pair[0]]; ok { - if len(pair) == 2 && val == pair[1] { - return true - } - if len(pair) == 1 { - return true - } - } - return false - } - case "before-image": - return func(image *storage.Image, info *inspect.ImageData) bool { - if params == nil || params.BeforeImage.IsZero() { - return true - } - return info.Created.Before(params.BeforeImage) - } - case "since-image": - return func(image *storage.Image, info *inspect.ImageData) bool { - if params == nil || params.SinceImage.IsZero() { - return true - } - return info.Created.After(params.SinceImage) + for i := 0; i < v.NumField(); i++ { + key := v.Type().Field(i).Name + value := key + if value == "ID" { + value = "Image" + value } - case "dangling": - return func(image *storage.Image, info *inspect.ImageData) bool { - if params == nil || params.Dangling == "" { - return true - } - if common.IsFalse(params.Dangling) && params.ImageName != "" { - return true + values[key] = strings.ToUpper(splitCamelCase(value)) + } + return values +} + +// CreateFilterFuncs returns an array of filter functions based on the user inputs +// and is later used to filter images for output +func CreateFilterFuncs(r *libpod.Runtime, c *cli.Context, userInput string) ([]libpod.ImageResultFilter, error) { + var filterFuncs []libpod.ImageResultFilter + for _, filter := range c.StringSlice("filter") { + splitFilter := strings.Split(filter, "=") + switch splitFilter[0] { + case "before": + before := r.NewImage(splitFilter[1]) + _, beforeID, _ := before.GetLocalImageName() + + if before.LocalName == "" { + return nil, errors.Errorf("unable to find image % in local stores", splitFilter[1]) } - if common.IsTrue(params.Dangling) && params.ImageName == "" { - return true + img, err := r.GetImage(beforeID) + if err != nil { + return nil, err } - return false - } - case "reference": - return func(image *storage.Image, info *inspect.ImageData) bool { - if params == nil || params.ReferencePattern == "" { - return true + filterFuncs = append(filterFuncs, libpod.ImageCreatedBefore(img.Created)) + case "after": + after := r.NewImage(splitFilter[1]) + _, afterID, _ := after.GetLocalImageName() + + if after.LocalName == "" { + return nil, errors.Errorf("unable to find image % in local stores", splitFilter[1]) } - return libpod.MatchesReference(params.ImageName, params.ReferencePattern) - } - case "image-input": - return func(image *storage.Image, info *inspect.ImageData) bool { - if params == nil || params.ImageInput == "" { - return true + img, err := r.GetImage(afterID) + if err != nil { + return nil, err } - return libpod.MatchesReference(params.ImageName, params.ImageInput) + filterFuncs = append(filterFuncs, libpod.ImageCreatedAfter(img.Created)) + case "dangling": + filterFuncs = append(filterFuncs, libpod.ImageDangling()) + case "label": + labelFilter := strings.Join(splitFilter[1:], "=") + filterFuncs = append(filterFuncs, libpod.ImageLabel(labelFilter)) + default: + return nil, errors.Errorf("invalid filter %s ", splitFilter[0]) } - default: - fmt.Println("invalid filter type", filterType) - return nil } + if len(strings.TrimSpace(userInput)) != 0 { + filterFuncs = append(filterFuncs, libpod.OutputImageFilter(userInput)) + } + return filterFuncs, nil } diff --git a/libpod/runtime_img.go b/libpod/runtime_img.go index 76687351d..cc6275494 100644 --- a/libpod/runtime_img.go +++ b/libpod/runtime_img.go @@ -108,6 +108,9 @@ type imageDecomposeStruct struct { transport string } +// ImageResultFilter is a mock function for image filtering +type ImageResultFilter func(inspect.ImageResult) bool + func (k *Image) assembleFqName() string { return fmt.Sprintf("%s/%s:%s", k.Registry, k.ImageName, k.Tag) } @@ -1262,3 +1265,178 @@ func getPolicyContext(ctx *types.SystemContext) (*signature.PolicyContext, error } return policyContext, nil } + +// sizer knows its size. +type sizer interface { + Size() (int64, error) +} + +func imageSize(img types.ImageSource) *uint64 { + if s, ok := img.(sizer); ok { + if sum, err := s.Size(); err == nil { + usum := uint64(sum) + return &usum + } + } + return nil +} + +func reposToMap(repotags []string) map[string]string { + // map format is repo -> tag + repos := make(map[string]string) + for _, repo := range repotags { + var repository, tag string + if len(repo) > 0 { + li := strings.LastIndex(repo, ":") + repository = repo[0:li] + tag = repo[li+1:] + } + repos[repository] = tag + } + if len(repos) == 0 { + repos[""] = "", 2)) + Expect(session.LineInOuputStartsWith("docker.io/library/alpine")).To(BeTrue()) + Expect(session.LineInOuputStartsWith("docker.io/library/busybox")).To(BeTrue()) + }) + It("podman images in JSON format", func() { session := podmanTest.Podman([]string{"images", "--format=json"}) session.WaitWithDefaultTimeout() @@ -44,10 +53,58 @@ var _ = Describe("Podman images", func() { Expect(session.IsJSONOutputValid()).To(BeTrue()) }) + It("podman images in GO template format", func() { + session := podmanTest.Podman([]string{"images", "--format={{.ID}}"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + }) + It("podman images with short options", func() { session := podmanTest.Podman([]string{"images", "-qn"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) Expect(len(session.OutputToStringArray())).To(BeNumerically(">", 1)) }) + + It("podman images filter by image name", func() { + session := podmanTest.Podman([]string{"images", "-q", ALPINE}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(1)) + }) + + It("podman images filter before image", func() { + dockerfile := `FROM docker.io/library/alpine:latest +` + podmanTest.BuildImage(dockerfile, "foobar.com/before:latest") + result := podmanTest.Podman([]string{"images", "-q", "-f", "before=foobar.com/before:latest"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(len(result.OutputToStringArray())).To(Equal(2)) + }) + + It("podman images filter after image", func() { + rmi := podmanTest.Podman([]string{"rmi", "busybox"}) + rmi.WaitWithDefaultTimeout() + Expect(rmi.ExitCode()).To(Equal(0)) + + dockerfile := `FROM docker.io/library/alpine:latest +` + podmanTest.BuildImage(dockerfile, "foobar.com/before:latest") + result := podmanTest.Podman([]string{"images", "-q", "-f", "after=docker.io/library/alpine:latest"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(len(result.OutputToStringArray())).To(Equal(1)) + }) + + It("podman images filter dangling", func() { + dockerfile := `FROM docker.io/library/alpine:latest +` + podmanTest.BuildImage(dockerfile, "foobar.com/before:latest") + podmanTest.BuildImage(dockerfile, "foobar.com/before:latest") + result := podmanTest.Podman([]string{"images", "-q", "-f", "dangling=true"}) + result.WaitWithDefaultTimeout() + Expect(result.ExitCode()).To(Equal(0)) + Expect(len(result.OutputToStringArray())).To(Equal(1)) + }) }) -- cgit v1.2.3-54-g00ecf