From 87c9f4cc226a8b709d39de473f09dd341b1a4fc3 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Mon, 18 Feb 2019 16:14:34 +0100 Subject: podman-search: refactor code to libpod/image/search.go Refactor the image-search logic from cmd/podman/search.go to libpod/image/search.go and update podman-search and the Varlink API to use it. Signed-off-by: Valentin Rothberg --- libpod/image/search.go | 280 +++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+) create mode 100644 libpod/image/search.go (limited to 'libpod/image/search.go') diff --git a/libpod/image/search.go b/libpod/image/search.go new file mode 100644 index 000000000..e9612ef47 --- /dev/null +++ b/libpod/image/search.go @@ -0,0 +1,280 @@ +package image + +import ( + "context" + "reflect" + "strconv" + "strings" + "sync" + + "github.com/containers/image/docker" + "github.com/containers/image/types" + "github.com/containers/libpod/libpod/common" + sysreg "github.com/containers/libpod/pkg/registries" + "github.com/fatih/camelcase" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "golang.org/x/sync/semaphore" +) + +const ( + descriptionTruncLength = 44 + maxQueries = 25 + maxParallelSearches = int64(6) +) + +// SearchResult is holding image-search related data. +type SearchResult struct { + // Index is the image index (e.g., "docker.io" or "quay.io") + Index string + // Name is the canoncical name of the image (e.g., "docker.io/library/alpine"). + Name string + // Description of the image. + Description string + // Stars is the number of stars of the image. + Stars int + // Official indicates if it's an official image. + Official string + // Automated indicates if the image was created by an automated build. + Automated string +} + +// SearchOptions are used to control the behaviour of SearchImages. +type SearchOptions struct { + // Filter allows to filter the results. + Filter []string + // Limit limits the number of queries per index (default: 25). Must be + // greater than 0 to overwrite the default value. + Limit int + // NoTrunc avoids the output to be truncated. + NoTrunc bool + // Authfile is the path to the authentication file. + Authfile string + // InsecureSkipTLSVerify allows to skip TLS verification. + InsecureSkipTLSVerify types.OptionalBool +} + +type searchFilterParams struct { + stars int + isAutomated *bool + isOfficial *bool +} + +func splitCamelCase(src string) string { + entries := camelcase.Split(src) + return strings.Join(entries, " ") +} + +// HeaderMap returns the headers of a SearchResult. +func (s *SearchResult) HeaderMap() map[string]string { + v := reflect.Indirect(reflect.ValueOf(s)) + values := make(map[string]string, v.NumField()) + + for i := 0; i < v.NumField(); i++ { + key := v.Type().Field(i).Name + value := key + values[key] = strings.ToUpper(splitCamelCase(value)) + } + return values +} + +// SearchImages searches images based on term and the specified SearchOptions +// in all registries. +func SearchImages(term string, options SearchOptions) ([]SearchResult, error) { + filter, err := parseSearchFilter(&options) + if err != nil { + return nil, err + } + + // Check if search term has a registry in it + registry, err := sysreg.GetRegistry(term) + if err != nil { + return nil, errors.Wrapf(err, "error getting registry from %q", term) + } + if registry != "" { + term = term[len(registry)+1:] + } + + registries, err := getRegistries(registry) + if err != nil { + return nil, err + } + + // searchOutputData is used as a return value for searching in parallel. + type searchOutputData struct { + data []SearchResult + err error + } + + // Let's follow Firefox by limiting parallel downloads to 6. + sem := semaphore.NewWeighted(maxParallelSearches) + wg := sync.WaitGroup{} + wg.Add(len(registries)) + data := make([]searchOutputData, len(registries)) + + searchImageInRegistryHelper := func(index int, registry string) { + defer sem.Release(1) + defer wg.Done() + searchOutput, err := searchImageInRegistry(term, registry, options, filter) + data[index] = searchOutputData{data: searchOutput, err: err} + } + + ctx := context.Background() + for i := range registries { + sem.Acquire(ctx, 1) + go searchImageInRegistryHelper(i, registries[i]) + } + + wg.Wait() + results := []SearchResult{} + for _, d := range data { + if d.err != nil { + return nil, d.err + } + results = append(results, d.data...) + } + return results, nil +} + +// getRegistries returns the list of registries to search, depending on an optional registry specification +func getRegistries(registry string) ([]string, error) { + var registries []string + if registry != "" { + registries = append(registries, registry) + } else { + var err error + registries, err = sysreg.GetRegistries() + if err != nil { + return nil, errors.Wrapf(err, "error getting registries to search") + } + } + return registries, nil +} + +func searchImageInRegistry(term string, registry string, options SearchOptions, filter *searchFilterParams) ([]SearchResult, error) { + // Max number of queries by default is 25 + limit := maxQueries + if options.Limit > 0 { + limit = options.Limit + } + + sc := common.GetSystemContext("", options.Authfile, false) + sc.DockerInsecureSkipTLSVerify = options.InsecureSkipTLSVerify + // FIXME: Set this more globally. Probably no reason not to have it in + // every types.SystemContext, and to compute the value just once in one + // place. + sc.SystemRegistriesConfPath = sysreg.SystemRegistriesConfPath() + results, err := docker.SearchRegistry(context.TODO(), sc, registry, term, limit) + if err != nil { + logrus.Errorf("error searching registry %q: %v", registry, err) + return []SearchResult{}, nil + } + index := registry + arr := strings.Split(registry, ".") + if len(arr) > 2 { + index = strings.Join(arr[len(arr)-2:], ".") + } + + // limit is the number of results to output + // if the total number of results is less than the limit, output all + // if the limit has been set by the user, output those number of queries + limit = maxQueries + if len(results) < limit { + limit = len(results) + } + if options.Limit != 0 && options.Limit < len(results) { + limit = options.Limit + } + + paramsArr := []SearchResult{} + for i := 0; i < limit; i++ { + if len(options.Filter) > 0 { + // Check whether query matches filters + if !(matchesAutomatedFilter(filter, results[i]) && matchesOfficialFilter(filter, results[i]) && matchesStarFilter(filter, results[i])) { + continue + } + } + official := "" + if results[i].IsOfficial { + official = "[OK]" + } + automated := "" + if results[i].IsAutomated { + automated = "[OK]" + } + description := strings.Replace(results[i].Description, "\n", " ", -1) + if len(description) > 44 && !options.NoTrunc { + description = description[:descriptionTruncLength] + "..." + } + name := registry + "/" + results[i].Name + if index == "docker.io" && !strings.Contains(results[i].Name, "/") { + name = index + "/library/" + results[i].Name + } + params := SearchResult{ + Index: index, + Name: name, + Description: description, + Official: official, + Automated: automated, + Stars: results[i].StarCount, + } + paramsArr = append(paramsArr, params) + } + return paramsArr, nil +} + +func parseSearchFilter(options *SearchOptions) (*searchFilterParams, error) { + filterParams := &searchFilterParams{} + ptrTrue := true + ptrFalse := false + for _, filter := range options.Filter { + arr := strings.Split(filter, "=") + switch arr[0] { + case "stars": + if len(arr) < 2 { + return nil, errors.Errorf("invalid `stars` filter %q, should be stars=", filter) + } + stars, err := strconv.Atoi(arr[1]) + if err != nil { + return nil, errors.Wrapf(err, "incorrect value type for stars filter") + } + filterParams.stars = stars + break + case "is-automated": + if len(arr) == 2 && arr[1] == "false" { + filterParams.isAutomated = &ptrFalse + } else { + filterParams.isAutomated = &ptrTrue + } + break + case "is-official": + if len(arr) == 2 && arr[1] == "false" { + filterParams.isOfficial = &ptrFalse + } else { + filterParams.isOfficial = &ptrTrue + } + break + default: + return nil, errors.Errorf("invalid filter type %q", filter) + } + } + return filterParams, nil +} + +func matchesStarFilter(filter *searchFilterParams, result docker.SearchResult) bool { + return result.StarCount >= filter.stars +} + +func matchesAutomatedFilter(filter *searchFilterParams, result docker.SearchResult) bool { + if filter.isAutomated != nil { + return result.IsAutomated == *filter.isAutomated + } + return true +} + +func matchesOfficialFilter(filter *searchFilterParams, result docker.SearchResult) bool { + if filter.isOfficial != nil { + return result.IsOfficial == *filter.isOfficial + } + return true +} -- cgit v1.2.3-54-g00ecf From 8a49b59ed403b25340f315b3cb9ea90482ad6cd9 Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Tue, 19 Feb 2019 15:09:19 +0100 Subject: image.SearchImages: use SearchFilter type Use an `image.SearchFilter` instead of a `[]string` in the SearchImages API. Signed-off-by: Valentin Rothberg --- cmd/podman/search.go | 7 ++++- libpod/image/search.go | 73 +++++++++++++++++++++++------------------------- pkg/varlinkapi/images.go | 7 ++++- 3 files changed, 47 insertions(+), 40 deletions(-) (limited to 'libpod/image/search.go') diff --git a/cmd/podman/search.go b/cmd/podman/search.go index a772827a6..f63131c84 100644 --- a/cmd/podman/search.go +++ b/cmd/podman/search.go @@ -58,10 +58,15 @@ func searchCmd(c *cliconfig.SearchValues) error { } term := args[0] + filter, err := image.ParseSearchFilter(c.Filter) + if err != nil { + return err + } + searchOptions := image.SearchOptions{ NoTrunc: c.NoTrunc, Limit: c.Limit, - Filter: c.Filter, + Filter: *filter, Authfile: getAuthFile(c.Authfile), } if c.Flag("tls-verify").Changed { diff --git a/libpod/image/search.go b/libpod/image/search.go index e9612ef47..212eff00b 100644 --- a/libpod/image/search.go +++ b/libpod/image/search.go @@ -42,7 +42,7 @@ type SearchResult struct { // SearchOptions are used to control the behaviour of SearchImages. type SearchOptions struct { // Filter allows to filter the results. - Filter []string + Filter SearchFilter // Limit limits the number of queries per index (default: 25). Must be // greater than 0 to overwrite the default value. Limit int @@ -54,10 +54,14 @@ type SearchOptions struct { InsecureSkipTLSVerify types.OptionalBool } -type searchFilterParams struct { - stars int - isAutomated *bool - isOfficial *bool +// SearchFilter allows filtering the results of SearchImages. +type SearchFilter struct { + // Stars describes the minimal amount of starts of an image. + Stars int + // IsAutomated decides if only images from automated builds are displayed. + IsAutomated types.OptionalBool + // IsOfficial decides if only official images are displayed. + IsOfficial types.OptionalBool } func splitCamelCase(src string) string { @@ -81,11 +85,6 @@ func (s *SearchResult) HeaderMap() map[string]string { // SearchImages searches images based on term and the specified SearchOptions // in all registries. func SearchImages(term string, options SearchOptions) ([]SearchResult, error) { - filter, err := parseSearchFilter(&options) - if err != nil { - return nil, err - } - // Check if search term has a registry in it registry, err := sysreg.GetRegistry(term) if err != nil { @@ -115,7 +114,7 @@ func SearchImages(term string, options SearchOptions) ([]SearchResult, error) { searchImageInRegistryHelper := func(index int, registry string) { defer sem.Release(1) defer wg.Done() - searchOutput, err := searchImageInRegistry(term, registry, options, filter) + searchOutput, err := searchImageInRegistry(term, registry, options) data[index] = searchOutputData{data: searchOutput, err: err} } @@ -151,7 +150,7 @@ func getRegistries(registry string) ([]string, error) { return registries, nil } -func searchImageInRegistry(term string, registry string, options SearchOptions, filter *searchFilterParams) ([]SearchResult, error) { +func searchImageInRegistry(term string, registry string, options SearchOptions) ([]SearchResult, error) { // Max number of queries by default is 25 limit := maxQueries if options.Limit > 0 { @@ -188,11 +187,9 @@ func searchImageInRegistry(term string, registry string, options SearchOptions, paramsArr := []SearchResult{} for i := 0; i < limit; i++ { - if len(options.Filter) > 0 { - // Check whether query matches filters - if !(matchesAutomatedFilter(filter, results[i]) && matchesOfficialFilter(filter, results[i]) && matchesStarFilter(filter, results[i])) { - continue - } + // Check whether query matches filters + if !(options.Filter.matchesAutomatedFilter(results[i]) && options.Filter.matchesOfficialFilter(results[i]) && options.Filter.matchesStarFilter(results[i])) { + continue } official := "" if results[i].IsOfficial { @@ -223,12 +220,12 @@ func searchImageInRegistry(term string, registry string, options SearchOptions, return paramsArr, nil } -func parseSearchFilter(options *SearchOptions) (*searchFilterParams, error) { - filterParams := &searchFilterParams{} - ptrTrue := true - ptrFalse := false - for _, filter := range options.Filter { - arr := strings.Split(filter, "=") +// ParseSearchFilter turns the filter into a SearchFilter that can be used for +// searching images. +func ParseSearchFilter(filter []string) (*SearchFilter, error) { + sFilter := new(SearchFilter) + for _, f := range filter { + arr := strings.Split(f, "=") switch arr[0] { case "stars": if len(arr) < 2 { @@ -238,43 +235,43 @@ func parseSearchFilter(options *SearchOptions) (*searchFilterParams, error) { if err != nil { return nil, errors.Wrapf(err, "incorrect value type for stars filter") } - filterParams.stars = stars + sFilter.Stars = stars break case "is-automated": if len(arr) == 2 && arr[1] == "false" { - filterParams.isAutomated = &ptrFalse + sFilter.IsAutomated = types.OptionalBoolFalse } else { - filterParams.isAutomated = &ptrTrue + sFilter.IsAutomated = types.OptionalBoolTrue } break case "is-official": if len(arr) == 2 && arr[1] == "false" { - filterParams.isOfficial = &ptrFalse + sFilter.IsOfficial = types.OptionalBoolFalse } else { - filterParams.isOfficial = &ptrTrue + sFilter.IsOfficial = types.OptionalBoolTrue } break default: - return nil, errors.Errorf("invalid filter type %q", filter) + return nil, errors.Errorf("invalid filter type %q", f) } } - return filterParams, nil + return sFilter, nil } -func matchesStarFilter(filter *searchFilterParams, result docker.SearchResult) bool { - return result.StarCount >= filter.stars +func (f *SearchFilter) matchesStarFilter(result docker.SearchResult) bool { + return result.StarCount >= f.Stars } -func matchesAutomatedFilter(filter *searchFilterParams, result docker.SearchResult) bool { - if filter.isAutomated != nil { - return result.IsAutomated == *filter.isAutomated +func (f *SearchFilter) matchesAutomatedFilter(result docker.SearchResult) bool { + if f.IsAutomated != types.OptionalBoolUndefined { + return result.IsAutomated == (f.IsAutomated == types.OptionalBoolTrue) } return true } -func matchesOfficialFilter(filter *searchFilterParams, result docker.SearchResult) bool { - if filter.isOfficial != nil { - return result.IsOfficial == *filter.isOfficial +func (f *SearchFilter) matchesOfficialFilter(result docker.SearchResult) bool { + if f.IsOfficial != types.OptionalBoolUndefined { + return result.IsOfficial == (f.IsOfficial == types.OptionalBoolTrue) } return true } diff --git a/pkg/varlinkapi/images.go b/pkg/varlinkapi/images.go index f4b8a7e18..032160b1b 100644 --- a/pkg/varlinkapi/images.go +++ b/pkg/varlinkapi/images.go @@ -435,9 +435,14 @@ func (i *LibpodAPI) RemoveImage(call iopodman.VarlinkCall, name string, force bo // SearchImages searches all registries configured in /etc/containers/registries.conf for an image // Requires an image name and a search limit as int func (i *LibpodAPI) SearchImages(call iopodman.VarlinkCall, query string, limit *int64, tlsVerify *bool, filter []string) error { + sFilter, err := image.ParseSearchFilter(filter) + if err != nil { + return call.ReplyErrorOccurred(err.Error()) + } + searchOptions := image.SearchOptions{ Limit: 1000, - Filter: filter, + Filter: *sFilter, InsecureSkipTLSVerify: types.NewOptionalBool(!*tlsVerify), } results, err := image.SearchImages(query, searchOptions) -- cgit v1.2.3-54-g00ecf