From 30d2964ff83387e3c3fa7447776c57f4342707e6 Mon Sep 17 00:00:00 2001 From: Brent Baude Date: Wed, 15 Apr 2020 10:52:12 -0500 Subject: v2 bloat pruning phase 2 this is second phase of removing unneeded bloat in the remote client. this is important to be able to reduce the client size as well as possible native compilation for windows/mac. Signed-off-by: Brent Baude --- cmd/podman/runlabel.go | 33 +++++++++++++++++++++++-- cmd/podmanV2/common/create.go | 11 ++++----- cmd/podmanV2/common/default.go | 13 ++++++++++ pkg/api/handlers/compat/containers_prune.go | 16 ++++++++---- pkg/api/handlers/libpod/volumes.go | 4 +-- pkg/api/handlers/utils/containers.go | 21 ++-------------- pkg/api/handlers/utils/pods.go | 33 +++++++++++-------------- pkg/domain/entities/container_ps.go | 4 +-- pkg/domain/infra/abi/containers.go | 15 +++++++++--- pkg/ps/define/types.go | 8 ++++++ pkg/ps/ps.go | 38 +++++++++++++++++++++-------- pkg/util/utils.go | 33 ------------------------- 12 files changed, 127 insertions(+), 102 deletions(-) create mode 100644 pkg/ps/define/types.go diff --git a/cmd/podman/runlabel.go b/cmd/podman/runlabel.go index 1ec4da650..193cc5aec 100644 --- a/cmd/podman/runlabel.go +++ b/cmd/podman/runlabel.go @@ -13,11 +13,11 @@ import ( "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/libpod/define" "github.com/containers/libpod/libpod/image" - "github.com/containers/libpod/pkg/util" "github.com/containers/libpod/utils" "github.com/pkg/errors" "github.com/sirupsen/logrus" "github.com/spf13/cobra" + "github.com/spf13/pflag" ) var ( @@ -157,7 +157,7 @@ func runlabelCmd(c *cliconfig.RunlabelValues) error { return errors.Errorf("%s does not have a label of %s", runlabelImage, label) } - globalOpts := util.GetGlobalOpts(c) + globalOpts := GetGlobalOpts(c) cmd, env, err := shared.GenerateRunlabelCommand(runLabel, imageName, c.Name, opts, extraArgs, globalOpts) if err != nil { return err @@ -193,3 +193,32 @@ func runlabelCmd(c *cliconfig.RunlabelValues) error { return utils.ExecCmdWithStdStreams(stdIn, stdOut, stdErr, env, cmd[0], cmd[1:]...) } + +// GetGlobalOpts checks all global flags and generates the command string +func GetGlobalOpts(c *cliconfig.RunlabelValues) string { + globalFlags := map[string]bool{ + "cgroup-manager": true, "cni-config-dir": true, "conmon": true, "default-mounts-file": true, + "hooks-dir": true, "namespace": true, "root": true, "runroot": true, + "runtime": true, "storage-driver": true, "storage-opt": true, "syslog": true, + "trace": true, "network-cmd-path": true, "config": true, "cpu-profile": true, + "log-level": true, "tmpdir": true} + const stringSliceType string = "stringSlice" + + var optsCommand []string + c.PodmanCommand.Command.Flags().VisitAll(func(f *pflag.Flag) { + if !f.Changed { + return + } + if _, exist := globalFlags[f.Name]; exist { + if f.Value.Type() == stringSliceType { + flagValue := strings.TrimSuffix(strings.TrimPrefix(f.Value.String(), "["), "]") + for _, value := range strings.Split(flagValue, ",") { + optsCommand = append(optsCommand, fmt.Sprintf("--%s %s", f.Name, value)) + } + } else { + optsCommand = append(optsCommand, fmt.Sprintf("--%s %s", f.Name, f.Value.String())) + } + } + }) + return strings.Join(optsCommand, " ") +} diff --git a/cmd/podmanV2/common/create.go b/cmd/podmanV2/common/create.go index e2eb8cbda..ecaaf38fb 100644 --- a/cmd/podmanV2/common/create.go +++ b/cmd/podmanV2/common/create.go @@ -6,7 +6,6 @@ import ( buildahcli "github.com/containers/buildah/pkg/cli" "github.com/containers/common/pkg/config" - "github.com/containers/libpod/cmd/podman/cliconfig" "github.com/sirupsen/logrus" "github.com/spf13/pflag" ) @@ -214,22 +213,22 @@ func GetCreateFlags(cf *ContainerCLIOpts) *pflag.FlagSet { ) createFlags.StringVar( &cf.HealthInterval, - "health-interval", cliconfig.DefaultHealthCheckInterval, + "health-interval", DefaultHealthCheckInterval, "set an interval for the healthchecks (a value of disable results in no automatic timer setup)", ) createFlags.UintVar( &cf.HealthRetries, - "health-retries", cliconfig.DefaultHealthCheckRetries, + "health-retries", DefaultHealthCheckRetries, "the number of retries allowed before a healthcheck is considered to be unhealthy", ) createFlags.StringVar( &cf.HealthStartPeriod, - "health-start-period", cliconfig.DefaultHealthCheckStartPeriod, + "health-start-period", DefaultHealthCheckStartPeriod, "the initialization time needed for a container to bootstrap", ) createFlags.StringVar( &cf.HealthTimeout, - "health-timeout", cliconfig.DefaultHealthCheckTimeout, + "health-timeout", DefaultHealthCheckTimeout, "the maximum time allowed to complete the healthcheck before an interval is considered failed", ) createFlags.StringVarP( @@ -244,7 +243,7 @@ func GetCreateFlags(cf *ContainerCLIOpts) *pflag.FlagSet { ) createFlags.StringVar( &cf.ImageVolume, - "image-volume", cliconfig.DefaultImageVolume, + "image-volume", DefaultImageVolume, `Tells podman how to handle the builtin image volumes ("bind"|"tmpfs"|"ignore")`, ) createFlags.BoolVar( diff --git a/cmd/podmanV2/common/default.go b/cmd/podmanV2/common/default.go index b71fcb6f0..bd793f168 100644 --- a/cmd/podmanV2/common/default.go +++ b/cmd/podmanV2/common/default.go @@ -12,6 +12,19 @@ import ( "github.com/opencontainers/selinux/go-selinux" ) +var ( + // DefaultHealthCheckInterval default value + DefaultHealthCheckInterval = "30s" + // DefaultHealthCheckRetries default value + DefaultHealthCheckRetries uint = 3 + // DefaultHealthCheckStartPeriod default value + DefaultHealthCheckStartPeriod = "0s" + // DefaultHealthCheckTimeout default value + DefaultHealthCheckTimeout = "30s" + // DefaultImageVolume default value + DefaultImageVolume = "bind" +) + // TODO these options are directly embedded into many of the CLI cobra values, as such // this approach will not work in a remote client. so we will need to likely do something like a // supported and unsupported approach here and backload these options into the specgen diff --git a/pkg/api/handlers/compat/containers_prune.go b/pkg/api/handlers/compat/containers_prune.go index bf3aecd65..b4e98ac1f 100644 --- a/pkg/api/handlers/compat/containers_prune.go +++ b/pkg/api/handlers/compat/containers_prune.go @@ -4,6 +4,7 @@ import ( "net/http" "github.com/containers/libpod/libpod" + lpfilters "github.com/containers/libpod/libpod/filters" "github.com/containers/libpod/pkg/api/handlers/utils" "github.com/containers/libpod/pkg/domain/entities" "github.com/docker/docker/api/types" @@ -15,6 +16,7 @@ func PruneContainers(w http.ResponseWriter, r *http.Request) { var ( delContainers []string space int64 + filterFuncs []libpod.ContainerFilter ) runtime := r.Context().Value("runtime").(*libpod.Runtime) decoder := r.Context().Value("decoder").(*schema.Decoder) @@ -26,11 +28,15 @@ func PruneContainers(w http.ResponseWriter, r *http.Request) { utils.Error(w, "Something went wrong.", http.StatusBadRequest, errors.Wrapf(err, "Failed to parse parameters for %s", r.URL.String())) return } - - filterFuncs, err := utils.GenerateFilterFuncsFromMap(runtime, query.Filters) - if err != nil { - utils.InternalServerError(w, err) - return + for k, v := range query.Filters { + for _, val := range v { + generatedFunc, err := lpfilters.GenerateContainerFilterFuncs(k, val, runtime) + if err != nil { + utils.InternalServerError(w, err) + return + } + filterFuncs = append(filterFuncs, generatedFunc) + } } prunedContainers, pruneErrors, err := runtime.PruneContainers(filterFuncs) if err != nil { diff --git a/pkg/api/handlers/libpod/volumes.go b/pkg/api/handlers/libpod/volumes.go index 5a6fc021e..18c561a0d 100644 --- a/pkg/api/handlers/libpod/volumes.go +++ b/pkg/api/handlers/libpod/volumes.go @@ -4,12 +4,12 @@ import ( "encoding/json" "net/http" - "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod/define" "github.com/containers/libpod/pkg/api/handlers/utils" "github.com/containers/libpod/pkg/domain/entities" "github.com/containers/libpod/pkg/domain/filters" + "github.com/containers/libpod/pkg/domain/infra/abi/parse" "github.com/gorilla/schema" "github.com/pkg/errors" ) @@ -46,7 +46,7 @@ func CreateVolume(w http.ResponseWriter, r *http.Request) { volumeOptions = append(volumeOptions, libpod.WithVolumeLabels(input.Label)) } if len(input.Options) > 0 { - parsedOptions, err := shared.ParseVolumeOptions(input.Options) + parsedOptions, err := parse.ParseVolumeOptions(input.Options) if err != nil { utils.InternalServerError(w, err) return diff --git a/pkg/api/handlers/utils/containers.go b/pkg/api/handlers/utils/containers.go index bbe4cee3c..d1107f67c 100644 --- a/pkg/api/handlers/utils/containers.go +++ b/pkg/api/handlers/utils/containers.go @@ -6,9 +6,10 @@ import ( "time" "github.com/containers/libpod/cmd/podman/shared" + createconfig "github.com/containers/libpod/pkg/spec" + "github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod/define" - createconfig "github.com/containers/libpod/pkg/spec" "github.com/gorilla/schema" "github.com/pkg/errors" ) @@ -68,24 +69,6 @@ func WaitContainer(w http.ResponseWriter, r *http.Request) (int32, error) { return con.WaitForConditionWithInterval(interval, condition) } -// GenerateFilterFuncsFromMap is used to generate un-executed functions that can be used to filter -// containers. It is specifically designed for the RESTFUL API input. -func GenerateFilterFuncsFromMap(r *libpod.Runtime, filters map[string][]string) ([]libpod.ContainerFilter, error) { - var ( - filterFuncs []libpod.ContainerFilter - ) - for k, v := range filters { - for _, val := range v { - f, err := shared.GenerateContainerFilterFuncs(k, val, r) - if err != nil { - return filterFuncs, err - } - filterFuncs = append(filterFuncs, f) - } - } - return filterFuncs, nil -} - func CreateContainer(ctx context.Context, w http.ResponseWriter, runtime *libpod.Runtime, cc *createconfig.CreateConfig) { var pod *libpod.Pod ctr, err := shared.CreateContainerFromCreateConfig(runtime, cc, ctx, pod) diff --git a/pkg/api/handlers/utils/pods.go b/pkg/api/handlers/utils/pods.go index d47053eda..fb795fa6a 100644 --- a/pkg/api/handlers/utils/pods.go +++ b/pkg/api/handlers/utils/pods.go @@ -1,20 +1,19 @@ package utils import ( - "fmt" "net/http" - "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/libpod" + lpfilters "github.com/containers/libpod/libpod/filters" "github.com/containers/libpod/pkg/domain/entities" "github.com/gorilla/schema" ) func GetPods(w http.ResponseWriter, r *http.Request) ([]*entities.ListPodsReport, error) { var ( - lps []*entities.ListPodsReport - pods []*libpod.Pod - podErr error + lps []*entities.ListPodsReport + pods []*libpod.Pod + filters []libpod.PodFilter ) runtime := r.Context().Value("runtime").(*libpod.Runtime) decoder := r.Context().Value("decoder").(*schema.Decoder) @@ -28,28 +27,24 @@ func GetPods(w http.ResponseWriter, r *http.Request) ([]*entities.ListPodsReport if err := decoder.Decode(&query, r.URL.Query()); err != nil { return nil, err } - var filters = []string{} if _, found := r.URL.Query()["digests"]; found && query.Digests { UnSupportedParameter("digests") } - if len(query.Filters) > 0 { - for k, v := range query.Filters { - for _, val := range v { - filters = append(filters, fmt.Sprintf("%s=%s", k, val)) + for k, v := range query.Filters { + for _, filter := range v { + f, err := lpfilters.GeneratePodFilterFunc(k, filter) + if err != nil { + return nil, err } + filters = append(filters, f) } - filterFuncs, err := shared.GenerateFilterFunction(runtime, filters) - if err != nil { - return nil, err - } - pods, podErr = shared.FilterAllPodsWithFilterFunc(runtime, filterFuncs...) - } else { - pods, podErr = runtime.GetAllPods() } - if podErr != nil { - return nil, podErr + pods, err := runtime.Pods(filters...) + if err != nil { + return nil, err } + for _, pod := range pods { status, err := pod.GetPodStatus() if err != nil { diff --git a/pkg/domain/entities/container_ps.go b/pkg/domain/entities/container_ps.go index ceafecebc..33f5d0500 100644 --- a/pkg/domain/entities/container_ps.go +++ b/pkg/domain/entities/container_ps.go @@ -4,8 +4,8 @@ import ( "sort" "strings" - "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/libpod" + "github.com/containers/libpod/pkg/ps/define" "github.com/cri-o/ocicni/pkg/ocicni" "github.com/pkg/errors" ) @@ -48,7 +48,7 @@ type ListContainer struct { // Port mappings Ports []ocicni.PortMapping // Size of the container rootfs. Requires the size boolean to be true - Size *shared.ContainerSize + Size *define.ContainerSize // Time when container started StartedAt int64 // State of container diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index fc62a6c29..4279fb756 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -11,6 +11,8 @@ import ( "strings" "sync" + lpfilters "github.com/containers/libpod/libpod/filters" + "github.com/containers/buildah" "github.com/containers/common/pkg/config" "github.com/containers/image/v5/manifest" @@ -19,7 +21,6 @@ import ( "github.com/containers/libpod/libpod/events" "github.com/containers/libpod/libpod/image" "github.com/containers/libpod/libpod/logs" - "github.com/containers/libpod/pkg/api/handlers/utils" "github.com/containers/libpod/pkg/checkpoint" "github.com/containers/libpod/pkg/domain/entities" "github.com/containers/libpod/pkg/domain/infra/abi/terminal" @@ -175,9 +176,15 @@ func (ic *ContainerEngine) ContainerStop(ctx context.Context, namesOrIds []strin } func (ic *ContainerEngine) ContainerPrune(ctx context.Context, options entities.ContainerPruneOptions) (*entities.ContainerPruneReport, error) { - filterFuncs, err := utils.GenerateFilterFuncsFromMap(ic.Libpod, options.Filters) - if err != nil { - return nil, err + var filterFuncs []libpod.ContainerFilter + for k, v := range options.Filters { + for _, val := range v { + generatedFunc, err := lpfilters.GenerateContainerFilterFuncs(k, val, ic.Libpod) + if err != nil { + return nil, err + } + filterFuncs = append(filterFuncs, generatedFunc) + } } prunedContainers, pruneErrors, err := ic.Libpod.PruneContainers(filterFuncs) if err != nil { diff --git a/pkg/ps/define/types.go b/pkg/ps/define/types.go new file mode 100644 index 000000000..878653c3a --- /dev/null +++ b/pkg/ps/define/types.go @@ -0,0 +1,8 @@ +package define + +// ContainerSize holds the size of the container's root filesystem and top +// read-write layer. +type ContainerSize struct { + RootFsSize int64 `json:"rootFsSize"` + RwSize int64 `json:"rwSize"` +} diff --git a/pkg/ps/ps.go b/pkg/ps/ps.go index 58fcc2c21..8b62fc307 100644 --- a/pkg/ps/ps.go +++ b/pkg/ps/ps.go @@ -1,16 +1,19 @@ package ps import ( + "os" "path/filepath" + "regexp" "sort" "strconv" + "strings" "time" - "github.com/containers/libpod/cmd/podman/shared" "github.com/containers/libpod/libpod" "github.com/containers/libpod/libpod/define" lpfilters "github.com/containers/libpod/libpod/filters" "github.com/containers/libpod/pkg/domain/entities" + psdefine "github.com/containers/libpod/pkg/ps/define" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -80,7 +83,7 @@ func ListContainerBatch(rt *libpod.Runtime, ctr *libpod.Container, opts entities exitCode int32 exited bool pid int - size *shared.ContainerSize + size *psdefine.ContainerSize startedTime time.Time exitedTime time.Time cgroup, ipc, mnt, net, pidns, user, uts string @@ -116,16 +119,16 @@ func ListContainerBatch(rt *libpod.Runtime, ctr *libpod.Container, opts entities return errors.Wrapf(err, "unable to obtain container pid") } ctrPID := strconv.Itoa(pid) - cgroup, _ = shared.GetNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "cgroup")) - ipc, _ = shared.GetNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "ipc")) - mnt, _ = shared.GetNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "mnt")) - net, _ = shared.GetNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "net")) - pidns, _ = shared.GetNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "pid")) - user, _ = shared.GetNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "user")) - uts, _ = shared.GetNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "uts")) + cgroup, _ = getNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "cgroup")) + ipc, _ = getNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "ipc")) + mnt, _ = getNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "mnt")) + net, _ = getNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "net")) + pidns, _ = getNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "pid")) + user, _ = getNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "user")) + uts, _ = getNamespaceInfo(filepath.Join("/proc", ctrPID, "ns", "uts")) } if opts.Size { - size = new(shared.ContainerSize) + size = new(psdefine.ContainerSize) rootFsSize, err := c.RootFsSize() if err != nil { @@ -187,3 +190,18 @@ func ListContainerBatch(rt *libpod.Runtime, ctr *libpod.Container, opts entities } return ps, nil } + +func getNamespaceInfo(path string) (string, error) { + val, err := os.Readlink(path) + if err != nil { + return "", errors.Wrapf(err, "error getting info from %q", path) + } + return getStrFromSquareBrackets(val), nil +} + +// getStrFromSquareBrackets gets the string inside [] from a string. +func getStrFromSquareBrackets(cmd string) string { + reg := regexp.MustCompile(`.*\[|\].*`) + arr := strings.Split(reg.ReplaceAllLiteralString(cmd, ""), ",") + return strings.Join(arr, ",") +} diff --git a/pkg/util/utils.go b/pkg/util/utils.go index 1051ed311..3906ed19f 100644 --- a/pkg/util/utils.go +++ b/pkg/util/utils.go @@ -14,7 +14,6 @@ import ( "github.com/BurntSushi/toml" "github.com/containers/image/v5/types" - "github.com/containers/libpod/cmd/podman/cliconfig" "github.com/containers/libpod/pkg/errorhandling" "github.com/containers/libpod/pkg/namespaces" "github.com/containers/libpod/pkg/rootless" @@ -24,7 +23,6 @@ import ( v1 "github.com/opencontainers/image-spec/specs-go/v1" "github.com/pkg/errors" "github.com/sirupsen/logrus" - "github.com/spf13/pflag" "golang.org/x/crypto/ssh/terminal" ) @@ -515,37 +513,6 @@ func ParseInputTime(inputTime string) (time.Time, error) { return time.Now().Add(-duration), nil } -// GetGlobalOpts checks all global flags and generates the command string -// FIXME: Port input to config.Config -// TODO: Is there a "better" way to reverse values to flags? This seems brittle. -func GetGlobalOpts(c *cliconfig.RunlabelValues) string { - globalFlags := map[string]bool{ - "cgroup-manager": true, "cni-config-dir": true, "conmon": true, "default-mounts-file": true, - "hooks-dir": true, "namespace": true, "root": true, "runroot": true, - "runtime": true, "storage-driver": true, "storage-opt": true, "syslog": true, - "trace": true, "network-cmd-path": true, "config": true, "cpu-profile": true, - "log-level": true, "tmpdir": true} - const stringSliceType string = "stringSlice" - - var optsCommand []string - c.PodmanCommand.Command.Flags().VisitAll(func(f *pflag.Flag) { - if !f.Changed { - return - } - if _, exist := globalFlags[f.Name]; exist { - if f.Value.Type() == stringSliceType { - flagValue := strings.TrimSuffix(strings.TrimPrefix(f.Value.String(), "["), "]") - for _, value := range strings.Split(flagValue, ",") { - optsCommand = append(optsCommand, fmt.Sprintf("--%s %s", f.Name, value)) - } - } else { - optsCommand = append(optsCommand, fmt.Sprintf("--%s %s", f.Name, f.Value.String())) - } - } - }) - return strings.Join(optsCommand, " ") -} - // OpenExclusiveFile opens a file for writing and ensure it doesn't already exist func OpenExclusiveFile(path string) (*os.File, error) { baseDir := filepath.Dir(path) -- cgit v1.2.3-54-g00ecf