From 581afbb86f441c94c6dd49fe93dff935de7af340 Mon Sep 17 00:00:00 2001 From: Daniel J Walsh Date: Tue, 25 Aug 2020 13:41:52 -0400 Subject: Show c/storage (Buildah/CRI-O) containers in ps The `podman ps --all` command will now show containers that are under the control of other c/storage container systems and the new `ps --storage` option will show only containers that are in c/storage but are not controlled by libpod. In the below examples, the '*working-container' entries were created by Buildah. ``` podman ps -a CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES 9257ef8c786c docker.io/library/busybox:latest ls /etc 8 hours ago Exited (0) 8 hours ago gifted_jang d302c81856da docker.io/library/busybox:latest buildah 30 hours ago storage busybox-working-container 7a5a7b099d33 localhost/tom:latest ls -alF 30 hours ago Exited (0) 30 hours ago hopeful_hellman 01d601fca090 localhost/tom:latest ls -alf 30 hours ago Exited (1) 30 hours ago determined_panini ee58f429ff26 localhost/tom:latest buildah 33 hours ago storage alpine-working-container podman ps --external CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES d302c81856da docker.io/library/busybox:latest buildah 30 hours ago external busybox-working-container ee58f429ff26 localhost/tom:latest buildah 33 hours ago external alpine-working-container ``` Signed-off-by: TomSweeneyRedHat Signed-off-by: Daniel J Walsh --- cmd/podman/containers/ps.go | 12 ++++- cmd/podman/utils/alias.go | 2 + completions/bash/podman | 1 + docs/source/markdown/podman-build.1.md | 4 +- docs/source/markdown/podman-ps.1.md | 19 +++++++- docs/source/markdown/podman-rm.1.md | 8 ++-- libpod/define/errors.go | 4 ++ libpod/image/image.go | 8 ++++ libpod/image/prune.go | 2 +- libpod/runtime_ctr.go | 33 +++++++++++++ pkg/domain/entities/containers.go | 1 + pkg/ps/ps.go | 85 ++++++++++++++++++++++++++++++++-- 12 files changed, 167 insertions(+), 12 deletions(-) diff --git a/cmd/podman/containers/ps.go b/cmd/podman/containers/ps.go index ebb6ed98f..2aa3b3a9b 100644 --- a/cmd/podman/containers/ps.go +++ b/cmd/podman/containers/ps.go @@ -13,6 +13,7 @@ import ( tm "github.com/buger/goterm" "github.com/containers/buildah/pkg/formats" "github.com/containers/podman/v2/cmd/podman/registry" + "github.com/containers/podman/v2/cmd/podman/utils" "github.com/containers/podman/v2/cmd/podman/validate" "github.com/containers/podman/v2/pkg/domain/entities" "github.com/cri-o/ocicni/pkg/ocicni" @@ -56,9 +57,9 @@ func init() { func listFlagSet(flags *pflag.FlagSet) { flags.BoolVarP(&listOpts.All, "all", "a", false, "Show all the containers, default is only running containers") flags.StringSliceVarP(&filters, "filter", "f", []string{}, "Filter output based on conditions given") + flags.BoolVar(&listOpts.Storage, "storage", false, "Show containers in storage not controlled by Podman") flags.StringVar(&listOpts.Format, "format", "", "Pretty-print containers to JSON or using a Go template") flags.IntVarP(&listOpts.Last, "last", "n", -1, "Print the n last created containers (all states)") - flags.BoolVar(&listOpts.Namespace, "namespace", false, "Display namespace information") flags.BoolVar(&listOpts.Namespace, "ns", false, "Display namespace information") flags.BoolVar(&noTrunc, "no-trunc", false, "Display the extended information") flags.BoolVarP(&listOpts.Pod, "pod", "p", false, "Print the ID and name of the pod the containers are associated with") @@ -69,6 +70,7 @@ func listFlagSet(flags *pflag.FlagSet) { sort := validate.Value(&listOpts.Sort, "command", "created", "id", "image", "names", "runningfor", "size", "status") flags.Var(sort, "sort", "Sort output by: "+sort.Choices()) + flags.SetNormalizeFunc(utils.AliasFlags) } func checkFlags(c *cobra.Command) error { // latest, and last are mutually exclusive. @@ -102,6 +104,14 @@ func checkFlags(c *cobra.Command) error { if listOpts.Watch > 0 && listOpts.Latest { return errors.New("the watch and latest flags cannot be used together") } + cfg := registry.PodmanConfig() + if cfg.Engine.Namespace != "" { + if c.Flag("storage").Changed && listOpts.Storage { + return errors.New("--namespace and --storage flags can not both be set") + } + listOpts.Storage = false + } + return nil } diff --git a/cmd/podman/utils/alias.go b/cmd/podman/utils/alias.go index e484461c5..ff31e82ea 100644 --- a/cmd/podman/utils/alias.go +++ b/cmd/podman/utils/alias.go @@ -19,6 +19,8 @@ func AliasFlags(f *pflag.FlagSet, name string) pflag.NormalizedName { name = "network" case "timeout": name = "time" + case "namespace": + name = "ns" } return pflag.NormalizedName(name) } diff --git a/completions/bash/podman b/completions/bash/podman index 3b50af1a9..e250f344b 100644 --- a/completions/bash/podman +++ b/completions/bash/podman @@ -2679,6 +2679,7 @@ _podman_ps() { --pod -p --quiet -q --size -s + --storage --namespace --ns --sync " diff --git a/docs/source/markdown/podman-build.1.md b/docs/source/markdown/podman-build.1.md index 6618df1b9..c38424a11 100644 --- a/docs/source/markdown/podman-build.1.md +++ b/docs/source/markdown/podman-build.1.md @@ -23,6 +23,8 @@ When the URL is an Containerfile, the Containerfile is downloaded to a temporary When a Git repository is set as the URL, the repository is cloned locally and then set as the context. +NOTE: `podman build` uses code sourced from the `buildah` project to build container images. This `buildah` code creates `buildah` containers for the `RUN` options in container storage. In certain situations, when the `podman build` crashes or users kill the `podman build` process, these external containers can be left in container storage. Use the `podman ps --all --storage` command to see these contaienrs. External containers can be removed with the `podman rm --storage` command. + ## OPTIONS **--add-host**=*host* @@ -804,7 +806,7 @@ If you are using a useradd command within a Containerfile with a large UID/GID, If you are using `useradd` within your build script, you should pass the `--no-log-init or -l` option to the `useradd` command. This option tells useradd to stop creating the lastlog file. ## SEE ALSO -podman(1), buildah(1), containers-registries.conf(5), crun(8), runc(8), useradd(8) +podman(1), buildah(1), containers-registries.conf(5), crun(8), runc(8), useradd(8), podman-ps(1), podman-rm(1) ## HISTORY Aug 2020, Additional options and .dockerignore added by Dan Walsh diff --git a/docs/source/markdown/podman-ps.1.md b/docs/source/markdown/podman-ps.1.md index 2f8112aab..58d3358e5 100644 --- a/docs/source/markdown/podman-ps.1.md +++ b/docs/source/markdown/podman-ps.1.md @@ -32,12 +32,18 @@ all the containers information. By default it lists: **--all**, **-a** -Show all the containers, default is only running containers +Show all the containers created by Podman, default is only running containers. + +Note: Podman shares containers storage with other tools such as Buildah and CRI-O. In some cases these `external` containers might also exist in the same storage. Use the `--storage` option to see these external containers. External containers show the 'storage' status. **--pod**, **-p** Display the pods the containers are associated with +**--storage** + +Display external containers that are not controlled by Podman but are stored in containers storage. These external containers are generally created via other container technology such as Buildah or CRI-O and may depend on the same container images that Podman is also using. External containers are denoted with either a 'buildah' or 'storage' in the COMMAND and STATUS column of the ps output. Only used with the --all option. + **--no-trunc** Display the extended information @@ -174,11 +180,20 @@ CONTAINER ID IMAGE COMMAND CREATED STATUS ``` +``` +$ podman ps --storage -a +CONTAINER ID IMAGE COMMAND CREATED STATUS PORTS NAMES +69ed779d8ef9f redis:alpine "redis-server" 25 hours ago Created 6379/tcp k8s_container1_podsandbox1_redhat.test.crio_redhat-test-crio_1 +38a8a78596f9 docker.io/library/busybox:latest buildah 2 hours ago storage busybox-working-container +fd7b786b5c32 docker.io/library/alpine:latest buildah 2 hours ago storage alpine-working-container +f78620804e00 scratch buildah 2 hours ago storage working-container +``` + ## ps Print a list of containers ## SEE ALSO -podman(1) +podman(1), buildah(1), crio(8) ## HISTORY August 2017, Originally compiled by Urvashi Mohnani diff --git a/docs/source/markdown/podman-rm.1.md b/docs/source/markdown/podman-rm.1.md index cddf06e3e..990af0cd1 100644 --- a/docs/source/markdown/podman-rm.1.md +++ b/docs/source/markdown/podman-rm.1.md @@ -45,9 +45,9 @@ The latest option is not supported on the remote client. **--storage** -Remove the container from the storage library only. -This is only possible with containers that are not present in libpod (cannot be seen by **podman ps**). -It is used to remove containers from **podman build** and **buildah**, and orphan containers which were only partially removed by **podman rm**. +Remove external containers from the storage library. +This is only possible with containers that are not present in libpod can be seen by **podman ps --all --storage**). +It is used to remove external containers from **podman build** and **buildah**, and orphan containers which were only partially removed by **podman rm**. The storage option conflicts with the **--all**, **--latest**, and **--volumes** options. **--volumes**, **-v** @@ -96,7 +96,7 @@ $ podman rm -f --latest **125** The command fails for a reason other than container did not exist or is paused/running ## SEE ALSO -podman(1), podman-image-rm(1) +podman(1), podman-image-rm(1), podman-ps(1), podman-build(1) ## HISTORY August 2017, Originally compiled by Ryan Cole diff --git a/libpod/define/errors.go b/libpod/define/errors.go index f80b1d6e3..7714ebbf0 100644 --- a/libpod/define/errors.go +++ b/libpod/define/errors.go @@ -161,4 +161,8 @@ var ( // ErrNetworkOnPodContainer indicates the user wishes to alter network attributes on a container // in a pod. This cannot be done as the infra container has all the network information ErrNetworkOnPodContainer = errors.New("network cannot be configured when it is shared with a pod") + + // ErrStoreNotInitialized indicates that the container storage was never + // initilized. + ErrStoreNotInitialized = errors.New("the container storage was never initilized") ) diff --git a/libpod/image/image.go b/libpod/image/image.go index 2d055cc44..3a8152ee2 100644 --- a/libpod/image/image.go +++ b/libpod/image/image.go @@ -466,6 +466,14 @@ func (ir *Runtime) getImage(image string) (*storage.Image, error) { return img, nil } +func (ir *Runtime) ImageNames(id string) ([]string, error) { + myImage, err := ir.getImage(id) + if err != nil { + return nil, errors.Wrapf(err, "error getting image %s ", id) + } + return myImage.Names, nil +} + // GetImages retrieves all images present in storage func (ir *Runtime) GetImages() ([]*Image, error) { return ir.getImages(false) diff --git a/libpod/image/prune.go b/libpod/image/prune.go index 5a9ca5d8e..fcc65fb03 100644 --- a/libpod/image/prune.go +++ b/libpod/image/prune.go @@ -137,7 +137,7 @@ func (ir *Runtime) PruneImages(ctx context.Context, all bool, filter []string) ( } if err := p.Remove(ctx, true); err != nil { if errors.Cause(err) == storage.ErrImageUsedByContainer { - logrus.Warnf("Failed to prune image %s as it is in use: %v", p.ID(), err) + logrus.Warnf("Failed to prune image %s as it is in use: %v.\nA container associated with containers/storage i.e. Buildah, CRI-O, etc., maybe associated with this image.\nUsing the rmi command with the --force option will remove the container and image, but may cause failures for other dependent systems.", p.ID(), err) continue } return nil, errors.Wrap(err, "failed to prune image") diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go index fa91fe002..936dce2e9 100644 --- a/libpod/runtime_ctr.go +++ b/libpod/runtime_ctr.go @@ -8,11 +8,13 @@ import ( "strings" "time" + "github.com/containers/buildah" "github.com/containers/common/pkg/config" "github.com/containers/podman/v2/libpod/define" "github.com/containers/podman/v2/libpod/events" "github.com/containers/podman/v2/pkg/cgroups" "github.com/containers/podman/v2/pkg/rootless" + "github.com/containers/storage" "github.com/containers/storage/pkg/stringid" "github.com/docker/go-units" spec "github.com/opencontainers/runtime-spec/specs-go" @@ -905,3 +907,34 @@ func (r *Runtime) PruneContainers(filterFuncs []ContainerFilter) (map[string]int } return prunedContainers, pruneErrors, nil } + +// StorageContainers returns a list of containers from containers/storage that +// are not currently known to Podman. +func (r *Runtime) StorageContainers() ([]storage.Container, error) { + + if r.store == nil { + return nil, define.ErrStoreNotInitialized + } + + storeContainers, err := r.store.Containers() + if err != nil { + return nil, errors.Wrapf(err, "error reading list of all storage containers") + } + retCtrs := []storage.Container{} + for _, container := range storeContainers { + exists, err := r.state.HasContainer(container.ID) + if err != nil && err != define.ErrNoSuchCtr { + return nil, errors.Wrapf(err, "failed to check if %s container exists in database", container.ID) + } + if exists { + continue + } + retCtrs = append(retCtrs, container) + } + + return retCtrs, nil +} + +func (r *Runtime) IsBuildahContainer(id string) (bool, error) { + return buildah.IsContainer(id, r.store) +} diff --git a/pkg/domain/entities/containers.go b/pkg/domain/entities/containers.go index c8894300b..16997cdd1 100644 --- a/pkg/domain/entities/containers.go +++ b/pkg/domain/entities/containers.go @@ -282,6 +282,7 @@ type ContainerListOptions struct { Quiet bool Size bool Sort string + Storage bool Sync bool Watch uint } diff --git a/pkg/ps/ps.go b/pkg/ps/ps.go index 4c5f60844..8087507e2 100644 --- a/pkg/ps/ps.go +++ b/pkg/ps/ps.go @@ -14,6 +14,7 @@ import ( lpfilters "github.com/containers/podman/v2/libpod/filters" "github.com/containers/podman/v2/pkg/domain/entities" psdefine "github.com/containers/podman/v2/pkg/ps/define" + "github.com/containers/storage" "github.com/pkg/errors" "github.com/sirupsen/logrus" ) @@ -54,12 +55,12 @@ func GetContainerLists(runtime *libpod.Runtime, options entities.ContainerListOp return nil, err } if options.Last > 0 { - // Sort the containers we got + // Sort the libpod containers sort.Sort(SortCreateTime{SortContainers: cons}) // we should perform the lopping before we start getting // the expensive information on containers if options.Last < len(cons) { - cons = cons[len(cons)-options.Last:] + cons = cons[:options.Last] } } for _, con := range cons { @@ -68,7 +69,31 @@ func GetContainerLists(runtime *libpod.Runtime, options entities.ContainerListOp return nil, err } pss = append(pss, listCon) + } + + if options.All && options.Storage { + externCons, err := runtime.StorageContainers() + if err != nil { + return nil, err + } + + for _, con := range externCons { + listCon, err := ListStorageContainer(runtime, con, options) + if err != nil { + return nil, err + } + pss = append(pss, listCon) + } + } + + // Sort the containers we got + sort.Sort(SortPSCreateTime{SortPSContainers: pss}) + if options.Last > 0 { + // only return the "last" containers caller requested + if options.Last < len(pss) { + pss = pss[:options.Last] + } } return pss, nil } @@ -199,6 +224,48 @@ func ListContainerBatch(rt *libpod.Runtime, ctr *libpod.Container, opts entities return ps, nil } +func ListStorageContainer(rt *libpod.Runtime, ctr storage.Container, opts entities.ContainerListOptions) (entities.ListContainer, error) { + name := "unknown" + if len(ctr.Names) > 0 { + name = ctr.Names[0] + } + + ps := entities.ListContainer{ + ID: ctr.ID, + Created: ctr.Created.Unix(), + ImageID: ctr.ImageID, + State: "storage", + Names: []string{name}, + } + + buildahCtr, err := rt.IsBuildahContainer(ctr.ID) + if err != nil { + return ps, errors.Wrapf(err, "error determining buildah container for container %s", ctr.ID) + } + + if buildahCtr { + ps.Command = []string{"buildah"} + } else { + ps.Command = []string{"storage"} + } + + imageName := "" + if ctr.ImageID != "" { + names, err := rt.ImageRuntime().ImageNames(ctr.ImageID) + if err != nil { + return ps, err + } + if len(names) > 0 { + imageName = names[0] + } + } else if buildahCtr { + imageName = "scratch" + } + + ps.Image = imageName + return ps, nil +} + func getNamespaceInfo(path string) (string, error) { val, err := os.Readlink(path) if err != nil { @@ -223,5 +290,17 @@ func (a SortContainers) Swap(i, j int) { a[i], a[j] = a[j], a[i] } type SortCreateTime struct{ SortContainers } func (a SortCreateTime) Less(i, j int) bool { - return a.SortContainers[i].CreatedTime().Before(a.SortContainers[j].CreatedTime()) + return a.SortContainers[i].CreatedTime().After(a.SortContainers[j].CreatedTime()) +} + +// SortPSContainers helps us set-up ability to sort by createTime +type SortPSContainers []entities.ListContainer + +func (a SortPSContainers) Len() int { return len(a) } +func (a SortPSContainers) Swap(i, j int) { a[i], a[j] = a[j], a[i] } + +type SortPSCreateTime struct{ SortPSContainers } + +func (a SortPSCreateTime) Less(i, j int) bool { + return a.SortPSContainers[i].Created > a.SortPSContainers[j].Created } -- cgit v1.2.3-54-g00ecf