From 5923656f321a9ca7b222c81cdb5f3387cc7cd3ad Mon Sep 17 00:00:00 2001 From: Baron Lenardson Date: Mon, 21 Dec 2020 10:35:21 -0600 Subject: Add volume filters to system prune This change was missed in pull/8689. Now that volume pruneing supports filters system pruneing can pass its filters down to the volume pruneing. Additionally this change adds tests for the following components * podman system prune subcommand with `--volumes` & `--filter` options * apiv2 api tests for `/system/` and `/libpod/system` endpoints Relates to #8453, #8672 Signed-off-by: Baron Lenardson --- cmd/podman/system/prune.go | 18 ++++------- libpod/filters/helpers.go | 20 ++++++++++++ pkg/bindings/test/system_test.go | 57 +++++++++++++++++++++++++++++++++ pkg/domain/entities/system.go | 6 ++-- pkg/domain/infra/abi/system.go | 12 +++++-- pkg/domain/infra/tunnel/system.go | 2 +- test/apiv2/45-system.at | 67 +++++++++++++++++++++++++++++++++++++++ test/e2e/prune_test.go | 60 +++++++++++++++++++++++++++++++++++ 8 files changed, 225 insertions(+), 17 deletions(-) create mode 100644 libpod/filters/helpers.go create mode 100644 test/apiv2/45-system.at diff --git a/cmd/podman/system/prune.go b/cmd/podman/system/prune.go index ea47e271f..a74363684 100644 --- a/cmd/podman/system/prune.go +++ b/cmd/podman/system/prune.go @@ -4,7 +4,6 @@ import ( "bufio" "context" "fmt" - "net/url" "os" "strings" @@ -12,8 +11,8 @@ import ( "github.com/containers/podman/v2/cmd/podman/registry" "github.com/containers/podman/v2/cmd/podman/utils" "github.com/containers/podman/v2/cmd/podman/validate" + lpfilters "github.com/containers/podman/v2/libpod/filters" "github.com/containers/podman/v2/pkg/domain/entities" - "github.com/pkg/errors" "github.com/spf13/cobra" ) @@ -55,6 +54,8 @@ func init() { } func prune(cmd *cobra.Command, args []string) error { + var err error + // Prompt for confirmation if --force is not set if !force { reader := bufio.NewReader(os.Stdin) @@ -79,16 +80,11 @@ Are you sure you want to continue? [y/N] `, volumeString) } } - pruneOptions.ContainerPruneOptions = entities.ContainerPruneOptions{} - for _, f := range filters { - t := strings.SplitN(f, "=", 2) - pruneOptions.ContainerPruneOptions.Filters = make(url.Values) - if len(t) < 2 { - return errors.Errorf("filter input must be in the form of filter=value: %s is invalid", f) - } - pruneOptions.ContainerPruneOptions.Filters.Add(t[0], t[1]) + pruneOptions.Filters, err = lpfilters.ParseFilterArgumentsIntoFilters(filters) + if err != nil { + return err } - // TODO: support for filters in system prune + response, err := registry.ContainerEngine().SystemPrune(context.Background(), pruneOptions) if err != nil { return err diff --git a/libpod/filters/helpers.go b/libpod/filters/helpers.go new file mode 100644 index 000000000..859db3a9a --- /dev/null +++ b/libpod/filters/helpers.go @@ -0,0 +1,20 @@ +package lpfilters + +import ( + "net/url" + "strings" + + "github.com/pkg/errors" +) + +func ParseFilterArgumentsIntoFilters(filters []string) (url.Values, error) { + parsedFilters := make(url.Values) + for _, f := range filters { + t := strings.SplitN(f, "=", 2) + if len(t) < 2 { + return parsedFilters, errors.Errorf("filter input must be in the form of filter=value: %s is invalid", f) + } + parsedFilters.Add(t[0], t[1]) + } + return parsedFilters, nil +} diff --git a/pkg/bindings/test/system_test.go b/pkg/bindings/test/system_test.go index 5cb0f9125..25fda5575 100644 --- a/pkg/bindings/test/system_test.go +++ b/pkg/bindings/test/system_test.go @@ -158,4 +158,61 @@ var _ = Describe("Podman system", func() { // Volume should be pruned now as flag set true Expect(len(systemPruneResponse.VolumePruneReport)).To(Equal(1)) }) + + It("podman system prune running alpine container volume prune --filter", func() { + // Start a pod and leave it running + _, err := pods.Start(bt.conn, newpod, nil) + Expect(err).To(BeNil()) + + // Start and stop a container to enter in exited state. + var name = "top" + _, err = bt.RunTopContainer(&name, bindings.PFalse, nil) + Expect(err).To(BeNil()) + err = containers.Stop(bt.conn, name, nil) + Expect(err).To(BeNil()) + + // Start second container and leave in running + var name2 = "top2" + _, err = bt.RunTopContainer(&name2, bindings.PFalse, nil) + Expect(err).To(BeNil()) + + // Adding an unused volume should work + _, err = volumes.Create(bt.conn, entities.VolumeCreateOptions{}, nil) + Expect(err).To(BeNil()) + + // Adding an unused volume with label should work + _, err = volumes.Create(bt.conn, entities.VolumeCreateOptions{Label: map[string]string{ + "label1": "value1", + }}, nil) + Expect(err).To(BeNil()) + + f := make(map[string][]string) + f["label"] = []string{"label1=idontmatch"} + + options := new(system.PruneOptions).WithAll(true).WithVolumes(true).WithFilters(f) + systemPruneResponse, err := system.Prune(bt.conn, options) + Expect(err).To(BeNil()) + Expect(len(systemPruneResponse.PodPruneReport)).To(Equal(0)) + // TODO fix system filter handling so all components can handle filters + // This check **should** be "Equal(0)" since we are passing label + // filters however the Prune function doesn't seem to pass filters + // to each component. + Expect(len(systemPruneResponse.ContainerPruneReport.ID)).To(Equal(1)) + Expect(len(systemPruneResponse.ImagePruneReport.Report.Id)). + To(BeNumerically(">", 0)) + // Alpine image should not be pruned as used by running container + Expect(systemPruneResponse.ImagePruneReport.Report.Id). + ToNot(ContainElement("docker.io/library/alpine:latest")) + // Volume shouldn't be pruned because the PruneOptions filters doesn't match + Expect(len(systemPruneResponse.VolumePruneReport)).To(Equal(0)) + + // Fix filter and re prune + f["label"] = []string{"label1=value1"} + options = new(system.PruneOptions).WithAll(true).WithVolumes(true).WithFilters(f) + systemPruneResponse, err = system.Prune(bt.conn, options) + Expect(err).To(BeNil()) + + // Volume should be pruned because the PruneOptions filters now match + Expect(len(systemPruneResponse.VolumePruneReport)).To(Equal(1)) + }) }) diff --git a/pkg/domain/entities/system.go b/pkg/domain/entities/system.go index 5266dc4cf..d5118f6a8 100644 --- a/pkg/domain/entities/system.go +++ b/pkg/domain/entities/system.go @@ -17,9 +17,9 @@ type ServiceOptions struct { // SystemPruneOptions provides options to prune system. type SystemPruneOptions struct { - All bool - Volume bool - ContainerPruneOptions + All bool + Volume bool + Filters map[string][]string `json:"filters" schema:"filters"` } // SystemPruneReport provides report after system prune is executed. diff --git a/pkg/domain/infra/abi/system.go b/pkg/domain/infra/abi/system.go index 17faa7fff..5f6c95d4f 100644 --- a/pkg/domain/infra/abi/system.go +++ b/pkg/domain/infra/abi/system.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "io/ioutil" + "net/url" "os" "os/exec" "path/filepath" @@ -180,7 +181,12 @@ func (ic *ContainerEngine) SystemPrune(ctx context.Context, options entities.Sys found = true } systemPruneReport.PodPruneReport = append(systemPruneReport.PodPruneReport, podPruneReport...) - containerPruneReport, err := ic.ContainerPrune(ctx, options.ContainerPruneOptions) + + // TODO: Figure out cleaner way to handle all of the different PruneOptions + containerPruneOptions := entities.ContainerPruneOptions{} + containerPruneOptions.Filters = (url.Values)(options.Filters) + + containerPruneReport, err := ic.ContainerPrune(ctx, containerPruneOptions) if err != nil { return nil, err } @@ -217,7 +223,9 @@ func (ic *ContainerEngine) SystemPrune(ctx context.Context, options entities.Sys systemPruneReport.ImagePruneReport.Report.Id = append(systemPruneReport.ImagePruneReport.Report.Id, results...) } if options.Volume { - volumePruneReport, err := ic.pruneVolumesHelper(ctx, nil) + volumePruneOptions := entities.VolumePruneOptions{} + volumePruneOptions.Filters = (url.Values)(options.Filters) + volumePruneReport, err := ic.VolumePrune(ctx, volumePruneOptions) if err != nil { return nil, err } diff --git a/pkg/domain/infra/tunnel/system.go b/pkg/domain/infra/tunnel/system.go index 1e4b2e0b4..9013d44c2 100644 --- a/pkg/domain/infra/tunnel/system.go +++ b/pkg/domain/infra/tunnel/system.go @@ -20,7 +20,7 @@ func (ic *ContainerEngine) SetupRootless(_ context.Context, cmd *cobra.Command) // SystemPrune prunes unused data from the system. func (ic *ContainerEngine) SystemPrune(ctx context.Context, opts entities.SystemPruneOptions) (*entities.SystemPruneReport, error) { - options := new(system.PruneOptions).WithAll(opts.All).WithVolumes(opts.Volume).WithFilters(opts.ContainerPruneOptions.Filters) + options := new(system.PruneOptions).WithAll(opts.All).WithVolumes(opts.Volume).WithFilters(opts.Filters) return system.Prune(ic.ClientCxt, options) } diff --git a/test/apiv2/45-system.at b/test/apiv2/45-system.at new file mode 100644 index 000000000..7d14fd4b3 --- /dev/null +++ b/test/apiv2/45-system.at @@ -0,0 +1,67 @@ +# -*- sh -*- +# +# system related tests +# + +## ensure system is clean +t POST 'libpod/system/prune?volumes=true&all=true' params='' 200 + +## podman system df +t GET system/df 200 '{"LayersSize":0,"Images":[],"Containers":[],"Volumes":[],"BuildCache":[],"BuilderSize":0}' +t GET libpod/system/df 200 '{"Images":[],"Containers":[],"Volumes":[]}' + +# Create volume. We expect df to report this volume next invocation of system/df +t GET libpod/info 200 +volumepath=$(jq -r ".store.volumePath" <<<"$output") +t POST libpod/volumes/create name=foo1 201 \ + .Name=foo1 \ + .Driver=local \ + .Mountpoint=$volumepath/foo1/_data \ + .CreatedAt~[0-9]\\{4\\}-[0-9]\\{2\\}-[0-9]\\{2\\}.* \ + .Labels={} \ + .Options=null + +t GET system/df 200 '.Volumes[0].Name=foo1' + +t GET libpod/system/df 200 '.Volumes[0].VolumeName=foo1' + +# Create two more volumes to test pruneing +t POST libpod/volumes/create \ + '"Name":"foo2","Label":{"testlabel1":""},"Options":{"type":"tmpfs","o":"nodev,noexec"}}' 201 \ + .Name=foo2 \ + .Driver=local \ + .Mountpoint=$volumepath/foo2/_data \ + .CreatedAt~[0-9]\\{4\\}-[0-9]\\{2\\}-[0-9]\\{2\\}.* \ + .Labels.testlabel1="" \ + .Options.o=nodev,noexec + +t POST libpod/volumes/create \ + '"Name":"foo3","Label":{"testlabel1":"testonly"},"Options":{"type":"tmpfs","o":"nodev,noexec"}}' 201 \ + .Name=foo3 \ + .Driver=local \ + .Mountpoint=$volumepath/foo3/_data \ + .CreatedAt~[0-9]\\{4\\}-[0-9]\\{2\\}-[0-9]\\{2\\}.* \ + .Labels.testlabel1=testonly \ + .Options.o=nodev,noexec + +t GET system/df 200 '.Volumes | length=3' +t GET libpod/system/df 200 '.Volumes | length=3' + +# Prune volumes + +# -G --data-urlencode 'volumes=true&filters={"label":["testlabel1=idontmatch"]}' +t POST 'libpod/system/prune?volumes=true&filters=%7B%22label%22:%5B%22testlabel1=idontmatch%22%5D%7D' params='' 200 + +# nothing should have been pruned +t GET system/df 200 '.Volumes | length=3' +t GET libpod/system/df 200 '.Volumes | length=3' + +# -G --data-urlencode 'volumes=true&filters={"label":["testlabel1=testonly"]}' +# only foo3 should be pruned because of filter +t POST 'libpod/system/prune?volumes=true&filters=%7B%22label%22:%5B%22testlabel1=testonly%22%5D%7D' params='' 200 .VolumePruneReport[0].Id=foo3 +# only foo2 should be pruned because of filter +t POST 'libpod/system/prune?volumes=true&filters=%7B%22label%22:%5B%22testlabel1%22%5D%7D' params='' 200 .VolumePruneReport[0].Id=foo2 +# foo1, the last remaining volume should be pruned without any filters applied +t POST 'libpod/system/prune?volumes=true' params='' 200 .VolumePruneReport[0].Id=foo1 + +# TODO add other system prune tests for pods / images diff --git a/test/e2e/prune_test.go b/test/e2e/prune_test.go index c02ed5a50..3bc1012df 100644 --- a/test/e2e/prune_test.go +++ b/test/e2e/prune_test.go @@ -349,4 +349,64 @@ var _ = Describe("Podman prune", func() { // all images are unused, so they all should be deleted! Expect(len(images.OutputToStringArray())).To(Equal(len(CACHE_IMAGES))) }) + + It("podman system prune --volumes --filter", func() { + session := podmanTest.Podman([]string{"volume", "create", "--label", "label1=value1", "myvol1"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "create", "--label", "sharedlabel1=slv1", "myvol2"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "create", "--label", "sharedlabel1=slv2", "myvol3"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "create", "--label", "sharedlabel1", "myvol4"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"create", "-v", "myvol5:/myvol5", ALPINE, "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"create", "-v", "myvol6:/myvol6", ALPINE, "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(7)) + + session = podmanTest.Podman([]string{"system", "prune", "--force", "--volumes", "--filter", "label=label1=value1"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(6)) + + session = podmanTest.Podman([]string{"system", "prune", "--force", "--volumes", "--filter", "label=sharedlabel1=slv1"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(5)) + + session = podmanTest.Podman([]string{"system", "prune", "--force", "--volumes", "--filter", "label=sharedlabel1"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + session = podmanTest.Podman([]string{"volume", "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + Expect(len(session.OutputToStringArray())).To(Equal(3)) + + podmanTest.Cleanup() + }) }) -- cgit v1.2.3-54-g00ecf