diff options
26 files changed, 784 insertions, 23 deletions
diff --git a/cmd/podman/container.go b/cmd/podman/container.go index b6262f890..b0232c874 100644 --- a/cmd/podman/container.go +++ b/cmd/podman/container.go @@ -22,7 +22,7 @@ var ( mountCommand, pauseCommand, portCommand, - // pruneCommand, + pruneContainersCommand, refreshCommand, restartCommand, restoreCommand, diff --git a/cmd/podman/containers_prune.go b/cmd/podman/containers_prune.go new file mode 100644 index 000000000..92604e82f --- /dev/null +++ b/cmd/podman/containers_prune.go @@ -0,0 +1,74 @@ +package main + +import ( + "github.com/containers/libpod/cmd/podman/libpodruntime" + "github.com/containers/libpod/cmd/podman/shared" + "github.com/containers/libpod/libpod" + "github.com/pkg/errors" + "github.com/sirupsen/logrus" + "github.com/urfave/cli" +) + +var ( + pruneContainersDescription = ` + podman container prune + + Removes all exited containers +` + + pruneContainersCommand = cli.Command{ + Name: "prune", + Usage: "Remove all stopped containers", + Description: pruneContainersDescription, + Action: pruneContainersCmd, + OnUsageError: usageErrorHandler, + } +) + +func pruneContainersCmd(c *cli.Context) error { + var ( + deleteFuncs []shared.ParallelWorkerInput + ) + + ctx := getContext() + runtime, err := libpodruntime.GetRuntime(c) + if err != nil { + return errors.Wrapf(err, "could not get runtime") + } + defer runtime.Shutdown(false) + + filter := func(c *libpod.Container) bool { + state, _ := c.State() + if state == libpod.ContainerStateStopped || (state == libpod.ContainerStateExited && err == nil && c.PodID() == "") { + return true + } + return false + } + delContainers, err := runtime.GetContainers(filter) + if err != nil { + return err + } + if len(delContainers) < 1 { + return nil + } + for _, container := range delContainers { + con := container + f := func() error { + return runtime.RemoveContainer(ctx, con, c.Bool("force")) + } + + deleteFuncs = append(deleteFuncs, shared.ParallelWorkerInput{ + ContainerID: con.ID(), + ParallelFunc: f, + }) + } + maxWorkers := shared.Parallelize("rm") + if c.GlobalIsSet("max-workers") { + maxWorkers = c.GlobalInt("max-workers") + } + logrus.Debugf("Setting maximum workers to %d", maxWorkers) + + // Run the parallel funcs + deleteErrors, errCount := shared.ParallelExecuteWorkerPool(maxWorkers, deleteFuncs) + return printParallelOutput(deleteErrors, errCount) +} diff --git a/cmd/podman/image.go b/cmd/podman/image.go index 418b442e3..95af36df5 100644 --- a/cmd/podman/image.go +++ b/cmd/podman/image.go @@ -13,7 +13,7 @@ var ( inspectCommand, loadCommand, lsImagesCommand, - // pruneCommand, + pruneImagesCommand, pullCommand, pushCommand, rmImageCommand, diff --git a/cmd/podman/images_prune.go b/cmd/podman/images_prune.go new file mode 100644 index 000000000..cb72a498f --- /dev/null +++ b/cmd/podman/images_prune.go @@ -0,0 +1,34 @@ +package main + +import ( + "github.com/containers/libpod/cmd/podman/libpodruntime" + "github.com/containers/libpod/cmd/podman/shared" + "github.com/pkg/errors" + "github.com/urfave/cli" +) + +var ( + pruneImagesDescription = ` + podman image prune + + Removes all unnamed images from local storage +` + + pruneImagesCommand = cli.Command{ + Name: "prune", + Usage: "Remove unused images", + Description: pruneImagesDescription, + Action: pruneImagesCmd, + OnUsageError: usageErrorHandler, + } +) + +func pruneImagesCmd(c *cli.Context) error { + runtime, err := libpodruntime.GetRuntime(c) + if err != nil { + return errors.Wrapf(err, "could not get runtime") + } + defer runtime.Shutdown(false) + + return shared.Prune(runtime.ImageRuntime()) +} diff --git a/cmd/podman/ps.go b/cmd/podman/ps.go index 0b03388a2..7a4a80769 100644 --- a/cmd/podman/ps.go +++ b/cmd/podman/ps.go @@ -200,6 +200,10 @@ var ( Usage: "Sort output by command, created, id, image, names, runningfor, size, or status", Value: "created", }, + cli.BoolFlag{ + Name: "sync", + Usage: "Sync container state with OCI runtime", + }, } psDescription = "Prints out information about the containers" psCommand = cli.Command{ @@ -260,6 +264,7 @@ func psCmd(c *cli.Context) error { Size: c.Bool("size"), Namespace: c.Bool("namespace"), Sort: c.String("sort"), + Sync: c.Bool("sync"), } filters := c.StringSlice("filter") diff --git a/cmd/podman/shared/container.go b/cmd/podman/shared/container.go index d0e892961..90ce193f7 100644 --- a/cmd/podman/shared/container.go +++ b/cmd/podman/shared/container.go @@ -45,6 +45,7 @@ type PsOptions struct { Sort string Label string Namespace bool + Sync bool } // BatchContainerStruct is the return obkect from BatchContainer and contains @@ -126,6 +127,12 @@ func NewBatchContainer(ctr *libpod.Container, opts PsOptions) (PsContainerOutput pso PsContainerOutput ) batchErr := ctr.Batch(func(c *libpod.Container) error { + if opts.Sync { + if err := c.Sync(); err != nil { + return err + } + } + conState, err = c.State() if err != nil { return errors.Wrapf(err, "unable to obtain container state") diff --git a/cmd/podman/shared/prune.go b/cmd/podman/shared/prune.go new file mode 100644 index 000000000..90cfe4475 --- /dev/null +++ b/cmd/podman/shared/prune.go @@ -0,0 +1,24 @@ +package shared + +import ( + "fmt" + "github.com/pkg/errors" + + "github.com/containers/libpod/libpod/image" +) + +// Prune removes all unnamed and unused images from the local store +func Prune(ir *image.Runtime) error { + pruneImages, err := ir.GetPruneImages() + if err != nil { + return err + } + + for _, i := range pruneImages { + if err := i.Remove(true); err != nil { + return errors.Wrapf(err, "failed to remove %s", i.ID()) + } + fmt.Println(i.ID()) + } + return nil +} diff --git a/completions/bash/podman b/completions/bash/podman index 7cec8bf80..3382b6c5a 100644 --- a/completions/bash/podman +++ b/completions/bash/podman @@ -888,6 +888,7 @@ _podman_container() { create diff exec + exists export inspect kill @@ -896,6 +897,7 @@ _podman_container() { mount pause port + prune refresh restart restore @@ -1227,11 +1229,13 @@ _podman_image() { " subcommands=" build + exists history import inspect load ls + prune pull push rm @@ -2054,6 +2058,7 @@ _podman_ps() { --quiet -q --size -s --namespace --ns + --sync " _complete_ "$options_with_args" "$boolean_options" } @@ -2244,6 +2249,26 @@ _podman_container_runlabel() { esac } +_podman_images_prune() { + local options_with_args=" + " + + local boolean_options=" + -h + --help + " +} + +_podman_container_prune() { + local options_with_args=" + " + + local boolean_options=" + -h + --help + " +} + _podman_container_exists() { local options_with_args=" " diff --git a/docs/podman-container-prune.1.md b/docs/podman-container-prune.1.md new file mode 100644 index 000000000..1f3ef1d41 --- /dev/null +++ b/docs/podman-container-prune.1.md @@ -0,0 +1,31 @@ +% PODMAN(1) Podman Man Pages +% Brent Baude +% December 2018 +# NAME +podman-container-prune - Remove all stopped containers + +# SYNOPSIS +**podman container prune** +[**-h**|**--help**] + +# DESCRIPTION +**podman container prune** removes all stopped containers from local storage. + +## Examples ## + +Remove all stopped containers from local storage +``` +$ sudo podman container prune +878392adf2e6c5c9bb1fc19b69d37d2e98c8abf9d539c0bce4b15b46bbcce471 +37664467fbe3618bf9479c34393ac29c02696675addf1750f9e346581636cde7 +ed0c6468b8e1cb641b4621d1fe30cb477e1fefc5c0bceb66feaf2f7cb50e5962 +6ac6c8f0067b7a4682e6b8e18902665b57d1a0e07e885d9abcd382232a543ccd +fff1c5b6c3631746055ec40598ce8ecaa4b82aef122f9e3a85b03b55c0d06c23 +602d343cd47e7cb3dfc808282a9900a3e4555747787ec6723bb68cedab8384d5 +``` + +## SEE ALSO +podman(1), podman-ps + +# HISTORY +December 2018, Originally compiled by Brent Baude (bbaude at redhat dot com) diff --git a/docs/podman-container.1.md b/docs/podman-container.1.md index aa5dfa82c..3675d9719 100644 --- a/docs/podman-container.1.md +++ b/docs/podman-container.1.md @@ -29,6 +29,7 @@ The container command allows you to manage containers | mount | [podman-mount(1)](podman-mount.1.md) | Mount a working container's root filesystem. | | pause | [podman-pause(1)](podman-pause.1.md) | Pause one or more containers. | | port | [podman-port(1)](podman-port.1.md) | List port mappings for the container. | +| prune | [podman-container-prune(1)](podman-container-prune.1.md) | Remove all stopped containers from local storage | | refresh | [podman-refresh(1)](podman-container-refresh.1.md) | Refresh the state of all containers | | restart | [podman-restart(1)](podman-restart.1.md) | Restart one or more containers. | | restore | [podman-container-restore(1)](podman-container-restore.1.md) | Restores one or more containers from a checkpoint. | diff --git a/docs/podman-image-prune.1.md b/docs/podman-image-prune.1.md new file mode 100644 index 000000000..db76b26e0 --- /dev/null +++ b/docs/podman-image-prune.1.md @@ -0,0 +1,32 @@ +% PODMAN(1) Podman Man Pages +% Brent Baude +% December 2018 +# NAME +podman-image-prune - Remove all unused images + +# SYNOPSIS +**podman image prune** +[**-h**|**--help**] + +# DESCRIPTION +**podman image prune** removes all unused images from local storage. An unused image +is defined as an image that does not have any containers based on it. + +## Examples ## + +Remove all unused images from local storage +``` +$ sudo podman image prune +f3e20dc537fb04cb51672a5cb6fdf2292e61d411315549391a0d1f64e4e3097e +324a7a3b2e0135f4226ffdd473e4099fd9e477a74230cdc35de69e84c0f9d907 +6125002719feb1ddf3030acab1df6156da7ce0e78e571e9b6e9c250424d6220c +91e732da5657264c6f4641b8d0c4001c218ae6c1adb9dcef33ad00cafd37d8b6 +e4e5109420323221f170627c138817770fb64832da7d8fe2babd863148287fca +77a57fa8285e9656dbb7b23d9efa837a106957409ddd702f995605af27a45ebe +``` + +## SEE ALSO +podman(1), podman-images + +# HISTORY +December 2018, Originally compiled by Brent Baude (bbaude at redhat dot com) diff --git a/docs/podman-image.1.md b/docs/podman-image.1.md index 446f8667d..8b812af11 100644 --- a/docs/podman-image.1.md +++ b/docs/podman-image.1.md @@ -21,6 +21,7 @@ The image command allows you to manage images | load | [podman-load(1)](podman-load.1.md) | Load an image from the docker archive. | | ls | [podman-images(1)](podman-images.1.md) | Prints out information about images. | | pull | [podman-pull(1)](podman-pull.1.md) | Pull an image from a registry. | +| prune| [podman-container-prune(1)](podman-container-prune.1.md) | Removed all unused images from the local store | | push | [podman-push(1)](podman-push.1.md) | Push an image from local storage to elsewhere. | | rm | [podman-rm(1)](podman-rmi.1.md) | Removes one or more locally stored images. | | save | [podman-save(1)](podman-save.1.md) | Save an image to docker-archive or oci. | diff --git a/docs/podman-ps.1.md b/docs/podman-ps.1.md index 7333a1095..8b86703d8 100644 --- a/docs/podman-ps.1.md +++ b/docs/podman-ps.1.md @@ -103,6 +103,13 @@ Valid filters are listed below: Print usage statement +**--sync** + +Force a sync of container state with the OCI runtime. +In some cases, a container's state in the runtime can become out of sync with Podman's state. +This will update Podman's state based on what the OCI runtime reports. +Forcibly syncing is much slower, but can resolve inconsistent state issues. + ## EXAMPLES ``` diff --git a/docs/podman-rmi.1.md b/docs/podman-rmi.1.md index f035897ee..9c080c9f1 100644 --- a/docs/podman-rmi.1.md +++ b/docs/podman-rmi.1.md @@ -19,15 +19,25 @@ Remove all images in the local storage. This option will cause podman to remove all containers that are using the image before removing the image from the system. -## EXAMPLE - -podman rmi imageID +Remove an image by its short ID +``` +podman rmi c0ed59d05ff7 +``` +Remove an image and its associated containers. +``` podman rmi --force imageID +```` -podman rmi imageID1 imageID2 imageID3 +Remove multiple images by their shortened IDs. +``` +podman rmi c4dfb1609ee2 93fd78260bd1 c0ed59d05ff7 +``` +Remove all images and containers. +``` podman rmi -a -f +``` ## SEE ALSO podman(1) diff --git a/docs/tutorials/podman_tutorial.md b/docs/tutorials/podman_tutorial.md index ce94d7d15..659973b28 100644 --- a/docs/tutorials/podman_tutorial.md +++ b/docs/tutorials/podman_tutorial.md @@ -24,7 +24,7 @@ acquire the source, and build it. sudo dnf install -y git runc libassuan-devel golang golang-github-cpuguy83-go-md2man glibc-static \ gpgme-devel glib2-devel device-mapper-devel libseccomp-devel \ atomic-registries iptables skopeo-containers containernetworking-cni \ - conmon + conmon ostree-devel ``` ### Building and installing podman @@ -54,7 +54,7 @@ tutorial. For this tutorial, the Ubuntu **artful-server-cloudimg** image was use #### Installing base packages ```console sudo apt-get update -sudo apt-get install libdevmapper-dev libglib2.0-dev libgpgme11-dev golang libseccomp-dev \ +sudo apt-get install libdevmapper-dev libglib2.0-dev libgpgme11-dev golang libseccomp-dev libostree-dev \ go-md2man libprotobuf-dev libprotobuf-c0-dev libseccomp-dev python3-setuptools ``` #### Building and installing conmon diff --git a/libpod/container_api.go b/libpod/container_api.go index bc92cae69..09bc46905 100644 --- a/libpod/container_api.go +++ b/libpod/container_api.go @@ -675,22 +675,27 @@ func (c *Container) Batch(batchFunc func(*Container) error) error { return err } -// Sync updates the current state of the container, checking whether its state -// has changed -// Sync can only be used inside Batch() - otherwise, it will be done -// automatically. -// When called outside Batch(), Sync() is a no-op +// Sync updates the status of a container by querying the OCI runtime. +// If the container has not been created inside the OCI runtime, nothing will be +// done. +// Most of the time, Podman does not explicitly query the OCI runtime for +// container status, and instead relies upon exit files created by conmon. +// This can cause a disconnect between running state and what Podman sees in +// cases where Conmon was killed unexpected, or runc was upgraded. +// Running a manual Sync() ensures that container state will be correct in +// such situations. func (c *Container) Sync() error { if !c.batched { - return nil + c.lock.Lock() + defer c.lock.Unlock() } // If runtime knows about the container, update its status in runtime // And then save back to disk if (c.state.State != ContainerStateUnknown) && - (c.state.State != ContainerStateConfigured) { + (c.state.State != ContainerStateConfigured) && + (c.state.State != ContainerStateExited) { oldState := c.state.State - // TODO: optionally replace this with a stat for the exit file if err := c.runtime.ociRuntime.updateContainerStatus(c, true); err != nil { return err } diff --git a/libpod/image/prune.go b/libpod/image/prune.go new file mode 100644 index 000000000..6a1f160d5 --- /dev/null +++ b/libpod/image/prune.go @@ -0,0 +1,26 @@ +package image + +// GetPruneImages returns a slice of images that have no names/unused +func (ir *Runtime) GetPruneImages() ([]*Image, error) { + var ( + unamedImages []*Image + ) + allImages, err := ir.GetImages() + if err != nil { + return nil, err + } + for _, i := range allImages { + if len(i.Names()) == 0 { + unamedImages = append(unamedImages, i) + continue + } + containers, err := i.Containers() + if err != nil { + return nil, err + } + if len(containers) < 1 { + unamedImages = append(unamedImages, i) + } + } + return unamedImages, nil +} diff --git a/test/e2e/prune_test.go b/test/e2e/prune_test.go new file mode 100644 index 000000000..6679a676c --- /dev/null +++ b/test/e2e/prune_test.go @@ -0,0 +1,88 @@ +package integration + +import ( + "fmt" + "os" + + . "github.com/containers/libpod/test/utils" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" +) + +var pruneImage = ` +FROM alpine:latest +LABEL RUN podman --version +RUN apk update +RUN apk add bash` + +var _ = Describe("Podman rm", func() { + var ( + tempdir string + err error + podmanTest *PodmanTestIntegration + ) + + BeforeEach(func() { + tempdir, err = CreateTempDirInTempDir() + if err != nil { + os.Exit(1) + } + podmanTest = PodmanTestCreate(tempdir) + podmanTest.RestoreAllArtifacts() + }) + + AfterEach(func() { + podmanTest.Cleanup() + f := CurrentGinkgoTestDescription() + timedResult := fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds()) + GinkgoWriter.Write([]byte(timedResult)) + }) + + It("podman container prune containers", func() { + top := podmanTest.RunTopContainer("") + top.WaitWithDefaultTimeout() + Expect(top.ExitCode()).To(Equal(0)) + + session := podmanTest.Podman([]string{"run", ALPINE, "ls"}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + + prune := podmanTest.Podman([]string{"container", "prune"}) + prune.WaitWithDefaultTimeout() + Expect(prune.ExitCode()).To(Equal(0)) + + Expect(podmanTest.NumberOfContainers()).To(Equal(1)) + }) + + It("podman image prune none images", func() { + podmanTest.BuildImage(pruneImage, "alpine_bash:latest", "true") + + none := podmanTest.Podman([]string{"images", "-a"}) + none.WaitWithDefaultTimeout() + Expect(none.ExitCode()).To(Equal(0)) + hasNone, _ := none.GrepString("<none>") + Expect(hasNone).To(BeTrue()) + + prune := podmanTest.Podman([]string{"image", "prune"}) + prune.WaitWithDefaultTimeout() + Expect(prune.ExitCode()).To(Equal(0)) + + after := podmanTest.Podman([]string{"images", "-a"}) + after.WaitWithDefaultTimeout() + Expect(none.ExitCode()).To(Equal(0)) + hasNoneAfter, _ := after.GrepString("<none>") + Expect(hasNoneAfter).To(BeFalse()) + }) + + It("podman image prune unused images", func() { + prune := podmanTest.Podman([]string{"image", "prune"}) + prune.WaitWithDefaultTimeout() + Expect(prune.ExitCode()).To(Equal(0)) + + images := podmanTest.Podman([]string{"images", "-a"}) + images.WaitWithDefaultTimeout() + // all images are unused, so they all should be deleted! + Expect(len(images.OutputToStringArray())).To(Equal(0)) + }) + +}) diff --git a/vendor.conf b/vendor.conf index 51907f763..94eb6fccc 100644 --- a/vendor.conf +++ b/vendor.conf @@ -12,7 +12,7 @@ github.com/containerd/continuity master github.com/containernetworking/cni v0.7.0-alpha1 github.com/containernetworking/plugins 1562a1e60ed101aacc5e08ed9dbeba8e9f3d4ec1 github.com/containers/image bd10b1b53b2976f215b3f2f848fb8e7cad779aeb -github.com/containers/storage ad0f9c4dfa38fcb160f430ff1d653dc3dae03810 +github.com/containers/storage db40f96d853dfced60c563e61fb66ba231ce7c8d github.com/containers/psgo 5dde6da0bc8831b35243a847625bcf18183bd1ee github.com/coreos/go-systemd v14 github.com/cri-o/ocicni 2d2983e40c242322a56c22a903785e7f83eb378c diff --git a/vendor/github.com/containers/storage/drivers/copy/copy.go b/vendor/github.com/containers/storage/drivers/copy/copy.go new file mode 100644 index 000000000..2617824c5 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/copy/copy.go @@ -0,0 +1,277 @@ +// +build linux + +package copy + +/* +#include <linux/fs.h> + +#ifndef FICLONE +#define FICLONE _IOW(0x94, 9, int) +#endif +*/ +import "C" +import ( + "container/list" + "fmt" + "io" + "os" + "path/filepath" + "syscall" + "time" + + "github.com/containers/storage/pkg/pools" + "github.com/containers/storage/pkg/system" + rsystem "github.com/opencontainers/runc/libcontainer/system" + "golang.org/x/sys/unix" +) + +// Mode indicates whether to use hardlink or copy content +type Mode int + +const ( + // Content creates a new file, and copies the content of the file + Content Mode = iota + // Hardlink creates a new hardlink to the existing file + Hardlink +) + +func copyRegular(srcPath, dstPath string, fileinfo os.FileInfo, copyWithFileRange, copyWithFileClone *bool) error { + srcFile, err := os.Open(srcPath) + if err != nil { + return err + } + defer srcFile.Close() + + // If the destination file already exists, we shouldn't blow it away + dstFile, err := os.OpenFile(dstPath, os.O_WRONLY|os.O_CREATE|os.O_EXCL, fileinfo.Mode()) + if err != nil { + return err + } + defer dstFile.Close() + + if *copyWithFileClone { + _, _, err = unix.Syscall(unix.SYS_IOCTL, dstFile.Fd(), C.FICLONE, srcFile.Fd()) + if err == nil { + return nil + } + + *copyWithFileClone = false + if err == unix.EXDEV { + *copyWithFileRange = false + } + } + if *copyWithFileRange { + err = doCopyWithFileRange(srcFile, dstFile, fileinfo) + // Trying the file_clone may not have caught the exdev case + // as the ioctl may not have been available (therefore EINVAL) + if err == unix.EXDEV || err == unix.ENOSYS { + *copyWithFileRange = false + } else { + return err + } + } + return legacyCopy(srcFile, dstFile) +} + +func doCopyWithFileRange(srcFile, dstFile *os.File, fileinfo os.FileInfo) error { + amountLeftToCopy := fileinfo.Size() + + for amountLeftToCopy > 0 { + n, err := unix.CopyFileRange(int(srcFile.Fd()), nil, int(dstFile.Fd()), nil, int(amountLeftToCopy), 0) + if err != nil { + return err + } + + amountLeftToCopy = amountLeftToCopy - int64(n) + } + + return nil +} + +func legacyCopy(srcFile io.Reader, dstFile io.Writer) error { + _, err := pools.Copy(dstFile, srcFile) + + return err +} + +func copyXattr(srcPath, dstPath, attr string) error { + data, err := system.Lgetxattr(srcPath, attr) + if err != nil { + return err + } + if data != nil { + if err := system.Lsetxattr(dstPath, attr, data, 0); err != nil { + return err + } + } + return nil +} + +type fileID struct { + dev uint64 + ino uint64 +} + +type dirMtimeInfo struct { + dstPath *string + stat *syscall.Stat_t +} + +// DirCopy copies or hardlinks the contents of one directory to another, +// properly handling xattrs, and soft links +// +// Copying xattrs can be opted out of by passing false for copyXattrs. +func DirCopy(srcDir, dstDir string, copyMode Mode, copyXattrs bool) error { + copyWithFileRange := true + copyWithFileClone := true + + // This is a map of source file inodes to dst file paths + copiedFiles := make(map[fileID]string) + + dirsToSetMtimes := list.New() + err := filepath.Walk(srcDir, func(srcPath string, f os.FileInfo, err error) error { + if err != nil { + return err + } + + // Rebase path + relPath, err := filepath.Rel(srcDir, srcPath) + if err != nil { + return err + } + + dstPath := filepath.Join(dstDir, relPath) + if err != nil { + return err + } + + stat, ok := f.Sys().(*syscall.Stat_t) + if !ok { + return fmt.Errorf("Unable to get raw syscall.Stat_t data for %s", srcPath) + } + + isHardlink := false + + switch f.Mode() & os.ModeType { + case 0: // Regular file + id := fileID{dev: stat.Dev, ino: stat.Ino} + if copyMode == Hardlink { + isHardlink = true + if err2 := os.Link(srcPath, dstPath); err2 != nil { + return err2 + } + } else if hardLinkDstPath, ok := copiedFiles[id]; ok { + if err2 := os.Link(hardLinkDstPath, dstPath); err2 != nil { + return err2 + } + } else { + if err2 := copyRegular(srcPath, dstPath, f, ©WithFileRange, ©WithFileClone); err2 != nil { + return err2 + } + copiedFiles[id] = dstPath + } + + case os.ModeDir: + if err := os.Mkdir(dstPath, f.Mode()); err != nil && !os.IsExist(err) { + return err + } + + case os.ModeSymlink: + link, err := os.Readlink(srcPath) + if err != nil { + return err + } + + if err := os.Symlink(link, dstPath); err != nil { + return err + } + + case os.ModeNamedPipe: + fallthrough + case os.ModeSocket: + if err := unix.Mkfifo(dstPath, stat.Mode); err != nil { + return err + } + + case os.ModeDevice: + if rsystem.RunningInUserNS() { + // cannot create a device if running in user namespace + return nil + } + if err := unix.Mknod(dstPath, stat.Mode, int(stat.Rdev)); err != nil { + return err + } + + default: + return fmt.Errorf("unknown file type for %s", srcPath) + } + + // Everything below is copying metadata from src to dst. All this metadata + // already shares an inode for hardlinks. + if isHardlink { + return nil + } + + if err := os.Lchown(dstPath, int(stat.Uid), int(stat.Gid)); err != nil { + return err + } + + if copyXattrs { + if err := doCopyXattrs(srcPath, dstPath); err != nil { + return err + } + } + + isSymlink := f.Mode()&os.ModeSymlink != 0 + + // There is no LChmod, so ignore mode for symlink. Also, this + // must happen after chown, as that can modify the file mode + if !isSymlink { + if err := os.Chmod(dstPath, f.Mode()); err != nil { + return err + } + } + + // system.Chtimes doesn't support a NOFOLLOW flag atm + // nolint: unconvert + if f.IsDir() { + dirsToSetMtimes.PushFront(&dirMtimeInfo{dstPath: &dstPath, stat: stat}) + } else if !isSymlink { + aTime := time.Unix(int64(stat.Atim.Sec), int64(stat.Atim.Nsec)) + mTime := time.Unix(int64(stat.Mtim.Sec), int64(stat.Mtim.Nsec)) + if err := system.Chtimes(dstPath, aTime, mTime); err != nil { + return err + } + } else { + ts := []syscall.Timespec{stat.Atim, stat.Mtim} + if err := system.LUtimesNano(dstPath, ts); err != nil { + return err + } + } + return nil + }) + if err != nil { + return err + } + for e := dirsToSetMtimes.Front(); e != nil; e = e.Next() { + mtimeInfo := e.Value.(*dirMtimeInfo) + ts := []syscall.Timespec{mtimeInfo.stat.Atim, mtimeInfo.stat.Mtim} + if err := system.LUtimesNano(*mtimeInfo.dstPath, ts); err != nil { + return err + } + } + + return nil +} + +func doCopyXattrs(srcPath, dstPath string) error { + if err := copyXattr(srcPath, dstPath, "security.capability"); err != nil { + return err + } + + // We need to copy this attribute if it appears in an overlay upper layer, as + // this function is used to copy those. It is set by overlay if a directory + // is removed and then re-created and should not inherit anything from the + // same dir in the lower dir. + return copyXattr(srcPath, dstPath, "trusted.overlay.opaque") +} diff --git a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go index 2801dfdc5..b6f22e90a 100644 --- a/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go +++ b/vendor/github.com/containers/storage/drivers/devmapper/deviceset.go @@ -2401,7 +2401,7 @@ func (devices *DeviceSet) MountDevice(hash, path string, moptions graphdriver.Mo addNouuid := strings.Contains("nouuid", mountOptions) mountOptions = strings.Join(moptions.Options, ",") if addNouuid { - mountOptions = fmt.Sprintf("nouuid,", mountOptions) + mountOptions = fmt.Sprintf("nouuid,%s", mountOptions) } } diff --git a/vendor/github.com/containers/storage/drivers/vfs/copy_linux.go b/vendor/github.com/containers/storage/drivers/vfs/copy_linux.go new file mode 100644 index 000000000..8137fcf67 --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/vfs/copy_linux.go @@ -0,0 +1,7 @@ +package vfs + +import "github.com/containers/storage/drivers/copy" + +func dirCopy(srcDir, dstDir string) error { + return copy.DirCopy(srcDir, dstDir, copy.Content, false) +} diff --git a/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go b/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go new file mode 100644 index 000000000..8ac80ee1d --- /dev/null +++ b/vendor/github.com/containers/storage/drivers/vfs/copy_unsupported.go @@ -0,0 +1,9 @@ +// +build !linux + +package vfs // import "github.com/containers/storage/drivers/vfs" + +import "github.com/containers/storage/pkg/chrootarchive" + +func dirCopy(srcDir, dstDir string) error { + return chrootarchive.NewArchiver(nil).CopyWithTar(srcDir, dstDir) +} diff --git a/vendor/github.com/containers/storage/drivers/vfs/driver.go b/vendor/github.com/containers/storage/drivers/vfs/driver.go index e3a67a69b..f7f3c75ba 100644 --- a/vendor/github.com/containers/storage/drivers/vfs/driver.go +++ b/vendor/github.com/containers/storage/drivers/vfs/driver.go @@ -7,7 +7,6 @@ import ( "strings" "github.com/containers/storage/drivers" - "github.com/containers/storage/pkg/chrootarchive" "github.com/containers/storage/pkg/idtools" "github.com/containers/storage/pkg/ostree" "github.com/containers/storage/pkg/system" @@ -15,8 +14,8 @@ import ( ) var ( - // CopyWithTar defines the copy method to use. - CopyWithTar = chrootarchive.NewArchiver(nil).CopyWithTar + // CopyDir defines the copy method to use. + CopyDir = dirCopy ) func init() { @@ -141,7 +140,7 @@ func (d *Driver) create(id, parent string, opts *graphdriver.CreateOpts, ro bool if err != nil { return fmt.Errorf("%s: %s", parent, err) } - if err := CopyWithTar(parentDir, dir); err != nil { + if err := dirCopy(parentDir, dir); err != nil { return err } } diff --git a/vendor/github.com/containers/storage/pkg/archive/example_changes.go b/vendor/github.com/containers/storage/pkg/archive/example_changes.go new file mode 100644 index 000000000..70f9c5564 --- /dev/null +++ b/vendor/github.com/containers/storage/pkg/archive/example_changes.go @@ -0,0 +1,97 @@ +// +build ignore + +// Simple tool to create an archive stream from an old and new directory +// +// By default it will stream the comparison of two temporary directories with junk files +package main + +import ( + "flag" + "fmt" + "io" + "io/ioutil" + "os" + "path" + + "github.com/containers/storage/pkg/archive" + "github.com/sirupsen/logrus" +) + +var ( + flDebug = flag.Bool("D", false, "debugging output") + flNewDir = flag.String("newdir", "", "") + flOldDir = flag.String("olddir", "", "") + log = logrus.New() +) + +func main() { + flag.Usage = func() { + fmt.Println("Produce a tar from comparing two directory paths. By default a demo tar is created of around 200 files (including hardlinks)") + fmt.Printf("%s [OPTIONS]\n", os.Args[0]) + flag.PrintDefaults() + } + flag.Parse() + log.Out = os.Stderr + if (len(os.Getenv("DEBUG")) > 0) || *flDebug { + logrus.SetLevel(logrus.DebugLevel) + } + var newDir, oldDir string + + if len(*flNewDir) == 0 { + var err error + newDir, err = ioutil.TempDir("", "storage-test-newDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(newDir) + if _, err := prepareUntarSourceDirectory(100, newDir, true); err != nil { + log.Fatal(err) + } + } else { + newDir = *flNewDir + } + + if len(*flOldDir) == 0 { + oldDir, err := ioutil.TempDir("", "storage-test-oldDir") + if err != nil { + log.Fatal(err) + } + defer os.RemoveAll(oldDir) + } else { + oldDir = *flOldDir + } + + changes, err := archive.ChangesDirs(newDir, oldDir) + if err != nil { + log.Fatal(err) + } + + a, err := archive.ExportChanges(newDir, changes) + if err != nil { + log.Fatal(err) + } + defer a.Close() + + i, err := io.Copy(os.Stdout, a) + if err != nil && err != io.EOF { + log.Fatal(err) + } + fmt.Fprintf(os.Stderr, "wrote archive of %d bytes", i) +} + +func prepareUntarSourceDirectory(numberOfFiles int, targetPath string, makeLinks bool) (int, error) { + fileData := []byte("fooo") + for n := 0; n < numberOfFiles; n++ { + fileName := fmt.Sprintf("file-%d", n) + if err := ioutil.WriteFile(path.Join(targetPath, fileName), fileData, 0700); err != nil { + return 0, err + } + if makeLinks { + if err := os.Link(path.Join(targetPath, fileName), path.Join(targetPath, fileName+"-link")); err != nil { + return 0, err + } + } + } + totalSize := numberOfFiles * len(fileData) + return totalSize, nil +} diff --git a/vendor/github.com/containers/storage/vendor.conf b/vendor/github.com/containers/storage/vendor.conf index 059ae94f0..fa52584d7 100644 --- a/vendor/github.com/containers/storage/vendor.conf +++ b/vendor/github.com/containers/storage/vendor.conf @@ -9,7 +9,7 @@ github.com/mistifyio/go-zfs c0224de804d438efd11ea6e52ada8014537d6062 github.com/opencontainers/go-digest master github.com/opencontainers/runc 6c22e77604689db8725fa866f0f2ec0b3e8c3a07 github.com/opencontainers/selinux 36a9bc45a08c85f2c52bd9eb32e20267876773bd -github.com/ostreedev/ostree-go aeb02c6b6aa2889db3ef62f7855650755befd460 +github.com/ostreedev/ostree-go master github.com/pborman/uuid 1b00554d822231195d1babd97ff4a781231955c9 github.com/pkg/errors master github.com/pmezard/go-difflib v1.0.0 @@ -21,3 +21,5 @@ github.com/tchap/go-patricia v2.2.6 github.com/vbatts/tar-split v0.10.2 golang.org/x/net 7dcfb8076726a3fdd9353b6b8a1f1b6be6811bd6 golang.org/x/sys 07c182904dbd53199946ba614a412c61d3c548f5 +gotest.tools master +github.com/google/go-cmp master |