From 3ae91aff9ee67acac64f3e97908e55c5912a33fe Mon Sep 17 00:00:00 2001 From: Kazım SARIKAYA Date: Sun, 3 Jan 2021 19:04:25 +0300 Subject: podman-remote fix sending tar content MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1.) podman cannot send proper dockerfile when it is not inside root folder. 2.) support for sending symlinks and folders inside context dir 3.) when sending context dir as tar to remote, prevent sending items inside .dockerignore Signed-off-by: Kazım SARIKAYA --- pkg/bindings/images/build.go | 138 ++++++++++++++++++++++----- test/e2e/build_test.go | 216 ++++++++++++++++++++++++++++++++++++++++++- 2 files changed, 330 insertions(+), 24 deletions(-) diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go index d34ab87d9..02765816f 100644 --- a/pkg/bindings/images/build.go +++ b/pkg/bindings/images/build.go @@ -2,6 +2,7 @@ package images import ( "archive/tar" + "compress/gzip" "context" "encoding/json" "io" @@ -18,6 +19,7 @@ import ( "github.com/containers/podman/v2/pkg/auth" "github.com/containers/podman/v2/pkg/bindings" "github.com/containers/podman/v2/pkg/domain/entities" + "github.com/containers/storage/pkg/fileutils" "github.com/docker/go-units" "github.com/hashicorp/go-multierror" jsoniter "github.com/json-iterator/go" @@ -138,12 +140,38 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO entries := make([]string, len(containerFiles)) copy(entries, containerFiles) entries = append(entries, options.ContextDirectory) - tarfile, err := nTar(entries...) + + excludes := options.Excludes + if len(excludes) == 0 { + excludes, err = parseDockerignore(options.ContextDirectory) + if err != nil { + return nil, err + } + } + + tarfile, err := nTar(excludes, entries...) if err != nil { + logrus.Errorf("cannot tar container entries %v error: %v", entries, err) return nil, err } defer tarfile.Close() - params.Set("dockerfile", filepath.Base(containerFiles[0])) + + containerFile, err := filepath.Abs(entries[0]) + if err != nil { + logrus.Errorf("cannot find absolute path of %v: %v", entries[0], err) + return nil, err + } + contextDir, err := filepath.Abs(entries[1]) + if err != nil { + logrus.Errorf("cannot find absolute path of %v: %v", entries[1], err) + return nil, err + } + + if strings.HasPrefix(containerFile, contextDir+string(filepath.Separator)) { + containerFile = strings.TrimPrefix(containerFile, contextDir+string(filepath.Separator)) + } + + params.Set("dockerfile", containerFile) conn, err := bindings.GetClient(ctx) if err != nil { @@ -200,52 +228,116 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO } } -func nTar(sources ...string) (io.ReadCloser, error) { +func nTar(excludes []string, sources ...string) (io.ReadCloser, error) { + pm, err := fileutils.NewPatternMatcher(excludes) + if err != nil { + return nil, errors.Wrapf(err, "error processing excludes list %v", excludes) + } + if len(sources) == 0 { return nil, errors.New("No source(s) provided for build") } pr, pw := io.Pipe() - tw := tar.NewWriter(pw) + gw := gzip.NewWriter(pw) + tw := tar.NewWriter(gw) var merr error go func() { defer pw.Close() + defer gw.Close() defer tw.Close() for _, src := range sources { - s := src - err := filepath.Walk(s, func(path string, info os.FileInfo, err error) error { + s, err := filepath.Abs(src) + if err != nil { + logrus.Errorf("cannot stat one of source context: %v", err) + merr = multierror.Append(merr, err) + return + } + + err = filepath.Walk(s, func(path string, info os.FileInfo, err error) error { if err != nil { return err } - if !info.Mode().IsRegular() || path == s { - return nil - } - f, lerr := os.Open(path) - if lerr != nil { - return lerr + if path == s { + return nil // skip root dir } name := strings.TrimPrefix(path, s+string(filepath.Separator)) - hdr, lerr := tar.FileInfoHeader(info, name) - if lerr != nil { - f.Close() - return lerr + + excluded, err := pm.Matches(filepath.ToSlash(name)) // nolint:staticcheck + if err != nil { + return errors.Wrapf(err, "error checking if %q is excluded", name) } - hdr.Name = name - if lerr := tw.WriteHeader(hdr); lerr != nil { - f.Close() - return lerr + if excluded { + return nil } - _, cerr := io.Copy(tw, f) - f.Close() - return cerr + if info.Mode().IsRegular() { // add file item + f, lerr := os.Open(path) + if lerr != nil { + return lerr + } + + hdr, lerr := tar.FileInfoHeader(info, name) + if lerr != nil { + f.Close() + return lerr + } + hdr.Name = name + if lerr := tw.WriteHeader(hdr); lerr != nil { + f.Close() + return lerr + } + + _, cerr := io.Copy(tw, f) + f.Close() + return cerr + } else if info.Mode().IsDir() { // add folders + hdr, lerr := tar.FileInfoHeader(info, name) + if lerr != nil { + return lerr + } + hdr.Name = name + if lerr := tw.WriteHeader(hdr); lerr != nil { + return lerr + } + } else if info.Mode()&os.ModeSymlink != 0 { // add symlinks as it, not content + link, err := os.Readlink(path) + if err != nil { + return err + } + hdr, lerr := tar.FileInfoHeader(info, link) + if lerr != nil { + return lerr + } + hdr.Name = name + if lerr := tw.WriteHeader(hdr); lerr != nil { + return lerr + } + } //skip other than file,folder and symlinks + return nil }) merr = multierror.Append(merr, err) } }() return pr, merr } + +func parseDockerignore(root string) ([]string, error) { + ignore, err := ioutil.ReadFile(filepath.Join(root, ".dockerignore")) + if err != nil && !os.IsNotExist(err) { + return nil, errors.Wrapf(err, "error reading .dockerignore: '%s'", root) + } + rawexcludes := strings.Split(string(ignore), "\n") + excludes := make([]string, 0, len(rawexcludes)) + for _, e := range rawexcludes { + if len(e) == 0 || e[0] == '#' { + continue + } + excludes = append(excludes, e) + } + return excludes, nil +} diff --git a/test/e2e/build_test.go b/test/e2e/build_test.go index ac9481797..21f98d3d0 100644 --- a/test/e2e/build_test.go +++ b/test/e2e/build_test.go @@ -234,7 +234,7 @@ RUN printenv http_proxy` }) It("podman build and check identity", func() { - session := podmanTest.Podman([]string{"build", "-f", "Containerfile.path", "--no-cache", "-t", "test", "build/basicalpine"}) + session := podmanTest.Podman([]string{"build", "-f", "build/basicalpine/Containerfile.path", "--no-cache", "-t", "test", "build/basicalpine"}) session.WaitWithDefaultTimeout() Expect(session.ExitCode()).To(Equal(0)) @@ -244,4 +244,218 @@ RUN printenv http_proxy` data := inspect.OutputToString() Expect(data).To(ContainSubstring(buildah.Version)) }) + + It("podman remote test container/docker file is not inside context dir", func() { + // Given + // Switch to temp dir and restore it afterwards + cwd, err := os.Getwd() + Expect(err).To(BeNil()) + + podmanTest.AddImageToRWStore(ALPINE) + + // Write target and fake files + targetPath, err := CreateTempDirInTempDir() + Expect(err).To(BeNil()) + targetSubPath := filepath.Join(targetPath, "subdir") + err = os.Mkdir(targetSubPath, 0755) + Expect(err).To(BeNil()) + dummyFile := filepath.Join(targetSubPath, "dummy") + err = ioutil.WriteFile(dummyFile, []byte("dummy"), 0644) + Expect(err).To(BeNil()) + + containerfile := `FROM quay.io/libpod/alpine:latest +ADD . /test +RUN find /test` + + containerfilePath := filepath.Join(targetPath, "Containerfile") + err = ioutil.WriteFile(containerfilePath, []byte(containerfile), 0644) + Expect(err).To(BeNil()) + + defer func() { + Expect(os.Chdir(cwd)).To(BeNil()) + Expect(os.RemoveAll(targetPath)).To(BeNil()) + }() + + // make cwd as context root path + Expect(os.Chdir(targetPath)).To(BeNil()) + + session := podmanTest.Podman([]string{"build", "-t", "test", "-f", "Containerfile", targetSubPath}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + ok, _ := session.GrepString("/test/dummy") + Expect(ok).To(BeTrue()) + }) + + It("podman remote test container/docker file is not at root of context dir", func() { + if IsRemote() { + podmanTest.StopRemoteService() + podmanTest.StartRemoteService() + } else { + Skip("Only valid at remote test") + } + // Given + // Switch to temp dir and restore it afterwards + cwd, err := os.Getwd() + Expect(err).To(BeNil()) + + podmanTest.AddImageToRWStore(ALPINE) + + // Write target and fake files + targetPath, err := CreateTempDirInTempDir() + Expect(err).To(BeNil()) + targetSubPath := filepath.Join(targetPath, "subdir") + err = os.Mkdir(targetSubPath, 0755) + Expect(err).To(BeNil()) + + containerfile := `FROM quay.io/libpod/alpine:latest` + + containerfilePath := filepath.Join(targetSubPath, "Containerfile") + err = ioutil.WriteFile(containerfilePath, []byte(containerfile), 0644) + Expect(err).To(BeNil()) + + defer func() { + Expect(os.Chdir(cwd)).To(BeNil()) + Expect(os.RemoveAll(targetPath)).To(BeNil()) + }() + + // make cwd as context root path + Expect(os.Chdir(targetPath)).To(BeNil()) + + session := podmanTest.Podman([]string{"build", "-t", "test", "-f", "subdir/Containerfile", "."}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + }) + + It("podman remote test .dockerignore", func() { + if IsRemote() { + podmanTest.StopRemoteService() + podmanTest.StartRemoteService() + } else { + Skip("Only valid at remote test") + } + // Given + // Switch to temp dir and restore it afterwards + cwd, err := os.Getwd() + Expect(err).To(BeNil()) + + podmanTest.AddImageToRWStore(ALPINE) + + // Write target and fake files + targetPath, err := CreateTempDirInTempDir() + Expect(err).To(BeNil()) + + containerfile := `FROM quay.io/libpod/alpine:latest +ADD . /testfilter/ +RUN find /testfilter/` + + containerfilePath := filepath.Join(targetPath, "Containerfile") + err = ioutil.WriteFile(containerfilePath, []byte(containerfile), 0644) + Expect(err).To(BeNil()) + + targetSubPath := filepath.Join(targetPath, "subdir") + err = os.Mkdir(targetSubPath, 0755) + Expect(err).To(BeNil()) + + dummyFile1 := filepath.Join(targetPath, "dummy1") + err = ioutil.WriteFile(dummyFile1, []byte("dummy1"), 0644) + Expect(err).To(BeNil()) + + dummyFile2 := filepath.Join(targetPath, "dummy2") + err = ioutil.WriteFile(dummyFile2, []byte("dummy2"), 0644) + Expect(err).To(BeNil()) + + dummyFile3 := filepath.Join(targetSubPath, "dummy3") + err = ioutil.WriteFile(dummyFile3, []byte("dummy3"), 0644) + Expect(err).To(BeNil()) + + defer func() { + Expect(os.Chdir(cwd)).To(BeNil()) + Expect(os.RemoveAll(targetPath)).To(BeNil()) + }() + + // make cwd as context root path + Expect(os.Chdir(targetPath)).To(BeNil()) + + dockerignoreContent := `dummy1 +subdir**` + dockerignoreFile := filepath.Join(targetPath, ".dockerignore") + + // test .dockerignore + By("Test .dockererignore") + err = ioutil.WriteFile(dockerignoreFile, []byte(dockerignoreContent), 0644) + Expect(err).To(BeNil()) + + session := podmanTest.Podman([]string{"build", "-t", "test", "."}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + ok, _ := session.GrepString("/testfilter/dummy1") + Expect(ok).NotTo(BeTrue()) + ok, _ = session.GrepString("/testfilter/dummy2") + Expect(ok).To(BeTrue()) + ok, _ = session.GrepString("/testfilter/subdir") + Expect(ok).NotTo(BeTrue()) //.dockerignore filters both subdir and inside subdir + ok, _ = session.GrepString("/testfilter/subdir/dummy3") + Expect(ok).NotTo(BeTrue()) + }) + + It("podman remote test context dir contains empty dirs and symlinks", func() { + if IsRemote() { + podmanTest.StopRemoteService() + podmanTest.StartRemoteService() + } else { + Skip("Only valid at remote test") + } + // Given + // Switch to temp dir and restore it afterwards + cwd, err := os.Getwd() + Expect(err).To(BeNil()) + + podmanTest.AddImageToRWStore(ALPINE) + + // Write target and fake files + targetPath, err := CreateTempDirInTempDir() + Expect(err).To(BeNil()) + targetSubPath := filepath.Join(targetPath, "subdir") + err = os.Mkdir(targetSubPath, 0755) + Expect(err).To(BeNil()) + dummyFile := filepath.Join(targetSubPath, "dummy") + err = ioutil.WriteFile(dummyFile, []byte("dummy"), 0644) + Expect(err).To(BeNil()) + + emptyDir := filepath.Join(targetSubPath, "emptyDir") + err = os.Mkdir(emptyDir, 0755) + Expect(err).To(BeNil()) + Expect(os.Chdir(targetSubPath)).To(BeNil()) + Expect(os.Symlink("dummy", "dummy-symlink")).To(BeNil()) + + containerfile := `FROM quay.io/libpod/alpine:latest +ADD . /test +RUN find /test +RUN [[ -L /test/dummy-symlink ]] && echo SYMLNKOK || echo SYMLNKERR` + + containerfilePath := filepath.Join(targetSubPath, "Containerfile") + err = ioutil.WriteFile(containerfilePath, []byte(containerfile), 0644) + Expect(err).To(BeNil()) + + defer func() { + Expect(os.Chdir(cwd)).To(BeNil()) + Expect(os.RemoveAll(targetPath)).To(BeNil()) + }() + + // make cwd as context root path + Expect(os.Chdir(targetPath)).To(BeNil()) + + session := podmanTest.Podman([]string{"build", "-t", "test", targetSubPath}) + session.WaitWithDefaultTimeout() + Expect(session.ExitCode()).To(Equal(0)) + ok, _ := session.GrepString("/test/dummy") + Expect(ok).To(BeTrue()) + ok, _ = session.GrepString("/test/emptyDir") + Expect(ok).To(BeTrue()) + ok, _ = session.GrepString("/test/dummy-symlink") + Expect(ok).To(BeTrue()) + ok, _ = session.GrepString("SYMLNKOK") + Expect(ok).To(BeTrue()) + }) + }) -- cgit v1.2.3-54-g00ecf