From 7e7a79b075f7d65657d95169f02c2c1c03198b93 Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Tue, 16 Aug 2022 18:30:19 -0400 Subject: podman manifest create: accept --amend and --insecure flags Accept a --amend flag in `podman manifest create`, and treat `--insecure` as we would `--tls-verify=false` in `podman manifest`'s "add", "create", and "push" subcommands. Signed-off-by: Nalin Dahyabhai --- pkg/bindings/manifests/types.go | 3 ++- pkg/bindings/manifests/types_create_options.go | 15 +++++++++++++++ 2 files changed, 17 insertions(+), 1 deletion(-) (limited to 'pkg/bindings') diff --git a/pkg/bindings/manifests/types.go b/pkg/bindings/manifests/types.go index e23ef798d..5f2557fe1 100644 --- a/pkg/bindings/manifests/types.go +++ b/pkg/bindings/manifests/types.go @@ -8,7 +8,8 @@ type InspectOptions struct { //go:generate go run ../generator/generator.go CreateOptions // CreateOptions are optional options for creating manifests type CreateOptions struct { - All *bool + All *bool + Amend *bool } //go:generate go run ../generator/generator.go ExistsOptions diff --git a/pkg/bindings/manifests/types_create_options.go b/pkg/bindings/manifests/types_create_options.go index 960332a82..09942c00a 100644 --- a/pkg/bindings/manifests/types_create_options.go +++ b/pkg/bindings/manifests/types_create_options.go @@ -31,3 +31,18 @@ func (o *CreateOptions) GetAll() bool { } return *o.All } + +// WithAmend set field Amend to given value +func (o *CreateOptions) WithAmend(value bool) *CreateOptions { + o.Amend = &value + return o +} + +// GetAmend returns value of field Amend +func (o *CreateOptions) GetAmend() bool { + if o.Amend == nil { + var z bool + return z + } + return *o.Amend +} -- cgit v1.2.3-54-g00ecf From 3bf52aa338b33de719e087e15402081568453284 Mon Sep 17 00:00:00 2001 From: Vladimir Kochnev Date: Fri, 19 Aug 2022 00:41:22 +0300 Subject: Add ProgressWriter to PullOptions Signed-off-by: Vladimir Kochnev --- cmd/podman/images/pull.go | 5 +++++ pkg/bindings/images/pull.go | 13 ++++++++----- pkg/bindings/images/push.go | 9 +++++---- pkg/bindings/images/types.go | 2 ++ pkg/bindings/images/types_pull_options.go | 16 ++++++++++++++++ pkg/bindings/manifests/manifests.go | 6 ++++-- pkg/bindings/test/images_test.go | 24 ++++++++++++++++++++++-- pkg/bindings/test/manifests_test.go | 23 ++++++++++++++++++++--- pkg/domain/entities/images.go | 2 ++ pkg/domain/infra/abi/images.go | 3 ++- pkg/domain/infra/tunnel/images.go | 1 + test/e2e/manifest_test.go | 27 +++++++++++++++++++++++++++ test/e2e/pull_test.go | 14 ++++++++++++++ 13 files changed, 128 insertions(+), 17 deletions(-) (limited to 'pkg/bindings') diff --git a/cmd/podman/images/pull.go b/cmd/podman/images/pull.go index 8211ceba5..fe9d1e9b6 100644 --- a/cmd/podman/images/pull.go +++ b/cmd/podman/images/pull.go @@ -155,6 +155,11 @@ func imagePull(cmd *cobra.Command, args []string) error { pullOptions.Username = creds.Username pullOptions.Password = creds.Password } + + if !pullOptions.Quiet { + pullOptions.Writer = os.Stderr + } + // Let's do all the remaining Yoga in the API to prevent us from // scattering logic across (too) many parts of the code. var errs utils.OutputErrors diff --git a/pkg/bindings/images/pull.go b/pkg/bindings/images/pull.go index 1a4aa3038..109981c63 100644 --- a/pkg/bindings/images/pull.go +++ b/pkg/bindings/images/pull.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "os" "strconv" @@ -57,10 +56,14 @@ func Pull(ctx context.Context, rawImage string, options *PullOptions) ([]string, return nil, response.Process(err) } - // Historically pull writes status to stderr - stderr := io.Writer(os.Stderr) + var writer io.Writer if options.GetQuiet() { - stderr = ioutil.Discard + writer = io.Discard + } else if progressWriter := options.GetProgressWriter(); progressWriter != nil { + writer = progressWriter + } else { + // Historically push writes status to stderr + writer = os.Stderr } dec := json.NewDecoder(response.Body) @@ -84,7 +87,7 @@ func Pull(ctx context.Context, rawImage string, options *PullOptions) ([]string, switch { case report.Stream != "": - fmt.Fprint(stderr, report.Stream) + fmt.Fprint(writer, report.Stream) case report.Error != "": pullErrors = append(pullErrors, errors.New(report.Error)) case len(report.Images) > 0: diff --git a/pkg/bindings/images/push.go b/pkg/bindings/images/push.go index 5069dd780..f1e059f8c 100644 --- a/pkg/bindings/images/push.go +++ b/pkg/bindings/images/push.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "os" "strconv" @@ -58,12 +57,14 @@ func Push(ctx context.Context, source string, destination string, options *PushO return response.Process(err) } - // Historically push writes status to stderr - writer := io.Writer(os.Stderr) + var writer io.Writer if options.GetQuiet() { - writer = ioutil.Discard + writer = io.Discard } else if progressWriter := options.GetProgressWriter(); progressWriter != nil { writer = progressWriter + } else { + // Historically push writes status to stderr + writer = os.Stderr } dec := json.NewDecoder(response.Body) diff --git a/pkg/bindings/images/types.go b/pkg/bindings/images/types.go index 7b28c499e..3ecfb9e09 100644 --- a/pkg/bindings/images/types.go +++ b/pkg/bindings/images/types.go @@ -182,6 +182,8 @@ type PullOptions struct { Policy *string // Password for authenticating against the registry. Password *string + // ProgressWriter is a writer where pull progress are sent. + ProgressWriter *io.Writer // Quiet can be specified to suppress pull progress when pulling. Ignored // for remote calls. Quiet *bool diff --git a/pkg/bindings/images/types_pull_options.go b/pkg/bindings/images/types_pull_options.go index 4cd525185..c1a88fd9e 100644 --- a/pkg/bindings/images/types_pull_options.go +++ b/pkg/bindings/images/types_pull_options.go @@ -2,6 +2,7 @@ package images import ( + "io" "net/url" "github.com/containers/podman/v4/pkg/bindings/internal/util" @@ -107,6 +108,21 @@ func (o *PullOptions) GetPassword() string { return *o.Password } +// WithProgressWriter set field ProgressWriter to given value +func (o *PullOptions) WithProgressWriter(value io.Writer) *PullOptions { + o.ProgressWriter = &value + return o +} + +// GetProgressWriter returns value of field ProgressWriter +func (o *PullOptions) GetProgressWriter() io.Writer { + if o.ProgressWriter == nil { + var z io.Writer + return z + } + return *o.ProgressWriter +} + // WithQuiet set field Quiet to given value func (o *PullOptions) WithQuiet(value bool) *PullOptions { o.Quiet = &value diff --git a/pkg/bindings/manifests/manifests.go b/pkg/bindings/manifests/manifests.go index 49e4089f5..0163d21a0 100644 --- a/pkg/bindings/manifests/manifests.go +++ b/pkg/bindings/manifests/manifests.go @@ -182,12 +182,14 @@ func Push(ctx context.Context, name, destination string, options *images.PushOpt return "", response.Process(err) } - // Historically push writes status to stderr - writer := io.Writer(os.Stderr) + var writer io.Writer if options.GetQuiet() { writer = io.Discard } else if progressWriter := options.GetProgressWriter(); progressWriter != nil { writer = progressWriter + } else { + // Historically push writes status to stderr + writer = os.Stderr } dec := json.NewDecoder(response.Body) diff --git a/pkg/bindings/test/images_test.go b/pkg/bindings/test/images_test.go index 9c9796661..53c5a1e83 100644 --- a/pkg/bindings/test/images_test.go +++ b/pkg/bindings/test/images_test.go @@ -1,11 +1,14 @@ package bindings_test import ( + "bytes" + "fmt" "net/http" "os" "path/filepath" "time" + podmanRegistry "github.com/containers/podman/v4/hack/podman-registry-go" "github.com/containers/podman/v4/pkg/bindings" "github.com/containers/podman/v4/pkg/bindings/containers" "github.com/containers/podman/v4/pkg/bindings/images" @@ -362,9 +365,14 @@ var _ = Describe("Podman images", func() { It("Image Pull", func() { rawImage := "docker.io/library/busybox:latest" - pulledImages, err := images.Pull(bt.conn, rawImage, nil) + var writer bytes.Buffer + pullOpts := new(images.PullOptions).WithProgressWriter(&writer) + pulledImages, err := images.Pull(bt.conn, rawImage, pullOpts) Expect(err).NotTo(HaveOccurred()) Expect(len(pulledImages)).To(Equal(1)) + output := writer.String() + Expect(output).To(ContainSubstring("Trying to pull ")) + Expect(output).To(ContainSubstring("Getting image source signatures")) exists, err := images.Exists(bt.conn, rawImage, nil) Expect(err).NotTo(HaveOccurred()) @@ -380,7 +388,19 @@ var _ = Describe("Podman images", func() { }) It("Image Push", func() { - Skip("TODO: implement test for image push to registry") + registry, err := podmanRegistry.Start() + Expect(err).To(BeNil()) + + var writer bytes.Buffer + pushOpts := new(images.PushOptions).WithUsername(registry.User).WithPassword(registry.Password).WithSkipTLSVerify(true).WithProgressWriter(&writer).WithQuiet(false) + err = images.Push(bt.conn, alpine.name, fmt.Sprintf("localhost:%s/test:latest", registry.Port), pushOpts) + Expect(err).ToNot(HaveOccurred()) + + output := writer.String() + Expect(output).To(ContainSubstring("Copying blob ")) + Expect(output).To(ContainSubstring("Copying config ")) + Expect(output).To(ContainSubstring("Writing manifest to image destination")) + Expect(output).To(ContainSubstring("Storing signatures")) }) It("Build no options", func() { diff --git a/pkg/bindings/test/manifests_test.go b/pkg/bindings/test/manifests_test.go index 6a34ef5a6..d6749f920 100644 --- a/pkg/bindings/test/manifests_test.go +++ b/pkg/bindings/test/manifests_test.go @@ -1,9 +1,12 @@ package bindings_test import ( + "bytes" + "fmt" "net/http" "time" + podmanRegistry "github.com/containers/podman/v4/hack/podman-registry-go" "github.com/containers/podman/v4/pkg/bindings" "github.com/containers/podman/v4/pkg/bindings/images" "github.com/containers/podman/v4/pkg/bindings/manifests" @@ -12,7 +15,7 @@ import ( "github.com/onsi/gomega/gexec" ) -var _ = Describe("podman manifest", func() { +var _ = Describe("Podman manifests", func() { var ( bt *bindingTest s *gexec.Session @@ -172,7 +175,21 @@ var _ = Describe("podman manifest", func() { Expect(list.Manifests[0].Platform.OS).To(Equal("foo")) }) - It("push manifest", func() { - Skip("TODO: implement test for manifest push to registry") + It("Manifest Push", func() { + registry, err := podmanRegistry.Start() + Expect(err).To(BeNil()) + + name := "quay.io/libpod/foobar:latest" + _, err = manifests.Create(bt.conn, name, []string{alpine.name}, nil) + Expect(err).ToNot(HaveOccurred()) + + var writer bytes.Buffer + pushOpts := new(images.PushOptions).WithUsername(registry.User).WithPassword(registry.Password).WithAll(true).WithSkipTLSVerify(true).WithProgressWriter(&writer).WithQuiet(false) + _, err = manifests.Push(bt.conn, name, fmt.Sprintf("localhost:%s/test:latest", registry.Port), pushOpts) + Expect(err).ToNot(HaveOccurred()) + + output := writer.String() + Expect(output).To(ContainSubstring("Writing manifest list to image destination")) + Expect(output).To(ContainSubstring("Storing list signatures")) }) }) diff --git a/pkg/domain/entities/images.go b/pkg/domain/entities/images.go index 21c1372b9..cad11b0ab 100644 --- a/pkg/domain/entities/images.go +++ b/pkg/domain/entities/images.go @@ -156,6 +156,8 @@ type ImagePullOptions struct { SkipTLSVerify types.OptionalBool // PullPolicy whether to pull new image PullPolicy config.PullPolicy + // Writer is used to display copy information including progress bars. + Writer io.Writer } // ImagePullReport is the response from pulling one or more images. diff --git a/pkg/domain/infra/abi/images.go b/pkg/domain/infra/abi/images.go index 77d1bf0db..f9839f62f 100644 --- a/pkg/domain/infra/abi/images.go +++ b/pkg/domain/infra/abi/images.go @@ -237,8 +237,9 @@ func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, options entiti pullOptions.Variant = options.Variant pullOptions.SignaturePolicyPath = options.SignaturePolicy pullOptions.InsecureSkipTLSVerify = options.SkipTLSVerify + pullOptions.Writer = options.Writer - if !options.Quiet { + if !options.Quiet && pullOptions.Writer == nil { pullOptions.Writer = os.Stderr } diff --git a/pkg/domain/infra/tunnel/images.go b/pkg/domain/infra/tunnel/images.go index bb3014099..2716aaf2a 100644 --- a/pkg/domain/infra/tunnel/images.go +++ b/pkg/domain/infra/tunnel/images.go @@ -110,6 +110,7 @@ func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, opts entities. options.WithAllTags(opts.AllTags).WithAuthfile(opts.Authfile).WithArch(opts.Arch).WithOS(opts.OS) options.WithVariant(opts.Variant).WithPassword(opts.Password) options.WithQuiet(opts.Quiet).WithUsername(opts.Username).WithPolicy(opts.PullPolicy.String()) + options.WithProgressWriter(opts.Writer) if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined { if s == types.OptionalBoolTrue { options.WithSkipTLSVerify(true) diff --git a/test/e2e/manifest_test.go b/test/e2e/manifest_test.go index 1c4aad710..60b72dcaa 100644 --- a/test/e2e/manifest_test.go +++ b/test/e2e/manifest_test.go @@ -350,6 +350,33 @@ var _ = Describe("Podman manifest", func() { Expect(foundZstdFile).To(BeTrue()) }) + It("push progress", func() { + SkipIfRemote("manifest push to dir not supported in remote mode") + + session := podmanTest.Podman([]string{"manifest", "create", "foo", imageList}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + dest := filepath.Join(podmanTest.TempDir, "pushed") + err := os.MkdirAll(dest, os.ModePerm) + Expect(err).To(BeNil()) + defer func() { + os.RemoveAll(dest) + }() + + session = podmanTest.Podman([]string{"push", "foo", "-q", "dir:" + dest}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + Expect(session.ErrorToString()).To(BeEmpty()) + + session = podmanTest.Podman([]string{"push", "foo", "dir:" + dest}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + output := session.ErrorToString() + Expect(output).To(ContainSubstring("Writing manifest list to image destination")) + Expect(output).To(ContainSubstring("Storing list signatures")) + }) + It("authenticated push", func() { registryOptions := &podmanRegistry.Options{ Image: "docker-archive:" + imageTarPath(REGISTRY_IMAGE), diff --git a/test/e2e/pull_test.go b/test/e2e/pull_test.go index 12f14fdc8..ba717f393 100644 --- a/test/e2e/pull_test.go +++ b/test/e2e/pull_test.go @@ -545,4 +545,18 @@ var _ = Describe("Podman pull", func() { Expect(data[0]).To(HaveField("Os", runtime.GOOS)) Expect(data[0]).To(HaveField("Architecture", "arm64")) }) + + It("podman pull progress", func() { + session := podmanTest.Podman([]string{"pull", ALPINE}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + output := session.ErrorToString() + Expect(output).To(ContainSubstring("Getting image source signatures")) + Expect(output).To(ContainSubstring("Copying blob ")) + + session = podmanTest.Podman([]string{"pull", "-q", ALPINE}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + Expect(session.ErrorToString()).To(BeEmpty()) + }) }) -- cgit v1.2.3-54-g00ecf From e00272cd99ec7ccfc73ccf1e67e123e98f2ab3f0 Mon Sep 17 00:00:00 2001 From: Aditya R Date: Thu, 25 Aug 2022 12:10:53 +0530 Subject: remote: fix implementation of build with --userns=auto for API `podman-remote` and Libpod API does not supports build with `--userns=auto` since `IDMappingOptions` were not implemented for API and bindings, following PR implements passing `IDMappingOptions` via bindings to API. Closes: https://github.com/containers/podman/issues/15476 Signed-off-by: Aditya R --- pkg/api/handlers/compat/images_build.go | 10 ++++++++++ pkg/bindings/images/build.go | 7 +++++++ test/e2e/build/Containerfile.userns-auto | 2 ++ test/e2e/run_userns_test.go | 30 ++++++++++++++++++++++++++++++ 4 files changed, 49 insertions(+) create mode 100644 test/e2e/build/Containerfile.userns-auto (limited to 'pkg/bindings') diff --git a/pkg/api/handlers/compat/images_build.go b/pkg/api/handlers/compat/images_build.go index 020991cc7..7ba1029a7 100644 --- a/pkg/api/handlers/compat/images_build.go +++ b/pkg/api/handlers/compat/images_build.go @@ -101,6 +101,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { ForceRm bool `schema:"forcerm"` From string `schema:"from"` HTTPProxy bool `schema:"httpproxy"` + IDMappingOptions string `schema:"idmappingoptions"` IdentityLabel bool `schema:"identitylabel"` Ignore bool `schema:"ignore"` Isolation string `schema:"isolation"` @@ -389,6 +390,14 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { } } + var idMappingOptions buildahDefine.IDMappingOptions + if _, found := r.URL.Query()["idmappingoptions"]; found { + if err := json.Unmarshal([]byte(query.IDMappingOptions), &idMappingOptions); err != nil { + utils.BadRequest(w, "idmappingoptions", query.IDMappingOptions, err) + return + } + } + var cacheFrom reference.Named if _, found := r.URL.Query()["cachefrom"]; found { cacheFrom, err = parse.RepoNameToNamedReference(query.CacheFrom) @@ -644,6 +653,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { Excludes: excludes, ForceRmIntermediateCtrs: query.ForceRm, From: fromImage, + IDMappingOptions: &idMappingOptions, IgnoreUnrecognizedInstructions: query.Ignore, Isolation: isolation, Jobs: &jobs, diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go index 2615bc516..8348ac54b 100644 --- a/pkg/bindings/images/build.go +++ b/pkg/bindings/images/build.go @@ -88,6 +88,13 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO } params.Set("additionalbuildcontexts", string(additionalBuildContextMap)) } + if options.IDMappingOptions != nil { + idmappingsOptions, err := jsoniter.Marshal(options.IDMappingOptions) + if err != nil { + return nil, err + } + params.Set("idmappingoptions", string(idmappingsOptions)) + } if buildArgs := options.Args; len(buildArgs) > 0 { bArgs, err := jsoniter.MarshalToString(buildArgs) if err != nil { diff --git a/test/e2e/build/Containerfile.userns-auto b/test/e2e/build/Containerfile.userns-auto new file mode 100644 index 000000000..921610982 --- /dev/null +++ b/test/e2e/build/Containerfile.userns-auto @@ -0,0 +1,2 @@ +FROM alpine +RUN cat /proc/self/uid_map diff --git a/test/e2e/run_userns_test.go b/test/e2e/run_userns_test.go index f247b2dac..62e512d3a 100644 --- a/test/e2e/run_userns_test.go +++ b/test/e2e/run_userns_test.go @@ -8,6 +8,7 @@ import ( "strings" . "github.com/containers/podman/v4/test/utils" + "github.com/containers/storage" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" . "github.com/onsi/gomega/gexec" @@ -42,6 +43,33 @@ var _ = Describe("Podman UserNS support", func() { }) + // Note: Lot of tests for build with --userns=auto are already there in buildah + // but they are skipped in podman CI because bud tests are executed in rootfull + // environment ( where mappings for the `containers` user is not present in /etc/subuid ) + // causing them to skip hence this is a redundant test for sanity to make sure + // we don't break this feature for podman-remote. + It("podman build with --userns=auto", func() { + u, err := user.Current() + Expect(err).To(BeNil()) + name := u.Name + if name == "root" { + name = "containers" + } + content, err := ioutil.ReadFile("/etc/subuid") + if err != nil { + Skip("cannot read /etc/subuid") + } + if !strings.Contains(string(content), name) { + Skip("cannot find mappings for the current user") + } + session := podmanTest.Podman([]string{"build", "-f", "build/Containerfile.userns-auto", "-t", "test", "--userns=auto"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + // `1024` is the default size or length of the range of user IDs + // that is mapped between the two user namespaces by --userns=auto. + Expect(session.OutputToString()).To(ContainSubstring(fmt.Sprintf("%d", storage.AutoUserNsMinSize))) + }) + It("podman uidmapping and gidmapping", func() { session := podmanTest.Podman([]string{"run", "--uidmap=0:100:5000", "--gidmap=0:200:5000", "alpine", "echo", "hello"}) session.WaitWithDefaultTimeout() @@ -157,6 +185,8 @@ var _ = Describe("Podman UserNS support", func() { session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) l := session.OutputToString() + // `1024` is the default size or length of the range of user IDs + // that is mapped between the two user namespaces by --userns=auto. Expect(l).To(ContainSubstring("1024")) m[l] = l } -- cgit v1.2.3-54-g00ecf