From 0dbbb1cb3f6ed2983105620bc49191e3b0436f37 Mon Sep 17 00:00:00 2001 From: Toshiki Sonoda Date: Fri, 12 Aug 2022 09:22:53 +0900 Subject: Add restart --cidfile, --filter --cidfile : Read container ID from the specified file and restart the container. --filter : restart the filtered container. Signed-off-by: Toshiki Sonoda --- pkg/domain/entities/containers.go | 6 ++++-- pkg/domain/infra/abi/containers.go | 27 +++++++++++++++++++-------- pkg/domain/infra/tunnel/containers.go | 11 ++++++++--- 3 files changed, 31 insertions(+), 13 deletions(-) (limited to 'pkg/domain') diff --git a/pkg/domain/entities/containers.go b/pkg/domain/entities/containers.go index 3ba507750..91ccdc2b2 100644 --- a/pkg/domain/entities/containers.go +++ b/pkg/domain/entities/containers.go @@ -119,6 +119,7 @@ type KillReport struct { } type RestartOptions struct { + Filters map[string][]string All bool Latest bool Running bool @@ -126,8 +127,9 @@ type RestartOptions struct { } type RestartReport struct { - Err error - Id string //nolint:revive,stylecheck + Err error + Id string //nolint:revive,stylecheck + RawInput string } type RmOptions struct { diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index 5b5bc665e..08d845d70 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -309,31 +309,42 @@ func (ic *ContainerEngine) ContainerKill(ctx context.Context, namesOrIds []strin func (ic *ContainerEngine) ContainerRestart(ctx context.Context, namesOrIds []string, options entities.RestartOptions) ([]*entities.RestartReport, error) { var ( - ctrs []*libpod.Container - err error + ctrs []*libpod.Container + err error + rawInputs = []string{} ) if options.Running { ctrs, err = ic.Libpod.GetRunningContainers() + for _, candidate := range ctrs { + rawInputs = append(rawInputs, candidate.ID()) + } + if err != nil { return nil, err } } else { - ctrs, err = getContainersByContext(options.All, options.Latest, namesOrIds, ic.Libpod) + ctrs, rawInputs, err = getContainersAndInputByContext(options.All, options.Latest, namesOrIds, options.Filters, ic.Libpod) if err != nil { return nil, err } } - + idToRawInput := map[string]string{} + if len(rawInputs) == len(ctrs) { + for i := range ctrs { + idToRawInput[ctrs[i].ID()] = rawInputs[i] + } + } reports := make([]*entities.RestartReport, 0, len(ctrs)) - for _, con := range ctrs { - timeout := con.StopTimeout() + for _, c := range ctrs { + timeout := c.StopTimeout() if options.Timeout != nil { timeout = *options.Timeout } reports = append(reports, &entities.RestartReport{ - Id: con.ID(), - Err: con.RestartWithTimeout(ctx, timeout), + Id: c.ID(), + Err: c.RestartWithTimeout(ctx, timeout), + RawInput: idToRawInput[c.ID()], }) } return reports, nil diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go index d49f029d5..046509140 100644 --- a/pkg/domain/infra/tunnel/containers.go +++ b/pkg/domain/infra/tunnel/containers.go @@ -183,17 +183,22 @@ func (ic *ContainerEngine) ContainerRestart(ctx context.Context, namesOrIds []st if to := opts.Timeout; to != nil { options.WithTimeout(int(*to)) } - ctrs, err := getContainersByContext(ic.ClientCtx, opts.All, false, namesOrIds) + ctrs, rawInputs, err := getContainersAndInputByContext(ic.ClientCtx, opts.All, false, namesOrIds, opts.Filters) if err != nil { return nil, err } + idToRawInput := map[string]string{} + for i := range ctrs { + idToRawInput[ctrs[i].ID] = rawInputs[i] + } for _, c := range ctrs { if opts.Running && c.State != define.ContainerStateRunning.String() { continue } reports = append(reports, &entities.RestartReport{ - Id: c.ID, - Err: containers.Restart(ic.ClientCtx, c.ID, options), + Id: c.ID, + Err: containers.Restart(ic.ClientCtx, c.ID, options), + RawInput: idToRawInput[c.ID], }) } return reports, nil -- cgit v1.2.3-54-g00ecf From 7e7a79b075f7d65657d95169f02c2c1c03198b93 Mon Sep 17 00:00:00 2001 From: Nalin Dahyabhai Date: Tue, 16 Aug 2022 18:30:19 -0400 Subject: podman manifest create: accept --amend and --insecure flags Accept a --amend flag in `podman manifest create`, and treat `--insecure` as we would `--tls-verify=false` in `podman manifest`'s "add", "create", and "push" subcommands. Signed-off-by: Nalin Dahyabhai --- cmd/podman/manifest/add.go | 12 ++++++++- cmd/podman/manifest/create.go | 32 ++++++++++++++++++++++-- cmd/podman/manifest/push.go | 15 ++++++++--- docs/source/markdown/podman-manifest-create.1.md | 12 +++++++++ pkg/api/handlers/libpod/manifests.go | 3 ++- pkg/api/server/register_manifest.go | 4 +++ pkg/bindings/manifests/types.go | 3 ++- pkg/bindings/manifests/types_create_options.go | 15 +++++++++++ pkg/domain/entities/manifest.go | 5 ++++ pkg/domain/infra/abi/manifest.go | 10 +++++++- pkg/domain/infra/tunnel/manifest.go | 2 +- test/e2e/manifest_test.go | 8 ++++++ 12 files changed, 111 insertions(+), 10 deletions(-) (limited to 'pkg/domain') diff --git a/cmd/podman/manifest/add.go b/cmd/podman/manifest/add.go index 35583ffcb..09a1a9a36 100644 --- a/cmd/podman/manifest/add.go +++ b/cmd/podman/manifest/add.go @@ -2,6 +2,7 @@ package manifest import ( "context" + "errors" "fmt" "github.com/containers/common/pkg/auth" @@ -20,6 +21,7 @@ type manifestAddOptsWrapper struct { entities.ManifestAddOptions TLSVerifyCLI bool // CLI only + Insecure bool // CLI only CredentialsCLI string } @@ -77,6 +79,8 @@ func init() { flags.StringVar(&manifestAddOpts.OSVersion, osVersionFlagName, "", "override the OS `version` of the specified image") _ = addCmd.RegisterFlagCompletionFunc(osVersionFlagName, completion.AutocompleteNone) + flags.BoolVar(&manifestAddOpts.Insecure, "insecure", false, "neither require HTTPS nor verify certificates when accessing the registry") + _ = flags.MarkHidden("insecure") flags.BoolVar(&manifestAddOpts.TLSVerifyCLI, "tls-verify", true, "require HTTPS and verify certificates when accessing the registry") variantFlagName := "variant" @@ -89,7 +93,7 @@ func init() { } func add(cmd *cobra.Command, args []string) error { - if err := auth.CheckAuthFile(manifestPushOpts.Authfile); err != nil { + if err := auth.CheckAuthFile(manifestAddOpts.Authfile); err != nil { return err } @@ -109,6 +113,12 @@ func add(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("tls-verify") { manifestAddOpts.SkipTLSVerify = types.NewOptionalBool(!manifestAddOpts.TLSVerifyCLI) } + if cmd.Flags().Changed("insecure") { + if manifestAddOpts.SkipTLSVerify != types.OptionalBoolUndefined { + return errors.New("--insecure may not be used with --tls-verify") + } + manifestAddOpts.SkipTLSVerify = types.NewOptionalBool(manifestAddOpts.Insecure) + } listID, err := registry.ImageEngine().ManifestAdd(context.Background(), args[0], args[1:], manifestAddOpts.ManifestAddOptions) if err != nil { diff --git a/cmd/podman/manifest/create.go b/cmd/podman/manifest/create.go index 435b4a57c..0a0ea1d88 100644 --- a/cmd/podman/manifest/create.go +++ b/cmd/podman/manifest/create.go @@ -1,16 +1,26 @@ package manifest import ( + "errors" "fmt" + "github.com/containers/image/v5/types" "github.com/containers/podman/v4/cmd/podman/common" "github.com/containers/podman/v4/cmd/podman/registry" "github.com/containers/podman/v4/pkg/domain/entities" "github.com/spf13/cobra" ) +// manifestCreateOptsWrapper wraps entities.ManifestCreateOptions and prevents leaking +// CLI-only fields into the API types. +type manifestCreateOptsWrapper struct { + entities.ManifestCreateOptions + + TLSVerifyCLI, Insecure bool // CLI only +} + var ( - manifestCreateOpts = entities.ManifestCreateOptions{} + manifestCreateOpts = manifestCreateOptsWrapper{} createCmd = &cobra.Command{ Use: "create [options] LIST [IMAGE...]", Short: "Create manifest list or image index", @@ -32,10 +42,28 @@ func init() { }) flags := createCmd.Flags() flags.BoolVar(&manifestCreateOpts.All, "all", false, "add all of the lists' images if the images to add are lists") + flags.BoolVar(&manifestCreateOpts.Amend, "amend", false, "modify an existing list if one with the desired name already exists") + flags.BoolVar(&manifestCreateOpts.Insecure, "insecure", false, "neither require HTTPS nor verify certificates when accessing the registry") + _ = flags.MarkHidden("insecure") + flags.BoolVar(&manifestCreateOpts.TLSVerifyCLI, "tls-verify", true, "require HTTPS and verify certificates when accessing the registry") } func create(cmd *cobra.Command, args []string) error { - imageID, err := registry.ImageEngine().ManifestCreate(registry.Context(), args[0], args[1:], manifestCreateOpts) + // TLS verification in c/image is controlled via a `types.OptionalBool` + // which allows for distinguishing among set-true, set-false, unspecified + // which is important to implement a sane way of dealing with defaults of + // boolean CLI flags. + if cmd.Flags().Changed("tls-verify") { + manifestCreateOpts.SkipTLSVerify = types.NewOptionalBool(!manifestCreateOpts.TLSVerifyCLI) + } + if cmd.Flags().Changed("insecure") { + if manifestCreateOpts.SkipTLSVerify != types.OptionalBoolUndefined { + return errors.New("--insecure may not be used with --tls-verify") + } + manifestCreateOpts.SkipTLSVerify = types.NewOptionalBool(manifestCreateOpts.Insecure) + } + + imageID, err := registry.ImageEngine().ManifestCreate(registry.Context(), args[0], args[1:], manifestCreateOpts.ManifestCreateOptions) if err != nil { return err } diff --git a/cmd/podman/manifest/push.go b/cmd/podman/manifest/push.go index 756ed2a74..fd67769b8 100644 --- a/cmd/podman/manifest/push.go +++ b/cmd/podman/manifest/push.go @@ -1,6 +1,7 @@ package manifest import ( + "errors" "fmt" "io/ioutil" @@ -20,9 +21,9 @@ import ( type manifestPushOptsWrapper struct { entities.ImagePushOptions - TLSVerifyCLI bool // CLI only - CredentialsCLI string - SignPassphraseFileCLI string + TLSVerifyCLI, Insecure bool // CLI only + CredentialsCLI string + SignPassphraseFileCLI string } var ( @@ -82,6 +83,8 @@ func init() { _ = pushCmd.RegisterFlagCompletionFunc(signPassphraseFileFlagName, completion.AutocompleteDefault) flags.BoolVar(&manifestPushOpts.TLSVerifyCLI, "tls-verify", true, "require HTTPS and verify certificates when accessing the registry") + flags.BoolVar(&manifestPushOpts.Insecure, "insecure", false, "neither require HTTPS nor verify certificates when accessing the registry") + _ = flags.MarkHidden("insecure") flags.BoolVarP(&manifestPushOpts.Quiet, "quiet", "q", false, "don't output progress information when pushing lists") flags.SetNormalizeFunc(utils.AliasFlags) @@ -130,6 +133,12 @@ func push(cmd *cobra.Command, args []string) error { if cmd.Flags().Changed("tls-verify") { manifestPushOpts.SkipTLSVerify = types.NewOptionalBool(!manifestPushOpts.TLSVerifyCLI) } + if cmd.Flags().Changed("insecure") { + if manifestPushOpts.SkipTLSVerify != types.OptionalBoolUndefined { + return errors.New("--insecure may not be used with --tls-verify") + } + manifestPushOpts.SkipTLSVerify = types.NewOptionalBool(manifestPushOpts.Insecure) + } digest, err := registry.ImageEngine().ManifestPush(registry.Context(), args[0], args[1], manifestPushOpts.ImagePushOptions) if err != nil { return err diff --git a/docs/source/markdown/podman-manifest-create.1.md b/docs/source/markdown/podman-manifest-create.1.md index 77a4b9db6..f2aac6069 100644 --- a/docs/source/markdown/podman-manifest-create.1.md +++ b/docs/source/markdown/podman-manifest-create.1.md @@ -22,11 +22,23 @@ If any of the images which should be added to the new list or index are themselves lists or indexes, add all of their contents. By default, only one image from such a list will be added to the newly-created list or index. +#### **--amend** + +If a manifest list named *listnameorindexname* already exists, modify the +preexisting list instead of exiting with an error. The contents of +*listnameorindexname* are not modified if no *imagename*s are given. + +#### **--tls-verify** + +Require HTTPS and verify certificates when talking to container registries. (defaults to true) + ## EXAMPLES ``` podman manifest create mylist:v1.11 9cfd24048d5fc80903f088f1531a21bff01172abe66effa8941a4c2308dc745f +podman manifest create --amend mylist:v1.11 +9cfd24048d5fc80903f088f1531a21bff01172abe66effa8941a4c2308dc745f ``` ``` diff --git a/pkg/api/handlers/libpod/manifests.go b/pkg/api/handlers/libpod/manifests.go index b0c93f3b9..fa83bbfe1 100644 --- a/pkg/api/handlers/libpod/manifests.go +++ b/pkg/api/handlers/libpod/manifests.go @@ -36,6 +36,7 @@ func ManifestCreate(w http.ResponseWriter, r *http.Request) { Name string `schema:"name"` Images []string `schema:"images"` All bool `schema:"all"` + Amend bool `schema:"amend"` }{ // Add defaults here once needed. } @@ -70,7 +71,7 @@ func ManifestCreate(w http.ResponseWriter, r *http.Request) { imageEngine := abi.ImageEngine{Libpod: runtime} - createOptions := entities.ManifestCreateOptions{All: query.All} + createOptions := entities.ManifestCreateOptions{All: query.All, Amend: query.Amend} manID, err := imageEngine.ManifestCreate(r.Context(), query.Name, query.Images, createOptions) if err != nil { utils.InternalServerError(w, err) diff --git a/pkg/api/server/register_manifest.go b/pkg/api/server/register_manifest.go index c22479cf9..7a55eaefe 100644 --- a/pkg/api/server/register_manifest.go +++ b/pkg/api/server/register_manifest.go @@ -117,6 +117,10 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // name: all // type: boolean // description: add all contents if given list + // - in: query + // name: amend + // type: boolean + // description: modify an existing list if one with the desired name already exists // - in: body // name: options // description: options for new manifest diff --git a/pkg/bindings/manifests/types.go b/pkg/bindings/manifests/types.go index e23ef798d..5f2557fe1 100644 --- a/pkg/bindings/manifests/types.go +++ b/pkg/bindings/manifests/types.go @@ -8,7 +8,8 @@ type InspectOptions struct { //go:generate go run ../generator/generator.go CreateOptions // CreateOptions are optional options for creating manifests type CreateOptions struct { - All *bool + All *bool + Amend *bool } //go:generate go run ../generator/generator.go ExistsOptions diff --git a/pkg/bindings/manifests/types_create_options.go b/pkg/bindings/manifests/types_create_options.go index 960332a82..09942c00a 100644 --- a/pkg/bindings/manifests/types_create_options.go +++ b/pkg/bindings/manifests/types_create_options.go @@ -31,3 +31,18 @@ func (o *CreateOptions) GetAll() bool { } return *o.All } + +// WithAmend set field Amend to given value +func (o *CreateOptions) WithAmend(value bool) *CreateOptions { + o.Amend = &value + return o +} + +// GetAmend returns value of field Amend +func (o *CreateOptions) GetAmend() bool { + if o.Amend == nil { + var z bool + return z + } + return *o.Amend +} diff --git a/pkg/domain/entities/manifest.go b/pkg/domain/entities/manifest.go index 126b76c62..f17079271 100644 --- a/pkg/domain/entities/manifest.go +++ b/pkg/domain/entities/manifest.go @@ -4,7 +4,12 @@ import "github.com/containers/image/v5/types" // ManifestCreateOptions provides model for creating manifest type ManifestCreateOptions struct { + // True when adding lists to include all images All bool `schema:"all"` + // Amend an extant list if there's already one with the desired name + Amend bool `schema:"amend"` + // Should TLS registry certificate be verified? + SkipTLSVerify types.OptionalBool `json:"-" schema:"-"` } // ManifestAddOptions provides model for adding digests to manifest list diff --git a/pkg/domain/infra/abi/manifest.go b/pkg/domain/infra/abi/manifest.go index e0c11267e..7e8c86526 100644 --- a/pkg/domain/infra/abi/manifest.go +++ b/pkg/domain/infra/abi/manifest.go @@ -32,7 +32,15 @@ func (ir *ImageEngine) ManifestCreate(ctx context.Context, name string, images [ manifestList, err := ir.Libpod.LibimageRuntime().CreateManifestList(name) if err != nil { - return "", err + if errors.Is(err, storage.ErrDuplicateName) && opts.Amend { + amendList, amendErr := ir.Libpod.LibimageRuntime().LookupManifestList(name) + if amendErr != nil { + return "", err + } + manifestList = amendList + } else { + return "", err + } } addOptions := &libimage.ManifestListAddOptions{All: opts.All} diff --git a/pkg/domain/infra/tunnel/manifest.go b/pkg/domain/infra/tunnel/manifest.go index 2a514861d..2e6134051 100644 --- a/pkg/domain/infra/tunnel/manifest.go +++ b/pkg/domain/infra/tunnel/manifest.go @@ -15,7 +15,7 @@ import ( // ManifestCreate implements manifest create via ImageEngine func (ir *ImageEngine) ManifestCreate(ctx context.Context, name string, images []string, opts entities.ManifestCreateOptions) (string, error) { - options := new(manifests.CreateOptions).WithAll(opts.All) + options := new(manifests.CreateOptions).WithAll(opts.All).WithAmend(opts.Amend) imageID, err := manifests.Create(ir.ClientCtx, name, images, options) if err != nil { return imageID, fmt.Errorf("error creating manifest: %w", err) diff --git a/test/e2e/manifest_test.go b/test/e2e/manifest_test.go index ee954a1a4..145a016ea 100644 --- a/test/e2e/manifest_test.go +++ b/test/e2e/manifest_test.go @@ -49,6 +49,14 @@ var _ = Describe("Podman manifest", func() { session := podmanTest.Podman([]string{"manifest", "create", "foo"}) session.WaitWithDefaultTimeout() Expect(session).Should(Exit(0)) + + session = podmanTest.Podman([]string{"manifest", "create", "foo"}) + session.WaitWithDefaultTimeout() + Expect(session).To(ExitWithError()) + + session = podmanTest.Podman([]string{"manifest", "create", "--amend", "foo"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) }) It("create w/ image", func() { -- cgit v1.2.3-54-g00ecf From 3bf52aa338b33de719e087e15402081568453284 Mon Sep 17 00:00:00 2001 From: Vladimir Kochnev Date: Fri, 19 Aug 2022 00:41:22 +0300 Subject: Add ProgressWriter to PullOptions Signed-off-by: Vladimir Kochnev --- cmd/podman/images/pull.go | 5 +++++ pkg/bindings/images/pull.go | 13 ++++++++----- pkg/bindings/images/push.go | 9 +++++---- pkg/bindings/images/types.go | 2 ++ pkg/bindings/images/types_pull_options.go | 16 ++++++++++++++++ pkg/bindings/manifests/manifests.go | 6 ++++-- pkg/bindings/test/images_test.go | 24 ++++++++++++++++++++++-- pkg/bindings/test/manifests_test.go | 23 ++++++++++++++++++++--- pkg/domain/entities/images.go | 2 ++ pkg/domain/infra/abi/images.go | 3 ++- pkg/domain/infra/tunnel/images.go | 1 + test/e2e/manifest_test.go | 27 +++++++++++++++++++++++++++ test/e2e/pull_test.go | 14 ++++++++++++++ 13 files changed, 128 insertions(+), 17 deletions(-) (limited to 'pkg/domain') diff --git a/cmd/podman/images/pull.go b/cmd/podman/images/pull.go index 8211ceba5..fe9d1e9b6 100644 --- a/cmd/podman/images/pull.go +++ b/cmd/podman/images/pull.go @@ -155,6 +155,11 @@ func imagePull(cmd *cobra.Command, args []string) error { pullOptions.Username = creds.Username pullOptions.Password = creds.Password } + + if !pullOptions.Quiet { + pullOptions.Writer = os.Stderr + } + // Let's do all the remaining Yoga in the API to prevent us from // scattering logic across (too) many parts of the code. var errs utils.OutputErrors diff --git a/pkg/bindings/images/pull.go b/pkg/bindings/images/pull.go index 1a4aa3038..109981c63 100644 --- a/pkg/bindings/images/pull.go +++ b/pkg/bindings/images/pull.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "os" "strconv" @@ -57,10 +56,14 @@ func Pull(ctx context.Context, rawImage string, options *PullOptions) ([]string, return nil, response.Process(err) } - // Historically pull writes status to stderr - stderr := io.Writer(os.Stderr) + var writer io.Writer if options.GetQuiet() { - stderr = ioutil.Discard + writer = io.Discard + } else if progressWriter := options.GetProgressWriter(); progressWriter != nil { + writer = progressWriter + } else { + // Historically push writes status to stderr + writer = os.Stderr } dec := json.NewDecoder(response.Body) @@ -84,7 +87,7 @@ func Pull(ctx context.Context, rawImage string, options *PullOptions) ([]string, switch { case report.Stream != "": - fmt.Fprint(stderr, report.Stream) + fmt.Fprint(writer, report.Stream) case report.Error != "": pullErrors = append(pullErrors, errors.New(report.Error)) case len(report.Images) > 0: diff --git a/pkg/bindings/images/push.go b/pkg/bindings/images/push.go index 5069dd780..f1e059f8c 100644 --- a/pkg/bindings/images/push.go +++ b/pkg/bindings/images/push.go @@ -6,7 +6,6 @@ import ( "errors" "fmt" "io" - "io/ioutil" "net/http" "os" "strconv" @@ -58,12 +57,14 @@ func Push(ctx context.Context, source string, destination string, options *PushO return response.Process(err) } - // Historically push writes status to stderr - writer := io.Writer(os.Stderr) + var writer io.Writer if options.GetQuiet() { - writer = ioutil.Discard + writer = io.Discard } else if progressWriter := options.GetProgressWriter(); progressWriter != nil { writer = progressWriter + } else { + // Historically push writes status to stderr + writer = os.Stderr } dec := json.NewDecoder(response.Body) diff --git a/pkg/bindings/images/types.go b/pkg/bindings/images/types.go index 7b28c499e..3ecfb9e09 100644 --- a/pkg/bindings/images/types.go +++ b/pkg/bindings/images/types.go @@ -182,6 +182,8 @@ type PullOptions struct { Policy *string // Password for authenticating against the registry. Password *string + // ProgressWriter is a writer where pull progress are sent. + ProgressWriter *io.Writer // Quiet can be specified to suppress pull progress when pulling. Ignored // for remote calls. Quiet *bool diff --git a/pkg/bindings/images/types_pull_options.go b/pkg/bindings/images/types_pull_options.go index 4cd525185..c1a88fd9e 100644 --- a/pkg/bindings/images/types_pull_options.go +++ b/pkg/bindings/images/types_pull_options.go @@ -2,6 +2,7 @@ package images import ( + "io" "net/url" "github.com/containers/podman/v4/pkg/bindings/internal/util" @@ -107,6 +108,21 @@ func (o *PullOptions) GetPassword() string { return *o.Password } +// WithProgressWriter set field ProgressWriter to given value +func (o *PullOptions) WithProgressWriter(value io.Writer) *PullOptions { + o.ProgressWriter = &value + return o +} + +// GetProgressWriter returns value of field ProgressWriter +func (o *PullOptions) GetProgressWriter() io.Writer { + if o.ProgressWriter == nil { + var z io.Writer + return z + } + return *o.ProgressWriter +} + // WithQuiet set field Quiet to given value func (o *PullOptions) WithQuiet(value bool) *PullOptions { o.Quiet = &value diff --git a/pkg/bindings/manifests/manifests.go b/pkg/bindings/manifests/manifests.go index 49e4089f5..0163d21a0 100644 --- a/pkg/bindings/manifests/manifests.go +++ b/pkg/bindings/manifests/manifests.go @@ -182,12 +182,14 @@ func Push(ctx context.Context, name, destination string, options *images.PushOpt return "", response.Process(err) } - // Historically push writes status to stderr - writer := io.Writer(os.Stderr) + var writer io.Writer if options.GetQuiet() { writer = io.Discard } else if progressWriter := options.GetProgressWriter(); progressWriter != nil { writer = progressWriter + } else { + // Historically push writes status to stderr + writer = os.Stderr } dec := json.NewDecoder(response.Body) diff --git a/pkg/bindings/test/images_test.go b/pkg/bindings/test/images_test.go index 9c9796661..53c5a1e83 100644 --- a/pkg/bindings/test/images_test.go +++ b/pkg/bindings/test/images_test.go @@ -1,11 +1,14 @@ package bindings_test import ( + "bytes" + "fmt" "net/http" "os" "path/filepath" "time" + podmanRegistry "github.com/containers/podman/v4/hack/podman-registry-go" "github.com/containers/podman/v4/pkg/bindings" "github.com/containers/podman/v4/pkg/bindings/containers" "github.com/containers/podman/v4/pkg/bindings/images" @@ -362,9 +365,14 @@ var _ = Describe("Podman images", func() { It("Image Pull", func() { rawImage := "docker.io/library/busybox:latest" - pulledImages, err := images.Pull(bt.conn, rawImage, nil) + var writer bytes.Buffer + pullOpts := new(images.PullOptions).WithProgressWriter(&writer) + pulledImages, err := images.Pull(bt.conn, rawImage, pullOpts) Expect(err).NotTo(HaveOccurred()) Expect(len(pulledImages)).To(Equal(1)) + output := writer.String() + Expect(output).To(ContainSubstring("Trying to pull ")) + Expect(output).To(ContainSubstring("Getting image source signatures")) exists, err := images.Exists(bt.conn, rawImage, nil) Expect(err).NotTo(HaveOccurred()) @@ -380,7 +388,19 @@ var _ = Describe("Podman images", func() { }) It("Image Push", func() { - Skip("TODO: implement test for image push to registry") + registry, err := podmanRegistry.Start() + Expect(err).To(BeNil()) + + var writer bytes.Buffer + pushOpts := new(images.PushOptions).WithUsername(registry.User).WithPassword(registry.Password).WithSkipTLSVerify(true).WithProgressWriter(&writer).WithQuiet(false) + err = images.Push(bt.conn, alpine.name, fmt.Sprintf("localhost:%s/test:latest", registry.Port), pushOpts) + Expect(err).ToNot(HaveOccurred()) + + output := writer.String() + Expect(output).To(ContainSubstring("Copying blob ")) + Expect(output).To(ContainSubstring("Copying config ")) + Expect(output).To(ContainSubstring("Writing manifest to image destination")) + Expect(output).To(ContainSubstring("Storing signatures")) }) It("Build no options", func() { diff --git a/pkg/bindings/test/manifests_test.go b/pkg/bindings/test/manifests_test.go index 6a34ef5a6..d6749f920 100644 --- a/pkg/bindings/test/manifests_test.go +++ b/pkg/bindings/test/manifests_test.go @@ -1,9 +1,12 @@ package bindings_test import ( + "bytes" + "fmt" "net/http" "time" + podmanRegistry "github.com/containers/podman/v4/hack/podman-registry-go" "github.com/containers/podman/v4/pkg/bindings" "github.com/containers/podman/v4/pkg/bindings/images" "github.com/containers/podman/v4/pkg/bindings/manifests" @@ -12,7 +15,7 @@ import ( "github.com/onsi/gomega/gexec" ) -var _ = Describe("podman manifest", func() { +var _ = Describe("Podman manifests", func() { var ( bt *bindingTest s *gexec.Session @@ -172,7 +175,21 @@ var _ = Describe("podman manifest", func() { Expect(list.Manifests[0].Platform.OS).To(Equal("foo")) }) - It("push manifest", func() { - Skip("TODO: implement test for manifest push to registry") + It("Manifest Push", func() { + registry, err := podmanRegistry.Start() + Expect(err).To(BeNil()) + + name := "quay.io/libpod/foobar:latest" + _, err = manifests.Create(bt.conn, name, []string{alpine.name}, nil) + Expect(err).ToNot(HaveOccurred()) + + var writer bytes.Buffer + pushOpts := new(images.PushOptions).WithUsername(registry.User).WithPassword(registry.Password).WithAll(true).WithSkipTLSVerify(true).WithProgressWriter(&writer).WithQuiet(false) + _, err = manifests.Push(bt.conn, name, fmt.Sprintf("localhost:%s/test:latest", registry.Port), pushOpts) + Expect(err).ToNot(HaveOccurred()) + + output := writer.String() + Expect(output).To(ContainSubstring("Writing manifest list to image destination")) + Expect(output).To(ContainSubstring("Storing list signatures")) }) }) diff --git a/pkg/domain/entities/images.go b/pkg/domain/entities/images.go index 21c1372b9..cad11b0ab 100644 --- a/pkg/domain/entities/images.go +++ b/pkg/domain/entities/images.go @@ -156,6 +156,8 @@ type ImagePullOptions struct { SkipTLSVerify types.OptionalBool // PullPolicy whether to pull new image PullPolicy config.PullPolicy + // Writer is used to display copy information including progress bars. + Writer io.Writer } // ImagePullReport is the response from pulling one or more images. diff --git a/pkg/domain/infra/abi/images.go b/pkg/domain/infra/abi/images.go index 77d1bf0db..f9839f62f 100644 --- a/pkg/domain/infra/abi/images.go +++ b/pkg/domain/infra/abi/images.go @@ -237,8 +237,9 @@ func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, options entiti pullOptions.Variant = options.Variant pullOptions.SignaturePolicyPath = options.SignaturePolicy pullOptions.InsecureSkipTLSVerify = options.SkipTLSVerify + pullOptions.Writer = options.Writer - if !options.Quiet { + if !options.Quiet && pullOptions.Writer == nil { pullOptions.Writer = os.Stderr } diff --git a/pkg/domain/infra/tunnel/images.go b/pkg/domain/infra/tunnel/images.go index bb3014099..2716aaf2a 100644 --- a/pkg/domain/infra/tunnel/images.go +++ b/pkg/domain/infra/tunnel/images.go @@ -110,6 +110,7 @@ func (ir *ImageEngine) Pull(ctx context.Context, rawImage string, opts entities. options.WithAllTags(opts.AllTags).WithAuthfile(opts.Authfile).WithArch(opts.Arch).WithOS(opts.OS) options.WithVariant(opts.Variant).WithPassword(opts.Password) options.WithQuiet(opts.Quiet).WithUsername(opts.Username).WithPolicy(opts.PullPolicy.String()) + options.WithProgressWriter(opts.Writer) if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined { if s == types.OptionalBoolTrue { options.WithSkipTLSVerify(true) diff --git a/test/e2e/manifest_test.go b/test/e2e/manifest_test.go index 1c4aad710..60b72dcaa 100644 --- a/test/e2e/manifest_test.go +++ b/test/e2e/manifest_test.go @@ -350,6 +350,33 @@ var _ = Describe("Podman manifest", func() { Expect(foundZstdFile).To(BeTrue()) }) + It("push progress", func() { + SkipIfRemote("manifest push to dir not supported in remote mode") + + session := podmanTest.Podman([]string{"manifest", "create", "foo", imageList}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + + dest := filepath.Join(podmanTest.TempDir, "pushed") + err := os.MkdirAll(dest, os.ModePerm) + Expect(err).To(BeNil()) + defer func() { + os.RemoveAll(dest) + }() + + session = podmanTest.Podman([]string{"push", "foo", "-q", "dir:" + dest}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + Expect(session.ErrorToString()).To(BeEmpty()) + + session = podmanTest.Podman([]string{"push", "foo", "dir:" + dest}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + output := session.ErrorToString() + Expect(output).To(ContainSubstring("Writing manifest list to image destination")) + Expect(output).To(ContainSubstring("Storing list signatures")) + }) + It("authenticated push", func() { registryOptions := &podmanRegistry.Options{ Image: "docker-archive:" + imageTarPath(REGISTRY_IMAGE), diff --git a/test/e2e/pull_test.go b/test/e2e/pull_test.go index 12f14fdc8..ba717f393 100644 --- a/test/e2e/pull_test.go +++ b/test/e2e/pull_test.go @@ -545,4 +545,18 @@ var _ = Describe("Podman pull", func() { Expect(data[0]).To(HaveField("Os", runtime.GOOS)) Expect(data[0]).To(HaveField("Architecture", "arm64")) }) + + It("podman pull progress", func() { + session := podmanTest.Podman([]string{"pull", ALPINE}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + output := session.ErrorToString() + Expect(output).To(ContainSubstring("Getting image source signatures")) + Expect(output).To(ContainSubstring("Copying blob ")) + + session = podmanTest.Podman([]string{"pull", "-q", ALPINE}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + Expect(session.ErrorToString()).To(BeEmpty()) + }) }) -- cgit v1.2.3-54-g00ecf From 716ac1c866787d7d6b1ee8b800da527fd5b980d0 Mon Sep 17 00:00:00 2001 From: Toshiki Sonoda Date: Tue, 23 Aug 2022 09:58:34 +0900 Subject: Refactor: About the RawInput process Refactor the RawInput process of the `rm` and `start` subcommands, like the other subcommands such as `restart, stop, etc`. [NO NEW TESTS NEEDED] Signed-off-by: Toshiki Sonoda --- cmd/podman/containers/rm.go | 7 +++++-- pkg/domain/infra/abi/containers.go | 24 +++++++++++++----------- pkg/domain/infra/tunnel/containers.go | 16 +++++++++------- 3 files changed, 27 insertions(+), 20 deletions(-) (limited to 'pkg/domain') diff --git a/cmd/podman/containers/rm.go b/cmd/podman/containers/rm.go index 1e3976389..9c760e752 100644 --- a/cmd/podman/containers/rm.go +++ b/cmd/podman/containers/rm.go @@ -149,7 +149,8 @@ func removeContainers(namesOrIDs []string, rmOptions entities.RmOptions, setExit return err } for _, r := range responses { - if r.Err != nil { + switch { + case r.Err != nil: if errors.Is(r.Err, define.ErrWillDeadlock) { logrus.Errorf("Potential deadlock detected - please run 'podman system renumber' to resolve") } @@ -160,8 +161,10 @@ func removeContainers(namesOrIDs []string, rmOptions entities.RmOptions, setExit setExitCode(r.Err) } errs = append(errs, r.Err) - } else { + case r.RawInput != "": fmt.Println(r.RawInput) + default: + fmt.Println(r.Id) } } return errs.PrintErrors() diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index 08d845d70..0a8e5bc2f 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -381,7 +381,7 @@ func (ic *ContainerEngine) ContainerRm(ctx context.Context, namesOrIds []string, // this will fail and code will fall through to removing the container from libpod.` tmpNames := []string{} for _, ctr := range names { - report := reports.RmReport{Id: ctr, RawInput: ctr} + report := reports.RmReport{Id: ctr} report.Err = ic.Libpod.RemoveStorageContainer(ctr, options.Force) //nolint:gocritic if report.Err == nil { @@ -938,13 +938,15 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri if err != nil { return nil, err } + idToRawInput := map[string]string{} + if len(rawInputs) == len(ctrs) { + for i := range ctrs { + idToRawInput[ctrs[i].ID()] = rawInputs[i] + } + } // There can only be one container if attach was used for i := range ctrs { ctr := ctrs[i] - rawInput := ctr.ID() - if !options.All { - rawInput = rawInputs[i] - } ctrState, err := ctr.State() if err != nil { return nil, err @@ -958,7 +960,7 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri // Exit cleanly immediately reports = append(reports, &entities.ContainerStartReport{ Id: ctr.ID(), - RawInput: rawInput, + RawInput: idToRawInput[ctr.ID()], Err: nil, ExitCode: 0, }) @@ -969,7 +971,7 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri logrus.Debugf("Deadlock error: %v", err) reports = append(reports, &entities.ContainerStartReport{ Id: ctr.ID(), - RawInput: rawInput, + RawInput: idToRawInput[ctr.ID()], Err: err, ExitCode: define.ExitCode(err), }) @@ -979,7 +981,7 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri if ctrRunning { reports = append(reports, &entities.ContainerStartReport{ Id: ctr.ID(), - RawInput: rawInput, + RawInput: idToRawInput[ctr.ID()], Err: nil, ExitCode: 0, }) @@ -989,7 +991,7 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri if err != nil { reports = append(reports, &entities.ContainerStartReport{ Id: ctr.ID(), - RawInput: rawInput, + RawInput: idToRawInput[ctr.ID()], Err: err, ExitCode: exitCode, }) @@ -1004,7 +1006,7 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri exitCode = ic.GetContainerExitCode(ctx, ctr) reports = append(reports, &entities.ContainerStartReport{ Id: ctr.ID(), - RawInput: rawInput, + RawInput: idToRawInput[ctr.ID()], Err: err, ExitCode: exitCode, }) @@ -1017,7 +1019,7 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri // If the container is in a pod, also set to recursively start dependencies report := &entities.ContainerStartReport{ Id: ctr.ID(), - RawInput: rawInput, + RawInput: idToRawInput[ctr.ID()], ExitCode: 125, } if err := ctr.Start(ctx, true); err != nil { diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go index 046509140..023bee430 100644 --- a/pkg/domain/infra/tunnel/containers.go +++ b/pkg/domain/infra/tunnel/containers.go @@ -672,10 +672,16 @@ func logIfRmError(id string, err error, reports []*reports.RmReport) { func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []string, options entities.ContainerStartOptions) ([]*entities.ContainerStartReport, error) { reports := []*entities.ContainerStartReport{} var exitCode = define.ExecErrorCodeGeneric - ctrs, namesOrIds, err := getContainersAndInputByContext(ic.ClientCtx, options.All, false, namesOrIds, options.Filters) + ctrs, rawInputs, err := getContainersAndInputByContext(ic.ClientCtx, options.All, false, namesOrIds, options.Filters) if err != nil { return nil, err } + idToRawInput := map[string]string{} + if len(rawInputs) == len(ctrs) { + for i := range ctrs { + idToRawInput[ctrs[i].ID] = rawInputs[i] + } + } removeOptions := new(containers.RemoveOptions).WithVolumes(true).WithForce(false) removeContainer := func(id string) { reports, err := containers.Remove(ic.ClientCtx, id, removeOptions) @@ -683,15 +689,11 @@ func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []stri } // There can only be one container if attach was used - for i, ctr := range ctrs { + for _, ctr := range ctrs { name := ctr.ID - rawInput := ctr.ID - if !options.All { - rawInput = namesOrIds[i] - } report := entities.ContainerStartReport{ Id: name, - RawInput: rawInput, + RawInput: idToRawInput[name], ExitCode: exitCode, } ctrRunning := ctr.State == define.ContainerStateRunning.String() -- cgit v1.2.3-54-g00ecf From 0f739355635d5bc4d538cf88009d7af533e7c289 Mon Sep 17 00:00:00 2001 From: Matthew Heon Date: Tue, 23 Aug 2022 14:46:43 -0400 Subject: Add support for containers.conf volume timeouts Also, do a general cleanup of all the timeout code. Changes include: - Convert from int to *uint where possible. Timeouts cannot be negative, hence the uint change; and a timeout of 0 is valid, so we need a new way to detect that the user set a timeout (hence, pointer). - Change name in the database to avoid conflicts between new data type and old one. This will cause timeouts set with 4.2.0 to be lost, but considering nobody is using the feature at present (and the lack of validation means we could have invalid, negative timeouts in the DB) this feels safe. - Ensure volume plugin timeouts can only be used with volumes created using a plugin. Timeouts on the local driver are nonsensical. - Remove the existing test, as it did not use a volume plugin. Write a new test that does. The actual plumbing of the containers.conf timeout in is one line in volume_api.go; the remainder are the above-described cleanups. Signed-off-by: Matthew Heon --- go.mod | 2 +- go.sum | 10 ++-- libpod/define/volume_inspect.go | 2 +- libpod/options.go | 14 ++++- libpod/plugin/volume_api.go | 19 +++--- libpod/runtime.go | 2 +- libpod/runtime_volume_linux.go | 2 +- libpod/volume.go | 2 +- libpod/volume_inspect.go | 7 ++- pkg/domain/infra/abi/parse/parse.go | 5 +- test/e2e/config/containers.conf | 2 + test/e2e/volume_create_test.go | 15 ----- test/e2e/volume_plugin_test.go | 34 +++++++++++ test/testvol/util.go | 2 +- .../Microsoft/hcsshim/internal/jobobject/iocp.go | 2 +- .../hcsshim/internal/jobobject/jobobject.go | 55 +++++++++++++++--- .../Microsoft/hcsshim/internal/jobobject/limits.go | 4 +- .../Microsoft/hcsshim/internal/queue/mq.go | 61 +++++++------------- .../Microsoft/hcsshim/internal/winapi/jobobject.go | 2 +- .../Microsoft/hcsshim/internal/winapi/process.go | 2 +- .../Microsoft/hcsshim/internal/winapi/system.go | 3 +- .../hcsshim/internal/winapi/zsyscall_windows.go | 6 +- .../containers/common/libimage/inspect.go | 2 +- .../github.com/containers/common/libimage/load.go | 2 +- .../containers/common/libnetwork/cni/network.go | 14 ++++- .../common/libnetwork/network/interface.go | 1 + .../containers/common/pkg/config/config.go | 67 ++++++++++++++++++++++ .../containers/common/pkg/config/config_darwin.go | 2 + .../containers/common/pkg/config/containers.conf | 17 +++++- .../containers/common/pkg/config/default.go | 6 +- .../containers/common/pkg/config/default_darwin.go | 5 ++ .../common/pkg/config/default_freebsd.go | 5 ++ .../containers/common/pkg/config/default_linux.go | 5 ++ .../common/pkg/config/default_windows.go | 5 ++ .../common/pkg/subscriptions/subscriptions.go | 2 +- vendor/modules.txt | 6 +- 36 files changed, 284 insertions(+), 108 deletions(-) (limited to 'pkg/domain') diff --git a/go.mod b/go.mod index 635c0a17d..c0a23d962 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/containernetworking/cni v1.1.2 github.com/containernetworking/plugins v1.1.1 github.com/containers/buildah v1.27.0 - github.com/containers/common v0.49.2-0.20220817132854-f6679f170eca + github.com/containers/common v0.49.2-0.20220823130605-72a7da3358ac github.com/containers/conmon v2.0.20+incompatible github.com/containers/image/v5 v5.22.0 github.com/containers/ocicrypt v1.1.5 diff --git a/go.sum b/go.sum index 5053589c5..a476729ee 100644 --- a/go.sum +++ b/go.sum @@ -140,8 +140,9 @@ github.com/Microsoft/hcsshim v0.8.21/go.mod h1:+w2gRZ5ReXQhFOrvSQeNfhrYB/dg3oDwT github.com/Microsoft/hcsshim v0.8.22/go.mod h1:91uVCVzvX2QD16sMCenoxxXo6L1wJnLMX2PSufFMtF0= github.com/Microsoft/hcsshim v0.8.23/go.mod h1:4zegtUJth7lAvFyc6cH2gGQ5B3OFQim01nnU2M8jKDg= github.com/Microsoft/hcsshim v0.9.2/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= -github.com/Microsoft/hcsshim v0.9.3 h1:k371PzBuRrz2b+ebGuI2nVgVhgsVX60jMfSw80NECxo= github.com/Microsoft/hcsshim v0.9.3/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= +github.com/Microsoft/hcsshim v0.9.4 h1:mnUj0ivWy6UzbB1uLFqKR6F+ZyiDc7j4iGgHTpO+5+I= +github.com/Microsoft/hcsshim v0.9.4/go.mod h1:7pLA8lDk46WKDWlVsENo92gC0XFa8rbKfyFRBqxEbCc= github.com/Microsoft/hcsshim/test v0.0.0-20201218223536-d3e5debf77da/go.mod h1:5hlzMzRKMLyo42nCZ9oml8AdTlq/0cvIaBv6tK1RehU= github.com/Microsoft/hcsshim/test v0.0.0-20210227013316-43a75bb4edd3/go.mod h1:mw7qgWloBUl75W/gVH3cQszUg1+gUITj7D6NY7ywVnY= github.com/NYTimes/gziphandler v0.0.0-20170623195520-56545f4a5d46/go.mod h1:3wb06e3pkSAbeQ52E9H9iFoQsEEwGN64994WTCIhntQ= @@ -323,8 +324,9 @@ github.com/containerd/containerd v1.5.7/go.mod h1:gyvv6+ugqY25TiXxcZC3L5yOeYgEw0 github.com/containerd/containerd v1.5.8/go.mod h1:YdFSv5bTFLpG2HIYmfqDpSYYTDX+mc5qtSuYx1YUb/s= github.com/containerd/containerd v1.5.9/go.mod h1:fvQqCfadDGga5HZyn3j4+dx56qj2I9YwBrlSdalvJYQ= github.com/containerd/containerd v1.6.1/go.mod h1:1nJz5xCZPusx6jJU8Frfct988y0NpumIq9ODB0kLtoE= -github.com/containerd/containerd v1.6.6 h1:xJNPhbrmz8xAMDNoVjHy9YHtWwEQNS+CDkcIRh7t8Y0= github.com/containerd/containerd v1.6.6/go.mod h1:ZoP1geJldzCVY3Tonoz7b1IXk8rIX0Nltt5QE4OMNk0= +github.com/containerd/containerd v1.6.8 h1:h4dOFDwzHmqFEP754PgfgTeVXFnLiRc6kiqC7tplDJs= +github.com/containerd/containerd v1.6.8/go.mod h1:By6p5KqPK0/7/CgO/A6t/Gz+CUYUu2zf1hUaaymVXB0= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20190815185530-f2a389ac0a02/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20191127005431-f65d91d395eb/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= @@ -395,8 +397,8 @@ github.com/containernetworking/plugins v1.1.1/go.mod h1:Sr5TH/eBsGLXK/h71HeLfX19 github.com/containers/buildah v1.27.0 h1:LJ1ks7vKxwPzJGr5BWVvigbtVL9w7XeHtNEmiIOPJqI= github.com/containers/buildah v1.27.0/go.mod h1:anH3ExvDXRNP9zLQCrOc1vWb5CrhqLF/aYFim4tslvA= github.com/containers/common v0.49.1/go.mod h1:ueM5hT0itKqCQvVJDs+EtjornAQtrHYxQJzP2gxeGIg= -github.com/containers/common v0.49.2-0.20220817132854-f6679f170eca h1:OjhEBVpFskIJ6Vq9nikYW7M6YXfkTxOBu+EQBoCyhuM= -github.com/containers/common v0.49.2-0.20220817132854-f6679f170eca/go.mod h1:eT2iSsNzjOlF5VFLkyj9OU2SXznURvEYndsioQImuoE= +github.com/containers/common v0.49.2-0.20220823130605-72a7da3358ac h1:rLbTzosxPKrQd+EgMRxfC1WYm3azPiQfig+Lr7mCQ4k= +github.com/containers/common v0.49.2-0.20220823130605-72a7da3358ac/go.mod h1:xC4qkLfW9R+YSDknlT9xU+NDNxIw017U8AyohGtr9Ec= github.com/containers/conmon v2.0.20+incompatible h1:YbCVSFSCqFjjVwHTPINGdMX1F6JXHGTUje2ZYobNrkg= github.com/containers/conmon v2.0.20+incompatible/go.mod h1:hgwZ2mtuDrppv78a/cOBNiCm6O0UMWGx1mu7P00nu5I= github.com/containers/image/v5 v5.22.0 h1:KemxPmD4D2YYOFZN2SgoTk7nBFcnwPiPW0MqjYtknSE= diff --git a/libpod/define/volume_inspect.go b/libpod/define/volume_inspect.go index 9279812da..76120647c 100644 --- a/libpod/define/volume_inspect.go +++ b/libpod/define/volume_inspect.go @@ -57,7 +57,7 @@ type InspectVolumeData struct { // UID/GID. NeedsChown bool `json:"NeedsChown,omitempty"` // Timeout is the specified driver timeout if given - Timeout int `json:"Timeout,omitempty"` + Timeout uint `json:"Timeout,omitempty"` } type VolumeReload struct { diff --git a/libpod/options.go b/libpod/options.go index 43ed1ff78..d31741094 100644 --- a/libpod/options.go +++ b/libpod/options.go @@ -1695,14 +1695,22 @@ func withSetAnon() VolumeCreateOption { } } -// WithVolumeDriverTimeout sets the volume creation timeout period -func WithVolumeDriverTimeout(timeout int) VolumeCreateOption { +// WithVolumeDriverTimeout sets the volume creation timeout period. +// Only usable if a non-local volume driver is in use. +func WithVolumeDriverTimeout(timeout uint) VolumeCreateOption { return func(volume *Volume) error { if volume.valid { return define.ErrVolumeFinalized } - volume.config.Timeout = timeout + if volume.config.Driver == "" || volume.config.Driver == define.VolumeDriverLocal { + return fmt.Errorf("Volume driver timeout can only be used with non-local volume drivers: %w", define.ErrInvalidArg) + } + + tm := timeout + + volume.config.Timeout = &tm + return nil } } diff --git a/libpod/plugin/volume_api.go b/libpod/plugin/volume_api.go index 0a5eaae53..b13578388 100644 --- a/libpod/plugin/volume_api.go +++ b/libpod/plugin/volume_api.go @@ -3,6 +3,7 @@ package plugin import ( "bytes" "context" + "errors" "fmt" "io/ioutil" "net" @@ -13,8 +14,7 @@ import ( "sync" "time" - "errors" - + "github.com/containers/common/pkg/config" "github.com/containers/podman/v4/libpod/define" "github.com/docker/go-plugins-helpers/sdk" "github.com/docker/go-plugins-helpers/volume" @@ -40,7 +40,6 @@ var ( ) const ( - defaultTimeout = 5 * time.Second volumePluginType = "VolumeDriver" ) @@ -129,7 +128,7 @@ func validatePlugin(newPlugin *VolumePlugin) error { // GetVolumePlugin gets a single volume plugin, with the given name, at the // given path. -func GetVolumePlugin(name string, path string, timeout int) (*VolumePlugin, error) { +func GetVolumePlugin(name string, path string, timeout *uint, cfg *config.Config) (*VolumePlugin, error) { pluginsLock.Lock() defer pluginsLock.Unlock() @@ -152,13 +151,11 @@ func GetVolumePlugin(name string, path string, timeout int) (*VolumePlugin, erro // Need an HTTP client to force a Unix connection. // And since we can reuse it, might as well cache it. client := new(http.Client) - client.Timeout = defaultTimeout - // if the user specified a non-zero timeout, use their value. Else, keep the default. - if timeout != 0 { - if time.Duration(timeout)*time.Second < defaultTimeout { - logrus.Warnf("the default timeout for volume creation is %d seconds, setting a time less than that may break this feature.", defaultTimeout) - } - client.Timeout = time.Duration(timeout) * time.Second + client.Timeout = 5 * time.Second + if timeout != nil { + client.Timeout = time.Duration(*timeout) * time.Second + } else if cfg != nil { + client.Timeout = time.Duration(cfg.Engine.VolumePluginTimeout) * time.Second } // This bit borrowed from pkg/bindings/connection.go client.Transport = &http.Transport{ diff --git a/libpod/runtime.go b/libpod/runtime.go index 684f4abd7..9b97fd724 100644 --- a/libpod/runtime.go +++ b/libpod/runtime.go @@ -1097,7 +1097,7 @@ func (r *Runtime) getVolumePlugin(volConfig *VolumeConfig) (*plugin.VolumePlugin return nil, fmt.Errorf("no volume plugin with name %s available: %w", name, define.ErrMissingPlugin) } - return plugin.GetVolumePlugin(name, pluginPath, timeout) + return plugin.GetVolumePlugin(name, pluginPath, timeout, r.config) } // GetSecretsStorageDir returns the directory that the secrets manager should take diff --git a/libpod/runtime_volume_linux.go b/libpod/runtime_volume_linux.go index 1f354e41b..65f2a1005 100644 --- a/libpod/runtime_volume_linux.go +++ b/libpod/runtime_volume_linux.go @@ -184,7 +184,7 @@ func (r *Runtime) UpdateVolumePlugins(ctx context.Context) *define.VolumeReload ) for driverName, socket := range r.config.Engine.VolumePlugins { - driver, err := volplugin.GetVolumePlugin(driverName, socket, 0) + driver, err := volplugin.GetVolumePlugin(driverName, socket, nil, r.config) if err != nil { errs = append(errs, err) continue diff --git a/libpod/volume.go b/libpod/volume.go index 2e8cd77a5..a054e4032 100644 --- a/libpod/volume.go +++ b/libpod/volume.go @@ -56,7 +56,7 @@ type VolumeConfig struct { // quota tracking. DisableQuota bool `json:"disableQuota,omitempty"` // Timeout allows users to override the default driver timeout of 5 seconds - Timeout int + Timeout *uint `json:"timeout,omitempty"` } // VolumeState holds the volume's mutable state. diff --git a/libpod/volume_inspect.go b/libpod/volume_inspect.go index dd2f3fd01..c3872bca7 100644 --- a/libpod/volume_inspect.go +++ b/libpod/volume_inspect.go @@ -64,7 +64,12 @@ func (v *Volume) Inspect() (*define.InspectVolumeData, error) { data.MountCount = v.state.MountCount data.NeedsCopyUp = v.state.NeedsCopyUp data.NeedsChown = v.state.NeedsChown - data.Timeout = v.config.Timeout + + if v.config.Timeout != nil { + data.Timeout = *v.config.Timeout + } else if v.UsesVolumeDriver() { + data.Timeout = v.runtime.config.Engine.VolumePluginTimeout + } return data, nil } diff --git a/pkg/domain/infra/abi/parse/parse.go b/pkg/domain/infra/abi/parse/parse.go index 19699589b..fb2876bb2 100644 --- a/pkg/domain/infra/abi/parse/parse.go +++ b/pkg/domain/infra/abi/parse/parse.go @@ -86,8 +86,11 @@ func VolumeOptions(opts map[string]string) ([]libpod.VolumeCreateOption, error) if err != nil { return nil, fmt.Errorf("cannot convert Timeout %s to an integer: %w", splitO[1], err) } + if intTimeout < 0 { + return nil, fmt.Errorf("volume timeout cannot be negative (got %d)", intTimeout) + } logrus.Debugf("Removing timeout from options and adding WithTimeout for Timeout %d", intTimeout) - libpodOptions = append(libpodOptions, libpod.WithVolumeDriverTimeout(intTimeout)) + libpodOptions = append(libpodOptions, libpod.WithVolumeDriverTimeout(uint(intTimeout))) default: finalVal = append(finalVal, o) } diff --git a/test/e2e/config/containers.conf b/test/e2e/config/containers.conf index c33f32ab4..94bb316b1 100644 --- a/test/e2e/config/containers.conf +++ b/test/e2e/config/containers.conf @@ -61,6 +61,8 @@ no_hosts=true network_cmd_options=["allow_host_loopback=true"] service_timeout=1234 +volume_plugin_timeout = 15 + # We need to ensure each test runs on a separate plugin instance... # For now, let's just make a bunch of plugin paths and have each test use one. [engine.volume_plugins] diff --git a/test/e2e/volume_create_test.go b/test/e2e/volume_create_test.go index 7a975f6a5..499283cab 100644 --- a/test/e2e/volume_create_test.go +++ b/test/e2e/volume_create_test.go @@ -162,19 +162,4 @@ var _ = Describe("Podman volume create", func() { Expect(inspectOpts).Should(Exit(0)) Expect(inspectOpts.OutputToString()).To(Equal(optionStrFormatExpect)) }) - - It("podman create volume with o=timeout", func() { - volName := "testVol" - timeout := 10 - timeoutStr := "10" - session := podmanTest.Podman([]string{"volume", "create", "--opt", fmt.Sprintf("o=timeout=%d", timeout), volName}) - session.WaitWithDefaultTimeout() - Expect(session).Should(Exit(0)) - - inspectTimeout := podmanTest.Podman([]string{"volume", "inspect", "--format", "{{ .Timeout }}", volName}) - inspectTimeout.WaitWithDefaultTimeout() - Expect(inspectTimeout).Should(Exit(0)) - Expect(inspectTimeout.OutputToString()).To(Equal(timeoutStr)) - - }) }) diff --git a/test/e2e/volume_plugin_test.go b/test/e2e/volume_plugin_test.go index b585f8dd8..a44e75a54 100644 --- a/test/e2e/volume_plugin_test.go +++ b/test/e2e/volume_plugin_test.go @@ -256,4 +256,38 @@ Removed: Expect(session.OutputToStringArray()).To(ContainElements(localvol, vol2)) Expect(session.ErrorToString()).To(Equal("")) // make no errors are shown }) + + It("volume driver timeouts test", func() { + podmanTest.AddImageToRWStore(volumeTest) + + pluginStatePath := filepath.Join(podmanTest.TempDir, "volumes") + err := os.Mkdir(pluginStatePath, 0755) + Expect(err).ToNot(HaveOccurred()) + + // Keep this distinct within tests to avoid multiple tests using the same plugin. + pluginName := "testvol6" + plugin := podmanTest.Podman([]string{"run", "--security-opt", "label=disable", "-v", "/run/docker/plugins:/run/docker/plugins", "-v", fmt.Sprintf("%v:%v", pluginStatePath, pluginStatePath), "-d", volumeTest, "--sock-name", pluginName, "--path", pluginStatePath}) + plugin.WaitWithDefaultTimeout() + Expect(plugin).Should(Exit(0)) + + volName := "testVolume1" + create := podmanTest.Podman([]string{"volume", "create", "--driver", pluginName, volName}) + create.WaitWithDefaultTimeout() + Expect(create).Should(Exit(0)) + + volInspect := podmanTest.Podman([]string{"volume", "inspect", "--format", "{{ .Timeout }}", volName}) + volInspect.WaitWithDefaultTimeout() + Expect(volInspect).Should(Exit(0)) + Expect(volInspect.OutputToString()).To(ContainSubstring("15")) + + volName2 := "testVolume2" + create2 := podmanTest.Podman([]string{"volume", "create", "--driver", pluginName, "--opt", "o=timeout=3", volName2}) + create2.WaitWithDefaultTimeout() + Expect(create2).Should(Exit(0)) + + volInspect2 := podmanTest.Podman([]string{"volume", "inspect", "--format", "{{ .Timeout }}", volName2}) + volInspect2.WaitWithDefaultTimeout() + Expect(volInspect2).Should(Exit(0)) + Expect(volInspect2.OutputToString()).To(ContainSubstring("3")) + }) }) diff --git a/test/testvol/util.go b/test/testvol/util.go index b50bb3afb..b4961e097 100644 --- a/test/testvol/util.go +++ b/test/testvol/util.go @@ -25,5 +25,5 @@ func getPluginName(pathOrName string) string { func getPlugin(sockNameOrPath string) (*plugin.VolumePlugin, error) { path := getSocketPath(sockNameOrPath) name := getPluginName(sockNameOrPath) - return plugin.GetVolumePlugin(name, path, 0) + return plugin.GetVolumePlugin(name, path, nil, nil) } diff --git a/vendor/github.com/Microsoft/hcsshim/internal/jobobject/iocp.go b/vendor/github.com/Microsoft/hcsshim/internal/jobobject/iocp.go index 3d640ac7b..5d6acd69e 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/jobobject/iocp.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/jobobject/iocp.go @@ -57,7 +57,7 @@ func pollIOCP(ctx context.Context, iocpHandle windows.Handle) { }).Warn("failed to parse job object message") continue } - if err := msq.Write(notification); err == queue.ErrQueueClosed { + if err := msq.Enqueue(notification); err == queue.ErrQueueClosed { // Write will only return an error when the queue is closed. // The only time a queue would ever be closed is when we call `Close` on // the job it belongs to which also removes it from the jobMap, so something diff --git a/vendor/github.com/Microsoft/hcsshim/internal/jobobject/jobobject.go b/vendor/github.com/Microsoft/hcsshim/internal/jobobject/jobobject.go index 9c2726416..c9fdd921a 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/jobobject/jobobject.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/jobobject/jobobject.go @@ -68,6 +68,9 @@ type Options struct { // `UseNTVariant` specifies if we should use the `Nt` variant of Open/CreateJobObject. // Defaults to false. UseNTVariant bool + // `IOTracking` enables tracking I/O statistics on the job object. More specifically this + // calls SetInformationJobObject with the JobObjectIoAttribution class. + EnableIOTracking bool } // Create creates a job object. @@ -134,6 +137,12 @@ func Create(ctx context.Context, options *Options) (_ *JobObject, err error) { job.mq = mq } + if options.EnableIOTracking { + if err := enableIOTracking(jobHandle); err != nil { + return nil, err + } + } + return job, nil } @@ -235,7 +244,7 @@ func (job *JobObject) PollNotification() (interface{}, error) { if job.mq == nil { return nil, ErrNotRegistered } - return job.mq.ReadOrWait() + return job.mq.Dequeue() } // UpdateProcThreadAttribute updates the passed in ProcThreadAttributeList to contain what is necessary to @@ -330,7 +339,7 @@ func (job *JobObject) Pids() ([]uint32, error) { err := winapi.QueryInformationJobObject( job.handle, winapi.JobObjectBasicProcessIdList, - uintptr(unsafe.Pointer(&info)), + unsafe.Pointer(&info), uint32(unsafe.Sizeof(info)), nil, ) @@ -356,7 +365,7 @@ func (job *JobObject) Pids() ([]uint32, error) { if err = winapi.QueryInformationJobObject( job.handle, winapi.JobObjectBasicProcessIdList, - uintptr(unsafe.Pointer(&buf[0])), + unsafe.Pointer(&buf[0]), uint32(len(buf)), nil, ); err != nil { @@ -384,7 +393,7 @@ func (job *JobObject) QueryMemoryStats() (*winapi.JOBOBJECT_MEMORY_USAGE_INFORMA if err := winapi.QueryInformationJobObject( job.handle, winapi.JobObjectMemoryUsageInformation, - uintptr(unsafe.Pointer(&info)), + unsafe.Pointer(&info), uint32(unsafe.Sizeof(info)), nil, ); err != nil { @@ -406,7 +415,7 @@ func (job *JobObject) QueryProcessorStats() (*winapi.JOBOBJECT_BASIC_ACCOUNTING_ if err := winapi.QueryInformationJobObject( job.handle, winapi.JobObjectBasicAccountingInformation, - uintptr(unsafe.Pointer(&info)), + unsafe.Pointer(&info), uint32(unsafe.Sizeof(info)), nil, ); err != nil { @@ -415,7 +424,9 @@ func (job *JobObject) QueryProcessorStats() (*winapi.JOBOBJECT_BASIC_ACCOUNTING_ return &info, nil } -// QueryStorageStats gets the storage (I/O) stats for the job object. +// QueryStorageStats gets the storage (I/O) stats for the job object. This call will error +// if either `EnableIOTracking` wasn't set to true on creation of the job, or SetIOTracking() +// hasn't been called since creation of the job. func (job *JobObject) QueryStorageStats() (*winapi.JOBOBJECT_IO_ATTRIBUTION_INFORMATION, error) { job.handleLock.RLock() defer job.handleLock.RUnlock() @@ -430,7 +441,7 @@ func (job *JobObject) QueryStorageStats() (*winapi.JOBOBJECT_IO_ATTRIBUTION_INFO if err := winapi.QueryInformationJobObject( job.handle, winapi.JobObjectIoAttribution, - uintptr(unsafe.Pointer(&info)), + unsafe.Pointer(&info), uint32(unsafe.Sizeof(info)), nil, ); err != nil { @@ -476,7 +487,7 @@ func (job *JobObject) QueryPrivateWorkingSet() (uint64, error) { status := winapi.NtQueryInformationProcess( h, winapi.ProcessVmCounters, - uintptr(unsafe.Pointer(&vmCounters)), + unsafe.Pointer(&vmCounters), uint32(unsafe.Sizeof(vmCounters)), nil, ) @@ -497,3 +508,31 @@ func (job *JobObject) QueryPrivateWorkingSet() (uint64, error) { return jobWorkingSetSize, nil } + +// SetIOTracking enables IO tracking for processes in the job object. +// This enables use of the QueryStorageStats method. +func (job *JobObject) SetIOTracking() error { + job.handleLock.RLock() + defer job.handleLock.RUnlock() + + if job.handle == 0 { + return ErrAlreadyClosed + } + + return enableIOTracking(job.handle) +} + +func enableIOTracking(job windows.Handle) error { + info := winapi.JOBOBJECT_IO_ATTRIBUTION_INFORMATION{ + ControlFlags: winapi.JOBOBJECT_IO_ATTRIBUTION_CONTROL_ENABLE, + } + if _, err := windows.SetInformationJobObject( + job, + winapi.JobObjectIoAttribution, + uintptr(unsafe.Pointer(&info)), + uint32(unsafe.Sizeof(info)), + ); err != nil { + return fmt.Errorf("failed to enable IO tracking on job object: %w", err) + } + return nil +} diff --git a/vendor/github.com/Microsoft/hcsshim/internal/jobobject/limits.go b/vendor/github.com/Microsoft/hcsshim/internal/jobobject/limits.go index 4be297788..4efde292c 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/jobobject/limits.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/jobobject/limits.go @@ -202,7 +202,7 @@ func (job *JobObject) getExtendedInformation() (*windows.JOBOBJECT_EXTENDED_LIMI if err := winapi.QueryInformationJobObject( job.handle, windows.JobObjectExtendedLimitInformation, - uintptr(unsafe.Pointer(&info)), + unsafe.Pointer(&info), uint32(unsafe.Sizeof(info)), nil, ); err != nil { @@ -224,7 +224,7 @@ func (job *JobObject) getCPURateControlInformation() (*winapi.JOBOBJECT_CPU_RATE if err := winapi.QueryInformationJobObject( job.handle, windows.JobObjectCpuRateControlInformation, - uintptr(unsafe.Pointer(&info)), + unsafe.Pointer(&info), uint32(unsafe.Sizeof(info)), nil, ); err != nil { diff --git a/vendor/github.com/Microsoft/hcsshim/internal/queue/mq.go b/vendor/github.com/Microsoft/hcsshim/internal/queue/mq.go index e177c9a62..4eb9bb9f1 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/queue/mq.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/queue/mq.go @@ -5,10 +5,7 @@ import ( "sync" ) -var ( - ErrQueueClosed = errors.New("the queue is closed for reading and writing") - ErrQueueEmpty = errors.New("the queue is empty") -) +var ErrQueueClosed = errors.New("the queue is closed for reading and writing") // MessageQueue represents a threadsafe message queue to be used to retrieve or // write messages to. @@ -29,8 +26,8 @@ func NewMessageQueue() *MessageQueue { } } -// Write writes `msg` to the queue. -func (mq *MessageQueue) Write(msg interface{}) error { +// Enqueue writes `msg` to the queue. +func (mq *MessageQueue) Enqueue(msg interface{}) error { mq.m.Lock() defer mq.m.Unlock() @@ -43,55 +40,37 @@ func (mq *MessageQueue) Write(msg interface{}) error { return nil } -// Read will read a value from the queue if available, otherwise return an error. -func (mq *MessageQueue) Read() (interface{}, error) { +// Dequeue will read a value from the queue and remove it. If the queue +// is empty, this will block until the queue is closed or a value gets enqueued. +func (mq *MessageQueue) Dequeue() (interface{}, error) { mq.m.Lock() defer mq.m.Unlock() - if mq.closed { - return nil, ErrQueueClosed - } - if mq.isEmpty() { - return nil, ErrQueueEmpty + + for !mq.closed && mq.size() == 0 { + mq.c.Wait() } - val := mq.messages[0] - mq.messages[0] = nil - mq.messages = mq.messages[1:] - return val, nil -} -// ReadOrWait will read a value from the queue if available, else it will wait for a -// value to become available. This will block forever if nothing gets written or until -// the queue gets closed. -func (mq *MessageQueue) ReadOrWait() (interface{}, error) { - mq.m.Lock() + // We got woken up, check if it's because the queue got closed. if mq.closed { - mq.m.Unlock() return nil, ErrQueueClosed } - if mq.isEmpty() { - for !mq.closed && mq.isEmpty() { - mq.c.Wait() - } - mq.m.Unlock() - return mq.Read() - } + val := mq.messages[0] mq.messages[0] = nil mq.messages = mq.messages[1:] - mq.m.Unlock() return val, nil } -// IsEmpty returns if the queue is empty -func (mq *MessageQueue) IsEmpty() bool { +// Size returns the size of the queue. +func (mq *MessageQueue) Size() int { mq.m.RLock() defer mq.m.RUnlock() - return len(mq.messages) == 0 + return mq.size() } -// Nonexported empty check that doesn't lock so we can call this in Read and Write. -func (mq *MessageQueue) isEmpty() bool { - return len(mq.messages) == 0 +// Nonexported size check to check if the queue is empty inside already locked functions. +func (mq *MessageQueue) size() int { + return len(mq.messages) } // Close closes the queue for future writes or reads. Any attempts to read or write from the @@ -99,13 +78,15 @@ func (mq *MessageQueue) isEmpty() bool { func (mq *MessageQueue) Close() { mq.m.Lock() defer mq.m.Unlock() - // Already closed + + // Already closed, noop if mq.closed { return } + mq.messages = nil mq.closed = true - // If there's anybody currently waiting on a value from ReadOrWait, we need to + // If there's anybody currently waiting on a value from Dequeue, we need to // broadcast so the read(s) can return ErrQueueClosed. mq.c.Broadcast() } diff --git a/vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go b/vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go index 479649db3..7eb13f8f0 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/winapi/jobobject.go @@ -175,7 +175,7 @@ type JOBOBJECT_ASSOCIATE_COMPLETION_PORT struct { // LPDWORD lpReturnLength // ); // -//sys QueryInformationJobObject(jobHandle windows.Handle, infoClass uint32, jobObjectInfo uintptr, jobObjectInformationLength uint32, lpReturnLength *uint32) (err error) = kernel32.QueryInformationJobObject +//sys QueryInformationJobObject(jobHandle windows.Handle, infoClass uint32, jobObjectInfo unsafe.Pointer, jobObjectInformationLength uint32, lpReturnLength *uint32) (err error) = kernel32.QueryInformationJobObject // HANDLE OpenJobObjectW( // DWORD dwDesiredAccess, diff --git a/vendor/github.com/Microsoft/hcsshim/internal/winapi/process.go b/vendor/github.com/Microsoft/hcsshim/internal/winapi/process.go index 5f9e03fd2..222529f43 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/winapi/process.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/winapi/process.go @@ -18,7 +18,7 @@ const ProcessVmCounters = 3 // [out, optional] PULONG ReturnLength // ); // -//sys NtQueryInformationProcess(processHandle windows.Handle, processInfoClass uint32, processInfo uintptr, processInfoLength uint32, returnLength *uint32) (status uint32) = ntdll.NtQueryInformationProcess +//sys NtQueryInformationProcess(processHandle windows.Handle, processInfoClass uint32, processInfo unsafe.Pointer, processInfoLength uint32, returnLength *uint32) (status uint32) = ntdll.NtQueryInformationProcess // typedef struct _VM_COUNTERS_EX // { diff --git a/vendor/github.com/Microsoft/hcsshim/internal/winapi/system.go b/vendor/github.com/Microsoft/hcsshim/internal/winapi/system.go index 327f57d7c..78fe01a4b 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/winapi/system.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/winapi/system.go @@ -12,7 +12,8 @@ const STATUS_INFO_LENGTH_MISMATCH = 0xC0000004 // ULONG SystemInformationLength, // PULONG ReturnLength // ); -//sys NtQuerySystemInformation(systemInfoClass int, systemInformation uintptr, systemInfoLength uint32, returnLength *uint32) (status uint32) = ntdll.NtQuerySystemInformation +// +//sys NtQuerySystemInformation(systemInfoClass int, systemInformation unsafe.Pointer, systemInfoLength uint32, returnLength *uint32) (status uint32) = ntdll.NtQuerySystemInformation type SYSTEM_PROCESS_INFORMATION struct { NextEntryOffset uint32 // ULONG diff --git a/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go b/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go index 39fb3e1ad..1f16cf0b8 100644 --- a/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go +++ b/vendor/github.com/Microsoft/hcsshim/internal/winapi/zsyscall_windows.go @@ -100,7 +100,7 @@ func resizePseudoConsole(hPc windows.Handle, size uint32) (hr error) { return } -func NtQuerySystemInformation(systemInfoClass int, systemInformation uintptr, systemInfoLength uint32, returnLength *uint32) (status uint32) { +func NtQuerySystemInformation(systemInfoClass int, systemInformation unsafe.Pointer, systemInfoLength uint32, returnLength *uint32) (status uint32) { r0, _, _ := syscall.Syscall6(procNtQuerySystemInformation.Addr(), 4, uintptr(systemInfoClass), uintptr(systemInformation), uintptr(systemInfoLength), uintptr(unsafe.Pointer(returnLength)), 0, 0) status = uint32(r0) return @@ -152,7 +152,7 @@ func IsProcessInJob(procHandle windows.Handle, jobHandle windows.Handle, result return } -func QueryInformationJobObject(jobHandle windows.Handle, infoClass uint32, jobObjectInfo uintptr, jobObjectInformationLength uint32, lpReturnLength *uint32) (err error) { +func QueryInformationJobObject(jobHandle windows.Handle, infoClass uint32, jobObjectInfo unsafe.Pointer, jobObjectInformationLength uint32, lpReturnLength *uint32) (err error) { r1, _, e1 := syscall.Syscall6(procQueryInformationJobObject.Addr(), 5, uintptr(jobHandle), uintptr(infoClass), uintptr(jobObjectInfo), uintptr(jobObjectInformationLength), uintptr(unsafe.Pointer(lpReturnLength)), 0) if r1 == 0 { if e1 != 0 { @@ -244,7 +244,7 @@ func LocalFree(ptr uintptr) { return } -func NtQueryInformationProcess(processHandle windows.Handle, processInfoClass uint32, processInfo uintptr, processInfoLength uint32, returnLength *uint32) (status uint32) { +func NtQueryInformationProcess(processHandle windows.Handle, processInfoClass uint32, processInfo unsafe.Pointer, processInfoLength uint32, returnLength *uint32) (status uint32) { r0, _, _ := syscall.Syscall6(procNtQueryInformationProcess.Addr(), 5, uintptr(processHandle), uintptr(processInfoClass), uintptr(processInfo), uintptr(processInfoLength), uintptr(unsafe.Pointer(returnLength)), 0) status = uint32(r0) return diff --git a/vendor/github.com/containers/common/libimage/inspect.go b/vendor/github.com/containers/common/libimage/inspect.go index 5da8df1bf..c6632d9a2 100644 --- a/vendor/github.com/containers/common/libimage/inspect.go +++ b/vendor/github.com/containers/common/libimage/inspect.go @@ -190,7 +190,7 @@ func (i *Image) Inspect(ctx context.Context, options *InspectOptions) (*ImageDat // NOTE: Health checks may be listed in the container config or // the config. data.HealthCheck = dockerManifest.ContainerConfig.Healthcheck - if data.HealthCheck == nil { + if data.HealthCheck == nil && dockerManifest.Config != nil { data.HealthCheck = dockerManifest.Config.Healthcheck } } diff --git a/vendor/github.com/containers/common/libimage/load.go b/vendor/github.com/containers/common/libimage/load.go index 89faa4635..593eef04b 100644 --- a/vendor/github.com/containers/common/libimage/load.go +++ b/vendor/github.com/containers/common/libimage/load.go @@ -99,7 +99,7 @@ func (r *Runtime) Load(ctx context.Context, path string, options *LoadOptions) ( } // loadMultiImageDockerArchive loads the docker archive specified by ref. In -// case the path@reference notation was used, only the specifiec image will be +// case the path@reference notation was used, only the specified image will be // loaded. Otherwise, all images will be loaded. func (r *Runtime) loadMultiImageDockerArchive(ctx context.Context, ref types.ImageReference, options *CopyOptions) ([]string, error) { // If we cannot stat the path, it either does not exist OR the correct diff --git a/vendor/github.com/containers/common/libnetwork/cni/network.go b/vendor/github.com/containers/common/libnetwork/cni/network.go index fce8f0066..11f1bbe14 100644 --- a/vendor/github.com/containers/common/libnetwork/cni/network.go +++ b/vendor/github.com/containers/common/libnetwork/cni/network.go @@ -19,6 +19,7 @@ import ( "github.com/containers/common/pkg/config" "github.com/containers/storage/pkg/lockfile" "github.com/sirupsen/logrus" + "golang.org/x/sys/unix" ) type cniNetwork struct { @@ -62,6 +63,8 @@ type InitConfig struct { CNIConfigDir string // CNIPluginDirs is a list of directories where cni should look for the plugins. CNIPluginDirs []string + // RunDir is a directory where temporary files can be stored. + RunDir string // DefaultNetwork is the name for the default network. DefaultNetwork string @@ -81,7 +84,16 @@ func NewCNINetworkInterface(conf *InitConfig) (types.ContainerNetwork, error) { // TODO: consider using a shared memory lock lock, err := lockfile.GetLockfile(filepath.Join(conf.CNIConfigDir, "cni.lock")) if err != nil { - return nil, err + // If we're on a read-only filesystem, there is no risk of + // contention. Fall back to a local lockfile. + if errors.Is(err, unix.EROFS) { + lock, err = lockfile.GetLockfile(filepath.Join(conf.RunDir, "cni.lock")) + if err != nil { + return nil, err + } + } else { + return nil, err + } } defaultNetworkName := conf.DefaultNetwork diff --git a/vendor/github.com/containers/common/libnetwork/network/interface.go b/vendor/github.com/containers/common/libnetwork/network/interface.go index 639ff4e45..545655fd3 100644 --- a/vendor/github.com/containers/common/libnetwork/network/interface.go +++ b/vendor/github.com/containers/common/libnetwork/network/interface.go @@ -169,6 +169,7 @@ func getCniInterface(conf *config.Config) (types.ContainerNetwork, error) { return cni.NewCNINetworkInterface(&cni.InitConfig{ CNIConfigDir: confDir, CNIPluginDirs: conf.Network.CNIPluginDirs, + RunDir: conf.Engine.TmpDir, DefaultNetwork: conf.Network.DefaultNetwork, DefaultSubnet: conf.Network.DefaultSubnet, DefaultsubnetPools: conf.Network.DefaultSubnetPools, diff --git a/vendor/github.com/containers/common/pkg/config/config.go b/vendor/github.com/containers/common/pkg/config/config.go index de1d91ae3..858f961b6 100644 --- a/vendor/github.com/containers/common/pkg/config/config.go +++ b/vendor/github.com/containers/common/pkg/config/config.go @@ -7,6 +7,7 @@ import ( "os" "os/exec" "path/filepath" + "runtime" "sort" "strings" "sync" @@ -27,6 +28,8 @@ const ( _configPath = "containers/containers.conf" // UserOverrideContainersConfig holds the containers config path overridden by the rootless user UserOverrideContainersConfig = ".config/" + _configPath + // Token prefix for looking for helper binary under $BINDIR + bindirPrefix = "$BINDIR" ) // RuntimeStateStore is a constant indicating which state store implementation @@ -454,6 +457,13 @@ type EngineConfig struct { // may not be by other drivers. VolumePath string `toml:"volume_path,omitempty"` + // VolumePluginTimeout sets the default timeout, in seconds, for + // operations that must contact a volume plugin. Plugins are external + // programs accessed via REST API; this sets a timeout for requests to + // that API. + // A value of 0 is treated as no timeout. + VolumePluginTimeout uint `toml:"volume_plugin_timeout,omitempty,omitzero"` + // VolumePlugins is a set of plugins that can be used as the backend for // Podman named volumes. Each volume is specified as a name (what Podman // will refer to the plugin as) mapped to a path, which must point to a @@ -815,6 +825,18 @@ func (c *Config) Validate() error { return nil } +// URI returns the URI Path to the machine image +func (m *MachineConfig) URI() string { + uri := m.Image + for _, val := range []string{"$ARCH", "$arch"} { + uri = strings.Replace(uri, val, runtime.GOARCH, 1) + } + for _, val := range []string{"$OS", "$os"} { + uri = strings.Replace(uri, val, runtime.GOOS, 1) + } + return uri +} + func (c *EngineConfig) findRuntime() string { // Search for crun first followed by runc, kata, runsc for _, name := range []string{"crun", "runc", "runj", "kata", "runsc"} { @@ -1241,10 +1263,37 @@ func (c *Config) ActiveDestination() (uri, identity string, err error) { return "", "", errors.New("no service destination configured") } +var ( + bindirFailed = false + bindirCached = "" +) + +func findBindir() string { + if bindirCached != "" || bindirFailed { + return bindirCached + } + execPath, err := os.Executable() + if err == nil { + // Resolve symbolic links to find the actual binary file path. + execPath, err = filepath.EvalSymlinks(execPath) + } + if err != nil { + // If failed to find executable (unlikely to happen), warn about it. + // The bindirFailed flag will track this, so we only warn once. + logrus.Warnf("Failed to find $BINDIR: %v", err) + bindirFailed = true + return "" + } + bindirCached = filepath.Dir(execPath) + return bindirCached +} + // FindHelperBinary will search the given binary name in the configured directories. // If searchPATH is set to true it will also search in $PATH. func (c *Config) FindHelperBinary(name string, searchPATH bool) (string, error) { dirList := c.Engine.HelperBinariesDir + bindirPath := "" + bindirSearched := false // If set, search this directory first. This is used in testing. if dir, found := os.LookupEnv("CONTAINERS_HELPER_BINARY_DIR"); found { @@ -1252,6 +1301,24 @@ func (c *Config) FindHelperBinary(name string, searchPATH bool) (string, error) } for _, path := range dirList { + if path == bindirPrefix || strings.HasPrefix(path, bindirPrefix+string(filepath.Separator)) { + // Calculate the path to the executable first time we encounter a $BINDIR prefix. + if !bindirSearched { + bindirSearched = true + bindirPath = findBindir() + } + // If there's an error, don't stop the search for the helper binary. + // findBindir() will have warned once during the first failure. + if bindirPath == "" { + continue + } + // Replace the $BINDIR prefix with the path to the directory of the current binary. + if path == bindirPrefix { + path = bindirPath + } else { + path = filepath.Join(bindirPath, strings.TrimPrefix(path, bindirPrefix+string(filepath.Separator))) + } + } fullpath := filepath.Join(path, name) if fi, err := os.Stat(fullpath); err == nil && fi.Mode().IsRegular() { return fullpath, nil diff --git a/vendor/github.com/containers/common/pkg/config/config_darwin.go b/vendor/github.com/containers/common/pkg/config/config_darwin.go index 0ab9e0294..5283665e1 100644 --- a/vendor/github.com/containers/common/pkg/config/config_darwin.go +++ b/vendor/github.com/containers/common/pkg/config/config_darwin.go @@ -35,4 +35,6 @@ var defaultHelperBinariesDir = []string{ "/usr/local/lib/podman", "/usr/libexec/podman", "/usr/lib/podman", + // Relative to the binary directory + "$BINDIR/../libexec/podman", } diff --git a/vendor/github.com/containers/common/pkg/config/containers.conf b/vendor/github.com/containers/common/pkg/config/containers.conf index d1ac7c0e8..5b5aaa00a 100644 --- a/vendor/github.com/containers/common/pkg/config/containers.conf +++ b/vendor/github.com/containers/common/pkg/config/containers.conf @@ -605,6 +605,12 @@ default_sysctls = [ # #volume_path = "/var/lib/containers/storage/volumes" +# Default timeout (in seconds) for volume plugin operations. +# Plugins are external programs accessed via a REST API; this sets a timeout +# for requests to that API. +# A value of 0 is treated as no timeout. +#volume_plugin_timeout = 5 + # Paths to look for a valid OCI runtime (crun, runc, kata, runsc, krun, etc) [engine.runtimes] #crun = [ @@ -665,9 +671,16 @@ default_sysctls = [ # #disk_size=10 -# The image used when creating a podman-machine VM. +# Default image URI when creating a new VM using `podman machine init`. +# Options: On Linux/Mac, `testing`, `stable`, `next`. On Windows, the major +# version of the OS (e.g `36`) for Fedora 36. For all platforms you can +# alternatively specify a custom download URL to an image. Container engines +# translate URIs $OS and $ARCH to the native OS and ARCH. URI +# "https://example.com/$OS/$ARCH/foobar.ami" becomes +# "https://example.com/linux/amd64/foobar.ami" on a Linux AMD machine. +# The default value is `testing`. # -#image = "testing" +# image = "testing" # Memory in MB a machine is created with. # diff --git a/vendor/github.com/containers/common/pkg/config/default.go b/vendor/github.com/containers/common/pkg/config/default.go index 6bca7312a..b0d62779b 100644 --- a/vendor/github.com/containers/common/pkg/config/default.go +++ b/vendor/github.com/containers/common/pkg/config/default.go @@ -168,6 +168,8 @@ const ( SeccompOverridePath = _etcDir + "/containers/seccomp.json" // SeccompDefaultPath defines the default seccomp path. SeccompDefaultPath = _installPrefix + "/share/containers/seccomp.json" + // DefaultVolumePluginTimeout is the default volume plugin timeout, in seconds + DefaultVolumePluginTimeout = 5 ) // DefaultConfig defines the default values from containers.conf. @@ -264,7 +266,7 @@ func defaultMachineConfig() MachineConfig { Image: getDefaultMachineImage(), Memory: 2048, User: getDefaultMachineUser(), - Volumes: []string{"$HOME:$HOME"}, + Volumes: getDefaultMachineVolumes(), } } @@ -304,6 +306,8 @@ func defaultConfigFromMemory() (*EngineConfig, error) { c.StaticDir = filepath.Join(storeOpts.GraphRoot, "libpod") c.VolumePath = filepath.Join(storeOpts.GraphRoot, "volumes") + c.VolumePluginTimeout = DefaultVolumePluginTimeout + c.HelperBinariesDir = defaultHelperBinariesDir if additionalHelperBinariesDir != "" { c.HelperBinariesDir = append(c.HelperBinariesDir, additionalHelperBinariesDir) diff --git a/vendor/github.com/containers/common/pkg/config/default_darwin.go b/vendor/github.com/containers/common/pkg/config/default_darwin.go index c502ea55e..5d857df4f 100644 --- a/vendor/github.com/containers/common/pkg/config/default_darwin.go +++ b/vendor/github.com/containers/common/pkg/config/default_darwin.go @@ -11,3 +11,8 @@ func getDefaultLockType() string { func getLibpodTmpDir() string { return "/run/libpod" } + +// getDefaultMachineVolumes returns default mounted volumes (possibly with env vars, which will be expanded) +func getDefaultMachineVolumes() []string { + return []string{"$HOME:$HOME"} +} diff --git a/vendor/github.com/containers/common/pkg/config/default_freebsd.go b/vendor/github.com/containers/common/pkg/config/default_freebsd.go index 8b10ac1f7..9c827dbfe 100644 --- a/vendor/github.com/containers/common/pkg/config/default_freebsd.go +++ b/vendor/github.com/containers/common/pkg/config/default_freebsd.go @@ -18,3 +18,8 @@ func getDefaultLockType() string { func getLibpodTmpDir() string { return "/var/run/libpod" } + +// getDefaultMachineVolumes returns default mounted volumes (possibly with env vars, which will be expanded) +func getDefaultMachineVolumes() []string { + return []string{"$HOME:$HOME"} +} diff --git a/vendor/github.com/containers/common/pkg/config/default_linux.go b/vendor/github.com/containers/common/pkg/config/default_linux.go index 86873beb1..15052c10e 100644 --- a/vendor/github.com/containers/common/pkg/config/default_linux.go +++ b/vendor/github.com/containers/common/pkg/config/default_linux.go @@ -70,3 +70,8 @@ func getDefaultLockType() string { func getLibpodTmpDir() string { return "/run/libpod" } + +// getDefaultMachineVolumes returns default mounted volumes (possibly with env vars, which will be expanded) +func getDefaultMachineVolumes() []string { + return []string{"$HOME:$HOME"} +} diff --git a/vendor/github.com/containers/common/pkg/config/default_windows.go b/vendor/github.com/containers/common/pkg/config/default_windows.go index 1ff88fc42..08a0bf223 100644 --- a/vendor/github.com/containers/common/pkg/config/default_windows.go +++ b/vendor/github.com/containers/common/pkg/config/default_windows.go @@ -44,3 +44,8 @@ func getDefaultLockType() string { func getLibpodTmpDir() string { return "/run/libpod" } + +// getDefaultMachineVolumes returns default mounted volumes (possibly with env vars, which will be expanded) +func getDefaultMachineVolumes() []string { + return []string{} +} diff --git a/vendor/github.com/containers/common/pkg/subscriptions/subscriptions.go b/vendor/github.com/containers/common/pkg/subscriptions/subscriptions.go index ff82b5a39..02b6dfb09 100644 --- a/vendor/github.com/containers/common/pkg/subscriptions/subscriptions.go +++ b/vendor/github.com/containers/common/pkg/subscriptions/subscriptions.go @@ -372,7 +372,7 @@ func mountExists(mounts []rspec.Mount, dest string) bool { return false } -// resolveSymbolicLink resolves a possbile symlink path. If the path is a symlink, returns resolved +// resolveSymbolicLink resolves symlink paths. If the path is a symlink, returns resolved // path; if not, returns the original path. func resolveSymbolicLink(path string) (string, error) { info, err := os.Lstat(path) diff --git a/vendor/modules.txt b/vendor/modules.txt index eb9c7a34d..170f8fb98 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -11,7 +11,7 @@ github.com/Microsoft/go-winio/backuptar github.com/Microsoft/go-winio/pkg/guid github.com/Microsoft/go-winio/pkg/security github.com/Microsoft/go-winio/vhd -# github.com/Microsoft/hcsshim v0.9.3 +# github.com/Microsoft/hcsshim v0.9.4 github.com/Microsoft/hcsshim github.com/Microsoft/hcsshim/computestorage github.com/Microsoft/hcsshim/internal/cow @@ -67,7 +67,7 @@ github.com/container-orchestrated-devices/container-device-interface/pkg/cdi github.com/container-orchestrated-devices/container-device-interface/specs-go # github.com/containerd/cgroups v1.0.3 github.com/containerd/cgroups/stats/v1 -# github.com/containerd/containerd v1.6.6 +# github.com/containerd/containerd v1.6.8 github.com/containerd/containerd/errdefs github.com/containerd/containerd/log github.com/containerd/containerd/pkg/userns @@ -114,7 +114,7 @@ github.com/containers/buildah/pkg/rusage github.com/containers/buildah/pkg/sshagent github.com/containers/buildah/pkg/util github.com/containers/buildah/util -# github.com/containers/common v0.49.2-0.20220817132854-f6679f170eca +# github.com/containers/common v0.49.2-0.20220823130605-72a7da3358ac ## explicit github.com/containers/common/libimage github.com/containers/common/libimage/define -- cgit v1.2.3-54-g00ecf From b4584ea8546be227c74174130ec3a04a93996d28 Mon Sep 17 00:00:00 2001 From: Aditya R Date: Wed, 24 Aug 2022 11:22:33 +0530 Subject: run,create: add support for --env-merge for preprocessing vars Allow end users to preprocess default environment variables before injecting them into container using `--env-merge` Usage ``` podman run -it --rm --env-merge some=${some}-edit --env-merge some2=${some2}-edit2 myimage sh ``` Closes: https://github.com/containers/podman/issues/15288 Signed-off-by: Aditya R --- cmd/podman/common/create.go | 8 ++++++++ docs/source/markdown/options/env-merge.md | 5 +++++ docs/source/markdown/podman-create.1.md.in | 2 ++ docs/source/markdown/podman-run.1.md.in | 2 ++ go.mod | 1 + pkg/api/handlers/compat/containers_create.go | 1 + pkg/api/handlers/types.go | 1 + pkg/domain/entities/pods.go | 1 + pkg/specgen/generate/container.go | 12 ++++++++++++ pkg/specgen/specgen.go | 3 +++ pkg/specgenutil/specgen.go | 3 +++ test/e2e/run_env_test.go | 11 +++++++++++ vendor/modules.txt | 1 + 13 files changed, 51 insertions(+) create mode 100644 docs/source/markdown/options/env-merge.md (limited to 'pkg/domain') diff --git a/cmd/podman/common/create.go b/cmd/podman/common/create.go index 00873b95b..1e573cc2d 100644 --- a/cmd/podman/common/create.go +++ b/cmd/podman/common/create.go @@ -124,6 +124,14 @@ func DefineCreateFlags(cmd *cobra.Command, cf *entities.ContainerCreateOptions, "This is a Docker specific option and is a NOOP", ) + envMergeFlagName := "env-merge" + createFlags.StringArrayVar( + &cf.EnvMerge, + envMergeFlagName, []string{}, + "Preprocess environment variables from image before injecting them into the container", + ) + _ = cmd.RegisterFlagCompletionFunc(envMergeFlagName, completion.AutocompleteNone) + envFlagName := "env" createFlags.StringArrayP( envFlagName, "e", Env(), diff --git a/docs/source/markdown/options/env-merge.md b/docs/source/markdown/options/env-merge.md new file mode 100644 index 000000000..aa1aa003d --- /dev/null +++ b/docs/source/markdown/options/env-merge.md @@ -0,0 +1,5 @@ +#### **--env-merge**=*env* + +Preprocess default environment variables for the containers. For example +if image contains environment variable `hello=world` user can preprocess +it using `--env-merge hello=${hello}-some` so new value will be `hello=world-some`. diff --git a/docs/source/markdown/podman-create.1.md.in b/docs/source/markdown/podman-create.1.md.in index 9518568bb..0e65e7e3a 100644 --- a/docs/source/markdown/podman-create.1.md.in +++ b/docs/source/markdown/podman-create.1.md.in @@ -208,6 +208,8 @@ Read in a line delimited file of environment variables. See **Environment** note @@option env-host +@@option env-merge + @@option expose #### **--gidmap**=*container_gid:host_gid:amount* diff --git a/docs/source/markdown/podman-run.1.md.in b/docs/source/markdown/podman-run.1.md.in index 3e31ccc45..98d57a9c1 100644 --- a/docs/source/markdown/podman-run.1.md.in +++ b/docs/source/markdown/podman-run.1.md.in @@ -243,6 +243,8 @@ Read in a line delimited file of environment variables. See **Environment** note @@option env-host +@@option env-merge + @@option expose #### **--gidmap**=*container_gid:host_gid:amount* diff --git a/go.mod b/go.mod index 635c0a17d..a694e6add 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( github.com/opencontainers/runtime-spec v1.0.3-0.20211214071223-8958f93039ab github.com/opencontainers/runtime-tools v0.9.1-0.20220714195903-17b3287fafb7 github.com/opencontainers/selinux v1.10.1 + github.com/openshift/imagebuilder v1.2.4-0.20220711175835-4151e43600df github.com/rootless-containers/rootlesskit v1.0.1 github.com/sirupsen/logrus v1.9.0 github.com/spf13/cobra v1.5.0 diff --git a/pkg/api/handlers/compat/containers_create.go b/pkg/api/handlers/compat/containers_create.go index 9fff8b4c8..d4f5d5f36 100644 --- a/pkg/api/handlers/compat/containers_create.go +++ b/pkg/api/handlers/compat/containers_create.go @@ -408,6 +408,7 @@ func cliOpts(cc handlers.CreateContainerConfig, rtc *config.Config) (*entities.C Systemd: "true", // podman default TmpFS: parsedTmp, TTY: cc.Config.Tty, + EnvMerge: cc.EnvMerge, UnsetEnv: cc.UnsetEnv, UnsetEnvAll: cc.UnsetEnvAll, User: cc.Config.User, diff --git a/pkg/api/handlers/types.go b/pkg/api/handlers/types.go index b533e131c..ebbc5f63a 100644 --- a/pkg/api/handlers/types.go +++ b/pkg/api/handlers/types.go @@ -127,6 +127,7 @@ type CreateContainerConfig struct { dockerContainer.Config // desired container configuration HostConfig dockerContainer.HostConfig // host dependent configuration for container NetworkingConfig dockerNetwork.NetworkingConfig // network configuration for container + EnvMerge []string // preprocess env variables from image before injecting into containers UnsetEnv []string // unset specified default environment variables UnsetEnvAll bool // unset all default environment variables } diff --git a/pkg/domain/entities/pods.go b/pkg/domain/entities/pods.go index 14ce370c1..33ca2c807 100644 --- a/pkg/domain/entities/pods.go +++ b/pkg/domain/entities/pods.go @@ -263,6 +263,7 @@ type ContainerCreateOptions struct { TTY bool Timezone string Umask string + EnvMerge []string UnsetEnv []string UnsetEnvAll bool UIDMap []string diff --git a/pkg/specgen/generate/container.go b/pkg/specgen/generate/container.go index 85cd8f5ca..e293ce010 100644 --- a/pkg/specgen/generate/container.go +++ b/pkg/specgen/generate/container.go @@ -19,6 +19,7 @@ import ( "github.com/containers/podman/v4/pkg/signal" "github.com/containers/podman/v4/pkg/specgen" spec "github.com/opencontainers/runtime-spec/specs-go" + "github.com/openshift/imagebuilder" "github.com/sirupsen/logrus" "golang.org/x/sys/unix" ) @@ -131,6 +132,17 @@ func CompleteSpec(ctx context.Context, r *libpod.Runtime, s *specgen.SpecGenerat defaultEnvs = envLib.Join(envLib.DefaultEnvVariables(), envLib.Join(defaultEnvs, envs)) } + for _, e := range s.EnvMerge { + processedWord, err := imagebuilder.ProcessWord(e, envLib.Slice(defaultEnvs)) + if err != nil { + return nil, fmt.Errorf("unable to process variables for --env-merge %s: %w", e, err) + } + splitWord := strings.Split(processedWord, "=") + if _, ok := defaultEnvs[splitWord[0]]; ok { + defaultEnvs[splitWord[0]] = splitWord[1] + } + } + for _, e := range s.UnsetEnv { delete(defaultEnvs, e) } diff --git a/pkg/specgen/specgen.go b/pkg/specgen/specgen.go index b90f07ef8..51b6736a9 100644 --- a/pkg/specgen/specgen.go +++ b/pkg/specgen/specgen.go @@ -204,6 +204,9 @@ type ContainerBasicConfig struct { // The execution domain system allows Linux to provide limited support // for binaries compiled under other UNIX-like operating systems. Personality *spec.LinuxPersonality `json:"personality,omitempty"` + // EnvMerge takes the specified environment variables from image and preprocess them before injecting them into the + // container. + EnvMerge []string `json:"envmerge,omitempty"` // UnsetEnv unsets the specified default environment variables from the image or from buildin or containers.conf // Optional. UnsetEnv []string `json:"unsetenv,omitempty"` diff --git a/pkg/specgenutil/specgen.go b/pkg/specgenutil/specgen.go index 7392e7b44..aab2eebd5 100644 --- a/pkg/specgenutil/specgen.go +++ b/pkg/specgenutil/specgen.go @@ -839,6 +839,9 @@ func FillOutSpecGen(s *specgen.SpecGenerator, c *entities.ContainerCreateOptions if !s.Volatile { s.Volatile = c.Rm } + if len(s.EnvMerge) == 0 || len(c.EnvMerge) != 0 { + s.EnvMerge = c.EnvMerge + } if len(s.UnsetEnv) == 0 || len(c.UnsetEnv) != 0 { s.UnsetEnv = c.UnsetEnv } diff --git a/test/e2e/run_env_test.go b/test/e2e/run_env_test.go index bab52efc5..9e78e150a 100644 --- a/test/e2e/run_env_test.go +++ b/test/e2e/run_env_test.go @@ -82,6 +82,17 @@ var _ = Describe("Podman run", func() { Expect(session.OutputToString()).To(ContainSubstring("HOSTNAME")) }) + It("podman run with --env-merge", func() { + dockerfile := `FROM quay.io/libpod/alpine:latest +ENV hello=world +` + podmanTest.BuildImage(dockerfile, "test", "false") + session := podmanTest.Podman([]string{"run", "--rm", "--env-merge", "hello=${hello}-earth", "test", "env"}) + session.WaitWithDefaultTimeout() + Expect(session).Should(Exit(0)) + Expect(session.OutputToString()).To(ContainSubstring("world-earth")) + }) + It("podman run --env-host environment test", func() { env := append(os.Environ(), "FOO=BAR") session := podmanTest.PodmanAsUser([]string{"run", "--rm", "--env-host", ALPINE, "/bin/printenv", "FOO"}, 0, 0, "", env) diff --git a/vendor/modules.txt b/vendor/modules.txt index eb9c7a34d..feb9f00d5 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -620,6 +620,7 @@ github.com/opencontainers/selinux/go-selinux/label github.com/opencontainers/selinux/pkg/pwalk github.com/opencontainers/selinux/pkg/pwalkdir # github.com/openshift/imagebuilder v1.2.4-0.20220711175835-4151e43600df +## explicit github.com/openshift/imagebuilder github.com/openshift/imagebuilder/dockerfile/command github.com/openshift/imagebuilder/dockerfile/parser -- cgit v1.2.3-54-g00ecf From df1d8d0e9353528708a87609b932ae6b833c7ec0 Mon Sep 17 00:00:00 2001 From: Miloslav Trmač Date: Wed, 24 Aug 2022 19:57:43 +0200 Subject: Remove commented out code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We can always recover it from git, but it seems to serve no purpose anyway. Should not change behavior. Signed-off-by: Miloslav Trmač --- pkg/domain/infra/abi/trust.go | 4 ---- 1 file changed, 4 deletions(-) (limited to 'pkg/domain') diff --git a/pkg/domain/infra/abi/trust.go b/pkg/domain/infra/abi/trust.go index 0e3d8fad9..cefe76da7 100644 --- a/pkg/domain/infra/abi/trust.go +++ b/pkg/domain/infra/abi/trust.go @@ -142,16 +142,12 @@ func getPolicyShowOutput(policyContentStruct trust.PolicyContent, systemRegistri Transport: transport, Type: trustTypeDescription(repoval[0].Type), } - // TODO - keyarr is not used and I don't know its intent; commenting out for now for someone to fix later - // keyarr := []string{} uids := []string{} for _, repoele := range repoval { if len(repoele.KeyPath) > 0 { - // keyarr = append(keyarr, repoele.KeyPath) uids = append(uids, trust.GetGPGIdFromKeyPath(repoele.KeyPath)...) } if len(repoele.KeyData) > 0 { - // keyarr = append(keyarr, string(repoele.KeyData)) uids = append(uids, trust.GetGPGIdFromKeyData(repoele.KeyData)...) } } -- cgit v1.2.3-54-g00ecf From 4f68075306efb9381de3c5ea5762a3f843137b56 Mon Sep 17 00:00:00 2001 From: Miloslav Trmač Date: Wed, 24 Aug 2022 19:33:00 +0200 Subject: Add a variable for scope MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Only process the incoming args[] (which is a single-element array for some reason) once, and use a semantic variable name for the value we care about. Should not change behavior, the only caller already supposedly ensures that len(args) == 1. Signed-off-by: Miloslav Trmač --- pkg/domain/infra/abi/trust.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'pkg/domain') diff --git a/pkg/domain/infra/abi/trust.go b/pkg/domain/infra/abi/trust.go index cefe76da7..61bf03727 100644 --- a/pkg/domain/infra/abi/trust.go +++ b/pkg/domain/infra/abi/trust.go @@ -46,6 +46,11 @@ func (ir *ImageEngine) ShowTrust(ctx context.Context, args []string, options ent } func (ir *ImageEngine) SetTrust(ctx context.Context, args []string, options entities.SetTrustOptions) error { + if len(args) != 1 { + return fmt.Errorf("SetTrust called with unexpected %d args", len(args)) + } + scope := args[0] + var ( policyContentStruct trust.PolicyContent newReposContent []trust.RepoContent @@ -81,7 +86,7 @@ func (ir *ImageEngine) SetTrust(ctx context.Context, args []string, options enti } else { newReposContent = append(newReposContent, trust.RepoContent{Type: trustType}) } - if args[0] == "default" { + if scope == "default" { policyContentStruct.Default = newReposContent } else { if len(policyContentStruct.Default) == 0 { @@ -89,9 +94,9 @@ func (ir *ImageEngine) SetTrust(ctx context.Context, args []string, options enti } registryExists := false for transport, transportval := range policyContentStruct.Transports { - _, registryExists = transportval[args[0]] + _, registryExists = transportval[scope] if registryExists { - policyContentStruct.Transports[transport][args[0]] = newReposContent + policyContentStruct.Transports[transport][scope] = newReposContent break } } @@ -102,7 +107,7 @@ func (ir *ImageEngine) SetTrust(ctx context.Context, args []string, options enti if policyContentStruct.Transports["docker"] == nil { policyContentStruct.Transports["docker"] = make(map[string][]trust.RepoContent) } - policyContentStruct.Transports["docker"][args[0]] = append(policyContentStruct.Transports["docker"][args[0]], newReposContent...) + policyContentStruct.Transports["docker"][scope] = append(policyContentStruct.Transports["docker"][scope], newReposContent...) } } -- cgit v1.2.3-54-g00ecf From cbdbb025a3f6e6e5417cdade032075d679842056 Mon Sep 17 00:00:00 2001 From: Miloslav Trmač Date: Wed, 24 Aug 2022 21:39:14 +0200 Subject: Move most of imageEngine.SetTrust to pkg/trust.AddPolicyEntries MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will allow us to write unit tests without setting up the complete Podman runtime (and without the Linux dependency). Also, actually add a basic smoke test of the core functionality. Should not change behavior. Signed-off-by: Miloslav Trmač --- pkg/domain/infra/abi/trust.go | 68 +++------------------------------------ pkg/trust/policy.go | 74 +++++++++++++++++++++++++++++++++++++++++++ pkg/trust/policy_test.go | 71 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 150 insertions(+), 63 deletions(-) create mode 100644 pkg/trust/policy_test.go (limited to 'pkg/domain') diff --git a/pkg/domain/infra/abi/trust.go b/pkg/domain/infra/abi/trust.go index 61bf03727..381ea5deb 100644 --- a/pkg/domain/infra/abi/trust.go +++ b/pkg/domain/infra/abi/trust.go @@ -2,11 +2,8 @@ package abi import ( "context" - "encoding/json" - "errors" "fmt" "io/ioutil" - "os" "strings" "github.com/containers/podman/v4/pkg/domain/entities" @@ -51,71 +48,16 @@ func (ir *ImageEngine) SetTrust(ctx context.Context, args []string, options enti } scope := args[0] - var ( - policyContentStruct trust.PolicyContent - newReposContent []trust.RepoContent - ) - trustType := options.Type - if trustType == "accept" { - trustType = "insecureAcceptAnything" - } - - pubkeysfile := options.PubKeysFile - if len(pubkeysfile) == 0 && trustType == "signedBy" { - return errors.New("at least one public key must be defined for type 'signedBy'") - } - policyPath := trust.DefaultPolicyPath(ir.Libpod.SystemContext()) if len(options.PolicyPath) > 0 { policyPath = options.PolicyPath } - _, err := os.Stat(policyPath) - if !os.IsNotExist(err) { - policyContent, err := ioutil.ReadFile(policyPath) - if err != nil { - return err - } - if err := json.Unmarshal(policyContent, &policyContentStruct); err != nil { - return errors.New("could not read trust policies") - } - } - if len(pubkeysfile) != 0 { - for _, filepath := range pubkeysfile { - newReposContent = append(newReposContent, trust.RepoContent{Type: trustType, KeyType: "GPGKeys", KeyPath: filepath}) - } - } else { - newReposContent = append(newReposContent, trust.RepoContent{Type: trustType}) - } - if scope == "default" { - policyContentStruct.Default = newReposContent - } else { - if len(policyContentStruct.Default) == 0 { - return errors.New("default trust policy must be set") - } - registryExists := false - for transport, transportval := range policyContentStruct.Transports { - _, registryExists = transportval[scope] - if registryExists { - policyContentStruct.Transports[transport][scope] = newReposContent - break - } - } - if !registryExists { - if policyContentStruct.Transports == nil { - policyContentStruct.Transports = make(map[string]trust.RepoMap) - } - if policyContentStruct.Transports["docker"] == nil { - policyContentStruct.Transports["docker"] = make(map[string][]trust.RepoContent) - } - policyContentStruct.Transports["docker"][scope] = append(policyContentStruct.Transports["docker"][scope], newReposContent...) - } - } - data, err := json.MarshalIndent(policyContentStruct, "", " ") - if err != nil { - return fmt.Errorf("error setting trust policy: %w", err) - } - return ioutil.WriteFile(policyPath, data, 0644) + return trust.AddPolicyEntries(policyPath, trust.AddPolicyEntriesInput{ + Scope: scope, + Type: options.Type, + PubKeyFiles: options.PubKeysFile, + }) } func getPolicyShowOutput(policyContentStruct trust.PolicyContent, systemRegistriesDirPath string) ([]*trust.Policy, error) { diff --git a/pkg/trust/policy.go b/pkg/trust/policy.go index 62950131d..352be781c 100644 --- a/pkg/trust/policy.go +++ b/pkg/trust/policy.go @@ -5,6 +5,7 @@ import ( "bytes" "encoding/base64" "encoding/json" + "errors" "fmt" "io/ioutil" "os" @@ -123,3 +124,76 @@ func GetPolicy(policyPath string) (PolicyContent, error) { } return policyContentStruct, nil } + +// AddPolicyEntriesInput collects some parameters to AddPolicyEntries, +// primarily so that the callers use named values instead of just strings in a sequence. +type AddPolicyEntriesInput struct { + Scope string // "default" or a docker/atomic scope name + Type string + PubKeyFiles []string // For signature enforcement types, paths to public keys files (where the image needs to be signed by at least one key from _each_ of the files). File format depends on Type. +} + +// AddPolicyEntries adds one or more policy entries necessary to implement AddPolicyEntriesInput. +func AddPolicyEntries(policyPath string, input AddPolicyEntriesInput) error { + var ( + policyContentStruct PolicyContent + newReposContent []RepoContent + ) + trustType := input.Type + if trustType == "accept" { + trustType = "insecureAcceptAnything" + } + + pubkeysfile := input.PubKeyFiles + if len(pubkeysfile) == 0 && trustType == "signedBy" { + return errors.New("at least one public key must be defined for type 'signedBy'") + } + + _, err := os.Stat(policyPath) + if !os.IsNotExist(err) { + policyContent, err := ioutil.ReadFile(policyPath) + if err != nil { + return err + } + if err := json.Unmarshal(policyContent, &policyContentStruct); err != nil { + return errors.New("could not read trust policies") + } + } + if len(pubkeysfile) != 0 { + for _, filepath := range pubkeysfile { + newReposContent = append(newReposContent, RepoContent{Type: trustType, KeyType: "GPGKeys", KeyPath: filepath}) + } + } else { + newReposContent = append(newReposContent, RepoContent{Type: trustType}) + } + if input.Scope == "default" { + policyContentStruct.Default = newReposContent + } else { + if len(policyContentStruct.Default) == 0 { + return errors.New("default trust policy must be set") + } + registryExists := false + for transport, transportval := range policyContentStruct.Transports { + _, registryExists = transportval[input.Scope] + if registryExists { + policyContentStruct.Transports[transport][input.Scope] = newReposContent + break + } + } + if !registryExists { + if policyContentStruct.Transports == nil { + policyContentStruct.Transports = make(map[string]RepoMap) + } + if policyContentStruct.Transports["docker"] == nil { + policyContentStruct.Transports["docker"] = make(map[string][]RepoContent) + } + policyContentStruct.Transports["docker"][input.Scope] = append(policyContentStruct.Transports["docker"][input.Scope], newReposContent...) + } + } + + data, err := json.MarshalIndent(policyContentStruct, "", " ") + if err != nil { + return fmt.Errorf("error setting trust policy: %w", err) + } + return ioutil.WriteFile(policyPath, data, 0644) +} diff --git a/pkg/trust/policy_test.go b/pkg/trust/policy_test.go new file mode 100644 index 000000000..1f2f585c8 --- /dev/null +++ b/pkg/trust/policy_test.go @@ -0,0 +1,71 @@ +package trust + +import ( + "encoding/json" + "os" + "path/filepath" + "testing" + + "github.com/containers/image/v5/signature" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAddPolicyEntries(t *testing.T) { + tempDir := t.TempDir() + policyPath := filepath.Join(tempDir, "policy.json") + + minimalPolicy := &signature.Policy{ + Default: []signature.PolicyRequirement{ + signature.NewPRInsecureAcceptAnything(), + }, + } + minimalPolicyJSON, err := json.Marshal(minimalPolicy) + require.NoError(t, err) + err = os.WriteFile(policyPath, minimalPolicyJSON, 0600) + require.NoError(t, err) + + err = AddPolicyEntries(policyPath, AddPolicyEntriesInput{ + Scope: "default", + Type: "reject", + }) + assert.NoError(t, err) + err = AddPolicyEntries(policyPath, AddPolicyEntriesInput{ + Scope: "quay.io/accepted", + Type: "accept", + }) + assert.NoError(t, err) + err = AddPolicyEntries(policyPath, AddPolicyEntriesInput{ + Scope: "quay.io/multi-signed", + Type: "signedBy", + PubKeyFiles: []string{"/1.pub", "/2.pub"}, + }) + assert.NoError(t, err) + + // Test that the outcome is consumable, and compare it with the expected values. + parsedPolicy, err := signature.NewPolicyFromFile(policyPath) + require.NoError(t, err) + assert.Equal(t, &signature.Policy{ + Default: signature.PolicyRequirements{ + signature.NewPRReject(), + }, + Transports: map[string]signature.PolicyTransportScopes{ + "docker": { + "quay.io/accepted": { + signature.NewPRInsecureAcceptAnything(), + }, + "quay.io/multi-signed": { + xNewPRSignedByKeyPath(t, "/1.pub", signature.NewPRMMatchRepoDigestOrExact()), + xNewPRSignedByKeyPath(t, "/2.pub", signature.NewPRMMatchRepoDigestOrExact()), + }, + }, + }, + }, parsedPolicy) +} + +// xNewPRSignedByKeyPath is a wrapper for NewPRSignedByKeyPath which must not fail. +func xNewPRSignedByKeyPath(t *testing.T, keyPath string, signedIdentity signature.PolicyReferenceMatch) signature.PolicyRequirement { + pr, err := signature.NewPRSignedByKeyPath(signature.SBKeyTypeGPGKeys, keyPath, signedIdentity) + require.NoError(t, err) + return pr +} -- cgit v1.2.3-54-g00ecf From 7723a1ea654624b5cfcedc6d94e947169967c183 Mon Sep 17 00:00:00 2001 From: Miloslav Trmač Date: Wed, 24 Aug 2022 22:09:58 +0200 Subject: Move most of ImageEngine.ShowTrust into pkg/trust.PolicyDescription MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This will allow us to write unit tests without setting up the complete Podman runtime (and without the Linux dependency). Should not change behavior. Signed-off-by: Miloslav Trmač --- pkg/domain/infra/abi/trust.go | 68 +------------------------------------------ pkg/trust/policy.go | 10 +++++++ pkg/trust/trust.go | 68 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 79 insertions(+), 67 deletions(-) (limited to 'pkg/domain') diff --git a/pkg/domain/infra/abi/trust.go b/pkg/domain/infra/abi/trust.go index 381ea5deb..c58ddff06 100644 --- a/pkg/domain/infra/abi/trust.go +++ b/pkg/domain/infra/abi/trust.go @@ -4,11 +4,9 @@ import ( "context" "fmt" "io/ioutil" - "strings" "github.com/containers/podman/v4/pkg/domain/entities" "github.com/containers/podman/v4/pkg/trust" - "github.com/sirupsen/logrus" ) func (ir *ImageEngine) ShowTrust(ctx context.Context, args []string, options entities.ShowTrustOptions) (*entities.ShowTrustReport, error) { @@ -31,11 +29,7 @@ func (ir *ImageEngine) ShowTrust(ctx context.Context, args []string, options ent if len(options.RegistryPath) > 0 { report.SystemRegistriesDirPath = options.RegistryPath } - policyContentStruct, err := trust.GetPolicy(policyPath) - if err != nil { - return nil, fmt.Errorf("could not read trust policies: %w", err) - } - report.Policies, err = getPolicyShowOutput(policyContentStruct, report.SystemRegistriesDirPath) + report.Policies, err = trust.PolicyDescription(policyPath, report.SystemRegistriesDirPath) if err != nil { return nil, fmt.Errorf("could not show trust policies: %w", err) } @@ -59,63 +53,3 @@ func (ir *ImageEngine) SetTrust(ctx context.Context, args []string, options enti PubKeyFiles: options.PubKeysFile, }) } - -func getPolicyShowOutput(policyContentStruct trust.PolicyContent, systemRegistriesDirPath string) ([]*trust.Policy, error) { - var output []*trust.Policy - - registryConfigs, err := trust.LoadAndMergeConfig(systemRegistriesDirPath) - if err != nil { - return nil, err - } - - if len(policyContentStruct.Default) > 0 { - defaultPolicyStruct := trust.Policy{ - Transport: "all", - Name: "* (default)", - RepoName: "default", - Type: trustTypeDescription(policyContentStruct.Default[0].Type), - } - output = append(output, &defaultPolicyStruct) - } - for transport, transval := range policyContentStruct.Transports { - if transport == "docker" { - transport = "repository" - } - - for repo, repoval := range transval { - tempTrustShowOutput := trust.Policy{ - Name: repo, - RepoName: repo, - Transport: transport, - Type: trustTypeDescription(repoval[0].Type), - } - uids := []string{} - for _, repoele := range repoval { - if len(repoele.KeyPath) > 0 { - uids = append(uids, trust.GetGPGIdFromKeyPath(repoele.KeyPath)...) - } - if len(repoele.KeyData) > 0 { - uids = append(uids, trust.GetGPGIdFromKeyData(repoele.KeyData)...) - } - } - tempTrustShowOutput.GPGId = strings.Join(uids, ", ") - - registryNamespace := trust.HaveMatchRegistry(repo, registryConfigs) - if registryNamespace != nil { - tempTrustShowOutput.SignatureStore = registryNamespace.SigStore - } - output = append(output, &tempTrustShowOutput) - } - } - return output, nil -} - -var typeDescription = map[string]string{"insecureAcceptAnything": "accept", "signedBy": "signed", "reject": "reject"} - -func trustTypeDescription(trustType string) string { - trustDescription, exist := typeDescription[trustType] - if !exist { - logrus.Warnf("Invalid trust type %s", trustType) - } - return trustDescription -} diff --git a/pkg/trust/policy.go b/pkg/trust/policy.go index 3a31b9338..0dc46eac3 100644 --- a/pkg/trust/policy.go +++ b/pkg/trust/policy.go @@ -125,6 +125,16 @@ func GetPolicy(policyPath string) (PolicyContent, error) { return policyContentStruct, nil } +var typeDescription = map[string]string{"insecureAcceptAnything": "accept", "signedBy": "signed", "reject": "reject"} + +func trustTypeDescription(trustType string) string { + trustDescription, exist := typeDescription[trustType] + if !exist { + logrus.Warnf("Invalid trust type %s", trustType) + } + return trustDescription +} + // AddPolicyEntriesInput collects some parameters to AddPolicyEntries, // primarily so that the callers use named values instead of just strings in a sequence. type AddPolicyEntriesInput struct { diff --git a/pkg/trust/trust.go b/pkg/trust/trust.go index 6186d4cbd..2813b126d 100644 --- a/pkg/trust/trust.go +++ b/pkg/trust/trust.go @@ -1,5 +1,10 @@ package trust +import ( + "fmt" + "strings" +) + // Policy describes a basic trust policy configuration type Policy struct { Transport string `json:"transport"` @@ -10,3 +15,66 @@ type Policy struct { Type string `json:"type"` GPGId string `json:"gpg_id,omitempty"` } + +// PolicyDescription returns an user-focused description of the policy in policyPath and registries.d data from registriesDirPath. +func PolicyDescription(policyPath, registriesDirPath string) ([]*Policy, error) { + policyContentStruct, err := GetPolicy(policyPath) + if err != nil { + return nil, fmt.Errorf("could not read trust policies: %w", err) + } + res, err := getPolicyShowOutput(policyContentStruct, registriesDirPath) + if err != nil { + return nil, fmt.Errorf("could not show trust policies: %w", err) + } + return res, nil +} + +func getPolicyShowOutput(policyContentStruct PolicyContent, systemRegistriesDirPath string) ([]*Policy, error) { + var output []*Policy + + registryConfigs, err := LoadAndMergeConfig(systemRegistriesDirPath) + if err != nil { + return nil, err + } + + if len(policyContentStruct.Default) > 0 { + defaultPolicyStruct := Policy{ + Transport: "all", + Name: "* (default)", + RepoName: "default", + Type: trustTypeDescription(policyContentStruct.Default[0].Type), + } + output = append(output, &defaultPolicyStruct) + } + for transport, transval := range policyContentStruct.Transports { + if transport == "docker" { + transport = "repository" + } + + for repo, repoval := range transval { + tempTrustShowOutput := Policy{ + Name: repo, + RepoName: repo, + Transport: transport, + Type: trustTypeDescription(repoval[0].Type), + } + uids := []string{} + for _, repoele := range repoval { + if len(repoele.KeyPath) > 0 { + uids = append(uids, GetGPGIdFromKeyPath(repoele.KeyPath)...) + } + if len(repoele.KeyData) > 0 { + uids = append(uids, GetGPGIdFromKeyData(repoele.KeyData)...) + } + } + tempTrustShowOutput.GPGId = strings.Join(uids, ", ") + + registryNamespace := HaveMatchRegistry(repo, registryConfigs) + if registryNamespace != nil { + tempTrustShowOutput.SignatureStore = registryNamespace.SigStore + } + output = append(output, &tempTrustShowOutput) + } + } + return output, nil +} -- cgit v1.2.3-54-g00ecf From 0b3184a5acb68043a6cf44b7c223d84d8badd930 Mon Sep 17 00:00:00 2001 From: Doug Rabson Date: Fri, 26 Aug 2022 15:32:52 +0100 Subject: pkg/domain: Add terminal support for FreeBSD This just moves the code to files which can be shared with freebsd. [NO NEW TESTS NEEDED] Signed-off-by: Doug Rabson --- pkg/domain/infra/abi/terminal/sigproxy_commn.go | 63 ++++++++++++ pkg/domain/infra/abi/terminal/sigproxy_linux.go | 60 ----------- pkg/domain/infra/abi/terminal/terminal_common.go | 113 +++++++++++++++++++++ pkg/domain/infra/abi/terminal/terminal_linux.go | 110 -------------------- .../infra/abi/terminal/terminal_unsupported.go | 4 +- 5 files changed, 178 insertions(+), 172 deletions(-) create mode 100644 pkg/domain/infra/abi/terminal/sigproxy_commn.go delete mode 100644 pkg/domain/infra/abi/terminal/sigproxy_linux.go create mode 100644 pkg/domain/infra/abi/terminal/terminal_common.go delete mode 100644 pkg/domain/infra/abi/terminal/terminal_linux.go (limited to 'pkg/domain') diff --git a/pkg/domain/infra/abi/terminal/sigproxy_commn.go b/pkg/domain/infra/abi/terminal/sigproxy_commn.go new file mode 100644 index 000000000..3a0132ef3 --- /dev/null +++ b/pkg/domain/infra/abi/terminal/sigproxy_commn.go @@ -0,0 +1,63 @@ +//go:build linux || freebsd +// +build linux freebsd + +package terminal + +import ( + "errors" + "os" + "syscall" + + "github.com/containers/podman/v4/libpod" + "github.com/containers/podman/v4/libpod/define" + "github.com/containers/podman/v4/libpod/shutdown" + "github.com/containers/podman/v4/pkg/signal" + "github.com/sirupsen/logrus" +) + +// Make sure the signal buffer is sufficiently big. +// runc is using the same value. +const signalBufferSize = 2048 + +// ProxySignals ... +func ProxySignals(ctr *libpod.Container) { + // Stop catching the shutdown signals (SIGINT, SIGTERM) - they're going + // to the container now. + shutdown.Stop() //nolint: errcheck + + sigBuffer := make(chan os.Signal, signalBufferSize) + signal.CatchAll(sigBuffer) + + logrus.Debugf("Enabling signal proxying") + + go func() { + for s := range sigBuffer { + // Ignore SIGCHLD and SIGPIPE - these are mostly likely + // intended for the podman command itself. + // SIGURG was added because of golang 1.14 and its preemptive changes + // causing more signals to "show up". + // https://github.com/containers/podman/issues/5483 + if s == syscall.SIGCHLD || s == syscall.SIGPIPE || s == syscall.SIGURG { + continue + } + + if err := ctr.Kill(uint(s.(syscall.Signal))); err != nil { + if errors.Is(err, define.ErrCtrStateInvalid) { + logrus.Infof("Ceasing signal forwarding to container %s as it has stopped", ctr.ID()) + } else { + logrus.Errorf("forwarding signal %d to container %s: %v", s, ctr.ID(), err) + } + // If the container dies, and we find out here, + // we need to forward that one signal to + // ourselves so that it is not lost, and then + // we terminate the proxy and let the defaults + // play out. + signal.StopCatch(sigBuffer) + if err := syscall.Kill(syscall.Getpid(), s.(syscall.Signal)); err != nil { + logrus.Errorf("Failed to kill pid %d", syscall.Getpid()) + } + return + } + } + }() +} diff --git a/pkg/domain/infra/abi/terminal/sigproxy_linux.go b/pkg/domain/infra/abi/terminal/sigproxy_linux.go deleted file mode 100644 index 16d345f06..000000000 --- a/pkg/domain/infra/abi/terminal/sigproxy_linux.go +++ /dev/null @@ -1,60 +0,0 @@ -package terminal - -import ( - "errors" - "os" - "syscall" - - "github.com/containers/podman/v4/libpod" - "github.com/containers/podman/v4/libpod/define" - "github.com/containers/podman/v4/libpod/shutdown" - "github.com/containers/podman/v4/pkg/signal" - "github.com/sirupsen/logrus" -) - -// Make sure the signal buffer is sufficiently big. -// runc is using the same value. -const signalBufferSize = 2048 - -// ProxySignals ... -func ProxySignals(ctr *libpod.Container) { - // Stop catching the shutdown signals (SIGINT, SIGTERM) - they're going - // to the container now. - shutdown.Stop() //nolint: errcheck - - sigBuffer := make(chan os.Signal, signalBufferSize) - signal.CatchAll(sigBuffer) - - logrus.Debugf("Enabling signal proxying") - - go func() { - for s := range sigBuffer { - // Ignore SIGCHLD and SIGPIPE - these are mostly likely - // intended for the podman command itself. - // SIGURG was added because of golang 1.14 and its preemptive changes - // causing more signals to "show up". - // https://github.com/containers/podman/issues/5483 - if s == syscall.SIGCHLD || s == syscall.SIGPIPE || s == syscall.SIGURG { - continue - } - - if err := ctr.Kill(uint(s.(syscall.Signal))); err != nil { - if errors.Is(err, define.ErrCtrStateInvalid) { - logrus.Infof("Ceasing signal forwarding to container %s as it has stopped", ctr.ID()) - } else { - logrus.Errorf("forwarding signal %d to container %s: %v", s, ctr.ID(), err) - } - // If the container dies, and we find out here, - // we need to forward that one signal to - // ourselves so that it is not lost, and then - // we terminate the proxy and let the defaults - // play out. - signal.StopCatch(sigBuffer) - if err := syscall.Kill(syscall.Getpid(), s.(syscall.Signal)); err != nil { - logrus.Errorf("Failed to kill pid %d", syscall.Getpid()) - } - return - } - } - }() -} diff --git a/pkg/domain/infra/abi/terminal/terminal_common.go b/pkg/domain/infra/abi/terminal/terminal_common.go new file mode 100644 index 000000000..afae2c085 --- /dev/null +++ b/pkg/domain/infra/abi/terminal/terminal_common.go @@ -0,0 +1,113 @@ +//go:build linux || freebsd +// +build linux freebsd + +package terminal + +import ( + "bufio" + "context" + "fmt" + "os" + + "github.com/containers/common/pkg/resize" + "github.com/containers/podman/v4/libpod" + "github.com/containers/podman/v4/libpod/define" + "github.com/sirupsen/logrus" + "golang.org/x/term" +) + +// ExecAttachCtr execs and attaches to a container +func ExecAttachCtr(ctx context.Context, ctr *libpod.Container, execConfig *libpod.ExecConfig, streams *define.AttachStreams) (int, error) { + var resizechan chan resize.TerminalSize + haveTerminal := term.IsTerminal(int(os.Stdin.Fd())) + + // Check if we are attached to a terminal. If we are, generate resize + // events, and set the terminal to raw mode + if haveTerminal && execConfig.Terminal { + resizechan = make(chan resize.TerminalSize) + cancel, oldTermState, err := handleTerminalAttach(ctx, resizechan) + if err != nil { + return -1, err + } + defer cancel() + defer func() { + if err := restoreTerminal(oldTermState); err != nil { + logrus.Errorf("Unable to restore terminal: %q", err) + } + }() + } + return ctr.Exec(execConfig, streams, resizechan) +} + +// StartAttachCtr starts and (if required) attaches to a container +// if you change the signature of this function from os.File to io.Writer, it will trigger a downstream +// error. we may need to just lint disable this one. +func StartAttachCtr(ctx context.Context, ctr *libpod.Container, stdout, stderr, stdin *os.File, detachKeys string, sigProxy bool, startContainer bool) error { //nolint: interfacer + resize := make(chan resize.TerminalSize) + + haveTerminal := term.IsTerminal(int(os.Stdin.Fd())) + + // Check if we are attached to a terminal. If we are, generate resize + // events, and set the terminal to raw mode + if haveTerminal && ctr.Spec().Process.Terminal { + cancel, oldTermState, err := handleTerminalAttach(ctx, resize) + if err != nil { + return err + } + defer func() { + if err := restoreTerminal(oldTermState); err != nil { + logrus.Errorf("Unable to restore terminal: %q", err) + } + }() + defer cancel() + } + + streams := new(define.AttachStreams) + streams.OutputStream = stdout + streams.ErrorStream = stderr + streams.InputStream = bufio.NewReader(stdin) + streams.AttachOutput = true + streams.AttachError = true + streams.AttachInput = true + + if stdout == nil { + logrus.Debugf("Not attaching to stdout") + streams.AttachOutput = false + } + if stderr == nil { + logrus.Debugf("Not attaching to stderr") + streams.AttachError = false + } + if stdin == nil { + logrus.Debugf("Not attaching to stdin") + streams.AttachInput = false + } + + if !startContainer { + if sigProxy { + ProxySignals(ctr) + } + + return ctr.Attach(streams, detachKeys, resize) + } + + attachChan, err := ctr.StartAndAttach(ctx, streams, detachKeys, resize, true) + if err != nil { + return err + } + + if sigProxy { + ProxySignals(ctr) + } + + if stdout == nil && stderr == nil { + fmt.Printf("%s\n", ctr.ID()) + } + + err = <-attachChan + if err != nil { + return fmt.Errorf("error attaching to container %s: %w", ctr.ID(), err) + } + + return nil +} diff --git a/pkg/domain/infra/abi/terminal/terminal_linux.go b/pkg/domain/infra/abi/terminal/terminal_linux.go deleted file mode 100644 index 222590871..000000000 --- a/pkg/domain/infra/abi/terminal/terminal_linux.go +++ /dev/null @@ -1,110 +0,0 @@ -package terminal - -import ( - "bufio" - "context" - "fmt" - "os" - - "github.com/containers/common/pkg/resize" - "github.com/containers/podman/v4/libpod" - "github.com/containers/podman/v4/libpod/define" - "github.com/sirupsen/logrus" - "golang.org/x/term" -) - -// ExecAttachCtr execs and attaches to a container -func ExecAttachCtr(ctx context.Context, ctr *libpod.Container, execConfig *libpod.ExecConfig, streams *define.AttachStreams) (int, error) { - var resizechan chan resize.TerminalSize - haveTerminal := term.IsTerminal(int(os.Stdin.Fd())) - - // Check if we are attached to a terminal. If we are, generate resize - // events, and set the terminal to raw mode - if haveTerminal && execConfig.Terminal { - resizechan = make(chan resize.TerminalSize) - cancel, oldTermState, err := handleTerminalAttach(ctx, resizechan) - if err != nil { - return -1, err - } - defer cancel() - defer func() { - if err := restoreTerminal(oldTermState); err != nil { - logrus.Errorf("Unable to restore terminal: %q", err) - } - }() - } - return ctr.Exec(execConfig, streams, resizechan) -} - -// StartAttachCtr starts and (if required) attaches to a container -// if you change the signature of this function from os.File to io.Writer, it will trigger a downstream -// error. we may need to just lint disable this one. -func StartAttachCtr(ctx context.Context, ctr *libpod.Container, stdout, stderr, stdin *os.File, detachKeys string, sigProxy bool, startContainer bool) error { //nolint: interfacer - resize := make(chan resize.TerminalSize) - - haveTerminal := term.IsTerminal(int(os.Stdin.Fd())) - - // Check if we are attached to a terminal. If we are, generate resize - // events, and set the terminal to raw mode - if haveTerminal && ctr.Spec().Process.Terminal { - cancel, oldTermState, err := handleTerminalAttach(ctx, resize) - if err != nil { - return err - } - defer func() { - if err := restoreTerminal(oldTermState); err != nil { - logrus.Errorf("Unable to restore terminal: %q", err) - } - }() - defer cancel() - } - - streams := new(define.AttachStreams) - streams.OutputStream = stdout - streams.ErrorStream = stderr - streams.InputStream = bufio.NewReader(stdin) - streams.AttachOutput = true - streams.AttachError = true - streams.AttachInput = true - - if stdout == nil { - logrus.Debugf("Not attaching to stdout") - streams.AttachOutput = false - } - if stderr == nil { - logrus.Debugf("Not attaching to stderr") - streams.AttachError = false - } - if stdin == nil { - logrus.Debugf("Not attaching to stdin") - streams.AttachInput = false - } - - if !startContainer { - if sigProxy { - ProxySignals(ctr) - } - - return ctr.Attach(streams, detachKeys, resize) - } - - attachChan, err := ctr.StartAndAttach(ctx, streams, detachKeys, resize, true) - if err != nil { - return err - } - - if sigProxy { - ProxySignals(ctr) - } - - if stdout == nil && stderr == nil { - fmt.Printf("%s\n", ctr.ID()) - } - - err = <-attachChan - if err != nil { - return fmt.Errorf("error attaching to container %s: %w", ctr.ID(), err) - } - - return nil -} diff --git a/pkg/domain/infra/abi/terminal/terminal_unsupported.go b/pkg/domain/infra/abi/terminal/terminal_unsupported.go index 8fe325736..21ed6c8d4 100644 --- a/pkg/domain/infra/abi/terminal/terminal_unsupported.go +++ b/pkg/domain/infra/abi/terminal/terminal_unsupported.go @@ -1,5 +1,5 @@ -//go:build !linux -// +build !linux +//go:build !linux && !freebsd +// +build !linux,!freebsd package terminal -- cgit v1.2.3-54-g00ecf