From fde39edb9cc173871925a66df3941917e33bf45e Mon Sep 17 00:00:00 2001 From: Valentin Rothberg Date: Thu, 21 Jul 2022 14:09:44 +0200 Subject: remote push: show copy progress `podman-remote push` has shown absolutely no progress at all. Fix that by doing essentially the same as the remote-pull code does. The get-free-out-of-jail-card for backwards compatibility is to let the `quiet` parameter default to true. Since the --quioet flag wasn't working before either, older Podman clients do not set it. Also add regression tests to make sure we won't regress again. Fixes: #11554 Fixes: #14971 Signed-off-by: Valentin Rothberg --- pkg/api/handlers/libpod/images.go | 71 ---------------- pkg/api/handlers/libpod/images_push.go | 144 +++++++++++++++++++++++++++++++++ 2 files changed, 144 insertions(+), 71 deletions(-) create mode 100644 pkg/api/handlers/libpod/images_push.go (limited to 'pkg/api/handlers/libpod') diff --git a/pkg/api/handlers/libpod/images.go b/pkg/api/handlers/libpod/images.go index ed1c65f8e..67943ecf1 100644 --- a/pkg/api/handlers/libpod/images.go +++ b/pkg/api/handlers/libpod/images.go @@ -1,7 +1,6 @@ package libpod import ( - "context" "errors" "fmt" "io" @@ -14,13 +13,11 @@ import ( "github.com/containers/buildah" "github.com/containers/common/libimage" "github.com/containers/image/v5/manifest" - "github.com/containers/image/v5/types" "github.com/containers/podman/v4/libpod" "github.com/containers/podman/v4/libpod/define" "github.com/containers/podman/v4/pkg/api/handlers" "github.com/containers/podman/v4/pkg/api/handlers/utils" api "github.com/containers/podman/v4/pkg/api/types" - "github.com/containers/podman/v4/pkg/auth" "github.com/containers/podman/v4/pkg/domain/entities" "github.com/containers/podman/v4/pkg/domain/entities/reports" "github.com/containers/podman/v4/pkg/domain/infra/abi" @@ -416,74 +413,6 @@ func ImagesImport(w http.ResponseWriter, r *http.Request) { utils.WriteResponse(w, http.StatusOK, report) } -// PushImage is the handler for the compat http endpoint for pushing images. -func PushImage(w http.ResponseWriter, r *http.Request) { - decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) - runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) - - query := struct { - All bool `schema:"all"` - Destination string `schema:"destination"` - Format string `schema:"format"` - RemoveSignatures bool `schema:"removeSignatures"` - TLSVerify bool `schema:"tlsVerify"` - }{ - // This is where you can override the golang default value for one of fields - } - if err := decoder.Decode(&query, r.URL.Query()); err != nil { - utils.Error(w, http.StatusBadRequest, fmt.Errorf("failed to parse parameters for %s: %w", r.URL.String(), err)) - return - } - - source := strings.TrimSuffix(utils.GetName(r), "/push") // GetName returns the entire path - if _, err := utils.ParseStorageReference(source); err != nil { - utils.Error(w, http.StatusBadRequest, err) - return - } - - destination := query.Destination - if destination == "" { - destination = source - } - - if err := utils.IsRegistryReference(destination); err != nil { - utils.Error(w, http.StatusBadRequest, err) - return - } - - authconf, authfile, err := auth.GetCredentials(r) - if err != nil { - utils.Error(w, http.StatusBadRequest, err) - return - } - defer auth.RemoveAuthfile(authfile) - var username, password string - if authconf != nil { - username = authconf.Username - password = authconf.Password - } - options := entities.ImagePushOptions{ - All: query.All, - Authfile: authfile, - Format: query.Format, - Password: password, - Quiet: true, - RemoveSignatures: query.RemoveSignatures, - Username: username, - } - if _, found := r.URL.Query()["tlsVerify"]; found { - options.SkipTLSVerify = types.NewOptionalBool(!query.TLSVerify) - } - - imageEngine := abi.ImageEngine{Libpod: runtime} - if err := imageEngine.Push(context.Background(), source, destination, options); err != nil { - utils.Error(w, http.StatusBadRequest, fmt.Errorf("error pushing image %q: %w", destination, err)) - return - } - - utils.WriteResponse(w, http.StatusOK, "") -} - func CommitContainer(w http.ResponseWriter, r *http.Request) { var ( destImage string diff --git a/pkg/api/handlers/libpod/images_push.go b/pkg/api/handlers/libpod/images_push.go new file mode 100644 index 000000000..f427dc01b --- /dev/null +++ b/pkg/api/handlers/libpod/images_push.go @@ -0,0 +1,144 @@ +package libpod + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "strings" + + "github.com/containers/image/v5/types" + "github.com/containers/podman/v4/libpod" + "github.com/containers/podman/v4/pkg/api/handlers/utils" + api "github.com/containers/podman/v4/pkg/api/types" + "github.com/containers/podman/v4/pkg/auth" + "github.com/containers/podman/v4/pkg/channel" + "github.com/containers/podman/v4/pkg/domain/entities" + "github.com/containers/podman/v4/pkg/domain/infra/abi" + "github.com/gorilla/schema" + "github.com/sirupsen/logrus" +) + +// PushImage is the handler for the compat http endpoint for pushing images. +func PushImage(w http.ResponseWriter, r *http.Request) { + decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) + runtime := r.Context().Value(api.RuntimeKey).(*libpod.Runtime) + + query := struct { + All bool `schema:"all"` + Destination string `schema:"destination"` + Format string `schema:"format"` + RemoveSignatures bool `schema:"removeSignatures"` + TLSVerify bool `schema:"tlsVerify"` + Quiet bool `schema:"quiet"` + }{ + // #14971: older versions did not sent *any* data, so we need + // to be quiet by default to remain backwards compatible + Quiet: true, + } + if err := decoder.Decode(&query, r.URL.Query()); err != nil { + utils.Error(w, http.StatusBadRequest, fmt.Errorf("failed to parse parameters for %s: %w", r.URL.String(), err)) + return + } + + source := strings.TrimSuffix(utils.GetName(r), "/push") // GetName returns the entire path + if _, err := utils.ParseStorageReference(source); err != nil { + utils.Error(w, http.StatusBadRequest, err) + return + } + + destination := query.Destination + if destination == "" { + destination = source + } + + if err := utils.IsRegistryReference(destination); err != nil { + utils.Error(w, http.StatusBadRequest, err) + return + } + + authconf, authfile, err := auth.GetCredentials(r) + if err != nil { + utils.Error(w, http.StatusBadRequest, err) + return + } + defer auth.RemoveAuthfile(authfile) + + var username, password string + if authconf != nil { + username = authconf.Username + password = authconf.Password + } + options := entities.ImagePushOptions{ + All: query.All, + Authfile: authfile, + Format: query.Format, + Password: password, + Quiet: true, + RemoveSignatures: query.RemoveSignatures, + Username: username, + } + + if _, found := r.URL.Query()["tlsVerify"]; found { + options.SkipTLSVerify = types.NewOptionalBool(!query.TLSVerify) + } + + imageEngine := abi.ImageEngine{Libpod: runtime} + + // Let's keep thing simple when running in quiet mode and push directly. + if query.Quiet { + if err := imageEngine.Push(context.Background(), source, destination, options); err != nil { + utils.Error(w, http.StatusBadRequest, fmt.Errorf("error pushing image %q: %w", destination, err)) + return + } + utils.WriteResponse(w, http.StatusOK, "") + return + } + + writer := channel.NewWriter(make(chan []byte)) + defer writer.Close() + options.Writer = writer + + pushCtx, pushCancel := context.WithCancel(r.Context()) + var pushError error + go func() { + defer pushCancel() + pushError = imageEngine.Push(pushCtx, source, destination, options) + }() + + flush := func() { + if flusher, ok := w.(http.Flusher); ok { + flusher.Flush() + } + } + + w.WriteHeader(http.StatusOK) + w.Header().Set("Content-Type", "application/json") + flush() + + enc := json.NewEncoder(w) + enc.SetEscapeHTML(true) + for { + var report entities.ImagePushReport + select { + case s := <-writer.Chan(): + report.Stream = string(s) + if err := enc.Encode(report); err != nil { + logrus.Warnf("Failed to encode json: %v", err) + } + flush() + case <-pushCtx.Done(): + if pushError != nil { + report.Error = pushError.Error() + if err := enc.Encode(report); err != nil { + logrus.Warnf("Failed to encode json: %v", err) + } + } + flush() + return + case <-r.Context().Done(): + // Client has closed connection + return + } + } +} -- cgit v1.2.3-54-g00ecf From c4616510a2c7509be20120353e539804b161922d Mon Sep 17 00:00:00 2001 From: Paul Holzinger Date: Fri, 22 Jul 2022 14:16:25 +0200 Subject: API: libpod/create use correct default umask Make sure containers created via API have the correct umask from containers.conf set. Fixes #15036 Signed-off-by: Paul Holzinger --- pkg/api/handlers/libpod/containers_create.go | 3 +++ test/apiv2/20-containers.at | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) (limited to 'pkg/api/handlers/libpod') diff --git a/pkg/api/handlers/libpod/containers_create.go b/pkg/api/handlers/libpod/containers_create.go index e4964d602..1307c267a 100644 --- a/pkg/api/handlers/libpod/containers_create.go +++ b/pkg/api/handlers/libpod/containers_create.go @@ -31,6 +31,9 @@ func CreateContainer(w http.ResponseWriter, r *http.Request) { ContainerNetworkConfig: specgen.ContainerNetworkConfig{ UseImageHosts: conf.Containers.NoHosts, }, + ContainerSecurityConfig: specgen.ContainerSecurityConfig{ + Umask: conf.Containers.Umask, + }, } if err := json.NewDecoder(r.Body).Decode(&sg); err != nil { diff --git a/test/apiv2/20-containers.at b/test/apiv2/20-containers.at index 6ef4ef917..a8d9baef3 100644 --- a/test/apiv2/20-containers.at +++ b/test/apiv2/20-containers.at @@ -123,7 +123,8 @@ t GET libpod/containers/${cid}/json 200 \ .Id=$cid \ .State.Status~\\\(exited\\\|stopped\\\) \ .State.Running=false \ - .State.ExitCode=0 + .State.ExitCode=0 \ + .Config.Umask=0022 # regression check for #15036 t DELETE libpod/containers/$cid 200 .[0].Id=$cid CNAME=myfoo -- cgit v1.2.3-54-g00ecf From e6ebfbd1e0106d8ddcf19a1ec3f97052592f49ad Mon Sep 17 00:00:00 2001 From: Vladimir Kochnev Date: Mon, 25 Jul 2022 16:00:23 +0300 Subject: Set TLSVerify=true by default for API endpoints Option defaults in API must be the same as in CLI. ``` % podman image push --help % podman image pull --help % podman manifest push --help % podman image search --help ``` All of these CLI commands them have --tls-verify=true by default: ``` --tls-verify require HTTPS and verify certificates when accessing the registry (default true) ``` As for `podman image build`, it doesn't have any means to control `tlsVerify` parameter but it must be true by default. Signed-off-by: Vladimir Kochnev --- pkg/api/handlers/compat/images_build.go | 1 + pkg/api/handlers/compat/images_search.go | 1 + pkg/api/handlers/libpod/images_push.go | 1 + pkg/api/handlers/libpod/manifests.go | 1 + pkg/api/server/register_images.go | 8 ++++---- pkg/api/server/register_manifest.go | 10 +++++----- test/apiv2/12-imagesMore.at | 5 ++++- test/apiv2/15-manifest.at | 2 ++ 8 files changed, 19 insertions(+), 10 deletions(-) (limited to 'pkg/api/handlers/libpod') diff --git a/pkg/api/handlers/compat/images_build.go b/pkg/api/handlers/compat/images_build.go index a9185c3d3..15cfc824e 100644 --- a/pkg/api/handlers/compat/images_build.go +++ b/pkg/api/handlers/compat/images_build.go @@ -140,6 +140,7 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { Registry: "docker.io", Rm: true, ShmSize: 64 * 1024 * 1024, + TLSVerify: true, } decoder := r.Context().Value(api.DecoderKey).(*schema.Decoder) diff --git a/pkg/api/handlers/compat/images_search.go b/pkg/api/handlers/compat/images_search.go index a6fd3a3a1..2fc95e84e 100644 --- a/pkg/api/handlers/compat/images_search.go +++ b/pkg/api/handlers/compat/images_search.go @@ -26,6 +26,7 @@ func SearchImages(w http.ResponseWriter, r *http.Request) { ListTags bool `json:"listTags"` }{ // This is where you can override the golang default value for one of fields + TLSVerify: true, } if err := decoder.Decode(&query, r.URL.Query()); err != nil { diff --git a/pkg/api/handlers/libpod/images_push.go b/pkg/api/handlers/libpod/images_push.go index f427dc01b..9ee651f5b 100644 --- a/pkg/api/handlers/libpod/images_push.go +++ b/pkg/api/handlers/libpod/images_push.go @@ -32,6 +32,7 @@ func PushImage(w http.ResponseWriter, r *http.Request) { TLSVerify bool `schema:"tlsVerify"` Quiet bool `schema:"quiet"` }{ + TLSVerify: true, // #14971: older versions did not sent *any* data, so we need // to be quiet by default to remain backwards compatible Quiet: true, diff --git a/pkg/api/handlers/libpod/manifests.go b/pkg/api/handlers/libpod/manifests.go index 3235a2972..43c7139d3 100644 --- a/pkg/api/handlers/libpod/manifests.go +++ b/pkg/api/handlers/libpod/manifests.go @@ -310,6 +310,7 @@ func ManifestPush(w http.ResponseWriter, r *http.Request) { TLSVerify bool `schema:"tlsVerify"` }{ // Add defaults here once needed. + TLSVerify: true, } if err := decoder.Decode(&query, r.URL.Query()); err != nil { utils.Error(w, http.StatusBadRequest, diff --git a/pkg/api/server/register_images.go b/pkg/api/server/register_images.go index 11ab8cae0..1bfedd77e 100644 --- a/pkg/api/server/register_images.go +++ b/pkg/api/server/register_images.go @@ -192,8 +192,8 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // - in: query // name: tlsVerify // type: boolean - // default: false - // description: skip TLS verification for registries + // default: true + // description: Require HTTPS and verify signatures when contacting registries. // - in: query // name: listTags // type: boolean @@ -1120,8 +1120,8 @@ func (s *APIServer) registerImagesHandlers(r *mux.Router) error { // - in: query // name: tlsVerify // type: boolean - // default: false - // description: skip TLS verification for registries + // default: true + // description: Require HTTPS and verify signatures when contacting registries. // - in: query // name: listTags // type: boolean diff --git a/pkg/api/server/register_manifest.go b/pkg/api/server/register_manifest.go index 4fadb92fd..19b507047 100644 --- a/pkg/api/server/register_manifest.go +++ b/pkg/api/server/register_manifest.go @@ -69,12 +69,12 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // name: all // description: push all images // type: boolean - // default: false + // default: true // - in: query // name: tlsVerify // type: boolean - // default: false - // description: skip TLS verification for registries + // default: true + // description: Require HTTPS and verify signatures when contacting registries. // responses: // 200: // schema: @@ -195,8 +195,8 @@ func (s *APIServer) registerManifestHandlers(r *mux.Router) error { // - in: query // name: tlsVerify // type: boolean - // default: false - // description: skip TLS verification for registries + // default: true + // description: Require HTTPS and verify signatures when contacting registries. // - in: body // name: options // description: options for mutating a manifest diff --git a/test/apiv2/12-imagesMore.at b/test/apiv2/12-imagesMore.at index d4b09174f..498d67569 100644 --- a/test/apiv2/12-imagesMore.at +++ b/test/apiv2/12-imagesMore.at @@ -28,7 +28,10 @@ t GET libpod/images/$IMAGE/json 200 \ .RepoTags[1]=localhost:$REGISTRY_PORT/myrepo:mytag # Push to local registry... -t POST "images/localhost:$REGISTRY_PORT/myrepo/push?tlsVerify=false&tag=mytag" 200 +t POST "images/localhost:$REGISTRY_PORT/myrepo/push?tag=mytag" 200 \ + .error~".*x509: certificate signed by unknown authority" +t POST "images/localhost:$REGISTRY_PORT/myrepo/push?tlsVerify=false&tag=mytag" 200 \ + .error~null # ...and check output. We can't use our built-in checks because this output # is a sequence of JSON objects, i.e., individual ones, not in a JSON array. diff --git a/test/apiv2/15-manifest.at b/test/apiv2/15-manifest.at index 970bed5a8..6584ea8e4 100644 --- a/test/apiv2/15-manifest.at +++ b/test/apiv2/15-manifest.at @@ -31,6 +31,8 @@ t POST /v3.4.0/libpod/manifests/$id_abc/add images="[\"containers-storage:$id_ab t PUT /v4.0.0/libpod/manifests/$id_xyz operation='update' images="[\"containers-storage:$id_xyz_image\"]" 200 t POST "/v3.4.0/libpod/manifests/abc:latest/push?destination=localhost:$REGISTRY_PORT%2Fabc:latest&tlsVerify=false&all=true" 200 +t POST "/v4.0.0/libpod/manifests/xyz:latest/registry/localhost:$REGISTRY_PORT%2Fxyz:latest?all=true" 400 \ + .cause='x509: certificate signed by unknown authority' t POST "/v4.0.0/libpod/manifests/xyz:latest/registry/localhost:$REGISTRY_PORT%2Fxyz:latest?tlsVerify=false&all=true" 200 # /v3.x cannot delete a manifest list -- cgit v1.2.3-54-g00ecf