diff options
Diffstat (limited to 'pkg')
27 files changed, 439 insertions, 636 deletions
diff --git a/pkg/api/handlers/compat/images_build.go b/pkg/api/handlers/compat/images_build.go index 15cfc824e..a00f0b089 100644 --- a/pkg/api/handlers/compat/images_build.go +++ b/pkg/api/handlers/compat/images_build.go @@ -17,6 +17,7 @@ import ( "github.com/containers/buildah" buildahDefine "github.com/containers/buildah/define" "github.com/containers/buildah/pkg/parse" + "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/types" "github.com/containers/podman/v4/libpod" "github.com/containers/podman/v4/pkg/api/handlers/utils" @@ -78,6 +79,8 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { AppArmor string `schema:"apparmor"` BuildArgs string `schema:"buildargs"` CacheFrom string `schema:"cachefrom"` + CacheTo string `schema:"cacheto"` + CacheTTL string `schema:"cachettl"` CgroupParent string `schema:"cgroupparent"` Compression uint64 `schema:"compression"` ConfigureNetwork string `schema:"networkmode"` @@ -386,6 +389,31 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { } } + var cacheFrom reference.Named + if _, found := r.URL.Query()["cachefrom"]; found { + cacheFrom, err = parse.RepoNameToNamedReference(query.CacheFrom) + if err != nil { + utils.BadRequest(w, "cacheFrom", query.CacheFrom, err) + return + } + } + var cacheTo reference.Named + if _, found := r.URL.Query()["cacheto"]; found { + cacheTo, err = parse.RepoNameToNamedReference(query.CacheTo) + if err != nil { + utils.BadRequest(w, "cacheto", query.CacheTo, err) + return + } + } + var cacheTTL time.Duration + if _, found := r.URL.Query()["cachettl"]; found { + cacheTTL, err = time.ParseDuration(query.CacheTTL) + if err != nil { + utils.BadRequest(w, "cachettl", query.CacheTTL, err) + return + } + } + var buildArgs = map[string]string{} if _, found := r.URL.Query()["buildargs"]; found { if err := json.Unmarshal([]byte(query.BuildArgs), &buildArgs); err != nil { @@ -578,6 +606,9 @@ func BuildImage(w http.ResponseWriter, r *http.Request) { AdditionalTags: additionalTags, Annotations: annotations, CPPFlags: cppflags, + CacheFrom: cacheFrom, + CacheTo: cacheTo, + CacheTTL: cacheTTL, Args: buildArgs, AllPlatforms: query.AllPlatforms, CommonBuildOpts: &buildah.CommonBuildOptions{ diff --git a/pkg/api/handlers/libpod/images.go b/pkg/api/handlers/libpod/images.go index 67943ecf1..82c1971cd 100644 --- a/pkg/api/handlers/libpod/images.go +++ b/pkg/api/handlers/libpod/images.go @@ -12,6 +12,7 @@ import ( "github.com/containers/buildah" "github.com/containers/common/libimage" + "github.com/containers/common/pkg/ssh" "github.com/containers/image/v5/manifest" "github.com/containers/podman/v4/libpod" "github.com/containers/podman/v4/libpod/define" @@ -547,6 +548,7 @@ func ImagesBatchRemove(w http.ResponseWriter, r *http.Request) { Ignore bool `schema:"ignore"` LookupManifest bool `schema:"lookupManifest"` Images []string `schema:"images"` + NoPrune bool `schema:"noprune"` }{} if err := decoder.Decode(&query, r.URL.Query()); err != nil { @@ -554,7 +556,7 @@ func ImagesBatchRemove(w http.ResponseWriter, r *http.Request) { return } - opts := entities.ImageRemoveOptions{All: query.All, Force: query.Force, Ignore: query.Ignore, LookupManifest: query.LookupManifest} + opts := entities.ImageRemoveOptions{All: query.All, Force: query.Force, Ignore: query.Ignore, LookupManifest: query.LookupManifest, NoPrune: query.NoPrune} imageEngine := abi.ImageEngine{Libpod: runtime} rmReport, rmErrors := imageEngine.Remove(r.Context(), query.Images, opts) strErrs := errorhandling.ErrorsToStrings(rmErrors) @@ -617,7 +619,7 @@ func ImageScp(w http.ResponseWriter, r *http.Request) { sourceArg := utils.GetName(r) - rep, source, dest, _, err := domainUtils.ExecuteTransfer(sourceArg, query.Destination, []string{}, query.Quiet) + rep, source, dest, _, err := domainUtils.ExecuteTransfer(sourceArg, query.Destination, []string{}, query.Quiet, ssh.GolangMode) if err != nil { utils.Error(w, http.StatusInternalServerError, err) return diff --git a/pkg/bindings/connection.go b/pkg/bindings/connection.go index b994a5857..6d7b052b7 100644 --- a/pkg/bindings/connection.go +++ b/pkg/bindings/connection.go @@ -14,11 +14,9 @@ import ( "time" "github.com/blang/semver/v4" - "github.com/containers/podman/v4/pkg/terminal" + "github.com/containers/common/pkg/ssh" "github.com/containers/podman/v4/version" "github.com/sirupsen/logrus" - "golang.org/x/crypto/ssh" - "golang.org/x/crypto/ssh/agent" ) type APIResponse struct { @@ -74,8 +72,7 @@ func NewConnection(ctx context.Context, uri string) (context.Context, error) { // or ssh://<user>@<host>[:port]/run/podman/podman.sock?secure=True func NewConnectionWithIdentity(ctx context.Context, uri string, identity string) (context.Context, error) { var ( - err error - secure bool + err error ) if v, found := os.LookupEnv("CONTAINER_HOST"); found && uri == "" { uri = v @@ -85,11 +82,6 @@ func NewConnectionWithIdentity(ctx context.Context, uri string, identity string) identity = v } - passPhrase := "" - if v, found := os.LookupEnv("CONTAINER_PASSPHRASE"); found { - passPhrase = v - } - _url, err := url.Parse(uri) if err != nil { return nil, fmt.Errorf("value of CONTAINER_HOST is not a valid url: %s: %w", uri, err) @@ -99,11 +91,26 @@ func NewConnectionWithIdentity(ctx context.Context, uri string, identity string) var connection Connection switch _url.Scheme { case "ssh": - secure, err = strconv.ParseBool(_url.Query().Get("secure")) + port, err := strconv.Atoi(_url.Port()) if err != nil { - secure = false + return nil, err } - connection, err = sshClient(_url, secure, passPhrase, identity) + conn, err := ssh.Dial(&ssh.ConnectionDialOptions{ + Host: uri, + Identity: identity, + User: _url.User, + Port: port, + }, "golang") + if err != nil { + return nil, err + } + connection = Connection{URI: _url} + connection.Client = &http.Client{ + Transport: &http.Transport{ + DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { + return ssh.DialNet(conn, "unix", _url) + }, + }} case "unix": if !strings.HasPrefix(uri, "unix:///") { // autofix unix://path_element vs unix:///path_element @@ -184,124 +191,6 @@ func pingNewConnection(ctx context.Context) (*semver.Version, error) { return nil, fmt.Errorf("ping response was %d", response.StatusCode) } -func sshClient(_url *url.URL, secure bool, passPhrase string, identity string) (Connection, error) { - // if you modify the authmethods or their conditionals, you will also need to make similar - // changes in the client (currently cmd/podman/system/connection/add getUDS). - - var signers []ssh.Signer // order Signers are appended to this list determines which key is presented to server - - if len(identity) > 0 { - s, err := terminal.PublicKey(identity, []byte(passPhrase)) - if err != nil { - return Connection{}, fmt.Errorf("failed to parse identity %q: %w", identity, err) - } - - signers = append(signers, s) - logrus.Debugf("SSH Ident Key %q %s %s", identity, ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type()) - } - - if sock, found := os.LookupEnv("SSH_AUTH_SOCK"); found { - logrus.Debugf("Found SSH_AUTH_SOCK %q, ssh-agent signer(s) enabled", sock) - - c, err := net.Dial("unix", sock) - if err != nil { - return Connection{}, err - } - - agentSigners, err := agent.NewClient(c).Signers() - if err != nil { - return Connection{}, err - } - signers = append(signers, agentSigners...) - - if logrus.IsLevelEnabled(logrus.DebugLevel) { - for _, s := range agentSigners { - logrus.Debugf("SSH Agent Key %s %s", ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type()) - } - } - } - - var authMethods []ssh.AuthMethod - if len(signers) > 0 { - var dedup = make(map[string]ssh.Signer) - // Dedup signers based on fingerprint, ssh-agent keys override CONTAINER_SSHKEY - for _, s := range signers { - fp := ssh.FingerprintSHA256(s.PublicKey()) - if _, found := dedup[fp]; found { - logrus.Debugf("Dedup SSH Key %s %s", ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type()) - } - dedup[fp] = s - } - - var uniq []ssh.Signer - for _, s := range dedup { - uniq = append(uniq, s) - } - authMethods = append(authMethods, ssh.PublicKeysCallback(func() ([]ssh.Signer, error) { - return uniq, nil - })) - } - - if pw, found := _url.User.Password(); found { - authMethods = append(authMethods, ssh.Password(pw)) - } - - if len(authMethods) == 0 { - callback := func() (string, error) { - pass, err := terminal.ReadPassword("Login password:") - return string(pass), err - } - authMethods = append(authMethods, ssh.PasswordCallback(callback)) - } - - port := _url.Port() - if port == "" { - port = "22" - } - - callback := ssh.InsecureIgnoreHostKey() - if secure { - host := _url.Hostname() - if port != "22" { - host = fmt.Sprintf("[%s]:%s", host, port) - } - key := terminal.HostKey(host) - if key != nil { - callback = ssh.FixedHostKey(key) - } - } - - bastion, err := ssh.Dial("tcp", - net.JoinHostPort(_url.Hostname(), port), - &ssh.ClientConfig{ - User: _url.User.Username(), - Auth: authMethods, - HostKeyCallback: callback, - HostKeyAlgorithms: []string{ - ssh.KeyAlgoRSA, - ssh.KeyAlgoDSA, - ssh.KeyAlgoECDSA256, - ssh.KeyAlgoECDSA384, - ssh.KeyAlgoECDSA521, - ssh.KeyAlgoED25519, - }, - Timeout: 5 * time.Second, - }, - ) - if err != nil { - return Connection{}, fmt.Errorf("connection to bastion host (%s) failed: %w", _url.String(), err) - } - - connection := Connection{URI: _url} - connection.Client = &http.Client{ - Transport: &http.Transport{ - DialContext: func(ctx context.Context, _, _ string) (net.Conn, error) { - return bastion.Dial("unix", _url.Path) - }, - }} - return connection, nil -} - func unixClient(_url *url.URL) Connection { connection := Connection{URI: _url} connection.Client = &http.Client{ diff --git a/pkg/bindings/images/build.go b/pkg/bindings/images/build.go index 6883585e2..2615bc516 100644 --- a/pkg/bindings/images/build.go +++ b/pkg/bindings/images/build.go @@ -224,6 +224,15 @@ func Build(ctx context.Context, containerFiles []string, options entities.BuildO if len(options.Manifest) > 0 { params.Set("manifest", options.Manifest) } + if options.CacheFrom != nil { + params.Set("cachefrom", options.CacheFrom.String()) + } + if options.CacheTo != nil { + params.Set("cacheto", options.CacheTo.String()) + } + if int64(options.CacheTTL) != 0 { + params.Set("cachettl", options.CacheTTL.String()) + } if memSwap := options.CommonBuildOpts.MemorySwap; memSwap > 0 { params.Set("memswap", strconv.Itoa(int(memSwap))) } diff --git a/pkg/bindings/images/push.go b/pkg/bindings/images/push.go index 8db3726e6..5069dd780 100644 --- a/pkg/bindings/images/push.go +++ b/pkg/bindings/images/push.go @@ -62,6 +62,8 @@ func Push(ctx context.Context, source string, destination string, options *PushO writer := io.Writer(os.Stderr) if options.GetQuiet() { writer = ioutil.Discard + } else if progressWriter := options.GetProgressWriter(); progressWriter != nil { + writer = progressWriter } dec := json.NewDecoder(response.Body) diff --git a/pkg/bindings/images/types.go b/pkg/bindings/images/types.go index 0664afc1b..7b28c499e 100644 --- a/pkg/bindings/images/types.go +++ b/pkg/bindings/images/types.go @@ -1,6 +1,8 @@ package images import ( + "io" + buildahDefine "github.com/containers/buildah/define" ) @@ -15,6 +17,8 @@ type RemoveOptions struct { Ignore *bool // Confirms if given name is a manifest list and removes it, otherwise returns error. LookupManifest *bool + // Does not remove dangling parent images + NoPrune *bool } //go:generate go run ../generator/generator.go DiffOptions @@ -129,6 +133,10 @@ type PushOptions struct { Format *string // Password for authenticating against the registry. Password *string + // ProgressWriter is a writer where push progress are sent. + // Since API handler for image push is quiet by default, WithQuiet(false) is necessary for + // the writer to receive progress messages. + ProgressWriter *io.Writer // SkipTLSVerify to skip HTTPS and certificate verification. SkipTLSVerify *bool // RemoveSignatures Discard any pre-existing signatures in the image. diff --git a/pkg/bindings/images/types_push_options.go b/pkg/bindings/images/types_push_options.go index 1ae031824..817d873f8 100644 --- a/pkg/bindings/images/types_push_options.go +++ b/pkg/bindings/images/types_push_options.go @@ -2,6 +2,7 @@ package images import ( + "io" "net/url" "github.com/containers/podman/v4/pkg/bindings/internal/util" @@ -107,6 +108,21 @@ func (o *PushOptions) GetPassword() string { return *o.Password } +// WithProgressWriter set field ProgressWriter to given value +func (o *PushOptions) WithProgressWriter(value io.Writer) *PushOptions { + o.ProgressWriter = &value + return o +} + +// GetProgressWriter returns value of field ProgressWriter +func (o *PushOptions) GetProgressWriter() io.Writer { + if o.ProgressWriter == nil { + var z io.Writer + return z + } + return *o.ProgressWriter +} + // WithSkipTLSVerify set field SkipTLSVerify to given value func (o *PushOptions) WithSkipTLSVerify(value bool) *PushOptions { o.SkipTLSVerify = &value diff --git a/pkg/bindings/images/types_remove_options.go b/pkg/bindings/images/types_remove_options.go index 559ebcfd5..8972ac93c 100644 --- a/pkg/bindings/images/types_remove_options.go +++ b/pkg/bindings/images/types_remove_options.go @@ -76,3 +76,18 @@ func (o *RemoveOptions) GetLookupManifest() bool { } return *o.LookupManifest } + +// WithNoPrune set field NoPrune to given value +func (o *RemoveOptions) WithNoPrune(value bool) *RemoveOptions { + o.NoPrune = &value + return o +} + +// GetNoPrune returns value of field NoPrune +func (o *RemoveOptions) GetNoPrune() bool { + if o.NoPrune == nil { + var z bool + return z + } + return *o.NoPrune +} diff --git a/pkg/bindings/test/images_test.go b/pkg/bindings/test/images_test.go index 8f76ce456..9c9796661 100644 --- a/pkg/bindings/test/images_test.go +++ b/pkg/bindings/test/images_test.go @@ -379,6 +379,10 @@ var _ = Describe("Podman images", func() { Expect(err).To(HaveOccurred()) }) + It("Image Push", func() { + Skip("TODO: implement test for image push to registry") + }) + It("Build no options", func() { results, err := images.Build(bt.conn, []string{"fixture/Containerfile"}, entities.BuildOptions{}) Expect(err).ToNot(HaveOccurred()) diff --git a/pkg/domain/entities/containers.go b/pkg/domain/entities/containers.go index 7048cd1d2..3ba507750 100644 --- a/pkg/domain/entities/containers.go +++ b/pkg/domain/entities/containers.go @@ -202,6 +202,7 @@ type CheckpointOptions struct { type CheckpointReport struct { Err error `json:"-"` Id string `json:"Id"` //nolint:revive,stylecheck + RawInput string `json:"RawInput"` RuntimeDuration int64 `json:"runtime_checkpoint_duration"` CRIUStatistics *define.CRIUCheckpointRestoreStatistics `json:"criu_statistics"` } @@ -228,6 +229,7 @@ type RestoreOptions struct { type RestoreReport struct { Err error `json:"-"` Id string `json:"Id"` //nolint:revive,stylecheck + RawInput string `json:"RawInput"` RuntimeDuration int64 `json:"runtime_restore_duration"` CRIUStatistics *define.CRIUCheckpointRestoreStatistics `json:"criu_statistics"` } @@ -374,6 +376,7 @@ type ContainerCleanupOptions struct { type ContainerCleanupReport struct { CleanErr error Id string //nolint:revive,stylecheck + RawInput string RmErr error RmiErr error } @@ -388,8 +391,9 @@ type ContainerInitOptions struct { // ContainerInitReport describes the results of a // container init type ContainerInitReport struct { - Err error - Id string //nolint:revive,stylecheck + Err error + Id string //nolint:revive,stylecheck + RawInput string } // ContainerMountOptions describes the input values for mounting containers diff --git a/pkg/domain/entities/engine.go b/pkg/domain/entities/engine.go index 32faa74af..c1a4ffdf3 100644 --- a/pkg/domain/entities/engine.go +++ b/pkg/domain/entities/engine.go @@ -52,4 +52,5 @@ type PodmanConfig struct { Runroot string StorageDriver string StorageOpts []string + SSHMode string } diff --git a/pkg/domain/entities/engine_image.go b/pkg/domain/entities/engine_image.go index 5f76ae50b..b8b694873 100644 --- a/pkg/domain/entities/engine_image.go +++ b/pkg/domain/entities/engine_image.go @@ -4,6 +4,7 @@ import ( "context" "github.com/containers/common/pkg/config" + "github.com/containers/common/pkg/ssh" "github.com/containers/podman/v4/pkg/domain/entities/reports" ) @@ -22,7 +23,7 @@ type ImageEngine interface { Push(ctx context.Context, source string, destination string, opts ImagePushOptions) error Remove(ctx context.Context, images []string, opts ImageRemoveOptions) (*ImageRemoveReport, []error) Save(ctx context.Context, nameOrID string, tags []string, options ImageSaveOptions) error - Scp(ctx context.Context, src, dst string, parentFlags []string, quiet bool) error + Scp(ctx context.Context, src, dst string, parentFlags []string, quiet bool, sshMode ssh.EngineMode) error Search(ctx context.Context, term string, opts ImageSearchOptions) ([]ImageSearchReport, error) SetTrust(ctx context.Context, args []string, options SetTrustOptions) error ShowTrust(ctx context.Context, args []string, options ShowTrustOptions) (*ShowTrustReport, error) diff --git a/pkg/domain/entities/images.go b/pkg/domain/entities/images.go index dad2dc6cc..21c1372b9 100644 --- a/pkg/domain/entities/images.go +++ b/pkg/domain/entities/images.go @@ -94,6 +94,8 @@ type ImageRemoveOptions struct { Ignore bool // Confirms if given name is a manifest list and removes it, otherwise returns error. LookupManifest bool + // NoPrune will not remove dangling images + NoPrune bool } // ImageRemoveReport is the response for removing one or more image(s) from storage diff --git a/pkg/domain/infra/abi/containers.go b/pkg/domain/infra/abi/containers.go index 2820032c9..5b5bc665e 100644 --- a/pkg/domain/infra/abi/containers.go +++ b/pkg/domain/infra/abi/containers.go @@ -39,6 +39,7 @@ import ( // is specified. It also returns a list of the corresponding input name used to lookup each container. func getContainersAndInputByContext(all, latest bool, names []string, filters map[string][]string, runtime *libpod.Runtime) (ctrs []*libpod.Container, rawInput []string, err error) { var ctr *libpod.Container + var filteredCtrs []*libpod.Container ctrs = []*libpod.Container{} filterFuncs := make([]libpod.ContainerFilter, 0, len(filters)) @@ -57,7 +58,17 @@ func getContainersAndInputByContext(all, latest bool, names []string, filters ma } rawInput = []string{} for _, candidate := range ctrs { - rawInput = append(rawInput, candidate.ID()) + if len(names) > 0 { + for _, name := range names { + if candidate.ID() == name || candidate.Name() == name { + rawInput = append(rawInput, candidate.ID()) + filteredCtrs = append(filteredCtrs, candidate) + } + } + ctrs = filteredCtrs + } else { + rawInput = append(rawInput, candidate.ID()) + } } case all: ctrs, err = runtime.GetAllContainers() @@ -610,8 +621,9 @@ func (ic *ContainerEngine) ContainerExport(ctx context.Context, nameOrID string, func (ic *ContainerEngine) ContainerCheckpoint(ctx context.Context, namesOrIds []string, options entities.CheckpointOptions) ([]*entities.CheckpointReport, error) { var ( - err error - cons []*libpod.Container + ctrs []*libpod.Container + rawInputs []string + err error ) checkOpts := libpod.ContainerCheckpointOptions{ Keep: options.Keep, @@ -628,24 +640,34 @@ func (ic *ContainerEngine) ContainerCheckpoint(ctx context.Context, namesOrIds [ CreateImage: options.CreateImage, } + idToRawInput := map[string]string{} if options.All { running := func(c *libpod.Container) bool { state, _ := c.State() return state == define.ContainerStateRunning } - cons, err = ic.Libpod.GetContainers(running) + ctrs, err = ic.Libpod.GetContainers(running) + if err != nil { + return nil, err + } } else { - cons, err = getContainersByContext(false, options.Latest, namesOrIds, ic.Libpod) - } - if err != nil { - return nil, err + ctrs, rawInputs, err = getContainersAndInputByContext(false, options.Latest, namesOrIds, nil, ic.Libpod) + if err != nil { + return nil, err + } + if len(rawInputs) == len(ctrs) { + for i := range ctrs { + idToRawInput[ctrs[i].ID()] = rawInputs[i] + } + } } - reports := make([]*entities.CheckpointReport, 0, len(cons)) - for _, con := range cons { - criuStatistics, runtimeCheckpointDuration, err := con.Checkpoint(ctx, checkOpts) + reports := make([]*entities.CheckpointReport, 0, len(ctrs)) + for _, c := range ctrs { + criuStatistics, runtimeCheckpointDuration, err := c.Checkpoint(ctx, checkOpts) reports = append(reports, &entities.CheckpointReport{ Err: err, - Id: con.ID(), + Id: c.ID(), + RawInput: idToRawInput[c.ID()], RuntimeDuration: runtimeCheckpointDuration, CRIUStatistics: criuStatistics, }) @@ -655,7 +677,7 @@ func (ic *ContainerEngine) ContainerCheckpoint(ctx context.Context, namesOrIds [ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []string, options entities.RestoreOptions) ([]*entities.RestoreReport, error) { var ( - containers []*libpod.Container + ctrs []*libpod.Container checkpointImageImportErrors []error err error ) @@ -682,19 +704,21 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st }, } + idToRawInput := map[string]string{} switch { case options.Import != "": - containers, err = checkpoint.CRImportCheckpointTar(ctx, ic.Libpod, options) + ctrs, err = checkpoint.CRImportCheckpointTar(ctx, ic.Libpod, options) case options.All: - containers, err = ic.Libpod.GetContainers(filterFuncs...) + ctrs, err = ic.Libpod.GetContainers(filterFuncs...) case options.Latest: - containers, err = getContainersByContext(false, options.Latest, namesOrIds, ic.Libpod) + ctrs, err = getContainersByContext(false, options.Latest, namesOrIds, ic.Libpod) default: for _, nameOrID := range namesOrIds { logrus.Debugf("look up container: %q", nameOrID) - ctr, err := ic.Libpod.LookupContainer(nameOrID) + c, err := ic.Libpod.LookupContainer(nameOrID) if err == nil { - containers = append(containers, ctr) + ctrs = append(ctrs, c) + idToRawInput[c.ID()] = nameOrID } else { // If container was not found, check if this is a checkpoint image logrus.Debugf("look up image: %q", nameOrID) @@ -712,7 +736,7 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st if err != nil { return nil, err } - importedContainers, err := checkpoint.CRImportCheckpoint(ctx, ic.Libpod, options, mountPoint) + importedCtrs, err := checkpoint.CRImportCheckpoint(ctx, ic.Libpod, options, mountPoint) if err != nil { // CRImportCheckpoint is expected to import exactly one container from checkpoint image checkpointImageImportErrors = append( @@ -720,7 +744,7 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st fmt.Errorf("unable to import checkpoint from image: %q: %v", nameOrID, err), ) } else { - containers = append(containers, importedContainers[0]) + ctrs = append(ctrs, importedCtrs[0]) } } } @@ -729,12 +753,13 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st return nil, err } - reports := make([]*entities.RestoreReport, 0, len(containers)) - for _, con := range containers { - criuStatistics, runtimeRestoreDuration, err := con.Restore(ctx, restoreOptions) + reports := make([]*entities.RestoreReport, 0, len(ctrs)) + for _, c := range ctrs { + criuStatistics, runtimeRestoreDuration, err := c.Restore(ctx, restoreOptions) reports = append(reports, &entities.RestoreReport{ Err: err, - Id: con.ID(), + Id: c.ID(), + RawInput: idToRawInput[c.ID()], RuntimeDuration: runtimeRestoreDuration, CRIUStatistics: criuStatistics, }) @@ -898,38 +923,7 @@ func (ic *ContainerEngine) ContainerExecDetached(ctx context.Context, nameOrID s func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []string, options entities.ContainerStartOptions) ([]*entities.ContainerStartReport, error) { reports := []*entities.ContainerStartReport{} var exitCode = define.ExecErrorCodeGeneric - containersNamesOrIds := namesOrIds - all := options.All - if len(options.Filters) > 0 { - all = false - filterFuncs := make([]libpod.ContainerFilter, 0, len(options.Filters)) - if len(options.Filters) > 0 { - for k, v := range options.Filters { - generatedFunc, err := dfilters.GenerateContainerFilterFuncs(k, v, ic.Libpod) - if err != nil { - return nil, err - } - filterFuncs = append(filterFuncs, generatedFunc) - } - } - candidates, err := ic.Libpod.GetContainers(filterFuncs...) - if err != nil { - return nil, err - } - containersNamesOrIds = []string{} - for _, candidate := range candidates { - if options.All { - containersNamesOrIds = append(containersNamesOrIds, candidate.ID()) - continue - } - for _, nameOrID := range namesOrIds { - if nameOrID == candidate.ID() || nameOrID == candidate.Name() { - containersNamesOrIds = append(containersNamesOrIds, nameOrID) - } - } - } - } - ctrs, rawInputs, err := getContainersAndInputByContext(all, options.Latest, containersNamesOrIds, options.Filters, ic.Libpod) + ctrs, rawInputs, err := getContainersAndInputByContext(options.All, options.Latest, namesOrIds, options.Filters, ic.Libpod) if err != nil { return nil, err } @@ -1223,14 +1217,20 @@ func (ic *ContainerEngine) ContainerLogs(ctx context.Context, containers []strin } func (ic *ContainerEngine) ContainerCleanup(ctx context.Context, namesOrIds []string, options entities.ContainerCleanupOptions) ([]*entities.ContainerCleanupReport, error) { - reports := []*entities.ContainerCleanupReport{} - ctrs, err := getContainersByContext(options.All, options.Latest, namesOrIds, ic.Libpod) + ctrs, rawInputs, err := getContainersAndInputByContext(options.All, options.Latest, namesOrIds, nil, ic.Libpod) if err != nil { return nil, err } + idToRawInput := map[string]string{} + if len(rawInputs) == len(ctrs) { + for i := range ctrs { + idToRawInput[ctrs[i].ID()] = rawInputs[i] + } + } + reports := []*entities.ContainerCleanupReport{} for _, ctr := range ctrs { var err error - report := entities.ContainerCleanupReport{Id: ctr.ID()} + report := entities.ContainerCleanupReport{Id: ctr.ID(), RawInput: idToRawInput[ctr.ID()]} if options.Exec != "" { if options.Remove { @@ -1271,13 +1271,19 @@ func (ic *ContainerEngine) ContainerCleanup(ctx context.Context, namesOrIds []st } func (ic *ContainerEngine) ContainerInit(ctx context.Context, namesOrIds []string, options entities.ContainerInitOptions) ([]*entities.ContainerInitReport, error) { - ctrs, err := getContainersByContext(options.All, options.Latest, namesOrIds, ic.Libpod) + ctrs, rawInputs, err := getContainersAndInputByContext(options.All, options.Latest, namesOrIds, nil, ic.Libpod) if err != nil { return nil, err } + idToRawInput := map[string]string{} + if len(rawInputs) == len(ctrs) { + for i := range ctrs { + idToRawInput[ctrs[i].ID()] = rawInputs[i] + } + } reports := make([]*entities.ContainerInitReport, 0, len(ctrs)) for _, ctr := range ctrs { - report := entities.ContainerInitReport{Id: ctr.ID()} + report := entities.ContainerInitReport{Id: ctr.ID(), RawInput: idToRawInput[ctr.ID()]} err := ctr.Init(ctx, ctr.PodID() != "") // If we're initializing all containers, ignore invalid state errors diff --git a/pkg/domain/infra/abi/images.go b/pkg/domain/infra/abi/images.go index 94178a8e2..77d1bf0db 100644 --- a/pkg/domain/infra/abi/images.go +++ b/pkg/domain/infra/abi/images.go @@ -18,6 +18,7 @@ import ( "github.com/containers/common/libimage" "github.com/containers/common/pkg/config" + "github.com/containers/common/pkg/ssh" "github.com/containers/image/v5/docker" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/manifest" @@ -565,6 +566,7 @@ func (ir *ImageEngine) Remove(ctx context.Context, images []string, opts entitie libimageOptions.Force = opts.Force libimageOptions.Ignore = opts.Ignore libimageOptions.LookupManifest = opts.LookupManifest + libimageOptions.NoPrune = opts.NoPrune if !opts.All { libimageOptions.Filters = append(libimageOptions.Filters, "intermediate=false") } @@ -581,7 +583,7 @@ func (ir *ImageEngine) Remove(ctx context.Context, images []string, opts entitie rmErrors = libimageErrors - return + return report, rmErrors } // Shutdown Libpod engine @@ -682,8 +684,8 @@ func (ir *ImageEngine) Sign(ctx context.Context, names []string, options entitie return nil, nil } -func (ir *ImageEngine) Scp(ctx context.Context, src, dst string, parentFlags []string, quiet bool) error { - rep, source, dest, flags, err := domainUtils.ExecuteTransfer(src, dst, parentFlags, quiet) +func (ir *ImageEngine) Scp(ctx context.Context, src, dst string, parentFlags []string, quiet bool, sshMode ssh.EngineMode) error { + rep, source, dest, flags, err := domainUtils.ExecuteTransfer(src, dst, parentFlags, quiet, sshMode) if err != nil { return err } diff --git a/pkg/domain/infra/tunnel/containers.go b/pkg/domain/infra/tunnel/containers.go index 225aee017..d49f029d5 100644 --- a/pkg/domain/infra/tunnel/containers.go +++ b/pkg/domain/infra/tunnel/containers.go @@ -362,6 +362,12 @@ func (ic *ContainerEngine) ContainerExport(ctx context.Context, nameOrID string, } func (ic *ContainerEngine) ContainerCheckpoint(ctx context.Context, namesOrIds []string, opts entities.CheckpointOptions) ([]*entities.CheckpointReport, error) { + var ( + err error + ctrs []entities.ListContainer + rawInputs []string + idToRawInput = map[string]string{} + ) options := new(containers.CheckpointOptions) options.WithFileLocks(opts.FileLocks) options.WithIgnoreRootfs(opts.IgnoreRootFS) @@ -374,11 +380,6 @@ func (ic *ContainerEngine) ContainerCheckpoint(ctx context.Context, namesOrIds [ options.WithLeaveRunning(opts.LeaveRunning) options.WithWithPrevious(opts.WithPrevious) - var ( - err error - ctrs = []entities.ListContainer{} - ) - if opts.All { allCtrs, err := getContainersByContext(ic.ClientCtx, true, false, []string{}) if err != nil { @@ -391,10 +392,15 @@ func (ic *ContainerEngine) ContainerCheckpoint(ctx context.Context, namesOrIds [ } } } else { - ctrs, err = getContainersByContext(ic.ClientCtx, false, false, namesOrIds) + ctrs, rawInputs, err = getContainersAndInputByContext(ic.ClientCtx, false, false, namesOrIds, nil) if err != nil { return nil, err } + if len(rawInputs) == len(ctrs) { + for i := range ctrs { + idToRawInput[ctrs[i].ID] = rawInputs[i] + } + } } reports := make([]*entities.CheckpointReport, 0, len(ctrs)) for _, c := range ctrs { @@ -402,6 +408,7 @@ func (ic *ContainerEngine) ContainerCheckpoint(ctx context.Context, namesOrIds [ if err != nil { reports = append(reports, &entities.CheckpointReport{Id: c.ID, Err: err}) } else { + report.RawInput = idToRawInput[report.Id] reports = append(reports, report) } } @@ -413,6 +420,10 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st return nil, fmt.Errorf("--import-previous is not supported on the remote client") } + var ( + ids []string + idToRawInput = map[string]string{} + ) options := new(containers.RestoreOptions) options.WithFileLocks(opts.FileLocks) options.WithIgnoreRootfs(opts.IgnoreRootFS) @@ -431,10 +442,6 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st report, err := containers.Restore(ic.ClientCtx, "", options) return []*entities.RestoreReport{report}, err } - - var ( - ids = []string{} - ) if opts.All { allCtrs, err := getContainersByContext(ic.ClientCtx, true, false, []string{}) if err != nil { @@ -457,6 +464,7 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st ctrData, _, err := ic.ContainerInspect(ic.ClientCtx, []string{nameOrID}, entities.InspectOptions{}) if err == nil && len(ctrData) > 0 { ids = append(ids, ctrData[0].ID) + idToRawInput[ctrData[0].ID] = nameOrID } else { // If container was not found, check if this is a checkpoint image inspectReport, err := images.GetImage(ic.ClientCtx, nameOrID, getImageOptions) @@ -480,6 +488,7 @@ func (ic *ContainerEngine) ContainerRestore(ctx context.Context, namesOrIds []st if err != nil { reports = append(reports, &entities.RestoreReport{Id: id, Err: err}) } + report.RawInput = idToRawInput[report.Id] reports = append(reports, report) } return reports, nil @@ -658,36 +667,7 @@ func logIfRmError(id string, err error, reports []*reports.RmReport) { func (ic *ContainerEngine) ContainerStart(ctx context.Context, namesOrIds []string, options entities.ContainerStartOptions) ([]*entities.ContainerStartReport, error) { reports := []*entities.ContainerStartReport{} var exitCode = define.ExecErrorCodeGeneric - containersNamesOrIds := namesOrIds - all := options.All - if len(options.Filters) > 0 { - all = false - containersNamesOrIds = []string{} - opts := new(containers.ListOptions).WithFilters(options.Filters).WithAll(true) - candidates, listErr := containers.List(ic.ClientCtx, opts) - if listErr != nil { - return nil, listErr - } - for _, candidate := range candidates { - if options.All { - containersNamesOrIds = append(containersNamesOrIds, candidate.ID) - continue - } - for _, nameOrID := range namesOrIds { - if nameOrID == candidate.ID { - containersNamesOrIds = append(containersNamesOrIds, nameOrID) - continue - } - for _, containerName := range candidate.Names { - if containerName == nameOrID { - containersNamesOrIds = append(containersNamesOrIds, nameOrID) - continue - } - } - } - } - } - ctrs, err := getContainersByContext(ic.ClientCtx, all, false, containersNamesOrIds) + ctrs, namesOrIds, err := getContainersAndInputByContext(ic.ClientCtx, options.All, false, namesOrIds, options.Filters) if err != nil { return nil, err } @@ -935,21 +915,28 @@ func (ic *ContainerEngine) ContainerCleanup(ctx context.Context, namesOrIds []st } func (ic *ContainerEngine) ContainerInit(ctx context.Context, namesOrIds []string, options entities.ContainerInitOptions) ([]*entities.ContainerInitReport, error) { - ctrs, err := getContainersByContext(ic.ClientCtx, options.All, false, namesOrIds) + ctrs, rawInputs, err := getContainersAndInputByContext(ic.ClientCtx, options.All, false, namesOrIds, nil) if err != nil { return nil, err } + idToRawInput := map[string]string{} + if len(rawInputs) == len(ctrs) { + for i := range ctrs { + idToRawInput[ctrs[i].ID] = rawInputs[i] + } + } reports := make([]*entities.ContainerInitReport, 0, len(ctrs)) - for _, ctr := range ctrs { - err := containers.ContainerInit(ic.ClientCtx, ctr.ID, nil) + for _, c := range ctrs { + err := containers.ContainerInit(ic.ClientCtx, c.ID, nil) // When using all, it is NOT considered an error if a container // has already been init'd. if err != nil && options.All && strings.Contains(err.Error(), define.ErrCtrStateInvalid.Error()) { err = nil } reports = append(reports, &entities.ContainerInitReport{ - Err: err, - Id: ctr.ID, + Err: err, + RawInput: idToRawInput[c.ID], + Id: c.ID, }) } return reports, nil diff --git a/pkg/domain/infra/tunnel/helpers.go b/pkg/domain/infra/tunnel/helpers.go index a0b01dd71..90d558119 100644 --- a/pkg/domain/infra/tunnel/helpers.go +++ b/pkg/domain/infra/tunnel/helpers.go @@ -31,8 +31,17 @@ func getContainersAndInputByContext(contextWithConnection context.Context, all, rawInputs := []string{} switch { case len(filters) > 0: + namesOrIDs = nil for i := range allContainers { - namesOrIDs = append(namesOrIDs, allContainers[i].ID) + if len(namesOrIDs) > 0 { + for _, name := range namesOrIDs { + if name == allContainers[i].ID { + namesOrIDs = append(namesOrIDs, allContainers[i].ID) + } + } + } else { + namesOrIDs = append(namesOrIDs, allContainers[i].ID) + } } case all: for i := range allContainers { diff --git a/pkg/domain/infra/tunnel/images.go b/pkg/domain/infra/tunnel/images.go index 4f79325fd..bb3014099 100644 --- a/pkg/domain/infra/tunnel/images.go +++ b/pkg/domain/infra/tunnel/images.go @@ -12,6 +12,7 @@ import ( "github.com/containers/common/libimage" "github.com/containers/common/pkg/config" + "github.com/containers/common/pkg/ssh" "github.com/containers/image/v5/docker/reference" "github.com/containers/image/v5/types" "github.com/containers/podman/v4/pkg/bindings/images" @@ -28,7 +29,7 @@ func (ir *ImageEngine) Exists(_ context.Context, nameOrID string) (*entities.Boo } func (ir *ImageEngine) Remove(ctx context.Context, imagesArg []string, opts entities.ImageRemoveOptions) (*entities.ImageRemoveReport, []error) { - options := new(images.RemoveOptions).WithForce(opts.Force).WithIgnore(opts.Ignore).WithAll(opts.All).WithLookupManifest(opts.LookupManifest) + options := new(images.RemoveOptions).WithForce(opts.Force).WithIgnore(opts.Ignore).WithAll(opts.All).WithLookupManifest(opts.LookupManifest).WithNoPrune(opts.NoPrune) return images.Remove(ir.ClientCtx, imagesArg, options) } @@ -240,7 +241,7 @@ func (ir *ImageEngine) Import(ctx context.Context, opts entities.ImageImportOpti func (ir *ImageEngine) Push(ctx context.Context, source string, destination string, opts entities.ImagePushOptions) error { options := new(images.PushOptions) - options.WithAll(opts.All).WithCompress(opts.Compress).WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithFormat(opts.Format).WithRemoveSignatures(opts.RemoveSignatures).WithQuiet(opts.Quiet).WithCompressionFormat(opts.CompressionFormat) + options.WithAll(opts.All).WithCompress(opts.Compress).WithUsername(opts.Username).WithPassword(opts.Password).WithAuthfile(opts.Authfile).WithFormat(opts.Format).WithRemoveSignatures(opts.RemoveSignatures).WithQuiet(opts.Quiet).WithCompressionFormat(opts.CompressionFormat).WithProgressWriter(opts.Writer) if s := opts.SkipTLSVerify; s != types.OptionalBoolUndefined { if s == types.OptionalBoolTrue { @@ -364,7 +365,7 @@ func (ir *ImageEngine) Sign(ctx context.Context, names []string, options entitie return nil, errors.New("not implemented yet") } -func (ir *ImageEngine) Scp(ctx context.Context, src, dst string, parentFlags []string, quiet bool) error { +func (ir *ImageEngine) Scp(ctx context.Context, src, dst string, parentFlags []string, quiet bool, sshMode ssh.EngineMode) error { options := new(images.ScpOptions) var destination *string diff --git a/pkg/domain/utils/scp.go b/pkg/domain/utils/scp.go index 3c73cddd1..44a0d94d7 100644 --- a/pkg/domain/utils/scp.go +++ b/pkg/domain/utils/scp.go @@ -1,31 +1,24 @@ package utils import ( - "bytes" "fmt" "io/ioutil" - "net" "net/url" "os" "os/exec" "os/user" "strconv" "strings" - "time" - - scpD "github.com/dtylman/scp" "github.com/containers/common/pkg/config" + "github.com/containers/common/pkg/ssh" + "github.com/containers/image/v5/transports/alltransports" "github.com/containers/podman/v4/libpod/define" "github.com/containers/podman/v4/pkg/domain/entities" - "github.com/containers/podman/v4/pkg/terminal" - "github.com/docker/distribution/reference" "github.com/sirupsen/logrus" - "golang.org/x/crypto/ssh" - "golang.org/x/crypto/ssh/agent" ) -func ExecuteTransfer(src, dst string, parentFlags []string, quiet bool) (*entities.ImageLoadReport, *entities.ImageScpOptions, *entities.ImageScpOptions, []string, error) { +func ExecuteTransfer(src, dst string, parentFlags []string, quiet bool, sshMode ssh.EngineMode) (*entities.ImageLoadReport, *entities.ImageScpOptions, *entities.ImageScpOptions, []string, error) { source := entities.ImageScpOptions{} dest := entities.ImageScpOptions{} sshInfo := entities.ImageScpConnections{} @@ -46,10 +39,6 @@ func ExecuteTransfer(src, dst string, parentFlags []string, quiet bool) (*entiti return nil, nil, nil, nil, fmt.Errorf("could not make config: %w", err) } - cfg, err := config.ReadCustomConfig() // get ready to set ssh destination if necessary - if err != nil { - return nil, nil, nil, nil, err - } locations := []*entities.ImageScpOptions{} cliConnections := []string{} args := []string{src} @@ -83,9 +72,7 @@ func ExecuteTransfer(src, dst string, parentFlags []string, quiet bool) (*entiti source.Quiet = quiet source.File = f.Name() // after parsing the arguments, set the file for the save/load dest.File = source.File - if err = os.Remove(source.File); err != nil { // remove the file and simply use its name so podman creates the file upon save. avoids umask errors - return nil, nil, nil, nil, err - } + defer os.Remove(source.File) allLocal := true // if we are all localhost, do not validate connections but if we are using one localhost and one non we need to use sshd for _, val := range cliConnections { @@ -98,6 +85,10 @@ func ExecuteTransfer(src, dst string, parentFlags []string, quiet bool) (*entiti cliConnections = []string{} } + cfg, err := config.ReadCustomConfig() // get ready to set ssh destination if necessary + if err != nil { + return nil, nil, nil, nil, err + } var serv map[string]config.Destination serv, err = GetServiceInformation(&sshInfo, cliConnections, cfg) if err != nil { @@ -109,12 +100,12 @@ func ExecuteTransfer(src, dst string, parentFlags []string, quiet bool) (*entiti switch { case source.Remote: // if we want to load FROM the remote, dest can either be local or remote in this case - err = SaveToRemote(source.Image, source.File, "", sshInfo.URI[0], sshInfo.Identities[0]) + err = SaveToRemote(source.Image, source.File, "", sshInfo.URI[0], sshInfo.Identities[0], sshMode) if err != nil { return nil, nil, nil, nil, err } if dest.Remote { // we want to load remote -> remote, both source and dest are remote - rep, id, err := LoadToRemote(dest, dest.File, "", sshInfo.URI[1], sshInfo.Identities[1]) + rep, id, err := LoadToRemote(dest, dest.File, "", sshInfo.URI[1], sshInfo.Identities[1], sshMode) if err != nil { return nil, nil, nil, nil, err } @@ -138,7 +129,8 @@ func ExecuteTransfer(src, dst string, parentFlags []string, quiet bool) (*entiti if err != nil { return nil, nil, nil, nil, err } - rep, id, err := LoadToRemote(dest, source.File, "", sshInfo.URI[0], sshInfo.Identities[0]) + + rep, id, err := LoadToRemote(dest, source.File, "", sshInfo.URI[0], sshInfo.Identities[0], sshMode) if err != nil { return nil, nil, nil, nil, err } @@ -220,34 +212,37 @@ func LoginUser(user string) (*exec.Cmd, error) { // loadToRemote takes image and remote connection information. it connects to the specified client // and copies the saved image dir over to the remote host and then loads it onto the machine // returns a string containing output or an error -func LoadToRemote(dest entities.ImageScpOptions, localFile string, tag string, url *url.URL, iden string) (string, string, error) { - dial, remoteFile, err := CreateConnection(url, iden) +func LoadToRemote(dest entities.ImageScpOptions, localFile string, tag string, url *url.URL, iden string, sshEngine ssh.EngineMode) (string, string, error) { + port, err := strconv.Atoi(url.Port()) if err != nil { return "", "", err } - defer dial.Close() - n, err := scpD.CopyTo(dial, localFile, remoteFile) + remoteFile, err := ssh.Exec(&ssh.ConnectionExecOptions{Host: url.String(), Port: port, User: url.User, Args: []string{"mktemp"}}, sshEngine) if err != nil { - errOut := strconv.Itoa(int(n)) + " Bytes copied before error" - return " ", "", fmt.Errorf("%v: %w", errOut, err) + return "", "", err } - var run string - if tag != "" { - return "", "", fmt.Errorf("renaming of an image is currently not supported: %w", define.ErrInvalidArg) + + opts := ssh.ConnectionScpOptions{User: url.User, Identity: iden, Port: port, Source: localFile, Destination: "ssh://" + url.User.String() + "@" + url.Hostname() + ":" + remoteFile} + scpRep, err := ssh.Scp(&opts, sshEngine) + if err != nil { + return "", "", err } - podman := os.Args[0] - run = podman + " image load --input=" + remoteFile + ";rm " + remoteFile // run ssh image load of the file copied via scp - out, err := ExecRemoteCommand(dial, run) + out, err := ssh.Exec(&ssh.ConnectionExecOptions{Host: url.String(), Port: port, User: url.User, Args: []string{"podman", "image", "load", "--input=" + scpRep + ";", "rm", scpRep}}, sshEngine) if err != nil { return "", "", err } - rep := strings.TrimSuffix(string(out), "\n") + if tag != "" { + return "", "", fmt.Errorf("renaming of an image is currently not supported: %w", define.ErrInvalidArg) + } + rep := strings.TrimSuffix(out, "\n") outArr := strings.Split(rep, " ") id := outArr[len(outArr)-1] if len(dest.Tag) > 0 { // tag the remote image using the output ID - run = podman + " tag " + id + " " + dest.Tag - _, err = ExecRemoteCommand(dial, run) + _, err := ssh.Exec(&ssh.ConnectionExecOptions{Host: url.Hostname(), Port: port, User: url.User, Args: []string{"podman", "image", "tag", id, dest.Tag}}, sshEngine) + if err != nil { + return "", "", err + } if err != nil { return "", "", err } @@ -258,94 +253,37 @@ func LoadToRemote(dest entities.ImageScpOptions, localFile string, tag string, u // saveToRemote takes image information and remote connection information. it connects to the specified client // and saves the specified image on the remote machine and then copies it to the specified local location // returns an error if one occurs. -func SaveToRemote(image, localFile string, tag string, uri *url.URL, iden string) error { - dial, remoteFile, err := CreateConnection(uri, iden) - - if err != nil { - return err - } - defer dial.Close() - +func SaveToRemote(image, localFile string, tag string, uri *url.URL, iden string, sshEngine ssh.EngineMode) error { if tag != "" { return fmt.Errorf("renaming of an image is currently not supported: %w", define.ErrInvalidArg) } - podman := os.Args[0] - run := podman + " image save " + image + " --format=oci-archive --output=" + remoteFile // run ssh image load of the file copied via scp. Files are reverse in this case... - _, err = ExecRemoteCommand(dial, run) + + port, err := strconv.Atoi(uri.Port()) if err != nil { return err } - n, err := scpD.CopyFrom(dial, remoteFile, localFile) - if _, conErr := ExecRemoteCommand(dial, "rm "+remoteFile); conErr != nil { - logrus.Errorf("Removing file on endpoint: %v", conErr) - } - if err != nil { - errOut := strconv.Itoa(int(n)) + " Bytes copied before error" - return fmt.Errorf("%v: %w", errOut, err) - } - return nil -} -// makeRemoteFile creates the necessary remote file on the host to -// save or load the image to. returns a string with the file name or an error -func MakeRemoteFile(dial *ssh.Client) (string, error) { - run := "mktemp" - remoteFile, err := ExecRemoteCommand(dial, run) + remoteFile, err := ssh.Exec(&ssh.ConnectionExecOptions{Host: uri.String(), Port: port, User: uri.User, Args: []string{"mktemp"}}, sshEngine) if err != nil { - return "", err + return err } - return strings.TrimSuffix(string(remoteFile), "\n"), nil -} -// createConnections takes a boolean determining which ssh client to dial -// and returns the dials client, its newly opened remote file, and an error if applicable. -func CreateConnection(url *url.URL, iden string) (*ssh.Client, string, error) { - cfg, err := ValidateAndConfigure(url, iden) + _, err = ssh.Exec(&ssh.ConnectionExecOptions{Host: uri.String(), Port: port, User: uri.User, Args: []string{"podman", "image", "save", image, "--format", "oci-archive", "--output", remoteFile}}, sshEngine) if err != nil { - return nil, "", err + return err } - dialAdd, err := ssh.Dial("tcp", url.Host, cfg) // dial the client + + opts := ssh.ConnectionScpOptions{User: uri.User, Identity: iden, Port: port, Source: "ssh://" + uri.User.String() + "@" + uri.Hostname() + ":" + remoteFile, Destination: localFile} + scpRep, err := ssh.Scp(&opts, sshEngine) if err != nil { - return nil, "", fmt.Errorf("failed to connect: %w", err) + return err } - file, err := MakeRemoteFile(dialAdd) + _, err = ssh.Exec(&ssh.ConnectionExecOptions{Host: uri.String(), Port: port, User: uri.User, Args: []string{"rm", scpRep}}, sshEngine) if err != nil { - return nil, "", err + logrus.Errorf("Removing file on endpoint: %v", err) } - return dialAdd, file, nil -} - -// GetSerivceInformation takes the parsed list of hosts to connect to and validates the information -func GetServiceInformation(sshInfo *entities.ImageScpConnections, cliConnections []string, cfg *config.Config) (map[string]config.Destination, error) { - var serv map[string]config.Destination - var urlS string - var iden string - for i, val := range cliConnections { - splitEnv := strings.SplitN(val, "::", 2) - sshInfo.Connections = append(sshInfo.Connections, splitEnv[0]) - conn, found := cfg.Engine.ServiceDestinations[sshInfo.Connections[i]] - if found { - urlS = conn.URI - iden = conn.Identity - } else { // no match, warn user and do a manual connection. - urlS = "ssh://" + sshInfo.Connections[i] - iden = "" - logrus.Warnf("Unknown connection name given. Please use system connection add to specify the default remote socket location") - } - urlFinal, err := url.Parse(urlS) // create an actual url to pass to exec command - if err != nil { - return nil, err - } - if urlFinal.User.Username() == "" { - if urlFinal.User, err = GetUserInfo(urlFinal); err != nil { - return nil, err - } - } - sshInfo.URI = append(sshInfo.URI, urlFinal) - sshInfo.Identities = append(sshInfo.Identities, iden) - } - return serv, nil + return nil } // execPodman executes the podman save/load command given the podman binary @@ -413,18 +351,32 @@ func ParseImageSCPArg(arg string) (*entities.ImageScpOptions, []string, error) { return &location, cliConnections, nil } -// validateImagePortion is a helper function to validate the image name in an SCP argument func ValidateImagePortion(location entities.ImageScpOptions, arg string) (entities.ImageScpOptions, error) { if RemoteArgLength(arg, 1) > 0 { - err := ValidateImageName(strings.Split(arg, "::")[1]) - if err != nil { - return location, err - } - location.Image = strings.Split(arg, "::")[1] // this will get checked/set again once we validate connections + before := strings.Split(arg, "::")[1] + name := ValidateImageName(before) + if before != name { + location.Image = name + } else { + location.Image = before + } // this will get checked/set again once we validate connections } return location, nil } +// validateImageName makes sure that the image given is valid and no injections are occurring +// we simply use this for error checking, bot setting the image +func ValidateImageName(input string) string { + // ParseNormalizedNamed transforms a shortname image into its + // full name reference so busybox => docker.io/library/busybox + // we want to keep our shortnames, so only return an error if + // we cannot parse what the user has given us + if ref, err := alltransports.ParseImageName(input); err == nil { + return ref.Transport().Name() + } + return input +} + // validateSCPArgs takes the array of source and destination options and checks for common errors func ValidateSCPArgs(locations []*entities.ImageScpOptions) error { if len(locations) > 2 { @@ -440,17 +392,6 @@ func ValidateSCPArgs(locations []*entities.ImageScpOptions) error { return nil } -// validateImageName makes sure that the image given is valid and no injections are occurring -// we simply use this for error checking, bot setting the image -func ValidateImageName(input string) error { - // ParseNormalizedNamed transforms a shortname image into its - // full name reference so busybox => docker.io/library/busybox - // we want to keep our shortnames, so only return an error if - // we cannot parse what the user has given us - _, err := reference.ParseNormalizedNamed(input) - return err -} - // remoteArgLength is a helper function to simplify the extracting of host argument data // returns an int which contains the length of a specified index in a host::image string func RemoteArgLength(input string, side int) int { @@ -460,23 +401,36 @@ func RemoteArgLength(input string, side int) int { return -1 } -// ExecRemoteCommand takes a ssh client connection and a command to run and executes the -// command on the specified client. The function returns the Stdout from the client or the Stderr -func ExecRemoteCommand(dial *ssh.Client, run string) ([]byte, error) { - sess, err := dial.NewSession() // new ssh client session - if err != nil { - return nil, err - } - defer sess.Close() - - var buffer bytes.Buffer - var bufferErr bytes.Buffer - sess.Stdout = &buffer // output from client funneled into buffer - sess.Stderr = &bufferErr // err form client funneled into buffer - if err := sess.Run(run); err != nil { // run the command on the ssh client - return nil, fmt.Errorf("%v: %w", bufferErr.String(), err) +// GetSerivceInformation takes the parsed list of hosts to connect to and validates the information +func GetServiceInformation(sshInfo *entities.ImageScpConnections, cliConnections []string, cfg *config.Config) (map[string]config.Destination, error) { + var serv map[string]config.Destination + var urlS string + var iden string + for i, val := range cliConnections { + splitEnv := strings.SplitN(val, "::", 2) + sshInfo.Connections = append(sshInfo.Connections, splitEnv[0]) + conn, found := cfg.Engine.ServiceDestinations[sshInfo.Connections[i]] + if found { + urlS = conn.URI + iden = conn.Identity + } else { // no match, warn user and do a manual connection. + urlS = "ssh://" + sshInfo.Connections[i] + iden = "" + logrus.Warnf("Unknown connection name given. Please use system connection add to specify the default remote socket location") + } + urlFinal, err := url.Parse(urlS) // create an actual url to pass to exec command + if err != nil { + return nil, err + } + if urlFinal.User.Username() == "" { + if urlFinal.User, err = GetUserInfo(urlFinal); err != nil { + return nil, err + } + } + sshInfo.URI = append(sshInfo.URI, urlFinal) + sshInfo.Identities = append(sshInfo.Identities, iden) } - return buffer.Bytes(), nil + return serv, nil } func GetUserInfo(uri *url.URL) (*url.Userinfo, error) { @@ -502,79 +456,3 @@ func GetUserInfo(uri *url.URL) (*url.Userinfo, error) { } return url.User(usr.Username), nil } - -// ValidateAndConfigure will take a ssh url and an identity key (rsa and the like) and ensure the information given is valid -// iden iden can be blank to mean no identity key -// once the function validates the information it creates and returns an ssh.ClientConfig. -func ValidateAndConfigure(uri *url.URL, iden string) (*ssh.ClientConfig, error) { - var signers []ssh.Signer - passwd, passwdSet := uri.User.Password() - if iden != "" { // iden might be blank if coming from image scp or if no validation is needed - value := iden - s, err := terminal.PublicKey(value, []byte(passwd)) - if err != nil { - return nil, fmt.Errorf("failed to read identity %q: %w", value, err) - } - signers = append(signers, s) - logrus.Debugf("SSH Ident Key %q %s %s", value, ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type()) - } - if sock, found := os.LookupEnv("SSH_AUTH_SOCK"); found { // validate ssh information, specifically the unix file socket used by the ssh agent. - logrus.Debugf("Found SSH_AUTH_SOCK %q, ssh-agent signer enabled", sock) - - c, err := net.Dial("unix", sock) - if err != nil { - return nil, err - } - agentSigners, err := agent.NewClient(c).Signers() - if err != nil { - return nil, err - } - - signers = append(signers, agentSigners...) - - if logrus.IsLevelEnabled(logrus.DebugLevel) { - for _, s := range agentSigners { - logrus.Debugf("SSH Agent Key %s %s", ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type()) - } - } - } - var authMethods []ssh.AuthMethod // now we validate and check for the authorization methods, most notaibly public key authorization - if len(signers) > 0 { - var dedup = make(map[string]ssh.Signer) - for _, s := range signers { - fp := ssh.FingerprintSHA256(s.PublicKey()) - if _, found := dedup[fp]; found { - logrus.Debugf("Dedup SSH Key %s %s", ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type()) - } - dedup[fp] = s - } - - var uniq []ssh.Signer - for _, s := range dedup { - uniq = append(uniq, s) - } - authMethods = append(authMethods, ssh.PublicKeysCallback(func() ([]ssh.Signer, error) { - return uniq, nil - })) - } - if passwdSet { // if password authentication is given and valid, add to the list - authMethods = append(authMethods, ssh.Password(passwd)) - } - if len(authMethods) == 0 { - authMethods = append(authMethods, ssh.PasswordCallback(func() (string, error) { - pass, err := terminal.ReadPassword(fmt.Sprintf("%s's login password:", uri.User.Username())) - return string(pass), err - })) - } - tick, err := time.ParseDuration("40s") - if err != nil { - return nil, err - } - cfg := &ssh.ClientConfig{ - User: uri.User.Username(), - Auth: authMethods, - HostKeyCallback: ssh.InsecureIgnoreHostKey(), - Timeout: tick, - } - return cfg, nil -} diff --git a/pkg/machine/applehv/machine.go b/pkg/machine/applehv/machine.go new file mode 100644 index 000000000..35a8e9851 --- /dev/null +++ b/pkg/machine/applehv/machine.go @@ -0,0 +1,70 @@ +//go:build arm64 && !windows && !linux +// +build darwin + +package applehv + +import ( + "time" + + "github.com/containers/podman/v4/pkg/machine" +) + +type Provider struct{} + +var ( + hvProvider = &Provider{} + // vmtype refers to qemu (vs libvirt, krun, etc). + vmtype = "apple" +) + +func GetVirtualizationProvider() machine.Provider { + return hvProvider +} + +const ( + // Some of this will need to change when we are closer to having + // working code. + VolumeTypeVirtfs = "virtfs" + MountType9p = "9p" + dockerSock = "/var/run/docker.sock" + dockerConnectTimeout = 5 * time.Second + apiUpTimeout = 20 * time.Second +) + +type apiForwardingState int + +const ( + noForwarding apiForwardingState = iota + claimUnsupported + notInstalled + machineLocal + dockerGlobal +) + +func (p *Provider) NewMachine(opts machine.InitOptions) (machine.VM, error) { + return nil, machine.ErrNotImplemented +} + +func (p *Provider) LoadVMByName(name string) (machine.VM, error) { + return nil, machine.ErrNotImplemented +} + +func (p *Provider) List(opts machine.ListOptions) ([]*machine.ListResponse, error) { + return nil, machine.ErrNotImplemented +} + +func (p *Provider) IsValidVMName(name string) (bool, error) { + return false, machine.ErrNotImplemented +} + +func (p *Provider) CheckExclusiveActiveVM() (bool, string, error) { + return false, "", machine.ErrNotImplemented +} + +func (p *Provider) RemoveAndCleanMachines() error { + return machine.ErrNotImplemented +} + +func (p *Provider) VMType() string { + return vmtype +} diff --git a/pkg/machine/config.go b/pkg/machine/config.go index 253601dad..5162006db 100644 --- a/pkg/machine/config.go +++ b/pkg/machine/config.go @@ -66,6 +66,7 @@ var ( ErrVMAlreadyExists = errors.New("VM already exists") ErrVMAlreadyRunning = errors.New("VM already running or starting") ErrMultipleActiveVM = errors.New("only one VM can be active at a time") + ErrNotImplemented = errors.New("functionality not implemented") ForwarderBinaryName = "gvproxy" ) diff --git a/pkg/machine/e2e/init_test.go b/pkg/machine/e2e/init_test.go index b246dc4da..859a3ca46 100644 --- a/pkg/machine/e2e/init_test.go +++ b/pkg/machine/e2e/init_test.go @@ -3,7 +3,7 @@ package e2e_test import ( "io/ioutil" "os" - "runtime" + "strconv" "time" "github.com/containers/podman/v4/pkg/machine" @@ -80,7 +80,7 @@ var _ = Describe("podman machine init", func() { It("machine init with cpus, disk size, memory, timezone", func() { name := randomString() i := new(initMachine) - session, err := mb.setName(name).setCmd(i.withImagePath(mb.imagePath).withCPUs(2).withDiskSize(102).withMemory(4000).withTimezone("Pacific/Honolulu")).run() + session, err := mb.setName(name).setCmd(i.withImagePath(mb.imagePath).withCPUs(2).withDiskSize(102).withMemory(4096).withTimezone("Pacific/Honolulu")).run() Expect(err).To(BeNil()) Expect(session).To(Exit(0)) @@ -102,18 +102,13 @@ var _ = Describe("podman machine init", func() { Expect(diskSession.outputToString()).To(ContainSubstring("102 GiB")) sshMemory := sshMachine{} - memorySession, err := mb.setName(name).setCmd(sshMemory.withSSHComand([]string{"cat", "/proc/meminfo", "|", "numfmt", "--field", "2", "--from-unit=Ki", "--to-unit=Mi", "|", "sed", "'s/ kB/M/g'", "|", "grep", "MemTotal"})).run() + memorySession, err := mb.setName(name).setCmd(sshMemory.withSSHComand([]string{"cat", "/proc/meminfo", "|", "grep", "-i", "'memtotal'", "|", "grep", "-o", "'[[:digit:]]*'"})).run() Expect(err).To(BeNil()) Expect(memorySession).To(Exit(0)) - switch runtime.GOOS { - // os's handle memory differently - case "linux": - Expect(memorySession.outputToString()).To(ContainSubstring("3822")) - case "darwin": - Expect(memorySession.outputToString()).To(ContainSubstring("3824")) - default: - // add windows when testing on that platform - } + foundMemory, err := strconv.Atoi(memorySession.outputToString()) + Expect(err).To(BeNil()) + Expect(foundMemory).To(BeNumerically(">", 3800000)) + Expect(foundMemory).To(BeNumerically("<", 4200000)) sshTimezone := sshMachine{} timezoneSession, err := mb.setName(name).setCmd(sshTimezone.withSSHComand([]string{"date"})).run() diff --git a/pkg/machine/e2e/set_test.go b/pkg/machine/e2e/set_test.go index 4839e33da..a32bb72f2 100644 --- a/pkg/machine/e2e/set_test.go +++ b/pkg/machine/e2e/set_test.go @@ -1,7 +1,7 @@ package e2e_test import ( - "runtime" + "strconv" . "github.com/onsi/ginkgo" . "github.com/onsi/gomega" @@ -29,7 +29,7 @@ var _ = Describe("podman machine set", func() { Expect(session).To(Exit(0)) set := setMachine{} - setSession, err := mb.setName(name).setCmd(set.withCPUs(2).withDiskSize(102).withMemory(4000)).run() + setSession, err := mb.setName(name).setCmd(set.withCPUs(2).withDiskSize(102).withMemory(4096)).run() Expect(err).To(BeNil()) Expect(setSession).To(Exit(0)) @@ -56,18 +56,14 @@ var _ = Describe("podman machine set", func() { Expect(diskSession.outputToString()).To(ContainSubstring("102 GiB")) sshMemory := sshMachine{} - memorySession, err := mb.setName(name).setCmd(sshMemory.withSSHComand([]string{"cat", "/proc/meminfo", "|", "numfmt", "--field", "2", "--from-unit=Ki", "--to-unit=Mi", "|", "sed", "'s/ kB/M/g'", "|", "grep", "MemTotal"})).run() + memorySession, err := mb.setName(name).setCmd(sshMemory.withSSHComand([]string{"cat", "/proc/meminfo", "|", "grep", "-i", "'memtotal'", "|", "grep", "-o", "'[[:digit:]]*'"})).run() Expect(err).To(BeNil()) Expect(memorySession).To(Exit(0)) - switch runtime.GOOS { - // it seems macos and linux handle memory differently - case "linux": - Expect(memorySession.outputToString()).To(ContainSubstring("3822")) - case "darwin": - Expect(memorySession.outputToString()).To(ContainSubstring("3824")) - default: - // windows can go here if we ever run tests there - } + foundMemory, err := strconv.Atoi(memorySession.outputToString()) + Expect(err).To(BeNil()) + Expect(foundMemory).To(BeNumerically(">", 3800000)) + Expect(foundMemory).To(BeNumerically("<", 4200000)) + // Setting a running machine results in 125 runner, err := mb.setName(name).setCmd(set.withCPUs(4)).run() Expect(err).To(BeNil()) diff --git a/pkg/machine/qemu/machine.go b/pkg/machine/qemu/machine.go index 7974c261e..213f7ce5d 100644 --- a/pkg/machine/qemu/machine.go +++ b/pkg/machine/qemu/machine.go @@ -42,7 +42,7 @@ var ( vmtype = "qemu" ) -func GetQemuProvider() machine.Provider { +func GetVirtualizationProvider() machine.Provider { return qemuProvider } diff --git a/pkg/machine/wsl/machine.go b/pkg/machine/wsl/machine.go index 9a57102f0..8f6ef7a43 100644 --- a/pkg/machine/wsl/machine.go +++ b/pkg/machine/wsl/machine.go @@ -364,14 +364,6 @@ func (v *MachineVM) Init(opts machine.InitOptions) (bool, error) { return false, err } - if err := v.writeConfig(); err != nil { - return false, err - } - - if err := setupConnections(v, opts, sshDir); err != nil { - return false, err - } - dist, err := provisionWSLDist(v) if err != nil { return false, err @@ -393,6 +385,14 @@ func (v *MachineVM) Init(opts machine.InitOptions) (bool, error) { // Cycle so that user change goes into effect _ = terminateDist(dist) + if err := v.writeConfig(); err != nil { + return false, err + } + + if err := setupConnections(v, opts, sshDir); err != nil { + return false, err + } + return true, nil } diff --git a/pkg/specgen/generate/container.go b/pkg/specgen/generate/container.go index b5d10df8c..ec85f0f79 100644 --- a/pkg/specgen/generate/container.go +++ b/pkg/specgen/generate/container.go @@ -192,16 +192,24 @@ func CompleteSpec(ctx context.Context, r *libpod.Runtime, s *specgen.SpecGenerat // - "container" denotes the container should join the VM of the SandboxID // (the infra container) if len(s.Pod) > 0 { - annotations[ann.SandboxID] = s.Pod + p, err := r.LookupPod(s.Pod) + if err != nil { + return nil, err + } + sandboxID := p.ID() + if p.HasInfraContainer() { + infra, err := p.InfraContainer() + if err != nil { + return nil, err + } + sandboxID = infra.ID() + } + annotations[ann.SandboxID] = sandboxID annotations[ann.ContainerType] = ann.ContainerTypeContainer // Check if this is an init-ctr and if so, check if // the pod is running. we do not want to add init-ctrs to // a running pod because it creates confusion for us. if len(s.InitContainerType) > 0 { - p, err := r.LookupPod(s.Pod) - if err != nil { - return nil, err - } containerStatuses, err := p.Status() if err != nil { return nil, err diff --git a/pkg/terminal/util.go b/pkg/terminal/util.go deleted file mode 100644 index 0f0968c30..000000000 --- a/pkg/terminal/util.go +++ /dev/null @@ -1,134 +0,0 @@ -package terminal - -import ( - "bufio" - "errors" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "sync" - - "github.com/containers/storage/pkg/homedir" - "github.com/sirupsen/logrus" - "golang.org/x/crypto/ssh" - "golang.org/x/crypto/ssh/knownhosts" - "golang.org/x/term" -) - -var ( - passPhrase []byte - phraseSync sync.Once - password []byte - passwordSync sync.Once -) - -// ReadPassword prompts for a secret and returns value input by user from stdin -// Unlike terminal.ReadPassword(), $(echo $SECRET | podman...) is supported. -// Additionally, all input after `<secret>/n` is queued to podman command. -func ReadPassword(prompt string) (pw []byte, err error) { - fd := int(os.Stdin.Fd()) - if term.IsTerminal(fd) { - fmt.Fprint(os.Stderr, prompt) - pw, err = term.ReadPassword(fd) - fmt.Fprintln(os.Stderr) - return - } - - var b [1]byte - for { - n, err := os.Stdin.Read(b[:]) - // terminal.ReadPassword discards any '\r', so we do the same - if n > 0 && b[0] != '\r' { - if b[0] == '\n' { - return pw, nil - } - pw = append(pw, b[0]) - // limit size, so that a wrong input won't fill up the memory - if len(pw) > 1024 { - err = errors.New("password too long, 1024 byte limit") - } - } - if err != nil { - // terminal.ReadPassword accepts EOF-terminated passwords - // if non-empty, so we do the same - if err == io.EOF && len(pw) > 0 { - err = nil - } - return pw, err - } - } -} - -func PublicKey(path string, passphrase []byte) (ssh.Signer, error) { - key, err := ioutil.ReadFile(path) - if err != nil { - return nil, err - } - - signer, err := ssh.ParsePrivateKey(key) - if err != nil { - if _, ok := err.(*ssh.PassphraseMissingError); !ok { - return nil, err - } - if len(passphrase) == 0 { - passphrase = ReadPassphrase() - } - return ssh.ParsePrivateKeyWithPassphrase(key, passphrase) - } - return signer, nil -} - -func ReadPassphrase() []byte { - phraseSync.Do(func() { - secret, err := ReadPassword("Key Passphrase: ") - if err != nil { - secret = []byte{} - } - passPhrase = secret - }) - return passPhrase -} - -func ReadLogin() []byte { - passwordSync.Do(func() { - secret, err := ReadPassword("Login password: ") - if err != nil { - secret = []byte{} - } - password = secret - }) - return password -} - -func HostKey(host string) ssh.PublicKey { - // parse OpenSSH known_hosts file - // ssh or use ssh-keyscan to get initial key - knownHosts := filepath.Join(homedir.Get(), ".ssh", "known_hosts") - fd, err := os.Open(knownHosts) - if err != nil { - logrus.Error(err) - return nil - } - - // support -H parameter for ssh-keyscan - hashhost := knownhosts.HashHostname(host) - - scanner := bufio.NewScanner(fd) - for scanner.Scan() { - _, hosts, key, _, _, err := ssh.ParseKnownHosts(scanner.Bytes()) - if err != nil { - logrus.Errorf("Failed to parse known_hosts: %s", scanner.Text()) - continue - } - - for _, h := range hosts { - if h == host || h == hashhost { - return key - } - } - } - - return nil -} |