Merge pull request #10828 from cdoern/scp

Created image scp feature

3 files changed, 445 insertions, 57 deletions

diff --git a/cmd/podman/images/scp.go b/cmd/podman/images/scp.go
new file mode 100644
index 000000000..71399e0b7
--- /dev/null
+++ b/cmd/podman/images/scp.go
@@ -0,0 +1,348 @@
+package images
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	urlP "net/url"
+	"os"
+	"strconv"
+	"strings"
+
+	"github.com/containers/common/pkg/config"
+	"github.com/containers/podman/v3/cmd/podman/common"
+	"github.com/containers/podman/v3/cmd/podman/parse"
+	"github.com/containers/podman/v3/cmd/podman/registry"
+	"github.com/containers/podman/v3/cmd/podman/system/connection"
+	"github.com/containers/podman/v3/libpod/define"
+	"github.com/containers/podman/v3/pkg/domain/entities"
+	"github.com/docker/distribution/reference"
+	scpD "github.com/dtylman/scp"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"github.com/spf13/cobra"
+	"golang.org/x/crypto/ssh"
+)
+
+var (
+	saveScpDescription = `Securely copy an image from one host to another.`
+	imageScpCommand    = &cobra.Command{
+		Use:               "scp [options] IMAGE [HOST::]",
+		Annotations:       map[string]string{registry.EngineMode: registry.ABIMode},
+		Long:              saveScpDescription,
+		Short:             "securely copy images",
+		RunE:              scp,
+		Args:              cobra.RangeArgs(1, 2),
+		ValidArgsFunction: common.AutocompleteImages,
+		Example:           `podman image scp myimage:latest otherhost::`,
+	}
+)
+
+var (
+	scpOpts entities.ImageScpOptions
+)
+
+func init() {
+	registry.Commands = append(registry.Commands, registry.CliCommand{
+		Command: imageScpCommand,
+		Parent:  imageCmd,
+	})
+	scpFlags(imageScpCommand)
+}
+
+func scpFlags(cmd *cobra.Command) {
+	flags := cmd.Flags()
+	flags.BoolVarP(&scpOpts.Save.Quiet, "quiet", "q", false, "Suppress the output")
+}
+
+func scp(cmd *cobra.Command, args []string) (finalErr error) {
+	var (
+		// TODO add tag support for images
+		err error
+	)
+	if scpOpts.Save.Quiet { // set quiet for both load and save
+		scpOpts.Load.Quiet = true
+	}
+	f, err := ioutil.TempFile("", "podman") // open temp file for load/save output
+	if err != nil {
+		return err
+	}
+	defer os.Remove(f.Name())
+
+	scpOpts.Save.Output = f.Name()
+	scpOpts.Load.Input = scpOpts.Save.Output
+	if err := parse.ValidateFileName(saveOpts.Output); err != nil {
+		return err
+	}
+	confR, err := config.NewConfig("") // create a hand made config for the remote engine since we might use remote and native at once
+	if err != nil {
+		return errors.Wrapf(err, "could not make config")
+	}
+	abiEng, err := registry.NewImageEngine(cmd, args) // abi native engine
+	if err != nil {
+		return err
+	}
+
+	cfg, err := config.ReadCustomConfig() // get ready to set ssh destination if necessary
+	if err != nil {
+		return err
+	}
+	serv, err := parseArgs(args, cfg) // parses connection data and "which way" we are loading and saving
+	if err != nil {
+		return err
+	}
+	// TODO: Add podman remote support
+	confR.Engine = config.EngineConfig{Remote: true, CgroupManager: "cgroupfs", ServiceDestinations: serv} // pass the service dest (either remote or something else) to engine
+	switch {
+	case scpOpts.FromRemote: // if we want to load FROM the remote
+		err = saveToRemote(scpOpts.SourceImageName, scpOpts.Save.Output, "", scpOpts.URI[0], scpOpts.Iden[0])
+		if err != nil {
+			return err
+		}
+		if scpOpts.ToRemote { // we want to load remote -> remote
+			rep, err := loadToRemote(scpOpts.Save.Output, "", scpOpts.URI[1], scpOpts.Iden[1])
+			if err != nil {
+				return err
+			}
+			fmt.Println(rep)
+			break
+		}
+		report, err := abiEng.Load(context.Background(), scpOpts.Load)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Loaded images(s): " + strings.Join(report.Names, ","))
+	case scpOpts.ToRemote: // remote host load
+		scpOpts.Save.Format = "oci-archive"
+		abiErr := abiEng.Save(context.Background(), scpOpts.SourceImageName, []string{}, scpOpts.Save) // save the image locally before loading it on remote, local, or ssh
+		if abiErr != nil {
+			errors.Wrapf(abiErr, "could not save image as specified")
+		}
+		rep, err := loadToRemote(scpOpts.Save.Output, "", scpOpts.URI[0], scpOpts.Iden[0])
+		if err != nil {
+			return err
+		}
+		fmt.Println(rep)
+	// TODO: Add podman remote support
+	default: // else native load
+		if scpOpts.Tag != "" {
+			return errors.Wrapf(define.ErrInvalidArg, "Renaming of an image is currently not supported")
+		}
+		scpOpts.Save.Format = "oci-archive"
+		abiErr := abiEng.Save(context.Background(), scpOpts.SourceImageName, []string{}, scpOpts.Save) // save the image locally before loading it on remote, local, or ssh
+		if abiErr != nil {
+			errors.Wrapf(abiErr, "could not save image as specified")
+		}
+		rep, err := abiEng.Load(context.Background(), scpOpts.Load)
+		if err != nil {
+			return err
+		}
+		fmt.Println("Loaded images(s): " + strings.Join(rep.Names, ","))
+	}
+	return nil
+}
+
+// loadToRemote takes image and remote connection information. it connects to the specified client
+// and copies the saved image dir over to the remote host and then loads it onto the machine
+// returns a string containing output or an error
+func loadToRemote(localFile string, tag string, url *urlP.URL, iden string) (string, error) {
+	dial, remoteFile, err := createConnection(url, iden)
+	if err != nil {
+		return "", err
+	}
+	defer dial.Close()
+
+	n, err := scpD.CopyTo(dial, localFile, remoteFile)
+	if err != nil {
+		errOut := (strconv.Itoa(int(n)) + " Bytes copied before error")
+		return " ", errors.Wrapf(err, errOut)
+	}
+	run := ""
+	if tag != "" {
+		return "", errors.Wrapf(define.ErrInvalidArg, "Renaming of an image is currently not supported")
+	}
+	podman := os.Args[0]
+	run = podman + " image load --input=" + remoteFile + ";rm " + remoteFile // run ssh image load of the file copied via scp
+	out, err := connection.ExecRemoteCommand(dial, run)
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSuffix(out, "\n"), nil
+}
+
+// saveToRemote takes image information and remote connection information. it connects to the specified client
+// and saves the specified image on the remote machine and then copies it to the specified local location
+// returns an error if one occurs.
+func saveToRemote(image, localFile string, tag string, uri *urlP.URL, iden string) error {
+	dial, remoteFile, err := createConnection(uri, iden)
+
+	if err != nil {
+		return err
+	}
+	defer dial.Close()
+
+	if tag != "" {
+		return errors.Wrapf(define.ErrInvalidArg, "Renaming of an image is currently not supported")
+	}
+	podman := os.Args[0]
+	run := podman + " image save " + image + " --format=oci-archive --output=" + remoteFile // run ssh image load of the file copied via scp. Files are reverse in thie case...
+	_, err = connection.ExecRemoteCommand(dial, run)
+	if err != nil {
+		return nil
+	}
+	n, err := scpD.CopyFrom(dial, remoteFile, localFile)
+	connection.ExecRemoteCommand(dial, "rm "+remoteFile)
+	if err != nil {
+		errOut := (strconv.Itoa(int(n)) + " Bytes copied before error")
+		return errors.Wrapf(err, errOut)
+	}
+	return nil
+}
+
+// makeRemoteFile creates the necessary remote file on the host to
+// save or load the image to. returns a string with the file name or an error
+func makeRemoteFile(dial *ssh.Client) (string, error) {
+	run := "mktemp"
+	remoteFile, err := connection.ExecRemoteCommand(dial, run)
+	if err != nil {
+		return "", err
+	}
+	remoteFile = strings.TrimSuffix(remoteFile, "\n")
+	if err != nil {
+		return "", err
+	}
+	return remoteFile, nil
+}
+
+// createConnections takes a boolean determining which ssh client to dial
+// and returns the dials client, its newly opened remote file, and an error if applicable.
+func createConnection(url *urlP.URL, iden string) (*ssh.Client, string, error) {
+	cfg, err := connection.ValidateAndConfigure(url, iden)
+	if err != nil {
+		return nil, "", err
+	}
+	dialAdd, err := ssh.Dial("tcp", url.Host, cfg) // dial the client
+	if err != nil {
+		return nil, "", errors.Wrapf(err, "failed to connect")
+	}
+	file, err := makeRemoteFile(dialAdd)
+	if err != nil {
+		return nil, "", err
+	}
+
+	return dialAdd, file, nil
+}
+
+// validateImageName makes sure that the image given is valid and no injections are occurring
+// we simply use this for error checking, bot setting the image
+func validateImageName(input string) error {
+	// ParseNormalizedNamed transforms a shortname image into its
+	// full name reference so busybox => docker.io/library/busybox
+	// we want to keep our shortnames, so only return an error if
+	// we cannot parse what th euser has given us
+	_, err := reference.ParseNormalizedNamed(input)
+	return err
+}
+
+// remoteArgLength is a helper function to simplify the extracting of host argument data
+// returns an int which contains the length of a specified index in a host::image string
+func remoteArgLength(input string, side int) int {
+	return len((strings.Split(input, "::"))[side])
+}
+
+// parseArgs returns the valid connection data based off of the information provided by the user
+// args is an array of the command arguments and cfg is tooling configuration used to get service destinations
+// returned is serv and an error if applicable. serv is a map of service destinations with the connection name as the index
+// this connection name is intended to be used as EngineConfig.ServiceDestinations
+// this function modifies the global scpOpt entities: FromRemote, ToRemote, Connections, and SourceImageName
+func parseArgs(args []string, cfg *config.Config) (map[string]config.Destination, error) {
+	serv := map[string]config.Destination{}
+	cliConnections := []string{}
+	switch len(args) {
+	case 1:
+		if strings.Contains(args[0], "::") {
+			scpOpts.FromRemote = true
+			cliConnections = append(cliConnections, args[0])
+		} else {
+			err := validateImageName(args[0])
+			if err != nil {
+				return nil, err
+			}
+			scpOpts.SourceImageName = args[0]
+		}
+	case 2:
+		if strings.Contains(args[0], "::") {
+			if !(strings.Contains(args[1], "::")) && remoteArgLength(args[0], 1) == 0 { // if an image is specified, this mean we are loading to our client
+				cliConnections = append(cliConnections, args[0])
+				scpOpts.ToRemote = true
+				scpOpts.SourceImageName = args[1]
+			} else if strings.Contains(args[1], "::") { // both remote clients
+				scpOpts.FromRemote = true
+				scpOpts.ToRemote = true
+				if remoteArgLength(args[0], 1) == 0 { // is save->load w/ one image name
+					cliConnections = append(cliConnections, args[0])
+					cliConnections = append(cliConnections, args[1])
+				} else if remoteArgLength(args[0], 1) > 0 && remoteArgLength(args[1], 1) > 0 {
+					//in the future, this function could, instead of rejecting renames, also set a DestImageName field
+					return nil, errors.Wrapf(define.ErrInvalidArg, "cannot specify an image rename")
+				} else { // else its a load save (order of args)
+					cliConnections = append(cliConnections, args[1])
+					cliConnections = append(cliConnections, args[0])
+				}
+			} else {
+				//in the future, this function could, instead of rejecting renames, also set a DestImageName field
+				return nil, errors.Wrapf(define.ErrInvalidArg, "cannot specify an image rename")
+			}
+		} else if strings.Contains(args[1], "::") { // if we are given image host::
+			if remoteArgLength(args[1], 1) > 0 {
+				//in the future, this function could, instead of rejecting renames, also set a DestImageName field
+				return nil, errors.Wrapf(define.ErrInvalidArg, "cannot specify an image rename")
+			}
+			err := validateImageName(args[0])
+			if err != nil {
+				return nil, err
+			}
+			scpOpts.SourceImageName = args[0]
+			scpOpts.ToRemote = true
+			cliConnections = append(cliConnections, args[1])
+		} else {
+			//in the future, this function could, instead of rejecting renames, also set a DestImageName field
+			return nil, errors.Wrapf(define.ErrInvalidArg, "cannot specify an image rename")
+		}
+	}
+	var url string
+	var iden string
+	for i, val := range cliConnections {
+		splitEnv := strings.SplitN(val, "::", 2)
+		scpOpts.Connections = append(scpOpts.Connections, splitEnv[0])
+		if len(splitEnv[1]) != 0 {
+			err := validateImageName(splitEnv[1])
+			if err != nil {
+				return nil, err
+			}
+			scpOpts.SourceImageName = splitEnv[1]
+			//TODO: actually use the new name given by the user
+		}
+		conn, found := cfg.Engine.ServiceDestinations[scpOpts.Connections[i]]
+		if found {
+			url = conn.URI
+			iden = conn.Identity
+		} else { // no match, warn user and do a manual connection.
+			url = "ssh://" + scpOpts.Connections[i]
+			iden = ""
+			logrus.Warnf("Unknown connection name given. Please use system connection add to specify the default remote socket location")
+		}
+		urlT, err := urlP.Parse(url) // create an actual url to pass to exec command
+		if err != nil {
+			return nil, err
+		}
+		if urlT.User.Username() == "" {
+			if urlT.User, err = connection.GetUserInfo(urlT); err != nil {
+				return nil, err
+			}
+		}
+		scpOpts.URI = append(scpOpts.URI, urlT)
+		scpOpts.Iden = append(scpOpts.Iden, iden)
+	}
+	return serv, nil
+}
diff --git a/cmd/podman/system/connection/add.go b/cmd/podman/system/connection/add.go
index 912193d0b..290b3c245 100644
--- a/cmd/podman/system/connection/add.go
+++ b/cmd/podman/system/connection/add.go
@@ -1,7 +1,6 @@
 package connection
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
 	"net"
@@ -9,6 +8,7 @@ import (
 	"os"
 	"os/user"
 	"regexp"
+	"time"
 
 	"github.com/containers/common/pkg/completion"
 	"github.com/containers/common/pkg/config"
@@ -83,7 +83,6 @@ func add(cmd *cobra.Command, args []string) error {
 	} else if !match {
 		dest = "ssh://" + dest
 	}
-
 	uri, err := url.Parse(dest)
 	if err != nil {
 		return err
@@ -96,7 +95,7 @@ func add(cmd *cobra.Command, args []string) error {
 	switch uri.Scheme {
 	case "ssh":
 		if uri.User.Username() == "" {
-			if uri.User, err = getUserInfo(uri); err != nil {
+			if uri.User, err = GetUserInfo(uri); err != nil {
 				return err
 			}
 		}
@@ -108,9 +107,12 @@ func add(cmd *cobra.Command, args []string) error {
 		if uri.Port() == "" {
 			uri.Host = net.JoinHostPort(uri.Hostname(), cmd.Flag("port").DefValue)
 		}
-
+		iden := ""
+		if cmd.Flags().Changed("identity") {
+			iden = cOpts.Identity
+		}
 		if uri.Path == "" || uri.Path == "/" {
-			if uri.Path, err = getUDS(cmd, uri); err != nil {
+			if uri.Path, err = getUDS(cmd, uri, iden); err != nil {
 				return err
 			}
 		}
@@ -178,7 +180,7 @@ func add(cmd *cobra.Command, args []string) error {
 	return cfg.Write()
 }
 
-func getUserInfo(uri *url.URL) (*url.Userinfo, error) {
+func GetUserInfo(uri *url.URL) (*url.Userinfo, error) {
 	var (
 		usr *user.User
 		err error
@@ -202,30 +204,74 @@ func getUserInfo(uri *url.URL) (*url.Userinfo, error) {
 	return url.User(usr.Username), nil
 }
 
-func getUDS(cmd *cobra.Command, uri *url.URL) (string, error) {
-	var signers []ssh.Signer
+func getUDS(cmd *cobra.Command, uri *url.URL, iden string) (string, error) {
+	cfg, err := ValidateAndConfigure(uri, iden)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to validate")
+	}
+	dial, err := ssh.Dial("tcp", uri.Host, cfg)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to connect")
+	}
+	defer dial.Close()
+
+	session, err := dial.NewSession()
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to create new ssh session on %q", uri.Host)
+	}
+	defer session.Close()
+
+	// Override podman binary for testing etc
+	podman := "podman"
+	if v, found := os.LookupEnv("PODMAN_BINARY"); found {
+		podman = v
+	}
+	run := podman + " info --format=json"
+	out, err := ExecRemoteCommand(dial, run)
+	if err != nil {
+		return "", err
+	}
+	infoJSON, err := json.Marshal(out)
+	if err != nil {
+		return "", err
+	}
+
+	var info define.Info
+	if err := json.Unmarshal(infoJSON, &info); err != nil {
+		return "", errors.Wrapf(err, "failed to parse 'podman info' results")
+	}
+
+	if info.Host.RemoteSocket == nil || len(info.Host.RemoteSocket.Path) == 0 {
+		return "", errors.Errorf("remote podman %q failed to report its UDS socket", uri.Host)
+	}
+	return info.Host.RemoteSocket.Path, nil
+}
 
+// ValidateAndConfigure will take a ssh url and an identity key (rsa and the like) and ensure the information given is valid
+// iden iden can be blank to mean no identity key
+// once the function validates the information it creates and returns an ssh.ClientConfig
+func ValidateAndConfigure(uri *url.URL, iden string) (*ssh.ClientConfig, error) {
+	var signers []ssh.Signer
 	passwd, passwdSet := uri.User.Password()
-	if cmd.Flags().Changed("identity") {
-		value := cmd.Flag("identity").Value.String()
+	if iden != "" { // iden might be blank if coming from image scp or if no validation is needed
+		value := iden
 		s, err := terminal.PublicKey(value, []byte(passwd))
 		if err != nil {
-			return "", errors.Wrapf(err, "failed to read identity %q", value)
+			return nil, errors.Wrapf(err, "failed to read identity %q", value)
 		}
 		signers = append(signers, s)
 		logrus.Debugf("SSH Ident Key %q %s %s", value, ssh.FingerprintSHA256(s.PublicKey()), s.PublicKey().Type())
 	}
-
-	if sock, found := os.LookupEnv("SSH_AUTH_SOCK"); found {
+	if sock, found := os.LookupEnv("SSH_AUTH_SOCK"); found { // validate ssh information, specifically the unix file socket used by the ssh agent.
 		logrus.Debugf("Found SSH_AUTH_SOCK %q, ssh-agent signer enabled", sock)
 
 		c, err := net.Dial("unix", sock)
 		if err != nil {
-			return "", err
+			return nil, err
 		}
 		agentSigners, err := agent.NewClient(c).Signers()
 		if err != nil {
-			return "", err
+			return nil, err
 		}
 
 		signers = append(signers, agentSigners...)
@@ -236,11 +282,9 @@ func getUDS(cmd *cobra.Command, uri *url.URL) (string, error) {
 			}
 		}
 	}
-
-	var authMethods []ssh.AuthMethod
+	var authMethods []ssh.AuthMethod // now we validate and check for the authorization methods, most notaibly public key authorization
 	if len(signers) > 0 {
 		var dedup = make(map[string]ssh.Signer)
-		// Dedup signers based on fingerprint, ssh-agent keys override CONTAINER_SSHKEY
 		for _, s := range signers {
 			fp := ssh.FingerprintSHA256(s.PublicKey())
 			if _, found := dedup[fp]; found {
@@ -253,60 +297,28 @@ func getUDS(cmd *cobra.Command, uri *url.URL) (string, error) {
 		for _, s := range dedup {
 			uniq = append(uniq, s)
 		}
-
 		authMethods = append(authMethods, ssh.PublicKeysCallback(func() ([]ssh.Signer, error) {
 			return uniq, nil
 		}))
 	}
-
-	if passwdSet {
+	if passwdSet { // if password authentication is given and valid, add to the list
 		authMethods = append(authMethods, ssh.Password(passwd))
 	}
-
 	if len(authMethods) == 0 {
 		authMethods = append(authMethods, ssh.PasswordCallback(func() (string, error) {
 			pass, err := terminal.ReadPassword(fmt.Sprintf("%s's login password:", uri.User.Username()))
 			return string(pass), err
 		}))
 	}
-
+	tick, err := time.ParseDuration("40s")
+	if err != nil {
+		return nil, err
+	}
 	cfg := &ssh.ClientConfig{
 		User:            uri.User.Username(),
 		Auth:            authMethods,
 		HostKeyCallback: ssh.InsecureIgnoreHostKey(),
+		Timeout:         tick,
 	}
-	dial, err := ssh.Dial("tcp", uri.Host, cfg)
-	if err != nil {
-		return "", errors.Wrapf(err, "failed to connect")
-	}
-	defer dial.Close()
-
-	session, err := dial.NewSession()
-	if err != nil {
-		return "", errors.Wrapf(err, "failed to create new ssh session on %q", uri.Host)
-	}
-	defer session.Close()
-
-	// Override podman binary for testing etc
-	podman := "podman"
-	if v, found := os.LookupEnv("PODMAN_BINARY"); found {
-		podman = v
-	}
-	run := podman + " info --format=json"
-
-	var buffer bytes.Buffer
-	session.Stdout = &buffer
-	if err := session.Run(run); err != nil {
-		return "", err
-	}
-
-	var info define.Info
-	if err := json.Unmarshal(buffer.Bytes(), &info); err != nil {
-		return "", errors.Wrapf(err, "failed to parse 'podman info' results")
-	}
-
-	if info.Host.RemoteSocket == nil || len(info.Host.RemoteSocket.Path) == 0 {
-		return "", errors.Errorf("remote podman %q failed to report its UDS socket", uri.Host)
-	}
-	return info.Host.RemoteSocket.Path, nil
+	return cfg, nil
 }
diff --git a/cmd/podman/system/connection/shared.go b/cmd/podman/system/connection/shared.go
new file mode 100644
index 000000000..3fd7c59fb
--- /dev/null
+++ b/cmd/podman/system/connection/shared.go
@@ -0,0 +1,28 @@
+package connection
+
+import (
+	"bytes"
+
+	"github.com/pkg/errors"
+	"golang.org/x/crypto/ssh"
+)
+
+// ExecRemoteCommand takes a ssh client connection and a command to run and executes the
+// command on the specified client. The function returns the Stdout from the client or the Stderr
+func ExecRemoteCommand(dial *ssh.Client, run string) (string, error) {
+	sess, err := dial.NewSession() // new ssh client session
+	if err != nil {
+		return "", err
+	}
+	defer sess.Close()
+
+	var buffer bytes.Buffer
+	var bufferErr bytes.Buffer
+	sess.Stdout = &buffer                 // output from client funneled into buffer
+	sess.Stderr = &bufferErr              // err form client funneled into buffer
+	if err := sess.Run(run); err != nil { // run the command on the ssh client
+		return "", errors.Wrapf(err, bufferErr.String())
+	}
+	out := buffer.String() // output from command
+	return out, nil
+}