test connection add

* Fix connection JSON encoding
* Add custom ginkgo matchers for connection testing
* Cleanup code

Fixes #11984

Signed-off-by: Jhon Honce <jhonce@redhat.com>

10 files changed, 451 insertions, 299 deletions

diff --git a/cmd/podman/images/scp.go b/cmd/podman/images/scp.go
index 8402d9a10..67a531e6b 100644
--- a/cmd/podman/images/scp.go
+++ b/cmd/podman/images/scp.go
@@ -168,7 +168,7 @@ func loadToRemote(localFile string, tag string, url *urlP.URL, iden string) (str
 
 	n, err := scpD.CopyTo(dial, localFile, remoteFile)
 	if err != nil {
-		errOut := (strconv.Itoa(int(n)) + " Bytes copied before error")
+		errOut := strconv.Itoa(int(n)) + " Bytes copied before error"
 		return " ", errors.Wrapf(err, errOut)
 	}
 	run := ""
@@ -181,7 +181,7 @@ func loadToRemote(localFile string, tag string, url *urlP.URL, iden string) (str
 	if err != nil {
 		return "", err
 	}
-	return strings.TrimSuffix(out, "\n"), nil
+	return strings.TrimSuffix(string(out), "\n"), nil
 }
 
 // saveToRemote takes image information and remote connection information. it connects to the specified client
@@ -207,7 +207,7 @@ func saveToRemote(image, localFile string, tag string, uri *urlP.URL, iden strin
 	n, err := scpD.CopyFrom(dial, remoteFile, localFile)
 	connection.ExecRemoteCommand(dial, "rm "+remoteFile)
 	if err != nil {
-		errOut := (strconv.Itoa(int(n)) + " Bytes copied before error")
+		errOut := strconv.Itoa(int(n)) + " Bytes copied before error"
 		return errors.Wrapf(err, errOut)
 	}
 	return nil
@@ -221,11 +221,7 @@ func makeRemoteFile(dial *ssh.Client) (string, error) {
 	if err != nil {
 		return "", err
 	}
-	remoteFile = strings.TrimSuffix(remoteFile, "\n")
-	if err != nil {
-		return "", err
-	}
-	return remoteFile, nil
+	return strings.TrimSuffix(string(remoteFile), "\n"), nil
 }
 
 // createConnections takes a boolean determining which ssh client to dial
diff --git a/cmd/podman/system/connection/add.go b/cmd/podman/system/connection/add.go
index 290b3c245..ee237d7d0 100644
--- a/cmd/podman/system/connection/add.go
+++ b/cmd/podman/system/connection/add.go
@@ -226,12 +226,7 @@ func getUDS(cmd *cobra.Command, uri *url.URL, iden string) (string, error) {
 	if v, found := os.LookupEnv("PODMAN_BINARY"); found {
 		podman = v
 	}
-	run := podman + " info --format=json"
-	out, err := ExecRemoteCommand(dial, run)
-	if err != nil {
-		return "", err
-	}
-	infoJSON, err := json.Marshal(out)
+	infoJSON, err := ExecRemoteCommand(dial, podman+" info --format=json")
 	if err != nil {
 		return "", err
 	}
diff --git a/cmd/podman/system/connection/shared.go b/cmd/podman/system/connection/shared.go
index 3fd7c59fb..714ae827d 100644
--- a/cmd/podman/system/connection/shared.go
+++ b/cmd/podman/system/connection/shared.go
@@ -9,10 +9,10 @@ import (
 
 // ExecRemoteCommand takes a ssh client connection and a command to run and executes the
 // command on the specified client. The function returns the Stdout from the client or the Stderr
-func ExecRemoteCommand(dial *ssh.Client, run string) (string, error) {
+func ExecRemoteCommand(dial *ssh.Client, run string) ([]byte, error) {
 	sess, err := dial.NewSession() // new ssh client session
 	if err != nil {
-		return "", err
+		return nil, err
 	}
 	defer sess.Close()
 
@@ -21,8 +21,7 @@ func ExecRemoteCommand(dial *ssh.Client, run string) (string, error) {
 	sess.Stdout = &buffer                 // output from client funneled into buffer
 	sess.Stderr = &bufferErr              // err form client funneled into buffer
 	if err := sess.Run(run); err != nil { // run the command on the ssh client
-		return "", errors.Wrapf(err, bufferErr.String())
+		return nil, errors.Wrapf(err, bufferErr.String())
 	}
-	out := buffer.String() // output from command
-	return out, nil
+	return buffer.Bytes(), nil
 }
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 9b7c613f5..cff8f4b3f 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -166,30 +166,42 @@ setup_rootless() {
     useradd -g $rootless_gid -u $rootless_uid --no-user-group --create-home $ROOTLESS_USER
     chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOPATH" "$GOSRC"
 
-    msg "creating ssh key pair for $USER"
+    mkdir -p "$HOME/.ssh" "/home/$ROOTLESS_USER/.ssh"
+
+    msg "Creating ssh key pairs"
     [[ -r "$HOME/.ssh/id_rsa" ]] || \
-        ssh-keygen -P "" -f "$HOME/.ssh/id_rsa"
+        ssh-keygen -t rsa -P "" -f "$HOME/.ssh/id_rsa"
+    ssh-keygen -t ed25519 -P "" -f "/home/$ROOTLESS_USER/.ssh/id_ed25519"
+    ssh-keygen -t rsa -P "" -f "/home/$ROOTLESS_USER/.ssh/id_rsa"
 
-    msg "Allowing ssh key for $ROOTLESS_USER"
-    akfilepath="/home/$ROOTLESS_USER/.ssh/authorized_keys"
-    (umask 077 && mkdir "/home/$ROOTLESS_USER/.ssh")
-    chown -R $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.ssh"
-    install -o $ROOTLESS_USER -g $ROOTLESS_USER -m 0600 \
-        "$HOME/.ssh/id_rsa.pub" "$akfilepath"
-    # Makes debugging easier
-    cat /root/.ssh/authorized_keys >> "$akfilepath"
+    msg "Setup authorized_keys"
+    cat $HOME/.ssh/*.pub /home/$ROOTLESS_USER/.ssh/*.pub >> $HOME/.ssh/authorized_keys
+    cat $HOME/.ssh/*.pub /home/$ROOTLESS_USER/.ssh/*.pub >> /home/$ROOTLESS_USER/.ssh/authorized_keys
 
     msg "Ensure the ssh daemon is up and running within 5 minutes"
     systemctl start sshd
-    sshcmd="ssh $ROOTLESS_USER@localhost
-           -o UserKnownHostsFile=/dev/null
-           -o StrictHostKeyChecking=no
-           -o CheckHostIP=no"
-    lilto $sshcmd true  # retry until sshd is up
-
-    msg "Configuring rootless user self-access to ssh to localhost"
-    $sshcmd ssh-keygen -P '""' -f "/home/$ROOTLESS_USER/.ssh/id_rsa"
-    cat "/home/$ROOTLESS_USER/.ssh/id_rsa" >> "$akfilepath"
+    lilto systemctl is-active sshd
+
+    msg "Configure ssh file permissions"
+    chmod -R 700 "$HOME/.ssh"
+    chmod -R 700 "/home/$ROOTLESS_USER/.ssh"
+    chown -R $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.ssh"
+
+    msg "   setup known_hosts for $USER"
+    ssh -q root@localhost \
+        -o UserKnownHostsFile=/root/.ssh/known_hosts \
+        -o UpdateHostKeys=yes \
+        -o StrictHostKeyChecking=no \
+        -o CheckHostIP=no \
+        true
+
+    msg "   setup known_hosts for $ROOTLESS_USER"
+    su $ROOTLESS_USER -c "ssh -q $ROOTLESS_USER@localhost \
+        -o UserKnownHostsFile=/home/$ROOTLESS_USER/.ssh/known_hosts \
+        -o UpdateHostKeys=yes \
+        -o StrictHostKeyChecking=no \
+        -o CheckHostIP=no \
+        true"
 }
 
 install_test_configs() {
diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go
index e598f7ab9..200faae2d 100644
--- a/test/e2e/common_test.go
+++ b/test/e2e/common_test.go
@@ -208,9 +208,7 @@ var _ = SynchronizedAfterSuite(func() {},
 
 // PodmanTestCreate creates a PodmanTestIntegration instance for the tests
 func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration {
-	var (
-		podmanRemoteBinary string
-	)
+	var podmanRemoteBinary string
 
 	host := GetHostDistributionInfo()
 	cwd, _ := os.Getwd()
@@ -220,12 +218,11 @@ func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration {
 		podmanBinary = os.Getenv("PODMAN_BINARY")
 	}
 
-	if remote {
-		podmanRemoteBinary = filepath.Join(cwd, "../../bin/podman-remote")
-		if os.Getenv("PODMAN_REMOTE_BINARY") != "" {
-			podmanRemoteBinary = os.Getenv("PODMAN_REMOTE_BINARY")
-		}
+	podmanRemoteBinary = filepath.Join(cwd, "../../bin/podman-remote")
+	if os.Getenv("PODMAN_REMOTE_BINARY") != "" {
+		podmanRemoteBinary = os.Getenv("PODMAN_REMOTE_BINARY")
 	}
+
 	conmonBinary := filepath.Join("/usr/libexec/podman/conmon")
 	altConmonBinary := "/usr/bin/conmon"
 	if _, err := os.Stat(conmonBinary); os.IsNotExist(err) {
@@ -271,12 +268,13 @@ func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration {
 
 	p := &PodmanTestIntegration{
 		PodmanTest: PodmanTest{
-			PodmanBinary:  podmanBinary,
-			ArtifactPath:  ARTIFACT_DIR,
-			TempDir:       tempDir,
-			RemoteTest:    remote,
-			ImageCacheFS:  storageFs,
-			ImageCacheDir: ImageCacheDir,
+			PodmanBinary:       podmanBinary,
+			RemotePodmanBinary: podmanRemoteBinary,
+			ArtifactPath:       ARTIFACT_DIR,
+			TempDir:            tempDir,
+			RemoteTest:         remote,
+			ImageCacheFS:       storageFs,
+			ImageCacheDir:      ImageCacheDir,
 		},
 		ConmonBinary:        conmonBinary,
 		CrioRoot:            filepath.Join(tempDir, "crio"),
@@ -289,8 +287,8 @@ func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration {
 		CgroupManager:       cgroupManager,
 		Host:                host,
 	}
+
 	if remote {
-		p.PodmanTest.RemotePodmanBinary = podmanRemoteBinary
 		uuid := stringid.GenerateNonCryptoID()
 		if !rootless.IsRootless() {
 			p.RemoteSocket = fmt.Sprintf("unix:/run/podman/podman-%s.sock", uuid)
@@ -632,6 +630,19 @@ func SkipIfNotRootless(reason string) {
 	}
 }
 
+func SkipIfSystemdNotRunning(reason string) {
+	checkReason(reason)
+
+	cmd := exec.Command("systemctl", "list-units")
+	err := cmd.Run()
+	if err != nil {
+		if _, ok := err.(*exec.Error); ok {
+			ginkgo.Skip("[notSystemd]: not running " + reason)
+		}
+		Expect(err).ToNot(HaveOccurred())
+	}
+}
+
 func SkipIfNotSystemd(manager, reason string) {
 	checkReason(reason)
 	if manager != "systemd" {
@@ -683,6 +694,41 @@ func SkipIfContainerized(reason string) {
 	}
 }
 
+func SkipIfRemote(reason string) {
+	checkReason(reason)
+	if !IsRemote() {
+		return
+	}
+	ginkgo.Skip("[remote]: " + reason)
+}
+
+// SkipIfInContainer skips a test if the test is run inside a container
+func SkipIfInContainer(reason string) {
+	checkReason(reason)
+	if os.Getenv("TEST_ENVIRON") == "container" {
+		Skip("[container]: " + reason)
+	}
+}
+
+// SkipIfNotActive skips a test if the given systemd unit is not active
+func SkipIfNotActive(unit string, reason string) {
+	checkReason(reason)
+
+	var buffer bytes.Buffer
+	cmd := exec.Command("systemctl", "is-active", unit)
+	cmd.Stdout = &buffer
+	err := cmd.Start()
+	Expect(err).ToNot(HaveOccurred())
+
+	err = cmd.Wait()
+	Expect(err).ToNot(HaveOccurred())
+
+	Expect(err).ToNot(HaveOccurred())
+	if strings.TrimSpace(buffer.String()) != "active" {
+		Skip(fmt.Sprintf("[systemd]: unit %s is not active: %s", unit, reason))
+	}
+}
+
 // PodmanAsUser is the exec call to podman on the filesystem with the specified uid/gid and environment
 func (p *PodmanTestIntegration) PodmanAsUser(args []string, uid, gid uint32, cwd string, env []string) *PodmanSessionIntegration {
 	podmanSession := p.PodmanAsUserBase(args, uid, gid, cwd, env, false, false, nil, nil)
diff --git a/test/e2e/libpod_suite_remote_test.go b/test/e2e/libpod_suite_remote_test.go
index ad511cc9e..1fa29daa1 100644
--- a/test/e2e/libpod_suite_remote_test.go
+++ b/test/e2e/libpod_suite_remote_test.go
@@ -16,20 +16,12 @@ import (
 	"time"
 
 	"github.com/containers/podman/v3/pkg/rootless"
-	"github.com/onsi/ginkgo"
 )
 
 func IsRemote() bool {
 	return true
 }
 
-func SkipIfRemote(reason string) {
-	if len(reason) < 5 {
-		panic("SkipIfRemote must specify a reason to skip")
-	}
-	ginkgo.Skip("[remote]: " + reason)
-}
-
 // Podman is the exec call to podman on the filesystem
 func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration {
 	var remoteArgs = []string{"--remote", "--url", p.RemoteSocket}
diff --git a/test/e2e/libpod_suite_test.go b/test/e2e/libpod_suite_test.go
index 6d2d3fee8..001a869b1 100644
--- a/test/e2e/libpod_suite_test.go
+++ b/test/e2e/libpod_suite_test.go
@@ -16,9 +16,6 @@ func IsRemote() bool {
 	return false
 }
 
-func SkipIfRemote(string) {
-}
-
 // Podman is the exec call to podman on the filesystem
 func (p *PodmanTestIntegration) Podman(args []string) *PodmanSessionIntegration {
 	podmanSession := p.PodmanBase(args, false, false)
diff --git a/test/e2e/system_connection_test.go b/test/e2e/system_connection_test.go
index d80e6d5a0..c0e29d525 100644
--- a/test/e2e/system_connection_test.go
+++ b/test/e2e/system_connection_test.go
@@ -3,7 +3,11 @@ package integration
 import (
 	"fmt"
 	"io/ioutil"
+	"net/url"
 	"os"
+	"os/exec"
+	"os/user"
+	"path/filepath"
 
 	"github.com/containers/common/pkg/config"
 	. "github.com/containers/podman/v3/test/utils"
@@ -19,22 +23,16 @@ var _ = Describe("podman system connection", func() {
 		IsSet bool
 	}{}
 
-	var (
-		podmanTest *PodmanTestIntegration
-	)
+	var podmanTest *PodmanTestIntegration
 
 	BeforeEach(func() {
 		ConfPath.Value, ConfPath.IsSet = os.LookupEnv("CONTAINERS_CONF")
 		conf, err := ioutil.TempFile("", "containersconf")
-		if err != nil {
-			panic(err)
-		}
+		Expect(err).ToNot(HaveOccurred())
 		os.Setenv("CONTAINERS_CONF", conf.Name())
 
 		tempdir, err := CreateTempDirInTempDir()
-		if err != nil {
-			panic(err)
-		}
+		Expect(err).ToNot(HaveOccurred())
 		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.Setup()
 	})
@@ -49,221 +47,241 @@ var _ = Describe("podman system connection", func() {
 		}
 
 		f := CurrentGinkgoTestDescription()
-		timedResult := fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds())
-		GinkgoWriter.Write([]byte(timedResult))
+		GinkgoWriter.Write(
+			[]byte(
+				fmt.Sprintf("Test: %s completed in %f seconds", f.TestText, f.Duration.Seconds())))
 	})
 
-	It("add ssh://", func() {
-		cmd := []string{"system", "connection", "add",
-			"--default",
-			"--identity", "~/.ssh/id_rsa",
-			"QA",
-			"ssh://root@server.fubar.com:2222/run/podman/podman.sock",
-		}
-		session := podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
-		Expect(session.Out).Should(Say(""))
-
-		cfg, err := config.ReadCustomConfig()
-		Expect(err).ShouldNot(HaveOccurred())
-		Expect(cfg.Engine.ActiveService).To(Equal("QA"))
-		Expect(cfg.Engine.ServiceDestinations["QA"]).To(Equal(
-			config.Destination{
-				URI:      "ssh://root@server.fubar.com:2222/run/podman/podman.sock",
-				Identity: "~/.ssh/id_rsa",
-			},
-		))
-
-		cmd = []string{"system", "connection", "rename",
-			"QA",
-			"QE",
-		}
-		session = podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
-
-		cfg, err = config.ReadCustomConfig()
-		Expect(err).ShouldNot(HaveOccurred())
-		Expect(cfg.Engine.ActiveService).To(Equal("QE"))
-		Expect(cfg.Engine.ServiceDestinations["QE"]).To(Equal(
-			config.Destination{
-				URI:      "ssh://root@server.fubar.com:2222/run/podman/podman.sock",
-				Identity: "~/.ssh/id_rsa",
-			},
-		))
-	})
+	Context("without running API service", func() {
+		It("add ssh://", func() {
+			cmd := []string{"system", "connection", "add",
+				"--default",
+				"--identity", "~/.ssh/id_rsa",
+				"QA",
+				"ssh://root@server.fubar.com:2222/run/podman/podman.sock",
+			}
+			session := podmanTest.Podman(cmd)
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+			Expect(session.Out.Contents()).Should(BeEmpty())
 
-	It("add UDS", func() {
-		cmd := []string{"system", "connection", "add",
-			"QA-UDS",
-			"unix:///run/podman/podman.sock",
-		}
-		session := podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
-		Expect(session.Out).Should(Say(""))
-
-		cfg, err := config.ReadCustomConfig()
-		Expect(err).ShouldNot(HaveOccurred())
-		Expect(cfg.Engine.ActiveService).To(Equal("QA-UDS"))
-		Expect(cfg.Engine.ServiceDestinations["QA-UDS"]).To(Equal(
-			config.Destination{
-				URI:      "unix:///run/podman/podman.sock",
-				Identity: "",
-			},
-		))
-
-		cmd = []string{"system", "connection", "add",
-			"QA-UDS1",
-			"--socket-path", "/run/user/podman/podman.sock",
-			"unix:///run/podman/podman.sock",
-		}
-		session = podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
-		Expect(session.Out).Should(Say(""))
-
-		cfg, err = config.ReadCustomConfig()
-		Expect(err).ShouldNot(HaveOccurred())
-		Expect(cfg.Engine.ActiveService).To(Equal("QA-UDS"))
-		Expect(cfg.Engine.ServiceDestinations["QA-UDS1"]).To(Equal(
-			config.Destination{
-				URI:      "unix:///run/user/podman/podman.sock",
-				Identity: "",
-			},
-		))
-	})
+			cfg, err := config.ReadCustomConfig()
+			Expect(err).ShouldNot(HaveOccurred())
+			Expect(cfg).To(HaveActiveService("QA"))
+			Expect(cfg).Should(VerifyService(
+				"QA",
+				"ssh://root@server.fubar.com:2222/run/podman/podman.sock",
+				"~/.ssh/id_rsa",
+			))
 
-	It("add tcp", func() {
-		cmd := []string{"system", "connection", "add",
-			"QA-TCP",
-			"tcp://localhost:8888",
-		}
-		session := podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
-		Expect(session.Out).Should(Say(""))
-
-		cfg, err := config.ReadCustomConfig()
-		Expect(err).ShouldNot(HaveOccurred())
-		Expect(cfg.Engine.ActiveService).To(Equal("QA-TCP"))
-		Expect(cfg.Engine.ServiceDestinations["QA-TCP"]).To(Equal(
-			config.Destination{
-				URI:      "tcp://localhost:8888",
-				Identity: "",
-			},
-		))
-	})
+			cmd = []string{"system", "connection", "rename",
+				"QA",
+				"QE",
+			}
+			session = podmanTest.Podman(cmd)
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
 
-	It("remove", func() {
-		cmd := []string{"system", "connection", "add",
-			"--default",
-			"--identity", "~/.ssh/id_rsa",
-			"QA",
-			"ssh://root@server.fubar.com:2222/run/podman/podman.sock",
-		}
-		session := podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
+			Expect(config.ReadCustomConfig()).To(HaveActiveService("QE"))
+		})
 
-		for i := 0; i < 2; i++ {
-			cmd = []string{"system", "connection", "remove", "QA"}
+		It("add UDS", func() {
+			cmd := []string{"system", "connection", "add",
+				"QA-UDS",
+				"unix:///run/podman/podman.sock",
+			}
+			session := podmanTest.Podman(cmd)
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+			Expect(session.Out.Contents()).Should(BeEmpty())
+
+			Expect(config.ReadCustomConfig()).Should(VerifyService(
+				"QA-UDS",
+				"unix:///run/podman/podman.sock",
+				"",
+			))
+
+			cmd = []string{"system", "connection", "add",
+				"QA-UDS1",
+				"--socket-path", "/run/user/podman/podman.sock",
+				"unix:///run/podman/podman.sock",
+			}
 			session = podmanTest.Podman(cmd)
 			session.WaitWithDefaultTimeout()
 			Expect(session).Should(Exit(0))
-			Expect(session.Out).Should(Say(""))
+			Expect(session.Out.Contents()).Should(BeEmpty())
 
-			cfg, err := config.ReadCustomConfig()
-			Expect(err).ShouldNot(HaveOccurred())
-			Expect(cfg.Engine.ActiveService).To(BeEmpty())
-			Expect(cfg.Engine.ServiceDestinations).To(BeEmpty())
-		}
-	})
-
-	It("remove --all", func() {
-		cmd := []string{"system", "connection", "add",
-			"--default",
-			"--identity", "~/.ssh/id_rsa",
-			"QA",
-			"ssh://root@server.fubar.com:2222/run/podman/podman.sock",
-		}
-		session := podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
-
-		cmd = []string{"system", "connection", "remove", "--all"}
-		session = podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
-		Expect(session.Out).Should(Say(""))
-
-		cmd = []string{"system", "connection", "list"}
-		session = podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
-		Expect(session.Out).Should(Say(""))
-		Expect(session.Err).Should(Say(""))
-	})
+			Expect(config.ReadCustomConfig()).Should(HaveActiveService("QA-UDS"))
+			Expect(config.ReadCustomConfig()).Should(VerifyService(
+				"QA-UDS1",
+				"unix:///run/user/podman/podman.sock",
+				"",
+			))
+		})
 
-	It("default", func() {
-		for _, name := range []string{"devl", "qe"} {
+		It("add tcp", func() {
 			cmd := []string{"system", "connection", "add",
+				"QA-TCP",
+				"tcp://localhost:8888",
+			}
+			session := podmanTest.Podman(cmd)
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+			Expect(session.Out.Contents()).Should(BeEmpty())
+
+			Expect(config.ReadCustomConfig()).Should(VerifyService(
+				"QA-TCP",
+				"tcp://localhost:8888",
+				"",
+			))
+		})
+
+		It("remove", func() {
+			session := podmanTest.Podman([]string{"system", "connection", "add",
 				"--default",
 				"--identity", "~/.ssh/id_rsa",
-				name,
+				"QA",
 				"ssh://root@server.fubar.com:2222/run/podman/podman.sock",
+			})
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+
+			// two passes to test that removing non-existent connection is not an error
+			for i := 0; i < 2; i++ {
+				session = podmanTest.Podman([]string{"system", "connection", "remove", "QA"})
+				session.WaitWithDefaultTimeout()
+				Expect(session).Should(Exit(0))
+				Expect(session.Out.Contents()).Should(BeEmpty())
+
+				cfg, err := config.ReadCustomConfig()
+				Expect(err).ShouldNot(HaveOccurred())
+				Expect(cfg.Engine.ActiveService).To(BeEmpty())
+				Expect(cfg.Engine.ServiceDestinations).To(BeEmpty())
+			}
+		})
+
+		It("remove --all", func() {
+			session := podmanTest.Podman([]string{"system", "connection", "add",
+				"--default",
+				"--identity", "~/.ssh/id_rsa",
+				"QA",
+				"ssh://root@server.fubar.com:2222/run/podman/podman.sock",
+			})
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+
+			session = podmanTest.Podman([]string{"system", "connection", "remove", "--all"})
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+			Expect(session.Out.Contents()).Should(BeEmpty())
+			Expect(session.Err.Contents()).Should(BeEmpty())
+
+			session = podmanTest.Podman([]string{"system", "connection", "list"})
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+		})
+
+		It("default", func() {
+			for _, name := range []string{"devl", "qe"} {
+				cmd := []string{"system", "connection", "add",
+					"--default",
+					"--identity", "~/.ssh/id_rsa",
+					name,
+					"ssh://root@server.fubar.com:2222/run/podman/podman.sock",
+				}
+				session := podmanTest.Podman(cmd)
+				session.WaitWithDefaultTimeout()
+				Expect(session).Should(Exit(0))
 			}
+
+			cmd := []string{"system", "connection", "default", "devl"}
 			session := podmanTest.Podman(cmd)
 			session.WaitWithDefaultTimeout()
 			Expect(session).Should(Exit(0))
-		}
+			Expect(session.Out.Contents()).Should(BeEmpty())
 
-		cmd := []string{"system", "connection", "default", "devl"}
-		session := podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
-		Expect(session.Out).Should(Say(""))
-
-		cfg, err := config.ReadCustomConfig()
-		Expect(err).ShouldNot(HaveOccurred())
-		Expect(cfg.Engine.ActiveService).To(Equal("devl"))
-
-		cmd = []string{"system", "connection", "list"}
-		session = podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
-		Expect(session.Out).Should(Say("Name *URI *Identity *Default"))
-
-		cmd = []string{"system", "connection", "list", "--format", "{{.Name}}"}
-		session = podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
-		Expect(session.OutputToString()).Should(Equal("devl qe"))
-	})
+			Expect(config.ReadCustomConfig()).Should(HaveActiveService("devl"))
 
-	It("failed default", func() {
-		cmd := []string{"system", "connection", "default", "devl"}
-		session := podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).ShouldNot(Exit(0))
-		Expect(session.Err).Should(Say("destination is not defined"))
-	})
+			cmd = []string{"system", "connection", "list"}
+			session = podmanTest.Podman(cmd)
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+			Expect(session.Out).Should(Say("Name *URI *Identity *Default"))
 
-	It("failed rename", func() {
-		cmd := []string{"system", "connection", "rename", "devl", "QE"}
-		session := podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).ShouldNot(Exit(0))
-		Expect(session.Err).Should(Say("destination is not defined"))
+			cmd = []string{"system", "connection", "list", "--format", "{{.Name}}"}
+			session = podmanTest.Podman(cmd)
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+			Expect(session.OutputToString()).Should(Equal("devl qe"))
+		})
+
+		It("failed default", func() {
+			cmd := []string{"system", "connection", "default", "devl"}
+			session := podmanTest.Podman(cmd)
+			session.WaitWithDefaultTimeout()
+			Expect(session).ShouldNot(Exit(0))
+			Expect(session.Err).Should(Say("destination is not defined"))
+		})
+
+		It("failed rename", func() {
+			cmd := []string{"system", "connection", "rename", "devl", "QE"}
+			session := podmanTest.Podman(cmd)
+			session.WaitWithDefaultTimeout()
+			Expect(session).ShouldNot(Exit(0))
+			Expect(session.Err).Should(Say("destination is not defined"))
+		})
+
+		It("empty list", func() {
+			cmd := []string{"system", "connection", "list"}
+			session := podmanTest.Podman(cmd)
+			session.WaitWithDefaultTimeout()
+			Expect(session).Should(Exit(0))
+			Expect(session.Out.Contents()).Should(BeEmpty())
+			Expect(session.Err.Contents()).Should(BeEmpty())
+		})
 	})
 
-	It("empty list", func() {
-		cmd := []string{"system", "connection", "list"}
-		session := podmanTest.Podman(cmd)
-		session.WaitWithDefaultTimeout()
-		Expect(session).Should(Exit(0))
-		Expect(session.Out).Should(Say(""))
-		Expect(session.Err).Should(Say(""))
+	Context("sshd and API services required", func() {
+		BeforeEach(func() {
+			// These tests are unique in as much as they require podman, podman-remote, systemd and sshd.
+			// podman-remote commands will be executed by ginkgo directly.
+			SkipIfContainerized("sshd is not available when running in a container")
+			SkipIfRemote("connection heuristic requires both podman and podman-remote binaries")
+			SkipIfNotRootless("FIXME: setup ssh keys when root")
+			SkipIfSystemdNotRunning("cannot test connection heuristic if systemd is not running")
+			SkipIfNotActive("sshd", "cannot test connection heuristic if sshd is not running")
+		})
+
+		It("add ssh:// socket path using connection heuristic", func() {
+			u, err := user.Current()
+			Expect(err).ShouldNot(HaveOccurred())
+
+			cmd := exec.Command(podmanTest.RemotePodmanBinary,
+				"system", "connection", "add",
+				"--default",
+				"--identity", filepath.Join(u.HomeDir, ".ssh", "id_ed25519"),
+				"QA",
+				fmt.Sprintf("ssh://%s@localhost", u.Username))
+
+			session, err := Start(cmd, GinkgoWriter, GinkgoWriter)
+			Expect(err).ToNot(HaveOccurred(), fmt.Sprintf("%q failed to execute", podmanTest.RemotePodmanBinary))
+			Eventually(session, DefaultWaitTimeout).Should(Exit(0))
+			Expect(session.Out.Contents()).Should(BeEmpty())
+			Expect(session.Err.Contents()).Should(BeEmpty())
+
+			uri := url.URL{
+				Scheme: "ssh",
+				User:   url.User(u.Username),
+				Host:   "localhost:22",
+				Path:   fmt.Sprintf("/run/user/%s/podman/podman.sock", u.Uid),
+			}
+
+			Expect(config.ReadCustomConfig()).Should(HaveActiveService("QA"))
+			Expect(config.ReadCustomConfig()).Should(VerifyService(
+				"QA",
+				uri.String(),
+				filepath.Join(u.HomeDir, ".ssh", "id_ed25519"),
+			))
+		})
 	})
 })
diff --git a/test/utils/matchers.go b/test/utils/matchers.go
index 07c1232e7..17ff3ea75 100644
--- a/test/utils/matchers.go
+++ b/test/utils/matchers.go
@@ -2,57 +2,164 @@ package utils
 
 import (
 	"fmt"
+	"net/url"
 
+	"github.com/containers/common/pkg/config"
+	. "github.com/onsi/gomega"
 	"github.com/onsi/gomega/format"
 	"github.com/onsi/gomega/gexec"
+	"github.com/onsi/gomega/matchers"
+	"github.com/onsi/gomega/types"
 )
 
+// HaveActiveService verifies the given service is the active service
+func HaveActiveService(name interface{}) OmegaMatcher {
+	return WithTransform(
+		func(cfg *config.Config) string {
+			return cfg.Engine.ActiveService
+		},
+		Equal(name))
+}
+
+type ServiceMatcher struct {
+	types.GomegaMatcher
+	Name                  interface{}
+	URI                   interface{}
+	Identity              interface{}
+	failureMessage        string
+	negatedFailureMessage string
+}
+
+func VerifyService(name, uri, identity interface{}) OmegaMatcher {
+	return &ServiceMatcher{
+		Name:     name,
+		URI:      uri,
+		Identity: identity,
+	}
+}
+
+func (matcher *ServiceMatcher) Match(actual interface{}) (success bool, err error) {
+	cfg, ok := actual.(*config.Config)
+	if !ok {
+		return false, fmt.Errorf("ServiceMatcher matcher expects a config.Config")
+	}
+
+	if _, err = url.Parse(matcher.URI.(string)); err != nil {
+		return false, err
+	}
+
+	success, err = HaveKey(matcher.Name).Match(cfg.Engine.ServiceDestinations)
+	if !success || err != nil {
+		matcher.failureMessage = HaveKey(matcher.Name).FailureMessage(cfg.Engine.ServiceDestinations)
+		matcher.negatedFailureMessage = HaveKey(matcher.Name).NegatedFailureMessage(cfg.Engine.ServiceDestinations)
+		return
+	}
+
+	sd := cfg.Engine.ServiceDestinations[matcher.Name.(string)]
+	success, err = Equal(matcher.URI).Match(sd.URI)
+	if !success || err != nil {
+		matcher.failureMessage = Equal(matcher.URI).FailureMessage(sd.URI)
+		matcher.negatedFailureMessage = Equal(matcher.URI).NegatedFailureMessage(sd.URI)
+		return
+	}
+
+	success, err = Equal(matcher.Identity).Match(sd.Identity)
+	if !success || err != nil {
+		matcher.failureMessage = Equal(matcher.Identity).FailureMessage(sd.Identity)
+		matcher.negatedFailureMessage = Equal(matcher.Identity).NegatedFailureMessage(sd.Identity)
+		return
+	}
+
+	return true, nil
+}
+
+func (matcher *ServiceMatcher) FailureMessage(_ interface{}) string {
+	return matcher.failureMessage
+}
+
+func (matcher *ServiceMatcher) NegatedFailureMessage(_ interface{}) string {
+	return matcher.negatedFailureMessage
+}
+
+type URLMatcher struct {
+	matchers.EqualMatcher
+}
+
+// VerifyURL matches when actual is a valid URL and matches expected
+func VerifyURL(uri interface{}) OmegaMatcher {
+	return &URLMatcher{matchers.EqualMatcher{Expected: uri}}
+}
+
+func (matcher *URLMatcher) Match(actual interface{}) (bool, error) {
+	e, ok := matcher.Expected.(string)
+	if !ok {
+		return false, fmt.Errorf("VerifyURL requires string inputs %T is not supported", matcher.Expected)
+	}
+	e_uri, err := url.Parse(e)
+	if err != nil {
+		return false, err
+	}
+
+	a, ok := actual.(string)
+	if !ok {
+		return false, fmt.Errorf("VerifyURL requires string inputs %T is not supported", actual)
+	}
+	a_uri, err := url.Parse(a)
+	if err != nil {
+		return false, err
+	}
+
+	return (&matchers.EqualMatcher{Expected: e_uri}).Match(a_uri)
+}
+
+type ExitMatcher struct {
+	types.GomegaMatcher
+	Expected int
+	Actual   int
+}
+
 // ExitWithError matches when assertion is > argument.  Default 0
-// Modeled after the gomega Exit() matcher
-func ExitWithError(optionalExitCode ...int) *exitMatcher {
+// Modeled after the gomega Exit() matcher and also operates on sessions.
+func ExitWithError(optionalExitCode ...int) *ExitMatcher {
 	exitCode := 0
 	if len(optionalExitCode) > 0 {
 		exitCode = optionalExitCode[0]
 	}
-	return &exitMatcher{exitCode: exitCode}
+	return &ExitMatcher{Expected: exitCode}
 }
 
-type exitMatcher struct {
-	exitCode       int
-	actualExitCode int
-}
-
-func (m *exitMatcher) Match(actual interface{}) (success bool, err error) {
+// Match follows gexec.Matcher interface
+func (matcher *ExitMatcher) Match(actual interface{}) (success bool, err error) {
 	exiter, ok := actual.(gexec.Exiter)
 	if !ok {
 		return false, fmt.Errorf("ExitWithError must be passed a gexec.Exiter (Missing method ExitCode() int) Got:\n#{format.Object(actual, 1)}")
 	}
 
-	m.actualExitCode = exiter.ExitCode()
-	if m.actualExitCode == -1 {
+	matcher.Actual = exiter.ExitCode()
+	if matcher.Actual == -1 {
 		return false, nil
 	}
-	return m.actualExitCode > m.exitCode, nil
+	return matcher.Actual > matcher.Expected, nil
 }
 
-func (m *exitMatcher) FailureMessage(actual interface{}) (message string) {
-	if m.actualExitCode == -1 {
+func (matcher *ExitMatcher) FailureMessage(_ interface{}) (message string) {
+	if matcher.Actual == -1 {
 		return "Expected process to exit.  It did not."
 	}
-	return format.Message(m.actualExitCode, "to be greater than exit code:", m.exitCode)
+	return format.Message(matcher.Actual, "to be greater than exit code: ", matcher.Expected)
 }
 
-func (m *exitMatcher) NegatedFailureMessage(actual interface{}) (message string) {
-	if m.actualExitCode == -1 {
+func (matcher *ExitMatcher) NegatedFailureMessage(_ interface{}) (message string) {
+	switch {
+	case matcher.Actual == -1:
 		return "you really shouldn't be able to see this!"
-	} else {
-		if m.exitCode == -1 {
-			return "Expected process not to exit.  It did."
-		}
-		return format.Message(m.actualExitCode, "is less than or equal to exit code:", m.exitCode)
+	case matcher.Expected == -1:
+		return "Expected process not to exit.  It did."
 	}
+	return format.Message(matcher.Actual, "is less than or equal to exit code: ", matcher.Expected)
 }
-func (m *exitMatcher) MatchMayChangeInTheFuture(actual interface{}) bool {
+
+func (matcher *ExitMatcher) MatchMayChangeInTheFuture(actual interface{}) bool {
 	session, ok := actual.(*gexec.Session)
 	if ok {
 		return session.ExitCode() == -1
diff --git a/test/utils/utils.go b/test/utils/utils.go
index d4d5e6e2f..8d1edb23a 100644
--- a/test/utils/utils.go
+++ b/test/utils/utils.go
@@ -397,7 +397,7 @@ func tagOutputToMap(imagesOutput []string) map[string]map[string]bool {
 	return m
 }
 
-// GetHostDistributionInfo returns a struct with its distribution name and version
+// GetHostDistributionInfo returns a struct with its distribution Name and version
 func GetHostDistributionInfo() HostOS {
 	f, err := os.Open(OSReleasePath)
 	defer f.Close()
@@ -491,13 +491,3 @@ func RandomString(n int) string {
 	}
 	return string(b)
 }
-
-//SkipIfInContainer skips a test if the test is run inside a container
-func SkipIfInContainer(reason string) {
-	if len(reason) < 5 {
-		panic("SkipIfInContainer must specify a reason to skip")
-	}
-	if os.Getenv("TEST_ENVIRON") == "container" {
-		Skip("[container]: " + reason)
-	}
-}