Implement podman-remote rm

* refactor command output to use one function
* Add new worker pool parallel operations
* Implement podman-remote umount
* Refactored podman wait to use printCmdOutput()

Signed-off-by: Jhon Honce <jhonce@redhat.com>

4 files changed, 135 insertions, 63 deletions

diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go
index b20b3b37e..58f94f27e 100644
--- a/test/e2e/common_test.go
+++ b/test/e2e/common_test.go
@@ -3,7 +3,6 @@ package integration
 import (
 	"encoding/json"
 	"fmt"
-	"github.com/containers/libpod/pkg/rootless"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -12,6 +11,7 @@ import (
 	"strings"
 	"testing"
 
+	"github.com/containers/libpod/pkg/rootless"
 	"github.com/containers/storage"
 
 	"github.com/containers/libpod/pkg/inspect"
@@ -86,7 +86,7 @@ func TestLibpod(t *testing.T) {
 }
 
 var _ = SynchronizedBeforeSuite(func() []byte {
-	//Cache images
+	// Cache images
 	cwd, _ := os.Getwd()
 	INTEGRATION_ROOT = filepath.Join(cwd, "../../")
 	podman := PodmanTestCreate("/tmp")
@@ -134,18 +134,18 @@ func (p *PodmanTestIntegration) Setup() {
 	p.ArtifactPath = ARTIFACT_DIR
 }
 
-//var _ = BeforeSuite(func() {
-//	cwd, _ := os.Getwd()
-//	INTEGRATION_ROOT = filepath.Join(cwd, "../../")
-//	podman := PodmanTestCreate("/tmp")
-//	podman.ArtifactPath = ARTIFACT_DIR
-//	if _, err := os.Stat(ARTIFACT_DIR); os.IsNotExist(err) {
-//		if err = os.Mkdir(ARTIFACT_DIR, 0777); err != nil {
-//			fmt.Printf("%q\n", err)
-//			os.Exit(1)
-//		}
-//	}
-//})
+// var _ = BeforeSuite(func() {
+// 	cwd, _ := os.Getwd()
+// 	INTEGRATION_ROOT = filepath.Join(cwd, "../../")
+// 	podman := PodmanTestCreate("/tmp")
+// 	podman.ArtifactPath = ARTIFACT_DIR
+// 	if _, err := os.Stat(ARTIFACT_DIR); os.IsNotExist(err) {
+// 		if err = os.Mkdir(ARTIFACT_DIR, 0777); err != nil {
+// 			fmt.Printf("%q\n", err)
+// 			os.Exit(1)
+// 		}
+// 	}
+// })
 // for _, image := range CACHE_IMAGES {
 // 	if err := podman.CreateArtifact(image); err != nil {
 // 		fmt.Printf("%q\n", err)
@@ -172,7 +172,7 @@ func (p *PodmanTestIntegration) Setup() {
 // 	os.Exit(1)
 // }
 // LockTmpDir = path
-//})
+// })
 
 var _ = AfterSuite(func() {
 	sort.Sort(testResultsSortedLength{testResults})
diff --git a/test/e2e/libpod_suite_test.go b/test/e2e/libpod_suite_test.go
index 685a08340..a69c1ba9a 100644
--- a/test/e2e/libpod_suite_test.go
+++ b/test/e2e/libpod_suite_test.go
@@ -61,9 +61,12 @@ func (p *PodmanTestIntegration) PodmanPID(args []string) (*PodmanSessionIntegrat
 func (p *PodmanTestIntegration) Cleanup() {
 	// Remove all containers
 	stopall := p.Podman([]string{"stop", "-a", "--timeout", "0"})
-	stopall.WaitWithDefaultTimeout()
+	// stopall.WaitWithDefaultTimeout()
+	stopall.Wait(90)
+
 	session := p.Podman([]string{"rm", "-fa"})
 	session.Wait(90)
+
 	// Nuke tempdir
 	if err := os.RemoveAll(p.TempDir); err != nil {
 		fmt.Printf("%q\n", err)
@@ -141,7 +144,7 @@ func (p *PodmanTestIntegration) CreatePod(name string) (*PodmanSessionIntegratio
 	return session, session.ExitCode(), session.OutputToString()
 }
 
-//RunTopContainer runs a simple container in the background that
+// RunTopContainer runs a simple container in the background that
 // runs top.  If the name passed != "", it will have a name
 func (p *PodmanTestIntegration) RunTopContainer(name string) *PodmanSessionIntegration {
 	var podmanArgs = []string{"run"}
@@ -161,7 +164,7 @@ func (p *PodmanTestIntegration) RunTopContainerInPod(name, pod string) *PodmanSe
 	return p.Podman(podmanArgs)
 }
 
-//RunLsContainer runs a simple container in the background that
+// RunLsContainer runs a simple container in the background that
 // simply runs ls. If the name passed != "", it will have a name
 func (p *PodmanTestIntegration) RunLsContainer(name string) (*PodmanSessionIntegration, int, string) {
 	var podmanArgs = []string{"run"}
@@ -215,13 +218,19 @@ func PodmanTestCreate(tempDir string) *PodmanTestIntegration {
 	return PodmanTestCreateUtil(tempDir, false)
 }
 
-//MakeOptions assembles all the podman main options
+// MakeOptions assembles all the podman main options
 func (p *PodmanTestIntegration) makeOptions(args []string) []string {
-	podmanOptions := strings.Split(fmt.Sprintf("--root %s --runroot %s --runtime %s --conmon %s --cni-config-dir %s --cgroup-manager %s --tmpdir %s",
-		p.CrioRoot, p.RunRoot, p.OCIRuntime, p.ConmonBinary, p.CNIConfigDir, p.CgroupManager, p.TmpDir), " ")
+	var debug string
+	if _, ok := os.LookupEnv("DEBUG"); ok {
+		debug = "--log-level=debug --syslog=true "
+	}
+
+	podmanOptions := strings.Split(fmt.Sprintf("%s--root %s --runroot %s --runtime %s --conmon %s --cni-config-dir %s --cgroup-manager %s --tmpdir %s",
+		debug, p.CrioRoot, p.RunRoot, p.OCIRuntime, p.ConmonBinary, p.CNIConfigDir, p.CgroupManager, p.TmpDir), " ")
 	if os.Getenv("HOOK_OPTION") != "" {
 		podmanOptions = append(podmanOptions, os.Getenv("HOOK_OPTION"))
 	}
+
 	podmanOptions = append(podmanOptions, strings.Split(p.StorageOptions, " ")...)
 	podmanOptions = append(podmanOptions, args...)
 	return podmanOptions
diff --git a/test/e2e/search_test.go b/test/e2e/search_test.go
index 589389b3b..61d581c6d 100644
--- a/test/e2e/search_test.go
+++ b/test/e2e/search_test.go
@@ -3,47 +3,79 @@
 package integration
 
 import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
 	"os"
 	"strconv"
+	"text/template"
 
 	. "github.com/containers/libpod/test/utils"
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
 )
 
+type endpoint struct {
+	Host string
+	Port string
+}
+
+func (e *endpoint) Address() string {
+	return fmt.Sprintf("%s:%s", e.Host, e.Port)
+}
+
 var _ = Describe("Podman search", func() {
 	var (
 		tempdir    string
 		err        error
 		podmanTest *PodmanTestIntegration
 	)
+
+	var registryEndpoints = []endpoint{
+		{"localhost", "5001"},
+		{"localhost", "5002"},
+		{"localhost", "5003"},
+		{"localhost", "5004"},
+		{"localhost", "5005"},
+		{"localhost", "5006"},
+		{"localhost", "5007"},
+		{"localhost", "5008"},
+		{"localhost", "5009"},
+	}
+
 	const regFileContents = `
-	[registries.search]
-	registries = ['localhost:5000']
+[registries.search]
+registries = ['{{.Host}}:{{.Port}}']
 
-	[registries.insecure]
-	registries = ['localhost:5000']`
+[registries.insecure]
+registries = ['{{.Host}}:{{.Port}}']`
+	registryFileTmpl := template.Must(template.New("registryFile").Parse(regFileContents))
 
 	const badRegFileContents = `
-	[registries.search]
-	registries = ['localhost:5000']
-    # empty
-	[registries.insecure]
-	registries = []`
+[registries.search]
+registries = ['{{.Host}}:{{.Port}}']
+# empty
+[registries.insecure]
+registries = []`
+	registryFileBadTmpl := template.Must(template.New("registryFileBad").Parse(badRegFileContents))
 
 	const regFileContents2 = `
-	[registries.search]
-	registries = ['localhost:5000', 'localhost:6000']
+[registries.search]
+registries = ['{{.Host}}:{{.Port}}', '{{.Host}}:6000']
+
+[registries.insecure]
+registries = ['{{.Host}}:{{.Port}}']`
+	registryFileTwoTmpl := template.Must(template.New("registryFileTwo").Parse(regFileContents2))
 
-	[registries.insecure]
-	registries = ['localhost:5000']`
 	BeforeEach(func() {
 		tempdir, err = CreateTempDirInTempDir()
 		if err != nil {
 			os.Exit(1)
 		}
+
 		podmanTest = PodmanTestCreate(tempdir)
 		podmanTest.Setup()
+
 		podmanTest.RestoreAllArtifacts()
 	})
 
@@ -51,7 +83,6 @@ var _ = Describe("Podman search", func() {
 		podmanTest.Cleanup()
 		f := CurrentGinkgoTestDescription()
 		processTestResult(f)
-
 	})
 
 	It("podman search", func() {
@@ -134,11 +165,13 @@ var _ = Describe("Podman search", func() {
 		if podmanTest.Host.Arch == "ppc64le" {
 			Skip("No registry image for ppc64le")
 		}
-		lock := GetPortLock("5000")
+		lock := GetPortLock(registryEndpoints[0].Port)
 		defer lock.Unlock()
 
 		podmanTest.RestoreArtifact(registry)
-		fakereg := podmanTest.Podman([]string{"run", "-d", "--name", "registry", "-p", "5000:5000", registry, "/entrypoint.sh", "/etc/docker/registry/config.yml"})
+		fakereg := podmanTest.Podman([]string{"run", "-d", "--name", "registry",
+			"-p", fmt.Sprintf("%s:5000", registryEndpoints[0].Port),
+			registry, "/entrypoint.sh", "/etc/docker/registry/config.yml"})
 		fakereg.WaitWithDefaultTimeout()
 		Expect(fakereg.ExitCode()).To(Equal(0))
 
@@ -146,7 +179,8 @@ var _ = Describe("Podman search", func() {
 			Skip("Can not start docker registry.")
 		}
 
-		search := podmanTest.Podman([]string{"search", "localhost:5000/fake/image:andtag", "--tls-verify=false"})
+		search := podmanTest.Podman([]string{"search",
+			fmt.Sprintf("%s/fake/image:andtag", registryEndpoints[0].Address()), "--tls-verify=false"})
 		search.WaitWithDefaultTimeout()
 
 		// if this test succeeded, there will be no output (there is no entry named fake/image:andtag in an empty registry)
@@ -160,10 +194,12 @@ var _ = Describe("Podman search", func() {
 		if podmanTest.Host.Arch == "ppc64le" {
 			Skip("No registry image for ppc64le")
 		}
-		lock := GetPortLock("5000")
+		lock := GetPortLock(registryEndpoints[3].Port)
 		defer lock.Unlock()
 		podmanTest.RestoreArtifact(registry)
-		registry := podmanTest.Podman([]string{"run", "-d", "--name", "registry3", "-p", "5000:5000", registry, "/entrypoint.sh", "/etc/docker/registry/config.yml"})
+		registry := podmanTest.Podman([]string{"run", "-d", "--name", "registry3",
+			"-p", fmt.Sprintf("%s:5000", registryEndpoints[3].Port), registry,
+			"/entrypoint.sh", "/etc/docker/registry/config.yml"})
 		registry.WaitWithDefaultTimeout()
 		Expect(registry.ExitCode()).To(Equal(0))
 
@@ -171,10 +207,11 @@ var _ = Describe("Podman search", func() {
 			Skip("Can not start docker registry.")
 		}
 
-		push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, "localhost:5000/my-alpine"})
+		image := fmt.Sprintf("%s/my-alpine", registryEndpoints[3].Address())
+		push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, image})
 		push.WaitWithDefaultTimeout()
 		Expect(push.ExitCode()).To(Equal(0))
-		search := podmanTest.Podman([]string{"search", "localhost:5000/my-alpine", "--tls-verify=false"})
+		search := podmanTest.Podman([]string{"search", image, "--tls-verify=false"})
 		search.WaitWithDefaultTimeout()
 
 		Expect(search.ExitCode()).To(Equal(0))
@@ -185,10 +222,12 @@ var _ = Describe("Podman search", func() {
 		if podmanTest.Host.Arch == "ppc64le" {
 			Skip("No registry image for ppc64le")
 		}
-		lock := GetPortLock("5000")
+
+		lock := GetPortLock(registryEndpoints[4].Port)
 		defer lock.Unlock()
 		podmanTest.RestoreArtifact(registry)
-		registry := podmanTest.Podman([]string{"run", "-d", "--name", "registry4", "-p", "5000:5000", registry, "/entrypoint.sh", "/etc/docker/registry/config.yml"})
+		registry := podmanTest.Podman([]string{"run", "-d", "-p", fmt.Sprintf("%s:5000", registryEndpoints[4].Port),
+			"--name", "registry4", registry, "/entrypoint.sh", "/etc/docker/registry/config.yml"})
 		registry.WaitWithDefaultTimeout()
 		Expect(registry.ExitCode()).To(Equal(0))
 
@@ -196,14 +235,18 @@ var _ = Describe("Podman search", func() {
 			Skip("Can not start docker registry.")
 		}
 
-		push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, "localhost:5000/my-alpine"})
+		image := fmt.Sprintf("%s/my-alpine", registryEndpoints[4].Address())
+		push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, image})
 		push.WaitWithDefaultTimeout()
 		Expect(push.ExitCode()).To(Equal(0))
 
 		// registries.conf set up
-		podmanTest.setRegistriesConfigEnv([]byte(regFileContents))
+		var buffer bytes.Buffer
+		registryFileTmpl.Execute(&buffer, registryEndpoints[4])
+		podmanTest.setRegistriesConfigEnv(buffer.Bytes())
+		ioutil.WriteFile(fmt.Sprintf("%s/registry4.conf", tempdir), buffer.Bytes(), 0644)
 
-		search := podmanTest.Podman([]string{"search", "localhost:5000/my-alpine"})
+		search := podmanTest.Podman([]string{"search", image})
 		search.WaitWithDefaultTimeout()
 
 		Expect(search.ExitCode()).To(Equal(0))
@@ -219,24 +262,29 @@ var _ = Describe("Podman search", func() {
 		if podmanTest.Host.Arch == "ppc64le" {
 			Skip("No registry image for ppc64le")
 		}
-		lock := GetPortLock("5000")
+		lock := GetPortLock(registryEndpoints[5].Port)
 		defer lock.Unlock()
 		podmanTest.RestoreArtifact(registry)
-		registry := podmanTest.Podman([]string{"run", "-d", "-p", "5000:5000", "--name", "registry5", registry})
+		registry := podmanTest.Podman([]string{"run", "-d", "-p", fmt.Sprintf("%s:5000", registryEndpoints[5].Port),
+			"--name", "registry5", registry})
 		registry.WaitWithDefaultTimeout()
 		Expect(registry.ExitCode()).To(Equal(0))
 
 		if !WaitContainerReady(podmanTest, "registry5", "listening on", 20, 1) {
 			Skip("Can not start docker registry.")
 		}
-		push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, "localhost:5000/my-alpine"})
+
+		image := fmt.Sprintf("%s/my-alpine", registryEndpoints[5].Address())
+		push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, image})
 		push.WaitWithDefaultTimeout()
 		Expect(push.ExitCode()).To(Equal(0))
 
-		// registries.conf set up
-		podmanTest.setRegistriesConfigEnv([]byte(regFileContents))
+		var buffer bytes.Buffer
+		registryFileTmpl.Execute(&buffer, registryEndpoints[5])
+		podmanTest.setRegistriesConfigEnv(buffer.Bytes())
+		ioutil.WriteFile(fmt.Sprintf("%s/registry5.conf", tempdir), buffer.Bytes(), 0644)
 
-		search := podmanTest.Podman([]string{"search", "localhost:5000/my-alpine", "--tls-verify=true"})
+		search := podmanTest.Podman([]string{"search", image, "--tls-verify=true"})
 		search.WaitWithDefaultTimeout()
 
 		Expect(search.ExitCode()).To(Equal(0))
@@ -252,24 +300,29 @@ var _ = Describe("Podman search", func() {
 		if podmanTest.Host.Arch == "ppc64le" {
 			Skip("No registry image for ppc64le")
 		}
-		lock := GetPortLock("5000")
+		lock := GetPortLock(registryEndpoints[6].Port)
 		defer lock.Unlock()
 		podmanTest.RestoreArtifact(registry)
-		registry := podmanTest.Podman([]string{"run", "-d", "-p", "5000:5000", "--name", "registry6", registry})
+		registry := podmanTest.Podman([]string{"run", "-d", "-p", fmt.Sprintf("%s:5000", registryEndpoints[6].Port),
+			"--name", "registry6", registry})
 		registry.WaitWithDefaultTimeout()
 		Expect(registry.ExitCode()).To(Equal(0))
 
 		if !WaitContainerReady(podmanTest, "registry6", "listening on", 20, 1) {
 			Skip("Can not start docker registry.")
 		}
-		push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, "localhost:5000/my-alpine"})
+
+		image := fmt.Sprintf("%s/my-alpine", registryEndpoints[6].Address())
+		push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, image})
 		push.WaitWithDefaultTimeout()
 		Expect(push.ExitCode()).To(Equal(0))
 
-		// registries.conf set up
-		podmanTest.setRegistriesConfigEnv([]byte(badRegFileContents))
+		var buffer bytes.Buffer
+		registryFileBadTmpl.Execute(&buffer, registryEndpoints[6])
+		podmanTest.setRegistriesConfigEnv(buffer.Bytes())
+		ioutil.WriteFile(fmt.Sprintf("%s/registry6.conf", tempdir), buffer.Bytes(), 0644)
 
-		search := podmanTest.Podman([]string{"search", "localhost:5000/my-alpine"})
+		search := podmanTest.Podman([]string{"search", image})
 		search.WaitWithDefaultTimeout()
 
 		Expect(search.ExitCode()).To(Equal(0))
@@ -285,10 +338,14 @@ var _ = Describe("Podman search", func() {
 		if podmanTest.Host.Arch == "ppc64le" {
 			Skip("No registry image for ppc64le")
 		}
-		lock := GetPortLock("5000")
-		defer lock.Unlock()
+		lock7 := GetPortLock(registryEndpoints[7].Port)
+		defer lock7.Unlock()
+		lock8 := GetPortLock("6000")
+		defer lock8.Unlock()
+
 		podmanTest.RestoreArtifact(registry)
-		registryLocal := podmanTest.Podman([]string{"run", "-d", "-p", "5000:5000", "--name", "registry7", registry})
+		registryLocal := podmanTest.Podman([]string{"run", "-d", "-p", fmt.Sprintf("%s:5000", registryEndpoints[7].Port),
+			"--name", "registry7", registry})
 		registryLocal.WaitWithDefaultTimeout()
 		Expect(registryLocal.ExitCode()).To(Equal(0))
 
@@ -303,12 +360,16 @@ var _ = Describe("Podman search", func() {
 		if !WaitContainerReady(podmanTest, "registry8", "listening on", 20, 1) {
 			Skip("Can not start docker registry.")
 		}
+
 		push := podmanTest.Podman([]string{"push", "--tls-verify=false", "--remove-signatures", ALPINE, "localhost:6000/my-alpine"})
 		push.WaitWithDefaultTimeout()
 		Expect(push.ExitCode()).To(Equal(0))
 
 		// registries.conf set up
-		podmanTest.setRegistriesConfigEnv([]byte(regFileContents2))
+		var buffer bytes.Buffer
+		registryFileTwoTmpl.Execute(&buffer, registryEndpoints[8])
+		podmanTest.setRegistriesConfigEnv(buffer.Bytes())
+		ioutil.WriteFile(fmt.Sprintf("%s/registry8.conf", tempdir), buffer.Bytes(), 0644)
 
 		search := podmanTest.Podman([]string{"search", "my-alpine"})
 		search.WaitWithDefaultTimeout()
diff --git a/test/utils/utils.go b/test/utils/utils.go
index 499466f5a..6308197b8 100644
--- a/test/utils/utils.go
+++ b/test/utils/utils.go
@@ -311,6 +311,8 @@ func (s *PodmanSession) IsJSONOutputValid() bool {
 // WaitWithDefaultTimeout waits for process finished with defaultWaitTimeout
 func (s *PodmanSession) WaitWithDefaultTimeout() {
 	s.Wait(defaultWaitTimeout)
+	os.Stdout.Sync()
+	os.Stderr.Sync()
 	fmt.Println("output:", s.OutputToString())
 }