podman image tree: restore previous behavior

The initial version of libimage changed the order of layers which has
now been restored to remain backwards compatible.

Further changes:

 * Fix a bug in the journald logging which requires to strip trailing
   new lines from the message.  The system tests did not pass due to
   empty new lines.  Triggered by changing the default logger to
   journald in containers/common.

 * Fix another bug in the journald logging which embedded the container
   ID inside the message rather than the specifid field.  That surfaced
   in a preceeding whitespace of each log line which broke the system
   tests.

 * Alter the system tests to make sure that the k8s-file and the
   journald logging drivers are executed.

 * A number of e2e tests have been changed to force the k8s-file driver
   to make them pass when running inside a root container.

 * Increase the timeout in a kill test which seems to take longer now.
   Reasons are unknown.  Tests passed earlier and no signal-related
   changes happend.  It may be CI VM flake since some system tests but
   other flaked.

Signed-off-by: Valentin Rothberg <rothberg@redhat.com>

8 files changed, 45 insertions, 30 deletions

diff --git a/test/e2e/create_staticip_test.go b/test/e2e/create_staticip_test.go
index 340ea31f3..2cf552274 100644
--- a/test/e2e/create_staticip_test.go
+++ b/test/e2e/create_staticip_test.go
@@ -60,8 +60,10 @@ var _ = Describe("Podman create with --ip flag", func() {
 	})
 
 	It("Podman create with specified static IP has correct IP", func() {
+		// NOTE: we force the k8s-file log driver to make sure the
+		// tests are passing inside a container.
 		ip := GetRandomIPAddress()
-		result := podmanTest.Podman([]string{"create", "--name", "test", "--ip", ip, ALPINE, "ip", "addr"})
+		result := podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test", "--ip", ip, ALPINE, "ip", "addr"})
 		result.WaitWithDefaultTimeout()
 		// Rootless static ip assignment without network should error
 		if rootless.IsRootless() {
@@ -83,10 +85,10 @@ var _ = Describe("Podman create with --ip flag", func() {
 	It("Podman create two containers with the same IP", func() {
 		SkipIfRootless("--ip not supported without network in rootless mode")
 		ip := GetRandomIPAddress()
-		result := podmanTest.Podman([]string{"create", "--name", "test1", "--ip", ip, ALPINE, "sleep", "999"})
+		result := podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test1", "--ip", ip, ALPINE, "sleep", "999"})
 		result.WaitWithDefaultTimeout()
 		Expect(result.ExitCode()).To(Equal(0))
-		result = podmanTest.Podman([]string{"create", "--name", "test2", "--ip", ip, ALPINE, "ip", "addr"})
+		result = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test2", "--ip", ip, ALPINE, "ip", "addr"})
 		result.WaitWithDefaultTimeout()
 		Expect(result.ExitCode()).To(Equal(0))
 		result = podmanTest.Podman([]string{"start", "test1"})
diff --git a/test/e2e/create_test.go b/test/e2e/create_test.go
index 1f1786dbe..e4db6b845 100644
--- a/test/e2e/create_test.go
+++ b/test/e2e/create_test.go
@@ -160,9 +160,12 @@ var _ = Describe("Podman create", func() {
 		if podmanTest.Host.Arch == "ppc64le" {
 			Skip("skip failing test on ppc64le")
 		}
+		// NOTE: we force the k8s-file log driver to make sure the
+		// tests are passing inside a container.
+
 		mountPath := filepath.Join(podmanTest.TempDir, "secrets")
 		os.Mkdir(mountPath, 0755)
-		session := podmanTest.Podman([]string{"create", "--name", "test", "--mount", fmt.Sprintf("type=bind,src=%s,target=/create/test", mountPath), ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
+		session := podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test", "--mount", fmt.Sprintf("type=bind,src=%s,target=/create/test", mountPath), ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 		session = podmanTest.Podman([]string{"start", "test"})
@@ -173,7 +176,7 @@ var _ = Describe("Podman create", func() {
 		Expect(session.ExitCode()).To(Equal(0))
 		Expect(session.OutputToString()).To(ContainSubstring("/create/test rw"))
 
-		session = podmanTest.Podman([]string{"create", "--name", "test_ro", "--mount", fmt.Sprintf("type=bind,src=%s,target=/create/test,ro", mountPath), ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
+		session = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test_ro", "--mount", fmt.Sprintf("type=bind,src=%s,target=/create/test,ro", mountPath), ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 		session = podmanTest.Podman([]string{"start", "test_ro"})
@@ -184,7 +187,7 @@ var _ = Describe("Podman create", func() {
 		Expect(session.ExitCode()).To(Equal(0))
 		Expect(session.OutputToString()).To(ContainSubstring("/create/test ro"))
 
-		session = podmanTest.Podman([]string{"create", "--name", "test_shared", "--mount", fmt.Sprintf("type=bind,src=%s,target=/create/test,shared", mountPath), ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
+		session = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test_shared", "--mount", fmt.Sprintf("type=bind,src=%s,target=/create/test,shared", mountPath), ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 		session = podmanTest.Podman([]string{"start", "test_shared"})
@@ -200,7 +203,7 @@ var _ = Describe("Podman create", func() {
 
 		mountPath = filepath.Join(podmanTest.TempDir, "scratchpad")
 		os.Mkdir(mountPath, 0755)
-		session = podmanTest.Podman([]string{"create", "--name", "test_tmpfs", "--mount", "type=tmpfs,target=/create/test", ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
+		session = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test_tmpfs", "--mount", "type=tmpfs,target=/create/test", ALPINE, "grep", "/create/test", "/proc/self/mountinfo"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 		session = podmanTest.Podman([]string{"start", "test_tmpfs"})
diff --git a/test/e2e/run_networking_test.go b/test/e2e/run_networking_test.go
index 4c66e2823..37e837b1d 100644
--- a/test/e2e/run_networking_test.go
+++ b/test/e2e/run_networking_test.go
@@ -649,11 +649,13 @@ var _ = Describe("Podman run networking", func() {
 		defer podmanTest.removeCNINetwork(netName)
 
 		name := "nc-server"
-		run := podmanTest.Podman([]string{"run", "-d", "--name", name, "--net", netName, ALPINE, "nc", "-l", "-p", "8080"})
+		run := podmanTest.Podman([]string{"run", "--log-driver", "k8s-file", "-d", "--name", name, "--net", netName, ALPINE, "nc", "-l", "-p", "8080"})
 		run.WaitWithDefaultTimeout()
 		Expect(run.ExitCode()).To(Equal(0))
 
-		run = podmanTest.Podman([]string{"run", "--rm", "--net", netName, "--uidmap", "0:1:4096", ALPINE, "sh", "-c", fmt.Sprintf("echo podman | nc -w 1 %s.dns.podman 8080", name)})
+		// NOTE: we force the k8s-file log driver to make sure the
+		// tests are passing inside a container.
+		run = podmanTest.Podman([]string{"run", "--log-driver", "k8s-file", "--rm", "--net", netName, "--uidmap", "0:1:4096", ALPINE, "sh", "-c", fmt.Sprintf("echo podman | nc -w 1 %s.dns.podman 8080", name)})
 		run.WaitWithDefaultTimeout()
 		Expect(run.ExitCode()).To(Equal(0))
 
diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go
index 59220cf01..f27ded5d2 100644
--- a/test/e2e/run_test.go
+++ b/test/e2e/run_test.go
@@ -712,7 +712,7 @@ USER bin`, BB)
 
 	It("podman run log-opt", func() {
 		log := filepath.Join(podmanTest.TempDir, "/container.log")
-		session := podmanTest.Podman([]string{"run", "--rm", "--log-opt", fmt.Sprintf("path=%s", log), ALPINE, "ls"})
+		session := podmanTest.Podman([]string{"run", "--rm", "--log-driver", "k8s-file", "--log-opt", fmt.Sprintf("path=%s", log), ALPINE, "ls"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 		_, err := os.Stat(log)
diff --git a/test/e2e/toolbox_test.go b/test/e2e/toolbox_test.go
index 986f856bf..16300bebc 100644
--- a/test/e2e/toolbox_test.go
+++ b/test/e2e/toolbox_test.go
@@ -215,7 +215,7 @@ var _ = Describe("Toolbox-specific testing", func() {
 		useradd := fmt.Sprintf("useradd --home-dir %s --shell %s --uid %s %s",
 			homeDir, shell, uid, username)
 		passwd := fmt.Sprintf("passwd --delete %s", username)
-		session = podmanTest.Podman([]string{"create", "--name", "test", "--userns=keep-id", "--user", "root:root", fedoraToolbox, "sh", "-c",
+		session = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test", "--userns=keep-id", "--user", "root:root", fedoraToolbox, "sh", "-c",
 			fmt.Sprintf("%s; %s; echo READY; sleep 1000", useradd, passwd)})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
@@ -250,7 +250,7 @@ var _ = Describe("Toolbox-specific testing", func() {
 
 		groupadd := fmt.Sprintf("groupadd --gid %s %s", gid, groupName)
 
-		session = podmanTest.Podman([]string{"create", "--name", "test", "--userns=keep-id", "--user", "root:root", fedoraToolbox, "sh", "-c",
+		session = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test", "--userns=keep-id", "--user", "root:root", fedoraToolbox, "sh", "-c",
 			fmt.Sprintf("%s; echo READY; sleep 1000", groupadd)})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
@@ -294,7 +294,7 @@ var _ = Describe("Toolbox-specific testing", func() {
 		usermod := fmt.Sprintf("usermod --append --groups wheel --home %s --shell %s --uid %s --gid %s %s",
 			homeDir, shell, uid, gid, username)
 
-		session = podmanTest.Podman([]string{"create", "--name", "test", "--userns=keep-id", "--user", "root:root", fedoraToolbox, "sh", "-c",
+		session = podmanTest.Podman([]string{"create", "--log-driver", "k8s-file", "--name", "test", "--userns=keep-id", "--user", "root:root", fedoraToolbox, "sh", "-c",
 			fmt.Sprintf("%s; %s; %s; echo READY; sleep 1000", useradd, groupadd, usermod)})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
@@ -339,6 +339,7 @@ var _ = Describe("Toolbox-specific testing", func() {
 		// These should be most of the switches that Toolbox uses to create a "toolbox" container
 		// https://github.com/containers/toolbox/blob/master/src/cmd/create.go
 		session = podmanTest.Podman([]string{"create",
+			"--log-driver", "k8s-file",
 			"--dns", "none",
 			"--hostname", "toolbox",
 			"--ipc", "host",
diff --git a/test/system/035-logs.bats b/test/system/035-logs.bats
index bac153b8e..3dd88e5eb 100644
--- a/test/system/035-logs.bats
+++ b/test/system/035-logs.bats
@@ -27,13 +27,22 @@ load helpers
     run_podman rm $cid
 }
 
-@test "podman logs - multi" {
+function _log_test_multi() {
+    local driver=$1
+
     skip_if_remote "logs does not support multiple containers when run remotely"
 
+    # Under k8s file, 'podman logs' returns just the facts, Ma'am.
+    # Under journald, there may be other cruft (e.g. container removals)
+    local etc=
+    if [[ $driver =~ journal ]]; then
+        etc='.*'
+    fi
+
     # Simple helper to make the container starts, below, easier to read
     local -a cid
     doit() {
-        run_podman run --rm -d --name "$1" $IMAGE sh -c "$2";
+        run_podman run --log-driver=$driver --rm -d --name "$1" $IMAGE sh -c "$2";
         cid+=($(echo "${output:0:12}"))
     }
 
@@ -47,24 +56,21 @@ load helpers
 
     run_podman logs -f c1 c2
     is "$output" \
-       "${cid[0]} a
-${cid[1]} b
-${cid[1]} c
+       "${cid[0]} a$etc
+${cid[1]} b$etc
+${cid[1]} c$etc
 ${cid[0]} d"   "Sequential output from logs"
 }
 
-@test "podman logs over journald" {
+@test "podman logs - multi k8s-file" {
+    _log_test_multi k8s-file
+}
+
+@test "podman logs - multi journald" {
     # We can't use journald on RHEL as rootless: rhbz#1895105
     skip_if_journald_unavailable
 
-    msg=$(random_string 20)
-
-    run_podman run --name myctr --log-driver journald $IMAGE echo $msg
-
-    run_podman logs myctr
-    is "$output" "$msg" "check that log output equals the container output"
-
-    run_podman rm myctr
+    _log_test_multi journald
 }
 
 # vim: filetype=sh
diff --git a/test/system/070-build.bats b/test/system/070-build.bats
index a2c8ae588..d2d56c051 100644
--- a/test/system/070-build.bats
+++ b/test/system/070-build.bats
@@ -393,9 +393,9 @@ Labels.$label_name | $label_value
        "image tree: third line"
     is "${lines[3]}" "Image Layers" \
        "image tree: fourth line"
-    is "${lines[4]}"  ".* ID: [0-9a-f]\{12\} Size: .* Top Layer of: \[localhost/build_test:latest]" \
+    is "${lines[4]}" ".* ID: [0-9a-f]\{12\} Size: .* Top Layer of: \[$IMAGE]" \
        "image tree: first layer line"
-    is "${lines[-1]}" ".* ID: [0-9a-f]\{12\} Size: .* Top Layer of: \[$IMAGE]" \
+    is "${lines[-1]}"  ".* ID: [0-9a-f]\{12\} Size: .* Top Layer of: \[localhost/build_test:latest]" \
        "image tree: last layer line"
 
     # FIXME: 'image tree --whatrequires' does not work via remote
diff --git a/test/system/130-kill.bats b/test/system/130-kill.bats
index 3770eac27..1b02b4976 100644
--- a/test/system/130-kill.bats
+++ b/test/system/130-kill.bats
@@ -8,7 +8,8 @@ load helpers
 @test "podman kill - test signal handling in containers" {
     # Start a container that will handle all signals by emitting 'got: N'
     local -a signals=(1 2 3 4 5 6 8 10 12 13 14 15 16 20 21 22 23 24 25 26 64)
-    run_podman run -d $IMAGE sh -c \
+    # Force the k8s-file driver until #10323 is fixed.
+    run_podman run --log-driver=k8s-file -d $IMAGE sh -c \
         "for i in ${signals[*]}; do trap \"echo got: \$i\" \$i; done;
         echo READY;
         while ! test -e /stop; do sleep 0.05; done;