Podman-remote run should wait for exit code

This change matches what is happening on the podman local side
and should eliminate a race condition.

Also exit commands on the server side should start to return to client.
Signed-off-by: Daniel J Walsh <dwalsh@redhat.com>

8 files changed, 76 insertions, 29 deletions

diff --git a/libpod/container_api.go b/libpod/container_api.go
index b1c2caa81..4f0d5301c 100644
--- a/libpod/container_api.go
+++ b/libpod/container_api.go
@@ -821,3 +821,12 @@ func (c *Container) Restore(ctx context.Context, options ContainerCheckpointOpti
 	defer c.newContainerEvent(events.Restore)
 	return c.restore(ctx, options)
 }
+
+// AutoRemove indicates whether the container will be removed after it is executed
+func (c *Container) AutoRemove() bool {
+	spec := c.config.Spec
+	if spec.Annotations == nil {
+		return false
+	}
+	return c.Spec().Annotations[InspectAnnotationAutoremove] == InspectResponseTrue
+}
diff --git a/pkg/adapter/containers.go b/pkg/adapter/containers.go
index 2ce437828..47db5c0dc 100644
--- a/pkg/adapter/containers.go
+++ b/pkg/adapter/containers.go
@@ -396,14 +396,7 @@ func (r *LocalRuntime) Run(ctx context.Context, c *cliconfig.RunValues, exitCode
 		// Do not perform cleanup, or wait for container exit code
 		// Just exit immediately
 		if errors.Cause(err) == define.ErrDetach {
-			exitCode = 0
-			return exitCode, nil
-		}
-		// This means the command did not exist
-		exitCode = 127
-		e := strings.ToLower(err.Error())
-		if strings.Contains(e, "permission denied") || strings.Contains(e, "operation not permitted") {
-			exitCode = 126
+			return 0, nil
 		}
 		if c.IsSet("rm") {
 			if deleteError := r.Runtime.RemoveContainer(ctx, ctr, true, false); deleteError != nil {
diff --git a/pkg/adapter/containers_remote.go b/pkg/adapter/containers_remote.go
index b3661ed16..01e008e87 100644
--- a/pkg/adapter/containers_remote.go
+++ b/pkg/adapter/containers_remote.go
@@ -464,19 +464,22 @@ func (r *LocalRuntime) Run(ctx context.Context, c *cliconfig.RunValues, exitCode
 	results := shared.NewIntermediateLayer(&c.PodmanCommand, true)
 	cid, err := iopodman.CreateContainer().Call(r.Conn, results.MakeVarlink())
 	if err != nil {
-		return 0, err
+		return exitCode, err
 	}
 	if c.Bool("detach") {
-		_, err := iopodman.StartContainer().Call(r.Conn, cid)
+		if _, err := iopodman.StartContainer().Call(r.Conn, cid); err != nil {
+			return exitCode, err
+		}
 		fmt.Println(cid)
-		return 0, err
+		return 0, nil
 	}
-	errChan, err := r.attach(ctx, os.Stdin, os.Stdout, cid, true, c.String("detach-keys"))
+	exitChan, errChan, err := r.attach(ctx, os.Stdin, os.Stdout, cid, true, c.String("detach-keys"))
 	if err != nil {
-		return 0, err
+		return exitCode, err
 	}
+	exitCode = <-exitChan
 	finalError := <-errChan
-	return 0, finalError
+	return exitCode, finalError
 }
 
 func ReadExitFile(runtimeTmp, ctrID string) (int, error) {
@@ -572,7 +575,7 @@ func (r *LocalRuntime) Attach(ctx context.Context, c *cliconfig.AttachValues) er
 			return err
 		}
 	}
-	errChan, err := r.attach(ctx, inputStream, os.Stdout, c.InputArgs[0], false, c.DetachKeys)
+	_, errChan, err := r.attach(ctx, inputStream, os.Stdout, c.InputArgs[0], false, c.DetachKeys)
 	if err != nil {
 		return err
 	}
@@ -686,12 +689,13 @@ func (r *LocalRuntime) Start(ctx context.Context, c *cliconfig.StartValues, sigP
 	}
 	// start.go makes sure that if attach, there can be only one ctr
 	if c.Attach {
-		errChan, err := r.attach(ctx, inputStream, os.Stdout, containerIDs[0], true, c.DetachKeys)
+		exitChan, errChan, err := r.attach(ctx, inputStream, os.Stdout, containerIDs[0], true, c.DetachKeys)
 		if err != nil {
 			return exitCode, nil
 		}
+		exitCode := <-exitChan
 		err = <-errChan
-		return 0, err
+		return exitCode, err
 	}
 
 	// TODO the notion of starting a pod container and its deps still needs to be worked through
@@ -710,13 +714,13 @@ func (r *LocalRuntime) Start(ctx context.Context, c *cliconfig.StartValues, sigP
 	return exitCode, finalErr
 }
 
-func (r *LocalRuntime) attach(ctx context.Context, stdin, stdout *os.File, cid string, start bool, detachKeys string) (chan error, error) {
+func (r *LocalRuntime) attach(ctx context.Context, stdin, stdout *os.File, cid string, start bool, detachKeys string) (chan int, chan error, error) {
 	var (
 		oldTermState *term.State
 	)
 	spec, err := r.Spec(cid)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	resize := make(chan remotecommand.TerminalSize, 5)
 	haveTerminal := terminal.IsTerminal(int(os.Stdin.Fd()))
@@ -726,7 +730,7 @@ func (r *LocalRuntime) attach(ctx context.Context, stdin, stdout *os.File, cid s
 	if haveTerminal && spec.Process.Terminal {
 		cancel, oldTermState, err := handleTerminalAttach(ctx, resize)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		defer cancel()
 		defer restoreTerminal(oldTermState)
@@ -738,7 +742,7 @@ func (r *LocalRuntime) attach(ctx context.Context, stdin, stdout *os.File, cid s
 	reply, err := iopodman.Attach().Send(r.Conn, varlink.Upgrade, cid, detachKeys, start)
 	if err != nil {
 		restoreTerminal(oldTermState)
-		return nil, err
+		return nil, nil, err
 	}
 
 	// See if the server accepts the upgraded connection or returns an error
@@ -746,11 +750,12 @@ func (r *LocalRuntime) attach(ctx context.Context, stdin, stdout *os.File, cid s
 
 	if err != nil {
 		restoreTerminal(oldTermState)
-		return nil, err
+		return nil, nil, err
 	}
 
-	errChan := configureVarlinkAttachStdio(r.Conn.Reader, r.Conn.Writer, stdin, stdout, oldTermState, resize, nil)
-	return errChan, nil
+	ecChan := make(chan int, 1)
+	errChan := configureVarlinkAttachStdio(r.Conn.Reader, r.Conn.Writer, stdin, stdout, oldTermState, resize, ecChan)
+	return ecChan, errChan, nil
 }
 
 // PauseContainers pauses container(s) based on CLI inputs.
diff --git a/pkg/util/utils.go b/pkg/util/utils.go
index 2261934f0..583bf5d18 100644
--- a/pkg/util/utils.go
+++ b/pkg/util/utils.go
@@ -377,3 +377,19 @@ func ValidatePullType(pullType string) (PullType, error) {
 		return PullImageMissing, errors.Errorf("invalid pull type %q", pullType)
 	}
 }
+
+// ExitCode reads the error message when failing to executing container process
+// and then returns 0 if no error, 126 if command does not exist, or 127 for
+// all other errors
+func ExitCode(err error) int {
+	if err == nil {
+		return 0
+	}
+	e := strings.ToLower(err.Error())
+	if strings.Contains(e, "file not found") ||
+		strings.Contains(e, "no such file or directory") {
+		return 127
+	}
+
+	return 126
+}
diff --git a/pkg/varlinkapi/attach.go b/pkg/varlinkapi/attach.go
index 1f8d48eb9..3bd487849 100644
--- a/pkg/varlinkapi/attach.go
+++ b/pkg/varlinkapi/attach.go
@@ -9,7 +9,9 @@ import (
 	"github.com/containers/libpod/cmd/podman/varlink"
 	"github.com/containers/libpod/libpod"
 	"github.com/containers/libpod/libpod/define"
+	"github.com/containers/libpod/libpod/events"
 	"github.com/containers/libpod/pkg/varlinkapi/virtwriter"
+	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"k8s.io/client-go/tools/remotecommand"
 )
@@ -79,11 +81,36 @@ func (i *LibpodAPI) Attach(call iopodman.VarlinkCall, name string, detachKeys st
 		finalErr = startAndAttach(ctr, streams, detachKeys, resize, errChan)
 	}
 
+	exitCode := define.ExitCode(finalErr)
 	if finalErr != define.ErrDetach && finalErr != nil {
 		logrus.Error(finalErr)
+	} else {
+		if ecode, err := ctr.Wait(); err != nil {
+			if errors.Cause(err) == define.ErrNoSuchCtr {
+				// Check events
+				event, err := i.Runtime.GetLastContainerEvent(ctr.ID(), events.Exited)
+				if err != nil {
+					logrus.Errorf("Cannot get exit code: %v", err)
+					exitCode = define.ExecErrorCodeNotFound
+				} else {
+					exitCode = event.ContainerExitCode
+				}
+			} else {
+				exitCode = define.ExitCode(err)
+			}
+		} else {
+			exitCode = int(ecode)
+		}
+	}
+
+	if ctr.AutoRemove() {
+		err := i.Runtime.RemoveContainer(getContext(), ctr, false, false)
+		if err != nil {
+			logrus.Errorf("Failed to remove container %s: %s", ctr.ID(), err.Error())
+		}
 	}
 
-	if err = virtwriter.HangUp(writer, 0); err != nil {
+	if err = virtwriter.HangUp(writer, uint32(exitCode)); err != nil {
 		logrus.Errorf("Failed to HANG-UP attach to %s: %s", ctr.ID(), err.Error())
 	}
 	return call.Writer.Flush()
diff --git a/test/e2e/run_exit_test.go b/test/e2e/run_exit_test.go
index fc8b52120..374705879 100644
--- a/test/e2e/run_exit_test.go
+++ b/test/e2e/run_exit_test.go
@@ -1,5 +1,3 @@
-// +build !remoteclient
-
 package integration
 
 import (
diff --git a/test/e2e/run_selinux_test.go b/test/e2e/run_selinux_test.go
index dfe71531a..0c78ab15b 100644
--- a/test/e2e/run_selinux_test.go
+++ b/test/e2e/run_selinux_test.go
@@ -162,7 +162,7 @@ var _ = Describe("Podman run", func() {
 
 		session = podmanTest.Podman([]string{"run", "-it", "--security-opt", "label=type:spc_t", "--security-opt", "label=filetype:foobar", fedoraMinimal, "ls", "-Z", "/dev"})
 		session.WaitWithDefaultTimeout()
-		Expect(session.ExitCode()).To(Equal(127))
+		Expect(session.ExitCode()).To(Equal(126))
 	})
 
 })
diff --git a/test/system/030-run.bats b/test/system/030-run.bats
index f279a0c75..760ed6a18 100644
--- a/test/system/030-run.bats
+++ b/test/system/030-run.bats
@@ -60,7 +60,6 @@ echo $rand        |   0 | $rand
 # 'run --rm' goes through different code paths and may lose exit status.
 # See https://github.com/containers/libpod/issues/3795
 @test "podman run --rm" {
-    skip_if_remote "podman-remote does not handle exit codes"
 
     run_podman 0 run --rm $IMAGE /bin/true
     run_podman 1 run --rm $IMAGE /bin/false