Merge pull request #3143 from haircommander/conmon-exec

use conmon for exec

33 files changed, 1415 insertions, 775 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 75b6fba8a..6302cf388 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -31,9 +31,9 @@ env:
     ####
     #### Cache-image names to test with
     ###
-    FEDORA_CACHE_IMAGE_NAME: "fedora-30-libpod-5925244995371008"
-    PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5925244995371008"
-    UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5925244995371008"
+    FEDORA_CACHE_IMAGE_NAME: "fedora-30-libpod-5479475851034624"
+    PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5479475851034624"
+    UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5479475851034624"
 
     ####
     #### Variables for composing new cache-images (used in PR testing) from
diff --git a/API.md b/API.md
index ec8512fd4..31bffeb16 100755
--- a/API.md
+++ b/API.md
@@ -41,6 +41,8 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in
 
 [func Diff(name: string) DiffInfo](#Diff)
 
+[func ExecContainer(opts: ExecOpts) ](#ExecContainer)
+
 [func ExportContainer(name: string, path: string) string](#ExportContainer)
 
 [func ExportImage(name: string, destination: string, compress: bool, tags: []string) string](#ExportImage)
@@ -203,6 +205,8 @@ in the [API.md](https://github.com/containers/libpod/blob/master/API.md) file in
 
 [type Event](#Event)
 
+[type ExecOpts](#ExecOpts)
+
 [type Image](#Image)
 
 [type ImageHistory](#ImageHistory)
@@ -439,6 +443,11 @@ $ varlink call -m unix:/run/podman/io.podman/io.podman.DeleteUnusedImages
 
 method Diff(name: [string](https://godoc.org/builtin#string)) [DiffInfo](#DiffInfo)</div>
 Diff returns a diff between libpod objects
+### <a name="ExecContainer"></a>func ExecContainer
+<div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;">
+
+method ExecContainer(opts: [ExecOpts](#ExecOpts)) </div>
+ExecContainer executes a command in the given container.
 ### <a name="ExportContainer"></a>func ExportContainer
 <div style="background-color: #E8E8E8; padding: 15px; margin: 10px; border-radius: 10px;">
 
@@ -1565,6 +1574,23 @@ status [string](https://godoc.org/builtin#string)
 time [string](https://godoc.org/builtin#string)
 
 type [string](https://godoc.org/builtin#string)
+### <a name="ExecOpts"></a>type ExecOpts
+
+
+
+name [string](https://godoc.org/builtin#string)
+
+tty [bool](https://godoc.org/builtin#bool)
+
+privileged [bool](https://godoc.org/builtin#bool)
+
+cmd [[]string](#[]string)
+
+user [?string](#?string)
+
+workdir [?string](#?string)
+
+env [?[]string](#?[]string)
 ### <a name="Image"></a>type Image
 
 
diff --git a/cmd/podman/cliconfig/config.go b/cmd/podman/cliconfig/config.go
index be380cda0..025f40cf6 100644
--- a/cmd/podman/cliconfig/config.go
+++ b/cmd/podman/cliconfig/config.go
@@ -119,14 +119,15 @@ type DiffValues struct {
 
 type ExecValues struct {
 	PodmanCommand
-	Env          []string
-	Privileged   bool
-	Interfactive bool
-	Tty          bool
-	User         string
-	Latest       bool
-	Workdir      string
-	PreserveFDs  int
+	DetachKeys  string
+	Env         []string
+	Privileged  bool
+	Interactive bool
+	Tty         bool
+	User        string
+	Latest      bool
+	Workdir     string
+	PreserveFDs int
 }
 
 type ImageExistsValues struct {
diff --git a/cmd/podman/commands.go b/cmd/podman/commands.go
index 9229191ff..27f3fc214 100644
--- a/cmd/podman/commands.go
+++ b/cmd/podman/commands.go
@@ -11,7 +11,6 @@ const remoteclient = false
 // Commands that the local client implements
 func getMainCommands() []*cobra.Command {
 	rootCommands := []*cobra.Command{
-		_execCommand,
 		_playCommand,
 		_loginCommand,
 		_logoutCommand,
@@ -41,7 +40,6 @@ func getContainerSubCommands() []*cobra.Command {
 
 	return []*cobra.Command{
 		_cleanupCommand,
-		_execCommand,
 		_mountCommand,
 		_refreshCommand,
 		_runlabelCommand,
diff --git a/cmd/podman/container.go b/cmd/podman/container.go
index cb54317c0..557f5fafa 100644
--- a/cmd/podman/container.go
+++ b/cmd/podman/container.go
@@ -57,6 +57,7 @@ var (
 		_contInspectSubCommand,
 		_cpCommand,
 		_diffCommand,
+		_execCommand,
 		_exportCommand,
 		_createCommand,
 		_initCommand,
diff --git a/cmd/podman/exec.go b/cmd/podman/exec.go
index 799ed9f38..2e9b9e47e 100644
--- a/cmd/podman/exec.go
+++ b/cmd/podman/exec.go
@@ -35,8 +35,9 @@ func init() {
 	execCommand.SetUsageTemplate(UsageTemplate())
 	flags := execCommand.Flags()
 	flags.SetInterspersed(false)
+	flags.StringVar(&execCommand.DetachKeys, "detach-keys", "", "Override the key sequence for detaching a container. Format is a single character [a-Z] or ctrl-<value> where <value> is one of: a-z, @, ^, [, , or _")
 	flags.StringArrayVarP(&execCommand.Env, "env", "e", []string{}, "Set environment variables")
-	flags.BoolVarP(&execCommand.Interfactive, "interactive", "i", false, "Not supported.  All exec commands are interactive by default")
+	flags.BoolVarP(&execCommand.Interactive, "interactive", "i", false, "Keep STDIN open even if not attached")
 	flags.BoolVarP(&execCommand.Latest, "latest", "l", false, "Act on the latest container podman is aware of")
 	flags.BoolVar(&execCommand.Privileged, "privileged", false, "Give the process extended Linux capabilities inside the container.  The default is false")
 	flags.BoolVarP(&execCommand.Tty, "tty", "t", false, "Allocate a pseudo-TTY. The default is false")
@@ -45,30 +46,35 @@ func init() {
 	flags.IntVar(&execCommand.PreserveFDs, "preserve-fds", 0, "Pass N additional file descriptors to the container")
 	flags.StringVarP(&execCommand.Workdir, "workdir", "w", "", "Working directory inside the container")
 	markFlagHiddenForRemoteClient("latest", flags)
+	markFlagHiddenForRemoteClient("preserve-fds", flags)
 }
 
 func execCmd(c *cliconfig.ExecValues) error {
-	args := c.InputArgs
-	argStart := 1
-	if len(args) < 1 && !c.Latest {
-		return errors.Errorf("you must provide one container name or id")
-	}
-	if len(args) < 2 && !c.Latest {
-		return errors.Errorf("you must provide a command to exec")
-	}
+	argLen := len(c.InputArgs)
 	if c.Latest {
-		argStart = 0
+		if argLen < 1 {
+			return errors.Errorf("you must provide a command to exec")
+		}
+	} else {
+		if argLen < 1 {
+			return errors.Errorf("you must provide one container name or id")
+		}
+		if argLen < 2 {
+			return errors.Errorf("you must provide a command to exec")
+		}
 	}
-	cmd := args[argStart:]
 	runtime, err := adapter.GetRuntimeNoStore(getContext(), &c.PodmanCommand)
 	if err != nil {
 		return errors.Wrapf(err, "error creating libpod runtime")
 	}
 	defer runtime.DeferredShutdown(false)
 
-	err = runtime.Exec(c, cmd)
-	if errors.Cause(err) == define.ErrCtrStateInvalid {
+	exitCode, err = runtime.ExecContainer(getContext(), c)
+	if errors.Cause(err) == define.ErrOCIRuntimePermissionDenied {
 		exitCode = 126
 	}
+	if errors.Cause(err) == define.ErrOCIRuntimeNotFound {
+		exitCode = 127
+	}
 	return err
 }
diff --git a/cmd/podman/healthcheck_run.go b/cmd/podman/healthcheck_run.go
index 66ca4580a..3a2a8f333 100644
--- a/cmd/podman/healthcheck_run.go
+++ b/cmd/podman/healthcheck_run.go
@@ -44,6 +44,9 @@ func healthCheckCmd(c *cliconfig.HealthCheckValues) error {
 	}
 	defer runtime.DeferredShutdown(false)
 	status, err := runtime.HealthCheck(c)
+	if err == nil && status == "unhealthy" {
+		exitCode = 1
+	}
 	fmt.Println(status)
 	return err
 }
diff --git a/cmd/podman/main.go b/cmd/podman/main.go
index a8478bda6..72d1754ac 100644
--- a/cmd/podman/main.go
+++ b/cmd/podman/main.go
@@ -35,6 +35,7 @@ var mainCommands = []*cobra.Command{
 	_diffCommand,
 	_createCommand,
 	_eventsCommand,
+	_execCommand,
 	_exportCommand,
 	_generateCommand,
 	_historyCommand,
diff --git a/cmd/podman/varlink/io.podman.varlink b/cmd/podman/varlink/io.podman.varlink
index 72b15c328..cc66e7df0 100644
--- a/cmd/podman/varlink/io.podman.varlink
+++ b/cmd/podman/varlink/io.podman.varlink
@@ -500,6 +500,23 @@ type DiffInfo(
     changeType: string
 )
 
+type ExecOpts(
+    # container name or id
+    name: string,
+    # Create pseudo tty
+    tty: bool,
+    # privileged access in container
+    privileged: bool,
+    # command to execute in container
+    cmd: []string,
+    # user to use in container
+    user: ?string,
+    # workdir to run command in container
+    workdir: ?string,
+    # slice of keyword=value environment variables
+    env: ?[]string
+)
+
 # GetVersion returns version and build information of the podman service
 method GetVersion() -> (
     version: string,
@@ -1100,6 +1117,9 @@ method ContainerRestore(name: string, keep: bool, tcpEstablished: bool) -> (id:
 # ContainerRunlabel runs executes a command as described by a given container image label.
 method ContainerRunlabel(runlabel: Runlabel) -> ()
 
+# ExecContainer executes a command in the given container.
+method ExecContainer(opts: ExecOpts) -> ()
+
 # ListContainerMounts gathers all the mounted container mount points and returns them as an array
 # of strings
 # #### Example
diff --git a/completions/bash/podman b/completions/bash/podman
index 969f1054e..c8f00a3cb 100644
--- a/completions/bash/podman
+++ b/completions/bash/podman
@@ -1234,6 +1234,7 @@ _podman_diff() {
 
 _podman_exec() {
     local options_with_args="
+    --detach-keys
     -e
     --env
     --user
diff --git a/docs/podman-exec.1.md b/docs/podman-exec.1.md
index 07d24e295..ab8d75626 100644
--- a/docs/podman-exec.1.md
+++ b/docs/podman-exec.1.md
@@ -10,19 +10,24 @@ podman\-exec - Execute a command in a running container
 **podman exec** executes a command in a running container.
 
 ## OPTIONS
+
+**--detach-keys**=*sequence*
+
+Override the key sequence for detaching a container. Format is a single character `[a-Z]` or `ctrl-<value>` where `<value>` is one of: `a-z`, `@`, `^`, `[`, `,` or `_`.
+
 **--env**, **-e**
 
 You may specify arbitrary environment variables that are available for the
 command to be executed.
 
-**--interactive**, **-i**
+**--interactive**, **-i**=*true|false*
 
-Not supported.  All exec commands are interactive by default.
+When set to true, keep stdin open even if not attached. The default is *false*.
 
 **--latest**, **-l**
 
 Instead of providing the container name or ID, use the last created container. If you use methods other than Podman
-to run containers such as CRI-O, the last started  container could be from either of those methods.
+to run containers such as CRI-O, the last started container could be from either of those methods.
 
 The latest option is not supported on the remote client.
 
diff --git a/docs/podman-run.1.md b/docs/podman-run.1.md
index 6d95d6045..d6c7ae055 100644
--- a/docs/podman-run.1.md
+++ b/docs/podman-run.1.md
@@ -366,9 +366,7 @@ Path to the container-init binary.
 
 **--interactive**, **-i**=*true|false*
 
-Keep STDIN open even if not attached. The default is *false*.
-
-When set to true, keep stdin open even if not attached. The default is false.
+When set to true, keep stdin open even if not attached. The default is *false*.
 
 **--ip6**=*ip*
 
diff --git a/libpod/container_api.go b/libpod/container_api.go
index 50fa1759d..0cce6ca22 100644
--- a/libpod/container_api.go
+++ b/libpod/container_api.go
@@ -2,16 +2,13 @@ package libpod
 
 import (
 	"context"
-	"fmt"
 	"io"
 	"io/ioutil"
 	"os"
-	"strconv"
 	"time"
 
 	"github.com/containers/libpod/libpod/define"
 	"github.com/containers/libpod/libpod/events"
-	"github.com/containers/libpod/pkg/lookup"
 	"github.com/containers/storage/pkg/stringid"
 	"github.com/docker/docker/oci/caps"
 	"github.com/opentracing/opentracing-go"
@@ -21,6 +18,11 @@ import (
 	"k8s.io/client-go/tools/remotecommand"
 )
 
+const (
+	defaultExecExitCode             = 125
+	defaultExecExitCodeCannotInvoke = 126
+)
+
 // Init creates a container in the OCI runtime
 func (c *Container) Init(ctx context.Context) (err error) {
 	span, _ := opentracing.StartSpanFromContext(ctx, "containerInit")
@@ -220,23 +222,19 @@ func (c *Container) Kill(signal uint) error {
 }
 
 // Exec starts a new process inside the container
+// Returns an exit code and an error. If Exec was not able to exec in the container before a failure, an exit code of 126 is returned.
+// If another generic error happens, an exit code of 125 is returned.
+// Sometimes, the $RUNTIME exec call errors, and if that is the case, the exit code is the exit code of the call.
+// Otherwise, the exit code will be the exit code of the executed call inside of the container.
 // TODO investigate allowing exec without attaching
-func (c *Container) Exec(tty, privileged bool, env, cmd []string, user, workDir string, streams *AttachStreams, preserveFDs int) error {
+func (c *Container) Exec(tty, privileged bool, env, cmd []string, user, workDir string, streams *AttachStreams, preserveFDs int, resize chan remotecommand.TerminalSize, detachKeys string) (int, error) {
 	var capList []string
-
-	locked := false
 	if !c.batched {
-		locked = true
-
 		c.lock.Lock()
-		defer func() {
-			if locked {
-				c.lock.Unlock()
-			}
-		}()
+		defer c.lock.Unlock()
 
 		if err := c.syncContainer(); err != nil {
-			return err
+			return defaultExecExitCodeCannotInvoke, err
 		}
 	}
 
@@ -244,25 +242,13 @@ func (c *Container) Exec(tty, privileged bool, env, cmd []string, user, workDir
 
 	// TODO can probably relax this once we track exec sessions
 	if conState != define.ContainerStateRunning {
-		return errors.Wrapf(define.ErrCtrStateInvalid, "cannot exec into container that is not running")
+		return defaultExecExitCodeCannotInvoke, errors.Wrapf(define.ErrCtrStateInvalid, "cannot exec into container that is not running")
 	}
+
 	if privileged || c.config.Privileged {
 		capList = caps.GetAllCapabilities()
 	}
 
-	// If user was set, look it up in the container to get a UID to use on
-	// the host
-	hostUser := ""
-	if user != "" {
-		execUser, err := lookup.GetUserGroupInfo(c.state.Mountpoint, user, nil)
-		if err != nil {
-			return err
-		}
-
-		// runc expects user formatted as uid:gid
-		hostUser = fmt.Sprintf("%d:%d", execUser.Uid, execUser.Gid)
-	}
-
 	// Generate exec session ID
 	// Ensure we don't conflict with an existing session ID
 	sessionID := stringid.GenerateNonCryptoID()
@@ -282,55 +268,27 @@ func (c *Container) Exec(tty, privileged bool, env, cmd []string, user, workDir
 	}
 
 	logrus.Debugf("Creating new exec session in container %s with session id %s", c.ID(), sessionID)
-
-	execCmd, err := c.ociRuntime.execContainer(c, cmd, capList, env, tty, workDir, hostUser, sessionID, streams, preserveFDs)
-	if err != nil {
-		return errors.Wrapf(err, "error exec %s", c.ID())
+	if err := c.createExecBundle(sessionID); err != nil {
+		return defaultExecExitCodeCannotInvoke, err
 	}
-	chWait := make(chan error)
-	go func() {
-		chWait <- execCmd.Wait()
-		close(chWait)
-	}()
 
-	pidFile := c.execPidPath(sessionID)
-	// 60 second seems a reasonable time to wait
-	// https://github.com/containers/libpod/issues/1495
-	// https://github.com/containers/libpod/issues/1816
-	const pidWaitTimeout = 60000
-
-	// Wait until the runtime makes the pidfile
-	exited, err := WaitForFile(pidFile, chWait, pidWaitTimeout*time.Millisecond)
-	if err != nil {
-		if exited {
-			// If the runtime exited, propagate the error we got from the process.
-			// We need to remove PID files to ensure no memory leaks
-			if err2 := os.Remove(pidFile); err2 != nil {
-				logrus.Errorf("Error removing exit file for container %s exec session %s: %v", c.ID(), sessionID, err2)
-			}
-
-			return err
+	defer func() {
+		// cleanup exec bundle
+		if err := c.cleanupExecBundle(sessionID); err != nil {
+			logrus.Errorf("Error removing exec session %s bundle path for container %s: %v", sessionID, c.ID(), err)
 		}
-		return errors.Wrapf(err, "timed out waiting for runtime to create pidfile for exec session in container %s", c.ID())
-	}
+	}()
 
-	// Pidfile exists, read it
-	contents, err := ioutil.ReadFile(pidFile)
-	// We need to remove PID files to ensure no memory leaks
-	if err2 := os.Remove(pidFile); err2 != nil {
-		logrus.Errorf("Error removing exit file for container %s exec session %s: %v", c.ID(), sessionID, err2)
-	}
-	if err != nil {
-		// We don't know the PID of the exec session
-		// However, it may still be alive
-		// TODO handle this better
-		return errors.Wrapf(err, "could not read pidfile for exec session %s in container %s", sessionID, c.ID())
-	}
-	pid, err := strconv.ParseInt(string(contents), 10, 32)
+	pid, attachChan, err := c.ociRuntime.execContainer(c, cmd, capList, env, tty, workDir, user, sessionID, streams, preserveFDs, resize, detachKeys)
 	if err != nil {
-		// As above, we don't have a valid PID, but the exec session is likely still alive
-		// TODO handle this better
-		return errors.Wrapf(err, "error parsing PID of exec session %s in container %s", sessionID, c.ID())
+		ec := defaultExecExitCode
+		// Conmon will pass a non-zero exit code from the runtime as a pid here.
+		// we differentiate a pid with an exit code by sending it as negative, so reverse
+		// that change and return the exit code the runtime failed with.
+		if pid < 0 {
+			ec = -1 * pid
+		}
+		return ec, err
 	}
 
 	// We have the PID, add it to state
@@ -340,12 +298,12 @@ func (c *Container) Exec(tty, privileged bool, env, cmd []string, user, workDir
 	session := new(ExecSession)
 	session.ID = sessionID
 	session.Command = cmd
-	session.PID = int(pid)
+	session.PID = pid
 	c.state.ExecSessions[sessionID] = session
 	if err := c.save(); err != nil {
 		// Now we have a PID but we can't save it in the DB
 		// TODO handle this better
-		return errors.Wrapf(err, "error saving exec sessions %s for container %s", sessionID, c.ID())
+		return defaultExecExitCode, errors.Wrapf(err, "error saving exec sessions %s for container %s", sessionID, c.ID())
 	}
 	c.newContainerEvent(events.Exec)
 	logrus.Debugf("Successfully started exec session %s in container %s", sessionID, c.ID())
@@ -353,23 +311,33 @@ func (c *Container) Exec(tty, privileged bool, env, cmd []string, user, workDir
 	// Unlock so other processes can use the container
 	if !c.batched {
 		c.lock.Unlock()
-		locked = false
 	}
 
-	var waitErr error
-	if !exited {
-		waitErr = <-chWait
+	lastErr := <-attachChan
+
+	exitCode, err := c.readExecExitCode(sessionID)
+	if err != nil {
+		if lastErr != nil {
+			logrus.Errorf(lastErr.Error())
+		}
+		lastErr = err
+	}
+	if exitCode != 0 {
+		if lastErr != nil {
+			logrus.Errorf(lastErr.Error())
+		}
+		lastErr = errors.Wrapf(define.ErrOCIRuntime, "non zero exit code: %d", exitCode)
 	}
 
 	// Lock again
 	if !c.batched {
-		locked = true
 		c.lock.Lock()
 	}
 
 	// Sync the container again to pick up changes in state
 	if err := c.syncContainer(); err != nil {
-		return errors.Wrapf(err, "error syncing container %s state to remove exec session %s", c.ID(), sessionID)
+		logrus.Errorf("error syncing container %s state to remove exec session %s", c.ID(), sessionID)
+		return exitCode, lastErr
 	}
 
 	// Remove the exec session from state
@@ -377,7 +345,7 @@ func (c *Container) Exec(tty, privileged bool, env, cmd []string, user, workDir
 	if err := c.save(); err != nil {
 		logrus.Errorf("Error removing exec session %s from container %s state: %v", sessionID, c.ID(), err)
 	}
-	return waitErr
+	return exitCode, lastErr
 }
 
 // AttachStreams contains streams that will be attached to the container
diff --git a/libpod/container_attach_linux.go b/libpod/container_attach_linux.go
deleted file mode 100644
index 837cbf916..000000000
--- a/libpod/container_attach_linux.go
+++ /dev/null
@@ -1,179 +0,0 @@
-//+build linux
-
-package libpod
-
-import (
-	"fmt"
-	"io"
-	"net"
-	"os"
-	"path/filepath"
-
-	"github.com/containers/libpod/libpod/define"
-	"github.com/containers/libpod/pkg/errorhandling"
-	"github.com/containers/libpod/pkg/kubeutils"
-	"github.com/containers/libpod/utils"
-	"github.com/docker/docker/pkg/term"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-	"golang.org/x/sys/unix"
-	"k8s.io/client-go/tools/remotecommand"
-)
-
-/* Sync with stdpipe_t in conmon.c */
-const (
-	AttachPipeStdin  = 1
-	AttachPipeStdout = 2
-	AttachPipeStderr = 3
-)
-
-// Attach to the given container
-// Does not check if state is appropriate
-func (c *Container) attach(streams *AttachStreams, keys string, resize <-chan remotecommand.TerminalSize, startContainer bool, started chan bool) error {
-	if !streams.AttachOutput && !streams.AttachError && !streams.AttachInput {
-		return errors.Wrapf(define.ErrInvalidArg, "must provide at least one stream to attach to")
-	}
-
-	logrus.Debugf("Attaching to container %s", c.ID())
-
-	return c.attachContainerSocket(resize, keys, streams, startContainer, started)
-}
-
-// attachContainerSocket connects to the container's attach socket and deals with the IO.
-// started is only required if startContainer is true
-// TODO add a channel to allow interrupting
-func (c *Container) attachContainerSocket(resize <-chan remotecommand.TerminalSize, keys string, streams *AttachStreams, startContainer bool, started chan bool) error {
-	if startContainer && started == nil {
-		return errors.Wrapf(define.ErrInternal, "started chan not passed when startContainer set")
-	}
-
-	// Use default detach keys when keys aren't passed or specified in libpod.conf
-	if len(keys) == 0 {
-		keys = DefaultDetachKeys
-	}
-
-	// Check the validity of the provided keys
-	detachKeys := []byte{}
-	var err error
-	detachKeys, err = term.ToBytes(keys)
-	if err != nil {
-		return errors.Wrapf(err, "invalid detach keys")
-	}
-
-	kubeutils.HandleResizing(resize, func(size remotecommand.TerminalSize) {
-		controlPath := filepath.Join(c.bundlePath(), "ctl")
-		controlFile, err := os.OpenFile(controlPath, unix.O_WRONLY, 0)
-		if err != nil {
-			logrus.Debugf("Could not open ctl file: %v", err)
-			return
-		}
-		defer errorhandling.CloseQuiet(controlFile)
-
-		logrus.Debugf("Received a resize event: %+v", size)
-		if _, err = fmt.Fprintf(controlFile, "%d %d %d\n", 1, size.Height, size.Width); err != nil {
-			logrus.Warnf("Failed to write to control file to resize terminal: %v", err)
-		}
-	})
-
-	socketPath := c.AttachSocketPath()
-
-	maxUnixLength := unixPathLength()
-	if maxUnixLength < len(socketPath) {
-		socketPath = socketPath[0:maxUnixLength]
-	}
-
-	logrus.Debug("connecting to socket ", socketPath)
-
-	conn, err := net.DialUnix("unixpacket", nil, &net.UnixAddr{Name: socketPath, Net: "unixpacket"})
-	if err != nil {
-		return errors.Wrapf(err, "failed to connect to container's attach socket: %v", socketPath)
-	}
-	defer func() {
-		if err := conn.Close(); err != nil {
-			logrus.Errorf("unable to close socket: %q", err)
-		}
-	}()
-
-	// If starting was requested, start the container and notify when that's
-	// done.
-	if startContainer {
-		if err := c.start(); err != nil {
-			return err
-		}
-		started <- true
-	}
-
-	receiveStdoutError := make(chan error)
-	go func() {
-		receiveStdoutError <- redirectResponseToOutputStreams(streams.OutputStream, streams.ErrorStream, streams.AttachOutput, streams.AttachError, conn)
-	}()
-
-	stdinDone := make(chan error)
-	go func() {
-		var err error
-		if streams.AttachInput {
-			_, err = utils.CopyDetachable(conn, streams.InputStream, detachKeys)
-			if err := conn.CloseWrite(); err != nil {
-				logrus.Error("failed to close write in attach")
-			}
-		}
-		stdinDone <- err
-	}()
-
-	select {
-	case err := <-receiveStdoutError:
-		return err
-	case err := <-stdinDone:
-		if err == define.ErrDetach {
-			return err
-		}
-		if streams.AttachOutput || streams.AttachError {
-			return <-receiveStdoutError
-		}
-	}
-	return nil
-}
-
-func redirectResponseToOutputStreams(outputStream, errorStream io.Writer, writeOutput, writeError bool, conn io.Reader) error {
-	var err error
-	buf := make([]byte, 8192+1) /* Sync with conmon STDIO_BUF_SIZE */
-	for {
-		nr, er := conn.Read(buf)
-		if nr > 0 {
-			var dst io.Writer
-			var doWrite bool
-			switch buf[0] {
-			case AttachPipeStdout:
-				dst = outputStream
-				doWrite = writeOutput
-			case AttachPipeStderr:
-				dst = errorStream
-				doWrite = writeError
-			default:
-				logrus.Infof("Received unexpected attach type %+d", buf[0])
-			}
-			if dst == nil {
-				return errors.New("output destination cannot be nil")
-			}
-			if doWrite {
-				nw, ew := dst.Write(buf[1:nr])
-				if ew != nil {
-					err = ew
-					break
-				}
-				if nr != nw+1 {
-					err = io.ErrShortWrite
-					break
-				}
-			}
-		}
-		if er == io.EOF {
-			break
-		}
-		if er != nil {
-			err = er
-			break
-		}
-	}
-	return err
-}
diff --git a/libpod/container_internal.go b/libpod/container_internal.go
index ca27f5814..36732685b 100644
--- a/libpod/container_internal.go
+++ b/libpod/container_internal.go
@@ -31,7 +31,8 @@ import (
 
 const (
 	// name of the directory holding the artifacts
-	artifactsDir = "artifacts"
+	artifactsDir      = "artifacts"
+	execDirPermission = 0755
 )
 
 // rootFsSize gets the size of the container's root filesystem
@@ -132,14 +133,91 @@ func (c *Container) AttachSocketPath() string {
 	return filepath.Join(c.ociRuntime.socketsDir, c.ID(), "attach")
 }
 
+// exitFilePath gets the path to the container's exit file
+func (c *Container) exitFilePath() string {
+	return filepath.Join(c.ociRuntime.exitsDir, c.ID())
+}
+
+// create a bundle path and associated files for an exec session
+func (c *Container) createExecBundle(sessionID string) (err error) {
+	bundlePath := c.execBundlePath(sessionID)
+	if createErr := os.MkdirAll(bundlePath, execDirPermission); createErr != nil {
+		return createErr
+	}
+	defer func() {
+		if err != nil {
+			if err2 := os.RemoveAll(bundlePath); err != nil {
+				logrus.Warnf("error removing exec bundle after creation caused another error: %v", err2)
+			}
+		}
+	}()
+	if err2 := os.MkdirAll(c.execExitFileDir(sessionID), execDirPermission); err2 != nil {
+		// The directory is allowed to exist
+		if !os.IsExist(err2) {
+			err = errors.Wrapf(err2, "error creating OCI runtime exit file path %s", c.execExitFileDir(sessionID))
+		}
+	}
+	return
+}
+
+// cleanup an exec session after its done
+func (c *Container) cleanupExecBundle(sessionID string) error {
+	return os.RemoveAll(c.execBundlePath(sessionID))
+}
+
+// the path to a containers exec session bundle
+func (c *Container) execBundlePath(sessionID string) string {
+	return filepath.Join(c.bundlePath(), sessionID)
+}
+
 // Get PID file path for a container's exec session
 func (c *Container) execPidPath(sessionID string) string {
-	return filepath.Join(c.state.RunDir, "exec_pid_"+sessionID)
+	return filepath.Join(c.execBundlePath(sessionID), "exec_pid")
 }
 
-// exitFilePath gets the path to the container's exit file
-func (c *Container) exitFilePath() string {
-	return filepath.Join(c.ociRuntime.exitsDir, c.ID())
+// the log path for an exec session
+func (c *Container) execLogPath(sessionID string) string {
+	return filepath.Join(c.execBundlePath(sessionID), "exec_log")
+}
+
+// the socket conmon creates for an exec session
+func (c *Container) execAttachSocketPath(sessionID string) string {
+	return filepath.Join(c.ociRuntime.socketsDir, sessionID, "attach")
+}
+
+// execExitFileDir gets the path to the container's exit file
+func (c *Container) execExitFileDir(sessionID string) string {
+	return filepath.Join(c.execBundlePath(sessionID), "exit")
+}
+
+// execOCILog returns the file path for the exec sessions oci log
+func (c *Container) execOCILog(sessionID string) string {
+	if !c.ociRuntime.supportsJSON {
+		return ""
+	}
+	return filepath.Join(c.execBundlePath(sessionID), "oci-log")
+}
+
+// readExecExitCode reads the exit file for an exec session and returns
+// the exit code
+func (c *Container) readExecExitCode(sessionID string) (int, error) {
+	exitFile := filepath.Join(c.execExitFileDir(sessionID), c.ID())
+	chWait := make(chan error)
+	defer close(chWait)
+
+	_, err := WaitForFile(exitFile, chWait, time.Second*5)
+	if err != nil {
+		return -1, err
+	}
+	ec, err := ioutil.ReadFile(exitFile)
+	if err != nil {
+		return -1, err
+	}
+	ecInt, err := strconv.Atoi(string(ec))
+	if err != nil {
+		return -1, err
+	}
+	return ecInt, nil
 }
 
 // Wait for the container's exit file to appear.
@@ -849,8 +927,8 @@ func (c *Container) init(ctx context.Context, retainRetries bool) error {
 		return err
 	}
 
-	// With the newSpec complete, do an OCI create
-	if err := c.ociRuntime.createContainer(c, c.config.CgroupParent, nil); err != nil {
+	// With the spec complete, do an OCI create
+	if err := c.ociRuntime.createContainer(c, nil); err != nil {
 		return err
 	}
 
diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go
index 3dfd4c9e9..6e775cd28 100644
--- a/libpod/container_internal_linux.go
+++ b/libpod/container_internal_linux.go
@@ -834,7 +834,7 @@ func (c *Container) restore(ctx context.Context, options ContainerCheckpointOpti
 		}
 	}
 
-	if err := c.ociRuntime.createContainer(c, c.config.CgroupParent, &options); err != nil {
+	if err := c.ociRuntime.createContainer(c, &options); err != nil {
 		return err
 	}
 
diff --git a/libpod/define/errors.go b/libpod/define/errors.go
index 3b8313855..a4368a9aa 100644
--- a/libpod/define/errors.go
+++ b/libpod/define/errors.go
@@ -97,6 +97,14 @@ var (
 	// OS.
 	ErrOSNotSupported = errors.New("no support for this OS yet")
 
-	// ErrOCIRuntime indicates an error from the OCI runtime
+	// ErrOCIRuntime indicates a generic error from the OCI runtime
 	ErrOCIRuntime = errors.New("OCI runtime error")
+
+	// ErrOCIRuntimePermissionDenied indicates the OCI runtime attempted to invoke a command that returned
+	// a permission denied error
+	ErrOCIRuntimePermissionDenied = errors.New("OCI runtime permission denied error")
+
+	// ErrOCIRuntimeNotFound indicates the OCI runtime attempted to invoke a command
+	// that was not found
+	ErrOCIRuntimeNotFound = errors.New("OCI runtime command not found error")
 )
diff --git a/libpod/healthcheck.go b/libpod/healthcheck.go
index 1a19b88bb..0338828e4 100644
--- a/libpod/healthcheck.go
+++ b/libpod/healthcheck.go
@@ -141,10 +141,18 @@ func (c *Container) runHealthCheck() (HealthCheckStatus, error) {
 	logrus.Debugf("executing health check command %s for %s", strings.Join(newCommand, " "), c.ID())
 	timeStart := time.Now()
 	hcResult := HealthCheckSuccess
-	hcErr := c.Exec(false, false, []string{}, newCommand, "", "", streams, 0)
+	_, hcErr := c.Exec(false, false, []string{}, newCommand, "", "", streams, 0, nil, "")
 	if hcErr != nil {
+		errCause := errors.Cause(hcErr)
 		hcResult = HealthCheckFailure
-		returnCode = 1
+		if errCause == define.ErrOCIRuntimeNotFound ||
+			errCause == define.ErrOCIRuntimePermissionDenied ||
+			errCause == define.ErrOCIRuntime {
+			returnCode = 1
+			hcErr = nil
+		} else {
+			returnCode = 125
+		}
 	}
 	timeEnd := time.Now()
 	if c.HealthCheckConfig().StartPeriod > 0 {
diff --git a/libpod/oci.go b/libpod/oci.go
index 3daf9f834..193e66aaf 100644
--- a/libpod/oci.go
+++ b/libpod/oci.go
@@ -62,12 +62,6 @@ type OCIRuntime struct {
 	supportsJSON  bool
 }
 
-// syncInfo is used to return data from monitor process to daemon
-type syncInfo struct {
-	Pid     int    `json:"pid"`
-	Message string `json:"message,omitempty"`
-}
-
 // ociError is used to parse the OCI runtime JSON log.  It is not part of the
 // OCI runtime specifications, it follows what runc does
 type ociError struct {
@@ -245,6 +239,7 @@ func (r *OCIRuntime) updateContainerStatus(ctr *Container, useRuntime bool) erro
 
 	cmd := exec.Command(r.path, "state", ctr.ID())
 	cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir))
+
 	outPipe, err := cmd.StdoutPipe()
 	if err != nil {
 		return errors.Wrapf(err, "getting stdout pipe")
@@ -390,103 +385,6 @@ func (r *OCIRuntime) unpauseContainer(ctr *Container) error {
 	return utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, env, r.path, "resume", ctr.ID())
 }
 
-// execContainer executes a command in a running container
-// TODO: Add --detach support
-// TODO: Convert to use conmon
-// TODO: add --pid-file and use that to generate exec session tracking
-func (r *OCIRuntime) execContainer(c *Container, cmd, capAdd, env []string, tty bool, cwd, user, sessionID string, streams *AttachStreams, preserveFDs int) (*exec.Cmd, error) {
-	if len(cmd) == 0 {
-		return nil, errors.Wrapf(define.ErrInvalidArg, "must provide a command to execute")
-	}
-
-	if sessionID == "" {
-		return nil, errors.Wrapf(define.ErrEmptyID, "must provide a session ID for exec")
-	}
-
-	runtimeDir, err := util.GetRootlessRuntimeDir()
-	if err != nil {
-		return nil, err
-	}
-
-	args := []string{}
-
-	// TODO - should we maintain separate logpaths for exec sessions?
-	args = append(args, "exec")
-
-	if cwd != "" {
-		args = append(args, "--cwd", cwd)
-	}
-
-	args = append(args, "--pid-file", c.execPidPath(sessionID))
-
-	if tty {
-		args = append(args, "--tty")
-	} else {
-		args = append(args, "--tty=false")
-	}
-
-	if user != "" {
-		args = append(args, "--user", user)
-	}
-
-	if preserveFDs > 0 {
-		args = append(args, fmt.Sprintf("--preserve-fds=%d", preserveFDs))
-	}
-	if c.config.Spec.Process.NoNewPrivileges {
-		args = append(args, "--no-new-privs")
-	}
-
-	for _, capabilityAdd := range capAdd {
-		args = append(args, "--cap", capabilityAdd)
-	}
-
-	for _, envVar := range env {
-		args = append(args, "--env", envVar)
-	}
-
-	// Append container ID, name and command
-	args = append(args, c.ID())
-	args = append(args, cmd...)
-
-	logrus.Debugf("Starting runtime %s with following arguments: %v", r.path, args)
-
-	execCmd := exec.Command(r.path, args...)
-
-	if streams.AttachOutput {
-		execCmd.Stdout = streams.OutputStream
-	}
-	if streams.AttachInput {
-		execCmd.Stdin = streams.InputStream
-	}
-	if streams.AttachError {
-		execCmd.Stderr = streams.ErrorStream
-	}
-
-	execCmd.Env = append(execCmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir))
-
-	if preserveFDs > 0 {
-		for fd := 3; fd < 3+preserveFDs; fd++ {
-			execCmd.ExtraFiles = append(execCmd.ExtraFiles, os.NewFile(uintptr(fd), fmt.Sprintf("fd-%d", fd)))
-		}
-	}
-
-	if err := execCmd.Start(); err != nil {
-		return nil, errors.Wrapf(err, "cannot start container %s", c.ID())
-	}
-
-	if preserveFDs > 0 {
-		for fd := 3; fd < 3+preserveFDs; fd++ {
-			// These fds were passed down to the runtime.  Close them
-			// and not interfere
-			if err := os.NewFile(uintptr(fd), fmt.Sprintf("fd-%d", fd)).Close(); err != nil {
-				logrus.Debugf("unable to close file fd-%d", fd)
-			}
-		}
-	}
-
-	return execCmd, nil
-}
-
 // checkpointContainer checkpoints the given container
 func (r *OCIRuntime) checkpointContainer(ctr *Container, options ContainerCheckpointOptions) error {
 	if err := label.SetSocketLabel(ctr.ProcessLabel()); err != nil {
diff --git a/libpod/oci_attach_linux.go b/libpod/oci_attach_linux.go
new file mode 100644
index 000000000..7157ee2f7
--- /dev/null
+++ b/libpod/oci_attach_linux.go
@@ -0,0 +1,258 @@
+//+build linux
+
+package libpod
+
+import (
+	"fmt"
+	"io"
+	"net"
+	"os"
+	"path/filepath"
+
+	"github.com/containers/libpod/libpod/define"
+	"github.com/containers/libpod/pkg/errorhandling"
+	"github.com/containers/libpod/pkg/kubeutils"
+	"github.com/containers/libpod/utils"
+	"github.com/docker/docker/pkg/term"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+	"k8s.io/client-go/tools/remotecommand"
+)
+
+/* Sync with stdpipe_t in conmon.c */
+const (
+	AttachPipeStdin  = 1
+	AttachPipeStdout = 2
+	AttachPipeStderr = 3
+)
+
+// Attach to the given container
+// Does not check if state is appropriate
+// started is only required if startContainer is true
+func (c *Container) attach(streams *AttachStreams, keys string, resize <-chan remotecommand.TerminalSize, startContainer bool, started chan bool) error {
+	if !streams.AttachOutput && !streams.AttachError && !streams.AttachInput {
+		return errors.Wrapf(define.ErrInvalidArg, "must provide at least one stream to attach to")
+	}
+	if startContainer && started == nil {
+		return errors.Wrapf(define.ErrInternal, "started chan not passed when startContainer set")
+	}
+
+	detachKeys, err := processDetachKeys(keys)
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("Attaching to container %s", c.ID())
+
+	registerResizeFunc(resize, c.bundlePath())
+
+	socketPath := buildSocketPath(c.AttachSocketPath())
+
+	conn, err := net.DialUnix("unixpacket", nil, &net.UnixAddr{Name: socketPath, Net: "unixpacket"})
+	if err != nil {
+		return errors.Wrapf(err, "failed to connect to container's attach socket: %v", socketPath)
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			logrus.Errorf("unable to close socket: %q", err)
+		}
+	}()
+
+	// If starting was requested, start the container and notify when that's
+	// done.
+	if startContainer {
+		if err := c.start(); err != nil {
+			return err
+		}
+		started <- true
+	}
+
+	receiveStdoutError, stdinDone := setupStdioChannels(streams, conn, detachKeys)
+	return readStdio(streams, receiveStdoutError, stdinDone)
+}
+
+// Attach to the given container's exec session
+// attachFd and startFd must be open file descriptors
+// attachFd must be the output side of the fd. attachFd is used for two things:
+//  conmon will first send a nonse value across the pipe indicating it has set up its side of the console socket
+//    this ensures attachToExec gets all of the output of the called process
+//  conmon will then send the exit code of the exec process, or an error in the exec session
+// startFd must be the input side of the fd.
+//   conmon will wait to start the exec session until the parent process has setup the console socket.
+//   Once attachToExec successfully attaches to the console socket, the child conmon process responsible for calling runtime exec
+//     will read from the output side of start fd, thus learning to start the child process.
+// Thus, the order goes as follow:
+// 1. conmon parent process sets up its console socket. sends on attachFd
+// 2. attachToExec attaches to the console socket after reading on attachFd
+// 3. child waits on startFd for attachToExec to attach to said console socket
+// 4. attachToExec sends on startFd, signalling it has attached to the socket and child is ready to go
+// 5. child receives on startFd, runs the runtime exec command
+// attachToExec is responsible for closing startFd and attachFd
+func (c *Container) attachToExec(streams *AttachStreams, keys string, resize <-chan remotecommand.TerminalSize, sessionID string, startFd, attachFd *os.File) error {
+	if !streams.AttachOutput && !streams.AttachError && !streams.AttachInput {
+		return errors.Wrapf(define.ErrInvalidArg, "must provide at least one stream to attach to")
+	}
+	if startFd == nil || attachFd == nil {
+		return errors.Wrapf(define.ErrInvalidArg, "start sync pipe and attach sync pipe must be defined for exec attach")
+	}
+
+	defer errorhandling.CloseQuiet(startFd)
+	defer errorhandling.CloseQuiet(attachFd)
+
+	detachKeys, err := processDetachKeys(keys)
+	if err != nil {
+		return err
+	}
+
+	logrus.Debugf("Attaching to container %s exec session %s", c.ID(), sessionID)
+
+	registerResizeFunc(resize, c.execBundlePath(sessionID))
+
+	// set up the socket path, such that it is the correct length and location for exec
+	socketPath := buildSocketPath(c.execAttachSocketPath(sessionID))
+
+	// 2: read from attachFd that the parent process has set up the console socket
+	if _, err := readConmonPipeData(attachFd, ""); err != nil {
+		return err
+	}
+	// 2: then attach
+	conn, err := net.DialUnix("unixpacket", nil, &net.UnixAddr{Name: socketPath, Net: "unixpacket"})
+	if err != nil {
+		return errors.Wrapf(err, "failed to connect to container's attach socket: %v", socketPath)
+	}
+	defer func() {
+		if err := conn.Close(); err != nil {
+			logrus.Errorf("unable to close socket: %q", err)
+		}
+	}()
+
+	// start listening on stdio of the process
+	receiveStdoutError, stdinDone := setupStdioChannels(streams, conn, detachKeys)
+
+	// 4: send start message to child
+	if err := writeConmonPipeData(startFd); err != nil {
+		return err
+	}
+
+	return readStdio(streams, receiveStdoutError, stdinDone)
+}
+
+func processDetachKeys(keys string) ([]byte, error) {
+	// Check the validity of the provided keys first
+	if len(keys) == 0 {
+		keys = DefaultDetachKeys
+	}
+	detachKeys, err := term.ToBytes(keys)
+	if err != nil {
+		return nil, errors.Wrapf(err, "invalid detach keys")
+	}
+	return detachKeys, nil
+}
+
+func registerResizeFunc(resize <-chan remotecommand.TerminalSize, bundlePath string) {
+	kubeutils.HandleResizing(resize, func(size remotecommand.TerminalSize) {
+		controlPath := filepath.Join(bundlePath, "ctl")
+		controlFile, err := os.OpenFile(controlPath, unix.O_WRONLY, 0)
+		if err != nil {
+			logrus.Debugf("Could not open ctl file: %v", err)
+			return
+		}
+		defer controlFile.Close()
+
+		logrus.Debugf("Received a resize event: %+v", size)
+		if _, err = fmt.Fprintf(controlFile, "%d %d %d\n", 1, size.Height, size.Width); err != nil {
+			logrus.Warnf("Failed to write to control file to resize terminal: %v", err)
+		}
+	})
+}
+
+func buildSocketPath(socketPath string) string {
+	maxUnixLength := unixPathLength()
+	if maxUnixLength < len(socketPath) {
+		socketPath = socketPath[0:maxUnixLength]
+	}
+
+	logrus.Debug("connecting to socket ", socketPath)
+	return socketPath
+}
+
+func setupStdioChannels(streams *AttachStreams, conn *net.UnixConn, detachKeys []byte) (chan error, chan error) {
+	receiveStdoutError := make(chan error)
+	go func() {
+		receiveStdoutError <- redirectResponseToOutputStreams(streams.OutputStream, streams.ErrorStream, streams.AttachOutput, streams.AttachError, conn)
+	}()
+
+	stdinDone := make(chan error)
+	go func() {
+		var err error
+		if streams.AttachInput {
+			_, err = utils.CopyDetachable(conn, streams.InputStream, detachKeys)
+			conn.CloseWrite()
+		}
+		stdinDone <- err
+	}()
+
+	return receiveStdoutError, stdinDone
+}
+
+func redirectResponseToOutputStreams(outputStream, errorStream io.Writer, writeOutput, writeError bool, conn io.Reader) error {
+	var err error
+	buf := make([]byte, 8192+1) /* Sync with conmon STDIO_BUF_SIZE */
+	for {
+		nr, er := conn.Read(buf)
+		if nr > 0 {
+			var dst io.Writer
+			var doWrite bool
+			switch buf[0] {
+			case AttachPipeStdout:
+				dst = outputStream
+				doWrite = writeOutput
+			case AttachPipeStderr:
+				dst = errorStream
+				doWrite = writeError
+			default:
+				logrus.Infof("Received unexpected attach type %+d", buf[0])
+			}
+			if dst == nil {
+				return errors.New("output destination cannot be nil")
+			}
+
+			if doWrite {
+				nw, ew := dst.Write(buf[1:nr])
+				if ew != nil {
+					err = ew
+					break
+				}
+				if nr != nw+1 {
+					err = io.ErrShortWrite
+					break
+				}
+			}
+		}
+		if er == io.EOF {
+			break
+		}
+		if er != nil {
+			err = er
+			break
+		}
+	}
+	return err
+}
+
+func readStdio(streams *AttachStreams, receiveStdoutError, stdinDone chan error) error {
+	var err error
+	select {
+	case err = <-receiveStdoutError:
+		return err
+	case err = <-stdinDone:
+		if err == define.ErrDetach {
+			return err
+		}
+		if streams.AttachOutput || streams.AttachError {
+			return <-receiveStdoutError
+		}
+	}
+	return nil
+}
diff --git a/libpod/container_attach_linux_cgo.go b/libpod/oci_attach_linux_cgo.go
index d81243360..d81243360 100644
--- a/libpod/container_attach_linux_cgo.go
+++ b/libpod/oci_attach_linux_cgo.go
diff --git a/libpod/container_attach_linux_nocgo.go b/libpod/oci_attach_linux_nocgo.go
index a514a555d..a514a555d 100644
--- a/libpod/container_attach_linux_nocgo.go
+++ b/libpod/oci_attach_linux_nocgo.go
diff --git a/libpod/container_attach_unsupported.go b/libpod/oci_attach_unsupported.go
index c27ce0799..987d2c973 100644
--- a/libpod/container_attach_unsupported.go
+++ b/libpod/oci_attach_unsupported.go
@@ -3,6 +3,8 @@
 package libpod
 
 import (
+	"os"
+
 	"github.com/containers/libpod/libpod/define"
 	"k8s.io/client-go/tools/remotecommand"
 )
@@ -10,3 +12,7 @@ import (
 func (c *Container) attach(streams *AttachStreams, keys string, resize <-chan remotecommand.TerminalSize, startContainer bool, started chan bool) error {
 	return define.ErrNotImplemented
 }
+
+func (c *Container) attachToExec(streams *AttachStreams, keys string, resize <-chan remotecommand.TerminalSize, sessionID string, startFd *os.File, attachFd *os.File) error {
+	return define.ErrNotImplemented
+}
diff --git a/libpod/oci_internal_linux.go b/libpod/oci_internal_linux.go
new file mode 100644
index 000000000..1d8654eca
--- /dev/null
+++ b/libpod/oci_internal_linux.go
@@ -0,0 +1,493 @@
+// +build linux
+
+package libpod
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"regexp"
+	"runtime"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/containers/libpod/libpod/define"
+	"github.com/containers/libpod/pkg/cgroups"
+	"github.com/containers/libpod/pkg/lookup"
+	"github.com/containers/libpod/pkg/util"
+	"github.com/containers/libpod/utils"
+	"github.com/coreos/go-systemd/activation"
+	spec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/selinux/go-selinux"
+	"github.com/opencontainers/selinux/go-selinux/label"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"golang.org/x/sys/unix"
+)
+
+// createOCIContainer generates this container's main conmon instance and prepares it for starting
+func (r *OCIRuntime) createOCIContainer(ctr *Container, restoreOptions *ContainerCheckpointOptions) (err error) {
+	var stderrBuf bytes.Buffer
+
+	runtimeDir, err := util.GetRootlessRuntimeDir()
+	if err != nil {
+		return err
+	}
+
+	parentSyncPipe, childSyncPipe, err := newPipe()
+	if err != nil {
+		return errors.Wrapf(err, "error creating socket pair")
+	}
+	defer parentSyncPipe.Close()
+
+	childStartPipe, parentStartPipe, err := newPipe()
+	if err != nil {
+		return errors.Wrapf(err, "error creating socket pair for start pipe")
+	}
+
+	defer parentStartPipe.Close()
+
+	var ociLog string
+	if logrus.GetLevel() != logrus.DebugLevel && r.supportsJSON {
+		ociLog = filepath.Join(ctr.state.RunDir, "oci-log")
+	}
+	args := r.sharedConmonArgs(ctr, ctr.ID(), ctr.bundlePath(), filepath.Join(ctr.state.RunDir, "pidfile"), ctr.LogPath(), r.exitsDir, ociLog)
+
+	if ctr.config.Spec.Process.Terminal {
+		args = append(args, "-t")
+	} else if ctr.config.Stdin {
+		args = append(args, "-i")
+	}
+
+	if ctr.config.ConmonPidFile != "" {
+		args = append(args, "--conmon-pidfile", ctr.config.ConmonPidFile)
+	}
+
+	if r.noPivot {
+		args = append(args, "--no-pivot")
+	}
+
+	if len(ctr.config.ExitCommand) > 0 {
+		args = append(args, "--exit-command", ctr.config.ExitCommand[0])
+		for _, arg := range ctr.config.ExitCommand[1:] {
+			args = append(args, []string{"--exit-command-arg", arg}...)
+		}
+	}
+
+	if restoreOptions != nil {
+		args = append(args, "--restore", ctr.CheckpointPath())
+		if restoreOptions.TCPEstablished {
+			args = append(args, "--runtime-opt", "--tcp-established")
+		}
+	}
+
+	logrus.WithFields(logrus.Fields{
+		"args": args,
+	}).Debugf("running conmon: %s", r.conmonPath)
+
+	cmd := exec.Command(r.conmonPath, args...)
+	cmd.Dir = ctr.bundlePath()
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+	// TODO this is probably a really bad idea for some uses
+	// Make this configurable
+	cmd.Stdin = os.Stdin
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if ctr.config.Spec.Process.Terminal {
+		cmd.Stderr = &stderrBuf
+	}
+
+	// 0, 1 and 2 are stdin, stdout and stderr
+	conmonEnv, envFiles, err := r.configureConmonEnv(runtimeDir)
+	if err != nil {
+		return err
+	}
+
+	cmd.Env = append(r.conmonEnv, fmt.Sprintf("_OCI_SYNCPIPE=%d", 3), fmt.Sprintf("_OCI_STARTPIPE=%d", 4))
+	cmd.Env = append(cmd.Env, conmonEnv...)
+	cmd.ExtraFiles = append(cmd.ExtraFiles, childSyncPipe, childStartPipe)
+	cmd.ExtraFiles = append(cmd.ExtraFiles, envFiles...)
+
+	if r.reservePorts && !ctr.config.NetMode.IsSlirp4netns() {
+		ports, err := bindPorts(ctr.config.PortMappings)
+		if err != nil {
+			return err
+		}
+
+		// Leak the port we bound in the conmon process.  These fd's won't be used
+		// by the container and conmon will keep the ports busy so that another
+		// process cannot use them.
+		cmd.ExtraFiles = append(cmd.ExtraFiles, ports...)
+	}
+
+	if ctr.config.NetMode.IsSlirp4netns() {
+		ctr.rootlessSlirpSyncR, ctr.rootlessSlirpSyncW, err = os.Pipe()
+		if err != nil {
+			return errors.Wrapf(err, "failed to create rootless network sync pipe")
+		}
+		// Leak one end in conmon, the other one will be leaked into slirp4netns
+		cmd.ExtraFiles = append(cmd.ExtraFiles, ctr.rootlessSlirpSyncW)
+	}
+
+	err = startCommandGivenSelinux(cmd)
+	// regardless of whether we errored or not, we no longer need the children pipes
+	childSyncPipe.Close()
+	childStartPipe.Close()
+	if err != nil {
+		return err
+	}
+	if err := r.moveConmonToCgroupAndSignal(ctr, cmd, parentStartPipe, ctr.ID()); err != nil {
+		return err
+	}
+	/* Wait for initial setup and fork, and reap child */
+	err = cmd.Wait()
+	if err != nil {
+		return err
+	}
+
+	pid, err := readConmonPipeData(parentSyncPipe, ociLog)
+	if err != nil {
+		if err2 := r.deleteContainer(ctr); err2 != nil {
+			logrus.Errorf("Error removing container %s from runtime after creation failed", ctr.ID())
+		}
+		return err
+	}
+	ctr.state.PID = pid
+
+	conmonPID, err := readConmonPidFile(ctr.config.ConmonPidFile)
+	if err != nil {
+		logrus.Warnf("error reading conmon pid file for container %s: %s", ctr.ID(), err.Error())
+	} else if conmonPID > 0 {
+		// conmon not having a pid file is a valid state, so don't set it if we don't have it
+		logrus.Infof("Got Conmon PID as %d", conmonPID)
+		ctr.state.ConmonPID = conmonPID
+	}
+
+	return nil
+}
+
+// prepareProcessExec returns the path of the process.json used in runc exec -p
+// caller is responsible to close the returned *os.File if needed.
+func prepareProcessExec(c *Container, cmd, env []string, tty bool, cwd, user, sessionID string) (*os.File, error) {
+	f, err := ioutil.TempFile(c.execBundlePath(sessionID), "exec-process-")
+	if err != nil {
+		return nil, err
+	}
+
+	pspec := c.config.Spec.Process
+	pspec.Args = cmd
+	// We need to default this to false else it will inherit terminal as true
+	// from the container.
+	pspec.Terminal = false
+	if tty {
+		pspec.Terminal = true
+	}
+	if len(env) > 0 {
+		pspec.Env = append(pspec.Env, env...)
+	}
+
+	if cwd != "" {
+		pspec.Cwd = cwd
+
+	}
+	// If user was set, look it up in the container to get a UID to use on
+	// the host
+	if user != "" {
+		execUser, err := lookup.GetUserGroupInfo(c.state.Mountpoint, user, nil)
+		if err != nil {
+			return nil, err
+		}
+		sgids := make([]uint32, 0, len(execUser.Sgids))
+		for _, sgid := range execUser.Sgids {
+			sgids = append(sgids, uint32(sgid))
+		}
+		processUser := spec.User{
+			UID:            uint32(execUser.Uid),
+			GID:            uint32(execUser.Gid),
+			AdditionalGids: sgids,
+		}
+
+		pspec.User = processUser
+	}
+
+	processJSON, err := json.Marshal(pspec)
+	if err != nil {
+		return nil, err
+	}
+
+	if err := ioutil.WriteFile(f.Name(), processJSON, 0644); err != nil {
+		return nil, err
+	}
+	return f, nil
+}
+
+// configureConmonEnv gets the environment values to add to conmon's exec struct
+// TODO this may want to be less hardcoded/more configurable in the future
+func (r *OCIRuntime) configureConmonEnv(runtimeDir string) ([]string, []*os.File, error) {
+	env := make([]string, 0, 6)
+	env = append(env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir))
+	env = append(env, fmt.Sprintf("_CONTAINERS_USERNS_CONFIGURED=%s", os.Getenv("_CONTAINERS_USERNS_CONFIGURED")))
+	env = append(env, fmt.Sprintf("_CONTAINERS_ROOTLESS_UID=%s", os.Getenv("_CONTAINERS_ROOTLESS_UID")))
+	home, err := homeDir()
+	if err != nil {
+		return nil, nil, err
+	}
+	env = append(env, fmt.Sprintf("HOME=%s", home))
+
+	extraFiles := make([]*os.File, 0)
+	if notify, ok := os.LookupEnv("NOTIFY_SOCKET"); ok {
+		env = append(env, fmt.Sprintf("NOTIFY_SOCKET=%s", notify))
+	}
+	if listenfds, ok := os.LookupEnv("LISTEN_FDS"); ok {
+		env = append(env, fmt.Sprintf("LISTEN_FDS=%s", listenfds), "LISTEN_PID=1")
+		fds := activation.Files(false)
+		extraFiles = append(extraFiles, fds...)
+	}
+	return env, extraFiles, nil
+}
+
+// sharedConmonArgs takes common arguments for exec and create/restore and formats them for the conmon CLI
+func (r *OCIRuntime) sharedConmonArgs(ctr *Container, cuuid, bundlePath, pidPath, logPath, exitDir, ociLogPath string) []string {
+	// set the conmon API version to be able to use the correct sync struct keys
+	args := []string{"--api-version", "1"}
+	if r.cgroupManager == SystemdCgroupsManager {
+		args = append(args, "-s")
+	}
+	args = append(args, "-c", ctr.ID())
+	args = append(args, "-u", cuuid)
+	args = append(args, "-r", r.path)
+	args = append(args, "-b", bundlePath)
+	args = append(args, "-p", pidPath)
+
+	var logDriver string
+	switch ctr.LogDriver() {
+	case JournaldLogging:
+		logDriver = JournaldLogging
+	case JSONLogging:
+		fallthrough
+	default:
+		// No case here should happen except JSONLogging, but keep this here in case the options are extended
+		logrus.Errorf("%s logging specified but not supported. Choosing k8s-file logging instead", ctr.LogDriver())
+		fallthrough
+	case KubernetesLogging:
+		logDriver = fmt.Sprintf("%s:%s", KubernetesLogging, logPath)
+	}
+
+	args = append(args, "-l", logDriver)
+	args = append(args, "--exit-dir", exitDir)
+	args = append(args, "--socket-dir-path", r.socketsDir)
+	if r.logSizeMax >= 0 {
+		args = append(args, "--log-size-max", fmt.Sprintf("%v", r.logSizeMax))
+	}
+
+	logLevel := logrus.GetLevel()
+	args = append(args, "--log-level", logLevel.String())
+
+	if logLevel == logrus.DebugLevel {
+		logrus.Debugf("%s messages will be logged to syslog", r.conmonPath)
+		args = append(args, "--syslog")
+	}
+	if ociLogPath != "" {
+		args = append(args, "--runtime-arg", "--log-format=json", "--runtime-arg", "--log", fmt.Sprintf("--runtime-arg=%s", ociLogPath))
+	}
+	return args
+}
+
+// startCommandGivenSelinux starts a container ensuring to set the labels of
+// the process to make sure SELinux doesn't block conmon communication, if SELinux is enabled
+func startCommandGivenSelinux(cmd *exec.Cmd) error {
+	if !selinux.GetEnabled() {
+		return cmd.Start()
+	}
+	// Set the label of the conmon process to be level :s0
+	// This will allow the container processes to talk to fifo-files
+	// passed into the container by conmon
+	var (
+		plabel string
+		con    selinux.Context
+		err    error
+	)
+	plabel, err = selinux.CurrentLabel()
+	if err != nil {
+		return errors.Wrapf(err, "Failed to get current SELinux label")
+	}
+
+	con, err = selinux.NewContext(plabel)
+	if err != nil {
+		return errors.Wrapf(err, "Failed to get new context from SELinux label")
+	}
+
+	runtime.LockOSThread()
+	if con["level"] != "s0" && con["level"] != "" {
+		con["level"] = "s0"
+		if err = label.SetProcessLabel(con.Get()); err != nil {
+			runtime.UnlockOSThread()
+			return err
+		}
+	}
+	err = cmd.Start()
+	// Ignore error returned from SetProcessLabel("") call,
+	// can't recover.
+	label.SetProcessLabel("")
+	runtime.UnlockOSThread()
+	return err
+}
+
+// moveConmonToCgroupAndSignal gets a container's cgroupParent and moves the conmon process to that cgroup
+// it then signals for conmon to start by sending nonse data down the start fd
+func (r *OCIRuntime) moveConmonToCgroupAndSignal(ctr *Container, cmd *exec.Cmd, startFd *os.File, uuid string) error {
+	cgroupParent := ctr.CgroupParent()
+	if os.Geteuid() == 0 {
+		if r.cgroupManager == SystemdCgroupsManager {
+			unitName := createUnitName("libpod-conmon", ctr.ID())
+
+			realCgroupParent := cgroupParent
+			splitParent := strings.Split(cgroupParent, "/")
+			if strings.HasSuffix(cgroupParent, ".slice") && len(splitParent) > 1 {
+				realCgroupParent = splitParent[len(splitParent)-1]
+			}
+
+			logrus.Infof("Running conmon under slice %s and unitName %s", realCgroupParent, unitName)
+			if err := utils.RunUnderSystemdScope(cmd.Process.Pid, realCgroupParent, unitName); err != nil {
+				logrus.Warnf("Failed to add conmon to systemd sandbox cgroup: %v", err)
+			}
+		} else {
+			cgroupPath := filepath.Join(ctr.config.CgroupParent, "conmon")
+			control, err := cgroups.New(cgroupPath, &spec.LinuxResources{})
+			if err != nil {
+				logrus.Warnf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err)
+			} else {
+				// we need to remove this defer and delete the cgroup once conmon exits
+				// maybe need a conmon monitor?
+				if err := control.AddPid(cmd.Process.Pid); err != nil {
+					logrus.Warnf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err)
+				}
+			}
+		}
+	}
+
+	/* We set the cgroup, now the child can start creating children */
+	if err := writeConmonPipeData(startFd); err != nil {
+		return err
+	}
+	return nil
+}
+
+// newPipe creates a unix socket pair for communication
+func newPipe() (parent *os.File, child *os.File, err error) {
+	fds, err := unix.Socketpair(unix.AF_LOCAL, unix.SOCK_SEQPACKET|unix.SOCK_CLOEXEC, 0)
+	if err != nil {
+		return nil, nil, err
+	}
+	return os.NewFile(uintptr(fds[1]), "parent"), os.NewFile(uintptr(fds[0]), "child"), nil
+}
+
+// readConmonPidFile attempts to read conmon's pid from its pid file
+func readConmonPidFile(pidFile string) (int, error) {
+	// Let's try reading the Conmon pid at the same time.
+	if pidFile != "" {
+		contents, err := ioutil.ReadFile(pidFile)
+		if err != nil {
+			return -1, err
+		}
+		// Convert it to an int
+		conmonPID, err := strconv.Atoi(string(contents))
+		if err != nil {
+			return -1, err
+		}
+		return conmonPID, nil
+	}
+	return 0, nil
+}
+
+// readConmonPipeData attempts to read a syncInfo struct from the pipe
+func readConmonPipeData(pipe *os.File, ociLog string) (int, error) {
+	// syncInfo is used to return data from monitor process to daemon
+	type syncInfo struct {
+		Data    int    `json:"data"`
+		Message string `json:"message,omitempty"`
+	}
+
+	// Wait to get container pid from conmon
+	type syncStruct struct {
+		si  *syncInfo
+		err error
+	}
+	ch := make(chan syncStruct)
+	go func() {
+		var si *syncInfo
+		rdr := bufio.NewReader(pipe)
+		b, err := rdr.ReadBytes('\n')
+		if err != nil {
+			ch <- syncStruct{err: err}
+		}
+		if err := json.Unmarshal(b, &si); err != nil {
+			ch <- syncStruct{err: err}
+			return
+		}
+		ch <- syncStruct{si: si}
+	}()
+
+	data := -1
+	select {
+	case ss := <-ch:
+		if ss.err != nil {
+			return -1, errors.Wrapf(ss.err, "error reading container (probably exited) json message")
+		}
+		logrus.Debugf("Received: %d", ss.si.Data)
+		if ss.si.Data < 0 {
+			if ociLog != "" {
+				ociLogData, err := ioutil.ReadFile(ociLog)
+				if err == nil {
+					var ociErr ociError
+					if err := json.Unmarshal(ociLogData, &ociErr); err == nil {
+						return ss.si.Data, getOCIRuntimeError(ociErr.Msg)
+					}
+				}
+			}
+			// If we failed to parse the JSON errors, then print the output as it is
+			if ss.si.Message != "" {
+				return ss.si.Data, getOCIRuntimeError(ss.si.Message)
+			}
+			return ss.si.Data, errors.Wrapf(define.ErrInternal, "container create failed")
+		}
+		data = ss.si.Data
+	case <-time.After(ContainerCreateTimeout):
+		return -1, errors.Wrapf(define.ErrInternal, "container creation timeout")
+	}
+	return data, nil
+}
+
+func getOCIRuntimeError(runtimeMsg string) error {
+	if match, _ := regexp.MatchString(".*permission denied.*", runtimeMsg); match {
+		return errors.Wrapf(define.ErrOCIRuntimePermissionDenied, "%s", strings.Trim(runtimeMsg, "\n"))
+	}
+	if match, _ := regexp.MatchString(".*executable file not found in.*", runtimeMsg); match {
+		return errors.Wrapf(define.ErrOCIRuntimeNotFound, "%s", strings.Trim(runtimeMsg, "\n"))
+	}
+	return errors.Wrapf(define.ErrOCIRuntime, "%s", strings.Trim(runtimeMsg, "\n"))
+}
+
+// writeConmonPipeData writes nonse data to a pipe
+func writeConmonPipeData(pipe *os.File) error {
+	someData := []byte{0}
+	_, err := pipe.Write(someData)
+	return err
+}
+
+// formatRuntimeOpts prepends opts passed to it with --runtime-opt for passing to conmon
+func formatRuntimeOpts(opts ...string) []string {
+	args := make([]string, 0, len(opts)*2)
+	for _, o := range opts {
+		args = append(args, "--runtime-opt", o)
+	}
+	return args
+}
diff --git a/libpod/oci_linux.go b/libpod/oci_linux.go
index 9ce836cb5..45365203e 100644
--- a/libpod/oci_linux.go
+++ b/libpod/oci_linux.go
@@ -3,78 +3,29 @@
 package libpod
 
 import (
-	"bufio"
-	"bytes"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
 	"syscall"
 	"time"
 
 	"github.com/containers/libpod/libpod/define"
-	"github.com/containers/libpod/pkg/cgroups"
 	"github.com/containers/libpod/pkg/errorhandling"
 	"github.com/containers/libpod/pkg/rootless"
 	"github.com/containers/libpod/pkg/util"
 	"github.com/containers/libpod/utils"
 	pmount "github.com/containers/storage/pkg/mount"
-	"github.com/coreos/go-systemd/activation"
-	spec "github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/opencontainers/selinux/go-selinux"
-	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/sys/unix"
+	"k8s.io/client-go/tools/remotecommand"
 )
 
 const unknownPackage = "Unknown"
 
-func (r *OCIRuntime) moveConmonToCgroup(ctr *Container, cgroupParent string, cmd *exec.Cmd) error {
-	if os.Geteuid() == 0 {
-		if r.cgroupManager == SystemdCgroupsManager {
-			unitName := createUnitName("libpod-conmon", ctr.ID())
-
-			realCgroupParent := cgroupParent
-			splitParent := strings.Split(cgroupParent, "/")
-			if strings.HasSuffix(cgroupParent, ".slice") && len(splitParent) > 1 {
-				realCgroupParent = splitParent[len(splitParent)-1]
-			}
-
-			logrus.Infof("Running conmon under slice %s and unitName %s", realCgroupParent, unitName)
-			if err := utils.RunUnderSystemdScope(cmd.Process.Pid, realCgroupParent, unitName); err != nil {
-				logrus.Warnf("Failed to add conmon to systemd sandbox cgroup: %v", err)
-			}
-		} else {
-			cgroupPath := filepath.Join(ctr.config.CgroupParent, "conmon")
-			control, err := cgroups.New(cgroupPath, &spec.LinuxResources{})
-			if err != nil {
-				logrus.Warnf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err)
-			} else {
-				// we need to remove this defer and delete the cgroup once conmon exits
-				// maybe need a conmon monitor?
-				if err := control.AddPid(cmd.Process.Pid); err != nil {
-					logrus.Warnf("Failed to add conmon to cgroupfs sandbox cgroup: %v", err)
-				}
-			}
-		}
-	}
-	return nil
-}
-
-// newPipe creates a unix socket pair for communication
-func newPipe() (parent *os.File, child *os.File, err error) {
-	fds, err := unix.Socketpair(unix.AF_LOCAL, unix.SOCK_STREAM|unix.SOCK_CLOEXEC, 0)
-	if err != nil {
-		return nil, nil, err
-	}
-	return os.NewFile(uintptr(fds[1]), "parent"), os.NewFile(uintptr(fds[0]), "child"), nil
-}
-
 // makeAccessible changes the path permission and each parent directory to have --x--x--x
 func makeAccessible(path string, uid, gid int) error {
 	for ; path != "/"; path = filepath.Dir(path) {
@@ -100,7 +51,7 @@ func makeAccessible(path string, uid, gid int) error {
 // CreateContainer creates a container in the OCI runtime
 // TODO terminal support for container
 // Presently just ignoring conmon opts related to it
-func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string, restoreOptions *ContainerCheckpointOptions) (err error) {
+func (r *OCIRuntime) createContainer(ctr *Container, restoreOptions *ContainerCheckpointOptions) (err error) {
 	if len(ctr.config.IDMappings.UIDMap) != 0 || len(ctr.config.IDMappings.GIDMap) != 0 {
 		for _, i := range []string{ctr.state.RunDir, ctr.runtime.config.TmpDir, ctr.config.StaticDir, ctr.state.Mountpoint, ctr.runtime.config.VolumePath} {
 			if err := makeAccessible(i, ctr.RootUID(), ctr.RootGID()); err != nil {
@@ -152,7 +103,7 @@ func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string, restor
 							return errors.Wrapf(err, "cannot unmount %s", m.Mountpoint)
 						}
 					}
-					return r.createOCIContainer(ctr, cgroupParent, restoreOptions)
+					return r.createOCIContainer(ctr, restoreOptions)
 				}()
 				ch <- err
 			}()
@@ -160,7 +111,7 @@ func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string, restor
 			return err
 		}
 	}
-	return r.createOCIContainer(ctr, cgroupParent, restoreOptions)
+	return r.createOCIContainer(ctr, restoreOptions)
 }
 
 func rpmVersion(path string) string {
@@ -195,293 +146,178 @@ func (r *OCIRuntime) conmonPackage() string {
 	return dpkgVersion(r.conmonPath)
 }
 
-func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, restoreOptions *ContainerCheckpointOptions) (err error) {
-	var stderrBuf bytes.Buffer
+// execContainer executes a command in a running container
+// TODO: Add --detach support
+// TODO: Convert to use conmon
+// TODO: add --pid-file and use that to generate exec session tracking
+func (r *OCIRuntime) execContainer(c *Container, cmd, capAdd, env []string, tty bool, cwd, user, sessionID string, streams *AttachStreams, preserveFDs int, resize chan remotecommand.TerminalSize, detachKeys string) (int, chan error, error) {
+	if len(cmd) == 0 {
+		return -1, nil, errors.Wrapf(define.ErrInvalidArg, "must provide a command to execute")
+	}
 
-	runtimeDir, err := util.GetRootlessRuntimeDir()
-	if err != nil {
-		return err
+	if sessionID == "" {
+		return -1, nil, errors.Wrapf(define.ErrEmptyID, "must provide a session ID for exec")
 	}
 
-	parentPipe, childPipe, err := newPipe()
+	// create sync pipe to receive the pid
+	parentSyncPipe, childSyncPipe, err := newPipe()
 	if err != nil {
-		return errors.Wrapf(err, "error creating socket pair")
+		return -1, nil, errors.Wrapf(err, "error creating socket pair")
 	}
 
+	defer errorhandling.CloseQuiet(parentSyncPipe)
+
+	// create start pipe to set the cgroup before running
+	// attachToExec is responsible for closing parentStartPipe
 	childStartPipe, parentStartPipe, err := newPipe()
 	if err != nil {
-		return errors.Wrapf(err, "error creating socket pair for start pipe")
+		return -1, nil, errors.Wrapf(err, "error creating socket pair")
 	}
 
-	defer errorhandling.CloseQuiet(parentPipe)
-	defer errorhandling.CloseQuiet(parentStartPipe)
+	// We want to make sure we close the parent{Start,Attach}Pipes if we fail
+	// but also don't want to close them after attach to exec is called
+	attachToExecCalled := false
 
-	ociLog := filepath.Join(ctr.state.RunDir, "oci-log")
-	logLevel := logrus.GetLevel()
+	defer func() {
+		if !attachToExecCalled {
+			errorhandling.CloseQuiet(parentStartPipe)
+		}
+	}()
 
-	args := []string{}
-	if r.cgroupManager == SystemdCgroupsManager {
-		args = append(args, "-s")
-	}
-	args = append(args, "-c", ctr.ID())
-	args = append(args, "-u", ctr.ID())
-	args = append(args, "-n", ctr.Name())
-	args = append(args, "-r", r.path)
-	args = append(args, "-b", ctr.bundlePath())
-	args = append(args, "-p", filepath.Join(ctr.state.RunDir, "pidfile"))
-	args = append(args, "--exit-dir", r.exitsDir)
-	if logLevel != logrus.DebugLevel && r.supportsJSON {
-		args = append(args, "--runtime-arg", "--log-format=json", "--runtime-arg", "--log", fmt.Sprintf("--runtime-arg=%s", ociLog))
-	}
-	if ctr.config.ConmonPidFile != "" {
-		args = append(args, "--conmon-pidfile", ctr.config.ConmonPidFile)
+	// create the attach pipe to allow attach socket to be created before
+	// $RUNTIME exec starts running. This is to make sure we can capture all output
+	// from the process through that socket, rather than half reading the log, half attaching to the socket
+	// attachToExec is responsible for closing parentAttachPipe
+	parentAttachPipe, childAttachPipe, err := newPipe()
+	if err != nil {
+		return -1, nil, errors.Wrapf(err, "error creating socket pair")
 	}
-	if len(ctr.config.ExitCommand) > 0 {
-		args = append(args, "--exit-command", ctr.config.ExitCommand[0])
-		for _, arg := range ctr.config.ExitCommand[1:] {
-			args = append(args, []string{"--exit-command-arg", arg}...)
+
+	defer func() {
+		if !attachToExecCalled {
+			errorhandling.CloseQuiet(parentAttachPipe)
 		}
-	}
-	args = append(args, "--socket-dir-path", r.socketsDir)
-	if ctr.config.Spec.Process.Terminal {
-		args = append(args, "-t")
-	} else if ctr.config.Stdin {
-		args = append(args, "-i")
-	}
-	if r.logSizeMax >= 0 {
-		args = append(args, "--log-size-max", fmt.Sprintf("%v", r.logSizeMax))
+	}()
+
+	childrenClosed := false
+	defer func() {
+		if !childrenClosed {
+			errorhandling.CloseQuiet(childSyncPipe)
+			errorhandling.CloseQuiet(childAttachPipe)
+			errorhandling.CloseQuiet(childStartPipe)
+		}
+	}()
+
+	runtimeDir, err := util.GetRootlessRuntimeDir()
+	if err != nil {
+		return -1, nil, err
 	}
 
-	logDriver := KubernetesLogging
-	if ctr.LogDriver() == JSONLogging {
-		logrus.Errorf("json-file logging specified but not supported. Choosing k8s-file logging instead")
-	} else if ctr.LogDriver() != "" {
-		logDriver = ctr.LogDriver()
+	processFile, err := prepareProcessExec(c, cmd, env, tty, cwd, user, sessionID)
+	if err != nil {
+		return -1, nil, err
 	}
-	args = append(args, "-l", fmt.Sprintf("%s:%s", logDriver, ctr.LogPath()))
 
-	if r.noPivot {
-		args = append(args, "--no-pivot")
+	var ociLog string
+	if logrus.GetLevel() != logrus.DebugLevel && r.supportsJSON {
+		ociLog = c.execOCILog(sessionID)
 	}
+	args := r.sharedConmonArgs(c, sessionID, c.execBundlePath(sessionID), c.execPidPath(sessionID), c.execLogPath(sessionID), c.execExitFileDir(sessionID), ociLog)
 
-	args = append(args, "--log-level", logLevel.String())
+	if preserveFDs > 0 {
+		args = append(args, formatRuntimeOpts("--preserve-fds", string(preserveFDs))...)
+	}
 
-	if logLevel == logrus.DebugLevel {
-		logrus.Debugf("%s messages will be logged to syslog", r.conmonPath)
-		args = append(args, "--syslog")
+	for _, capability := range capAdd {
+		args = append(args, formatRuntimeOpts("--cap", capability)...)
 	}
 
-	if restoreOptions != nil {
-		args = append(args, "--restore", ctr.CheckpointPath())
-		if restoreOptions.TCPEstablished {
-			args = append(args, "--restore-arg", "--tcp-established")
-		}
+	if tty {
+		args = append(args, "-t")
 	}
 
+	// Append container ID and command
+	args = append(args, "-e")
+	// TODO make this optional when we can detach
+	args = append(args, "--exec-attach")
+	args = append(args, "--exec-process-spec", processFile.Name())
+
 	logrus.WithFields(logrus.Fields{
 		"args": args,
 	}).Debugf("running conmon: %s", r.conmonPath)
+	execCmd := exec.Command(r.conmonPath, args...)
 
-	cmd := exec.Command(r.conmonPath, args...)
-	cmd.Dir = ctr.bundlePath()
-	cmd.SysProcAttr = &syscall.SysProcAttr{
-		Setpgid: true,
+	if streams.AttachInput {
+		execCmd.Stdin = streams.InputStream
 	}
-	// TODO this is probably a really bad idea for some uses
-	// Make this configurable
-	cmd.Stdin = os.Stdin
-	cmd.Stdout = os.Stdout
-	cmd.Stderr = os.Stderr
-	if ctr.config.Spec.Process.Terminal {
-		cmd.Stderr = &stderrBuf
-	}
-
-	cmd.ExtraFiles = append(cmd.ExtraFiles, childPipe, childStartPipe)
-	// 0, 1 and 2 are stdin, stdout and stderr
-	cmd.Env = append(r.conmonEnv, fmt.Sprintf("_OCI_SYNCPIPE=%d", 3))
-	cmd.Env = append(cmd.Env, fmt.Sprintf("_OCI_STARTPIPE=%d", 4))
-	cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir))
-	cmd.Env = append(cmd.Env, fmt.Sprintf("_CONTAINERS_USERNS_CONFIGURED=%s", os.Getenv("_CONTAINERS_USERNS_CONFIGURED")))
-	cmd.Env = append(cmd.Env, fmt.Sprintf("_CONTAINERS_ROOTLESS_UID=%s", os.Getenv("_CONTAINERS_ROOTLESS_UID")))
-	home, err := homeDir()
-	if err != nil {
-		return err
+	if streams.AttachOutput {
+		execCmd.Stdout = streams.OutputStream
+	}
+	if streams.AttachError {
+		execCmd.Stderr = streams.ErrorStream
 	}
-	cmd.Env = append(cmd.Env, fmt.Sprintf("HOME=%s", home))
-
-	if r.reservePorts && !ctr.config.NetMode.IsSlirp4netns() {
-		ports, err := bindPorts(ctr.config.PortMappings)
-		if err != nil {
-			return err
-		}
 
-		// Leak the port we bound in the conmon process.  These fd's won't be used
-		// by the container and conmon will keep the ports busy so that another
-		// process cannot use them.
-		cmd.ExtraFiles = append(cmd.ExtraFiles, ports...)
+	conmonEnv, extraFiles, err := r.configureConmonEnv(runtimeDir)
+	if err != nil {
+		return -1, nil, err
 	}
 
-	if ctr.config.NetMode.IsSlirp4netns() {
-		ctr.rootlessSlirpSyncR, ctr.rootlessSlirpSyncW, err = os.Pipe()
-		if err != nil {
-			return errors.Wrapf(err, "failed to create rootless network sync pipe")
-		}
-		// Leak one end in conmon, the other one will be leaked into slirp4netns
-		cmd.ExtraFiles = append(cmd.ExtraFiles, ctr.rootlessSlirpSyncW)
-	}
-
-	if notify, ok := os.LookupEnv("NOTIFY_SOCKET"); ok {
-		cmd.Env = append(cmd.Env, fmt.Sprintf("NOTIFY_SOCKET=%s", notify))
-	}
-	if listenfds, ok := os.LookupEnv("LISTEN_FDS"); ok {
-		cmd.Env = append(cmd.Env, fmt.Sprintf("LISTEN_FDS=%s", listenfds), "LISTEN_PID=1")
-		fds := activation.Files(false)
-		cmd.ExtraFiles = append(cmd.ExtraFiles, fds...)
-	}
-	if selinux.GetEnabled() {
-		// Set the label of the conmon process to be level :s0
-		// This will allow the container processes to talk to fifo-files
-		// passed into the container by conmon
-		var (
-			plabel string
-			con    selinux.Context
-		)
-		plabel, err = selinux.CurrentLabel()
-		if err != nil {
-			if err := childPipe.Close(); err != nil {
-				logrus.Errorf("failed to close child pipe: %q", err)
-			}
-			return errors.Wrapf(err, "Failed to get current SELinux label")
-		}
+	// we don't want to step on users fds they asked to preserve
+	// Since 0-2 are used for stdio, start the fds we pass in at preserveFDs+3
+	execCmd.Env = append(r.conmonEnv, fmt.Sprintf("_OCI_SYNCPIPE=%d", preserveFDs+3), fmt.Sprintf("_OCI_STARTPIPE=%d", preserveFDs+4), fmt.Sprintf("_OCI_ATTACHPIPE=%d", preserveFDs+5))
+	execCmd.Env = append(execCmd.Env, conmonEnv...)
 
-		con, err = selinux.NewContext(plabel)
-		if err != nil {
-			return errors.Wrapf(err, "Failed to get new context from SELinux label")
-		}
+	execCmd.ExtraFiles = append(execCmd.ExtraFiles, childSyncPipe, childStartPipe, childAttachPipe)
+	execCmd.ExtraFiles = append(execCmd.ExtraFiles, extraFiles...)
+	execCmd.Dir = c.execBundlePath(sessionID)
+	execCmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
 
-		runtime.LockOSThread()
-		if con["level"] != "s0" && con["level"] != "" {
-			con["level"] = "s0"
-			if err = label.SetProcessLabel(con.Get()); err != nil {
-				runtime.UnlockOSThread()
-				return err
-			}
+	if preserveFDs > 0 {
+		for fd := 3; fd < 3+preserveFDs; fd++ {
+			execCmd.ExtraFiles = append(execCmd.ExtraFiles, os.NewFile(uintptr(fd), fmt.Sprintf("fd-%d", fd)))
 		}
-		err = cmd.Start()
-		// Ignore error returned from SetProcessLabel("") call,
-		// can't recover.
-		if err := label.SetProcessLabel(""); err != nil {
-			_ = err
-		}
-		runtime.UnlockOSThread()
-	} else {
-		err = cmd.Start()
 	}
-	if err != nil {
-		errorhandling.CloseQuiet(childPipe)
-		return err
-	}
-	defer func() {
-		_ = cmd.Wait()
-	}()
 
-	// We don't need childPipe on the parent side
-	if err := childPipe.Close(); err != nil {
-		return err
-	}
-	if err := childStartPipe.Close(); err != nil {
-		return err
-	}
+	err = startCommandGivenSelinux(execCmd)
 
-	// Move conmon to specified cgroup
-	if err := r.moveConmonToCgroup(ctr, cgroupParent, cmd); err != nil {
-		return err
-	}
+	// We don't need children pipes  on the parent side
+	errorhandling.CloseQuiet(childSyncPipe)
+	errorhandling.CloseQuiet(childAttachPipe)
+	errorhandling.CloseQuiet(childStartPipe)
+	childrenClosed = true
 
-	/* We set the cgroup, now the child can start creating children */
-	someData := []byte{0}
-	_, err = parentStartPipe.Write(someData)
 	if err != nil {
-		return err
+		return -1, nil, errors.Wrapf(err, "cannot start container %s", c.ID())
 	}
-
-	/* Wait for initial setup and fork, and reap child */
-	err = cmd.Wait()
-	if err != nil {
-		return err
+	if err := r.moveConmonToCgroupAndSignal(c, execCmd, parentStartPipe, sessionID); err != nil {
+		return -1, nil, err
 	}
 
-	defer func() {
-		if err != nil {
-			if err2 := r.deleteContainer(ctr); err2 != nil {
-				logrus.Errorf("Error removing container %s from runtime after creation failed", ctr.ID())
+	if preserveFDs > 0 {
+		for fd := 3; fd < 3+preserveFDs; fd++ {
+			// These fds were passed down to the runtime.  Close them
+			// and not interfere
+			if err := os.NewFile(uintptr(fd), fmt.Sprintf("fd-%d", fd)).Close(); err != nil {
+				logrus.Debugf("unable to close file fd-%d", fd)
 			}
 		}
-	}()
-
-	// Wait to get container pid from conmon
-	type syncStruct struct {
-		si  *syncInfo
-		err error
 	}
-	ch := make(chan syncStruct)
+
+	// TODO Only create if !detach
+	// Attach to the container before starting it
+	attachChan := make(chan error)
 	go func() {
-		var si *syncInfo
-		rdr := bufio.NewReader(parentPipe)
-		b, err := rdr.ReadBytes('\n')
-		if err != nil {
-			ch <- syncStruct{err: err}
-		}
-		if err := json.Unmarshal(b, &si); err != nil {
-			ch <- syncStruct{err: err}
-			return
-		}
-		ch <- syncStruct{si: si}
+		// attachToExec is responsible for closing pipes
+		attachChan <- c.attachToExec(streams, detachKeys, resize, sessionID, parentStartPipe, parentAttachPipe)
+		close(attachChan)
 	}()
+	attachToExecCalled = true
 
-	select {
-	case ss := <-ch:
-		if ss.err != nil {
-			return errors.Wrapf(ss.err, "error reading container (probably exited) json message")
-		}
-		logrus.Debugf("Received container pid: %d", ss.si.Pid)
-		if ss.si.Pid == -1 {
-			if r.supportsJSON {
-				data, err := ioutil.ReadFile(ociLog)
-				if err == nil {
-					var ociErr ociError
-					if err := json.Unmarshal(data, &ociErr); err == nil {
-						return errors.Wrapf(define.ErrOCIRuntime, "%s", strings.Trim(ociErr.Msg, "\n"))
-					}
-				}
-			}
-			// If we failed to parse the JSON errors, then print the output as it is
-			if ss.si.Message != "" {
-				return errors.Wrapf(define.ErrOCIRuntime, "%s", ss.si.Message)
-			}
-			return errors.Wrapf(define.ErrInternal, "container create failed")
-		}
-		ctr.state.PID = ss.si.Pid
-		// Let's try reading the Conmon pid at the same time.
-		if ctr.config.ConmonPidFile != "" {
-			contents, err := ioutil.ReadFile(ctr.config.ConmonPidFile)
-			if err != nil {
-				logrus.Warnf("Error reading Conmon pidfile for container %s: %v", ctr.ID(), err)
-			} else {
-				// Convert it to an int
-				conmonPID, err := strconv.Atoi(string(contents))
-				if err != nil {
-					logrus.Warnf("Error decoding Conmon PID %q for container %s: %v", string(contents), ctr.ID(), err)
-				} else {
-					ctr.state.ConmonPID = conmonPID
-					logrus.Infof("Got Conmon PID as %d", conmonPID)
-				}
-			}
-		}
-	case <-time.After(ContainerCreateTimeout):
-		return errors.Wrapf(define.ErrInternal, "container creation timeout")
-	}
-	return nil
+	pid, err := readConmonPipeData(parentSyncPipe, ociLog)
+
+	return pid, attachChan, err
 }
 
 // Wait for a container which has been sent a signal to stop
diff --git a/libpod/oci_unsupported.go b/libpod/oci_unsupported.go
index cfdf70bc6..4a65d4d1d 100644
--- a/libpod/oci_unsupported.go
+++ b/libpod/oci_unsupported.go
@@ -7,6 +7,7 @@ import (
 	"os/exec"
 
 	"github.com/containers/libpod/libpod/define"
+	"k8s.io/client-go/tools/remotecommand"
 )
 
 func (r *OCIRuntime) moveConmonToCgroup(ctr *Container, cgroupParent string, cmd *exec.Cmd) error {
@@ -17,7 +18,7 @@ func newPipe() (parent *os.File, child *os.File, err error) {
 	return nil, nil, define.ErrNotImplemented
 }
 
-func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string, restoreOptions *ContainerCheckpointOptions) (err error) {
+func (r *OCIRuntime) createContainer(ctr *Container, restoreOptions *ContainerCheckpointOptions) (err error) {
 	return define.ErrNotImplemented
 }
 
@@ -40,3 +41,7 @@ func (r *OCIRuntime) execStopContainer(ctr *Container, timeout uint) error {
 func (r *OCIRuntime) stopContainer(ctr *Container, timeout uint) error {
 	return define.ErrOSNotSupported
 }
+
+func (r *OCIRuntime) execContainer(c *Container, cmd, capAdd, env []string, tty bool, cwd, user, sessionID string, streams *AttachStreams, preserveFDs int, resize chan remotecommand.TerminalSize, detachKeys string) (int, chan error, error) {
+	return -1, nil, define.ErrOSNotSupported
+}
diff --git a/pkg/adapter/containers.go b/pkg/adapter/containers.go
index 9726b237f..47f1b091e 100644
--- a/pkg/adapter/containers.go
+++ b/pkg/adapter/containers.go
@@ -924,13 +924,85 @@ func (r *LocalRuntime) execPS(c *libpod.Container, args []string) ([]string, err
 	}()
 
 	cmd := append([]string{"ps"}, args...)
-	if err := c.Exec(false, false, []string{}, cmd, "", "", streams, 0); err != nil {
+	ec, err := c.Exec(false, false, []string{}, cmd, "", "", streams, 0, nil, "")
+	if err != nil {
 		return nil, err
+	} else if ec != 0 {
+		return nil, errors.Errorf("Runtime failed with exit status: %d and output: %s", ec, strings.Join(psOutput, " "))
 	}
 
 	return psOutput, nil
 }
 
+// ExecContainer executes a command in the container
+func (r *LocalRuntime) ExecContainer(ctx context.Context, cli *cliconfig.ExecValues) (int, error) {
+	var (
+		ctr *Container
+		err error
+		cmd []string
+	)
+	// default invalid command exit code
+	ec := 125
+
+	if cli.Latest {
+		if ctr, err = r.GetLatestContainer(); err != nil {
+			return ec, err
+		}
+		cmd = cli.InputArgs[0:]
+	} else {
+		if ctr, err = r.LookupContainer(cli.InputArgs[0]); err != nil {
+			return ec, err
+		}
+		cmd = cli.InputArgs[1:]
+	}
+
+	if cli.PreserveFDs > 0 {
+		entries, err := ioutil.ReadDir("/proc/self/fd")
+		if err != nil {
+			return ec, errors.Wrapf(err, "unable to read /proc/self/fd")
+		}
+
+		m := make(map[int]bool)
+		for _, e := range entries {
+			i, err := strconv.Atoi(e.Name())
+			if err != nil {
+				return ec, errors.Wrapf(err, "cannot parse %s in /proc/self/fd", e.Name())
+			}
+			m[i] = true
+		}
+
+		for i := 3; i < 3+cli.PreserveFDs; i++ {
+			if _, found := m[i]; !found {
+				return ec, errors.New("invalid --preserve-fds=N specified. Not enough FDs available")
+			}
+		}
+	}
+
+	// Validate given environment variables
+	env := map[string]string{}
+	if err := parse.ReadKVStrings(env, []string{}, cli.Env); err != nil {
+		return ec, errors.Wrapf(err, "unable to process environment variables")
+	}
+
+	// Build env slice of key=value strings for Exec
+	envs := []string{}
+	for k, v := range env {
+		envs = append(envs, fmt.Sprintf("%s=%s", k, v))
+	}
+
+	streams := new(libpod.AttachStreams)
+	streams.OutputStream = os.Stdout
+	streams.ErrorStream = os.Stderr
+	if cli.Interactive {
+		streams.InputStream = os.Stdin
+		streams.AttachInput = true
+	}
+	streams.AttachOutput = true
+	streams.AttachError = true
+
+	return ExecAttachCtr(ctx, ctr.Container, cli.Tty, cli.Privileged, envs, cmd, cli.User, cli.Workdir, streams, cli.PreserveFDs, cli.DetachKeys)
+}
+
 // Prune removes stopped containers
 func (r *LocalRuntime) Prune(ctx context.Context, maxWorkers int, force bool) ([]string, map[string]error, error) {
 	var (
@@ -1129,59 +1201,3 @@ func (r *LocalRuntime) Commit(ctx context.Context, c *cliconfig.CommitValues, co
 	}
 	return newImage.ID(), nil
 }
-
-// Exec a command in a container
-func (r *LocalRuntime) Exec(c *cliconfig.ExecValues, cmd []string) error {
-	var ctr *Container
-	var err error
-
-	if c.Latest {
-		ctr, err = r.GetLatestContainer()
-	} else {
-		ctr, err = r.LookupContainer(c.InputArgs[0])
-	}
-	if err != nil {
-		return errors.Wrapf(err, "unable to exec into %s", c.InputArgs[0])
-	}
-
-	if c.PreserveFDs > 0 {
-		entries, err := ioutil.ReadDir("/proc/self/fd")
-		if err != nil {
-			return errors.Wrapf(err, "unable to read /proc/self/fd")
-		}
-		m := make(map[int]bool)
-		for _, e := range entries {
-			i, err := strconv.Atoi(e.Name())
-			if err != nil {
-				return errors.Wrapf(err, "cannot parse %s in /proc/self/fd", e.Name())
-			}
-			m[i] = true
-		}
-		for i := 3; i < 3+c.PreserveFDs; i++ {
-			if _, found := m[i]; !found {
-				return errors.New("invalid --preserve-fds=N specified. Not enough FDs available")
-			}
-		}
-	}
-
-	// ENVIRONMENT VARIABLES
-	env := map[string]string{}
-
-	if err := parse.ReadKVStrings(env, []string{}, c.Env); err != nil {
-		return errors.Wrapf(err, "unable to process environment variables")
-	}
-	envs := []string{}
-	for k, v := range env {
-		envs = append(envs, fmt.Sprintf("%s=%s", k, v))
-	}
-
-	streams := new(libpod.AttachStreams)
-	streams.OutputStream = os.Stdout
-	streams.ErrorStream = os.Stderr
-	streams.InputStream = os.Stdin
-	streams.AttachOutput = true
-	streams.AttachError = true
-	streams.AttachInput = true
-
-	return ctr.Exec(c.Tty, c.Privileged, envs, cmd, c.User, c.Workdir, streams, c.PreserveFDs)
-}
diff --git a/pkg/adapter/containers_remote.go b/pkg/adapter/containers_remote.go
index fc23381a4..6b9fc8ee7 100644
--- a/pkg/adapter/containers_remote.go
+++ b/pkg/adapter/containers_remote.go
@@ -14,6 +14,7 @@ import (
 
 	"github.com/containers/libpod/cmd/podman/cliconfig"
 	"github.com/containers/libpod/cmd/podman/shared"
+	"github.com/containers/libpod/cmd/podman/shared/parse"
 	iopodman "github.com/containers/libpod/cmd/podman/varlink"
 	"github.com/containers/libpod/libpod"
 	"github.com/containers/libpod/libpod/define"
@@ -1034,7 +1035,42 @@ func (r *LocalRuntime) Commit(ctx context.Context, c *cliconfig.CommitValues, co
 	return iid, nil
 }
 
-// Exec executes a container in a running container
-func (r *LocalRuntime) Exec(c *cliconfig.ExecValues, cmd []string) error {
-	return define.ErrNotImplemented
+// ExecContainer executes a command in the container
+func (r *LocalRuntime) ExecContainer(ctx context.Context, cli *cliconfig.ExecValues) (int, error) {
+	// default invalid command exit code
+	ec := 125
+	// Validate given environment variables
+	env := map[string]string{}
+	if err := parse.ReadKVStrings(env, []string{}, cli.Env); err != nil {
+		return -1, errors.Wrapf(err, "Exec unable to process environment variables")
+	}
+
+	// Build env slice of key=value strings for Exec
+	envs := []string{}
+	for k, v := range env {
+		envs = append(envs, fmt.Sprintf("%s=%s", k, v))
+	}
+
+	opts := iopodman.ExecOpts{
+		Name:       cli.InputArgs[0],
+		Tty:        cli.Tty,
+		Privileged: cli.Privileged,
+		Cmd:        cli.InputArgs[1:],
+		User:       &cli.User,
+		Workdir:    &cli.Workdir,
+		Env:        &envs,
+	}
+
+	receive, err := iopodman.ExecContainer().Send(r.Conn, varlink.Upgrade, opts)
+	if err != nil {
+		return ec, errors.Wrapf(err, "Exec failed to contact service for %s", cli.InputArgs)
+	}
+
+	_, err = receive()
+	if err != nil {
+		return ec, errors.Wrapf(err, "Exec operation failed for %s", cli.InputArgs)
+	}
+
+	// TODO return exit code from exec call
+	return 0, nil
 }
diff --git a/pkg/adapter/terminal_linux.go b/pkg/adapter/terminal_linux.go
index 9f6ddc2e6..6e63dd87b 100644
--- a/pkg/adapter/terminal_linux.go
+++ b/pkg/adapter/terminal_linux.go
@@ -13,6 +13,25 @@ import (
 	"k8s.io/client-go/tools/remotecommand"
 )
 
+// ExecAttachCtr execs and attaches to a container
+func ExecAttachCtr(ctx context.Context, ctr *libpod.Container, tty, privileged bool, env, cmd []string, user, workDir string, streams *libpod.AttachStreams, preserveFDs int, detachKeys string) (int, error) {
+	resize := make(chan remotecommand.TerminalSize)
+
+	haveTerminal := terminal.IsTerminal(int(os.Stdin.Fd()))
+
+	// Check if we are attached to a terminal. If we are, generate resize
+	// events, and set the terminal to raw mode
+	if haveTerminal && tty {
+		cancel, oldTermState, err := handleTerminalAttach(ctx, resize)
+		if err != nil {
+			return -1, err
+		}
+		defer cancel()
+		defer restoreTerminal(oldTermState)
+	}
+	return ctr.Exec(tty, privileged, env, cmd, user, workDir, streams, preserveFDs, resize, detachKeys)
+}
+
 // StartAttachCtr starts and (if required) attaches to a container
 // if you change the signature of this function from os.File to io.Writer, it will trigger a downstream
 // error. we may need to just lint disable this one.
@@ -24,28 +43,16 @@ func StartAttachCtr(ctx context.Context, ctr *libpod.Container, stdout, stderr,
 	// Check if we are attached to a terminal. If we are, generate resize
 	// events, and set the terminal to raw mode
 	if haveTerminal && ctr.Spec().Process.Terminal {
-		logrus.Debugf("Handling terminal attach")
-
-		subCtx, cancel := context.WithCancel(ctx)
-		defer cancel()
-
-		resizeTty(subCtx, resize)
-
-		oldTermState, err := term.SaveState(os.Stdin.Fd())
+		cancel, oldTermState, err := handleTerminalAttach(ctx, resize)
 		if err != nil {
-			return errors.Wrapf(err, "unable to save terminal state")
-		}
-
-		logrus.SetFormatter(&RawTtyFormatter{})
-		if _, err := term.SetRawTerminal(os.Stdin.Fd()); err != nil {
 			return err
 		}
-
 		defer func() {
 			if err := restoreTerminal(oldTermState); err != nil {
 				logrus.Errorf("unable to restore terminal: %q", err)
 			}
 		}()
+		defer cancel()
 	}
 
 	streams := new(libpod.AttachStreams)
@@ -97,3 +104,25 @@ func StartAttachCtr(ctx context.Context, ctr *libpod.Container, stdout, stderr,
 
 	return nil
 }
+
+func handleTerminalAttach(ctx context.Context, resize chan remotecommand.TerminalSize) (context.CancelFunc, *term.State, error) {
+	logrus.Debugf("Handling terminal attach")
+
+	subCtx, cancel := context.WithCancel(ctx)
+
+	resizeTty(subCtx, resize)
+
+	oldTermState, err := term.SaveState(os.Stdin.Fd())
+	if err != nil {
+		// allow caller to not have to do any cleaning up if we error here
+		cancel()
+		return nil, nil, errors.Wrapf(err, "unable to save terminal state")
+	}
+
+	logrus.SetFormatter(&RawTtyFormatter{})
+	if _, err := term.SetRawTerminal(os.Stdin.Fd()); err != nil {
+		return nil, nil, err
+	}
+
+	return cancel, oldTermState, nil
+}
diff --git a/pkg/varlinkapi/attach.go b/pkg/varlinkapi/attach.go
index afa88e6a3..97ba525a5 100644
--- a/pkg/varlinkapi/attach.go
+++ b/pkg/varlinkapi/attach.go
@@ -82,9 +82,10 @@ func (i *LibpodAPI) Attach(call iopodman.VarlinkCall, name string, detachKeys st
 	if finalErr != define.ErrDetach && finalErr != nil {
 		logrus.Error(finalErr)
 	}
-	quitWriter := virtwriter.NewVirtWriteCloser(writer, virtwriter.Quit)
-	_, err = quitWriter.Write([]byte("HANG-UP"))
-	// TODO error handling is not quite right here yet
+
+	if err = virtwriter.HangUp(writer); err != nil {
+		logrus.Errorf("Failed to HANG-UP attach to %s: %s", ctr.ID(), err.Error())
+	}
 	return call.Writer.Flush()
 }
 
diff --git a/pkg/varlinkapi/containers.go b/pkg/varlinkapi/containers.go
index 6f6909fac..19a8bfd2e 100644
--- a/pkg/varlinkapi/containers.go
+++ b/pkg/varlinkapi/containers.go
@@ -19,8 +19,11 @@ import (
 	"github.com/containers/libpod/libpod/define"
 	"github.com/containers/libpod/libpod/logs"
 	"github.com/containers/libpod/pkg/adapter/shortcuts"
+	"github.com/containers/libpod/pkg/varlinkapi/virtwriter"
 	"github.com/containers/storage/pkg/archive"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"k8s.io/client-go/tools/remotecommand"
 )
 
 // ListContainers ...
@@ -756,3 +759,82 @@ func (i *LibpodAPI) Top(call iopodman.VarlinkCall, nameOrID string, descriptors
 	}
 	return call.ReplyTop(topInfo)
 }
+
+// ExecContainer is the varlink endpoint to execute a command in a container
+func (i *LibpodAPI) ExecContainer(call iopodman.VarlinkCall, opts iopodman.ExecOpts) error {
+	if !call.WantsUpgrade() {
+		return call.ReplyErrorOccurred("client must use upgraded connection to exec")
+	}
+
+	ctr, err := i.Runtime.LookupContainer(opts.Name)
+	if err != nil {
+		return call.ReplyContainerNotFound(opts.Name, err.Error())
+	}
+
+	state, err := ctr.State()
+	if err != nil {
+		return call.ReplyErrorOccurred(
+			fmt.Sprintf("exec failed to obtain container %s state: %s", ctr.ID(), err.Error()))
+	}
+
+	if state != define.ContainerStateRunning {
+		return call.ReplyErrorOccurred(
+			fmt.Sprintf("exec requires a running container, %s is %s", ctr.ID(), state.String()))
+	}
+
+	envs := []string{}
+	if opts.Env != nil {
+		envs = *opts.Env
+	}
+
+	var user string
+	if opts.User != nil {
+		user = *opts.User
+	}
+
+	var workDir string
+	if opts.Workdir != nil {
+		workDir = *opts.Workdir
+	}
+
+	resizeChan := make(chan remotecommand.TerminalSize)
+	errChan := make(chan error)
+
+	reader, writer, _, pipeWriter, streams := setupStreams(call)
+
+	go func() {
+		fmt.Printf("ExecContainer Start Reader\n")
+		if err := virtwriter.Reader(reader, nil, nil, pipeWriter, resizeChan); err != nil {
+			fmt.Printf("ExecContainer Reader err %s, %s\n", err.Error(), errors.Cause(err).Error())
+			errChan <- err
+		}
+	}()
+
+	// Debugging...
+	time.Sleep(5 * time.Second)
+
+	go func() {
+		fmt.Printf("ExecContainer Start ctr.Exec\n")
+		// TODO detach keys and resize
+		// TODO add handling for exit code
+		// TODO capture exit code and return to main thread
+		_, err := ctr.Exec(opts.Tty, opts.Privileged, envs, opts.Cmd, user, workDir, streams, 0, nil, "")
+		if err != nil {
+			fmt.Printf("ExecContainer Exec err %s, %s\n", err.Error(), errors.Cause(err).Error())
+			errChan <- errors.Wrapf(err, "ExecContainer failed for container %s", ctr.ID())
+		}
+	}()
+
+	execErr := <-errChan
+
+	if execErr != nil && errors.Cause(execErr) != io.EOF {
+		fmt.Printf("ExecContainer err: %s\n", execErr.Error())
+		return call.ReplyErrorOccurred(execErr.Error())
+	}
+
+	if err = virtwriter.HangUp(writer); err != nil {
+		fmt.Printf("ExecContainer hangup err: %s\n", err.Error())
+		logrus.Errorf("ExecContainer failed to HANG-UP on %s: %s", ctr.ID(), err.Error())
+	}
+	return call.Writer.Flush()
+}
diff --git a/pkg/varlinkapi/virtwriter/virtwriter.go b/pkg/varlinkapi/virtwriter/virtwriter.go
index 5e88914b2..0da2a91fc 100644
--- a/pkg/varlinkapi/virtwriter/virtwriter.go
+++ b/pkg/varlinkapi/virtwriter/virtwriter.go
@@ -4,8 +4,9 @@ import (
 	"bufio"
 	"encoding/binary"
 	"encoding/json"
-	"errors"
 	"io"
+
+	"github.com/pkg/errors"
 	"k8s.io/client-go/tools/remotecommand"
 )
 
@@ -95,7 +96,7 @@ func Reader(r *bufio.Reader, output io.Writer, errput io.Writer, input io.Writer
 	for {
 		n, err := io.ReadFull(r, headerBytes)
 		if err != nil {
-			return err
+			return errors.Wrapf(err, "Virtual Read failed, %d", n)
 		}
 		if n < 8 {
 			return errors.New("short read and no full header read")
@@ -151,3 +152,19 @@ func Reader(r *bufio.Reader, output io.Writer, errput io.Writer, input io.Writer
 		}
 	}
 }
+
+// HangUp sends message to peer to close connection
+func HangUp(writer *bufio.Writer) (err error) {
+	n := 0
+	msg := []byte("HANG-UP")
+
+	writeQuit := NewVirtWriteCloser(writer, Quit)
+	if n, err = writeQuit.Write(msg); err != nil {
+		return
+	}
+
+	if n != len(msg) {
+		return errors.Errorf("Failed to send complete %s message", string(msg))
+	}
+	return
+}
diff --git a/test/e2e/exec_test.go b/test/e2e/exec_test.go
index f42831ebb..76515cf3e 100644
--- a/test/e2e/exec_test.go
+++ b/test/e2e/exec_test.go
@@ -169,4 +169,24 @@ var _ = Describe("Podman exec", func() {
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(1))
 	})
+
+	It("podman exec cannot be invoked", func() {
+		setup := podmanTest.RunTopContainer("test1")
+		setup.WaitWithDefaultTimeout()
+		Expect(setup.ExitCode()).To(Equal(0))
+
+		session := podmanTest.Podman([]string{"exec", "test1", "/etc"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(126))
+	})
+
+	It("podman exec command not found", func() {
+		setup := podmanTest.RunTopContainer("test1")
+		setup.WaitWithDefaultTimeout()
+		Expect(setup.ExitCode()).To(Equal(0))
+
+		session := podmanTest.Podman([]string{"exec", "test1", "notthere"})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(Equal(127))
+	})
 })