Merge pull request #5088 from mheon/begin_exec_rework

Begin exec rework

20 files changed, 1552 insertions, 448 deletions

diff --git a/libpod/boltdb_state.go b/libpod/boltdb_state.go
index 4918bf57a..2ce8e0de4 100644
--- a/libpod/boltdb_state.go
+++ b/libpod/boltdb_state.go
@@ -41,6 +41,8 @@ type BoltState struct {
 //   containing the path to the container's network namespace, a dependencies
 //   bucket containing the container's dependencies, and an optional pod key
 //   containing the ID of the pod the container is joined to.
+//   After updates to include exec sessions, may also include an exec bucket
+//   with the IDs of exec sessions currently in use by the container.
 // - allCtrsBkt: Map of ID to name containing only containers. Used for
 //   container lookup operations.
 // - podBkt: Contains a sub-bucket for each pod in the state.
@@ -49,6 +51,10 @@ type BoltState struct {
 //   containers in the pod.
 // - allPodsBkt: Map of ID to name containing only pods. Used for pod lookup
 //   operations.
+// - execBkt: Map of exec session ID to exec session - contains a sub-bucket for
+//   each exec session in the DB.
+// - execRegistryBkt: Map of exec session ID to nothing. Contains one entry for
+//   each exec session. Used for iterating through all exec sessions.
 // - runtimeConfigBkt: Contains configuration of the libpod instance that
 //   initially created the database. This must match for any further instances
 //   that access the database, to ensure that state mismatches with
@@ -86,6 +92,7 @@ func NewBoltState(path string, runtime *Runtime) (State, error) {
 		allPodsBkt,
 		volBkt,
 		allVolsBkt,
+		execBkt,
 		runtimeConfigBkt,
 	}
 
@@ -171,6 +178,11 @@ func (s *BoltState) Refresh() error {
 			return err
 		}
 
+		execBucket, err := getExecBucket(tx)
+		if err != nil {
+			return err
+		}
+
 		// Iterate through all IDs. Check if they are containers.
 		// If they are, unmarshal their state, and then clear
 		// PID, mountpoint, and state for all of them
@@ -245,6 +257,26 @@ func (s *BoltState) Refresh() error {
 				return errors.Wrapf(err, "error updating state for container %s in DB", string(id))
 			}
 
+			// Delete all exec sessions, if there are any
+			ctrExecBkt := ctrBkt.Bucket(execBkt)
+			if ctrExecBkt != nil {
+				// Can't delete in a ForEach, so build a list of
+				// what to remove then remove.
+				toRemove := []string{}
+				err = ctrExecBkt.ForEach(func(id, unused []byte) error {
+					toRemove = append(toRemove, string(id))
+					return nil
+				})
+				if err != nil {
+					return err
+				}
+				for _, execId := range toRemove {
+					if err := ctrExecBkt.Delete([]byte(execId)); err != nil {
+						return errors.Wrapf(err, "error removing exec session %s from container %s", execId, string(id))
+					}
+				}
+			}
+
 			return nil
 		})
 		if err != nil {
@@ -285,7 +317,30 @@ func (s *BoltState) Refresh() error {
 
 			return nil
 		})
-		return err
+		if err != nil {
+			return err
+		}
+
+		// Now refresh exec sessions
+		// We want to remove them all, but for-each can't modify buckets
+		// So we have to make a list of what to operate on, then do the
+		// work.
+		toRemoveExec := []string{}
+		err = execBucket.ForEach(func(id, unused []byte) error {
+			toRemoveExec = append(toRemoveExec, string(id))
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		for _, execSession := range toRemoveExec {
+			if err := execBucket.Delete([]byte(execSession)); err != nil {
+				return errors.Wrapf(err, "error deleting exec session %s registry from database", execSession)
+			}
+		}
+
+		return nil
 	})
 	return err
 }
@@ -895,6 +950,287 @@ func (s *BoltState) GetContainerConfig(id string) (*ContainerConfig, error) {
 	return config, nil
 }
 
+// AddExecSession adds an exec session to the state.
+func (s *BoltState) AddExecSession(ctr *Container, session *ExecSession) error {
+	if !s.valid {
+		return define.ErrDBClosed
+	}
+
+	if !ctr.valid {
+		return define.ErrCtrRemoved
+	}
+
+	db, err := s.getDBCon()
+	if err != nil {
+		return err
+	}
+	defer s.deferredCloseDBCon(db)
+
+	ctrID := []byte(ctr.ID())
+	sessionID := []byte(session.ID())
+
+	err = db.Update(func(tx *bolt.Tx) error {
+		execBucket, err := getExecBucket(tx)
+		if err != nil {
+			return err
+		}
+		ctrBucket, err := getCtrBucket(tx)
+		if err != nil {
+			return err
+		}
+
+		dbCtr := ctrBucket.Bucket(ctrID)
+		if dbCtr == nil {
+			ctr.valid = false
+			return errors.Wrapf(define.ErrNoSuchCtr, "container %s is not present in the database", ctr.ID())
+		}
+
+		ctrExecSessionBucket, err := dbCtr.CreateBucketIfNotExists(execBkt)
+		if err != nil {
+			return errors.Wrapf(err, "error creating exec sessions bucket for container %s", ctr.ID())
+		}
+
+		execExists := execBucket.Get(sessionID)
+		if execExists != nil {
+			return errors.Wrapf(define.ErrExecSessionExists, "an exec session with ID %s already exists", session.ID())
+		}
+
+		if err := execBucket.Put(sessionID, ctrID); err != nil {
+			return errors.Wrapf(err, "error adding exec session %s to DB", session.ID())
+		}
+
+		if err := ctrExecSessionBucket.Put(sessionID, ctrID); err != nil {
+			return errors.Wrapf(err, "error adding exec session %s to container %s in DB", session.ID(), ctr.ID())
+		}
+
+		return nil
+	})
+	return err
+}
+
+// GetExecSession returns the ID of the container an exec session is associated
+// with.
+func (s *BoltState) GetExecSession(id string) (string, error) {
+	if !s.valid {
+		return "", define.ErrDBClosed
+	}
+
+	if id == "" {
+		return "", define.ErrEmptyID
+	}
+
+	db, err := s.getDBCon()
+	if err != nil {
+		return "", err
+	}
+	defer s.deferredCloseDBCon(db)
+
+	ctrID := ""
+	err = db.View(func(tx *bolt.Tx) error {
+		execBucket, err := getExecBucket(tx)
+		if err != nil {
+			return err
+		}
+
+		ctr := execBucket.Get([]byte(id))
+		if ctr == nil {
+			return errors.Wrapf(define.ErrNoSuchExecSession, "no exec session with ID %s found", id)
+		}
+		ctrID = string(ctr)
+		return nil
+	})
+	return ctrID, err
+}
+
+// RemoveExecSession removes references to the given exec session in the
+// database.
+func (s *BoltState) RemoveExecSession(session *ExecSession) error {
+	if !s.valid {
+		return define.ErrDBClosed
+	}
+
+	db, err := s.getDBCon()
+	if err != nil {
+		return err
+	}
+	defer s.deferredCloseDBCon(db)
+
+	sessionID := []byte(session.ID())
+	containerID := []byte(session.ContainerID())
+	err = db.Update(func(tx *bolt.Tx) error {
+		execBucket, err := getExecBucket(tx)
+		if err != nil {
+			return err
+		}
+		ctrBucket, err := getCtrBucket(tx)
+		if err != nil {
+			return err
+		}
+
+		sessionExists := execBucket.Get(sessionID)
+		if sessionExists == nil {
+			return define.ErrNoSuchExecSession
+		}
+		// Check that container ID matches
+		if string(sessionExists) != session.ContainerID() {
+			return errors.Wrapf(define.ErrInternal, "database inconsistency: exec session %s points to container %s in state but %s in database", session.ID(), session.ContainerID(), string(sessionExists))
+		}
+
+		if err := execBucket.Delete(sessionID); err != nil {
+			return errors.Wrapf(err, "error removing exec session %s from database", session.ID())
+		}
+
+		dbCtr := ctrBucket.Bucket(containerID)
+		if dbCtr == nil {
+			// State is inconsistent. We refer to a container that
+			// is no longer in the state.
+			// Return without error, to attempt to recover.
+			return nil
+		}
+
+		ctrExecBucket := dbCtr.Bucket(execBkt)
+		if ctrExecBucket == nil {
+			// Again, state is inconsistent. We should have an exec
+			// bucket, and it should have this session.
+			// Again, nothing we can do, so proceed and try to
+			// recover.
+			return nil
+		}
+
+		ctrSessionExists := ctrExecBucket.Get(sessionID)
+		if ctrSessionExists != nil {
+			if err := ctrExecBucket.Delete(sessionID); err != nil {
+				return errors.Wrapf(err, "error removing exec session %s from container %s in database", session.ID(), session.ContainerID())
+			}
+		}
+
+		return nil
+	})
+	return err
+}
+
+// GetContainerExecSessions retrieves the IDs of all exec sessions running in a
+// container that the database is aware of (IE, were added via AddExecSession).
+func (s *BoltState) GetContainerExecSessions(ctr *Container) ([]string, error) {
+	if !s.valid {
+		return nil, define.ErrDBClosed
+	}
+
+	if !ctr.valid {
+		return nil, define.ErrCtrRemoved
+	}
+
+	db, err := s.getDBCon()
+	if err != nil {
+		return nil, err
+	}
+	defer s.deferredCloseDBCon(db)
+
+	ctrID := []byte(ctr.ID())
+	sessions := []string{}
+	err = db.View(func(tx *bolt.Tx) error {
+		ctrBucket, err := getCtrBucket(tx)
+		if err != nil {
+			return err
+		}
+
+		dbCtr := ctrBucket.Bucket(ctrID)
+		if dbCtr == nil {
+			ctr.valid = false
+			return define.ErrNoSuchCtr
+		}
+
+		ctrExecSessions := dbCtr.Bucket(execBkt)
+		if ctrExecSessions == nil {
+			return nil
+		}
+
+		return ctrExecSessions.ForEach(func(id, unused []byte) error {
+			sessions = append(sessions, string(id))
+			return nil
+		})
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return sessions, nil
+}
+
+// RemoveContainerExecSessions removes all exec sessions attached to a given
+// container.
+func (s *BoltState) RemoveContainerExecSessions(ctr *Container) error {
+	if !s.valid {
+		return define.ErrDBClosed
+	}
+
+	if !ctr.valid {
+		return define.ErrCtrRemoved
+	}
+
+	db, err := s.getDBCon()
+	if err != nil {
+		return err
+	}
+	defer s.deferredCloseDBCon(db)
+
+	ctrID := []byte(ctr.ID())
+	sessions := []string{}
+
+	err = db.Update(func(tx *bolt.Tx) error {
+		execBucket, err := getExecBucket(tx)
+		if err != nil {
+			return err
+		}
+		ctrBucket, err := getCtrBucket(tx)
+		if err != nil {
+			return err
+		}
+
+		dbCtr := ctrBucket.Bucket(ctrID)
+		if dbCtr == nil {
+			ctr.valid = false
+			return define.ErrNoSuchCtr
+		}
+
+		ctrExecSessions := dbCtr.Bucket(execBkt)
+		if ctrExecSessions == nil {
+			return nil
+		}
+
+		err = ctrExecSessions.ForEach(func(id, unused []byte) error {
+			sessions = append(sessions, string(id))
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+
+		for _, session := range sessions {
+			if err := ctrExecSessions.Delete([]byte(session)); err != nil {
+				return errors.Wrapf(err, "error removing container %s exec session %s from database", ctr.ID(), session)
+			}
+			// Check if the session exists in the global table
+			// before removing. It should, but in cases where the DB
+			// has become inconsistent, we should try and proceed
+			// so we can recover.
+			sessionExists := execBucket.Get([]byte(session))
+			if sessionExists == nil {
+				continue
+			}
+			if string(sessionExists) != ctr.ID() {
+				return errors.Wrapf(define.ErrInternal, "database mismatch: exec session %s is associated with containers %s and %s", session, ctr.ID(), string(sessionExists))
+			}
+			if err := execBucket.Delete([]byte(session)); err != nil {
+				return errors.Wrapf(err, "error removing container %s exec session %s from exec sessions", ctr.ID(), session)
+			}
+		}
+
+		return nil
+	})
+	return err
+}
+
 // RewriteContainerConfig rewrites a container's configuration.
 // WARNING: This function is DANGEROUS. Do not use without reading the full
 // comment on this function in state.go.
diff --git a/libpod/boltdb_state_internal.go b/libpod/boltdb_state_internal.go
index 3f09305f5..1c4c9e12d 100644
--- a/libpod/boltdb_state_internal.go
+++ b/libpod/boltdb_state_internal.go
@@ -24,6 +24,7 @@ const (
 	allPodsName       = "allPods"
 	volName           = "vol"
 	allVolsName       = "allVolumes"
+	execName          = "exec"
 	runtimeConfigName = "runtime-config"
 
 	configName         = "config"
@@ -54,6 +55,7 @@ var (
 	allPodsBkt       = []byte(allPodsName)
 	volBkt           = []byte(volName)
 	allVolsBkt       = []byte(allVolsName)
+	execBkt          = []byte(execName)
 	runtimeConfigBkt = []byte(runtimeConfigName)
 
 	configKey          = []byte(configName)
@@ -339,6 +341,14 @@ func getAllVolsBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
 	return bkt, nil
 }
 
+func getExecBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
+	bkt := tx.Bucket(execBkt)
+	if bkt == nil {
+		return nil, errors.Wrapf(define.ErrDBBadConfig, "exec bucket not found in DB")
+	}
+	return bkt, nil
+}
+
 func getRuntimeConfigBucket(tx *bolt.Tx) (*bolt.Bucket, error) {
 	bkt := tx.Bucket(runtimeConfigBkt)
 	if bkt == nil {
@@ -787,6 +797,23 @@ func (s *BoltState) removeContainer(ctr *Container, pod *Pod, tx *bolt.Tx) error
 		}
 	}
 
+	// Does the container have exec sessions?
+	ctrExecSessionsBkt := ctrExists.Bucket(execBkt)
+	if ctrExecSessionsBkt != nil {
+		sessions := []string{}
+		err = ctrExecSessionsBkt.ForEach(func(id, value []byte) error {
+			sessions = append(sessions, string(id))
+
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+		if len(sessions) > 0 {
+			return errors.Wrapf(define.ErrExecSessionExists, "container %s has active exec sessions: %s", ctr.ID(), strings.Join(sessions, ", "))
+		}
+	}
+
 	// Does the container have dependencies?
 	ctrDepsBkt := ctrExists.Bucket(dependenciesBkt)
 	if ctrDepsBkt == nil {
diff --git a/libpod/common_test.go b/libpod/common_test.go
index 63ea4f41b..747252cf4 100644
--- a/libpod/common_test.go
+++ b/libpod/common_test.go
@@ -58,14 +58,12 @@ func getTestContainer(id, name string, manager lock.Manager) (*Container, error)
 			PID:        1234,
 			ExecSessions: map[string]*ExecSession{
 				"abcd": {
-					ID:      "1",
-					Command: []string{"2", "3"},
-					PID:     9876,
+					Id:  "1",
+					PID: 9876,
 				},
 				"ef01": {
-					ID:      "5",
-					Command: []string{"hello", "world"},
-					PID:     46765,
+					Id:  "5",
+					PID: 46765,
 				},
 			},
 			BindMounts: map[string]string{
diff --git a/libpod/container.go b/libpod/container.go
index d83de93bb..e59fb9fe8 100644
--- a/libpod/container.go
+++ b/libpod/container.go
@@ -181,9 +181,13 @@ type ContainerState struct {
 	PID int `json:"pid,omitempty"`
 	// ConmonPID is the PID of the container's conmon
 	ConmonPID int `json:"conmonPid,omitempty"`
-	// ExecSessions contains active exec sessions for container
-	// Exec session ID is mapped to PID of exec process
-	ExecSessions map[string]*ExecSession `json:"execSessions,omitempty"`
+	// ExecSessions contains all exec sessions that are associated with this
+	// container.
+	ExecSessions map[string]*ExecSession `json:"newExecSessions,omitempty"`
+	// LegacyExecSessions are legacy exec sessions from older versions of
+	// Podman.
+	// These are DEPRECATED and will be removed in a future release.
+	LegacyExecSessions map[string]*legacyExecSession `json:"execSessions,omitempty"`
 	// NetworkStatus contains the configuration results for all networks
 	// the pod is attached to. Only populated if we created a network
 	// namespace for the container, and the network namespace is currently
@@ -214,13 +218,6 @@ type ContainerState struct {
 	containerPlatformState
 }
 
-// ExecSession contains information on an active exec session
-type ExecSession struct {
-	ID      string   `json:"id"`
-	Command []string `json:"command"`
-	PID     int      `json:"pid"`
-}
-
 // ContainerConfig contains all information that was used to create the
 // container. It may not be changed once created.
 // It is stored, read-only, on disk
@@ -944,13 +941,13 @@ func (c *Container) ExecSession(id string) (*ExecSession, error) {
 
 	session, ok := c.state.ExecSessions[id]
 	if !ok {
-		return nil, errors.Wrapf(define.ErrNoSuchCtr, "no exec session with ID %s found in container %s", id, c.ID())
+		return nil, errors.Wrapf(define.ErrNoSuchExecSession, "no exec session with ID %s found in container %s", id, c.ID())
 	}
 
 	returnSession := new(ExecSession)
-	returnSession.ID = session.ID
-	returnSession.Command = session.Command
-	returnSession.PID = session.PID
+	if err := JSONDeepCopy(session, returnSession); err != nil {
+		return nil, errors.Wrapf(err, "error copying contents of container %s exec session %s", c.ID(), session.ID())
+	}
 
 	return returnSession, nil
 }
diff --git a/libpod/container_api.go b/libpod/container_api.go
index 039619ea6..967180437 100644
--- a/libpod/container_api.go
+++ b/libpod/container_api.go
@@ -9,10 +9,8 @@ import (
 	"os"
 	"time"
 
-	"github.com/containers/common/pkg/capabilities"
 	"github.com/containers/libpod/libpod/define"
 	"github.com/containers/libpod/libpod/events"
-	"github.com/containers/storage/pkg/stringid"
 	"github.com/opentracing/opentracing-go"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -215,142 +213,6 @@ func (c *Container) Kill(signal uint) error {
 	return c.save()
 }
 
-// Exec starts a new process inside the container
-// Returns an exit code and an error. If Exec was not able to exec in the container before a failure, an exit code of define.ExecErrorCodeCannotInvoke is returned.
-// If another generic error happens, an exit code of define.ExecErrorCodeGeneric is returned.
-// Sometimes, the $RUNTIME exec call errors, and if that is the case, the exit code is the exit code of the call.
-// Otherwise, the exit code will be the exit code of the executed call inside of the container.
-// TODO investigate allowing exec without attaching
-func (c *Container) Exec(tty, privileged bool, env map[string]string, cmd []string, user, workDir string, streams *AttachStreams, preserveFDs uint, resize chan remotecommand.TerminalSize, detachKeys string) (int, error) {
-	var capList []string
-	if !c.batched {
-		c.lock.Lock()
-		defer c.lock.Unlock()
-
-		if err := c.syncContainer(); err != nil {
-			return define.ExecErrorCodeCannotInvoke, err
-		}
-	}
-
-	if c.state.State != define.ContainerStateRunning {
-		return define.ExecErrorCodeCannotInvoke, errors.Wrapf(define.ErrCtrStateInvalid, "cannot exec into container that is not running")
-	}
-
-	if privileged || c.config.Privileged {
-		capList = capabilities.AllCapabilities()
-	}
-
-	// Generate exec session ID
-	// Ensure we don't conflict with an existing session ID
-	sessionID := stringid.GenerateNonCryptoID()
-	found := true
-	// This really ought to be a do-while, but Go doesn't have those...
-	for found {
-		found = false
-		for id := range c.state.ExecSessions {
-			if id == sessionID {
-				found = true
-				break
-			}
-		}
-		if found {
-			sessionID = stringid.GenerateNonCryptoID()
-		}
-	}
-
-	logrus.Debugf("Creating new exec session in container %s with session id %s", c.ID(), sessionID)
-	if err := c.createExecBundle(sessionID); err != nil {
-		return define.ExecErrorCodeCannotInvoke, err
-	}
-
-	defer func() {
-		// cleanup exec bundle
-		if err := c.cleanupExecBundle(sessionID); err != nil {
-			logrus.Errorf("Error removing exec session %s bundle path for container %s: %v", sessionID, c.ID(), err)
-		}
-	}()
-
-	opts := new(ExecOptions)
-	opts.Cmd = cmd
-	opts.CapAdd = capList
-	opts.Env = env
-	opts.Terminal = tty
-	opts.Cwd = workDir
-	opts.User = user
-	opts.Streams = streams
-	opts.PreserveFDs = preserveFDs
-	opts.Resize = resize
-	opts.DetachKeys = detachKeys
-
-	pid, attachChan, err := c.ociRuntime.ExecContainer(c, sessionID, opts)
-	if err != nil {
-		ec := define.ExecErrorCodeGeneric
-		// Conmon will pass a non-zero exit code from the runtime as a pid here.
-		// we differentiate a pid with an exit code by sending it as negative, so reverse
-		// that change and return the exit code the runtime failed with.
-		if pid < 0 {
-			ec = -1 * pid
-		}
-		return ec, err
-	}
-
-	// We have the PID, add it to state
-	if c.state.ExecSessions == nil {
-		c.state.ExecSessions = make(map[string]*ExecSession)
-	}
-	session := new(ExecSession)
-	session.ID = sessionID
-	session.Command = cmd
-	session.PID = pid
-	c.state.ExecSessions[sessionID] = session
-	if err := c.save(); err != nil {
-		// Now we have a PID but we can't save it in the DB
-		// TODO handle this better
-		return define.ExecErrorCodeGeneric, errors.Wrapf(err, "error saving exec sessions %s for container %s", sessionID, c.ID())
-	}
-	c.newContainerEvent(events.Exec)
-	logrus.Debugf("Successfully started exec session %s in container %s", sessionID, c.ID())
-
-	// Unlock so other processes can use the container
-	if !c.batched {
-		c.lock.Unlock()
-	}
-
-	lastErr := <-attachChan
-
-	exitCode, err := c.readExecExitCode(sessionID)
-	if err != nil {
-		if lastErr != nil {
-			logrus.Errorf(lastErr.Error())
-		}
-		lastErr = err
-	}
-	if exitCode != 0 {
-		if lastErr != nil {
-			logrus.Errorf(lastErr.Error())
-		}
-		lastErr = errors.Wrapf(define.ErrOCIRuntime, "non zero exit code: %d", exitCode)
-	}
-
-	// Lock again
-	if !c.batched {
-		c.lock.Lock()
-	}
-
-	// Sync the container again to pick up changes in state
-	if err := c.syncContainer(); err != nil {
-		logrus.Errorf("error syncing container %s state to remove exec session %s", c.ID(), sessionID)
-		return exitCode, lastErr
-	}
-
-	// Remove the exec session from state
-	delete(c.state.ExecSessions, sessionID)
-	if err := c.save(); err != nil {
-		logrus.Errorf("Error removing exec session %s from container %s state: %v", sessionID, c.ID(), err)
-	}
-	return exitCode, lastErr
-}
-
 // AttachStreams contains streams that will be attached to the container
 type AttachStreams struct {
 	// OutputStream will be attached to container's STDOUT
@@ -493,7 +355,11 @@ func (c *Container) Unmount(force bool) error {
 			if c.ensureState(define.ContainerStateRunning, define.ContainerStatePaused) {
 				return errors.Wrapf(define.ErrCtrStateInvalid, "cannot unmount storage for container %s as it is running or paused", c.ID())
 			}
-			if len(c.state.ExecSessions) != 0 {
+			execSessions, err := c.getActiveExecSessions()
+			if err != nil {
+				return err
+			}
+			if len(execSessions) != 0 {
 				return errors.Wrapf(define.ErrCtrStateInvalid, "container %s has active exec sessions, refusing to unmount", c.ID())
 			}
 			return errors.Wrapf(define.ErrInternal, "can't unmount %s last mount, it is still in use", c.ID())
@@ -674,15 +540,15 @@ func (c *Container) Cleanup(ctx context.Context) error {
 
 	// If we didn't restart, we perform a normal cleanup
 
-	// Reap exec sessions first.
-	if err := c.reapExecSessions(); err != nil {
+	// Check for running exec sessions
+	sessions, err := c.getActiveExecSessions()
+	if err != nil {
 		return err
 	}
-
-	// Check if we have active exec sessions after reaping.
-	if len(c.state.ExecSessions) != 0 {
+	if len(sessions) > 0 {
 		return errors.Wrapf(define.ErrCtrStateInvalid, "container %s has active exec sessions, refusing to clean up", c.ID())
 	}
+
 	defer c.newContainerEvent(events.Cleanup)
 	return c.cleanup(ctx)
 }
@@ -757,114 +623,11 @@ func (c *Container) Sync() error {
 	return nil
 }
 
-// Refresh refreshes a container's state in the database, restarting the
-// container if it is running
+// Refresh is DEPRECATED and REMOVED.
 func (c *Container) Refresh(ctx context.Context) error {
-	if !c.batched {
-		c.lock.Lock()
-		defer c.lock.Unlock()
-
-		if err := c.syncContainer(); err != nil {
-			return err
-		}
-	}
-
-	if c.state.State == define.ContainerStateRemoving {
-		return errors.Wrapf(define.ErrCtrStateInvalid, "cannot refresh containers that are being removed")
-	}
-
-	wasCreated := false
-	if c.state.State == define.ContainerStateCreated {
-		wasCreated = true
-	}
-	wasRunning := false
-	if c.state.State == define.ContainerStateRunning {
-		wasRunning = true
-	}
-	wasPaused := false
-	if c.state.State == define.ContainerStatePaused {
-		wasPaused = true
-	}
-
-	// First, unpause the container if it's paused
-	if c.state.State == define.ContainerStatePaused {
-		if err := c.unpause(); err != nil {
-			return err
-		}
-	}
-
-	// Next, if the container is running, stop it
-	if c.state.State == define.ContainerStateRunning {
-		if err := c.stop(c.config.StopTimeout); err != nil {
-			return err
-		}
-	}
-
-	// If there are active exec sessions, we need to kill them
-	if len(c.state.ExecSessions) > 0 {
-		logrus.Infof("Killing %d exec sessions in container %s. They will not be restored after refresh.",
-			len(c.state.ExecSessions), c.ID())
-	}
-	for _, session := range c.state.ExecSessions {
-		if err := c.ociRuntime.ExecStopContainer(c, session.ID, c.StopTimeout()); err != nil {
-			return errors.Wrapf(err, "error stopping exec session %s of container %s", session.ID, c.ID())
-		}
-	}
-
-	// If the container is in ContainerStateStopped, we need to delete it
-	// from the runtime and clear conmon state
-	if c.state.State == define.ContainerStateStopped {
-		if err := c.delete(ctx); err != nil {
-			return err
-		}
-		if err := c.removeConmonFiles(); err != nil {
-			return err
-		}
-	}
-
-	// Fire cleanup code one more time unconditionally to ensure we are good
-	// to refresh
-	if err := c.cleanup(ctx); err != nil {
-		return err
-	}
-
-	logrus.Debugf("Resetting state of container %s", c.ID())
-
-	// We've finished unwinding the container back to its initial state
-	// Now safe to refresh container state
-	if err := resetState(c.state); err != nil {
-		return errors.Wrapf(err, "error resetting state of container %s", c.ID())
-	}
-	if err := c.refresh(); err != nil {
-		return err
-	}
-
-	logrus.Debugf("Successfully refresh container %s state", c.ID())
-
-	// Initialize the container if it was created in runc
-	if wasCreated || wasRunning || wasPaused {
-		if err := c.prepare(); err != nil {
-			return err
-		}
-		if err := c.init(ctx, false); err != nil {
-			return err
-		}
-	}
-
-	// If the container was running before, start it
-	if wasRunning || wasPaused {
-		if err := c.start(); err != nil {
-			return err
-		}
-	}
-
-	// If the container was paused before, re-pause it
-	if wasPaused {
-		if err := c.pause(); err != nil {
-			return err
-		}
-	}
-	return nil
+	// This has been deprecated for a long while, and is in the process of
+	// being removed.
+	return define.ErrNotImplemented
 }
 
 // ContainerCheckpointOptions is a struct used to pass the parameters
diff --git a/libpod/container_exec.go b/libpod/container_exec.go
new file mode 100644
index 000000000..7ed7a3343
--- /dev/null
+++ b/libpod/container_exec.go
@@ -0,0 +1,857 @@
+package libpod
+
+import (
+	"io/ioutil"
+	"os"
+	"path/filepath"
+	"strconv"
+	"time"
+
+	"github.com/containers/common/pkg/capabilities"
+	"github.com/containers/libpod/libpod/define"
+	"github.com/containers/libpod/libpod/events"
+	"github.com/containers/storage/pkg/stringid"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"k8s.io/client-go/tools/remotecommand"
+)
+
+// ExecConfig contains the configuration of an exec session
+type ExecConfig struct {
+	// Command the the command that will be invoked in the exec session.
+	// Must not be empty.
+	Command []string `json:"command"`
+	// Terminal is whether the exec session will allocate a pseudoterminal.
+	Terminal bool `json:"terminal,omitempty"`
+	// AttachStdin is whether the STDIN stream will be forwarded to the exec
+	// session's first process when attaching. Only available if Terminal is
+	// false.
+	AttachStdin bool `json:"attachStdin,omitempty"`
+	// AttachStdout is whether the STDOUT stream will be forwarded to the
+	// exec session's first process when attaching. Only available if
+	// Terminal is false.
+	AttachStdout bool `json:"attachStdout,omitempty"`
+	// AttachStderr is whether the STDERR stream will be forwarded to the
+	// exec session's first process when attaching. Only available if
+	// Terminal is false.
+	AttachStderr bool `json:"attachStderr,omitempty"`
+	// DetachKeys are keys that will be used to detach from the exec
+	// session. Here, nil will use the default detach keys, where a pointer
+	// to the empty string ("") will disable detaching via detach keys.
+	DetachKeys *string `json:"detachKeys,omitempty"`
+	// Environment is a set of environment variables that will be set for
+	// the first process started by the exec session.
+	Environment map[string]string `json:"environment,omitempty"`
+	// Privileged is whether the exec session will be privileged - that is,
+	// will be granted additional capabilities.
+	Privileged bool `json:"privileged,omitempty"`
+	// User is the user the exec session will be run as.
+	// If set to "" the exec session will be started as the same user the
+	// container was started as.
+	User string `json:"user,omitempty"`
+	// WorkDir is the working directory for the first process that will be
+	// launched by the exec session.
+	// If set to "" the exec session will be started in / within the
+	// container.
+	WorkDir string `json:"workDir,omitempty"`
+	// PreserveFDs indicates that a number of extra FDs from the process
+	// running libpod will be passed into the container. These are assumed
+	// to begin at 3 (immediately after the standard streams). The number
+	// given is the number that will be passed into the exec session,
+	// starting at 3.
+	PreserveFDs uint `json:"preserveFds,omitempty"`
+}
+
+// ExecSession contains information on a single exec session attached to a given
+// container.
+type ExecSession struct {
+	// Id is the ID of the exec session.
+	// Named somewhat strangely to not conflict with ID().
+	Id string `json:"id"`
+	// ContainerId is the ID of the container this exec session belongs to.
+	// Named somewhat strangely to not conflict with ContainerID().
+	ContainerId string `json:"containerId"`
+
+	// State is the state of the exec session.
+	State define.ContainerExecStatus `json:"state"`
+	// PID is the PID of the process created by the exec session.
+	PID int `json:"pid,omitempty"`
+	// ExitCode is the exit code of the exec session, if it has exited.
+	ExitCode int `json:"exitCode,omitempty"`
+
+	// Config is the configuration of this exec session.
+	// Cannot be empty.
+	Config *ExecConfig `json:"config"`
+}
+
+// ID returns the ID of an exec session.
+func (e *ExecSession) ID() string {
+	return e.Id
+}
+
+// ContainerID returns the ID of the container this exec session was started in.
+func (e *ExecSession) ContainerID() string {
+	return e.ContainerId
+}
+
+// InspectExecSession contains information about a given exec session.
+type InspectExecSession struct {
+	// CanRemove is legacy and used purely for compatibility reasons.
+	// Will always be set to true, unless the exec session is running.
+	CanRemove bool `json:"CanRemove"`
+	// ContainerID is the ID of the container this exec session is attached
+	// to.
+	ContainerID string `json:"ContainerID"`
+	// DetachKeys are the detach keys used by the exec session.
+	// If set to "" the default keys are being used.
+	// Will show "<none>" if no detach keys are set.
+	DetachKeys string `json:"DetachKeys"`
+	// ExitCode is the exit code of the exec session. Will be set to 0 if
+	// the exec session has not yet exited.
+	ExitCode int `json:"ExitCode"`
+	// ID is the ID of the exec session.
+	ID string `json:"ID"`
+	// OpenStderr is whether the container's STDERR stream will be attached.
+	// Always set to true if the exec session created a TTY.
+	OpenStderr bool `json:"OpenStderr"`
+	// OpenStdin is whether the container's STDIN stream will be attached
+	// to.
+	OpenStdin bool `json:"OpenStdin"`
+	// OpenStdout is whether the container's STDOUT stream will be attached.
+	// Always set to true if the exec session created a TTY.
+	OpenStdout bool `json:"OpenStdout"`
+	// Running is whether the exec session is running.
+	Running bool `json:"Running"`
+	// Pid is the PID of the exec session's process.
+	// Will be set to 0 if the exec session is not running.
+	Pid int `json:"Pid"`
+	// ProcessConfig contains information about the exec session's process.
+	ProcessConfig *InspectExecProcess `json:"ProcessConfig"`
+}
+
+// InspectExecProcess contains information about the process in a given exec
+// session.
+type InspectExecProcess struct {
+	// Arguments are the arguments to the entrypoint command of the exec
+	// session.
+	Arguments []string `json:"arguments"`
+	// Entrypoint is the entrypoint for the exec session (the command that
+	// will be executed in the container).
+	Entrypoint string `json:"entrypoint"`
+	// Privileged is whether the exec session will be started with elevated
+	// privileges.
+	Privileged bool `json:"privileged"`
+	// Tty is whether the exec session created a terminal.
+	Tty bool `json:"tty"`
+	// User is the user the exec session was started as.
+	User string `json:"user"`
+}
+
+// Inspect inspects the given exec session and produces detailed output on its
+// configuration and current state.
+func (e *ExecSession) Inspect() (*InspectExecSession, error) {
+	if e.Config == nil {
+		return nil, errors.Wrapf(define.ErrInternal, "given exec session does not have a configuration block")
+	}
+
+	output := new(InspectExecSession)
+	output.CanRemove = e.State != define.ExecStateRunning
+	output.ContainerID = e.ContainerId
+	if e.Config.DetachKeys != nil {
+		output.DetachKeys = *e.Config.DetachKeys
+	}
+	output.ExitCode = e.ExitCode
+	output.ID = e.Id
+	output.OpenStderr = e.Config.AttachStderr
+	output.OpenStdin = e.Config.AttachStdin
+	output.OpenStdout = e.Config.AttachStdout
+	output.Running = e.State == define.ExecStateRunning
+	output.Pid = e.PID
+	output.ProcessConfig = new(InspectExecProcess)
+	if len(e.Config.Command) > 0 {
+		output.ProcessConfig.Entrypoint = e.Config.Command[0]
+		if len(e.Config.Command) > 1 {
+			output.ProcessConfig.Arguments = make([]string, 0, len(e.Config.Command)-1)
+			output.ProcessConfig.Arguments = append(output.ProcessConfig.Arguments, e.Config.Command[1:]...)
+		}
+	}
+	output.ProcessConfig.Privileged = e.Config.Privileged
+	output.ProcessConfig.Tty = e.Config.Terminal
+	output.ProcessConfig.User = e.Config.User
+
+	return output, nil
+}
+
+// legacyExecSession contains information on an active exec session. It is a
+// holdover from a previous Podman version and is DEPRECATED.
+type legacyExecSession struct {
+	ID      string   `json:"id"`
+	Command []string `json:"command"`
+	PID     int      `json:"pid"`
+}
+
+// ExecCreate creates a new exec session for the container.
+// The session is not started. The ID of the new exec session will be returned.
+func (c *Container) ExecCreate(config *ExecConfig) (string, error) {
+	if !c.batched {
+		c.lock.Lock()
+		defer c.lock.Unlock()
+
+		if err := c.syncContainer(); err != nil {
+			return "", err
+		}
+	}
+
+	// Verify our config
+	if config == nil {
+		return "", errors.Wrapf(define.ErrInvalidArg, "must provide a configuration to ExecCreate")
+	}
+	if len(config.Command) == 0 {
+		return "", errors.Wrapf(define.ErrInvalidArg, "must provide a non-empty command to start an exec session")
+	}
+	if config.Terminal && (config.AttachStdin || config.AttachStdout || config.AttachStderr) {
+		return "", errors.Wrapf(define.ErrInvalidArg, "cannot specify streams to attach to when exec session has a pseudoterminal")
+	}
+
+	// Generate an ID for our new exec session
+	sessionID := stringid.GenerateNonCryptoID()
+	found := true
+	// This really ought to be a do-while, but Go doesn't have those...
+	for found {
+		found = false
+		for id := range c.state.ExecSessions {
+			if id == sessionID {
+				found = true
+				break
+			}
+		}
+		if found {
+			sessionID = stringid.GenerateNonCryptoID()
+		}
+	}
+
+	// Make our new exec session
+	session := new(ExecSession)
+	session.Id = sessionID
+	session.ContainerId = c.ID()
+	session.State = define.ExecStateCreated
+	session.Config = new(ExecConfig)
+	if err := JSONDeepCopy(config, session.Config); err != nil {
+		return "", errors.Wrapf(err, "error copying exec configuration into exec session")
+	}
+
+	if c.state.ExecSessions == nil {
+		c.state.ExecSessions = make(map[string]*ExecSession)
+	}
+
+	// Need to add to container state and exec session registry
+	c.state.ExecSessions[session.ID()] = session
+	if err := c.save(); err != nil {
+		return "", err
+	}
+	if err := c.runtime.state.AddExecSession(c, session); err != nil {
+		return "", err
+	}
+
+	logrus.Infof("Created exec session %s in container %s", session.ID(), c.ID())
+
+	return sessionID, nil
+}
+
+// ExecStart starts an exec session in the container, but does not attach to it.
+// Returns immediately upon starting the exec session.
+func (c *Container) ExecStart(sessionID string) error {
+	// Will be implemented in part 2, migrating Start and implementing
+	// detached Start.
+	return define.ErrNotImplemented
+}
+
+// ExecStartAndAttach starts and attaches to an exec session in a container.
+// TODO: Should we include detach keys in the signature to allow override?
+// TODO: How do we handle AttachStdin/AttachStdout/AttachStderr?
+func (c *Container) ExecStartAndAttach(sessionID string, streams *AttachStreams) error {
+	if !c.batched {
+		c.lock.Lock()
+		defer c.lock.Unlock()
+
+		if err := c.syncContainer(); err != nil {
+			return err
+		}
+	}
+
+	session, ok := c.state.ExecSessions[sessionID]
+	if !ok {
+		return errors.Wrapf(define.ErrNoSuchExecSession, "container %s has no exec session with ID %s", c.ID(), sessionID)
+	}
+
+	if session.State != define.ExecStateCreated {
+		return errors.Wrapf(define.ErrExecSessionStateInvalid, "can only start created exec sessions, while container %s session %s state is %q", c.ID(), session.ID(), session.State.String())
+	}
+
+	logrus.Infof("Going to start container %s exec session %s and attach to it", c.ID(), session.ID())
+
+	// TODO: check logic here - should we set Privileged if the container is
+	// privileged?
+	var capList []string
+	if session.Config.Privileged || c.config.Privileged {
+		capList = capabilities.AllCapabilities()
+	}
+
+	user := c.config.User
+	if session.Config.User != "" {
+		user = session.Config.User
+	}
+
+	if err := c.createExecBundle(session.ID()); err != nil {
+		return err
+	}
+
+	opts := new(ExecOptions)
+	opts.Cmd = session.Config.Command
+	opts.CapAdd = capList
+	opts.Env = session.Config.Environment
+	opts.Terminal = session.Config.Terminal
+	opts.Cwd = session.Config.WorkDir
+	opts.User = user
+	opts.Streams = streams
+	opts.PreserveFDs = session.Config.PreserveFDs
+	opts.DetachKeys = session.Config.DetachKeys
+
+	pid, attachChan, err := c.ociRuntime.ExecContainer(c, session.ID(), opts)
+	if err != nil {
+		return err
+	}
+
+	c.newContainerEvent(events.Exec)
+	logrus.Debugf("Successfully started exec session %s in container %s", session.ID(), c.ID())
+
+	var lastErr error
+
+	// Update and save session to reflect PID/running
+	session.PID = pid
+	session.State = define.ExecStateRunning
+
+	if err := c.save(); err != nil {
+		lastErr = err
+	}
+
+	// Unlock so other processes can use the container
+	if !c.batched {
+		c.lock.Unlock()
+	}
+
+	tmpErr := <-attachChan
+	if lastErr != nil {
+		logrus.Errorf("Container %s exec session %s error: %v", c.ID(), session.ID(), lastErr)
+	}
+	lastErr = tmpErr
+
+	exitCode, err := c.readExecExitCode(session.ID())
+	if err != nil {
+		if lastErr != nil {
+			logrus.Errorf("Container %s exec session %s error: %v", c.ID(), session.ID(), lastErr)
+		}
+		lastErr = err
+	}
+
+	logrus.Debugf("Container %s exec session %s completed with exit code %d", c.ID(), session.ID(), exitCode)
+
+	// Lock again
+	if !c.batched {
+		c.lock.Lock()
+	}
+
+	// Sync the container to pick up state changes
+	if err := c.syncContainer(); err != nil {
+		if lastErr != nil {
+			logrus.Errorf("Container %s exec session %s error: %v", c.ID(), session.ID(), lastErr)
+		}
+		return errors.Wrapf(err, "error syncing container %s state to remove exec session %s", c.ID(), session.ID())
+	}
+
+	// Update status
+	// Since we did a syncContainer, the old session has been overwritten.
+	// Grab a fresh one from the database.
+	session, ok = c.state.ExecSessions[sessionID]
+	if !ok {
+		// Exec session already removed.
+		logrus.Infof("Container %s exec session %s already removed from database", c.ID(), sessionID)
+		return nil
+	}
+	session.State = define.ExecStateStopped
+	session.ExitCode = exitCode
+	session.PID = 0
+
+	if err := c.save(); err != nil {
+		if lastErr != nil {
+			logrus.Errorf("Container %s exec session %s error: %v", c.ID(), session.ID(), lastErr)
+		}
+		lastErr = err
+	}
+
+	// Clean up after ourselves
+	if err := c.cleanupExecBundle(session.ID()); err != nil {
+		if lastErr != nil {
+			logrus.Errorf("Container %s exec session %s error: %v", c.ID(), session.ID(), lastErr)
+		}
+		lastErr = err
+	}
+
+	return lastErr
+}
+
+// ExecHTTPStartAndAttach starts and performs an HTTP attach to an exec session.
+func (c *Container) ExecHTTPStartAndAttach(sessionID string) error {
+	// Will be implemented in part 2, migrating Start.
+	return define.ErrNotImplemented
+}
+
+// ExecStop stops an exec session in the container.
+// If a timeout is provided, it will be used; otherwise, the timeout will
+// default to the stop timeout of the container.
+// Cleanup will be invoked automatically once the session is stopped.
+func (c *Container) ExecStop(sessionID string, timeout *uint) error {
+	if !c.batched {
+		c.lock.Lock()
+		defer c.lock.Unlock()
+
+		if err := c.syncContainer(); err != nil {
+			return err
+		}
+	}
+
+	session, ok := c.state.ExecSessions[sessionID]
+	if !ok {
+		return errors.Wrapf(define.ErrNoSuchExecSession, "container %s has no exec session with ID %s", c.ID(), sessionID)
+	}
+
+	if session.State != define.ExecStateRunning {
+		return errors.Wrapf(define.ErrExecSessionStateInvalid, "container %s exec session %s is %q, can only stop running sessions", c.ID(), session.ID(), session.State.String())
+	}
+
+	logrus.Infof("Stopping container %s exec session %s", c.ID(), session.ID())
+
+	finalTimeout := c.StopTimeout()
+	if timeout != nil {
+		finalTimeout = *timeout
+	}
+
+	// Stop the session
+	if err := c.ociRuntime.ExecStopContainer(c, session.ID(), finalTimeout); err != nil {
+		return err
+	}
+
+	var cleanupErr error
+
+	// Retrieve exit code and update status
+	exitCode, err := c.readExecExitCode(session.ID())
+	if err != nil {
+		cleanupErr = err
+	}
+	session.ExitCode = exitCode
+	session.PID = 0
+	session.State = define.ExecStateStopped
+
+	if err := c.save(); err != nil {
+		if cleanupErr != nil {
+			logrus.Errorf("Error stopping container %s exec session %s: %v", c.ID(), session.ID(), cleanupErr)
+		}
+		cleanupErr = err
+	}
+
+	if err := c.cleanupExecBundle(session.ID()); err != nil {
+		if cleanupErr != nil {
+			logrus.Errorf("Error stopping container %s exec session %s: %v", c.ID(), session.ID(), cleanupErr)
+		}
+		cleanupErr = err
+	}
+
+	return cleanupErr
+}
+
+// ExecCleanup cleans up an exec session in the container, removing temporary
+// files associated with it.
+func (c *Container) ExecCleanup(sessionID string) error {
+	if !c.batched {
+		c.lock.Lock()
+		defer c.lock.Unlock()
+
+		if err := c.syncContainer(); err != nil {
+			return err
+		}
+	}
+
+	session, ok := c.state.ExecSessions[sessionID]
+	if !ok {
+		return errors.Wrapf(define.ErrNoSuchExecSession, "container %s has no exec session with ID %s", c.ID(), sessionID)
+	}
+
+	if session.State == define.ExecStateRunning {
+		return errors.Wrapf(define.ErrExecSessionStateInvalid, "cannot clean up container %s exec session %s as it is running", c.ID(), session.ID())
+	}
+
+	logrus.Infof("Cleaning up container %s exec session %s", c.ID(), session.ID())
+
+	return c.cleanupExecBundle(session.ID())
+}
+
+// ExecRemove removes an exec session in the container.
+// If force is given, the session will be stopped first if it is running.
+func (c *Container) ExecRemove(sessionID string, force bool) error {
+	if !c.batched {
+		c.lock.Lock()
+		defer c.lock.Unlock()
+
+		if err := c.syncContainer(); err != nil {
+			return err
+		}
+	}
+
+	session, ok := c.state.ExecSessions[sessionID]
+	if !ok {
+		return errors.Wrapf(define.ErrNoSuchExecSession, "container %s has no exec session with ID %s", c.ID(), sessionID)
+	}
+
+	logrus.Infof("Removing container %s exec session %s", c.ID(), session.ID())
+
+	// Update status of exec session if running, so we cna check if it
+	// stopped in the meantime.
+	if session.State == define.ExecStateRunning {
+		stopped, err := c.ociRuntime.ExecUpdateStatus(c, session.ID())
+		if err != nil {
+			return err
+		}
+		if stopped {
+			session.State = define.ExecStateStopped
+			// TODO: should we retrieve exit code here?
+			// TODO: Might be worth saving state here.
+		}
+	}
+
+	if session.State == define.ExecStateRunning {
+		if !force {
+			return errors.Wrapf(define.ErrExecSessionStateInvalid, "container %s exec session %s is still running, cannot remove", c.ID(), session.ID())
+		}
+
+		// Stop the session
+		if err := c.ociRuntime.ExecStopContainer(c, session.ID(), c.StopTimeout()); err != nil {
+			return err
+		}
+
+		if err := c.cleanupExecBundle(session.ID()); err != nil {
+			return err
+		}
+	}
+
+	// First remove exec session from DB.
+	if err := c.runtime.state.RemoveExecSession(session); err != nil {
+		return err
+	}
+	// Next, remove it from the container and save state
+	delete(c.state.ExecSessions, sessionID)
+	if err := c.save(); err != nil {
+		return err
+	}
+
+	logrus.Debugf("Successfully removed container %s exec session %s", c.ID(), session.ID())
+
+	return nil
+}
+
+// ExecResize resizes the TTY of the given exec session. Only available if the
+// exec session created a TTY.
+func (c *Container) ExecResize(sessionID string, newSize remotecommand.TerminalSize) error {
+	if !c.batched {
+		c.lock.Lock()
+		defer c.lock.Unlock()
+
+		if err := c.syncContainer(); err != nil {
+			return err
+		}
+	}
+
+	session, ok := c.state.ExecSessions[sessionID]
+	if !ok {
+		return errors.Wrapf(define.ErrNoSuchExecSession, "container %s has no exec session with ID %s", c.ID(), sessionID)
+	}
+
+	logrus.Infof("Removing container %s exec session %s", c.ID(), session.ID())
+
+	if session.State != define.ExecStateRunning {
+		return errors.Wrapf(define.ErrExecSessionStateInvalid, "cannot resize container %s exec session %s as it is not running", c.ID(), session.ID())
+	}
+
+	return c.ociRuntime.ExecAttachResize(c, sessionID, newSize)
+}
+
+// Exec emulates the old Libpod exec API, providing a single call to create,
+// run, and remove an exec session. Returns exit code and error. Exit code is
+// not guaranteed to be set sanely if error is not nil.
+func (c *Container) Exec(config *ExecConfig, streams *AttachStreams, resize <-chan remotecommand.TerminalSize) (int, error) {
+	sessionID, err := c.ExecCreate(config)
+	if err != nil {
+		return -1, err
+	}
+	if err := c.ExecStartAndAttach(sessionID, streams); err != nil {
+		return -1, err
+	}
+
+	// Start resizing if we have a resize channel.
+	// This goroutine may likely leak, given that we cannot close it here.
+	// Not a big deal, since it should run for as long as the Podman process
+	// does. Could be a big deal for `podman service` but we don't need this
+	// API there.
+	// TODO: Refactor so this is closed here, before we remove the exec
+	// session.
+	if resize != nil {
+		go func() {
+			for resizeRequest := range resize {
+				if err := c.ExecResize(sessionID, resizeRequest); err != nil {
+					// Assume the exec session went down.
+					logrus.Warnf("Error resizing exec session %s: %v", sessionID, err)
+					return
+				}
+			}
+		}()
+	}
+
+	session, err := c.ExecSession(sessionID)
+	if err != nil {
+		return -1, err
+	}
+	exitCode := session.ExitCode
+	if err := c.ExecRemove(sessionID, false); err != nil {
+		return -1, err
+	}
+
+	if exitCode != 0 {
+		return exitCode, errors.Wrapf(define.ErrOCIRuntime, "exec session exited with non-zero exit code %d", exitCode)
+	}
+
+	return exitCode, nil
+}
+
+// cleanup an exec session after its done
+func (c *Container) cleanupExecBundle(sessionID string) error {
+	if err := os.RemoveAll(c.execBundlePath(sessionID)); err != nil && !os.IsNotExist(err) {
+		return err
+	}
+
+	return c.ociRuntime.ExecContainerCleanup(c, sessionID)
+}
+
+// the path to a containers exec session bundle
+func (c *Container) execBundlePath(sessionID string) string {
+	return filepath.Join(c.bundlePath(), sessionID)
+}
+
+// Get PID file path for a container's exec session
+func (c *Container) execPidPath(sessionID string) string {
+	return filepath.Join(c.execBundlePath(sessionID), "exec_pid")
+}
+
+// the log path for an exec session
+func (c *Container) execLogPath(sessionID string) string {
+	return filepath.Join(c.execBundlePath(sessionID), "exec_log")
+}
+
+// the socket conmon creates for an exec session
+func (c *Container) execAttachSocketPath(sessionID string) (string, error) {
+	return c.ociRuntime.ExecAttachSocketPath(c, sessionID)
+}
+
+// execExitFileDir gets the path to the container's exit file
+func (c *Container) execExitFileDir(sessionID string) string {
+	return filepath.Join(c.execBundlePath(sessionID), "exit")
+}
+
+// execOCILog returns the file path for the exec sessions oci log
+func (c *Container) execOCILog(sessionID string) string {
+	if !c.ociRuntime.SupportsJSONErrors() {
+		return ""
+	}
+	return filepath.Join(c.execBundlePath(sessionID), "oci-log")
+}
+
+// create a bundle path and associated files for an exec session
+func (c *Container) createExecBundle(sessionID string) (err error) {
+	bundlePath := c.execBundlePath(sessionID)
+	if createErr := os.MkdirAll(bundlePath, execDirPermission); createErr != nil {
+		return createErr
+	}
+	defer func() {
+		if err != nil {
+			if err2 := os.RemoveAll(bundlePath); err != nil {
+				logrus.Warnf("error removing exec bundle after creation caused another error: %v", err2)
+			}
+		}
+	}()
+	if err2 := os.MkdirAll(c.execExitFileDir(sessionID), execDirPermission); err2 != nil {
+		// The directory is allowed to exist
+		if !os.IsExist(err2) {
+			err = errors.Wrapf(err2, "error creating OCI runtime exit file path %s", c.execExitFileDir(sessionID))
+		}
+	}
+	return
+}
+
+// readExecExitCode reads the exit file for an exec session and returns
+// the exit code
+func (c *Container) readExecExitCode(sessionID string) (int, error) {
+	exitFile := filepath.Join(c.execExitFileDir(sessionID), c.ID())
+	chWait := make(chan error)
+	defer close(chWait)
+
+	_, err := WaitForFile(exitFile, chWait, time.Second*5)
+	if err != nil {
+		return -1, err
+	}
+	ec, err := ioutil.ReadFile(exitFile)
+	if err != nil {
+		return -1, err
+	}
+	ecInt, err := strconv.Atoi(string(ec))
+	if err != nil {
+		return -1, err
+	}
+	return ecInt, nil
+}
+
+// getExecSessionPID gets the PID of an active exec session
+func (c *Container) getExecSessionPID(sessionID string) (int, error) {
+	session, ok := c.state.ExecSessions[sessionID]
+	if ok {
+		return session.PID, nil
+	}
+	oldSession, ok := c.state.LegacyExecSessions[sessionID]
+	if ok {
+		return oldSession.PID, nil
+	}
+
+	return -1, errors.Wrapf(define.ErrNoSuchExecSession, "no exec session with ID %s found in container %s", sessionID, c.ID())
+}
+
+// getKnownExecSessions gets a list of all exec sessions we think are running,
+// but does not verify their current state.
+// Please use getActiveExecSessions() outside of container_exec.go, as this
+// function performs further checks to return an accurate list.
+func (c *Container) getKnownExecSessions() []string {
+	knownSessions := []string{}
+	// First check legacy sessions.
+	// TODO: This is DEPRECATED and will be removed in a future major
+	// release.
+	for sessionID := range c.state.LegacyExecSessions {
+		knownSessions = append(knownSessions, sessionID)
+	}
+	// Next check new exec sessions, but only if in running state
+	for sessionID, session := range c.state.ExecSessions {
+		if session.State == define.ExecStateRunning {
+			knownSessions = append(knownSessions, sessionID)
+		}
+	}
+
+	return knownSessions
+}
+
+// getActiveExecSessions checks if there are any active exec sessions in the
+// current container. Returns an array of active exec sessions.
+// Will continue through errors where possible.
+// Currently handles both new and legacy, deprecated exec sessions.
+func (c *Container) getActiveExecSessions() ([]string, error) {
+	activeSessions := []string{}
+	knownSessions := c.getKnownExecSessions()
+
+	// Instead of saving once per iteration, do it once at the end.
+	var lastErr error
+	needSave := false
+	for _, id := range knownSessions {
+		alive, err := c.ociRuntime.ExecUpdateStatus(c, id)
+		if err != nil {
+			if lastErr != nil {
+				logrus.Errorf("Error checking container %s exec sessions: %v", c.ID(), lastErr)
+			}
+			lastErr = err
+			continue
+		}
+		if !alive {
+			if err := c.cleanupExecBundle(id); err != nil {
+				if lastErr != nil {
+					logrus.Errorf("Error checking container %s exec sessions: %v", c.ID(), lastErr)
+				}
+				lastErr = err
+			}
+
+			_, isLegacy := c.state.LegacyExecSessions[id]
+			if isLegacy {
+				delete(c.state.LegacyExecSessions, id)
+				needSave = true
+			} else {
+				session := c.state.ExecSessions[id]
+				exitCode, err := c.readExecExitCode(session.ID())
+				if err != nil {
+					if lastErr != nil {
+						logrus.Errorf("Error checking container %s exec sessions: %v", c.ID(), lastErr)
+					}
+					lastErr = err
+				}
+				session.ExitCode = exitCode
+				session.PID = 0
+				session.State = define.ExecStateStopped
+
+				needSave = true
+			}
+		} else {
+			activeSessions = append(activeSessions, id)
+		}
+	}
+	if needSave {
+		if err := c.save(); err != nil {
+			if lastErr != nil {
+				logrus.Errorf("Error reaping exec sessions for container %s: %v", c.ID(), lastErr)
+			}
+			lastErr = err
+		}
+	}
+
+	return activeSessions, lastErr
+}
+
+// removeAllExecSessions stops and removes all the container's exec sessions
+func (c *Container) removeAllExecSessions() error {
+	knownSessions := c.getKnownExecSessions()
+
+	var lastErr error
+	for _, id := range knownSessions {
+		if err := c.ociRuntime.ExecStopContainer(c, id, c.StopTimeout()); err != nil {
+			if lastErr != nil {
+				logrus.Errorf("Error stopping container %s exec sessions: %v", c.ID(), lastErr)
+			}
+			lastErr = err
+			continue
+		}
+
+		if err := c.cleanupExecBundle(id); err != nil {
+			if lastErr != nil {
+				logrus.Errorf("Error stopping container %s exec sessions: %v", c.ID(), lastErr)
+			}
+			lastErr = err
+		}
+	}
+	// Delete all exec sessions
+	if err := c.runtime.state.RemoveContainerExecSessions(c); err != nil {
+		if lastErr != nil {
+			logrus.Errorf("Error stopping container %s exec sessions: %v", c.ID(), lastErr)
+		}
+		lastErr = err
+	}
+	c.state.ExecSessions = nil
+	c.state.LegacyExecSessions = nil
+	if err := c.save(); err != nil {
+		if lastErr != nil {
+			logrus.Errorf("Error stopping container %s exec sessions: %v", c.ID(), lastErr)
+		}
+		lastErr = err
+	}
+
+	return lastErr
+}
diff --git a/libpod/container_internal.go b/libpod/container_internal.go
index a0805c1fa..12a13a0ce 100644
--- a/libpod/container_internal.go
+++ b/libpod/container_internal.go
@@ -142,92 +142,6 @@ func (c *Container) exitFilePath() (string, error) {
 	return c.ociRuntime.ExitFilePath(c)
 }
 
-// create a bundle path and associated files for an exec session
-func (c *Container) createExecBundle(sessionID string) (err error) {
-	bundlePath := c.execBundlePath(sessionID)
-	if createErr := os.MkdirAll(bundlePath, execDirPermission); createErr != nil {
-		return createErr
-	}
-	defer func() {
-		if err != nil {
-			if err2 := os.RemoveAll(bundlePath); err != nil {
-				logrus.Warnf("error removing exec bundle after creation caused another error: %v", err2)
-			}
-		}
-	}()
-	if err2 := os.MkdirAll(c.execExitFileDir(sessionID), execDirPermission); err2 != nil {
-		// The directory is allowed to exist
-		if !os.IsExist(err2) {
-			err = errors.Wrapf(err2, "error creating OCI runtime exit file path %s", c.execExitFileDir(sessionID))
-		}
-	}
-	return
-}
-
-// cleanup an exec session after its done
-func (c *Container) cleanupExecBundle(sessionID string) error {
-	if err := os.RemoveAll(c.execBundlePath(sessionID)); err != nil && !os.IsNotExist(err) {
-		return err
-	}
-
-	return c.ociRuntime.ExecContainerCleanup(c, sessionID)
-}
-
-// the path to a containers exec session bundle
-func (c *Container) execBundlePath(sessionID string) string {
-	return filepath.Join(c.bundlePath(), sessionID)
-}
-
-// Get PID file path for a container's exec session
-func (c *Container) execPidPath(sessionID string) string {
-	return filepath.Join(c.execBundlePath(sessionID), "exec_pid")
-}
-
-// the log path for an exec session
-func (c *Container) execLogPath(sessionID string) string {
-	return filepath.Join(c.execBundlePath(sessionID), "exec_log")
-}
-
-// the socket conmon creates for an exec session
-func (c *Container) execAttachSocketPath(sessionID string) (string, error) {
-	return c.ociRuntime.ExecAttachSocketPath(c, sessionID)
-}
-
-// execExitFileDir gets the path to the container's exit file
-func (c *Container) execExitFileDir(sessionID string) string {
-	return filepath.Join(c.execBundlePath(sessionID), "exit")
-}
-
-// execOCILog returns the file path for the exec sessions oci log
-func (c *Container) execOCILog(sessionID string) string {
-	if !c.ociRuntime.SupportsJSONErrors() {
-		return ""
-	}
-	return filepath.Join(c.execBundlePath(sessionID), "oci-log")
-}
-
-// readExecExitCode reads the exit file for an exec session and returns
-// the exit code
-func (c *Container) readExecExitCode(sessionID string) (int, error) {
-	exitFile := filepath.Join(c.execExitFileDir(sessionID), c.ID())
-	chWait := make(chan error)
-	defer close(chWait)
-
-	_, err := WaitForFile(exitFile, chWait, time.Second*5)
-	if err != nil {
-		return -1, err
-	}
-	ec, err := ioutil.ReadFile(exitFile)
-	if err != nil {
-		return -1, err
-	}
-	ecInt, err := strconv.Atoi(string(ec))
-	if err != nil {
-		return -1, err
-	}
-	return ecInt, nil
-}
-
 // Wait for the container's exit file to appear.
 // When it does, update our state based on it.
 func (c *Container) waitForExitFileAndSync() error {
@@ -568,6 +482,7 @@ func resetState(state *ContainerState) error {
 		state.State = define.ContainerStateConfigured
 	}
 	state.ExecSessions = make(map[string]*ExecSession)
+	state.LegacyExecSessions = nil
 	state.NetworkStatus = nil
 	state.BindMounts = make(map[string]string)
 	state.StoppedByUser = false
@@ -1814,12 +1729,12 @@ func (c *Container) checkReadyForRemoval() error {
 		return errors.Wrapf(define.ErrCtrStateInvalid, "cannot remove container %s as it is %s - running or paused containers cannot be removed without force", c.ID(), c.state.State.String())
 	}
 
-	// Reap exec sessions
-	if err := c.reapExecSessions(); err != nil {
+	// Check exec sessions
+	sessions, err := c.getActiveExecSessions()
+	if err != nil {
 		return err
 	}
-
-	if len(c.state.ExecSessions) != 0 {
+	if len(sessions) != 0 {
 		return errors.Wrapf(define.ErrCtrStateInvalid, "cannot remove container %s as it has active exec sessions", c.ID())
 	}
 
@@ -1926,41 +1841,6 @@ func (c *Container) checkExitFile() error {
 	return c.handleExitFile(exitFile, info)
 }
 
-// Reap dead exec sessions
-func (c *Container) reapExecSessions() error {
-	// Instead of saving once per iteration, use a defer to do it once at
-	// the end.
-	var lastErr error
-	needSave := false
-	for id := range c.state.ExecSessions {
-		alive, err := c.ociRuntime.ExecUpdateStatus(c, id)
-		if err != nil {
-			if lastErr != nil {
-				logrus.Errorf("Error reaping exec sessions for container %s: %v", c.ID(), lastErr)
-			}
-			lastErr = err
-			continue
-		}
-		if !alive {
-			// Clean up lingering files and remove the exec session
-			if err := c.ociRuntime.ExecContainerCleanup(c, id); err != nil {
-				return errors.Wrapf(err, "error cleaning up container %s exec session %s files", c.ID(), id)
-			}
-			delete(c.state.ExecSessions, id)
-			needSave = true
-		}
-	}
-	if needSave {
-		if err := c.save(); err != nil {
-			if lastErr != nil {
-				logrus.Errorf("Error reaping exec sessions for container %s: %v", c.ID(), lastErr)
-			}
-			lastErr = err
-		}
-	}
-	return lastErr
-}
-
 func (c *Container) hasNamespace(namespace spec.LinuxNamespaceType) bool {
 	if c.config.Spec == nil || c.config.Spec.Linux == nil {
 		return false
diff --git a/libpod/container_top_linux.go b/libpod/container_top_linux.go
index 98edc340a..2a35a2ae9 100644
--- a/libpod/container_top_linux.go
+++ b/libpod/container_top_linux.go
@@ -134,7 +134,9 @@ func (c *Container) execPS(args []string) ([]string, error) {
 	}()
 
 	cmd := append([]string{"ps"}, args...)
-	ec, err := c.Exec(false, false, map[string]string{}, cmd, "", "", streams, 0, nil, "")
+	config := new(ExecConfig)
+	config.Command = cmd
+	ec, err := c.Exec(config, streams, nil)
 	if err != nil {
 		return nil, err
 	} else if ec != 0 {
diff --git a/libpod/define/containerstate.go b/libpod/define/containerstate.go
index e7d258e21..6da49a594 100644
--- a/libpod/define/containerstate.go
+++ b/libpod/define/containerstate.go
@@ -78,3 +78,37 @@ func StringToContainerStatus(status string) (ContainerStatus, error) {
 		return ContainerStateUnknown, errors.Wrapf(ErrInvalidArg, "unknown container state: %s", status)
 	}
 }
+
+// ContainerExecStatus is the status of an exec session within a container.
+type ContainerExecStatus int
+
+const (
+	// ExecStateUnknown indicates that the state of the exec session is not
+	// known.
+	ExecStateUnknown ContainerExecStatus = iota
+	// ExecStateCreated indicates that the exec session has been created but
+	// not yet started
+	ExecStateCreated ContainerExecStatus = iota
+	// ExecStateRunning indicates that the exec session has been started but
+	// has not yet exited.
+	ExecStateRunning ContainerExecStatus = iota
+	// ExecStateStopped indicates that the exec session has stopped and is
+	// no longer running.
+	ExecStateStopped ContainerExecStatus = iota
+)
+
+// String returns a string representation of a given exec state.
+func (s ContainerExecStatus) String() string {
+	switch s {
+	case ExecStateUnknown:
+		return "unknown"
+	case ExecStateCreated:
+		return "created"
+	case ExecStateRunning:
+		return "running"
+	case ExecStateStopped:
+		return "stopped"
+	default:
+		return "bad state"
+	}
+}
diff --git a/libpod/define/errors.go b/libpod/define/errors.go
index b79cf08dc..3ba343789 100644
--- a/libpod/define/errors.go
+++ b/libpod/define/errors.go
@@ -20,6 +20,10 @@ var (
 	// ErrNoSuchVolume indicates the requested volume does not exist
 	ErrNoSuchVolume = errors.New("no such volume")
 
+	// ErrNoSuchExecSession indicates that the requested exec session does
+	// not exist.
+	ErrNoSuchExecSession = errors.New("no such exec session")
+
 	// ErrCtrExists indicates a container with the same name or ID already
 	// exists
 	ErrCtrExists = errors.New("container already exists")
@@ -29,10 +33,16 @@ var (
 	ErrImageExists = errors.New("image already exists")
 	// ErrVolumeExists indicates a volume with the same name already exists
 	ErrVolumeExists = errors.New("volume already exists")
+	// ErrExecSessionExists indicates an exec session with the same ID
+	// already exists.
+	ErrExecSessionExists = errors.New("exec session already exists")
 
 	// ErrCtrStateInvalid indicates a container is in an improper state for
 	// the requested operation
 	ErrCtrStateInvalid = errors.New("container state improper")
+	// ErrExecSessionStateInvalid indicates that an exec session is in an
+	// improper state for the requested operation
+	ErrExecSessionStateInvalid = errors.New("exec session state improper")
 	// ErrVolumeBeingUsed indicates that a volume is being used by at least one container
 	ErrVolumeBeingUsed = errors.New("volume is being used")
 
@@ -90,6 +100,9 @@ var (
 	// ErrVolumeRemoved indicates that the volume has already been removed and
 	// no further operations can be performed on it
 	ErrVolumeRemoved = errors.New("volume has already been removed")
+	// ErrExecSessionRemoved indicates that the exec session has already
+	// been removed and no further operations can be performed on it.
+	ErrExecSessionRemoved = errors.New("exec session has already been removed")
 
 	// ErrDBClosed indicates that the connection to the state database has
 	// already been closed
diff --git a/libpod/healthcheck.go b/libpod/healthcheck.go
index 9c274c4f3..76b7a1fcf 100644
--- a/libpod/healthcheck.go
+++ b/libpod/healthcheck.go
@@ -143,7 +143,9 @@ func (c *Container) runHealthCheck() (HealthCheckStatus, error) {
 	logrus.Debugf("executing health check command %s for %s", strings.Join(newCommand, " "), c.ID())
 	timeStart := time.Now()
 	hcResult := HealthCheckSuccess
-	_, hcErr := c.Exec(false, false, map[string]string{}, newCommand, "", "", streams, 0, nil, "")
+	config := new(ExecConfig)
+	config.Command = newCommand
+	_, hcErr := c.Exec(config, streams, nil)
 	if hcErr != nil {
 		errCause := errors.Cause(hcErr)
 		hcResult = HealthCheckFailure
diff --git a/libpod/in_memory_state.go b/libpod/in_memory_state.go
index 2144671a5..ca562ab7e 100644
--- a/libpod/in_memory_state.go
+++ b/libpod/in_memory_state.go
@@ -20,10 +20,16 @@ type InMemoryState struct {
 	pods map[string]*Pod
 	// Maps container ID to container struct.
 	containers map[string]*Container
-	volumes    map[string]*Volume
+	// Maps volume ID to volume struct
+	volumes map[string]*Volume
+	// Maps exec session ID to ID of associated container
+	execSessions map[string]string
 	// Maps container ID to a list of IDs of dependencies.
-	ctrDepends    map[string][]string
+	ctrDepends map[string][]string
+	// Maps volume ID to IDs of dependencies
 	volumeDepends map[string][]string
+	// Maps container ID to IDs of associated exec sessions.
+	ctrExecSessions map[string][]string
 	// Maps pod ID to a map of container ID to container struct.
 	podContainers map[string]map[string]*Container
 	// Global name registry - ensures name uniqueness and performs lookups.
@@ -51,10 +57,13 @@ func NewInMemoryState() (State, error) {
 	state.pods = make(map[string]*Pod)
 	state.containers = make(map[string]*Container)
 	state.volumes = make(map[string]*Volume)
+	state.execSessions = make(map[string]string)
 
 	state.ctrDepends = make(map[string][]string)
 	state.volumeDepends = make(map[string][]string)
 
+	state.ctrExecSessions = make(map[string][]string)
+
 	state.podContainers = make(map[string]map[string]*Container)
 
 	state.nameIndex = registrar.NewRegistrar()
@@ -316,6 +325,13 @@ func (s *InMemoryState) RemoveContainer(ctr *Container) error {
 		return errors.Wrapf(define.ErrCtrExists, "the following containers depend on container %s: %s", ctr.ID(), depsStr)
 	}
 
+	// Ensure we don't have active exec sessions
+	ctrSessions := s.ctrExecSessions[ctr.ID()]
+	if len(ctrSessions) > 0 {
+		sessStr := strings.Join(ctrSessions, ", ")
+		return errors.Wrapf(define.ErrCtrExists, "the following exec sessions are running for container %s: %s", ctr.ID(), sessStr)
+	}
+
 	if _, ok := s.containers[ctr.ID()]; !ok {
 		ctr.valid = false
 		return errors.Wrapf(define.ErrNoSuchCtr, "no container exists in state with ID %s", ctr.ID())
@@ -437,6 +453,117 @@ func (s *InMemoryState) GetContainerConfig(id string) (*ContainerConfig, error)
 	return ctr.Config(), nil
 }
 
+// Add an exec session to the database
+func (s *InMemoryState) AddExecSession(ctr *Container, session *ExecSession) error {
+	if !ctr.valid {
+		return define.ErrCtrRemoved
+	}
+	if session.ContainerID() != ctr.ID() {
+		return errors.Wrapf(define.ErrInvalidArg, "container ID and exec session ID must match")
+	}
+	if _, ok := s.containers[ctr.ID()]; !ok {
+		return define.ErrNoSuchCtr
+	}
+
+	if _, ok := s.execSessions[session.ID()]; ok {
+		return define.ErrExecSessionExists
+	}
+
+	s.execSessions[session.ID()] = ctr.ID()
+
+	ctrSessions, ok := s.ctrExecSessions[ctr.ID()]
+	if !ok {
+		ctrSessions = []string{}
+	}
+
+	ctrSessions = append(ctrSessions, session.ID())
+	s.ctrExecSessions[ctr.ID()] = ctrSessions
+
+	return nil
+}
+
+// Get an exec session from the database by full or partial ID.
+func (s *InMemoryState) GetExecSession(id string) (string, error) {
+	if id == "" {
+		return "", define.ErrEmptyID
+	}
+
+	session, ok := s.execSessions[id]
+	if !ok {
+		return "", define.ErrNoSuchExecSession
+	}
+
+	return session, nil
+}
+
+// RemoveExecSession removes an exec session from the database.
+func (s *InMemoryState) RemoveExecSession(session *ExecSession) error {
+	if _, ok := s.execSessions[session.ID()]; !ok {
+		return define.ErrNoSuchExecSession
+	}
+
+	ctrSessions, ok := s.ctrExecSessions[session.ContainerID()]
+	// If !ok - internal state seems inconsistent, but the thing we wanted
+	// to remove is gone. Continue.
+	if ok {
+		newSessions := []string{}
+		for _, sess := range ctrSessions {
+			if sess != session.ID() {
+				newSessions = append(newSessions, sess)
+			}
+		}
+		s.ctrExecSessions[session.ContainerID()] = newSessions
+	}
+
+	delete(s.execSessions, session.ID())
+
+	return nil
+}
+
+// GetContainerExecSessions retrieves all exec sessions for the given container.
+func (s *InMemoryState) GetContainerExecSessions(ctr *Container) ([]string, error) {
+	if !ctr.valid {
+		return nil, define.ErrCtrRemoved
+	}
+	if _, ok := s.containers[ctr.ID()]; !ok {
+		ctr.valid = false
+		return nil, define.ErrNoSuchCtr
+	}
+
+	ctrSessions := s.ctrExecSessions[ctr.ID()]
+
+	return ctrSessions, nil
+}
+
+// RemoveContainerExecSessions removes all exec sessions for the given
+// container.
+func (s *InMemoryState) RemoveContainerExecSessions(ctr *Container) error {
+	if !ctr.valid {
+		return define.ErrCtrRemoved
+	}
+	if _, ok := s.containers[ctr.ID()]; !ok {
+		ctr.valid = false
+		return define.ErrNoSuchCtr
+	}
+
+	ctrSessions, ok := s.ctrExecSessions[ctr.ID()]
+	if !ok {
+		return nil
+	}
+
+	for _, sess := range ctrSessions {
+		if _, ok := s.execSessions[sess]; !ok {
+			// We have an internal state inconsistency
+			// Error out
+			return errors.Wrapf(define.ErrInternal, "inconsistent database state: exec session %s is missing", sess)
+		}
+		delete(s.execSessions, sess)
+	}
+	delete(s.ctrExecSessions, ctr.ID())
+
+	return nil
+}
+
 // RewriteContainerConfig rewrites a container's configuration.
 // This function is DANGEROUS, even with an in-memory state.
 // Please read the full comment on it in state.go before using it.
@@ -1056,6 +1183,13 @@ func (s *InMemoryState) RemoveContainerFromPod(pod *Pod, ctr *Container) error {
 		return errors.Wrapf(define.ErrCtrExists, "the following containers depend on container %s: %s", ctr.ID(), depsStr)
 	}
 
+	// Ensure we don't have active exec sessions
+	ctrSessions := s.ctrExecSessions[ctr.ID()]
+	if len(ctrSessions) > 0 {
+		sessStr := strings.Join(ctrSessions, ", ")
+		return errors.Wrapf(define.ErrCtrExists, "the following exec sessions are running for container %s: %s", ctr.ID(), sessStr)
+	}
+
 	// Retrieve pod containers
 	podCtrs, ok := s.podContainers[pod.ID()]
 	if !ok {
diff --git a/libpod/oci.go b/libpod/oci.go
index 27edebefc..ef46cf5c3 100644
--- a/libpod/oci.go
+++ b/libpod/oci.go
@@ -71,6 +71,9 @@ type OCIRuntime interface {
 	// Returns an int (exit code), error channel (errors from attach), and
 	// error (errors that occurred attempting to start the exec session).
 	ExecContainer(ctr *Container, sessionID string, options *ExecOptions) (int, chan error, error)
+	// ExecAttachResize resizes the terminal of a running exec session. Only
+	// allowed with sessions that were created with a TTY.
+	ExecAttachResize(ctr *Container, sessionID string, newSize remotecommand.TerminalSize) error
 	// ExecStopContainer stops a given exec session in a running container.
 	// SIGTERM with be sent initially, then SIGKILL after the given timeout.
 	// If timeout is 0, SIGKILL will be sent immediately, and SIGTERM will
@@ -143,12 +146,12 @@ type ExecOptions struct {
 	// to 0, 1, 2) that will be passed to the executed process. The total FDs
 	// passed will be 3 + PreserveFDs.
 	PreserveFDs uint
-	// Resize is a channel where terminal resize events are sent to be
-	// handled.
-	Resize chan remotecommand.TerminalSize
 	// DetachKeys is a set of keys that, when pressed in sequence, will
 	// detach from the container.
-	DetachKeys string
+	// If not provided, the default keys will be used.
+	// If provided but set to "", detaching from the container will be
+	// disabled.
+	DetachKeys *string
 }
 
 // HTTPAttachStreams informs the HTTPAttach endpoint which of the container's
diff --git a/libpod/oci_attach_linux.go b/libpod/oci_attach_linux.go
index 46c70e7eb..5fc46c31c 100644
--- a/libpod/oci_attach_linux.go
+++ b/libpod/oci_attach_linux.go
@@ -93,7 +93,7 @@ func (c *Container) attach(streams *AttachStreams, keys string, resize <-chan re
 // 4. attachToExec sends on startFd, signalling it has attached to the socket and child is ready to go
 // 5. child receives on startFd, runs the runtime exec command
 // attachToExec is responsible for closing startFd and attachFd
-func (c *Container) attachToExec(streams *AttachStreams, keys string, resize <-chan remotecommand.TerminalSize, sessionID string, startFd, attachFd *os.File) error {
+func (c *Container) attachToExec(streams *AttachStreams, keys *string, sessionID string, startFd, attachFd *os.File) error {
 	if !streams.AttachOutput && !streams.AttachError && !streams.AttachInput {
 		return errors.Wrapf(define.ErrInvalidArg, "must provide at least one stream to attach to")
 	}
@@ -104,7 +104,11 @@ func (c *Container) attachToExec(streams *AttachStreams, keys string, resize <-c
 	defer errorhandling.CloseQuiet(startFd)
 	defer errorhandling.CloseQuiet(attachFd)
 
-	detachKeys, err := processDetachKeys(keys)
+	detachString := define.DefaultDetachKeys
+	if keys != nil {
+		detachString = *keys
+	}
+	detachKeys, err := processDetachKeys(detachString)
 	if err != nil {
 		return err
 	}
@@ -134,10 +138,6 @@ func (c *Container) attachToExec(streams *AttachStreams, keys string, resize <-c
 		}
 	}()
 
-	// Register the resize func after we've read the attach socket, as we know at this point the
-	// 'ctl' file has been created in conmon
-	registerResizeFunc(resize, c.execBundlePath(sessionID))
-
 	// start listening on stdio of the process
 	receiveStdoutError, stdinDone := setupStdioChannels(streams, conn, detachKeys)
 
diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go
index d3c3bbcc5..82c5d7020 100644
--- a/libpod/oci_conmon_linux.go
+++ b/libpod/oci_conmon_linux.go
@@ -769,7 +769,7 @@ func (r *ConmonOCIRuntime) ExecContainer(c *Container, sessionID string, options
 	attachChan := make(chan error)
 	go func() {
 		// attachToExec is responsible for closing pipes
-		attachChan <- c.attachToExec(options.Streams, options.DetachKeys, options.Resize, sessionID, parentStartPipe, parentAttachPipe)
+		attachChan <- c.attachToExec(options.Streams, options.DetachKeys, sessionID, parentStartPipe, parentAttachPipe)
 		close(attachChan)
 	}()
 	attachToExecCalled = true
@@ -783,37 +783,54 @@ func (r *ConmonOCIRuntime) ExecContainer(c *Container, sessionID string, options
 	return pid, attachChan, err
 }
 
+// ExecAttachResize resizes the TTY of the given exec session.
+func (r *ConmonOCIRuntime) ExecAttachResize(ctr *Container, sessionID string, newSize remotecommand.TerminalSize) error {
+	// TODO: probably want a dedicated function to get ctl file path?
+	controlPath := filepath.Join(ctr.execBundlePath(sessionID), "ctl")
+	controlFile, err := os.OpenFile(controlPath, unix.O_WRONLY, 0)
+	if err != nil {
+		return errors.Wrapf(err, "could not open ctl file for terminal resize for container %s exec session %s", ctr.ID(), sessionID)
+	}
+	defer controlFile.Close()
+
+	logrus.Debugf("Received a resize event for container %s exec session %s: %+v", ctr.ID(), sessionID, newSize)
+	if _, err = fmt.Fprintf(controlFile, "%d %d %d\n", 1, newSize.Height, newSize.Width); err != nil {
+		return errors.Wrapf(err, "failed to write to ctl file to resize terminal")
+	}
+
+	return nil
+}
+
 // ExecStopContainer stops a given exec session in a running container.
 func (r *ConmonOCIRuntime) ExecStopContainer(ctr *Container, sessionID string, timeout uint) error {
-	session, ok := ctr.state.ExecSessions[sessionID]
-	if !ok {
-		// TODO This should probably be a separate error
-		return errors.Wrapf(define.ErrInvalidArg, "no exec session with ID %s found in container %s", sessionID, ctr.ID())
+	pid, err := ctr.getExecSessionPID(sessionID)
+	if err != nil {
+		return err
 	}
 
 	logrus.Debugf("Going to stop container %s exec session %s", ctr.ID(), sessionID)
 
 	// Is the session dead?
 	// Ping the PID with signal 0 to see if it still exists.
-	if err := unix.Kill(session.PID, 0); err != nil {
+	if err := unix.Kill(pid, 0); err != nil {
 		if err == unix.ESRCH {
 			return nil
 		}
-		return errors.Wrapf(err, "error pinging container %s exec session %s PID %d with signal 0", ctr.ID(), sessionID, session.PID)
+		return errors.Wrapf(err, "error pinging container %s exec session %s PID %d with signal 0", ctr.ID(), sessionID, pid)
 	}
 
 	if timeout > 0 {
 		// Use SIGTERM by default, then SIGSTOP after timeout.
-		logrus.Debugf("Killing exec session %s (PID %d) of container %s with SIGTERM", sessionID, session.PID, ctr.ID())
-		if err := unix.Kill(session.PID, unix.SIGTERM); err != nil {
+		logrus.Debugf("Killing exec session %s (PID %d) of container %s with SIGTERM", sessionID, pid, ctr.ID())
+		if err := unix.Kill(pid, unix.SIGTERM); err != nil {
 			if err == unix.ESRCH {
 				return nil
 			}
-			return errors.Wrapf(err, "error killing container %s exec session %s PID %d with SIGTERM", ctr.ID(), sessionID, session.PID)
+			return errors.Wrapf(err, "error killing container %s exec session %s PID %d with SIGTERM", ctr.ID(), sessionID, pid)
 		}
 
 		// Wait for the PID to stop
-		if err := waitPidStop(session.PID, time.Duration(timeout)*time.Second); err != nil {
+		if err := waitPidStop(pid, time.Duration(timeout)*time.Second); err != nil {
 			logrus.Warnf("Timed out waiting for container %s exec session %s to stop, resorting to SIGKILL", ctr.ID(), sessionID)
 		} else {
 			// No error, container is dead
@@ -822,17 +839,17 @@ func (r *ConmonOCIRuntime) ExecStopContainer(ctr *Container, sessionID string, t
 	}
 
 	// SIGTERM did not work. On to SIGKILL.
-	logrus.Debugf("Killing exec session %s (PID %d) of container %s with SIGKILL", sessionID, session.PID, ctr.ID())
-	if err := unix.Kill(session.PID, unix.SIGTERM); err != nil {
+	logrus.Debugf("Killing exec session %s (PID %d) of container %s with SIGKILL", sessionID, pid, ctr.ID())
+	if err := unix.Kill(pid, unix.SIGTERM); err != nil {
 		if err == unix.ESRCH {
 			return nil
 		}
-		return errors.Wrapf(err, "error killing container %s exec session %s PID %d with SIGKILL", ctr.ID(), sessionID, session.PID)
+		return errors.Wrapf(err, "error killing container %s exec session %s PID %d with SIGKILL", ctr.ID(), sessionID, pid)
 	}
 
 	// Wait for the PID to stop
-	if err := waitPidStop(session.PID, killContainerTimeout*time.Second); err != nil {
-		return errors.Wrapf(err, "timed out waiting for container %s exec session %s PID %d to stop after SIGKILL", ctr.ID(), sessionID, session.PID)
+	if err := waitPidStop(pid, killContainerTimeout*time.Second); err != nil {
+		return errors.Wrapf(err, "timed out waiting for container %s exec session %s PID %d to stop after SIGKILL", ctr.ID(), sessionID, pid)
 	}
 
 	return nil
@@ -840,21 +857,20 @@ func (r *ConmonOCIRuntime) ExecStopContainer(ctr *Container, sessionID string, t
 
 // ExecUpdateStatus checks if the given exec session is still running.
 func (r *ConmonOCIRuntime) ExecUpdateStatus(ctr *Container, sessionID string) (bool, error) {
-	session, ok := ctr.state.ExecSessions[sessionID]
-	if !ok {
-		// TODO This should probably be a separate error
-		return false, errors.Wrapf(define.ErrInvalidArg, "no exec session with ID %s found in container %s", sessionID, ctr.ID())
+	pid, err := ctr.getExecSessionPID(sessionID)
+	if err != nil {
+		return false, err
 	}
 
 	logrus.Debugf("Checking status of container %s exec session %s", ctr.ID(), sessionID)
 
 	// Is the session dead?
 	// Ping the PID with signal 0 to see if it still exists.
-	if err := unix.Kill(session.PID, 0); err != nil {
+	if err := unix.Kill(pid, 0); err != nil {
 		if err == unix.ESRCH {
 			return false, nil
 		}
-		return false, errors.Wrapf(err, "error pinging container %s exec session %s PID %d with signal 0", ctr.ID(), sessionID, session.PID)
+		return false, errors.Wrapf(err, "error pinging container %s exec session %s PID %d with signal 0", ctr.ID(), sessionID, pid)
 	}
 
 	return true, nil
diff --git a/libpod/oci_missing.go b/libpod/oci_missing.go
index ff7eea625..326591d89 100644
--- a/libpod/oci_missing.go
+++ b/libpod/oci_missing.go
@@ -125,6 +125,11 @@ func (r *MissingRuntime) ExecContainer(ctr *Container, sessionID string, options
 	return -1, nil, r.printError()
 }
 
+// ExecAttachResize is not available as the runtime is missing.
+func (r *MissingRuntime) ExecAttachResize(ctr *Container, sessionID string, newSize remotecommand.TerminalSize) error {
+	return r.printError()
+}
+
 // ExecStopContainer is not available as the runtime is missing.
 // TODO: We can also investigate using unix.Kill() on the PID of the exec
 // session here if we want to make stopping containers possible. Won't be
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
index 1f1cdc271..0b18436ca 100644
--- a/libpod/runtime_ctr.go
+++ b/libpod/runtime_ctr.go
@@ -463,11 +463,9 @@ func (r *Runtime) removeContainer(ctx context.Context, c *Container, force bool,
 		}
 	}
 
-	// Check that all of our exec sessions have finished
-	for _, session := range c.state.ExecSessions {
-		if err := c.ociRuntime.ExecStopContainer(c, session.ID, c.StopTimeout()); err != nil {
-			return errors.Wrapf(err, "error stopping exec session %s of container %s", session.ID, c.ID())
-		}
+	// Remove all active exec sessions
+	if err := c.removeAllExecSessions(); err != nil {
+		return err
 	}
 
 	// Check that no other containers depend on the container.
@@ -484,9 +482,8 @@ func (r *Runtime) removeContainer(ctx context.Context, c *Container, force bool,
 		}
 	}
 
-	// Set ContainerStateRemoving and remove exec sessions
+	// Set ContainerStateRemoving
 	c.state.State = define.ContainerStateRemoving
-	c.state.ExecSessions = nil
 
 	if err := c.save(); err != nil {
 		return errors.Wrapf(err, "unable to set container %s removing state in database", c.ID())
diff --git a/libpod/state.go b/libpod/state.go
index b246b5eac..9690e5819 100644
--- a/libpod/state.go
+++ b/libpod/state.go
@@ -72,6 +72,8 @@ type State interface {
 	// Removes container from state.
 	// Containers that are part of pods must use RemoveContainerFromPod.
 	// The container must be part of the set namespace.
+	// All dependencies must be removed first.
+	// All exec sessions referencing the container must be removed first.
 	RemoveContainer(ctr *Container) error
 	// UpdateContainer updates a container's state from the backing store.
 	// The container must be part of the set namespace.
@@ -95,6 +97,30 @@ type State interface {
 	// Return a container config from the database by full ID
 	GetContainerConfig(id string) (*ContainerConfig, error)
 
+	// Add creates a reference to an exec session in the database.
+	// The container the exec session is attached to will be recorded.
+	// The container state will not be modified.
+	// The actual exec session itself is part of the container's state.
+	// We assume higher-level callers will add the session by saving the
+	// container's state before calling this. This only ensures that the ID
+	// of the exec session is associated with the ID of the container.
+	// Implementations may, but are not required to, verify that the state
+	// of the given container has an exec session with the ID given.
+	AddExecSession(ctr *Container, session *ExecSession) error
+	// Get retrieves the container a given exec session is attached to.
+	GetExecSession(id string) (string, error)
+	// Remove a reference to an exec session from the database.
+	// This will not modify container state to remove the exec session there
+	// and instead only removes the session ID -> container ID reference
+	// added by AddExecSession.
+	RemoveExecSession(session *ExecSession) error
+	// Get the IDs of all exec sessions attached to a given container.
+	GetContainerExecSessions(ctr *Container) ([]string, error)
+	// Remove all exec sessions for a single container.
+	// Usually used as part of removing the container.
+	// As with RemoveExecSession, container state will not be modified.
+	RemoveContainerExecSessions(ctr *Container) error
+
 	// PLEASE READ FULL DESCRIPTION BEFORE USING.
 	// Rewrite a container's configuration.
 	// This function breaks libpod's normal prohibition on a read-only
diff --git a/pkg/adapter/terminal_linux.go b/pkg/adapter/terminal_linux.go
index 3dc5864e2..ef5a6f926 100644
--- a/pkg/adapter/terminal_linux.go
+++ b/pkg/adapter/terminal_linux.go
@@ -16,7 +16,6 @@ import (
 // ExecAttachCtr execs and attaches to a container
 func ExecAttachCtr(ctx context.Context, ctr *libpod.Container, tty, privileged bool, env map[string]string, cmd []string, user, workDir string, streams *libpod.AttachStreams, preserveFDs uint, detachKeys string) (int, error) {
 	resize := make(chan remotecommand.TerminalSize)
-
 	haveTerminal := terminal.IsTerminal(int(os.Stdin.Fd()))
 
 	// Check if we are attached to a terminal. If we are, generate resize
@@ -33,7 +32,18 @@ func ExecAttachCtr(ctx context.Context, ctr *libpod.Container, tty, privileged b
 			}
 		}()
 	}
-	return ctr.Exec(tty, privileged, env, cmd, user, workDir, streams, preserveFDs, resize, detachKeys)
+
+	execConfig := new(libpod.ExecConfig)
+	execConfig.Command = cmd
+	execConfig.Terminal = tty
+	execConfig.Privileged = privileged
+	execConfig.Environment = env
+	execConfig.User = user
+	execConfig.WorkDir = workDir
+	execConfig.DetachKeys = &detachKeys
+	execConfig.PreserveFDs = preserveFDs
+
+	return ctr.Exec(execConfig, streams, resize)
 }
 
 // StartAttachCtr starts and (if required) attaches to a container
diff --git a/pkg/varlinkapi/containers.go b/pkg/varlinkapi/containers.go
index 94726bbbd..55427771c 100644
--- a/pkg/varlinkapi/containers.go
+++ b/pkg/varlinkapi/containers.go
@@ -846,11 +846,6 @@ func (i *LibpodAPI) ExecContainer(call iopodman.VarlinkCall, opts iopodman.ExecO
 		workDir = *opts.Workdir
 	}
 
-	var detachKeys string
-	if opts.DetachKeys != nil {
-		detachKeys = *opts.DetachKeys
-	}
-
 	resizeChan := make(chan remotecommand.TerminalSize)
 
 	reader, writer, _, pipeWriter, streams := setupStreams(call)
@@ -870,8 +865,17 @@ func (i *LibpodAPI) ExecContainer(call iopodman.VarlinkCall, opts iopodman.ExecO
 		}
 	}()
 
+	execConfig := new(libpod.ExecConfig)
+	execConfig.Command = opts.Cmd
+	execConfig.Terminal = opts.Tty
+	execConfig.Privileged = opts.Privileged
+	execConfig.Environment = envs
+	execConfig.User = user
+	execConfig.WorkDir = workDir
+	execConfig.DetachKeys = opts.DetachKeys
+
 	go func() {
-		ec, err := ctr.Exec(opts.Tty, opts.Privileged, envs, opts.Cmd, user, workDir, streams, 0, resizeChan, detachKeys)
+		ec, err := ctr.Exec(execConfig, streams, resizeChan)
 		if err != nil {
 			logrus.Errorf(err.Error())
 		}