Merge branch 'master' of github.com:containers/libpod into vendor

32 files changed, 1598 insertions, 591 deletions

diff --git a/libpod/boltdb_state.go b/libpod/boltdb_state.go
index 42f029379..cb661d4e9 100644
--- a/libpod/boltdb_state.go
+++ b/libpod/boltdb_state.go
@@ -3,7 +3,6 @@ package libpod
 import (
 	"bytes"
 	"encoding/json"
-	"os"
 	"strings"
 	"sync"
 
@@ -19,7 +18,6 @@ type BoltState struct {
 	dbLock         sync.Mutex
 	namespace      string
 	namespaceBytes []byte
-	lockDir        string
 	runtime        *Runtime
 }
 
@@ -52,25 +50,15 @@ type BoltState struct {
 //   containers/storage do not occur.
 
 // NewBoltState creates a new bolt-backed state database
-func NewBoltState(path, lockDir string, runtime *Runtime) (State, error) {
+func NewBoltState(path string, runtime *Runtime) (State, error) {
 	state := new(BoltState)
 	state.dbPath = path
-	state.lockDir = lockDir
 	state.runtime = runtime
 	state.namespace = ""
 	state.namespaceBytes = nil
 
 	logrus.Debugf("Initializing boltdb state at %s", path)
 
-	// Make the directory that will hold container lockfiles
-	if err := os.MkdirAll(lockDir, 0750); err != nil {
-		// The directory is allowed to exist
-		if !os.IsExist(err) {
-			return nil, errors.Wrapf(err, "error creating lockfiles dir %s", lockDir)
-		}
-	}
-	state.lockDir = lockDir
-
 	db, err := bolt.Open(path, 0600, nil)
 	if err != nil {
 		return nil, errors.Wrapf(err, "error opening database %s", path)
@@ -115,11 +103,6 @@ func NewBoltState(path, lockDir string, runtime *Runtime) (State, error) {
 		return nil, errors.Wrapf(err, "error creating initial database layout")
 	}
 
-	// Check runtime configuration
-	if err := checkRuntimeConfig(db, runtime); err != nil {
-		return nil, err
-	}
-
 	state.valid = true
 
 	return state, nil
@@ -240,6 +223,72 @@ func (s *BoltState) Refresh() error {
 	return err
 }
 
+// GetDBConfig retrieves runtime configuration fields that were created when
+// the database was first initialized
+func (s *BoltState) GetDBConfig() (*DBConfig, error) {
+	if !s.valid {
+		return nil, ErrDBClosed
+	}
+
+	cfg := new(DBConfig)
+
+	db, err := s.getDBCon()
+	if err != nil {
+		return nil, err
+	}
+	defer s.closeDBCon(db)
+
+	err = db.View(func(tx *bolt.Tx) error {
+		configBucket, err := getRuntimeConfigBucket(tx)
+		if err != nil {
+			return nil
+		}
+
+		// Some of these may be nil
+		// When we convert to string, Go will coerce them to ""
+		// That's probably fine - we could raise an error if the key is
+		// missing, but just not including it is also OK.
+		libpodRoot := configBucket.Get(staticDirKey)
+		libpodTmp := configBucket.Get(tmpDirKey)
+		storageRoot := configBucket.Get(graphRootKey)
+		storageTmp := configBucket.Get(runRootKey)
+		graphDriver := configBucket.Get(graphDriverKey)
+
+		cfg.LibpodRoot = string(libpodRoot)
+		cfg.LibpodTmp = string(libpodTmp)
+		cfg.StorageRoot = string(storageRoot)
+		cfg.StorageTmp = string(storageTmp)
+		cfg.GraphDriver = string(graphDriver)
+
+		return nil
+	})
+	if err != nil {
+		return nil, err
+	}
+
+	return cfg, nil
+}
+
+// ValidateDBConfig validates paths in the given runtime against the database
+func (s *BoltState) ValidateDBConfig(runtime *Runtime) error {
+	if !s.valid {
+		return ErrDBClosed
+	}
+
+	db, err := s.getDBCon()
+	if err != nil {
+		return err
+	}
+	defer s.closeDBCon(db)
+
+	// Check runtime configuration
+	if err := checkRuntimeConfig(db, runtime); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // SetNamespace sets the namespace that will be used for container and pod
 // retrieval
 func (s *BoltState) SetNamespace(ns string) error {
diff --git a/libpod/boltdb_state_internal.go b/libpod/boltdb_state_internal.go
index cc7d106cc..3f00657ea 100644
--- a/libpod/boltdb_state_internal.go
+++ b/libpod/boltdb_state_internal.go
@@ -30,6 +30,13 @@ const (
 	containersName   = "containers"
 	podIDName        = "pod-id"
 	namespaceName    = "namespace"
+
+	staticDirName   = "static-dir"
+	tmpDirName      = "tmp-dir"
+	runRootName     = "run-root"
+	graphRootName   = "graph-root"
+	graphDriverName = "graph-driver-name"
+	osName          = "os"
 )
 
 var (
@@ -49,21 +56,19 @@ var (
 	containersBkt   = []byte(containersName)
 	podIDKey        = []byte(podIDName)
 	namespaceKey    = []byte(namespaceName)
+
+	staticDirKey   = []byte(staticDirName)
+	tmpDirKey      = []byte(tmpDirName)
+	runRootKey     = []byte(runRootName)
+	graphRootKey   = []byte(graphRootName)
+	graphDriverKey = []byte(graphDriverName)
+	osKey          = []byte(osName)
 )
 
 // Check if the configuration of the database is compatible with the
 // configuration of the runtime opening it
 // If there is no runtime configuration loaded, load our own
 func checkRuntimeConfig(db *bolt.DB, rt *Runtime) error {
-	var (
-		staticDir       = []byte("static-dir")
-		tmpDir          = []byte("tmp-dir")
-		runRoot         = []byte("run-root")
-		graphRoot       = []byte("graph-root")
-		graphDriverName = []byte("graph-driver-name")
-		osKey           = []byte("os")
-	)
-
 	err := db.Update(func(tx *bolt.Tx) error {
 		configBkt, err := getRuntimeConfigBucket(tx)
 		if err != nil {
@@ -74,31 +79,31 @@ func checkRuntimeConfig(db *bolt.DB, rt *Runtime) error {
 			return err
 		}
 
-		if err := validateDBAgainstConfig(configBkt, "static dir",
-			rt.config.StaticDir, staticDir, ""); err != nil {
+		if err := validateDBAgainstConfig(configBkt, "libpod root directory (staticdir)",
+			rt.config.StaticDir, staticDirKey, ""); err != nil {
 			return err
 		}
 
-		if err := validateDBAgainstConfig(configBkt, "tmp dir",
-			rt.config.TmpDir, tmpDir, ""); err != nil {
+		if err := validateDBAgainstConfig(configBkt, "libpod temporary files directory (tmpdir)",
+			rt.config.TmpDir, tmpDirKey, ""); err != nil {
 			return err
 		}
 
-		if err := validateDBAgainstConfig(configBkt, "run root",
-			rt.config.StorageConfig.RunRoot, runRoot,
+		if err := validateDBAgainstConfig(configBkt, "storage temporary directory (runroot)",
+			rt.config.StorageConfig.RunRoot, runRootKey,
 			storage.DefaultStoreOptions.RunRoot); err != nil {
 			return err
 		}
 
-		if err := validateDBAgainstConfig(configBkt, "graph root",
-			rt.config.StorageConfig.GraphRoot, graphRoot,
+		if err := validateDBAgainstConfig(configBkt, "storage graph root directory (graphroot)",
+			rt.config.StorageConfig.GraphRoot, graphRootKey,
 			storage.DefaultStoreOptions.GraphRoot); err != nil {
 			return err
 		}
 
-		return validateDBAgainstConfig(configBkt, "graph driver name",
+		return validateDBAgainstConfig(configBkt, "storage graph driver",
 			rt.config.StorageConfig.GraphDriverName,
-			graphDriverName,
+			graphDriverKey,
 			storage.DefaultStoreOptions.GraphDriverName)
 	})
 
@@ -261,7 +266,7 @@ func (s *BoltState) getContainerFromDB(id []byte, ctr *Container, ctrsBkt *bolt.
 	}
 
 	// Get the lock
-	lockPath := filepath.Join(s.lockDir, string(id))
+	lockPath := filepath.Join(s.runtime.lockDir, string(id))
 	lock, err := storage.GetLockfile(lockPath)
 	if err != nil {
 		return errors.Wrapf(err, "error retrieving lockfile for container %s", string(id))
@@ -297,7 +302,7 @@ func (s *BoltState) getPodFromDB(id []byte, pod *Pod, podBkt *bolt.Bucket) error
 	}
 
 	// Get the lock
-	lockPath := filepath.Join(s.lockDir, string(id))
+	lockPath := filepath.Join(s.runtime.lockDir, string(id))
 	lock, err := storage.GetLockfile(lockPath)
 	if err != nil {
 		return errors.Wrapf(err, "error retrieving lockfile for pod %s", string(id))
diff --git a/libpod/container.go b/libpod/container.go
index 7bb5b2687..b6155a809 100644
--- a/libpod/container.go
+++ b/libpod/container.go
@@ -9,6 +9,7 @@ import (
 
 	"github.com/containernetworking/cni/pkg/types"
 	cnitypes "github.com/containernetworking/cni/pkg/types/current"
+	"github.com/containers/libpod/pkg/namespaces"
 	"github.com/containers/storage"
 	"github.com/cri-o/ocicni/pkg/ocicni"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
@@ -296,6 +297,8 @@ type ContainerConfig struct {
 	HostAdd []string `json:"hostsAdd,omitempty"`
 	// Network names (CNI) to add container to. Empty to use default network.
 	Networks []string `json:"networks,omitempty"`
+	// Network mode specified for the default network.
+	NetMode namespaces.NetworkMode `json:"networkMode,omitempty"`
 
 	// Image Config
 
@@ -826,7 +829,7 @@ func (c *Container) IPs() ([]net.IPNet, error) {
 	}
 
 	if !c.config.CreateNetNS {
-		return nil, errors.Wrapf(ErrInvalidArg, "container %s network namespace is not managed by libpod")
+		return nil, errors.Wrapf(ErrInvalidArg, "container %s network namespace is not managed by libpod", c.ID())
 	}
 
 	ips := make([]net.IPNet, 0)
@@ -854,7 +857,7 @@ func (c *Container) Routes() ([]types.Route, error) {
 	}
 
 	if !c.config.CreateNetNS {
-		return nil, errors.Wrapf(ErrInvalidArg, "container %s network namespace is not managed by libpod")
+		return nil, errors.Wrapf(ErrInvalidArg, "container %s network namespace is not managed by libpod", c.ID())
 	}
 
 	routes := make([]types.Route, 0)
@@ -996,3 +999,15 @@ func (c *Container) IsInfra() bool {
 func (c *Container) IsReadOnly() bool {
 	return c.config.Spec.Root.Readonly
 }
+
+// NetworkDisabled returns whether the container is running with a disabled network
+func (c *Container) NetworkDisabled() bool {
+	if !c.config.PostConfigureNetNS {
+		for _, ns := range c.config.Spec.Linux.Namespaces {
+			if ns.Type == spec.NetworkNamespace {
+				return ns.Path == ""
+			}
+		}
+	}
+	return false
+}
diff --git a/libpod/container_api.go b/libpod/container_api.go
index e522038a3..bc92cae69 100644
--- a/libpod/container_api.go
+++ b/libpod/container_api.go
@@ -10,8 +10,8 @@ import (
 	"time"
 
 	"github.com/containers/libpod/libpod/driver"
-	"github.com/containers/libpod/pkg/chrootuser"
 	"github.com/containers/libpod/pkg/inspect"
+	"github.com/containers/libpod/pkg/lookup"
 	"github.com/containers/storage/pkg/stringid"
 	"github.com/docker/docker/daemon/caps"
 	"github.com/pkg/errors"
@@ -39,16 +39,13 @@ func (c *Container) Init(ctx context.Context) (err error) {
 
 	notRunning, err := c.checkDependenciesRunning()
 	if err != nil {
-		return errors.Wrapf(err, "error checking dependencies for container %s")
+		return errors.Wrapf(err, "error checking dependencies for container %s", c.ID())
 	}
 	if len(notRunning) > 0 {
 		depString := strings.Join(notRunning, ",")
 		return errors.Wrapf(ErrCtrStateInvalid, "some dependencies of container %s are not started: %s", c.ID(), depString)
 	}
 
-	if err := c.prepare(); err != nil {
-		return err
-	}
 	defer func() {
 		if err != nil {
 			if err2 := c.cleanup(ctx); err2 != nil {
@@ -57,6 +54,10 @@ func (c *Container) Init(ctx context.Context) (err error) {
 		}
 	}()
 
+	if err := c.prepare(); err != nil {
+		return err
+	}
+
 	if c.state.State == ContainerStateStopped {
 		// Reinitialize the container
 		return c.reinit(ctx)
@@ -92,16 +93,13 @@ func (c *Container) Start(ctx context.Context) (err error) {
 
 	notRunning, err := c.checkDependenciesRunning()
 	if err != nil {
-		return errors.Wrapf(err, "error checking dependencies for container %s")
+		return errors.Wrapf(err, "error checking dependencies for container %s", c.ID())
 	}
 	if len(notRunning) > 0 {
 		depString := strings.Join(notRunning, ",")
 		return errors.Wrapf(ErrCtrStateInvalid, "some dependencies of container %s are not started: %s", c.ID(), depString)
 	}
 
-	if err := c.prepare(); err != nil {
-		return err
-	}
 	defer func() {
 		if err != nil {
 			if err2 := c.cleanup(ctx); err2 != nil {
@@ -110,6 +108,10 @@ func (c *Container) Start(ctx context.Context) (err error) {
 		}
 	}()
 
+	if err := c.prepare(); err != nil {
+		return err
+	}
+
 	if c.state.State == ContainerStateStopped {
 		// Reinitialize the container if we need to
 		if err := c.reinit(ctx); err != nil {
@@ -157,16 +159,13 @@ func (c *Container) StartAndAttach(ctx context.Context, streams *AttachStreams,
 
 	notRunning, err := c.checkDependenciesRunning()
 	if err != nil {
-		return nil, errors.Wrapf(err, "error checking dependencies for container %s")
+		return nil, errors.Wrapf(err, "error checking dependencies for container %s", c.ID())
 	}
 	if len(notRunning) > 0 {
 		depString := strings.Join(notRunning, ",")
 		return nil, errors.Wrapf(ErrCtrStateInvalid, "some dependencies of container %s are not started: %s", c.ID(), depString)
 	}
 
-	if err := c.prepare(); err != nil {
-		return nil, err
-	}
 	defer func() {
 		if err != nil {
 			if err2 := c.cleanup(ctx); err2 != nil {
@@ -175,6 +174,10 @@ func (c *Container) StartAndAttach(ctx context.Context, streams *AttachStreams,
 		}
 	}()
 
+	if err := c.prepare(); err != nil {
+		return nil, err
+	}
+
 	if c.state.State == ContainerStateStopped {
 		// Reinitialize the container if we need to
 		if err := c.reinit(ctx); err != nil {
@@ -292,13 +295,13 @@ func (c *Container) Exec(tty, privileged bool, env, cmd []string, user string) e
 	// the host
 	hostUser := ""
 	if user != "" {
-		uid, gid, err := chrootuser.GetUser(c.state.Mountpoint, user)
+		execUser, err := lookup.GetUserGroupInfo(c.state.Mountpoint, user, nil)
 		if err != nil {
-			return errors.Wrapf(err, "error getting user to launch exec session as")
+			return err
 		}
 
 		// runc expects user formatted as uid:gid
-		hostUser = fmt.Sprintf("%d:%d", uid, gid)
+		hostUser = fmt.Sprintf("%d:%d", execUser.Uid, execUser.Gid)
 	}
 
 	// Generate exec session ID
@@ -325,25 +328,25 @@ func (c *Container) Exec(tty, privileged bool, env, cmd []string, user string) e
 	if err != nil {
 		return errors.Wrapf(err, "error exec %s", c.ID())
 	}
+	chWait := make(chan error)
+	go func() {
+		chWait <- execCmd.Wait()
+	}()
+	defer close(chWait)
 
 	pidFile := c.execPidPath(sessionID)
-	// 1 second seems a reasonable time to wait
-	// See https://github.com/containers/libpod/issues/1495
-	const pidWaitTimeout = 1000
+	// 60 second seems a reasonable time to wait
+	// https://github.com/containers/libpod/issues/1495
+	// https://github.com/containers/libpod/issues/1816
+	const pidWaitTimeout = 60000
 
 	// Wait until the runtime makes the pidfile
-	// TODO: If runtime errors before the PID file is created, we have to
-	// wait for timeout here
-	if err := WaitForFile(pidFile, pidWaitTimeout*time.Millisecond); err != nil {
-		logrus.Debugf("Timed out waiting for pidfile from runtime for container %s exec", c.ID())
-
-		// Check if an error occurred in the process before we made a pidfile
-		// TODO: Wait() here is a poor choice - is there a way to see if
-		// a process has finished, instead of waiting for it to finish?
-		if err := execCmd.Wait(); err != nil {
+	exited, err := WaitForFile(pidFile, chWait, pidWaitTimeout*time.Millisecond)
+	if err != nil {
+		if exited {
+			// If the runtime exited, propagate the error we got from the process.
 			return err
 		}
-
 		return errors.Wrapf(err, "timed out waiting for runtime to create pidfile for exec session in container %s", c.ID())
 	}
 
@@ -385,7 +388,10 @@ func (c *Container) Exec(tty, privileged bool, env, cmd []string, user string) e
 		locked = false
 	}
 
-	waitErr := execCmd.Wait()
+	var waitErr error
+	if !exited {
+		waitErr = <-chWait
+	}
 
 	// Lock again
 	if !c.batched {
@@ -685,7 +691,7 @@ func (c *Container) Sync() error {
 		(c.state.State != ContainerStateConfigured) {
 		oldState := c.state.State
 		// TODO: optionally replace this with a stat for the exit file
-		if err := c.runtime.ociRuntime.updateContainerStatus(c); err != nil {
+		if err := c.runtime.ociRuntime.updateContainerStatus(c, true); err != nil {
 			return err
 		}
 		// Only save back to DB if state changed
@@ -712,7 +718,7 @@ func (c *Container) RestartWithTimeout(ctx context.Context, timeout uint) (err e
 
 	notRunning, err := c.checkDependenciesRunning()
 	if err != nil {
-		return errors.Wrapf(err, "error checking dependencies for container %s")
+		return errors.Wrapf(err, "error checking dependencies for container %s", c.ID())
 	}
 	if len(notRunning) > 0 {
 		depString := strings.Join(notRunning, ",")
@@ -797,7 +803,7 @@ func (c *Container) Refresh(ctx context.Context) error {
 		return err
 	}
 
-	logrus.Debugf("Successfully refresh container %s state")
+	logrus.Debugf("Successfully refresh container %s state", c.ID())
 
 	// Initialize the container if it was created in runc
 	if wasCreated || wasRunning || wasPaused {
@@ -826,9 +832,22 @@ func (c *Container) Refresh(ctx context.Context) error {
 	return nil
 }
 
+// ContainerCheckpointOptions is a struct used to pass the parameters
+// for checkpointing (and restoring) to the corresponding functions
+type ContainerCheckpointOptions struct {
+	// Keep tells the API to not delete checkpoint artifacts
+	Keep bool
+	// KeepRunning tells the API to keep the container running
+	// after writing the checkpoint to disk
+	KeepRunning bool
+	// TCPEstablished tells the API to checkpoint a container
+	// even if it contains established TCP connections
+	TCPEstablished bool
+}
+
 // Checkpoint checkpoints a container
-func (c *Container) Checkpoint(ctx context.Context, keep bool) error {
-	logrus.Debugf("Trying to checkpoint container %s", c)
+func (c *Container) Checkpoint(ctx context.Context, options ContainerCheckpointOptions) error {
+	logrus.Debugf("Trying to checkpoint container %s", c.ID())
 	if !c.batched {
 		c.lock.Lock()
 		defer c.lock.Unlock()
@@ -838,12 +857,12 @@ func (c *Container) Checkpoint(ctx context.Context, keep bool) error {
 		}
 	}
 
-	return c.checkpoint(ctx, keep)
+	return c.checkpoint(ctx, options)
 }
 
 // Restore restores a container
-func (c *Container) Restore(ctx context.Context, keep bool) (err error) {
-	logrus.Debugf("Trying to restore container %s", c)
+func (c *Container) Restore(ctx context.Context, options ContainerCheckpointOptions) (err error) {
+	logrus.Debugf("Trying to restore container %s", c.ID())
 	if !c.batched {
 		c.lock.Lock()
 		defer c.lock.Unlock()
@@ -853,5 +872,5 @@ func (c *Container) Restore(ctx context.Context, keep bool) (err error) {
 		}
 	}
 
-	return c.restore(ctx, keep)
+	return c.restore(ctx, options)
 }
diff --git a/libpod/container_attach.go b/libpod/container_attach.go
index 3c4e0775d..f925c3897 100644
--- a/libpod/container_attach.go
+++ b/libpod/container_attach.go
@@ -16,6 +16,10 @@ import (
 	"k8s.io/client-go/tools/remotecommand"
 )
 
+//#include <sys/un.h>
+// extern int unix_path_length(){struct sockaddr_un addr; return sizeof(addr.sun_path) - 1;}
+import "C"
+
 /* Sync with stdpipe_t in conmon.c */
 const (
 	AttachPipeStdin  = 1
@@ -81,11 +85,19 @@ func (c *Container) attachContainerSocket(resize <-chan remotecommand.TerminalSi
 			logrus.Warnf("Failed to write to control file to resize terminal: %v", err)
 		}
 	})
-	logrus.Debug("connecting to socket ", c.AttachSocketPath())
 
-	conn, err := net.DialUnix("unixpacket", nil, &net.UnixAddr{Name: c.AttachSocketPath(), Net: "unixpacket"})
+	socketPath := c.AttachSocketPath()
+
+	maxUnixLength := int(C.unix_path_length())
+	if maxUnixLength < len(socketPath) {
+		socketPath = socketPath[0:maxUnixLength]
+	}
+
+	logrus.Debug("connecting to socket ", socketPath)
+
+	conn, err := net.DialUnix("unixpacket", nil, &net.UnixAddr{Name: socketPath, Net: "unixpacket"})
 	if err != nil {
-		return errors.Wrapf(err, "failed to connect to container's attach socket: %v", c.AttachSocketPath())
+		return errors.Wrapf(err, "failed to connect to container's attach socket: %v", socketPath)
 	}
 	defer conn.Close()
 
diff --git a/libpod/container_easyjson.go b/libpod/container_easyjson.go
index 041cc08ac..8bf5cb64f 100644
--- a/libpod/container_easyjson.go
+++ b/libpod/container_easyjson.go
@@ -8,6 +8,7 @@ import (
 	json "encoding/json"
 	types "github.com/containernetworking/cni/pkg/types"
 	current "github.com/containernetworking/cni/pkg/types/current"
+	namespaces "github.com/containers/libpod/pkg/namespaces"
 	storage "github.com/containers/storage"
 	idtools "github.com/containers/storage/pkg/idtools"
 	ocicni "github.com/cri-o/ocicni/pkg/ocicni"
@@ -1550,6 +1551,8 @@ func easyjson1dbef17bDecodeGithubComContainersLibpodLibpod2(in *jlexer.Lexer, ou
 				}
 				in.Delim(']')
 			}
+		case "networkMode":
+			out.NetMode = namespaces.NetworkMode(in.String())
 		case "userVolumes":
 			if in.IsNull() {
 				in.Skip()
@@ -2177,6 +2180,16 @@ func easyjson1dbef17bEncodeGithubComContainersLibpodLibpod2(out *jwriter.Writer,
 			out.RawByte(']')
 		}
 	}
+	if in.NetMode != "" {
+		const prefix string = ",\"networkMode\":"
+		if first {
+			first = false
+			out.RawString(prefix[1:])
+		} else {
+			out.RawString(prefix)
+		}
+		out.String(string(in.NetMode))
+	}
 	if len(in.UserVolumes) != 0 {
 		const prefix string = ",\"userVolumes\":"
 		if first {
diff --git a/libpod/container_graph_test.go b/libpod/container_graph_test.go
index bba3d7aad..25461f1f4 100644
--- a/libpod/container_graph_test.go
+++ b/libpod/container_graph_test.go
@@ -205,6 +205,7 @@ func TestBuildContainerGraphFourContainersNoEdges(t *testing.T) {
 	ctr3, err := getTestCtrN("3", tmpDir)
 	assert.NoError(t, err)
 	ctr4, err := getTestCtrN("4", tmpDir)
+	assert.NoError(t, err)
 
 	graph, err := buildContainerGraph([]*Container{ctr1, ctr2, ctr3, ctr4})
 	assert.NoError(t, err)
@@ -241,6 +242,7 @@ func TestBuildContainerGraphFourContainersTwoInCycle(t *testing.T) {
 	ctr3, err := getTestCtrN("3", tmpDir)
 	assert.NoError(t, err)
 	ctr4, err := getTestCtrN("4", tmpDir)
+	assert.NoError(t, err)
 	ctr1.config.IPCNsCtr = ctr2.config.ID
 	ctr2.config.UserNsCtr = ctr1.config.ID
 
@@ -260,6 +262,7 @@ func TestBuildContainerGraphFourContainersAllInCycle(t *testing.T) {
 	ctr3, err := getTestCtrN("3", tmpDir)
 	assert.NoError(t, err)
 	ctr4, err := getTestCtrN("4", tmpDir)
+	assert.NoError(t, err)
 	ctr1.config.IPCNsCtr = ctr2.config.ID
 	ctr2.config.UserNsCtr = ctr3.config.ID
 	ctr3.config.NetNsCtr = ctr4.config.ID
@@ -281,6 +284,7 @@ func TestBuildContainerGraphFourContainersNoneInCycle(t *testing.T) {
 	ctr3, err := getTestCtrN("3", tmpDir)
 	assert.NoError(t, err)
 	ctr4, err := getTestCtrN("4", tmpDir)
+	assert.NoError(t, err)
 	ctr1.config.IPCNsCtr = ctr2.config.ID
 	ctr1.config.NetNsCtr = ctr3.config.ID
 	ctr2.config.UserNsCtr = ctr3.config.ID
diff --git a/libpod/container_internal.go b/libpod/container_internal.go
index 2af216358..934ad7a22 100644
--- a/libpod/container_internal.go
+++ b/libpod/container_internal.go
@@ -12,14 +12,14 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	"github.com/containers/buildah/imagebuildah"
-	"github.com/containers/libpod/pkg/chrootuser"
+	"github.com/containers/libpod/pkg/ctime"
 	"github.com/containers/libpod/pkg/hooks"
 	"github.com/containers/libpod/pkg/hooks/exec"
-	"github.com/containers/libpod/pkg/resolvconf"
+	"github.com/containers/libpod/pkg/lookup"
 	"github.com/containers/libpod/pkg/rootless"
-	"github.com/containers/libpod/pkg/secrets"
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/archive"
 	"github.com/containers/storage/pkg/chrootarchive"
@@ -30,6 +30,7 @@ import (
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/text/language"
+	kwait "k8s.io/apimachinery/pkg/util/wait"
 )
 
 const (
@@ -146,6 +147,77 @@ func (c *Container) execPidPath(sessionID string) string {
 	return filepath.Join(c.state.RunDir, "exec_pid_"+sessionID)
 }
 
+// exitFilePath gets the path to the container's exit file
+func (c *Container) exitFilePath() string {
+	return filepath.Join(c.runtime.ociRuntime.exitsDir, c.ID())
+}
+
+// Wait for the container's exit file to appear.
+// When it does, update our state based on it.
+func (c *Container) waitForExitFileAndSync() error {
+	exitFile := c.exitFilePath()
+
+	err := kwait.ExponentialBackoff(
+		kwait.Backoff{
+			Duration: 500 * time.Millisecond,
+			Factor:   1.2,
+			Steps:    6,
+		},
+		func() (bool, error) {
+			_, err := os.Stat(exitFile)
+			if err != nil {
+				// wait longer
+				return false, nil
+			}
+			return true, nil
+		})
+	if err != nil {
+		// Exit file did not appear
+		// Reset our state
+		c.state.ExitCode = -1
+		c.state.FinishedTime = time.Now()
+		c.state.State = ContainerStateStopped
+
+		if err2 := c.save(); err2 != nil {
+			logrus.Errorf("Error saving container %s state: %v", c.ID(), err2)
+		}
+
+		return err
+	}
+
+	if err := c.runtime.ociRuntime.updateContainerStatus(c, false); err != nil {
+		return err
+	}
+
+	return c.save()
+}
+
+// Handle the container exit file.
+// The exit file is used to supply container exit time and exit code.
+// This assumes the exit file already exists.
+func (c *Container) handleExitFile(exitFile string, fi os.FileInfo) error {
+	c.state.FinishedTime = ctime.Created(fi)
+	statusCodeStr, err := ioutil.ReadFile(exitFile)
+	if err != nil {
+		return errors.Wrapf(err, "failed to read exit file for container %s", c.ID())
+	}
+	statusCode, err := strconv.Atoi(string(statusCodeStr))
+	if err != nil {
+		return errors.Wrapf(err, "error converting exit status code (%q) for container %s to int",
+			c.ID(), statusCodeStr)
+	}
+	c.state.ExitCode = int32(statusCode)
+
+	oomFilePath := filepath.Join(c.bundlePath(), "oom")
+	if _, err = os.Stat(oomFilePath); err == nil {
+		c.state.OOMKilled = true
+	}
+
+	c.state.Exited = true
+
+	return nil
+}
+
 // Sync this container with on-disk state and runtime status
 // Should only be called with container lock held
 // This function should suffice to ensure a container's state is accurate and
@@ -161,7 +233,7 @@ func (c *Container) syncContainer() error {
 		(c.state.State != ContainerStateExited) {
 		oldState := c.state.State
 		// TODO: optionally replace this with a stat for the exit file
-		if err := c.runtime.ociRuntime.updateContainerStatus(c); err != nil {
+		if err := c.runtime.ociRuntime.updateContainerStatus(c, false); err != nil {
 			return err
 		}
 		// Only save back to DB if state changed
@@ -201,6 +273,27 @@ func (c *Container) setupStorage(ctx context.Context) error {
 		},
 		LabelOpts: c.config.LabelOpts,
 	}
+	if c.config.Privileged {
+		privOpt := func(opt string) bool {
+			for _, privopt := range []string{"nodev", "nosuid", "noexec"} {
+				if opt == privopt {
+					return true
+				}
+			}
+			return false
+		}
+		defOptions, err := storage.GetDefaultMountOptions()
+		if err != nil {
+			return errors.Wrapf(err, "error getting default mount options")
+		}
+		var newOptions []string
+		for _, opt := range defOptions {
+			if !privOpt(opt) {
+				newOptions = append(newOptions, opt)
+			}
+		}
+		options.MountOpts = newOptions
+	}
 
 	if c.config.Rootfs == "" {
 		options.IDMappingOptions = c.config.IDMappings
@@ -508,13 +601,13 @@ func (c *Container) checkDependenciesRunningLocked(depCtrs map[string]*Container
 }
 
 func (c *Container) completeNetworkSetup() error {
-	if !c.config.PostConfigureNetNS {
+	if !c.config.PostConfigureNetNS || c.NetworkDisabled() {
 		return nil
 	}
 	if err := c.syncContainer(); err != nil {
 		return err
 	}
-	if rootless.IsRootless() {
+	if c.config.NetMode == "slirp4netns" {
 		return c.runtime.setupRootlessNetNS(c)
 	}
 	return c.runtime.setupNetNS(c)
@@ -522,10 +615,6 @@ func (c *Container) completeNetworkSetup() error {
 
 // Initialize a container, creating it in the runtime
 func (c *Container) init(ctx context.Context) error {
-	if err := c.makeBindMounts(); err != nil {
-		return err
-	}
-
 	// Generate the OCI spec
 	spec, err := c.generateSpec(ctx)
 	if err != nil {
@@ -538,7 +627,7 @@ func (c *Container) init(ctx context.Context) error {
 	}
 
 	// With the spec complete, do an OCI create
-	if err := c.runtime.ociRuntime.createContainer(c, c.config.CgroupParent, false); err != nil {
+	if err := c.runtime.ociRuntime.createContainer(c, c.config.CgroupParent, nil); err != nil {
 		return err
 	}
 
@@ -622,9 +711,6 @@ func (c *Container) initAndStart(ctx context.Context) (err error) {
 		return errors.Wrapf(ErrCtrStateInvalid, "cannot start paused container %s", c.ID())
 	}
 
-	if err := c.prepare(); err != nil {
-		return err
-	}
 	defer func() {
 		if err != nil {
 			if err2 := c.cleanup(ctx); err2 != nil {
@@ -633,6 +719,10 @@ func (c *Container) initAndStart(ctx context.Context) (err error) {
 		}
 	}()
 
+	if err := c.prepare(); err != nil {
+		return err
+	}
+
 	// If we are ContainerStateStopped we need to remove from runtime
 	// And reset to ContainerStateConfigured
 	if c.state.State == ContainerStateStopped {
@@ -672,13 +762,8 @@ func (c *Container) stop(timeout uint) error {
 		return err
 	}
 
-	// Sync the container's state to pick up return code
-	if err := c.runtime.ociRuntime.updateContainerStatus(c); err != nil {
-		return err
-	}
-
-	// Container should clean itself up
-	return nil
+	// Wait until we have an exit file, and sync once we do
+	return c.waitForExitFileAndSync()
 }
 
 // Internal, non-locking function to pause a container
@@ -718,9 +803,6 @@ func (c *Container) restartWithTimeout(ctx context.Context, timeout uint) (err e
 			return err
 		}
 	}
-	if err := c.prepare(); err != nil {
-		return err
-	}
 	defer func() {
 		if err != nil {
 			if err2 := c.cleanup(ctx); err2 != nil {
@@ -728,6 +810,9 @@ func (c *Container) restartWithTimeout(ctx context.Context, timeout uint) (err e
 			}
 		}
 	}()
+	if err := c.prepare(); err != nil {
+		return err
+	}
 
 	if c.state.State == ContainerStateStopped {
 		// Reinitialize the container if we need to
@@ -757,28 +842,22 @@ func (c *Container) mountStorage() (string, error) {
 		return c.state.Mountpoint, nil
 	}
 
-	if !rootless.IsRootless() {
-		// TODO: generalize this mount code so it will mount every mount in ctr.config.Mounts
-		mounted, err := mount.Mounted(c.config.ShmDir)
-		if err != nil {
-			return "", errors.Wrapf(err, "unable to determine if %q is mounted", c.config.ShmDir)
-		}
+	mounted, err := mount.Mounted(c.config.ShmDir)
+	if err != nil {
+		return "", errors.Wrapf(err, "unable to determine if %q is mounted", c.config.ShmDir)
+	}
 
+	if !mounted {
+		shmOptions := fmt.Sprintf("mode=1777,size=%d", c.config.ShmSize)
+		if err := c.mountSHM(shmOptions); err != nil {
+			return "", err
+		}
 		if err := os.Chown(c.config.ShmDir, c.RootUID(), c.RootGID()); err != nil {
 			return "", errors.Wrapf(err, "failed to chown %s", c.config.ShmDir)
 		}
-
-		if !mounted {
-			shmOptions := fmt.Sprintf("mode=1777,size=%d", c.config.ShmSize)
-			if err := c.mountSHM(shmOptions); err != nil {
-				return "", err
-			}
-			if err := os.Chown(c.config.ShmDir, c.RootUID(), c.RootGID()); err != nil {
-				return "", errors.Wrapf(err, "failed to chown %s", c.config.ShmDir)
-			}
-		}
 	}
 
+	// TODO: generalize this mount code so it will mount every mount in ctr.config.Mounts
 	mountPoint := c.config.Rootfs
 	if mountPoint == "" {
 		mountPoint, err = c.mount()
@@ -916,86 +995,6 @@ func (c *Container) postDeleteHooks(ctx context.Context) (err error) {
 	return nil
 }
 
-// Make standard bind mounts to include in the container
-func (c *Container) makeBindMounts() error {
-	if err := os.Chown(c.state.RunDir, c.RootUID(), c.RootGID()); err != nil {
-		return errors.Wrapf(err, "cannot chown run directory %s", c.state.RunDir)
-	}
-
-	if c.state.BindMounts == nil {
-		c.state.BindMounts = make(map[string]string)
-	}
-
-	// SHM is always added when we mount the container
-	c.state.BindMounts["/dev/shm"] = c.config.ShmDir
-
-	// Make /etc/resolv.conf
-	if _, ok := c.state.BindMounts["/etc/resolv.conf"]; ok {
-		// If it already exists, delete so we can recreate
-		delete(c.state.BindMounts, "/etc/resolv.conf")
-	}
-	newResolv, err := c.generateResolvConf()
-	if err != nil {
-		return errors.Wrapf(err, "error creating resolv.conf for container %s", c.ID())
-	}
-	c.state.BindMounts["/etc/resolv.conf"] = newResolv
-
-	newPasswd, err := c.generatePasswd()
-	if err != nil {
-		return errors.Wrapf(err, "error creating temporary passwd file for container %s", c.ID())
-	}
-	if newPasswd != "" {
-		// Make /etc/passwd
-		if _, ok := c.state.BindMounts["/etc/passwd"]; ok {
-			// If it already exists, delete so we can recreate
-			delete(c.state.BindMounts, "/etc/passwd")
-		}
-		logrus.Debugf("adding entry to /etc/passwd for non existent default user")
-		c.state.BindMounts["/etc/passwd"] = newPasswd
-	}
-	// Make /etc/hosts
-	if _, ok := c.state.BindMounts["/etc/hosts"]; ok {
-		// If it already exists, delete so we can recreate
-		delete(c.state.BindMounts, "/etc/hosts")
-	}
-	newHosts, err := c.generateHosts()
-	if err != nil {
-		return errors.Wrapf(err, "error creating hosts file for container %s", c.ID())
-	}
-	c.state.BindMounts["/etc/hosts"] = newHosts
-
-	// Make /etc/hostname
-	// This should never change, so no need to recreate if it exists
-	if _, ok := c.state.BindMounts["/etc/hostname"]; !ok {
-		hostnamePath, err := c.writeStringToRundir("hostname", c.Hostname())
-		if err != nil {
-			return errors.Wrapf(err, "error creating hostname file for container %s", c.ID())
-		}
-		c.state.BindMounts["/etc/hostname"] = hostnamePath
-	}
-
-	// Make .containerenv
-	// Empty file, so no need to recreate if it exists
-	if _, ok := c.state.BindMounts["/run/.containerenv"]; !ok {
-		// Empty string for now, but we may consider populating this later
-		containerenvPath, err := c.writeStringToRundir(".containerenv", "")
-		if err != nil {
-			return errors.Wrapf(err, "error creating containerenv file for container %s", c.ID())
-		}
-		c.state.BindMounts["/run/.containerenv"] = containerenvPath
-	}
-
-	// Add Secret Mounts
-	secretMounts := secrets.SecretMountsWithUIDGID(c.config.MountLabel, c.state.RunDir, c.runtime.config.DefaultMountsFile, c.state.DestinationRunDir, c.RootUID(), c.RootGID())
-	for _, mount := range secretMounts {
-		if _, ok := c.state.BindMounts[mount.Destination]; !ok {
-			c.state.BindMounts[mount.Destination] = mount.Source
-		}
-	}
-
-	return nil
-}
-
 // writeStringToRundir copies the provided file to the runtimedir
 func (c *Container) writeStringToRundir(destFile, output string) (string, error) {
 	destFileName := filepath.Join(c.state.RunDir, destFile)
@@ -1024,135 +1023,8 @@ func (c *Container) writeStringToRundir(destFile, output string) (string, error)
 	return filepath.Join(c.state.DestinationRunDir, destFile), nil
 }
 
-// generatePasswd generates a container specific passwd file,
-// iff g.config.User is a number
-func (c *Container) generatePasswd() (string, error) {
-	var (
-		groupspec string
-		gid       uint32
-	)
-	if c.config.User == "" {
-		return "", nil
-	}
-	spec := strings.SplitN(c.config.User, ":", 2)
-	userspec := spec[0]
-	if len(spec) > 1 {
-		groupspec = spec[1]
-	}
-	// If a non numeric User, then don't generate passwd
-	uid, err := strconv.ParseUint(userspec, 10, 32)
-	if err != nil {
-		return "", nil
-	}
-	// if UID exists inside of container rootfs /etc/passwd then
-	// don't generate passwd
-	if _, _, err := chrootuser.LookupUIDInContainer(c.state.Mountpoint, uid); err == nil {
-		return "", nil
-	}
-	if err == nil && groupspec != "" {
-		if !c.state.Mounted {
-			return "", errors.Wrapf(ErrCtrStateInvalid, "container %s must be mounted in order to translate group field for passwd record", c.ID())
-		}
-		gid, err = chrootuser.GetGroup(c.state.Mountpoint, groupspec)
-		if err != nil {
-			return "", errors.Wrapf(err, "unable to get gid from %s formporary passwd file")
-		}
-	}
-
-	originPasswdFile := filepath.Join(c.state.Mountpoint, "/etc/passwd")
-	orig, err := ioutil.ReadFile(originPasswdFile)
-	if err != nil {
-		return "", errors.Wrapf(err, "unable to read passwd file %s", originPasswdFile)
-	}
-
-	pwd := fmt.Sprintf("%s%d:x:%d:%d:container user:%s:/bin/sh\n", orig, uid, uid, gid, c.WorkingDir())
-	passwdFile, err := c.writeStringToRundir("passwd", pwd)
-	if err != nil {
-		return "", errors.Wrapf(err, "failed to create temporary passwd file")
-	}
-	if os.Chmod(passwdFile, 0644); err != nil {
-		return "", err
-	}
-	return passwdFile, nil
-}
-
-// generateResolvConf generates a containers resolv.conf
-func (c *Container) generateResolvConf() (string, error) {
-	// Determine the endpoint for resolv.conf in case it is a symlink
-	resolvPath, err := filepath.EvalSymlinks("/etc/resolv.conf")
-	if err != nil {
-		return "", err
-	}
-
-	contents, err := ioutil.ReadFile(resolvPath)
-	if err != nil {
-		return "", errors.Wrapf(err, "unable to read %s", resolvPath)
-	}
-
-	// Process the file to remove localhost nameservers
-	// TODO: set ipv6 enable bool more sanely
-	resolv, err := resolvconf.FilterResolvDNS(contents, true)
-	if err != nil {
-		return "", errors.Wrapf(err, "error parsing host resolv.conf")
-	}
-
-	// Make a new resolv.conf
-	nameservers := resolvconf.GetNameservers(resolv.Content)
-	if len(c.config.DNSServer) > 0 {
-		// We store DNS servers as net.IP, so need to convert to string
-		nameservers = []string{}
-		for _, server := range c.config.DNSServer {
-			nameservers = append(nameservers, server.String())
-		}
-	}
-
-	search := resolvconf.GetSearchDomains(resolv.Content)
-	if len(c.config.DNSSearch) > 0 {
-		search = c.config.DNSSearch
-	}
-
-	options := resolvconf.GetOptions(resolv.Content)
-	if len(c.config.DNSOption) > 0 {
-		options = c.config.DNSOption
-	}
-
-	destPath := filepath.Join(c.state.RunDir, "resolv.conf")
-
-	if err := os.Remove(destPath); err != nil && !os.IsNotExist(err) {
-		return "", errors.Wrapf(err, "error removing resolv.conf for container %s", c.ID())
-	}
-
-	// Build resolv.conf
-	if _, err = resolvconf.Build(destPath, nameservers, search, options); err != nil {
-		return "", errors.Wrapf(err, "error building resolv.conf for container %s")
-	}
-
-	// Relabel resolv.conf for the container
-	if err := label.Relabel(destPath, c.config.MountLabel, false); err != nil {
-		return "", err
-	}
-
-	return filepath.Join(c.state.DestinationRunDir, "resolv.conf"), nil
-}
-
-// generateHosts creates a containers hosts file
-func (c *Container) generateHosts() (string, error) {
-	orig, err := ioutil.ReadFile("/etc/hosts")
-	if err != nil {
-		return "", errors.Wrapf(err, "unable to read /etc/hosts")
-	}
-	hosts := string(orig)
-	if len(c.config.HostAdd) > 0 {
-		for _, host := range c.config.HostAdd {
-			// the host format has already been verified at this point
-			fields := strings.SplitN(host, ":", 2)
-			hosts += fmt.Sprintf("%s %s\n", fields[1], fields[0])
-		}
-	}
-	return c.writeStringToRundir("hosts", hosts)
-}
-
 func (c *Container) addLocalVolumes(ctx context.Context, g *generate.Generator) error {
+	var uid, gid int
 	mountPoint := c.state.Mountpoint
 	if !c.state.Mounted {
 		return errors.Wrapf(ErrInternal, "container is not mounted")
@@ -1176,6 +1048,18 @@ func (c *Container) addLocalVolumes(ctx context.Context, g *generate.Generator)
 		}
 	}
 
+	if c.config.User != "" {
+		if !c.state.Mounted {
+			return errors.Wrapf(ErrCtrStateInvalid, "container %s must be mounted in order to translate User field", c.ID())
+		}
+		execUser, err := lookup.GetUserGroupInfo(c.state.Mountpoint, c.config.User, nil)
+		if err != nil {
+			return err
+		}
+		uid = execUser.Uid
+		gid = execUser.Gid
+	}
+
 	for k := range imageData.ContainerConfig.Volumes {
 		mount := spec.Mount{
 			Destination: k,
@@ -1186,19 +1070,6 @@ func (c *Container) addLocalVolumes(ctx context.Context, g *generate.Generator)
 			continue
 		}
 		volumePath := filepath.Join(c.config.StaticDir, "volumes", k)
-		var (
-			uid uint32
-			gid uint32
-		)
-		if c.config.User != "" {
-			if !c.state.Mounted {
-				return errors.Wrapf(ErrCtrStateInvalid, "container %s must be mounted in order to translate User field", c.ID())
-			}
-			uid, gid, err = chrootuser.GetUser(c.state.Mountpoint, c.config.User)
-			if err != nil {
-				return err
-			}
-		}
 
 		// Ensure the symlinks are resolved
 		resolvedSymlink, err := imagebuildah.ResolveSymLink(mountPoint, k)
@@ -1218,7 +1089,7 @@ func (c *Container) addLocalVolumes(ctx context.Context, g *generate.Generator)
 				return errors.Wrapf(err, "error creating directory %q for volume %q in container %q", volumePath, k, c.ID())
 			}
 
-			if err = os.Chown(srcPath, int(uid), int(gid)); err != nil {
+			if err = os.Chown(srcPath, uid, gid); err != nil {
 				return errors.Wrapf(err, "error chowning directory %q for volume %q in container %q", srcPath, k, c.ID())
 			}
 		}
@@ -1228,7 +1099,7 @@ func (c *Container) addLocalVolumes(ctx context.Context, g *generate.Generator)
 				return errors.Wrapf(err, "error creating directory %q for volume %q in container %q", volumePath, k, c.ID())
 			}
 
-			if err = os.Chown(volumePath, int(uid), int(gid)); err != nil {
+			if err = os.Chown(volumePath, uid, gid); err != nil {
 				return errors.Wrapf(err, "error chowning directory %q for volume %q in container %q", volumePath, k, c.ID())
 			}
 
@@ -1297,10 +1168,6 @@ func (c *Container) saveSpec(spec *spec.Spec) error {
 }
 
 func (c *Container) setupOCIHooks(ctx context.Context, config *spec.Spec) (extensionStageHooks map[string][]spec.Hook, err error) {
-	if len(c.runtime.config.HooksDir) == 0 {
-		return nil, nil
-	}
-
 	var locale string
 	var ok bool
 	for _, envVar := range []string{
@@ -1328,25 +1195,39 @@ func (c *Container) setupOCIHooks(ctx context.Context, config *spec.Spec) (exten
 		}
 	}
 
-	allHooks := make(map[string][]spec.Hook)
-	for _, hDir := range c.runtime.config.HooksDir {
-		manager, err := hooks.New(ctx, []string{hDir}, []string{"poststop"}, lang)
-		if err != nil {
-			if c.runtime.config.HooksDirNotExistFatal || !os.IsNotExist(err) {
-				return nil, err
-			}
-			logrus.Warnf("failed to load hooks: {}", err)
+	if c.runtime.config.HooksDir == nil {
+		if rootless.IsRootless() {
 			return nil, nil
 		}
-		hooks, err := manager.Hooks(config, c.Spec().Annotations, len(c.config.UserVolumes) > 0)
-		if err != nil {
-			return nil, err
-		}
-		for i, hook := range hooks {
-			allHooks[i] = hook
+		allHooks := make(map[string][]spec.Hook)
+		for _, hDir := range []string{hooks.DefaultDir, hooks.OverrideDir} {
+			manager, err := hooks.New(ctx, []string{hDir}, []string{"poststop"}, lang)
+			if err != nil {
+				if os.IsNotExist(err) {
+					continue
+				}
+				return nil, err
+			}
+			hooks, err := manager.Hooks(config, c.Spec().Annotations, len(c.config.UserVolumes) > 0)
+			if err != nil {
+				return nil, err
+			}
+			if len(hooks) > 0 || config.Hooks != nil {
+				logrus.Warnf("implicit hook directories are deprecated; set --hooks-dir=%q explicitly to continue to load hooks from this directory", hDir)
+			}
+			for i, hook := range hooks {
+				allHooks[i] = hook
+			}
 		}
+		return allHooks, nil
 	}
-	return allHooks, nil
+
+	manager, err := hooks.New(ctx, c.runtime.config.HooksDir, []string{"poststop"}, lang)
+	if err != nil {
+		return nil, err
+	}
+
+	return manager.Hooks(config, c.Spec().Annotations, len(c.config.UserVolumes) > 0)
 }
 
 // mount mounts the container's root filesystem
diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go
index 0a1784ba7..b540bbeb8 100644
--- a/libpod/container_internal_linux.go
+++ b/libpod/container_internal_linux.go
@@ -11,6 +11,7 @@ import (
 	"os"
 	"path"
 	"path/filepath"
+	"strconv"
 	"strings"
 	"sync"
 	"syscall"
@@ -19,11 +20,12 @@ import (
 	cnitypes "github.com/containernetworking/cni/pkg/types/current"
 	"github.com/containernetworking/plugins/pkg/ns"
 	crioAnnotations "github.com/containers/libpod/pkg/annotations"
-	"github.com/containers/libpod/pkg/chrootuser"
 	"github.com/containers/libpod/pkg/criu"
+	"github.com/containers/libpod/pkg/lookup"
+	"github.com/containers/libpod/pkg/resolvconf"
 	"github.com/containers/libpod/pkg/rootless"
+	"github.com/containers/libpod/pkg/secrets"
 	"github.com/containers/storage/pkg/idtools"
-	"github.com/cyphar/filepath-securejoin"
 	"github.com/opencontainers/runc/libcontainer/user"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
@@ -59,7 +61,7 @@ func (c *Container) prepare() (err error) {
 		networkStatus                   []*cnitypes.Result
 		createNetNSErr, mountStorageErr error
 		mountPoint                      string
-		saveNetworkStatus               bool
+		tmpStateLock                    sync.Mutex
 	)
 
 	wg.Add(2)
@@ -68,17 +70,55 @@ func (c *Container) prepare() (err error) {
 		defer wg.Done()
 		// Set up network namespace if not already set up
 		if c.config.CreateNetNS && c.state.NetNS == nil && !c.config.PostConfigureNetNS {
-			saveNetworkStatus = true
 			netNS, networkStatus, createNetNSErr = c.runtime.createNetNS(c)
+
+			tmpStateLock.Lock()
+			defer tmpStateLock.Unlock()
+
+			// Assign NetNS attributes to container
+			if createNetNSErr == nil {
+				c.state.NetNS = netNS
+				c.state.NetworkStatus = networkStatus
+			}
 		}
 	}()
 	// Mount storage if not mounted
 	go func() {
 		defer wg.Done()
 		mountPoint, mountStorageErr = c.mountStorage()
+
+		if mountStorageErr != nil {
+			return
+		}
+
+		tmpStateLock.Lock()
+		defer tmpStateLock.Unlock()
+
+		// Finish up mountStorage
+		c.state.Mounted = true
+		c.state.Mountpoint = mountPoint
+		if c.state.UserNSRoot == "" {
+			c.state.RealMountpoint = c.state.Mountpoint
+		} else {
+			c.state.RealMountpoint = filepath.Join(c.state.UserNSRoot, "mountpoint")
+		}
+
+		logrus.Debugf("Created root filesystem for container %s at %s", c.ID(), c.state.Mountpoint)
+	}()
+
+	defer func() {
+		if err != nil {
+			if err2 := c.cleanupNetwork(); err2 != nil {
+				logrus.Errorf("Error cleaning up container %s network: %v", c.ID(), err2)
+			}
+			if err2 := c.cleanupStorage(); err2 != nil {
+				logrus.Errorf("Error cleaning up container %s storage: %v", c.ID(), err2)
+			}
+		}
 	}()
 
 	wg.Wait()
+
 	if createNetNSErr != nil {
 		if mountStorageErr != nil {
 			logrus.Error(createNetNSErr)
@@ -90,28 +130,15 @@ func (c *Container) prepare() (err error) {
 		return mountStorageErr
 	}
 
-	// Assign NetNS attributes to container
-	if saveNetworkStatus {
-		c.state.NetNS = netNS
-		c.state.NetworkStatus = networkStatus
-	}
-
-	// Finish up mountStorage
-	c.state.Mounted = true
-	c.state.Mountpoint = mountPoint
-	if c.state.UserNSRoot == "" {
-		c.state.RealMountpoint = c.state.Mountpoint
-	} else {
-		c.state.RealMountpoint = filepath.Join(c.state.UserNSRoot, "mountpoint")
-	}
-
-	logrus.Debugf("Created root filesystem for container %s at %s", c.ID(), c.state.Mountpoint)
 	// Save the container
 	return c.save()
 }
 
 // cleanupNetwork unmounts and cleans up the container's network
 func (c *Container) cleanupNetwork() error {
+	if c.NetworkDisabled() {
+		return nil
+	}
 	if c.state.NetNS == nil {
 		logrus.Debugf("Network is already cleaned up, skipping...")
 		return nil
@@ -135,6 +162,10 @@ func (c *Container) cleanupNetwork() error {
 // Generate spec for a container
 // Accepts a map of the container's dependencies
 func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
+	execUser, err := lookup.GetUserGroupInfo(c.state.Mountpoint, c.config.User, nil)
+	if err != nil {
+		return nil, err
+	}
 	g := generate.NewFromSpec(c.config.Spec)
 
 	// If network namespace was requested, add it now
@@ -145,6 +176,11 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 			g.AddOrReplaceLinuxNamespace(spec.NetworkNamespace, c.state.NetNS.Path())
 		}
 	}
+
+	if err := c.makeBindMounts(); err != nil {
+		return nil, err
+	}
+
 	// Check if the spec file mounts contain the label Relabel flags z or Z.
 	// If they do, relabel the source directory and then remove the option.
 	for _, m := range g.Mounts() {
@@ -188,11 +224,8 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 		}
 	}
 
-	var err error
-	if !rootless.IsRootless() {
-		if c.state.ExtensionStageHooks, err = c.setupOCIHooks(ctx, g.Config); err != nil {
-			return nil, errors.Wrapf(err, "error setting up OCI Hooks")
-		}
+	if c.state.ExtensionStageHooks, err = c.setupOCIHooks(ctx, g.Config); err != nil {
+		return nil, errors.Wrapf(err, "error setting up OCI Hooks")
 	}
 
 	// Bind builtin image volumes
@@ -203,28 +236,15 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 	}
 
 	if c.config.User != "" {
-		if !c.state.Mounted {
-			return nil, errors.Wrapf(ErrCtrStateInvalid, "container %s must be mounted in order to translate User field", c.ID())
-		}
-		uid, gid, err := chrootuser.GetUser(c.state.Mountpoint, c.config.User)
-		if err != nil {
-			return nil, err
-		}
 		// User and Group must go together
-		g.SetProcessUID(uid)
-		g.SetProcessGID(gid)
+		g.SetProcessUID(uint32(execUser.Uid))
+		g.SetProcessGID(uint32(execUser.Gid))
 	}
 
 	// Add addition groups if c.config.GroupAdd is not empty
 	if len(c.config.Groups) > 0 {
-		if !c.state.Mounted {
-			return nil, errors.Wrapf(ErrCtrStateInvalid, "container %s must be mounted in order to add additional groups", c.ID())
-		}
-		for _, group := range c.config.Groups {
-			gid, err := chrootuser.GetGroup(c.state.Mountpoint, group)
-			if err != nil {
-				return nil, err
-			}
+		gids, _ := lookup.GetContainerGroups(c.config.Groups, c.state.Mountpoint, nil)
+		for _, gid := range gids {
 			g.AddProcessAdditionalGid(gid)
 		}
 	}
@@ -237,26 +257,6 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 
 	// Look up and add groups the user belongs to, if a group wasn't directly specified
 	if !rootless.IsRootless() && !strings.Contains(c.config.User, ":") {
-		var groupDest, passwdDest string
-		defaultExecUser := user.ExecUser{
-			Uid:  0,
-			Gid:  0,
-			Home: "/",
-		}
-
-		// Make sure the /etc/group  and /etc/passwd destinations are not a symlink to something naughty
-		if groupDest, err = securejoin.SecureJoin(c.state.Mountpoint, "/etc/group"); err != nil {
-			logrus.Debug(err)
-			return nil, err
-		}
-		if passwdDest, err = securejoin.SecureJoin(c.state.Mountpoint, "/etc/passwd"); err != nil {
-			logrus.Debug(err)
-			return nil, err
-		}
-		execUser, err := user.GetExecUserPath(c.config.User, &defaultExecUser, passwdDest, groupDest)
-		if err != nil {
-			return nil, err
-		}
 		for _, gid := range execUser.Sgids {
 			g.AddProcessAdditionalGid(uint32(gid))
 		}
@@ -351,8 +351,34 @@ func (c *Container) generateSpec(ctx context.Context) (*spec.Spec, error) {
 	// Mounts need to be sorted so paths will not cover other paths
 	mounts := sortMounts(g.Mounts())
 	g.ClearMounts()
+
+	// Determine property of RootPropagation based on volume properties. If
+	// a volume is shared, then keep root propagation shared. This should
+	// work for slave and private volumes too.
+	//
+	// For slave volumes, it can be either [r]shared/[r]slave.
+	//
+	// For private volumes any root propagation value should work.
+	rootPropagation := ""
 	for _, m := range mounts {
 		g.AddMount(m)
+		for _, opt := range m.Options {
+			switch opt {
+			case MountShared, MountRShared:
+				if rootPropagation != MountShared && rootPropagation != MountRShared {
+					rootPropagation = MountShared
+				}
+			case MountSlave, MountRSlave:
+				if rootPropagation != MountShared && rootPropagation != MountRShared && rootPropagation != MountSlave && rootPropagation != MountRSlave {
+					rootPropagation = MountRSlave
+				}
+			}
+		}
+	}
+
+	if rootPropagation != "" {
+		logrus.Debugf("set root propagation to %q", rootPropagation)
+		g.SetLinuxRootPropagation(rootPropagation)
 	}
 	return g.Config, nil
 }
@@ -386,19 +412,31 @@ func (c *Container) setupSystemd(mounts []spec.Mount, g generate.Generator) erro
 		g.AddMount(tmpfsMnt)
 	}
 
-	cgroupPath, err := c.CGroupPath()
-	if err != nil {
-		return err
-	}
-	sourcePath := filepath.Join("/sys/fs/cgroup/systemd", cgroupPath)
+	// rootless containers have no write access to /sys/fs/cgroup, so don't
+	// add any mount into the container.
+	if !rootless.IsRootless() {
+		cgroupPath, err := c.CGroupPath()
+		if err != nil {
+			return err
+		}
+		sourcePath := filepath.Join("/sys/fs/cgroup/systemd", cgroupPath)
 
-	systemdMnt := spec.Mount{
-		Destination: "/sys/fs/cgroup/systemd",
-		Type:        "bind",
-		Source:      sourcePath,
-		Options:     []string{"bind", "private"},
+		systemdMnt := spec.Mount{
+			Destination: "/sys/fs/cgroup/systemd",
+			Type:        "bind",
+			Source:      sourcePath,
+			Options:     []string{"bind", "private"},
+		}
+		g.AddMount(systemdMnt)
+	} else {
+		systemdMnt := spec.Mount{
+			Destination: "/sys/fs/cgroup/systemd",
+			Type:        "bind",
+			Source:      "/sys/fs/cgroup/systemd",
+			Options:     []string{"bind", "nodev", "noexec", "nosuid"},
+		}
+		g.AddMount(systemdMnt)
 	}
-	g.AddMount(systemdMnt)
 
 	return nil
 }
@@ -423,7 +461,7 @@ func (c *Container) addNamespaceContainer(g *generate.Generator, ns LinuxNS, ctr
 	return nil
 }
 
-func (c *Container) checkpoint(ctx context.Context, keep bool) (err error) {
+func (c *Container) checkpoint(ctx context.Context, options ContainerCheckpointOptions) (err error) {
 
 	if !criu.CheckForCriu() {
 		return errors.Errorf("checkpointing a container requires at least CRIU %d", criu.MinCriuVersion)
@@ -432,7 +470,7 @@ func (c *Container) checkpoint(ctx context.Context, keep bool) (err error) {
 	if c.state.State != ContainerStateRunning {
 		return errors.Wrapf(ErrCtrStateInvalid, "%q is not running, cannot checkpoint", c.state.State)
 	}
-	if err := c.runtime.ociRuntime.checkpointContainer(c); err != nil {
+	if err := c.runtime.ociRuntime.checkpointContainer(c, options); err != nil {
 		return err
 	}
 
@@ -449,14 +487,16 @@ func (c *Container) checkpoint(ctx context.Context, keep bool) (err error) {
 
 	logrus.Debugf("Checkpointed container %s", c.ID())
 
-	c.state.State = ContainerStateStopped
+	if !options.KeepRunning {
+		c.state.State = ContainerStateStopped
 
-	// Cleanup Storage and Network
-	if err := c.cleanup(ctx); err != nil {
-		return err
+		// Cleanup Storage and Network
+		if err := c.cleanup(ctx); err != nil {
+			return err
+		}
 	}
 
-	if !keep {
+	if !options.Keep {
 		// Remove log file
 		os.Remove(filepath.Join(c.bundlePath(), "dump.log"))
 		// Remove statistic file
@@ -466,7 +506,7 @@ func (c *Container) checkpoint(ctx context.Context, keep bool) (err error) {
 	return c.save()
 }
 
-func (c *Container) restore(ctx context.Context, keep bool) (err error) {
+func (c *Container) restore(ctx context.Context, options ContainerCheckpointOptions) (err error) {
 
 	if !criu.CheckForCriu() {
 		return errors.Errorf("restoring a container requires at least CRIU %d", criu.MinCriuVersion)
@@ -510,9 +550,6 @@ func (c *Container) restore(ctx context.Context, keep bool) (err error) {
 		}
 	}
 
-	if err := c.prepare(); err != nil {
-		return err
-	}
 	defer func() {
 		if err != nil {
 			if err2 := c.cleanup(ctx); err2 != nil {
@@ -521,6 +558,10 @@ func (c *Container) restore(ctx context.Context, keep bool) (err error) {
 		}
 	}()
 
+	if err := c.prepare(); err != nil {
+		return err
+	}
+
 	// TODO: use existing way to request static IPs, once it is merged in ocicni
 	// https://github.com/cri-o/ocicni/pull/23/
 
@@ -553,7 +594,7 @@ func (c *Container) restore(ctx context.Context, keep bool) (err error) {
 	// Cleanup for a working restore.
 	c.removeConmonFiles()
 
-	if err := c.runtime.ociRuntime.createContainer(c, c.config.CgroupParent, true); err != nil {
+	if err := c.runtime.ociRuntime.createContainer(c, c.config.CgroupParent, &options); err != nil {
 		return err
 	}
 
@@ -561,7 +602,7 @@ func (c *Container) restore(ctx context.Context, keep bool) (err error) {
 
 	c.state.State = ContainerStateRunning
 
-	if !keep {
+	if !options.Keep {
 		// Delete all checkpoint related files. At this point, in theory, all files
 		// should exist. Still ignoring errors for now as the container should be
 		// restored and running. Not erroring out just because some cleanup operation
@@ -582,3 +623,225 @@ func (c *Container) restore(ctx context.Context, keep bool) (err error) {
 
 	return c.save()
 }
+
+// Make standard bind mounts to include in the container
+func (c *Container) makeBindMounts() error {
+	if err := os.Chown(c.state.RunDir, c.RootUID(), c.RootGID()); err != nil {
+		return errors.Wrapf(err, "cannot chown run directory %s", c.state.RunDir)
+	}
+
+	if c.state.BindMounts == nil {
+		c.state.BindMounts = make(map[string]string)
+	}
+
+	if !c.NetworkDisabled() {
+		// Make /etc/resolv.conf
+		if _, ok := c.state.BindMounts["/etc/resolv.conf"]; ok {
+			// If it already exists, delete so we can recreate
+			delete(c.state.BindMounts, "/etc/resolv.conf")
+		}
+		newResolv, err := c.generateResolvConf()
+		if err != nil {
+			return errors.Wrapf(err, "error creating resolv.conf for container %s", c.ID())
+		}
+		c.state.BindMounts["/etc/resolv.conf"] = newResolv
+
+		// Make /etc/hosts
+		if _, ok := c.state.BindMounts["/etc/hosts"]; ok {
+			// If it already exists, delete so we can recreate
+			delete(c.state.BindMounts, "/etc/hosts")
+		}
+		newHosts, err := c.generateHosts()
+		if err != nil {
+			return errors.Wrapf(err, "error creating hosts file for container %s", c.ID())
+		}
+		c.state.BindMounts["/etc/hosts"] = newHosts
+
+	}
+
+	// SHM is always added when we mount the container
+	c.state.BindMounts["/dev/shm"] = c.config.ShmDir
+
+	newPasswd, err := c.generatePasswd()
+	if err != nil {
+		return errors.Wrapf(err, "error creating temporary passwd file for container %s", c.ID())
+	}
+	if newPasswd != "" {
+		// Make /etc/passwd
+		if _, ok := c.state.BindMounts["/etc/passwd"]; ok {
+			// If it already exists, delete so we can recreate
+			delete(c.state.BindMounts, "/etc/passwd")
+		}
+		logrus.Debugf("adding entry to /etc/passwd for non existent default user")
+		c.state.BindMounts["/etc/passwd"] = newPasswd
+	}
+
+	// Make /etc/hostname
+	// This should never change, so no need to recreate if it exists
+	if _, ok := c.state.BindMounts["/etc/hostname"]; !ok {
+		hostnamePath, err := c.writeStringToRundir("hostname", c.Hostname())
+		if err != nil {
+			return errors.Wrapf(err, "error creating hostname file for container %s", c.ID())
+		}
+		c.state.BindMounts["/etc/hostname"] = hostnamePath
+	}
+
+	// Make .containerenv
+	// Empty file, so no need to recreate if it exists
+	if _, ok := c.state.BindMounts["/run/.containerenv"]; !ok {
+		// Empty string for now, but we may consider populating this later
+		containerenvPath, err := c.writeStringToRundir(".containerenv", "")
+		if err != nil {
+			return errors.Wrapf(err, "error creating containerenv file for container %s", c.ID())
+		}
+		c.state.BindMounts["/run/.containerenv"] = containerenvPath
+	}
+
+	// Add Secret Mounts
+	secretMounts := secrets.SecretMountsWithUIDGID(c.config.MountLabel, c.state.RunDir, c.runtime.config.DefaultMountsFile, c.state.DestinationRunDir, c.RootUID(), c.RootGID())
+	for _, mount := range secretMounts {
+		if _, ok := c.state.BindMounts[mount.Destination]; !ok {
+			c.state.BindMounts[mount.Destination] = mount.Source
+		}
+	}
+
+	return nil
+}
+
+// generateResolvConf generates a containers resolv.conf
+func (c *Container) generateResolvConf() (string, error) {
+	// Determine the endpoint for resolv.conf in case it is a symlink
+	resolvPath, err := filepath.EvalSymlinks("/etc/resolv.conf")
+	if err != nil {
+		return "", err
+	}
+
+	contents, err := ioutil.ReadFile(resolvPath)
+	if err != nil {
+		return "", errors.Wrapf(err, "unable to read %s", resolvPath)
+	}
+
+	// Ensure that the container's /etc/resolv.conf is compatible with its
+	// network configuration.
+	// TODO: set ipv6 enable bool more sanely
+	resolv, err := resolvconf.FilterResolvDNS(contents, true, c.config.CreateNetNS)
+	if err != nil {
+		return "", errors.Wrapf(err, "error parsing host resolv.conf")
+	}
+
+	// Make a new resolv.conf
+	nameservers := resolvconf.GetNameservers(resolv.Content)
+	if len(c.config.DNSServer) > 0 {
+		// We store DNS servers as net.IP, so need to convert to string
+		nameservers = []string{}
+		for _, server := range c.config.DNSServer {
+			nameservers = append(nameservers, server.String())
+		}
+	}
+
+	search := resolvconf.GetSearchDomains(resolv.Content)
+	if len(c.config.DNSSearch) > 0 {
+		search = c.config.DNSSearch
+	}
+
+	options := resolvconf.GetOptions(resolv.Content)
+	if len(c.config.DNSOption) > 0 {
+		options = c.config.DNSOption
+	}
+
+	destPath := filepath.Join(c.state.RunDir, "resolv.conf")
+
+	if err := os.Remove(destPath); err != nil && !os.IsNotExist(err) {
+		return "", errors.Wrapf(err, "error removing resolv.conf for container %s", c.ID())
+	}
+
+	// Build resolv.conf
+	if _, err = resolvconf.Build(destPath, nameservers, search, options); err != nil {
+		return "", errors.Wrapf(err, "error building resolv.conf for container %s", c.ID())
+	}
+
+	// Relabel resolv.conf for the container
+	if err := label.Relabel(destPath, c.config.MountLabel, false); err != nil {
+		return "", err
+	}
+
+	return filepath.Join(c.state.DestinationRunDir, "resolv.conf"), nil
+}
+
+// generateHosts creates a containers hosts file
+func (c *Container) generateHosts() (string, error) {
+	orig, err := ioutil.ReadFile("/etc/hosts")
+	if err != nil {
+		return "", errors.Wrapf(err, "unable to read /etc/hosts")
+	}
+	hosts := string(orig)
+	if len(c.config.HostAdd) > 0 {
+		for _, host := range c.config.HostAdd {
+			// the host format has already been verified at this point
+			fields := strings.SplitN(host, ":", 2)
+			hosts += fmt.Sprintf("%s %s\n", fields[1], fields[0])
+		}
+	}
+	if len(c.state.NetworkStatus) > 0 && len(c.state.NetworkStatus[0].IPs) > 0 {
+		ipAddress := strings.Split(c.state.NetworkStatus[0].IPs[0].Address.String(), "/")[0]
+		hosts += fmt.Sprintf("%s\t%s\n", ipAddress, c.Hostname())
+	}
+	return c.writeStringToRundir("hosts", hosts)
+}
+
+// generatePasswd generates a container specific passwd file,
+// iff g.config.User is a number
+func (c *Container) generatePasswd() (string, error) {
+	var (
+		groupspec string
+		gid       int
+	)
+	if c.config.User == "" {
+		return "", nil
+	}
+	spec := strings.SplitN(c.config.User, ":", 2)
+	userspec := spec[0]
+	if len(spec) > 1 {
+		groupspec = spec[1]
+	}
+	// If a non numeric User, then don't generate passwd
+	uid, err := strconv.ParseUint(userspec, 10, 32)
+	if err != nil {
+		return "", nil
+	}
+	// Lookup the user to see if it exists in the container image
+	_, err = lookup.GetUser(c.state.Mountpoint, userspec)
+	if err != nil && err != user.ErrNoPasswdEntries {
+		return "", err
+	}
+	if err == nil {
+		return "", nil
+	}
+	if groupspec != "" {
+		ugid, err := strconv.ParseUint(groupspec, 10, 32)
+		if err == nil {
+			gid = int(ugid)
+		} else {
+			group, err := lookup.GetGroup(c.state.Mountpoint, groupspec)
+			if err != nil {
+				return "", errors.Wrapf(err, "unable to get gid %s from group file", groupspec)
+			}
+			gid = group.Gid
+		}
+	}
+	originPasswdFile := filepath.Join(c.state.Mountpoint, "/etc/passwd")
+	orig, err := ioutil.ReadFile(originPasswdFile)
+	if err != nil && !os.IsNotExist(err) {
+		return "", errors.Wrapf(err, "unable to read passwd file %s", originPasswdFile)
+	}
+
+	pwd := fmt.Sprintf("%s%d:x:%d:%d:container user:%s:/bin/sh\n", orig, uid, uid, gid, c.WorkingDir())
+	passwdFile, err := c.writeStringToRundir("passwd", pwd)
+	if err != nil {
+		return "", errors.Wrapf(err, "failed to create temporary passwd file")
+	}
+	if os.Chmod(passwdFile, 0644); err != nil {
+		return "", err
+	}
+	return passwdFile, nil
+}
diff --git a/libpod/image/errors.go b/libpod/image/errors.go
new file mode 100644
index 000000000..4088946cb
--- /dev/null
+++ b/libpod/image/errors.go
@@ -0,0 +1,15 @@
+package image
+
+import (
+	"errors"
+)
+
+// Copied directly from libpod errors to avoid circular imports
+var (
+	// ErrNoSuchCtr indicates the requested container does not exist
+	ErrNoSuchCtr = errors.New("no such container")
+	// ErrNoSuchPod indicates the requested pod does not exist
+	ErrNoSuchPod = errors.New("no such pod")
+	// ErrNoSuchImage indicates the requested image does not exist
+	ErrNoSuchImage = errors.New("no such image")
+)
diff --git a/libpod/image/image.go b/libpod/image/image.go
index 7e520d97e..434f9031e 100644
--- a/libpod/image/image.go
+++ b/libpod/image/image.go
@@ -252,7 +252,7 @@ func (i *Image) getLocalImage() (*storage.Image, error) {
 	// The image has a registry name in it and we made sure we looked for it locally
 	// with a tag.  It cannot be local.
 	if decomposedImage.hasRegistry {
-		return nil, errors.Errorf("%s", imageError)
+		return nil, errors.Wrapf(ErrNoSuchImage, imageError)
 
 	}
 
@@ -275,7 +275,7 @@ func (i *Image) getLocalImage() (*storage.Image, error) {
 		return repoImage, nil
 	}
 
-	return nil, errors.Wrapf(err, imageError)
+	return nil, errors.Wrapf(ErrNoSuchImage, err.Error())
 }
 
 // ID returns the image ID as a string
@@ -869,6 +869,7 @@ func (i *Image) Inspect(ctx context.Context) (*inspect.ImageData, error) {
 		GraphDriver:  driver,
 		ManifestType: manifestType,
 		User:         ociv1Img.Config.User,
+		History:      ociv1Img.History,
 	}
 	return data, nil
 }
diff --git a/libpod/in_memory_state.go b/libpod/in_memory_state.go
index 78e765ccd..77eba0cc6 100644
--- a/libpod/in_memory_state.go
+++ b/libpod/in_memory_state.go
@@ -73,6 +73,18 @@ func (s *InMemoryState) Refresh() error {
 	return nil
 }
 
+// GetDBConfig is not implemented for in-memory state.
+// As we do not store a config, return an empty one.
+func (s *InMemoryState) GetDBConfig() (*DBConfig, error) {
+	return &DBConfig{}, nil
+}
+
+// ValidateDBConfig is not implemented for the in-memory state.
+// Since we do nothing just return no error.
+func (s *InMemoryState) ValidateDBConfig(runtime *Runtime) error {
+	return nil
+}
+
 // SetNamespace sets the namespace for container and pod retrieval.
 func (s *InMemoryState) SetNamespace(ns string) error {
 	s.namespace = ns
diff --git a/libpod/info.go b/libpod/info.go
index 4cbf3f734..5d8d160c8 100644
--- a/libpod/info.go
+++ b/libpod/info.go
@@ -12,6 +12,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/containers/libpod/pkg/rootless"
 	"github.com/containers/libpod/utils"
 	"github.com/containers/storage/pkg/system"
 	"github.com/pkg/errors"
@@ -30,6 +31,7 @@ func (r *Runtime) hostInfo() (map[string]interface{}, error) {
 	info["os"] = runtime.GOOS
 	info["arch"] = runtime.GOARCH
 	info["cpus"] = runtime.NumCPU()
+	info["rootless"] = rootless.IsRootless()
 	mi, err := system.ReadMemInfo()
 	if err != nil {
 		return nil, errors.Wrapf(err, "error reading memory info")
diff --git a/libpod/kube.go b/libpod/kube.go
new file mode 100644
index 000000000..1a5f80878
--- /dev/null
+++ b/libpod/kube.go
@@ -0,0 +1,270 @@
+package libpod
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/containers/libpod/pkg/lookup"
+	"github.com/containers/libpod/pkg/util"
+	"github.com/cri-o/ocicni/pkg/ocicni"
+	"github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	v12 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// InspectForKube takes a slice of libpod containers and generates
+// one v1.Pod description that includes just a single container.
+func (c *Container) InspectForKube() (*v1.Pod, error) {
+	// Generate the v1.Pod yaml description
+	return simplePodWithV1Container(c)
+}
+
+// simplePodWithV1Container is a function used by inspect when kube yaml needs to be generated
+// for a single container.  we "insert" that container description in a pod.
+func simplePodWithV1Container(ctr *Container) (*v1.Pod, error) {
+	var containers []v1.Container
+	result, err := containerToV1Container(ctr)
+	if err != nil {
+		return nil, err
+	}
+	containers = append(containers, result)
+
+	tm := v12.TypeMeta{
+		Kind:       "Pod",
+		APIVersion: "v1",
+	}
+
+	// Add a label called "app" with the containers name as a value
+	labels := make(map[string]string)
+	labels["app"] = removeUnderscores(ctr.Name())
+	om := v12.ObjectMeta{
+		// The name of the pod is container_name-libpod
+		Name:   fmt.Sprintf("%s-libpod", removeUnderscores(ctr.Name())),
+		Labels: labels,
+		// CreationTimestamp seems to be required, so adding it; in doing so, the timestamp
+		// will reflect time this is run (not container create time) because the conversion
+		// of the container create time to v1 Time is probably not warranted nor worthwhile.
+		CreationTimestamp: v12.Now(),
+	}
+	ps := v1.PodSpec{
+		Containers: containers,
+	}
+	p := v1.Pod{
+		TypeMeta:   tm,
+		ObjectMeta: om,
+		Spec:       ps,
+	}
+	return &p, nil
+}
+
+// containerToV1Container converts information we know about a libpod container
+// to a V1.Container specification.
+func containerToV1Container(c *Container) (v1.Container, error) {
+	kubeContainer := v1.Container{}
+	kubeSec, err := generateKubeSecurityContext(c)
+	if err != nil {
+		return kubeContainer, err
+	}
+
+	if len(c.config.Spec.Linux.Devices) > 0 {
+		// TODO Enable when we can support devices and their names
+		devices, err := generateKubeVolumeDeviceFromLinuxDevice(c.Spec().Linux.Devices)
+		if err != nil {
+			return kubeContainer, err
+		}
+		kubeContainer.VolumeDevices = devices
+		return kubeContainer, errors.Wrapf(ErrNotImplemented, "linux devices")
+	}
+
+	if len(c.config.UserVolumes) > 0 {
+		// TODO When we until we can resolve what the volume name should be, this is disabled
+		// Volume names need to be coordinated "globally" in the kube files.
+		volumes, err := libpodMountsToKubeVolumeMounts(c)
+		if err != nil {
+			return kubeContainer, err
+		}
+		kubeContainer.VolumeMounts = volumes
+		return kubeContainer, errors.Wrapf(ErrNotImplemented, "volume names")
+	}
+
+	envVariables, err := libpodEnvVarsToKubeEnvVars(c.config.Spec.Process.Env)
+	if err != nil {
+		return kubeContainer, nil
+	}
+
+	ports, err := ocicniPortMappingToContainerPort(c.PortMappings())
+	if err != nil {
+		return kubeContainer, nil
+	}
+
+	containerCommands := c.Command()
+	kubeContainer.Name = removeUnderscores(c.Name())
+
+	_, image := c.Image()
+	kubeContainer.Image = image
+	kubeContainer.Stdin = c.Stdin()
+	kubeContainer.Command = containerCommands
+	// TODO need to figure out how we handle command vs entry point.  Kube appears to prefer entrypoint.
+	// right now we just take the container's command
+	//container.Args = args
+	kubeContainer.WorkingDir = c.WorkingDir()
+	kubeContainer.Ports = ports
+	// This should not be applicable
+	//container.EnvFromSource =
+	kubeContainer.Env = envVariables
+	// TODO enable resources when we can support naming conventions
+	//container.Resources
+	kubeContainer.SecurityContext = kubeSec
+	kubeContainer.StdinOnce = false
+	kubeContainer.TTY = c.config.Spec.Process.Terminal
+
+	return kubeContainer, nil
+}
+
+// ocicniPortMappingToContainerPort takes an ocicni portmapping and converts
+// it to a v1.ContainerPort format for kube output
+func ocicniPortMappingToContainerPort(portMappings []ocicni.PortMapping) ([]v1.ContainerPort, error) {
+	var containerPorts []v1.ContainerPort
+	for _, p := range portMappings {
+		var protocol v1.Protocol
+		switch strings.ToUpper(p.Protocol) {
+		case "TCP":
+			protocol = v1.ProtocolTCP
+		case "UDP":
+			protocol = v1.ProtocolUDP
+		default:
+			return containerPorts, errors.Errorf("unknown network protocol %s", p.Protocol)
+		}
+		cp := v1.ContainerPort{
+			// Name will not be supported
+			HostPort:      p.HostPort,
+			HostIP:        p.HostIP,
+			ContainerPort: p.ContainerPort,
+			Protocol:      protocol,
+		}
+		containerPorts = append(containerPorts, cp)
+	}
+	return containerPorts, nil
+}
+
+// libpodEnvVarsToKubeEnvVars converts a key=value string slice to []v1.EnvVar
+func libpodEnvVarsToKubeEnvVars(envs []string) ([]v1.EnvVar, error) {
+	var envVars []v1.EnvVar
+	for _, e := range envs {
+		splitE := strings.SplitN(e, "=", 2)
+		if len(splitE) != 2 {
+			return envVars, errors.Errorf("environment variable %s is malformed; should be key=value", e)
+		}
+		ev := v1.EnvVar{
+			Name:  splitE[0],
+			Value: splitE[1],
+		}
+		envVars = append(envVars, ev)
+	}
+	return envVars, nil
+}
+
+// Is this worth it?
+func libpodMaxAndMinToResourceList(c *Container) (v1.ResourceList, v1.ResourceList) { //nolint
+	// It does not appear we can properly calculate CPU resources from the information
+	// we know in libpod.  Libpod knows CPUs by time, shares, etc.
+
+	// We also only know about a memory limit; no memory minimum
+	maxResources := make(map[v1.ResourceName]resource.Quantity)
+	minResources := make(map[v1.ResourceName]resource.Quantity)
+	config := c.Config()
+	maxMem := config.Spec.Linux.Resources.Memory.Limit
+
+	_ = maxMem
+
+	return maxResources, minResources
+}
+
+func generateKubeVolumeMount(hostSourcePath string, mounts []specs.Mount) (v1.VolumeMount, error) {
+	vm := v1.VolumeMount{}
+	for _, m := range mounts {
+		if m.Source == hostSourcePath {
+			// TODO Name is not provided and is required by Kube; therefore, this is disabled earlier
+			//vm.Name =
+			vm.MountPath = m.Source
+			vm.SubPath = m.Destination
+			if util.StringInSlice("ro", m.Options) {
+				vm.ReadOnly = true
+			}
+			return vm, nil
+		}
+	}
+	return vm, errors.New("unable to find mount source")
+}
+
+// libpodMountsToKubeVolumeMounts converts the containers mounts to a struct kube understands
+func libpodMountsToKubeVolumeMounts(c *Container) ([]v1.VolumeMount, error) {
+	// At this point, I dont think we can distinguish between the default
+	// volume mounts and user added ones.  For now, we pass them all.
+	var vms []v1.VolumeMount
+	for _, hostSourcePath := range c.config.UserVolumes {
+		vm, err := generateKubeVolumeMount(hostSourcePath, c.config.Spec.Mounts)
+		if err != nil {
+			return vms, err
+		}
+		vms = append(vms, vm)
+	}
+	return vms, nil
+}
+
+// generateKubeSecurityContext generates a securityContext based on the existing container
+func generateKubeSecurityContext(c *Container) (*v1.SecurityContext, error) {
+	priv := c.Privileged()
+	ro := c.IsReadOnly()
+	allowPrivEscalation := !c.Spec().Process.NoNewPrivileges
+
+	// TODO enable use of capabilities when we can figure out how to extract cap-add|remove
+	//caps := v1.Capabilities{
+	//	//Add: c.config.Spec.Process.Capabilities
+	//}
+	sc := v1.SecurityContext{
+		// TODO enable use of capabilities when we can figure out how to extract cap-add|remove
+		//Capabilities: &caps,
+		Privileged: &priv,
+		// TODO How do we know if selinux were passed into podman
+		//SELinuxOptions:
+		// RunAsNonRoot is an optional parameter; our first implementations should be root only; however
+		// I'm leaving this as a bread-crumb for later
+		//RunAsNonRoot:             &nonRoot,
+		ReadOnlyRootFilesystem:   &ro,
+		AllowPrivilegeEscalation: &allowPrivEscalation,
+	}
+
+	if c.User() != "" {
+		// It is *possible* that
+		logrus.Debugf("Looking in container for user: %s", c.User())
+		u, err := lookup.GetUser(c.state.Mountpoint, c.User())
+		if err != nil {
+			return nil, err
+		}
+		user := int64(u.Uid)
+		sc.RunAsUser = &user
+	}
+	return &sc, nil
+}
+
+// generateKubeVolumeDeviceFromLinuxDevice takes a list of devices and makes a VolumeDevice struct for kube
+func generateKubeVolumeDeviceFromLinuxDevice(devices []specs.LinuxDevice) ([]v1.VolumeDevice, error) {
+	var volumeDevices []v1.VolumeDevice
+	for _, d := range devices {
+		vd := v1.VolumeDevice{
+			// TBD How are we going to sync up these names
+			//Name:
+			DevicePath: d.Path,
+		}
+		volumeDevices = append(volumeDevices, vd)
+	}
+	return volumeDevices, nil
+}
+
+func removeUnderscores(s string) string {
+	return strings.Replace(s, "_", "", -1)
+}
diff --git a/libpod/mounts_linux.go b/libpod/mounts_linux.go
new file mode 100644
index 000000000..e6aa09eac
--- /dev/null
+++ b/libpod/mounts_linux.go
@@ -0,0 +1,18 @@
+// +build linux
+
+package libpod
+
+const (
+	// MountPrivate represents the private mount option.
+	MountPrivate = "private"
+	// MountRPrivate represents the rprivate mount option.
+	MountRPrivate = "rprivate"
+	// MountShared represents the shared mount option.
+	MountShared = "shared"
+	// MountRShared represents the rshared mount option.
+	MountRShared = "rshared"
+	// MountSlave represents the slave mount option.
+	MountSlave = "slave"
+	// MountRSlave represents the rslave mount option.
+	MountRSlave = "rslave"
+)
diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go
index 0d9ec2809..43d0a61a4 100644
--- a/libpod/networking_linux.go
+++ b/libpod/networking_linux.go
@@ -12,6 +12,7 @@ import (
 	"strconv"
 	"strings"
 	"syscall"
+	"time"
 
 	cnitypes "github.com/containernetworking/cni/pkg/types/current"
 	"github.com/containernetworking/plugins/pkg/ns"
@@ -63,20 +64,20 @@ func (r *Runtime) configureNetNS(ctr *Container, ctrNS ns.NetNS) ([]*cnitypes.Re
 		}
 	}()
 
-	networkStatus := make([]*cnitypes.Result, 1)
+	networkStatus := make([]*cnitypes.Result, 0)
 	for idx, r := range results {
 		logrus.Debugf("[%d] CNI result: %v", idx, r.String())
 		resultCurrent, err := cnitypes.GetResult(r)
 		if err != nil {
 			return nil, errors.Wrapf(err, "error parsing CNI plugin result %q: %v", r.String(), err)
 		}
-		networkStatus = append(ctr.state.NetworkStatus, resultCurrent)
+		networkStatus = append(networkStatus, resultCurrent)
 	}
 
 	// Add firewall rules to ensure the container has network access.
 	// Will not be necessary once CNI firewall plugin merges upstream.
 	// https://github.com/containernetworking/plugins/pull/75
-	for _, netStatus := range ctr.state.NetworkStatus {
+	for _, netStatus := range networkStatus {
 		firewallConf := &firewall.FirewallNetConf{
 			PrevResult: netStatus,
 		}
@@ -89,13 +90,16 @@ func (r *Runtime) configureNetNS(ctr *Container, ctrNS ns.NetNS) ([]*cnitypes.Re
 }
 
 // Create and configure a new network namespace for a container
-func (r *Runtime) createNetNS(ctr *Container) (ns.NetNS, []*cnitypes.Result, error) {
+func (r *Runtime) createNetNS(ctr *Container) (n ns.NetNS, q []*cnitypes.Result, err error) {
 	ctrNS, err := netns.NewNS()
 	if err != nil {
 		return nil, nil, errors.Wrapf(err, "error creating network namespace for container %s", ctr.ID())
 	}
 	defer func() {
 		if err != nil {
+			if err2 := netns.UnmountNS(ctrNS); err2 != nil {
+				logrus.Errorf("Error unmounting partially created network namespace for container %s: %v", ctr.ID(), err2)
+			}
 			if err2 := ctrNS.Close(); err2 != nil {
 				logrus.Errorf("Error closing partially created network namespace for container %s: %v", ctr.ID(), err2)
 			}
@@ -134,12 +138,33 @@ func (r *Runtime) setupRootlessNetNS(ctr *Container) (err error) {
 	cmd.ExtraFiles = append(cmd.ExtraFiles, ctr.rootlessSlirpSyncR, syncW)
 
 	if err := cmd.Start(); err != nil {
-		return errors.Wrapf(err, "failed to start process")
+		return errors.Wrapf(err, "failed to start slirp4netns process")
 	}
+	defer cmd.Process.Release()
 
 	b := make([]byte, 16)
-	if _, err := syncR.Read(b); err != nil {
-		return errors.Wrapf(err, "failed to read from sync pipe")
+	for {
+		if err := syncR.SetDeadline(time.Now().Add(1 * time.Second)); err != nil {
+			return errors.Wrapf(err, "error setting slirp4netns pipe timeout")
+		}
+		if _, err := syncR.Read(b); err == nil {
+			break
+		} else {
+			if os.IsTimeout(err) {
+				// Check if the process is still running.
+				var status syscall.WaitStatus
+				_, err := syscall.Wait4(cmd.Process.Pid, &status, syscall.WNOHANG, nil)
+				if err != nil {
+					return errors.Wrapf(err, "failed to read slirp4netns process status")
+				}
+				if status.Exited() || status.Signaled() {
+					return errors.New("slirp4netns failed")
+				}
+
+				continue
+			}
+			return errors.Wrapf(err, "failed to read from slirp4netns sync pipe")
+		}
 	}
 	return nil
 }
diff --git a/libpod/oci.go b/libpod/oci.go
index ca8f967c4..3222f9403 100644
--- a/libpod/oci.go
+++ b/libpod/oci.go
@@ -11,12 +11,10 @@ import (
 	"os/exec"
 	"path/filepath"
 	"runtime"
-	"strconv"
 	"strings"
 	"syscall"
 	"time"
 
-	"github.com/containers/libpod/pkg/ctime"
 	"github.com/containers/libpod/pkg/rootless"
 	"github.com/containers/libpod/pkg/util"
 	"github.com/coreos/go-systemd/activation"
@@ -229,7 +227,7 @@ func bindPorts(ports []ocicni.PortMapping) ([]*os.File, error) {
 	return files, nil
 }
 
-func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, restoreContainer bool) (err error) {
+func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, restoreOptions *ContainerCheckpointOptions) (err error) {
 	var stderrBuf bytes.Buffer
 
 	runtimeDir, err := util.GetRootlessRuntimeDir()
@@ -291,8 +289,11 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, res
 		args = append(args, "--syslog")
 	}
 
-	if restoreContainer {
+	if restoreOptions != nil {
 		args = append(args, "--restore", ctr.CheckpointPath())
+		if restoreOptions.TCPEstablished {
+			args = append(args, "--restore-arg", "--tcp-established")
+		}
 	}
 
 	logrus.WithFields(logrus.Fields{
@@ -318,6 +319,10 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, res
 	cmd.Env = append(r.conmonEnv, fmt.Sprintf("_OCI_SYNCPIPE=%d", 3))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("_OCI_STARTPIPE=%d", 4))
 	cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir))
+	cmd.Env = append(cmd.Env, fmt.Sprintf("_LIBPOD_USERNS_CONFIGURED=%s", os.Getenv("_LIBPOD_USERNS_CONFIGURED")))
+	cmd.Env = append(cmd.Env, fmt.Sprintf("_LIBPOD_ROOTLESS_UID=%s", os.Getenv("_LIBPOD_ROOTLESS_UID")))
+	cmd.Env = append(cmd.Env, fmt.Sprintf("HOME=%s", os.Getenv("HOME")))
+	cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir))
 
 	if r.reservePorts {
 		ports, err := bindPorts(ctr.config.PortMappings)
@@ -331,7 +336,7 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, res
 		cmd.ExtraFiles = append(cmd.ExtraFiles, ports...)
 	}
 
-	if rootless.IsRootless() {
+	if ctr.config.NetMode.IsSlirp4netns() {
 		ctr.rootlessSlirpSyncR, ctr.rootlessSlirpSyncW, err = os.Pipe()
 		if err != nil {
 			return errors.Wrapf(err, "failed to create rootless network sync pipe")
@@ -352,7 +357,8 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, res
 		// Set the label of the conmon process to be level :s0
 		// This will allow the container processes to talk to fifo-files
 		// passed into the container by conmon
-		plabel, err := selinux.CurrentLabel()
+		var plabel string
+		plabel, err = selinux.CurrentLabel()
 		if err != nil {
 			childPipe.Close()
 			return errors.Wrapf(err, "Failed to get current SELinux label")
@@ -362,7 +368,7 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, res
 		runtime.LockOSThread()
 		if c["level"] != "s0" && c["level"] != "" {
 			c["level"] = "s0"
-			if err := label.SetProcessLabel(c.Get()); err != nil {
+			if err = label.SetProcessLabel(c.Get()); err != nil {
 				runtime.UnlockOSThread()
 				return err
 			}
@@ -443,6 +449,7 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, res
 			}
 			return errors.Wrapf(ErrInternal, "container create failed")
 		}
+		ctr.state.PID = ss.si.Pid
 	case <-time.After(ContainerCreateTimeout):
 		return errors.Wrapf(ErrInternal, "container creation timeout")
 	}
@@ -451,17 +458,47 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, res
 
 // updateContainerStatus retrieves the current status of the container from the
 // runtime. It updates the container's state but does not save it.
-func (r *OCIRuntime) updateContainerStatus(ctr *Container) error {
-	state := new(spec.State)
+// If useRunc is false, we will not directly hit runc to see the container's
+// status, but will instead only check for the existence of the conmon exit file
+// and update state to stopped if it exists.
+func (r *OCIRuntime) updateContainerStatus(ctr *Container, useRunc bool) error {
+	exitFile := ctr.exitFilePath()
 
 	runtimeDir, err := util.GetRootlessRuntimeDir()
 	if err != nil {
 		return err
 	}
 
+	// If not using runc, we don't need to do most of this.
+	if !useRunc {
+		// If the container's not running, nothing to do.
+		if ctr.state.State != ContainerStateRunning {
+			return nil
+		}
+
+		// Check for the exit file conmon makes
+		info, err := os.Stat(exitFile)
+		if err != nil {
+			if os.IsNotExist(err) {
+				// Container is still running, no error
+				return nil
+			}
+
+			return errors.Wrapf(err, "error running stat on container %s exit file", ctr.ID())
+		}
+
+		// Alright, it exists. Transition to Stopped state.
+		ctr.state.State = ContainerStateStopped
+
+		// Read the exit file to get our stopped time and exit code.
+		return ctr.handleExitFile(exitFile, info)
+	}
+
 	// Store old state so we know if we were already stopped
 	oldState := ctr.state.State
 
+	state := new(spec.State)
+
 	cmd := exec.Command(r.path, "state", ctr.ID())
 	cmd.Env = append(cmd.Env, fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir))
 	outPipe, err := cmd.StdoutPipe()
@@ -480,6 +517,8 @@ func (r *OCIRuntime) updateContainerStatus(ctr *Container) error {
 		}
 		if strings.Contains(string(out), "does not exist") {
 			ctr.removeConmonFiles()
+			ctr.state.ExitCode = -1
+			ctr.state.FinishedTime = time.Now()
 			ctr.state.State = ContainerStateExited
 			return nil
 		}
@@ -514,7 +553,6 @@ func (r *OCIRuntime) updateContainerStatus(ctr *Container) error {
 	// Only grab exit status if we were not already stopped
 	// If we were, it should already be in the database
 	if ctr.state.State == ContainerStateStopped && oldState != ContainerStateStopped {
-		exitFile := filepath.Join(r.exitsDir, ctr.ID())
 		var fi os.FileInfo
 		err = kwait.ExponentialBackoff(
 			kwait.Backoff{
@@ -538,24 +576,7 @@ func (r *OCIRuntime) updateContainerStatus(ctr *Container) error {
 			return nil
 		}
 
-		ctr.state.FinishedTime = ctime.Created(fi)
-		statusCodeStr, err := ioutil.ReadFile(exitFile)
-		if err != nil {
-			return errors.Wrapf(err, "failed to read exit file for container %s", ctr.ID())
-		}
-		statusCode, err := strconv.Atoi(string(statusCodeStr))
-		if err != nil {
-			return errors.Wrapf(err, "error converting exit status code for container %s to int",
-				ctr.ID())
-		}
-		ctr.state.ExitCode = int32(statusCode)
-
-		oomFilePath := filepath.Join(ctr.bundlePath(), "oom")
-		if _, err = os.Stat(oomFilePath); err == nil {
-			ctr.state.OOMKilled = true
-		}
-
-		ctr.state.Exited = true
+		return ctr.handleExitFile(exitFile, fi)
 	}
 
 	return nil
@@ -570,6 +591,9 @@ func (r *OCIRuntime) startContainer(ctr *Container) error {
 		return err
 	}
 	env := []string{fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir)}
+	if notify, ok := os.LookupEnv("NOTIFY_SOCKET"); ok {
+		env = append(env, fmt.Sprintf("NOTIFY_SOCKET=%s", notify))
+	}
 	if err := utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, env, r.path, "start", ctr.ID()); err != nil {
 		return err
 	}
@@ -601,6 +625,8 @@ func (r *OCIRuntime) killContainer(ctr *Container, signal uint) error {
 // Does not set finished time for container, assumes you will run updateStatus
 // after to pull the exit code
 func (r *OCIRuntime) stopContainer(ctr *Container, timeout uint) error {
+	logrus.Debugf("Stopping container %s (PID %d)", ctr.ID(), ctr.state.PID)
+
 	// Ping the container to see if it's alive
 	// If it's not, it's already stopped, return
 	err := unix.Kill(ctr.state.PID, 0)
@@ -670,8 +696,12 @@ func (r *OCIRuntime) stopContainer(ctr *Container, timeout uint) error {
 
 // deleteContainer deletes a container from the OCI runtime
 func (r *OCIRuntime) deleteContainer(ctr *Container) error {
-	_, err := utils.ExecCmd(r.path, "delete", "--force", ctr.ID())
-	return err
+	runtimeDir, err := util.GetRootlessRuntimeDir()
+	if err != nil {
+		return err
+	}
+	env := []string{fmt.Sprintf("XDG_RUNTIME_DIR=%s", runtimeDir)}
+	return utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, env, r.path, "delete", "--force", ctr.ID())
 }
 
 // pauseContainer pauses the given container
@@ -725,6 +755,8 @@ func (r *OCIRuntime) execContainer(c *Container, cmd, capAdd, env []string, tty
 
 	if tty {
 		args = append(args, "--tty")
+	} else {
+		args = append(args, "--tty=false")
 	}
 
 	if user != "" {
@@ -828,13 +860,25 @@ func (r *OCIRuntime) execStopContainer(ctr *Container, timeout uint) error {
 }
 
 // checkpointContainer checkpoints the given container
-func (r *OCIRuntime) checkpointContainer(ctr *Container) error {
+func (r *OCIRuntime) checkpointContainer(ctr *Container, options ContainerCheckpointOptions) error {
 	// imagePath is used by CRIU to store the actual checkpoint files
 	imagePath := ctr.CheckpointPath()
 	// workPath will be used to store dump.log and stats-dump
 	workPath := ctr.bundlePath()
 	logrus.Debugf("Writing checkpoint to %s", imagePath)
 	logrus.Debugf("Writing checkpoint logs to %s", workPath)
-	return utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, nil, r.path, "checkpoint",
-		"--image-path", imagePath, "--work-path", workPath, ctr.ID())
+	args := []string{}
+	args = append(args, "checkpoint")
+	args = append(args, "--image-path")
+	args = append(args, imagePath)
+	args = append(args, "--work-path")
+	args = append(args, workPath)
+	if options.KeepRunning {
+		args = append(args, "--leave-running")
+	}
+	if options.TCPEstablished {
+		args = append(args, "--tcp-established")
+	}
+	args = append(args, ctr.ID())
+	return utils.ExecCmdWithStdStreams(os.Stdin, os.Stdout, os.Stderr, nil, r.path, args...)
 }
diff --git a/libpod/oci_linux.go b/libpod/oci_linux.go
index 0447670b3..2737a641e 100644
--- a/libpod/oci_linux.go
+++ b/libpod/oci_linux.go
@@ -19,6 +19,8 @@ import (
 	"golang.org/x/sys/unix"
 )
 
+const unknownPackage = "Unknown"
+
 func (r *OCIRuntime) moveConmonToCgroup(ctr *Container, cgroupParent string, cmd *exec.Cmd) error {
 	if os.Geteuid() == 0 {
 		if r.cgroupManager == SystemdCgroupsManager {
@@ -63,10 +65,10 @@ func newPipe() (parent *os.File, child *os.File, err error) {
 // CreateContainer creates a container in the OCI runtime
 // TODO terminal support for container
 // Presently just ignoring conmon opts related to it
-func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string, restoreContainer bool) (err error) {
+func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string, restoreOptions *ContainerCheckpointOptions) (err error) {
 	if ctr.state.UserNSRoot == "" {
 		// no need of an intermediate mount ns
-		return r.createOCIContainer(ctr, cgroupParent, restoreContainer)
+		return r.createOCIContainer(ctr, cgroupParent, restoreOptions)
 	}
 	var wg sync.WaitGroup
 	wg.Add(1)
@@ -74,7 +76,8 @@ func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string, restor
 		defer wg.Done()
 		runtime.LockOSThread()
 
-		fd, err := os.Open(fmt.Sprintf("/proc/%d/task/%d/ns/mnt", os.Getpid(), unix.Gettid()))
+		var fd *os.File
+		fd, err = os.Open(fmt.Sprintf("/proc/%d/task/%d/ns/mnt", os.Getpid(), unix.Gettid()))
 		if err != nil {
 			return
 		}
@@ -103,7 +106,7 @@ func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string, restor
 		if err != nil {
 			return
 		}
-		err = r.createOCIContainer(ctr, cgroupParent, restoreContainer)
+		err = r.createOCIContainer(ctr, cgroupParent, restoreOptions)
 	}()
 	wg.Wait()
 
@@ -111,7 +114,7 @@ func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string, restor
 }
 
 func rpmVersion(path string) string {
-	output := "Unknown"
+	output := unknownPackage
 	cmd := exec.Command("/usr/bin/rpm", "-q", "-f", path)
 	if outp, err := cmd.Output(); err == nil {
 		output = string(outp)
@@ -120,7 +123,7 @@ func rpmVersion(path string) string {
 }
 
 func dpkgVersion(path string) string {
-	output := "Unknown"
+	output := unknownPackage
 	cmd := exec.Command("/usr/bin/dpkg", "-S", path)
 	if outp, err := cmd.Output(); err == nil {
 		output = string(outp)
@@ -129,14 +132,14 @@ func dpkgVersion(path string) string {
 }
 
 func (r *OCIRuntime) pathPackage() string {
-	if out := rpmVersion(r.path); out != "Unknown" {
+	if out := rpmVersion(r.path); out != unknownPackage {
 		return out
 	}
 	return dpkgVersion(r.path)
 }
 
 func (r *OCIRuntime) conmonPackage() string {
-	if out := rpmVersion(r.conmonPath); out != "Unknown" {
+	if out := rpmVersion(r.conmonPath); out != unknownPackage {
 		return out
 	}
 	return dpkgVersion(r.conmonPath)
diff --git a/libpod/oci_unsupported.go b/libpod/oci_unsupported.go
index b133eb402..8c084d1e2 100644
--- a/libpod/oci_unsupported.go
+++ b/libpod/oci_unsupported.go
@@ -15,7 +15,7 @@ func newPipe() (parent *os.File, child *os.File, err error) {
 	return nil, nil, ErrNotImplemented
 }
 
-func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string, restoreContainer bool) (err error) {
+func (r *OCIRuntime) createContainer(ctr *Container, cgroupParent string, restoreOptions *ContainerCheckpointOptions) (err error) {
 	return ErrNotImplemented
 }
 
diff --git a/libpod/options.go b/libpod/options.go
index 8d044313b..3e43d73f0 100644
--- a/libpod/options.go
+++ b/libpod/options.go
@@ -7,6 +7,7 @@ import (
 	"regexp"
 	"syscall"
 
+	"github.com/containers/libpod/pkg/namespaces"
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/cri-o/ocicni/pkg/ocicni"
@@ -28,19 +29,40 @@ func WithStorageConfig(config storage.StoreOptions) RuntimeOption {
 			return ErrRuntimeFinalized
 		}
 
-		rt.config.StorageConfig.RunRoot = config.RunRoot
-		rt.config.StorageConfig.GraphRoot = config.GraphRoot
-		rt.config.StorageConfig.GraphDriverName = config.GraphDriverName
-		rt.config.StaticDir = filepath.Join(config.GraphRoot, "libpod")
+		if config.RunRoot != "" {
+			rt.config.StorageConfig.RunRoot = config.RunRoot
+			rt.configuredFrom.storageRunRootSet = true
+		}
 
-		rt.config.StorageConfig.GraphDriverOptions = make([]string, len(config.GraphDriverOptions))
-		copy(rt.config.StorageConfig.GraphDriverOptions, config.GraphDriverOptions)
+		if config.GraphRoot != "" {
+			rt.config.StorageConfig.GraphRoot = config.GraphRoot
+			rt.configuredFrom.storageGraphRootSet = true
 
-		rt.config.StorageConfig.UIDMap = make([]idtools.IDMap, len(config.UIDMap))
-		copy(rt.config.StorageConfig.UIDMap, config.UIDMap)
+			// Also set libpod static dir, so we are a subdirectory
+			// of the c/storage store by default
+			rt.config.StaticDir = filepath.Join(config.GraphRoot, "libpod")
+			rt.configuredFrom.libpodStaticDirSet = true
+		}
 
-		rt.config.StorageConfig.GIDMap = make([]idtools.IDMap, len(config.GIDMap))
-		copy(rt.config.StorageConfig.GIDMap, config.GIDMap)
+		if config.GraphDriverName != "" {
+			rt.config.StorageConfig.GraphDriverName = config.GraphDriverName
+			rt.configuredFrom.storageGraphDriverSet = true
+		}
+
+		if config.GraphDriverOptions != nil {
+			rt.config.StorageConfig.GraphDriverOptions = make([]string, len(config.GraphDriverOptions))
+			copy(rt.config.StorageConfig.GraphDriverOptions, config.GraphDriverOptions)
+		}
+
+		if config.UIDMap != nil {
+			rt.config.StorageConfig.UIDMap = make([]idtools.IDMap, len(config.UIDMap))
+			copy(rt.config.StorageConfig.UIDMap, config.UIDMap)
+		}
+
+		if config.GIDMap != nil {
+			rt.config.StorageConfig.GIDMap = make([]idtools.IDMap, len(config.GIDMap))
+			copy(rt.config.StorageConfig.GIDMap, config.GIDMap)
+		}
 
 		return nil
 	}
@@ -173,26 +195,26 @@ func WithStaticDir(dir string) RuntimeOption {
 		}
 
 		rt.config.StaticDir = dir
+		rt.configuredFrom.libpodStaticDirSet = true
 
 		return nil
 	}
 }
 
-// WithHooksDir sets the directory to look for OCI runtime hooks config.
-// Note we are not saving this in database, since this is really just for used
-// for testing.
-func WithHooksDir(hooksDir string) RuntimeOption {
+// WithHooksDir sets the directories to look for OCI runtime hook configuration.
+func WithHooksDir(hooksDirs ...string) RuntimeOption {
 	return func(rt *Runtime) error {
 		if rt.valid {
 			return ErrRuntimeFinalized
 		}
 
-		if hooksDir == "" {
-			return errors.Wrap(ErrInvalidArg, "empty-string hook directories are not supported")
+		for _, hooksDir := range hooksDirs {
+			if hooksDir == "" {
+				return errors.Wrap(ErrInvalidArg, "empty-string hook directories are not supported")
+			}
 		}
 
-		rt.config.HooksDir = []string{hooksDir}
-		rt.config.HooksDirNotExistFatal = true
+		rt.config.HooksDir = hooksDirs
 		return nil
 	}
 }
@@ -225,6 +247,7 @@ func WithTmpDir(dir string) RuntimeOption {
 		}
 
 		rt.config.TmpDir = dir
+		rt.configuredFrom.libpodTmpDirSet = true
 
 		return nil
 	}
@@ -817,7 +840,7 @@ func WithDependencyCtrs(ctrs []*Container) CtrCreateOption {
 // namespace with a minimal configuration.
 // An optional array of port mappings can be provided.
 // Conflicts with WithNetNSFrom().
-func WithNetNS(portMappings []ocicni.PortMapping, postConfigureNetNS bool, networks []string) CtrCreateOption {
+func WithNetNS(portMappings []ocicni.PortMapping, postConfigureNetNS bool, netmode string, networks []string) CtrCreateOption {
 	return func(ctr *Container) error {
 		if ctr.valid {
 			return ErrCtrFinalized
@@ -831,6 +854,7 @@ func WithNetNS(portMappings []ocicni.PortMapping, postConfigureNetNS bool, netwo
 		ctr.config.CreateNetNS = true
 		ctr.config.PortMappings = portMappings
 		ctr.config.Networks = networks
+		ctr.config.NetMode = namespaces.NetworkMode(netmode)
 
 		return nil
 	}
@@ -1295,3 +1319,14 @@ func WithInfraContainer() PodCreateOption {
 		return nil
 	}
 }
+
+// WithInfraContainerPorts tells the pod to add port bindings to the pause container
+func WithInfraContainerPorts(bindings []ocicni.PortMapping) PodCreateOption {
+	return func(pod *Pod) error {
+		if pod.valid {
+			return ErrPodFinalized
+		}
+		pod.config.InfraContainer.PortBindings = bindings
+		return nil
+	}
+}
diff --git a/libpod/pod.go b/libpod/pod.go
index 8ac976f6a..07f41f5c6 100644
--- a/libpod/pod.go
+++ b/libpod/pod.go
@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/containers/storage"
+	"github.com/cri-o/ocicni/pkg/ocicni"
 	"github.com/pkg/errors"
 )
 
@@ -96,7 +97,8 @@ type PodContainerInfo struct {
 
 // InfraContainerConfig is the configuration for the pod's infra container
 type InfraContainerConfig struct {
-	HasInfraContainer bool `json:"makeInfraContainer"`
+	HasInfraContainer bool                 `json:"makeInfraContainer"`
+	PortBindings      []ocicni.PortMapping `json:"infraPortBindings"`
 }
 
 // ID retrieves the pod's ID
diff --git a/libpod/pod_easyjson.go b/libpod/pod_easyjson.go
index 6c1c939f3..8ea9a5e72 100644
--- a/libpod/pod_easyjson.go
+++ b/libpod/pod_easyjson.go
@@ -6,6 +6,7 @@ package libpod
 
 import (
 	json "encoding/json"
+	ocicni "github.com/cri-o/ocicni/pkg/ocicni"
 	easyjson "github.com/mailru/easyjson"
 	jlexer "github.com/mailru/easyjson/jlexer"
 	jwriter "github.com/mailru/easyjson/jwriter"
@@ -721,6 +722,29 @@ func easyjsonBe091417DecodeGithubComContainersLibpodLibpod5(in *jlexer.Lexer, ou
 		switch key {
 		case "makeInfraContainer":
 			out.HasInfraContainer = bool(in.Bool())
+		case "infraPortBindings":
+			if in.IsNull() {
+				in.Skip()
+				out.PortBindings = nil
+			} else {
+				in.Delim('[')
+				if out.PortBindings == nil {
+					if !in.IsDelim(']') {
+						out.PortBindings = make([]ocicni.PortMapping, 0, 1)
+					} else {
+						out.PortBindings = []ocicni.PortMapping{}
+					}
+				} else {
+					out.PortBindings = (out.PortBindings)[:0]
+				}
+				for !in.IsDelim(']') {
+					var v6 ocicni.PortMapping
+					easyjsonBe091417DecodeGithubComContainersLibpodVendorGithubComCriOOcicniPkgOcicni(in, &v6)
+					out.PortBindings = append(out.PortBindings, v6)
+					in.WantComma()
+				}
+				in.Delim(']')
+			}
 		default:
 			in.SkipRecursive()
 		}
@@ -745,5 +769,109 @@ func easyjsonBe091417EncodeGithubComContainersLibpodLibpod5(out *jwriter.Writer,
 		}
 		out.Bool(bool(in.HasInfraContainer))
 	}
+	{
+		const prefix string = ",\"infraPortBindings\":"
+		if first {
+			first = false
+			out.RawString(prefix[1:])
+		} else {
+			out.RawString(prefix)
+		}
+		if in.PortBindings == nil && (out.Flags&jwriter.NilSliceAsEmpty) == 0 {
+			out.RawString("null")
+		} else {
+			out.RawByte('[')
+			for v7, v8 := range in.PortBindings {
+				if v7 > 0 {
+					out.RawByte(',')
+				}
+				easyjsonBe091417EncodeGithubComContainersLibpodVendorGithubComCriOOcicniPkgOcicni(out, v8)
+			}
+			out.RawByte(']')
+		}
+	}
+	out.RawByte('}')
+}
+func easyjsonBe091417DecodeGithubComContainersLibpodVendorGithubComCriOOcicniPkgOcicni(in *jlexer.Lexer, out *ocicni.PortMapping) {
+	isTopLevel := in.IsStart()
+	if in.IsNull() {
+		if isTopLevel {
+			in.Consumed()
+		}
+		in.Skip()
+		return
+	}
+	in.Delim('{')
+	for !in.IsDelim('}') {
+		key := in.UnsafeString()
+		in.WantColon()
+		if in.IsNull() {
+			in.Skip()
+			in.WantComma()
+			continue
+		}
+		switch key {
+		case "hostPort":
+			out.HostPort = int32(in.Int32())
+		case "containerPort":
+			out.ContainerPort = int32(in.Int32())
+		case "protocol":
+			out.Protocol = string(in.String())
+		case "hostIP":
+			out.HostIP = string(in.String())
+		default:
+			in.SkipRecursive()
+		}
+		in.WantComma()
+	}
+	in.Delim('}')
+	if isTopLevel {
+		in.Consumed()
+	}
+}
+func easyjsonBe091417EncodeGithubComContainersLibpodVendorGithubComCriOOcicniPkgOcicni(out *jwriter.Writer, in ocicni.PortMapping) {
+	out.RawByte('{')
+	first := true
+	_ = first
+	{
+		const prefix string = ",\"hostPort\":"
+		if first {
+			first = false
+			out.RawString(prefix[1:])
+		} else {
+			out.RawString(prefix)
+		}
+		out.Int32(int32(in.HostPort))
+	}
+	{
+		const prefix string = ",\"containerPort\":"
+		if first {
+			first = false
+			out.RawString(prefix[1:])
+		} else {
+			out.RawString(prefix)
+		}
+		out.Int32(int32(in.ContainerPort))
+	}
+	{
+		const prefix string = ",\"protocol\":"
+		if first {
+			first = false
+			out.RawString(prefix[1:])
+		} else {
+			out.RawString(prefix)
+		}
+		out.String(string(in.Protocol))
+	}
+	{
+		const prefix string = ",\"hostIP\":"
+		if first {
+			first = false
+			out.RawString(prefix[1:])
+		} else {
+			out.RawString(prefix)
+		}
+		out.String(string(in.HostIP))
+	}
 	out.RawByte('}')
 }
diff --git a/libpod/pod_internal.go b/libpod/pod_internal.go
index 46162c7ef..39a25c004 100644
--- a/libpod/pod_internal.go
+++ b/libpod/pod_internal.go
@@ -48,7 +48,7 @@ func (p *Pod) updatePod() error {
 // Save pod state to database
 func (p *Pod) save() error {
 	if err := p.runtime.state.SavePod(p); err != nil {
-		return errors.Wrapf(err, "error saving pod %s state")
+		return errors.Wrapf(err, "error saving pod %s state", p.ID())
 	}
 
 	return nil
diff --git a/libpod/runtime.go b/libpod/runtime.go
index 318cd0369..6b6a8cc2d 100644
--- a/libpod/runtime.go
+++ b/libpod/runtime.go
@@ -12,7 +12,6 @@ import (
 	"github.com/containers/image/types"
 	"github.com/containers/libpod/libpod/image"
 	"github.com/containers/libpod/pkg/firewall"
-	"github.com/containers/libpod/pkg/hooks"
 	sysreg "github.com/containers/libpod/pkg/registries"
 	"github.com/containers/libpod/pkg/rootless"
 	"github.com/containers/libpod/pkg/util"
@@ -84,6 +83,7 @@ type Runtime struct {
 	lock            sync.RWMutex
 	imageRuntime    *image.Runtime
 	firewallBackend firewall.FirewallBackend
+	configuredFrom  *runtimeConfiguredFrom
 }
 
 // RuntimeConfig contains configuration options used to set up the runtime
@@ -141,11 +141,11 @@ type RuntimeConfig struct {
 	// CNIDefaultNetwork is the network name of the default CNI network
 	// to attach pods to
 	CNIDefaultNetwork string `toml:"cni_default_network,omitempty"`
-	// HooksDir Path to the directory containing hooks configuration files
+	// HooksDir holds paths to the directories containing hooks
+	// configuration files. When the same filename is present in in
+	// multiple directories, the file in the directory listed last in
+	// this slice takes precedence.
 	HooksDir []string `toml:"hooks_dir"`
-	// HooksDirNotExistFatal switches between fatal errors and non-fatal
-	// warnings if the configured HooksDir does not exist.
-	HooksDirNotExistFatal bool `toml:"hooks_dir_not_exist_fatal"`
 	// DefaultMountsFile is the path to the default mounts file for testing
 	// purposes only
 	DefaultMountsFile string `toml:"-"`
@@ -175,6 +175,20 @@ type RuntimeConfig struct {
 	EnableLabeling bool `toml:"label"`
 }
 
+// runtimeConfiguredFrom is a struct used during early runtime init to help
+// assemble the full RuntimeConfig struct from defaults.
+// It indicated whether several fields in the runtime configuration were set
+// explicitly.
+// If they were not, we may override them with information from the database,
+// if it exists and differs from what is present in the system already.
+type runtimeConfiguredFrom struct {
+	storageGraphDriverSet bool
+	storageGraphRootSet   bool
+	storageRunRootSet     bool
+	libpodStaticDirSet    bool
+	libpodTmpDirSet       bool
+}
+
 var (
 	defaultRuntimeConfig = RuntimeConfig{
 		// Leave this empty so containers/storage will use its defaults
@@ -203,7 +217,6 @@ var (
 			"PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin",
 		},
 		CgroupManager:         SystemdCgroupsManager,
-		HooksDir:              []string{hooks.DefaultDir, hooks.OverrideDir},
 		StaticDir:             filepath.Join(storage.DefaultStoreOptions.GraphRoot, "libpod"),
 		TmpDir:                "",
 		MaxLogSize:            -1,
@@ -253,6 +266,7 @@ func SetXdgRuntimeDir(val string) error {
 func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) {
 	runtime = new(Runtime)
 	runtime.config = new(RuntimeConfig)
+	runtime.configuredFrom = new(runtimeConfiguredFrom)
 
 	// Copy the default configuration
 	tmpDir, err := getDefaultTmpDir()
@@ -262,8 +276,19 @@ func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) {
 	deepcopier.Copy(defaultRuntimeConfig).To(runtime.config)
 	runtime.config.TmpDir = tmpDir
 
+	if rootless.IsRootless() {
+		// If we're rootless, override the default storage config
+		storageConf, err := util.GetDefaultStoreOptions()
+		if err != nil {
+			return nil, errors.Wrapf(err, "error retrieving rootless storage config")
+		}
+		runtime.config.StorageConfig = storageConf
+		runtime.config.StaticDir = filepath.Join(storageConf.GraphRoot, "libpod")
+	}
+
 	configPath := ConfigPath
 	foundConfig := true
+	rootlessConfigPath := ""
 	if rootless.IsRootless() {
 		home := os.Getenv("HOME")
 		if runtime.config.SignaturePolicyPath == "" {
@@ -272,7 +297,10 @@ func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) {
 				runtime.config.SignaturePolicyPath = newPath
 			}
 		}
-		configPath = filepath.Join(home, ".config/containers/libpod.conf")
+
+		rootlessConfigPath = filepath.Join(home, ".config/containers/libpod.conf")
+
+		configPath = rootlessConfigPath
 		if _, err := os.Stat(configPath); err != nil {
 			foundConfig = false
 		}
@@ -303,6 +331,25 @@ func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) {
 		if err != nil {
 			return nil, errors.Wrapf(err, "error reading configuration file %s", configPath)
 		}
+
+		// This is ugly, but we need to decode twice.
+		// Once to check if libpod static and tmp dirs were explicitly
+		// set (not enough to check if they're not the default value,
+		// might have been explicitly configured to the default).
+		// A second time to actually get a usable config.
+		tmpConfig := new(RuntimeConfig)
+		if _, err := toml.Decode(string(contents), tmpConfig); err != nil {
+			return nil, errors.Wrapf(err, "error decoding configuration file %s",
+				configPath)
+		}
+
+		if tmpConfig.StaticDir != "" {
+			runtime.configuredFrom.libpodStaticDirSet = true
+		}
+		if tmpConfig.TmpDir != "" {
+			runtime.configuredFrom.libpodTmpDirSet = true
+		}
+
 		if _, err := toml.Decode(string(contents), runtime.config); err != nil {
 			return nil, errors.Wrapf(err, "error decoding configuration file %s", configPath)
 		}
@@ -317,6 +364,22 @@ func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) {
 	if err := makeRuntime(runtime); err != nil {
 		return nil, err
 	}
+
+	if !foundConfig && rootlessConfigPath != "" {
+		os.MkdirAll(filepath.Dir(rootlessConfigPath), 0755)
+		file, err := os.OpenFile(rootlessConfigPath, os.O_RDWR|os.O_CREATE|os.O_EXCL, 0666)
+		if err != nil && !os.IsExist(err) {
+			return nil, errors.Wrapf(err, "cannot open file %s", rootlessConfigPath)
+		}
+		if err == nil {
+			defer file.Close()
+			enc := toml.NewEncoder(file)
+			if err := enc.Encode(runtime.config); err != nil {
+				os.Remove(rootlessConfigPath)
+			}
+		}
+	}
+
 	return runtime, nil
 }
 
@@ -328,6 +391,7 @@ func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) {
 func NewRuntimeFromConfig(configPath string, options ...RuntimeOption) (runtime *Runtime, err error) {
 	runtime = new(Runtime)
 	runtime.config = new(RuntimeConfig)
+	runtime.configuredFrom = new(runtimeConfiguredFrom)
 
 	// Set two fields not in the TOML config
 	runtime.config.StateType = defaultRuntimeConfig.StateType
@@ -406,6 +470,77 @@ func makeRuntime(runtime *Runtime) (err error) {
 			runtime.config.ConmonPath)
 	}
 
+	// Make the static files directory if it does not exist
+	if err := os.MkdirAll(runtime.config.StaticDir, 0700); err != nil {
+		// The directory is allowed to exist
+		if !os.IsExist(err) {
+			return errors.Wrapf(err, "error creating runtime static files directory %s",
+				runtime.config.StaticDir)
+		}
+	}
+
+	// Set up the state
+	switch runtime.config.StateType {
+	case InMemoryStateStore:
+		state, err := NewInMemoryState()
+		if err != nil {
+			return err
+		}
+		runtime.state = state
+	case SQLiteStateStore:
+		return errors.Wrapf(ErrInvalidArg, "SQLite state is currently disabled")
+	case BoltDBStateStore:
+		dbPath := filepath.Join(runtime.config.StaticDir, "bolt_state.db")
+
+		state, err := NewBoltState(dbPath, runtime)
+		if err != nil {
+			return err
+		}
+		runtime.state = state
+	default:
+		return errors.Wrapf(ErrInvalidArg, "unrecognized state type passed")
+	}
+
+	// Grab config from the database so we can reset some defaults
+	dbConfig, err := runtime.state.GetDBConfig()
+	if err != nil {
+		return errors.Wrapf(err, "error retrieving runtime configuration from database")
+	}
+
+	// Reset defaults if they were not explicitly set
+	if !runtime.configuredFrom.storageGraphDriverSet && dbConfig.GraphDriver != "" {
+		runtime.config.StorageConfig.GraphDriverName = dbConfig.GraphDriver
+	}
+	if !runtime.configuredFrom.storageGraphRootSet && dbConfig.StorageRoot != "" {
+		runtime.config.StorageConfig.GraphRoot = dbConfig.StorageRoot
+	}
+	if !runtime.configuredFrom.storageRunRootSet && dbConfig.StorageTmp != "" {
+		runtime.config.StorageConfig.RunRoot = dbConfig.StorageTmp
+	}
+	if !runtime.configuredFrom.libpodStaticDirSet && dbConfig.LibpodRoot != "" {
+		runtime.config.StaticDir = dbConfig.LibpodRoot
+	}
+	if !runtime.configuredFrom.libpodTmpDirSet && dbConfig.LibpodTmp != "" {
+		runtime.config.TmpDir = dbConfig.LibpodTmp
+	}
+
+	logrus.Debugf("Using graph driver %s", runtime.config.StorageConfig.GraphDriverName)
+	logrus.Debugf("Using graph root %s", runtime.config.StorageConfig.GraphRoot)
+	logrus.Debugf("Using run root %s", runtime.config.StorageConfig.RunRoot)
+	logrus.Debugf("Using static dir %s", runtime.config.StaticDir)
+	logrus.Debugf("Using tmp dir %s", runtime.config.TmpDir)
+
+	// Validate our config against the database, now that we've set our
+	// final storage configuration
+	if err := runtime.state.ValidateDBConfig(runtime); err != nil {
+		return err
+	}
+
+	if err := runtime.state.SetNamespace(runtime.config.Namespace); err != nil {
+		return errors.Wrapf(err, "error setting libpod namespace in state")
+	}
+	logrus.Debugf("Set libpod namespace to %q", runtime.config.Namespace)
+
 	// Set up containers/storage
 	var store storage.Store
 	if rootless.SkipStorageSetup() {
@@ -473,15 +608,6 @@ func makeRuntime(runtime *Runtime) (err error) {
 	}
 	runtime.ociRuntime = ociRuntime
 
-	// Make the static files directory if it does not exist
-	if err := os.MkdirAll(runtime.config.StaticDir, 0755); err != nil {
-		// The directory is allowed to exist
-		if !os.IsExist(err) {
-			return errors.Wrapf(err, "error creating runtime static files directory %s",
-				runtime.config.StaticDir)
-		}
-	}
-
 	// Make a directory to hold container lockfiles
 	lockDir := filepath.Join(runtime.config.TmpDir, "lock")
 	if err := os.MkdirAll(lockDir, 0755); err != nil {
@@ -503,11 +629,13 @@ func makeRuntime(runtime *Runtime) (err error) {
 	}
 
 	// Set up the CNI net plugin
-	netPlugin, err := ocicni.InitCNI(runtime.config.CNIDefaultNetwork, runtime.config.CNIConfigDir, runtime.config.CNIPluginDir...)
-	if err != nil {
-		return errors.Wrapf(err, "error configuring CNI network plugin")
+	if !rootless.IsRootless() {
+		netPlugin, err := ocicni.InitCNI(runtime.config.CNIDefaultNetwork, runtime.config.CNIConfigDir, runtime.config.CNIPluginDir...)
+		if err != nil {
+			return errors.Wrapf(err, "error configuring CNI network plugin")
+		}
+		runtime.netPlugin = netPlugin
 	}
-	runtime.netPlugin = netPlugin
 
 	// Set up a firewall backend
 	backendType := ""
@@ -520,33 +648,6 @@ func makeRuntime(runtime *Runtime) (err error) {
 	}
 	runtime.firewallBackend = fwBackend
 
-	// Set up the state
-	switch runtime.config.StateType {
-	case InMemoryStateStore:
-		state, err := NewInMemoryState()
-		if err != nil {
-			return err
-		}
-		runtime.state = state
-	case SQLiteStateStore:
-		return errors.Wrapf(ErrInvalidArg, "SQLite state is currently disabled")
-	case BoltDBStateStore:
-		dbPath := filepath.Join(runtime.config.StaticDir, "bolt_state.db")
-
-		state, err := NewBoltState(dbPath, runtime.lockDir, runtime)
-		if err != nil {
-			return err
-		}
-		runtime.state = state
-	default:
-		return errors.Wrapf(ErrInvalidArg, "unrecognized state type passed")
-	}
-
-	if err := runtime.state.SetNamespace(runtime.config.Namespace); err != nil {
-		return errors.Wrapf(err, "error setting libpod namespace in state")
-	}
-	logrus.Debugf("Set libpod namespace to %q", runtime.config.Namespace)
-
 	// We now need to see if the system has restarted
 	// We check for the presence of a file in our tmp directory to verify this
 	// This check must be locked to prevent races
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
index b63726f29..09d0ec042 100644
--- a/libpod/runtime_ctr.go
+++ b/libpod/runtime_ctr.go
@@ -246,7 +246,19 @@ func (r *Runtime) removeContainer(ctx context.Context, c *Container, force bool)
 	}
 
 	if c.state.State == ContainerStatePaused {
-		return errors.Wrapf(ErrCtrStateInvalid, "container %s is paused, cannot remove until unpaused", c.ID())
+		if !force {
+			return errors.Wrapf(ErrCtrStateInvalid, "container %s is paused, cannot remove until unpaused", c.ID())
+		}
+		if err := c.runtime.ociRuntime.killContainer(c, 9); err != nil {
+			return err
+		}
+		if err := c.unpause(); err != nil {
+			return err
+		}
+		// Need to update container state to make sure we know it's stopped
+		if err := c.waitForExitFileAndSync(); err != nil {
+			return err
+		}
 	}
 
 	// Check that the container's in a good state to be removed
@@ -256,7 +268,7 @@ func (r *Runtime) removeContainer(ctx context.Context, c *Container, force bool)
 		}
 
 		// Need to update container state to make sure we know it's stopped
-		if err := c.syncContainer(); err != nil {
+		if err := c.waitForExitFileAndSync(); err != nil {
 			return err
 		}
 	} else if !(c.state.State == ContainerStateConfigured ||
diff --git a/libpod/runtime_pod_infra_linux.go b/libpod/runtime_pod_infra_linux.go
index fea79e994..8a5dbef56 100644
--- a/libpod/runtime_pod_infra_linux.go
+++ b/libpod/runtime_pod_infra_linux.go
@@ -7,7 +7,6 @@ import (
 
 	"github.com/containers/libpod/libpod/image"
 	"github.com/containers/libpod/pkg/rootless"
-	"github.com/cri-o/ocicni/pkg/ocicni"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/opencontainers/runtime-tools/generate"
 )
@@ -50,9 +49,12 @@ func (r *Runtime) makeInfraContainer(ctx context.Context, p *Pod, imgName, imgID
 	options = append(options, withIsInfra())
 
 	// Since user namespace sharing is not implemented, we only need to check if it's rootless
-	portMappings := make([]ocicni.PortMapping, 0)
 	networks := make([]string, 0)
-	options = append(options, WithNetNS(portMappings, isRootless, networks))
+	netmode := "bridge"
+	if isRootless {
+		netmode = "slirp4netns"
+	}
+	options = append(options, WithNetNS(p.config.InfraContainer.PortBindings, isRootless, netmode, networks))
 
 	return r.newContainer(ctx, g.Config, options...)
 }
diff --git a/libpod/runtime_pod_linux.go b/libpod/runtime_pod_linux.go
index eb3d471dd..3d6fad52f 100644
--- a/libpod/runtime_pod_linux.go
+++ b/libpod/runtime_pod_linux.go
@@ -265,15 +265,26 @@ func (r *Runtime) removePod(ctx context.Context, p *Pod, removeCtrs, force bool)
 			}
 		case CgroupfsCgroupsManager:
 			// Delete the cgroupfs cgroup
+			// Make sure the conmon cgroup is deleted first
+			// Since the pod is almost gone, don't bother failing
+			// hard - instead, just log errors.
 			v1CGroups := GetV1CGroups(getExcludedCGroups())
+			conmonCgroupPath := filepath.Join(p.state.CgroupPath, "conmon")
+			conmonCgroup, err := cgroups.Load(v1CGroups, cgroups.StaticPath(conmonCgroupPath))
+			if err != nil && err != cgroups.ErrCgroupDeleted {
+				return err
+			}
+			if err == nil {
+				if err := conmonCgroup.Delete(); err != nil {
+					logrus.Errorf("Error deleting pod %s conmon cgroup %s: %v", p.ID(), conmonCgroupPath, err)
+				}
+			}
 			cgroup, err := cgroups.Load(v1CGroups, cgroups.StaticPath(p.state.CgroupPath))
 			if err != nil && err != cgroups.ErrCgroupDeleted {
 				return err
-			} else if err == nil {
+			}
+			if err == nil {
 				if err := cgroup.Delete(); err != nil {
-					// The pod is already almost gone.
-					// No point in hard-failing if we fail
-					// this bit of cleanup.
 					logrus.Errorf("Error deleting pod %s cgroup %s: %v", p.ID(), p.state.CgroupPath, err)
 				}
 			}
diff --git a/libpod/state.go b/libpod/state.go
index 273e81318..06c2003d8 100644
--- a/libpod/state.go
+++ b/libpod/state.go
@@ -1,5 +1,15 @@
 package libpod
 
+// DBConfig is a set of Libpod runtime configuration settings that are saved
+// in a State when it is first created, and can subsequently be retrieved.
+type DBConfig struct {
+	LibpodRoot  string
+	LibpodTmp   string
+	StorageRoot string
+	StorageTmp  string
+	GraphDriver string
+}
+
 // State is a storage backend for libpod's current state.
 // A State is only initialized once per instance of libpod.
 // As such, initialization methods for State implementations may safely assume
@@ -21,6 +31,22 @@ type State interface {
 	// Refresh clears container and pod states after a reboot
 	Refresh() error
 
+	// GetDBConfig retrieves several paths configured within the database
+	// when it was created - namely, Libpod root and tmp dirs, c/storage
+	// root and tmp dirs, and c/storage graph driver.
+	// This is not implemented by the in-memory state, as it has no need to
+	// validate runtime configuration.
+	GetDBConfig() (*DBConfig, error)
+
+	// ValidateDBConfig validates the config in the given Runtime struct
+	// against paths stored in the configured database.
+	// Libpod root and tmp dirs and c/storage root and tmp dirs and graph
+	// driver are validated.
+	// This is not implemented by the in-memory state, as it has no need to
+	// validate runtime configuration that may change over multiple runs of
+	// the program.
+	ValidateDBConfig(runtime *Runtime) error
+
 	// SetNamespace() sets the namespace for the store, and will determine
 	// what containers are retrieved with container and pod retrieval calls.
 	// A namespace of "", the empty string, acts as no namespace, and
diff --git a/libpod/state_test.go b/libpod/state_test.go
index 04572fb29..d93a371f3 100644
--- a/libpod/state_test.go
+++ b/libpod/state_test.go
@@ -45,11 +45,16 @@ func getEmptyBoltState() (s State, p string, p2 string, err error) {
 	dbPath := filepath.Join(tmpDir, "db.sql")
 	lockDir := filepath.Join(tmpDir, "locks")
 
+	if err := os.Mkdir(lockDir, 0755); err != nil {
+		return nil, "", "", err
+	}
+
 	runtime := new(Runtime)
 	runtime.config = new(RuntimeConfig)
 	runtime.config.StorageConfig = storage.StoreOptions{}
+	runtime.lockDir = lockDir
 
-	state, err := NewBoltState(dbPath, lockDir, runtime)
+	state, err := NewBoltState(dbPath, runtime)
 	if err != nil {
 		return nil, "", "", err
 	}
diff --git a/libpod/testdata/config.toml b/libpod/testdata/config.toml
index e19d36017..1d78f2083 100644
--- a/libpod/testdata/config.toml
+++ b/libpod/testdata/config.toml
@@ -14,7 +14,7 @@
     seccomp_profile = "/etc/crio/seccomp.json"
     apparmor_profile = "crio-default"
     cgroup_manager = "cgroupfs"
-    hooks_dir_path = "/usr/share/containers/oci/hooks.d"
+    hooks_dir = ["/usr/share/containers/oci/hooks.d"]
     pids_limit = 2048
     container_exits_dir = "/var/run/podman/exits"
   [crio.image]
diff --git a/libpod/util.go b/libpod/util.go
index 7007b29cd..aa3494529 100644
--- a/libpod/util.go
+++ b/libpod/util.go
@@ -13,6 +13,7 @@ import (
 	"github.com/containers/image/signature"
 	"github.com/containers/image/types"
 	"github.com/containers/libpod/pkg/util"
+	"github.com/fsnotify/fsnotify"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 )
@@ -90,31 +91,64 @@ func MountExists(specMounts []spec.Mount, dest string) bool {
 }
 
 // WaitForFile waits until a file has been created or the given timeout has occurred
-func WaitForFile(path string, timeout time.Duration) error {
+func WaitForFile(path string, chWait chan error, timeout time.Duration) (bool, error) {
 	done := make(chan struct{})
 	chControl := make(chan struct{})
+
+	var inotifyEvents chan fsnotify.Event
+	var timer chan struct{}
+	watcher, err := fsnotify.NewWatcher()
+	if err == nil {
+		if err := watcher.Add(filepath.Dir(path)); err == nil {
+			inotifyEvents = watcher.Events
+		}
+		defer watcher.Close()
+	}
+	if inotifyEvents == nil {
+		// If for any reason we fail to create the inotify
+		// watcher, fallback to polling the file
+		timer = make(chan struct{})
+		go func() {
+			select {
+			case <-chControl:
+				close(timer)
+				return
+			default:
+				time.Sleep(25 * time.Millisecond)
+				timer <- struct{}{}
+			}
+		}()
+	}
+
 	go func() {
 		for {
 			select {
 			case <-chControl:
 				return
-			default:
+			case <-timer:
+				_, err := os.Stat(path)
+				if err == nil {
+					close(done)
+					return
+				}
+			case <-inotifyEvents:
 				_, err := os.Stat(path)
 				if err == nil {
 					close(done)
 					return
 				}
-				time.Sleep(25 * time.Millisecond)
 			}
 		}
 	}()
 
 	select {
+	case e := <-chWait:
+		return true, e
 	case <-done:
-		return nil
+		return false, nil
 	case <-time.After(timeout):
 		close(chControl)
-		return errors.Wrapf(ErrInternal, "timed out waiting for file %s", path)
+		return false, errors.Wrapf(ErrInternal, "timed out waiting for file %s", path)
 	}
 }
 
diff --git a/libpod/version.go b/libpod/version.go
index 5e7cd83c9..966588ae9 100644
--- a/libpod/version.go
+++ b/libpod/version.go
@@ -11,10 +11,10 @@ import (
 var (
 	// GitCommit is the commit that the binary is being built from.
 	// It will be populated by the Makefile.
-	GitCommit string
+	gitCommit string
 	// BuildInfo is the time at which the binary was built
 	// It will be populated by the Makefile.
-	BuildInfo string
+	buildInfo string
 )
 
 //Version is an output struct for varlink
@@ -30,9 +30,9 @@ type Version struct {
 func GetVersion() (Version, error) {
 	var err error
 	var buildTime int64
-	if BuildInfo != "" {
+	if buildInfo != "" {
 		// Converts unix time from string to int64
-		buildTime, err = strconv.ParseInt(BuildInfo, 10, 64)
+		buildTime, err = strconv.ParseInt(buildInfo, 10, 64)
 
 		if err != nil {
 			return Version{}, err
@@ -41,7 +41,7 @@ func GetVersion() (Version, error) {
 	return Version{
 		Version:   podmanVersion.Version,
 		GoVersion: runtime.Version(),
-		GitCommit: GitCommit,
+		GitCommit: gitCommit,
 		Built:     buildTime,
 		OsArch:    runtime.GOOS + "/" + runtime.GOARCH,
 	}, nil