17 files changed, 408 insertions, 312 deletions

diff --git a/pkg/adapter/checkpoint_restore.go b/pkg/adapter/checkpoint_restore.go
new file mode 100644
index 000000000..97ba5ecf7
--- /dev/null
+++ b/pkg/adapter/checkpoint_restore.go
@@ -0,0 +1,145 @@
+// +build !remoteclient
+
+package adapter
+
+import (
+	"context"
+	"github.com/containers/libpod/libpod"
+	"github.com/containers/libpod/libpod/image"
+	"github.com/containers/storage/pkg/archive"
+	jsoniter "github.com/json-iterator/go"
+	spec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/pkg/errors"
+	"io"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+)
+
+// Prefixing the checkpoint/restore related functions with 'cr'
+
+// crImportFromJSON imports the JSON files stored in the exported
+// checkpoint tarball
+func crImportFromJSON(filePath string, v interface{}) error {
+	jsonFile, err := os.Open(filePath)
+	if err != nil {
+		return errors.Wrapf(err, "Failed to open container definition %s for restore", filePath)
+	}
+	defer jsonFile.Close()
+
+	content, err := ioutil.ReadAll(jsonFile)
+	if err != nil {
+		return errors.Wrapf(err, "Failed to read container definition %s for restore", filePath)
+	}
+	json := jsoniter.ConfigCompatibleWithStandardLibrary
+	if err = json.Unmarshal([]byte(content), v); err != nil {
+		return errors.Wrapf(err, "Failed to unmarshal container definition %s for restore", filePath)
+	}
+
+	return nil
+}
+
+// crImportCheckpoint it the function which imports the information
+// from checkpoint tarball and re-creates the container from that information
+func crImportCheckpoint(ctx context.Context, runtime *libpod.Runtime, input string, name string) ([]*libpod.Container, error) {
+	// First get the container definition from the
+	// tarball to a temporary directory
+	archiveFile, err := os.Open(input)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Failed to open checkpoint archive %s for import", input)
+	}
+	defer archiveFile.Close()
+	options := &archive.TarOptions{
+		// Here we only need the files config.dump and spec.dump
+		ExcludePatterns: []string{
+			"checkpoint",
+			"artifacts",
+			"ctr.log",
+			"network.status",
+		},
+	}
+	dir, err := ioutil.TempDir("", "checkpoint")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(dir)
+	err = archive.Untar(archiveFile, dir, options)
+	if err != nil {
+		return nil, errors.Wrapf(err, "Unpacking of checkpoint archive %s failed", input)
+	}
+
+	// Load spec.dump from temporary directory
+	spec := new(spec.Spec)
+	if err := crImportFromJSON(filepath.Join(dir, "spec.dump"), spec); err != nil {
+		return nil, err
+	}
+
+	// Load config.dump from temporary directory
+	config := new(libpod.ContainerConfig)
+	if err = crImportFromJSON(filepath.Join(dir, "config.dump"), config); err != nil {
+		return nil, err
+	}
+
+	// This should not happen as checkpoints with these options are not exported.
+	if (len(config.Dependencies) > 0) || (len(config.NamedVolumes) > 0) {
+		return nil, errors.Errorf("Cannot import checkpoints of containers with named volumes or dependencies")
+	}
+
+	ctrID := config.ID
+	newName := false
+
+	// Check if the restored container gets a new name
+	if name != "" {
+		config.ID = ""
+		config.Name = name
+		newName = true
+	}
+
+	ctrName := config.Name
+
+	// The code to load the images is copied from create.go
+	var writer io.Writer
+	// In create.go this only set if '--quiet' does not exist.
+	writer = os.Stderr
+	rtc, err := runtime.GetConfig()
+	if err != nil {
+		return nil, err
+	}
+
+	_, err = runtime.ImageRuntime().New(ctx, config.RootfsImageName, rtc.SignaturePolicyPath, "", writer, nil, image.SigningOptions{}, false, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	// Now create a new container from the just loaded information
+	container, err := runtime.RestoreContainer(ctx, spec, config)
+	if err != nil {
+		return nil, err
+	}
+
+	var containers []*libpod.Container
+	if container == nil {
+		return nil, nil
+	}
+
+	containerConfig := container.Config()
+	if containerConfig.Name != ctrName {
+		return nil, errors.Errorf("Name of restored container (%s) does not match requested name (%s)", containerConfig.Name, ctrName)
+	}
+
+	if newName == false {
+		// Only check ID for a restore with the same name.
+		// Using -n to request a new name for the restored container, will also create a new ID
+		if containerConfig.ID != ctrID {
+			return nil, errors.Errorf("ID of restored container (%s) does not match requested ID (%s)", containerConfig.ID, ctrID)
+		}
+	}
+
+	// Check if the ExitCommand points to the correct container ID
+	if containerConfig.ExitCommand[len(containerConfig.ExitCommand)-1] != containerConfig.ID {
+		return nil, errors.Errorf("'ExitCommandID' uses ID %s instead of container ID %s", containerConfig.ExitCommand[len(containerConfig.ExitCommand)-1], containerConfig.ID)
+	}
+
+	containers = append(containers, container)
+	return containers, nil
+}
diff --git a/pkg/adapter/client.go b/pkg/adapter/client.go
index 01914834f..69aa3220a 100644
--- a/pkg/adapter/client.go
+++ b/pkg/adapter/client.go
@@ -6,42 +6,52 @@ import (
 	"fmt"
 	"os"
 
+	"github.com/containers/libpod/cmd/podman/remoteclientconfig"
 	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
 	"github.com/varlink/go/varlink"
 )
 
 var remoteEndpoint *Endpoint
 
 func (r RemoteRuntime) RemoteEndpoint() (remoteEndpoint *Endpoint, err error) {
-	if remoteEndpoint == nil {
-		remoteEndpoint = &Endpoint{Unknown, ""}
-	} else {
-		return remoteEndpoint, nil
-	}
+	remoteConfigConnections, _ := remoteclientconfig.ReadRemoteConfig(r.config)
 
-	// I'm leaving this here for now as a document of the birdge format.  It can be removed later once the bridge
-	// function is more flushed out.
-	// bridge := `ssh -T root@192.168.122.1 "/usr/bin/varlink -A '/usr/bin/podman varlink \$VARLINK_ADDRESS' bridge"`
-	if len(r.cmd.RemoteHost) > 0 {
-		// The user has provided a remote host endpoint
+	// If the user defines an env variable for podman_varlink_bridge
+	// we use that as passed.
+	if bridge := os.Getenv("PODMAN_VARLINK_BRIDGE"); bridge != "" {
+		logrus.Debug("creating a varlink bridge based on env variable")
+		remoteEndpoint, err = newBridgeConnection(bridge, nil, r.cmd.LogLevel)
+		// if an environment variable for podman_varlink_address is defined,
+		// we used that as passed
+	} else if address := os.Getenv("PODMAN_VARLINK_ADDRESS"); address != "" {
+		logrus.Debug("creating a varlink address based on env variable: %s", address)
+		remoteEndpoint, err = newSocketConnection(address)
+		//	if the user provides a remote host, we use it to configure a bridge connection
+	} else if len(r.cmd.RemoteHost) > 0 {
+		logrus.Debug("creating a varlink bridge based on user input")
 		if len(r.cmd.RemoteUserName) < 1 {
 			return nil, errors.New("you must provide a username when providing a remote host name")
 		}
-		remoteEndpoint.Type = BridgeConnection
-		remoteEndpoint.Connection = fmt.Sprintf(
-			`ssh -T %s@%s /usr/bin/varlink -A \'/usr/bin/podman --log-level=%s varlink \\\$VARLINK_ADDRESS\' bridge`,
-			r.cmd.RemoteUserName, r.cmd.RemoteHost, r.cmd.LogLevel)
-
-	} else if bridge := os.Getenv("PODMAN_VARLINK_BRIDGE"); bridge != "" {
-		remoteEndpoint.Type = BridgeConnection
-		remoteEndpoint.Connection = bridge
-	} else {
-		address := os.Getenv("PODMAN_VARLINK_ADDRESS")
-		if address == "" {
-			address = DefaultAddress
+		rc := remoteclientconfig.RemoteConnection{r.cmd.RemoteHost, r.cmd.RemoteUserName, false}
+		remoteEndpoint, err = newBridgeConnection("", &rc, r.cmd.LogLevel)
+		//  if the user has a config file with connections in it
+	} else if len(remoteConfigConnections.Connections) > 0 {
+		logrus.Debug("creating a varlink bridge based configuration file")
+		var rc *remoteclientconfig.RemoteConnection
+		if len(r.cmd.ConnectionName) > 0 {
+			rc, err = remoteConfigConnections.GetRemoteConnection(r.cmd.ConnectionName)
+		} else {
+			rc, err = remoteConfigConnections.GetDefault()
+		}
+		if err != nil {
+			return nil, err
 		}
-		remoteEndpoint.Type = DirectConnection
-		remoteEndpoint.Connection = address
+		remoteEndpoint, err = newBridgeConnection("", rc, r.cmd.LogLevel)
+		//	last resort is to make a socket connection with the default varlink address for root user
+	} else {
+		logrus.Debug("creating a varlink address based default root address")
+		remoteEndpoint, err = newSocketConnection(DefaultAddress)
 	}
 	return
 }
@@ -72,3 +82,12 @@ func (r RemoteRuntime) RefreshConnection() error {
 	r.Conn = newConn
 	return nil
 }
+
+// newSocketConnection returns an endpoint for a uds based connection
+func newSocketConnection(address string) (*Endpoint, error) {
+	endpoint := Endpoint{
+		Type:       DirectConnection,
+		Connection: address,
+	}
+	return &endpoint, nil
+}
diff --git a/pkg/adapter/client_unix.go b/pkg/adapter/client_unix.go
new file mode 100644
index 000000000..e0406567c
--- /dev/null
+++ b/pkg/adapter/client_unix.go
@@ -0,0 +1,30 @@
+// +build linux darwin
+// +build remoteclient
+
+package adapter
+
+import (
+	"fmt"
+
+	"github.com/containers/libpod/cmd/podman/remoteclientconfig"
+	"github.com/pkg/errors"
+)
+
+// newBridgeConnection creates a bridge type endpoint with username, destination, and log-level
+func newBridgeConnection(formattedBridge string, remoteConn *remoteclientconfig.RemoteConnection, logLevel string) (*Endpoint, error) {
+	endpoint := Endpoint{
+		Type: BridgeConnection,
+	}
+
+	if len(formattedBridge) < 1 && remoteConn == nil {
+		return nil, errors.New("bridge connections must either be created by string or remoteconnection")
+	}
+	if len(formattedBridge) > 0 {
+		endpoint.Connection = formattedBridge
+		return &endpoint, nil
+	}
+	endpoint.Connection = fmt.Sprintf(
+		`ssh -T %s@%s -- /usr/bin/varlink -A \'/usr/bin/podman --log-level=%s varlink \\\$VARLINK_ADDRESS\' bridge`,
+		remoteConn.Username, remoteConn.Destination, logLevel)
+	return &endpoint, nil
+}
diff --git a/pkg/adapter/client_windows.go b/pkg/adapter/client_windows.go
new file mode 100644
index 000000000..088550667
--- /dev/null
+++ b/pkg/adapter/client_windows.go
@@ -0,0 +1,15 @@
+// +build remoteclient
+
+package adapter
+
+import (
+	"github.com/containers/libpod/cmd/podman/remoteclientconfig"
+	"github.com/containers/libpod/libpod"
+)
+
+func newBridgeConnection(formattedBridge string, remoteConn *remoteclientconfig.RemoteConnection, logLevel string) (*Endpoint, error) {
+	// TODO
+	// Unix and Windows appear to quote their ssh implementations differently therefore once we figure out what
+	// windows ssh is doing here, we can then get the format correct.
+	return nil, libpod.ErrNotImplemented
+}
diff --git a/pkg/adapter/containers.go b/pkg/adapter/containers.go
index 34ee70d3d..40b1e6b43 100644
--- a/pkg/adapter/containers.go
+++ b/pkg/adapter/containers.go
@@ -190,12 +190,18 @@ func (r *LocalRuntime) RemoveContainers(ctx context.Context, cli *cliconfig.RmVa
 	}
 	logrus.Debugf("Setting maximum rm workers to %d", maxWorkers)
 
+	if cli.Storage {
+		for _, ctr := range cli.InputArgs {
+			if err := r.RemoveStorageContainer(ctr, cli.Force); err != nil {
+				failures[ctr] = err
+			}
+			ok = append(ok, ctr)
+		}
+		return ok, failures, nil
+	}
+
 	ctrs, err := shortcuts.GetContainersByContext(cli.All, cli.Latest, cli.InputArgs, r.Runtime)
 	if err != nil {
-		// Force may be used to remove containers no longer found in the database
-		if cli.Force && len(cli.InputArgs) > 0 && errors.Cause(err) == libpod.ErrNoSuchCtr {
-			r.RemoveContainersFromStorage(cli.InputArgs)
-		}
 		return ok, failures, err
 	}
 
@@ -526,7 +532,7 @@ func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues, options libpod.
 }
 
 // Restore one or more containers
-func (r *LocalRuntime) Restore(c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error {
+func (r *LocalRuntime) Restore(ctx context.Context, c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error {
 	var (
 		containers     []*libpod.Container
 		err, lastError error
@@ -538,7 +544,9 @@ func (r *LocalRuntime) Restore(c *cliconfig.RestoreValues, options libpod.Contai
 		return state == libpod.ContainerStateExited
 	})
 
-	if c.All {
+	if c.Import != "" {
+		containers, err = crImportCheckpoint(ctx, r.Runtime, c.Import, c.Name)
+	} else if c.All {
 		containers, err = r.GetContainers(filterFuncs...)
 	} else {
 		containers, err = shortcuts.GetContainersByContext(false, c.Latest, c.InputArgs, r.Runtime)
diff --git a/pkg/adapter/containers_remote.go b/pkg/adapter/containers_remote.go
index bc6a9cfcd..cf0b90b3a 100644
--- a/pkg/adapter/containers_remote.go
+++ b/pkg/adapter/containers_remote.go
@@ -16,7 +16,6 @@ import (
 	"github.com/containers/libpod/cmd/podman/shared"
 	iopodman "github.com/containers/libpod/cmd/podman/varlink"
 	"github.com/containers/libpod/libpod"
-	"github.com/containers/libpod/pkg/inspect"
 	"github.com/containers/libpod/pkg/varlinkapi/virtwriter"
 	"github.com/cri-o/ocicni/pkg/ocicni"
 	"github.com/docker/docker/pkg/term"
@@ -29,12 +28,12 @@ import (
 )
 
 // Inspect returns an inspect struct from varlink
-func (c *Container) Inspect(size bool) (*inspect.ContainerInspectData, error) {
+func (c *Container) Inspect(size bool) (*libpod.InspectContainerData, error) {
 	reply, err := iopodman.ContainerInspectData().Call(c.Runtime.Conn, c.ID(), size)
 	if err != nil {
 		return nil, err
 	}
-	data := inspect.ContainerInspectData{}
+	data := libpod.InspectContainerData{}
 	if err := json.Unmarshal([]byte(reply), &data); err != nil {
 		return nil, err
 	}
@@ -664,6 +663,10 @@ func (r *LocalRuntime) Attach(ctx context.Context, c *cliconfig.AttachValues) er
 
 // Checkpoint one or more containers
 func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues, options libpod.ContainerCheckpointOptions) error {
+	if c.Export != "" {
+		return errors.New("the remote client does not support exporting checkpoints")
+	}
+
 	var lastError error
 	ids, err := iopodman.GetContainersByContext().Call(r.Conn, c.All, c.Latest, c.InputArgs)
 	if err != nil {
@@ -699,7 +702,11 @@ func (r *LocalRuntime) Checkpoint(c *cliconfig.CheckpointValues, options libpod.
 }
 
 // Restore one or more containers
-func (r *LocalRuntime) Restore(c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error {
+func (r *LocalRuntime) Restore(ctx context.Context, c *cliconfig.RestoreValues, options libpod.ContainerCheckpointOptions) error {
+	if c.Import != "" {
+		return errors.New("the remote client does not support importing checkpoints")
+	}
+
 	var lastError error
 	ids, err := iopodman.GetContainersByContext().Call(r.Conn, c.All, c.Latest, c.InputArgs)
 	if err != nil {
diff --git a/pkg/adapter/runtime_remote.go b/pkg/adapter/runtime_remote.go
index e0c0898bd..a1d358f68 100644
--- a/pkg/adapter/runtime_remote.go
+++ b/pkg/adapter/runtime_remote.go
@@ -20,6 +20,7 @@ import (
 	"github.com/containers/image/docker/reference"
 	"github.com/containers/image/types"
 	"github.com/containers/libpod/cmd/podman/cliconfig"
+	"github.com/containers/libpod/cmd/podman/remoteclientconfig"
 	"github.com/containers/libpod/cmd/podman/varlink"
 	"github.com/containers/libpod/libpod"
 	"github.com/containers/libpod/libpod/events"
@@ -40,6 +41,7 @@ type RemoteRuntime struct {
 	Conn   *varlink.Connection
 	Remote bool
 	cmd    cliconfig.MainFlags
+	config io.Reader
 }
 
 // LocalRuntime describes a typical libpod runtime
@@ -49,10 +51,35 @@ type LocalRuntime struct {
 
 // GetRuntime returns a LocalRuntime struct with the actual runtime embedded in it
 func GetRuntime(ctx context.Context, c *cliconfig.PodmanCommand) (*LocalRuntime, error) {
+	var (
+		customConfig bool
+		err          error
+		f            *os.File
+	)
 	runtime := RemoteRuntime{
 		Remote: true,
 		cmd:    c.GlobalFlags,
 	}
+	configPath := remoteclientconfig.GetConfigFilePath()
+	if len(c.GlobalFlags.RemoteConfigFilePath) > 0 {
+		configPath = c.GlobalFlags.RemoteConfigFilePath
+		customConfig = true
+	}
+
+	f, err = os.Open(configPath)
+	if err != nil {
+		// If user does not explicitly provide a configuration file path and we cannot
+		// find a default, no error should occur.
+		if os.IsNotExist(err) && !customConfig {
+			logrus.Debugf("unable to load configuration file at %s", configPath)
+			runtime.config = nil
+		} else {
+			return nil, errors.Wrapf(err, "unable to load configuration file at %s", configPath)
+		}
+	} else {
+		// create the io reader for the remote client
+		runtime.config = bufio.NewReader(f)
+	}
 	conn, err := runtime.Connect()
 	if err != nil {
 		return nil, err
diff --git a/pkg/apparmor/apparmor_linux.go b/pkg/apparmor/apparmor_linux.go
index 2c5022c1f..0d01f41e9 100644
--- a/pkg/apparmor/apparmor_linux.go
+++ b/pkg/apparmor/apparmor_linux.go
@@ -225,8 +225,13 @@ func CheckProfileAndLoadDefault(name string) (string, error) {
 		}
 	}
 
-	if name != "" && !runcaa.IsEnabled() {
-		return "", fmt.Errorf("profile %q specified but AppArmor is disabled on the host", name)
+	// Check if AppArmor is disabled and error out if a profile is to be set.
+	if !runcaa.IsEnabled() {
+		if name == "" {
+			return "", nil
+		} else {
+			return "", fmt.Errorf("profile %q specified but AppArmor is disabled on the host", name)
+		}
 	}
 
 	// If the specified name is not empty or is not a default libpod one,
diff --git a/pkg/inspect/inspect.go b/pkg/inspect/inspect.go
index 2082bb3a6..ec3d98613 100644
--- a/pkg/inspect/inspect.go
+++ b/pkg/inspect/inspect.go
@@ -3,110 +3,11 @@ package inspect
 import (
 	"time"
 
-	"github.com/containers/image/manifest"
-	"github.com/cri-o/ocicni/pkg/ocicni"
-	"github.com/docker/go-connections/nat"
+	"github.com/containers/libpod/libpod/driver"
 	"github.com/opencontainers/go-digest"
 	"github.com/opencontainers/image-spec/specs-go/v1"
-	"github.com/opencontainers/runtime-spec/specs-go"
 )
 
-// ContainerData holds the podman inspect data for a container
-type ContainerData struct {
-	*ContainerInspectData
-	HostConfig *HostConfig `json:"HostConfig"`
-	Config     *CtrConfig  `json:"Config"`
-}
-
-// HostConfig represents the host configuration for the container
-type HostConfig struct {
-	ContainerIDFile      string                      `json:"ContainerIDFile"`
-	LogConfig            *LogConfig                  `json:"LogConfig"` //TODO
-	NetworkMode          string                      `json:"NetworkMode"`
-	PortBindings         nat.PortMap                 `json:"PortBindings"` //TODO
-	AutoRemove           bool                        `json:"AutoRemove"`
-	CapAdd               []string                    `json:"CapAdd"`
-	CapDrop              []string                    `json:"CapDrop"`
-	DNS                  []string                    `json:"DNS"`
-	DNSOptions           []string                    `json:"DNSOptions"`
-	DNSSearch            []string                    `json:"DNSSearch"`
-	ExtraHosts           []string                    `json:"ExtraHosts"`
-	GroupAdd             []uint32                    `json:"GroupAdd"`
-	IpcMode              string                      `json:"IpcMode"`
-	Cgroup               string                      `json:"Cgroup"`
-	OomScoreAdj          *int                        `json:"OomScoreAdj"`
-	PidMode              string                      `json:"PidMode"`
-	Privileged           bool                        `json:"Privileged"`
-	PublishAllPorts      bool                        `json:"PublishAllPorts"` //TODO
-	ReadOnlyRootfs       bool                        `json:"ReadonlyRootfs"`
-	ReadOnlyTmpfs        bool                        `json:"ReadonlyTmpfs"`
-	SecurityOpt          []string                    `json:"SecurityOpt"`
-	UTSMode              string                      `json:"UTSMode"`
-	UsernsMode           string                      `json:"UsernsMode"`
-	ShmSize              int64                       `json:"ShmSize"`
-	Runtime              string                      `json:"Runtime"`
-	ConsoleSize          *specs.Box                  `json:"ConsoleSize"`
-	CPUShares            *uint64                     `json:"CpuShares"`
-	Memory               int64                       `json:"Memory"`
-	NanoCPUs             int                         `json:"NanoCpus"`
-	CgroupParent         string                      `json:"CgroupParent"`
-	BlkioWeight          *uint16                     `json:"BlkioWeight"`
-	BlkioWeightDevice    []specs.LinuxWeightDevice   `json:"BlkioWeightDevice"`
-	BlkioDeviceReadBps   []specs.LinuxThrottleDevice `json:"BlkioDeviceReadBps"`
-	BlkioDeviceWriteBps  []specs.LinuxThrottleDevice `json:"BlkioDeviceWriteBps"`
-	BlkioDeviceReadIOps  []specs.LinuxThrottleDevice `json:"BlkioDeviceReadIOps"`
-	BlkioDeviceWriteIOps []specs.LinuxThrottleDevice `json:"BlkioDeviceWriteIOps"`
-	CPUPeriod            *uint64                     `json:"CpuPeriod"`
-	CPUQuota             *int64                      `json:"CpuQuota"`
-	CPURealtimePeriod    *uint64                     `json:"CpuRealtimePeriod"`
-	CPURealtimeRuntime   *int64                      `json:"CpuRealtimeRuntime"`
-	CPUSetCPUs           string                      `json:"CpuSetCpus"`
-	CPUSetMems           string                      `json:"CpuSetMems"`
-	Devices              []specs.LinuxDevice         `json:"Devices"`
-	DiskQuota            int                         `json:"DiskQuota"` //check type, TODO
-	KernelMemory         *int64                      `json:"KernelMemory"`
-	MemoryReservation    *int64                      `json:"MemoryReservation"`
-	MemorySwap           *int64                      `json:"MemorySwap"`
-	MemorySwappiness     *uint64                     `json:"MemorySwappiness"`
-	OomKillDisable       *bool                       `json:"OomKillDisable"`
-	PidsLimit            *int64                      `json:"PidsLimit"`
-	Ulimits              []string                    `json:"Ulimits"`
-	CPUCount             int                         `json:"CpuCount"`
-	CPUPercent           int                         `json:"CpuPercent"`
-	IOMaximumIOps        int                         `json:"IOMaximumIOps"`      //check type, TODO
-	IOMaximumBandwidth   int                         `json:"IOMaximumBandwidth"` //check type, TODO
-	Tmpfs                []string                    `json:"Tmpfs"`
-}
-
-// CtrConfig holds information about the container configuration
-type CtrConfig struct {
-	Hostname     string                        `json:"Hostname"`
-	DomainName   string                        `json:"Domainname"` //TODO
-	User         specs.User                    `json:"User"`
-	AttachStdin  bool                          `json:"AttachStdin"`  //TODO
-	AttachStdout bool                          `json:"AttachStdout"` //TODO
-	AttachStderr bool                          `json:"AttachStderr"` //TODO
-	Tty          bool                          `json:"Tty"`
-	OpenStdin    bool                          `json:"OpenStdin"`
-	StdinOnce    bool                          `json:"StdinOnce"` //TODO
-	Env          []string                      `json:"Env"`
-	Cmd          []string                      `json:"Cmd"`
-	Image        string                        `json:"Image"`
-	Volumes      map[string]struct{}           `json:"Volumes"`
-	WorkingDir   string                        `json:"WorkingDir"`
-	Entrypoint   string                        `json:"Entrypoint"`
-	Labels       map[string]string             `json:"Labels"`
-	Annotations  map[string]string             `json:"Annotations"`
-	StopSignal   uint                          `json:"StopSignal"`
-	Healthcheck  *manifest.Schema2HealthConfig `json:"Healthcheck,omitempty"`
-}
-
-// LogConfig holds the log information for a container
-type LogConfig struct {
-	Type   string            `json:"Type"`
-	Config map[string]string `json:"Config"` //idk type, TODO
-}
-
 // ImageData holds the inspect information of an image
 type ImageData struct {
 	ID           string            `json:"Id"`
@@ -123,7 +24,7 @@ type ImageData struct {
 	Os           string            `json:"Os"`
 	Size         int64             `json:"Size"`
 	VirtualSize  int64             `json:"VirtualSize"`
-	GraphDriver  *Data             `json:"GraphDriver"`
+	GraphDriver  *driver.Data      `json:"GraphDriver"`
 	RootFS       *RootFS           `json:"RootFS"`
 	Labels       map[string]string `json:"Labels"`
 	Annotations  map[string]string `json:"Annotations"`
@@ -138,86 +39,6 @@ type RootFS struct {
 	Layers []digest.Digest `json:"Layers"`
 }
 
-// Data handles the data for a storage driver
-type Data struct {
-	Name string            `json:"Name"`
-	Data map[string]string `json:"Data"`
-}
-
-// ContainerInspectData handles the data used when inspecting a container
-type ContainerInspectData struct {
-	ID              string                 `json:"ID"`
-	Created         time.Time              `json:"Created"`
-	Path            string                 `json:"Path"`
-	Args            []string               `json:"Args"`
-	State           *ContainerInspectState `json:"State"`
-	ImageID         string                 `json:"Image"`
-	ImageName       string                 `json:"ImageName"`
-	Rootfs          string                 `json:"Rootfs"`
-	ResolvConfPath  string                 `json:"ResolvConfPath"`
-	HostnamePath    string                 `json:"HostnamePath"`
-	HostsPath       string                 `json:"HostsPath"`
-	StaticDir       string                 `json:"StaticDir"`
-	LogPath         string                 `json:"LogPath"`
-	ConmonPidFile   string                 `json:"ConmonPidFile"`
-	Name            string                 `json:"Name"`
-	RestartCount    int32                  `json:"RestartCount"`
-	Driver          string                 `json:"Driver"`
-	MountLabel      string                 `json:"MountLabel"`
-	ProcessLabel    string                 `json:"ProcessLabel"`
-	AppArmorProfile string                 `json:"AppArmorProfile"`
-	EffectiveCaps   []string               `json:"EffectiveCaps"`
-	BoundingCaps    []string               `json:"BoundingCaps"`
-	ExecIDs         []string               `json:"ExecIDs"`
-	GraphDriver     *Data                  `json:"GraphDriver"`
-	SizeRw          int64                  `json:"SizeRw,omitempty"`
-	SizeRootFs      int64                  `json:"SizeRootFs,omitempty"`
-	Mounts          []specs.Mount          `json:"Mounts"`
-	Dependencies    []string               `json:"Dependencies"`
-	NetworkSettings *NetworkSettings       `json:"NetworkSettings"` //TODO
-	ExitCommand     []string               `json:"ExitCommand"`
-	Namespace       string                 `json:"Namespace"`
-	IsInfra         bool                   `json:"IsInfra"`
-}
-
-// ContainerInspectState represents the state of a container.
-type ContainerInspectState struct {
-	OciVersion  string             `json:"OciVersion"`
-	Status      string             `json:"Status"`
-	Running     bool               `json:"Running"`
-	Paused      bool               `json:"Paused"`
-	Restarting  bool               `json:"Restarting"` // TODO
-	OOMKilled   bool               `json:"OOMKilled"`
-	Dead        bool               `json:"Dead"`
-	Pid         int                `json:"Pid"`
-	ExitCode    int32              `json:"ExitCode"`
-	Error       string             `json:"Error"` // TODO
-	StartedAt   time.Time          `json:"StartedAt"`
-	FinishedAt  time.Time          `json:"FinishedAt"`
-	Healthcheck HealthCheckResults `json:"Healthcheck,omitempty"`
-}
-
-// NetworkSettings holds information about the newtwork settings of the container
-type NetworkSettings struct {
-	Bridge                 string               `json:"Bridge"`
-	SandboxID              string               `json:"SandboxID"`
-	HairpinMode            bool                 `json:"HairpinMode"`
-	LinkLocalIPv6Address   string               `json:"LinkLocalIPv6Address"`
-	LinkLocalIPv6PrefixLen int                  `json:"LinkLocalIPv6PrefixLen"`
-	Ports                  []ocicni.PortMapping `json:"Ports"`
-	SandboxKey             string               `json:"SandboxKey"`
-	SecondaryIPAddresses   []string             `json:"SecondaryIPAddresses"`
-	SecondaryIPv6Addresses []string             `json:"SecondaryIPv6Addresses"`
-	EndpointID             string               `json:"EndpointID"`
-	Gateway                string               `json:"Gateway"`
-	GlobalIPv6Address      string               `json:"GlobalIPv6Address"`
-	GlobalIPv6PrefixLen    int                  `json:"GlobalIPv6PrefixLen"`
-	IPAddress              string               `json:"IPAddress"`
-	IPPrefixLen            int                  `json:"IPPrefixLen"`
-	IPv6Gateway            string               `json:"IPv6Gateway"`
-	MacAddress             string               `json:"MacAddress"`
-}
-
 // ImageResult is used for podman images for collection and output
 type ImageResult struct {
 	Tag          string
@@ -232,25 +53,3 @@ type ImageResult struct {
 	Labels       map[string]string
 	Dangling     bool
 }
-
-// HealthCheckResults describes the results/logs from a healthcheck
-type HealthCheckResults struct {
-	// Status healthy or unhealthy
-	Status string `json:"Status"`
-	// FailingStreak is the number of consecutive failed healthchecks
-	FailingStreak int `json:"FailingStreak"`
-	// Log describes healthcheck attempts and results
-	Log []HealthCheckLog `json:"Log"`
-}
-
-// HealthCheckLog describes the results of a single healthcheck
-type HealthCheckLog struct {
-	// Start time as string
-	Start string `json:"Start"`
-	// End time as a string
-	End string `json:"End"`
-	// Exitcode is 0 or 1
-	ExitCode int `json:"ExitCode"`
-	// Output is the stdout/stderr from the healthcheck command
-	Output string `json:"Output"`
-}
diff --git a/pkg/registries/registries.go b/pkg/registries/registries.go
index 5c4ecd020..de63dcbf1 100644
--- a/pkg/registries/registries.go
+++ b/pkg/registries/registries.go
@@ -44,17 +44,7 @@ func getRegistries() ([]sysregistriesv2.Registry, error) {
 
 // GetRegistries obtains the list of search registries defined in the global registries file.
 func GetRegistries() ([]string, error) {
-	var searchRegistries []string
-	registries, err := getRegistries()
-	if err != nil {
-		return nil, err
-	}
-	for _, reg := range registries {
-		if reg.Search {
-			searchRegistries = append(searchRegistries, reg.Location)
-		}
-	}
-	return searchRegistries, nil
+	return sysregistriesv2.UnqualifiedSearchRegistries(&types.SystemContext{SystemRegistriesConfPath: SystemRegistriesConfPath()})
 }
 
 // GetBlockedRegistries obtains the list of blocked registries defined in the global registries file.
@@ -66,7 +56,7 @@ func GetBlockedRegistries() ([]string, error) {
 	}
 	for _, reg := range registries {
 		if reg.Blocked {
-			blockedRegistries = append(blockedRegistries, reg.Location)
+			blockedRegistries = append(blockedRegistries, reg.Prefix)
 		}
 	}
 	return blockedRegistries, nil
@@ -81,7 +71,7 @@ func GetInsecureRegistries() ([]string, error) {
 	}
 	for _, reg := range registries {
 		if reg.Insecure {
-			insecureRegistries = append(insecureRegistries, reg.Location)
+			insecureRegistries = append(insecureRegistries, reg.Prefix)
 		}
 	}
 	return insecureRegistries, nil
diff --git a/pkg/rootless/rootless_linux.c b/pkg/rootless/rootless_linux.c
index a2425c83e..eb62d55e9 100644
--- a/pkg/rootless/rootless_linux.c
+++ b/pkg/rootless/rootless_linux.c
@@ -489,6 +489,7 @@ reexec_userns_join (int userns, int mountns, char *pause_pid_file_path)
   char **argv;
   int pid;
   char *cwd = getcwd (NULL, 0);
+  sigset_t sigset, oldsigset;
 
   if (cwd == NULL)
     {
@@ -522,6 +523,22 @@ reexec_userns_join (int userns, int mountns, char *pause_pid_file_path)
       return pid;
     }
 
+  if (sigfillset (&sigset) < 0)
+    {
+      fprintf (stderr, "cannot fill sigset: %s\n", strerror (errno));
+      _exit (EXIT_FAILURE);
+    }
+  if (sigdelset (&sigset, SIGCHLD) < 0)
+    {
+      fprintf (stderr, "cannot sigdelset(SIGCHLD): %s\n", strerror (errno));
+      _exit (EXIT_FAILURE);
+    }
+  if (sigprocmask (SIG_BLOCK, &sigset, &oldsigset) < 0)
+    {
+      fprintf (stderr, "cannot block signals: %s\n", strerror (errno));
+      _exit (EXIT_FAILURE);
+    }
+
   setenv ("_CONTAINERS_USERNS_CONFIGURED", "init", 1);
   setenv ("_CONTAINERS_ROOTLESS_UID", uid, 1);
   setenv ("_CONTAINERS_ROOTLESS_GID", gid, 1);
@@ -570,6 +587,11 @@ reexec_userns_join (int userns, int mountns, char *pause_pid_file_path)
       /* We ignore errors here as we didn't create the namespace anyway.  */
       create_pause_process (pause_pid_file_path, argv);
     }
+  if (sigprocmask (SIG_SETMASK, &oldsigset, NULL) < 0)
+    {
+      fprintf (stderr, "cannot block signals: %s\n", strerror (errno));
+      _exit (EXIT_FAILURE);
+    }
 
   execvp (argv[0], argv);
 
diff --git a/pkg/rootless/rootless_linux.go b/pkg/rootless/rootless_linux.go
index d302b1777..3f78ffc67 100644
--- a/pkg/rootless/rootless_linux.go
+++ b/pkg/rootless/rootless_linux.go
@@ -24,6 +24,7 @@ import (
 /*
 #cgo remoteclient CFLAGS: -DDISABLE_JOIN_SHORTCUT
 #include <stdlib.h>
+#include <sys/types.h>
 extern uid_t rootless_uid();
 extern uid_t rootless_gid();
 extern int reexec_in_user_namespace(int ready, char *pause_pid_file_path, char *file_to_read, int fd);
@@ -169,6 +170,9 @@ func getUserNSFirstChild(fd uintptr) (*os.File, error) {
 	for {
 		nextFd, err := getParentUserNs(fd)
 		if err != nil {
+			if err == syscall.ENOTTY {
+				return os.NewFile(fd, "userns child"), nil
+			}
 			return nil, errors.Wrapf(err, "cannot get parent user namespace")
 		}
 
diff --git a/pkg/spec/createconfig.go b/pkg/spec/createconfig.go
index e4501aaac..a8413d6c7 100644
--- a/pkg/spec/createconfig.go
+++ b/pkg/spec/createconfig.go
@@ -162,6 +162,10 @@ func (c *CreateConfig) createExitCommand(runtime *libpod.Runtime) ([]string, err
 	if config.StorageConfig.GraphDriverName != "" {
 		command = append(command, []string{"--storage-driver", config.StorageConfig.GraphDriverName}...)
 	}
+	for _, opt := range config.StorageConfig.GraphDriverOptions {
+		command = append(command, []string{"--storage-opt", opt}...)
+	}
+
 	if c.Syslog {
 		command = append(command, "--syslog", "true")
 	}
@@ -320,7 +324,9 @@ func (c *CreateConfig) getContainerCreateOptions(runtime *libpod.Runtime, pod *l
 		options = append(options, libpod.WithLogPath(logPath))
 	}
 
-	options = append(options, libpod.WithLogDriver(c.LogDriver))
+	if c.LogDriver != "" {
+		options = append(options, libpod.WithLogDriver(c.LogDriver))
+	}
 
 	if c.IPAddress != "" {
 		ip := net.ParseIP(c.IPAddress)
diff --git a/pkg/spec/storage.go b/pkg/spec/storage.go
index e221b5cb5..283585ef8 100644
--- a/pkg/spec/storage.go
+++ b/pkg/spec/storage.go
@@ -384,7 +384,7 @@ func (config *CreateConfig) getMounts() (map[string]spec.Mount, map[string]*libp
 			}
 			finalNamedVolumes[volume.Dest] = volume
 		default:
-			return nil, nil, errors.Errorf("invalid fylesystem type %q", kv[1])
+			return nil, nil, errors.Errorf("invalid filesystem type %q", kv[1])
 		}
 	}
 
@@ -403,6 +403,8 @@ func getBindMount(args []string) (spec.Mount, error) {
 	for _, val := range args {
 		kv := strings.Split(val, "=")
 		switch kv[0] {
+		case "bind-nonrecursive":
+			newMount.Options = append(newMount.Options, "bind")
 		case "ro", "nosuid", "nodev", "noexec":
 			// TODO: detect duplication of these options.
 			// (Is this necessary?)
@@ -574,7 +576,7 @@ func ValidateVolumeCtrDir(ctrDir string) error {
 
 // ValidateVolumeOpts validates a volume's options
 func ValidateVolumeOpts(options []string) error {
-	var foundRootPropagation, foundRWRO, foundLabelChange int
+	var foundRootPropagation, foundRWRO, foundLabelChange, bindType int
 	for _, opt := range options {
 		switch opt {
 		case "rw", "ro":
@@ -592,6 +594,11 @@ func ValidateVolumeOpts(options []string) error {
 			if foundRootPropagation > 1 {
 				return errors.Errorf("invalid options %q, can only specify 1 '[r]shared', '[r]private' or '[r]slave' option", strings.Join(options, ", "))
 			}
+		case "bind", "rbind":
+			bindType++
+			if bindType > 1 {
+				return errors.Errorf("invalid options %q, can only specify 1 '[r]bind' option", strings.Join(options, ", "))
+			}
 		default:
 			return errors.Errorf("invalid option type %q", opt)
 		}
diff --git a/pkg/util/mountOpts.go b/pkg/util/mountOpts.go
index 59459807c..489e7eeef 100644
--- a/pkg/util/mountOpts.go
+++ b/pkg/util/mountOpts.go
@@ -17,10 +17,19 @@ var (
 // sensible and follow convention.
 func ProcessOptions(options []string) []string {
 	var (
-		foundrw, foundro bool
-		rootProp         string
+		foundbind, foundrw, foundro bool
+		rootProp                    string
 	)
-	options = append(options, "rbind")
+	for _, opt := range options {
+		switch opt {
+		case "bind", "rbind":
+			foundbind = true
+			break
+		}
+	}
+	if !foundbind {
+		options = append(options, "rbind")
+	}
 	for _, opt := range options {
 		switch opt {
 		case "rw":
diff --git a/pkg/util/utils.go b/pkg/util/utils.go
index a074f276c..61cdbbf38 100644
--- a/pkg/util/utils.go
+++ b/pkg/util/utils.go
@@ -99,7 +99,10 @@ func GetImageConfig(changes []string) (v1.ImageConfig, error) {
 			var st struct{}
 			exposedPorts[pair[1]] = st
 		case "ENV":
-			env = append(env, pair[1])
+			if len(pair) < 3 {
+				return v1.ImageConfig{}, errors.Errorf("no value given for environment variable %q", pair[1])
+			}
+			env = append(env, strings.Join(pair[1:], "="))
 		case "ENTRYPOINT":
 			entrypoint = append(entrypoint, pair[1])
 		case "CMD":
diff --git a/pkg/varlinkapi/virtwriter/virtwriter.go b/pkg/varlinkapi/virtwriter/virtwriter.go
index 3adaf6e17..e747984c7 100644
--- a/pkg/varlinkapi/virtwriter/virtwriter.go
+++ b/pkg/varlinkapi/virtwriter/virtwriter.go
@@ -91,65 +91,65 @@ func (v VirtWriteCloser) Write(input []byte) (int, error) {
 
 // Reader decodes the content that comes over the wire and directs it to the proper destination.
 func Reader(r *bufio.Reader, output, errput *os.File, input *io.PipeWriter, resize chan remotecommand.TerminalSize) error {
-	var saveb []byte
-	var eom int
+	var messageSize int64
+	headerBytes := make([]byte, 8)
+
 	for {
-		readb := make([]byte, 32*1024)
-		n, err := r.Read(readb)
-		// TODO, later may be worth checking in len of the read is 0
+		n, err := io.ReadFull(r, headerBytes)
 		if err != nil {
 			return err
 		}
-		b := append(saveb, readb[0:n]...)
-		// no sense in reading less than the header len
-		for len(b) > 7 {
-			eom = int(binary.BigEndian.Uint32(b[4:8])) + 8
-			// The message and header are togther
-			if len(b) >= eom {
-				out := append([]byte{}, b[8:eom]...)
-
-				switch IntToSocketDest(int(b[0])) {
-				case ToStdout:
-					n, err := output.Write(out)
-					if err != nil {
-						return err
-					}
-					if n < len(out) {
-						return errors.New("short write error occurred on stdout")
-					}
-				case ToStderr:
-					n, err := errput.Write(out)
-					if err != nil {
-						return err
-					}
-					if n < len(out) {
-						return errors.New("short write error occurred on stderr")
-					}
-				case ToStdin:
-					n, err := input.Write(out)
-					if err != nil {
-						return err
-					}
-					if n < len(out) {
-						return errors.New("short write error occurred on stdin")
-					}
-				case TerminalResize:
-					// Resize events come over in bytes, need to be reserialized
-					resizeEvent := remotecommand.TerminalSize{}
-					if err := json.Unmarshal(out, &resizeEvent); err != nil {
-						return err
-					}
-					resize <- resizeEvent
-				case Quit:
-					return nil
+		if n < 8 {
+			return errors.New("short read and no full header read")
+		}
+
+		messageSize = int64(binary.BigEndian.Uint32(headerBytes[4:8]))
+
+		switch IntToSocketDest(int(headerBytes[0])) {
+		case ToStdout:
+			_, err := io.CopyN(output, r, messageSize)
+			if err != nil {
+				return err
+			}
+		case ToStderr:
+			_, err := io.CopyN(errput, r, messageSize)
+			if err != nil {
+				return err
+			}
+		case ToStdin:
+			_, err := io.CopyN(input, r, messageSize)
+			if err != nil {
+				return err
+			}
+		case TerminalResize:
+			out := make([]byte, messageSize)
+			if messageSize > 0 {
+				_, err = io.ReadFull(r, out)
+
+				if err != nil {
+					return err
 				}
-				b = b[eom:]
-			} else {
-				//	We do not have the header and full message, need to slurp again
-				saveb = b
-				break
 			}
+			// Resize events come over in bytes, need to be reserialized
+			resizeEvent := remotecommand.TerminalSize{}
+			if err := json.Unmarshal(out, &resizeEvent); err != nil {
+				return err
+			}
+			resize <- resizeEvent
+		case Quit:
+			out := make([]byte, messageSize)
+			if messageSize > 0 {
+				_, err = io.ReadFull(r, out)
+
+				if err != nil {
+					return err
+				}
+			}
+			return nil
+
+		default:
+			// Something really went wrong
+			return errors.New("Unknown multiplex destination")
 		}
 	}
-	return nil
 }