25 files changed, 366 insertions, 267 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 764ec6a29..153a97f86 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -16,8 +16,10 @@ gce_instance:
 # Main collection of env. varss to set for all scripts. All others
 # are cooked in by $SCRIPT_BASE/setup_environment.sh
 env:
+    FEDORA_CNI_COMMIT: "412b6d31280682bb4fab4446f113c22ff1886554"
     CNI_COMMIT: "7480240de9749f9a0a5c8614b17f1f03e0c06ab9"
     CRIO_COMMIT: "662dbb31b5d4f5ed54511a47cde7190c61c28677"
+    CRIU_COMMIT: "584cbe4643c3fc7dc901ff08bf923ca0fe7326f9"
     RUNC_COMMIT: "ad0f5255060d36872be04de22f8731f38ef2d7b1"
     # File to update in home-dir with task-specific env. var values
     ENVLIB: ".bash_profile"
@@ -40,11 +42,11 @@ full_vm_testing_task:
         # 'matrix' combinations.  All run in parallel.
         matrix:
             # Images are generated separetly, from build_images_task (below)
-            image_name: "ubuntu-1804-bionic-v20180911-libpod-5763563410948096"
+            image_name: "ubuntu-1804-bionic-v20180911-libpod-fce09afe"
             # TODO: Make these work (also build_images_task below)
-            #image_name: "rhel-server-ec2-7-5-165-1-libpod-5358668723781632"
-            #image_name: "centos-7-v20180911-libpod-5358668723781632"
-            #image_name: "fedora-cloud-base-28-1-1-7-libpod-5358668723781632"
+            #image_name: "rhel-server-ec2-7-5-165-1-libpod-fce09afe"
+            #image_name: "centos-7-v20180911-libpod-fce09afe"
+            #image_name: "fedora-cloud-base-28-1-1-7-libpod-fce09afe"
 
     timeout_in: 120m
 
@@ -83,19 +85,21 @@ build_vm_images_task:
         PACKER_BUILDS: "ubuntu-18"
         # TODO: Make these work (also full_vm_testing_task above)
         # PACKER_BUILDS: "rhel-7,centos-7,fedora-28,ubuntu-18"
-        # Command to register a RHEL VM
-        RHSM_COMMAND: ENCRYPTED[fec01433222af1ed0b8e40e89e7d18f6ee2fa9f49a1e721dc72f7eed3c740661215d1bd05cb54ac66a1a62116b92bdce]
-        # Additional environment variables needed to build GCE images, within a GCE VM
-        SERVICE_ACCOUNT: ENCRYPTED[02e03838b1156eb9516c7cc1e888e287910759842275f3c7bc2b4d56075cc6740e29ffa0ab71ebdbbd079673361dd8c9]
-        GCE_SSH_USERNAME: ENCRYPTED[a19a4ec62423e3e0fe4e7d1a5c9f11eda8fde321b9047ab5ed5590c2b1d7a2d12091c2be1531f949eae927059c2ae531]
-        GCP_PROJECT_ID: ENCRYPTED[77cb2d392bbc8d17412547d7d91f8d190089bf6e6b96eab9927994bbff6ab2c691ba0329ac7a650ba6182fbbab9fb68d]
-        # Existing base values to use, output images get epoc stamped names
-        PACKER_VER: "1.3.1"
-        # low-level base VM image name inputs to packer
         CENTOS_BASE_IMAGE: "centos-7-v20180911"
         RHEL_BASE_IMAGE: "rhel-server-ec2-7-5-165-1"
         FEDORA_BASE_IMAGE: "fedora-cloud-base-28-1-1-7"
         UBUNTU_BASE_IMAGE: "ubuntu-1804-bionic-v20180911"
+        # low-level base VM image name inputs to packer
+
+        # Command to register a RHEL VM
+        RHSM_COMMAND: ENCRYPTED[5caa5ff8c5370c3d25c7a1a28168501ab0fa2e5e3b627926f6eaba02b3fed965a7638a6151657809661f8c905c7dc187]
+        # Additional environment variables needed to build GCE images, within a GCE VM
+        SERVICE_ACCOUNT: ENCRYPTED[99e9a0b1c23f8dd29e83dfdf164f064cfd17afd9b895ca3b5e4c41170bd4290a8366fe2ad8e7a210b9f751711d1d002a]
+        GCE_SSH_USERNAME: ENCRYPTED[a7706b9e4b8bbb47f76358df7407f4fffa2e8552531190cc0b3315180c4b50588f560c4f85731e99cb5f43a396778277]
+        GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
+        # Version of packer to use
+        PACKER_VER: "1.3.1"
+
 
     gce_instance:
         image_name: "image-builder-image"  # Simply CentOS 7 + packer dependencies
diff --git a/.papr.yml b/.papr.yml
index 74741a8b8..97068546d 100644
--- a/.papr.yml
+++ b/.papr.yml
@@ -133,42 +133,42 @@ required: false
 timeout: 90m
 context: "Fedora 28 Cloud"
 
-#---
-#
-#host:
-#    distro: fedora/29/cloud/pungi
-#    specs:
-#        ram: 8192
-#        cpus: 4
-#packages:
-#    - btrfs-progs-devel
-#    - glib2-devel
-#    - glibc-devel
-#    - glibc-static
-#    - git
-#    - go-md2man
-#    - gpgme-devel
-#    - libassuan-devel
-#    - libgpg-error-devel
-#    - libseccomp-devel
-#    - libselinux-devel
-#    - ostree-devel
-#    - pkgconfig
-#    - make
-#    - nc
-#    - go-compilers-golang-compiler
-#    - podman
-#    - python3-varlink
-#    - python3-dateutil
-#    - python3-psutil
-#    - container-selinux
-#
-#tests:
-#    - sed 's/^expand-check.*/expand-check=0/g' -i /etc/selinux/semanage.conf
-#    - dnf -y reinstall container-selinux
-#    - sh .papr.sh -b -i -t
-#
-#required: false
-#
-#timeout: 90m
-#context: "Fedora 29 Cloud"
+---
+
+host:
+    distro: fedora/29/cloud/pungi
+    specs:
+        ram: 8192
+        cpus: 4
+packages:
+    - btrfs-progs-devel
+    - glib2-devel
+    - glibc-devel
+    - glibc-static
+    - git
+    - go-md2man
+    - gpgme-devel
+    - libassuan-devel
+    - libgpg-error-devel
+    - libseccomp-devel
+    - libselinux-devel
+    - ostree-devel
+    - pkgconfig
+    - make
+    - nc
+    - go-compilers-golang-compiler
+    - podman
+    - python3-varlink
+    - python3-dateutil
+    - python3-psutil
+    - container-selinux
+
+tests:
+    - sed 's/^expand-check.*/expand-check=0/g' -i /etc/selinux/semanage.conf
+    - dnf -y reinstall container-selinux
+    - sh .papr.sh -b -i -t
+
+required: false
+
+timeout: 90m
+context: "Fedora 29 Cloud"
diff --git a/cmd/podman/common.go b/cmd/podman/common.go
index e342659ed..8ae1c9e0f 100644
--- a/cmd/podman/common.go
+++ b/cmd/podman/common.go
@@ -465,3 +465,11 @@ func getAuthFile(authfile string) string {
 	}
 	return os.Getenv("REGISTRY_AUTH_FILE")
 }
+
+// scrubServer removes 'http://' or 'https://' from the front of the
+// server/registry string if either is there.  This will be mostly used
+// for user input from 'podman login' and 'podman logout'.
+func scrubServer(server string) string {
+	server = strings.TrimPrefix(server, "https://")
+	return strings.TrimPrefix(server, "http://")
+}
diff --git a/cmd/podman/create_cli_test.go b/cmd/podman/create_cli_test.go
index fa128c8e6..9db007ff3 100644
--- a/cmd/podman/create_cli_test.go
+++ b/cmd/podman/create_cli_test.go
@@ -47,7 +47,7 @@ func TestGetAllLabels(t *testing.T) {
 }
 
 func TestGetAllLabelsBadKeyValue(t *testing.T) {
-	inLabels := []string{"ONE1", "TWO=2"}
+	inLabels := []string{"=badValue", "="}
 	fileLabels := []string{}
 	_, err := getAllLabels(fileLabels, inLabels)
 	assert.Error(t, err, assert.AnError)
diff --git a/cmd/podman/libpodruntime/runtime.go b/cmd/podman/libpodruntime/runtime.go
index a0d497e8e..df422eb81 100644
--- a/cmd/podman/libpodruntime/runtime.go
+++ b/cmd/podman/libpodruntime/runtime.go
@@ -1,21 +1,16 @@
 package libpodruntime
 
 import (
-	"fmt"
-	"os"
-	"path/filepath"
-
 	"github.com/containers/libpod/libpod"
 	"github.com/containers/libpod/pkg/rootless"
 	"github.com/containers/libpod/pkg/util"
 	"github.com/containers/storage"
-	"github.com/pkg/errors"
 	"github.com/urfave/cli"
 )
 
 // GetRuntime generates a new libpod runtime configured by command line options
 func GetRuntime(c *cli.Context) (*libpod.Runtime, error) {
-	storageOpts, err := GetDefaultStoreOptions()
+	storageOpts, err := util.GetDefaultStoreOptions()
 	if err != nil {
 		return nil, err
 	}
@@ -28,7 +23,7 @@ func GetContainerRuntime(c *cli.Context) (*libpod.Runtime, error) {
 	if err != nil {
 		return nil, err
 	}
-	storageOpts, err := GetDefaultStoreOptions()
+	storageOpts, err := util.GetDefaultStoreOptions()
 	if err != nil {
 		return nil, err
 	}
@@ -37,51 +32,6 @@ func GetContainerRuntime(c *cli.Context) (*libpod.Runtime, error) {
 	return GetRuntimeWithStorageOpts(c, &storageOpts)
 }
 
-func GetRootlessStorageOpts() (storage.StoreOptions, error) {
-	var opts storage.StoreOptions
-
-	rootlessRuntime, err := libpod.GetRootlessRuntimeDir()
-	if err != nil {
-		return opts, err
-	}
-	opts.RunRoot = filepath.Join(rootlessRuntime, "run")
-
-	dataDir := os.Getenv("XDG_DATA_HOME")
-	if dataDir == "" {
-		home := os.Getenv("HOME")
-		if home == "" {
-			return opts, fmt.Errorf("neither XDG_DATA_HOME nor HOME was set non-empty")
-		}
-		// runc doesn't like symlinks in the rootfs path, and at least
-		// on CoreOS /home is a symlink to /var/home, so resolve any symlink.
-		resolvedHome, err := filepath.EvalSymlinks(home)
-		if err != nil {
-			return opts, errors.Wrapf(err, "cannot resolve %s", home)
-		}
-		dataDir = filepath.Join(resolvedHome, ".local", "share")
-	}
-	opts.GraphRoot = filepath.Join(dataDir, "containers", "storage")
-	opts.GraphDriverName = "vfs"
-	return opts, nil
-}
-
-func GetDefaultStoreOptions() (storage.StoreOptions, error) {
-	storageOpts := storage.DefaultStoreOptions
-	if rootless.IsRootless() {
-		var err error
-		storageOpts, err = GetRootlessStorageOpts()
-		if err != nil {
-			return storageOpts, err
-		}
-
-		storageConf := filepath.Join(os.Getenv("HOME"), ".config/containers/storage.conf")
-		if _, err := os.Stat(storageConf); err == nil {
-			storage.ReloadConfigurationFile(storageConf, &storageOpts)
-		}
-	}
-	return storageOpts, nil
-}
-
 // GetRuntime generates a new libpod runtime configured by command line options
 func GetRuntimeWithStorageOpts(c *cli.Context, storageOpts *storage.StoreOptions) (*libpod.Runtime, error) {
 	options := []libpod.RuntimeOption{}
diff --git a/cmd/podman/login.go b/cmd/podman/login.go
index 76f0f50ff..aa26d1466 100644
--- a/cmd/podman/login.go
+++ b/cmd/podman/login.go
@@ -60,10 +60,7 @@ func loginCmd(c *cli.Context) error {
 	if len(args) == 0 {
 		return errors.Errorf("registry must be given")
 	}
-	var server string
-	if len(args) == 1 {
-		server = args[0]
-	}
+	server := scrubServer(args[0])
 	authfile := getAuthFile(c.String("authfile"))
 
 	sc := common.GetSystemContext("", authfile, false)
@@ -113,6 +110,10 @@ func getUserAndPass(username, password, userFromAuthFile string) (string, string
 		if err != nil {
 			return "", "", errors.Wrapf(err, "error reading username")
 		}
+		// If no username provided, use userFromAuthFile instead.
+		if strings.TrimSpace(username) == "" {
+			username = userFromAuthFile
+		}
 	}
 	if password == "" {
 		fmt.Print("Password: ")
diff --git a/cmd/podman/logout.go b/cmd/podman/logout.go
index 099464e4f..3cdb606b5 100644
--- a/cmd/podman/logout.go
+++ b/cmd/podman/logout.go
@@ -44,7 +44,7 @@ func logoutCmd(c *cli.Context) error {
 	}
 	var server string
 	if len(args) == 1 {
-		server = args[0]
+		server = scrubServer(args[0])
 	}
 	authfile := getAuthFile(c.String("authfile"))
 
@@ -54,14 +54,14 @@ func logoutCmd(c *cli.Context) error {
 		if err := config.RemoveAllAuthentication(sc); err != nil {
 			return err
 		}
-		fmt.Println("Remove login credentials for all registries")
+		fmt.Println("Removed login credentials for all registries")
 		return nil
 	}
 
 	err := config.RemoveAuthentication(sc, server)
 	switch err {
 	case nil:
-		fmt.Printf("Remove login credentials for %s\n", server)
+		fmt.Printf("Removed login credentials for %s\n", server)
 		return nil
 	case config.ErrNotLoggedIn:
 		return errors.Errorf("Not logged into %s\n", server)
diff --git a/cmd/podman/parse.go b/cmd/podman/parse.go
index ade592ddf..2e4959656 100644
--- a/cmd/podman/parse.go
+++ b/cmd/podman/parse.go
@@ -198,6 +198,11 @@ func readKVStrings(env map[string]string, files []string, override []string) err
 func parseEnv(env map[string]string, line string) error {
 	data := strings.SplitN(line, "=", 2)
 
+	// catch invalid variables such as "=" or "=A"
+	if data[0] == "" {
+		return errors.Errorf("invalid environment variable: %q", line)
+	}
+
 	// trim the front of a variable, but nothing else
 	name := strings.TrimLeft(data[0], whiteSpaces)
 	if strings.ContainsAny(name, whiteSpaces) {
@@ -208,10 +213,7 @@ func parseEnv(env map[string]string, line string) error {
 		env[name] = data[1]
 	} else {
 		// if only a pass-through variable is given, clean it up.
-		val, exists := os.LookupEnv(name)
-		if !exists {
-			return errors.Errorf("environment variable %q does not exist", name)
-		}
+		val, _ := os.LookupEnv(name)
 		env[name] = val
 	}
 	return nil
diff --git a/cmd/podman/rm.go b/cmd/podman/rm.go
index f64eca6f4..38b1546ff 100644
--- a/cmd/podman/rm.go
+++ b/cmd/podman/rm.go
@@ -3,6 +3,7 @@ package main
 import (
 	"fmt"
 	"os"
+	rt "runtime"
 
 	"github.com/containers/libpod/cmd/podman/libpodruntime"
 	"github.com/containers/libpod/libpod"
@@ -45,6 +46,12 @@ Running containers will not be removed without the -f option.
 
 // saveCmd saves the image to either docker-archive or oci
 func rmCmd(c *cli.Context) error {
+	var (
+		delContainers []*libpod.Container
+		lastError     error
+		deleteFuncs   []workerInput
+	)
+
 	ctx := getContext()
 	if err := validateFlags(c, rmFlags); err != nil {
 		return err
@@ -65,8 +72,6 @@ func rmCmd(c *cli.Context) error {
 		return errors.Errorf("specify one or more containers to remove")
 	}
 
-	var delContainers []*libpod.Container
-	var lastError error
 	if c.Bool("all") {
 		delContainers, err = runtime.GetContainers()
 		if err != nil {
@@ -89,16 +94,26 @@ func rmCmd(c *cli.Context) error {
 			delContainers = append(delContainers, container)
 		}
 	}
+
 	for _, container := range delContainers {
-		err = runtime.RemoveContainer(ctx, container, c.Bool("force"))
-		if err != nil {
-			if lastError != nil {
-				fmt.Fprintln(os.Stderr, lastError)
-			}
-			lastError = errors.Wrapf(err, "failed to delete container %v", container.ID())
-		} else {
-			fmt.Println(container.ID())
+		f := func() error {
+			return runtime.RemoveContainer(ctx, container, c.Bool("force"))
+		}
+
+		deleteFuncs = append(deleteFuncs, workerInput{
+			containerID:  container.ID(),
+			parallelFunc: f,
+		})
+	}
+
+	deleteErrors := parallelExecuteWorkerPool(rt.NumCPU()*3, deleteFuncs)
+	for cid, result := range deleteErrors {
+		if result != nil {
+			fmt.Println(result.Error())
+			lastError = result
+			continue
 		}
+		fmt.Println(cid)
 	}
 	return lastError
 }
diff --git a/cmd/podman/stats.go b/cmd/podman/stats.go
index dea351e88..f6beac1a8 100644
--- a/cmd/podman/stats.go
+++ b/cmd/podman/stats.go
@@ -84,8 +84,7 @@ func statsCmd(c *cli.Context) error {
 	if ctr > 1 {
 		return errors.Errorf("--all, --latest and containers cannot be used together")
 	} else if ctr == 0 {
-		// If user didn't specify, imply --all
-		all = true
+		return errors.Errorf("you must specify --all, --latest, or at least one container")
 	}
 
 	runtime, err := libpodruntime.GetRuntime(c)
@@ -126,6 +125,10 @@ func statsCmd(c *cli.Context) error {
 	for _, ctr := range ctrs {
 		initialStats, err := ctr.GetContainerStats(&libpod.ContainerStats{})
 		if err != nil {
+			// when doing "all", dont worry about containers that are not running
+			if c.Bool("all") && errors.Cause(err) == libpod.ErrCtrRemoved || errors.Cause(err) == libpod.ErrNoSuchCtr || errors.Cause(err) == libpod.ErrCtrStateInvalid {
+				continue
+			}
 			return err
 		}
 		containerStats[ctr.ID()] = initialStats
diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh
index 8538ee910..80c689a6c 100755
--- a/contrib/cirrus/build_vm_images.sh
+++ b/contrib/cirrus/build_vm_images.sh
@@ -13,7 +13,7 @@ UBUNTU_BASE_IMAGE $UBUNTU_BASE_IMAGE
 FEDORA_BASE_IMAGE $FEDORA_BASE_IMAGE
 RHEL_BASE_IMAGE $RHEL_BASE_IMAGE
 RHSM_COMMAND $RHSM_COMMAND
-CIRRUS_BUILD_ID $CIRRUS_BUILD_ID
+BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX
 SERVICE_ACCOUNT $SERVICE_ACCOUNT
 GCE_SSH_USERNAME $GCE_SSH_USERNAME
 GCP_PROJECT_ID $GCP_PROJECT_ID
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 098aded0a..1e0052a65 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -35,8 +35,11 @@ req_env_var() {
 # and useful variables.
 # ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables
 show_env_vars() {
+    # This is almost always multi-line, print it separately
+    echo "export CIRRUS_CHANGE_MESSAGE=$CIRRUS_CHANGE_MESSAGE"
     echo "
 BUILDTAGS $BUILDTAGS
+BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX
 CI $CI
 CIRRUS_CI $CIRRUS_CI
 CI_NODE_INDEX $CI_NODE_INDEX
@@ -47,7 +50,6 @@ CIRRUS_BASE_SHA $CIRRUS_BASE_SHA
 CIRRUS_BRANCH $CIRRUS_BRANCH
 CIRRUS_BUILD_ID $CIRRUS_BUILD_ID
 CIRRUS_CHANGE_IN_REPO $CIRRUS_CHANGE_IN_REPO
-CIRRUS_CHANGE_MESSAGE $CIRRUS_CHANGE_MESSAGE
 CIRRUS_CLONE_DEPTH $CIRRUS_CLONE_DEPTH
 CIRRUS_DEFAULT_BRANCH $CIRRUS_DEFAULT_BRANCH
 CIRRUS_PR $CIRRUS_PR
@@ -66,6 +68,7 @@ CIRRUS_USER_PERMISSION $CIRRUS_USER_PERMISSION
 CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR
 CIRRUS_HTTP_CACHE_HOST $CIRRUS_HTTP_CACHE_HOST
 $(go env)
+PACKER_BUILDS $PACKER_BUILDS
     " | while read NAME VALUE
     do
         [[ -z "$NAME" ]] || echo "export $NAME=\"$VALUE\""
@@ -212,6 +215,21 @@ install_conmon(){
     sudo install -D -m 755 bin/conmon /usr/libexec/podman/conmon
 }
 
+install_criu(){
+    echo "Installing CRIU from commit $CRIU_COMMIT"
+    req_env_var "
+    CRIU_COMMIT $CRIU_COMMIT
+    "
+    DEST="/tmp/criu"
+    rm -rf "$DEST"
+    ooe.sh git clone https://github.com/checkpoint-restore/criu.git "$DEST"
+    cd $DEST
+    ooe.sh git fetch origin --tags
+    ooe.sh git checkout -q "$CRIU_COMMIT"
+    ooe.sh make
+    sudo install -D -m 755  criu/criu /usr/sbin/
+}
+
 # Runs in testing VM, not image building
 install_testing_dependencies() {
     echo "Installing ginkgo, gomega, and easyjson into \$GOPATH=$GOPATH"
diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh
index 2253d7b35..7b2308739 100644
--- a/contrib/cirrus/packer/centos_setup.sh
+++ b/contrib/cirrus/packer/centos_setup.sh
@@ -12,6 +12,7 @@ req_env_var "
 SCRIPT_BASE $SCRIPT_BASE
 CNI_COMMIT $CNI_COMMIT
 CRIO_COMMIT $CRIO_COMMIT
+CRIU_COMMIT $CRIU_COMMIT
 "
 
 install_ooe
@@ -24,35 +25,46 @@ ooe.sh sudo yum -y update
 ooe.sh sudo yum -y install centos-release-scl epel-release
 
 ooe.sh sudo yum -y install \
-          atomic-registries \
-          btrfs-progs-devel \
-          bzip2 \
-          device-mapper-devel \
-          findutils \
-          glib2-devel \
-          glibc-static \
-          gnupg \
-          golang \
-          golang-github-cpuguy83-go-md2man \
-          golang-github-cpuguy83-go-md2man \
-          gpgme-devel \
-          iptables \
-          libassuan-devel \
-          libseccomp-devel \
-          libselinux-devel \
-          lsof \
-          make \
-          nmap-ncat \
-          ostree-devel \
-          python \
-          python3-dateutil \
-          python3-psutil \
-          python3-pytoml \
-          runc \
-          skopeo-containers \
-          unzip \
-          which \
-          xz
+    atomic-registries \
+    btrfs-progs-devel \
+    bzip2 \
+    device-mapper-devel \
+    findutils \
+    glib2-devel \
+    glibc-static \
+    gnupg \
+    golang \
+    golang-github-cpuguy83-go-md2man \
+    golang-github-cpuguy83-go-md2man \
+    gpgme-devel \
+    iptables \
+    libassuan-devel \
+    libcap-devel \
+    libnet \
+    libnet-devel \
+    libnl3-devel \
+    libseccomp-devel \
+    libselinux-devel \
+    lsof \
+    make \
+    nmap-ncat \
+    ostree-devel \
+    protobuf \
+    protobuf-c \
+    protobuf-c-devel \
+    protobuf-compiler \
+    protobuf-devel \
+    protobuf-python \
+    python \
+    python2-future \
+    python3-dateutil \
+    python3-psutil \
+    python3-pytoml \
+    runc \
+    skopeo-containers \
+    unzip \
+    which \
+    xz
 
 install_scl_git
 
@@ -62,6 +74,8 @@ install_buildah
 
 install_conmon
 
+install_criu
+
 install_packer_copied_files
 
 rh_finalize
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index 53709fbdd..16b6e4e6b 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -12,6 +12,7 @@ req_env_var "
 SCRIPT_BASE $SCRIPT_BASE
 CNI_COMMIT $CNI_COMMIT
 CRIO_COMMIT $CRIO_COMMIT
+CRIU_COMMIT $CRIU_COMMIT
 RUNC_COMMIT $RUNC_COMMIT
 "
 
@@ -40,6 +41,10 @@ ooe.sh sudo dnf install -y \
     gpgme-devel \
     iptables \
     libassuan-devel \
+    libcap-devel \
+    libnet \
+    libnet-devel \
+    libnl3-devel \
     libseccomp-devel \
     libselinux-devel \
     lsof \
@@ -47,14 +52,21 @@ ooe.sh sudo dnf install -y \
     nmap-ncat \
     ostree-devel \
     procps-ng \
+    protobuf \
+    protobuf-c \
+    protobuf-c-devel \
+    protobuf-compiler \
+    protobuf-devel \
+    protobuf-python \
     python \
+    python2-future \
     python3-dateutil \
     python3-psutil \
     python3-pytoml \
     runc \
     skopeo-containers \
     slirp4netns \
-    which\
+    which \
     xz
 
 install_varlink
@@ -65,6 +77,8 @@ install_buildah
 
 install_conmon
 
+install_criu
+
 install_packer_copied_files
 
 rh_finalize # N/B: Halts system!
diff --git a/contrib/cirrus/packer/libpod_images.json b/contrib/cirrus/packer/libpod_images.json
index 82a41ca25..9dac3e8ea 100644
--- a/contrib/cirrus/packer/libpod_images.json
+++ b/contrib/cirrus/packer/libpod_images.json
@@ -1,7 +1,9 @@
 {
   "variables": {
+    "FEDORA_CNI_COMMIT": "{{env `FEDORA_CNI_COMMIT`}}",
     "CNI_COMMIT": "{{env `CNI_COMMIT`}}",
     "CRIO_COMMIT": "{{env `CRIO_COMMIT`}}",
+    "CRIU_COMMIT": "{{env `CRIU_COMMIT`}}",
     "RUNC_COMMIT": "{{env `RUNC_COMMIT`}}",
 
     "CENTOS_BASE_IMAGE": "{{env `CENTOS_BASE_IMAGE`}}" ,
@@ -15,7 +17,7 @@
 
     "SERVICE_ACCOUNT": "{{env `SERVICE_ACCOUNT`}}",
     "GCP_PROJECT_ID": "{{env `GCP_PROJECT_ID`}}",
-    "CIRRUS_BUILD_ID": "{{env `CIRRUS_BUILD_ID`}}",
+    "BUILT_IMAGE_SUFFIX": "{{env `BUILT_IMAGE_SUFFIX`}}",
     "GCE_SSH_USERNAME": "{{env `GCE_SSH_USERNAME`}}",
     "RHSM_COMMAND": "{{env `RHSM_COMMAND`}}"
   },
@@ -29,7 +31,7 @@
       "project_id": "{{user `GCP_PROJECT_ID`}}",
       "zone": "us-central1-a",
       "source_image": "{{user `RHEL_BASE_IMAGE`}}",
-      "image_name": "{{user `RHEL_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}",
+      "image_name": "{{user `RHEL_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
       "image_family": "{{user `RHEL_BASE_IMAGE`}}-libpod",
       "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
       "communicator": "ssh",
@@ -41,7 +43,7 @@
       "project_id": "{{user `GCP_PROJECT_ID`}}",
       "zone": "us-central1-a",
       "source_image": "{{user `CENTOS_BASE_IMAGE`}}",
-      "image_name": "{{user `CENTOS_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}",
+      "image_name": "{{user `CENTOS_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
       "image_family": "{{user `CENTOS_BASE_IMAGE`}}-libpod",
       "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
       "communicator": "ssh",
@@ -53,7 +55,7 @@
       "project_id": "{{user `GCP_PROJECT_ID`}}",
       "zone": "us-central1-a",
       "source_image": "{{user `FEDORA_BASE_IMAGE`}}",
-      "image_name": "{{user `FEDORA_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}",
+      "image_name": "{{user `FEDORA_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
       "image_family": "{{user `FEDORA_BASE_IMAGE`}}-libpod",
       "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
       "communicator": "ssh",
@@ -65,7 +67,7 @@
       "project_id": "{{user `GCP_PROJECT_ID`}}",
       "zone": "us-central1-a",
       "source_image": "{{user `UBUNTU_BASE_IMAGE`}}",
-      "image_name": "{{user `UBUNTU_BASE_IMAGE`}}-libpod-{{user `CIRRUS_BUILD_ID`}}",
+      "image_name": "{{user `UBUNTU_BASE_IMAGE`}}{{user `BUILT_IMAGE_SUFFIX`}}",
       "image_family": "{{user `UBUNTU_BASE_IMAGE`}}-libpod",
       "service_account_email": "{{user `SERVICE_ACCOUNT`}}",
       "communicator": "ssh",
@@ -86,6 +88,7 @@
         "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
         "CNI_COMMIT={{user `CNI_COMMIT`}}",
         "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
+        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
         "RUNC_COMMIT={{user `RUNC_COMMIT`}}",
         "RHSM_COMMAND={{user `RHSM_COMMAND`}}"
       ]
@@ -97,6 +100,7 @@
         "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
         "CNI_COMMIT={{user `CNI_COMMIT`}}",
         "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
+        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
         "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
       ]
     },{
@@ -105,8 +109,9 @@
       "script": "{{user `GOSRC`}}/{{user `PACKER_BASE`}}/fedora_setup.sh",
       "environment_vars": [
         "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
-        "CNI_COMMIT={{user `CNI_COMMIT`}}",
+        "CNI_COMMIT={{user `FEDORA_CNI_COMMIT`}}",
         "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
+        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
         "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
       ]
     },{
@@ -117,6 +122,7 @@
         "SCRIPT_BASE={{user `SCRIPT_BASE`}}",
         "CNI_COMMIT={{user `CNI_COMMIT`}}",
         "CRIO_COMMIT={{user `CRIO_COMMIT`}}",
+        "CRIU_COMMIT={{user `CRIU_COMMIT`}}",
         "RUNC_COMMIT={{user `RUNC_COMMIT`}}"
       ]
     }
diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh
index b776a0d97..d296713fc 100644
--- a/contrib/cirrus/packer/rhel_setup.sh
+++ b/contrib/cirrus/packer/rhel_setup.sh
@@ -12,6 +12,7 @@ req_env_var "
 SCRIPT_BASE $SCRIPT_BASE
 CNI_COMMIT $CNI_COMMIT
 CRIO_COMMIT $CRIO_COMMIT
+CRIU_COMMIT $CRIU_COMMIT
 RHSM_COMMAND $RHSM_COMMAND
 "
 
@@ -62,37 +63,48 @@ gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg
 EOM
 
 ooe.sh sudo yum -y install \
-          atomic-registries \
-          btrfs-progs-devel \
-          bzip2 \
-          device-mapper-devel \
-          findutils \
-          glib2-devel \
-          glibc-static \
-          gnupg \
-          golang \
-          golang-github-cpuguy83-go-md2man \
-          golang-github-cpuguy83-go-md2man \
-          google-compute-engine \
-          google-compute-engine-oslogin \
-          gpgme-devel \
-          iptables \
-          libassuan-devel \
-          libseccomp-devel \
-          libselinux-devel \
-          lsof \
-          make \
-          nmap-ncat \
-          ostree-devel \
-          python \
-          python34-dateutil \
-          python34-psutil \
-          python34-pytoml \
-          runc \
-          skopeo-containers \
-          unzip \
-          which \
-          xz
+    atomic-registries \
+    btrfs-progs-devel \
+    bzip2 \
+    device-mapper-devel \
+    findutils \
+    glib2-devel \
+    glibc-static \
+    gnupg \
+    golang \
+    golang-github-cpuguy83-go-md2man \
+    golang-github-cpuguy83-go-md2man \
+    google-compute-engine \
+    google-compute-engine-oslogin \
+    gpgme-devel \
+    iptables \
+    libassuan-devel \
+    libcap-devel \
+    libnet \
+    libnet-devel \
+    libnl3-devel \
+    libseccomp-devel \
+    libselinux-devel \
+    lsof \
+    make \
+    nmap-ncat \
+    ostree-devel \
+    protobuf \
+    protobuf-c \
+    protobuf-c-devel \
+    protobuf-compiler \
+    protobuf-devel \
+    protobuf-python \
+    python \
+    python2-future \
+    python34-dateutil \
+    python34-psutil \
+    python34-pytoml \
+    runc \
+    skopeo-containers \
+    unzip \
+    which \
+    xz
 
 install_scl_git
 
@@ -102,6 +114,8 @@ install_buildah
 
 install_conmon
 
+install_criu
+
 install_packer_copied_files
 
 exit_handler  # release subscription!
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index 96b3a573f..ff20944dc 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -12,6 +12,7 @@ req_env_var "
 SCRIPT_BASE $SCRIPT_BASE
 CNI_COMMIT $CNI_COMMIT
 CRIO_COMMIT $CRIO_COMMIT
+CRIU_COMMIT $CRIU_COMMIT
 RUNC_COMMIT $RUNC_COMMIT
 "
 
@@ -34,8 +35,8 @@ ooe.sh sudo apt-get -qq install --no-install-recommends \
     e2fslibs-dev \
     gawk \
     gettext \
-    golang \
     go-md2man \
+    golang \
     iptables \
     libaio-dev \
     libapparmor-dev \
@@ -46,18 +47,22 @@ ooe.sh sudo apt-get -qq install --no-install-recommends \
     libglib2.0-dev \
     libgpgme11-dev \
     liblzma-dev \
+    libnet1 \
+    libnet1-dev \
+    libnl-3-dev \
     libostree-dev \
     libprotobuf-c0-dev \
     libprotobuf-dev \
     libtool \
-    libtool \
     libudev-dev \
     lsof \
     netcat \
     pkg-config \
     protobuf-c-compiler \
     protobuf-compiler \
+    python-future \
     python-minimal \
+    python-protobuf \
     python3-dateutil \
     python3-pip \
     python3-psutil \
@@ -77,6 +82,8 @@ install_runc
 
 install_conmon
 
+install_criu
+
 install_cni_plugins
 
 install_buildah
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 2302f0e15..167db127f 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -41,6 +41,7 @@ then
         "export OS_RELEASE_ID=\"$(os_release_id)\"" \
         "export OS_RELEASE_VER=\"$(os_release_ver)\"" \
         "export OS_REL_VER=\"${OS_RELEASE_ID}-${OS_RELEASE_VER}\"" \
+        "export BUILT_IMAGE_SUFFIX=\"-$CIRRUS_REPO_NAME-${CIRRUS_CHANGE_IN_REPO:0:8}\"" \
         "export GOPATH=\"/go\"" \
         'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \
         'export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"'
diff --git a/docs/podman.1.md b/docs/podman.1.md
index 3a0943d6b..085af97ff 100644
--- a/docs/podman.1.md
+++ b/docs/podman.1.md
@@ -42,11 +42,13 @@ When namespace is set, created containers and pods will join the given namespace
 
 **--root**=**value**
 
-Path to the root directory in which data, including images, is stored
+Storage root dir in which data, including images, is stored (default: "/var/lib/containers/storage" for UID 0, "$HOME/.local/share/containers/storage" for other users).
+Default root dir is configured in /etc/containers/storage.conf.
 
 **--runroot**=**value**
 
-Path to the 'run directory' where all state information is stored
+Storage state directory where all state information is stored (default: "/var/run/containers/storage" for UID 0, "/var/run/user/$UID/run" for other users).
+Default state dir is configured in /etc/containers/storage.conf.
 
 **--runtime**=**value**
 
@@ -73,7 +75,7 @@ Print the version
 
 ## Exit Status
 
-The exit code from `podman gives information about why the container
+The exit code from `podman` gives information about why the container
 failed to run or why it exited.  When `podman` commands exit with a non-zero code,
 the exit codes follow the `chroot` standard, see below:
 
diff --git a/libpod/oci.go b/libpod/oci.go
index f6d320017..2257cd42f 100644
--- a/libpod/oci.go
+++ b/libpod/oci.go
@@ -17,6 +17,7 @@ import (
 
 	"github.com/containers/libpod/pkg/ctime"
 	"github.com/containers/libpod/pkg/rootless"
+	"github.com/containers/libpod/pkg/util"
 	"github.com/coreos/go-systemd/activation"
 	"github.com/cri-o/ocicni/pkg/ocicni"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
@@ -230,7 +231,7 @@ func bindPorts(ports []ocicni.PortMapping) ([]*os.File, error) {
 func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, restoreContainer bool) (err error) {
 	var stderrBuf bytes.Buffer
 
-	runtimeDir, err := GetRootlessRuntimeDir()
+	runtimeDir, err := util.GetRootlessRuntimeDir()
 	if err != nil {
 		return err
 	}
@@ -377,6 +378,7 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, res
 		childPipe.Close()
 		return err
 	}
+	defer cmd.Wait()
 
 	// We don't need childPipe on the parent side
 	childPipe.Close()
@@ -446,7 +448,7 @@ func (r *OCIRuntime) createOCIContainer(ctr *Container, cgroupParent string, res
 func (r *OCIRuntime) updateContainerStatus(ctr *Container) error {
 	state := new(spec.State)
 
-	runtimeDir, err := GetRootlessRuntimeDir()
+	runtimeDir, err := util.GetRootlessRuntimeDir()
 	if err != nil {
 		return err
 	}
@@ -477,6 +479,7 @@ func (r *OCIRuntime) updateContainerStatus(ctr *Container) error {
 		}
 		return errors.Wrapf(err, "error getting container %s state. stderr/out: %s", ctr.ID(), out)
 	}
+	defer cmd.Wait()
 
 	errPipe.Close()
 	out, err := ioutil.ReadAll(outPipe)
@@ -556,7 +559,7 @@ func (r *OCIRuntime) updateContainerStatus(ctr *Container) error {
 // Sets time the container was started, but does not save it.
 func (r *OCIRuntime) startContainer(ctr *Container) error {
 	// TODO: streams should probably *not* be our STDIN/OUT/ERR - redirect to buffers?
-	runtimeDir, err := GetRootlessRuntimeDir()
+	runtimeDir, err := util.GetRootlessRuntimeDir()
 	if err != nil {
 		return err
 	}
@@ -573,7 +576,7 @@ func (r *OCIRuntime) startContainer(ctr *Container) error {
 // killContainer sends the given signal to the given container
 func (r *OCIRuntime) killContainer(ctr *Container, signal uint) error {
 	logrus.Debugf("Sending signal %d to container %s", signal, ctr.ID())
-	runtimeDir, err := GetRootlessRuntimeDir()
+	runtimeDir, err := util.GetRootlessRuntimeDir()
 	if err != nil {
 		return err
 	}
@@ -636,7 +639,7 @@ func (r *OCIRuntime) stopContainer(ctr *Container, timeout uint) error {
 		args = []string{"kill", "--all", ctr.ID(), "KILL"}
 	}
 
-	runtimeDir, err := GetRootlessRuntimeDir()
+	runtimeDir, err := util.GetRootlessRuntimeDir()
 	if err != nil {
 		return err
 	}
@@ -667,7 +670,7 @@ func (r *OCIRuntime) deleteContainer(ctr *Container) error {
 
 // pauseContainer pauses the given container
 func (r *OCIRuntime) pauseContainer(ctr *Container) error {
-	runtimeDir, err := GetRootlessRuntimeDir()
+	runtimeDir, err := util.GetRootlessRuntimeDir()
 	if err != nil {
 		return err
 	}
@@ -677,7 +680,7 @@ func (r *OCIRuntime) pauseContainer(ctr *Container) error {
 
 // unpauseContainer unpauses the given container
 func (r *OCIRuntime) unpauseContainer(ctr *Container) error {
-	runtimeDir, err := GetRootlessRuntimeDir()
+	runtimeDir, err := util.GetRootlessRuntimeDir()
 	if err != nil {
 		return err
 	}
@@ -698,7 +701,7 @@ func (r *OCIRuntime) execContainer(c *Container, cmd, capAdd, env []string, tty
 		return nil, errors.Wrapf(ErrEmptyID, "must provide a session ID for exec")
 	}
 
-	runtimeDir, err := GetRootlessRuntimeDir()
+	runtimeDir, err := util.GetRootlessRuntimeDir()
 	if err != nil {
 		return nil, err
 	}
@@ -780,7 +783,7 @@ func (r *OCIRuntime) execStopContainer(ctr *Container, timeout uint) error {
 	if len(execSessions) == 0 {
 		return nil
 	}
-	runtimeDir, err := GetRootlessRuntimeDir()
+	runtimeDir, err := util.GetRootlessRuntimeDir()
 	if err != nil {
 		return err
 	}
diff --git a/libpod/runtime.go b/libpod/runtime.go
index 985af2849..f012d66c2 100644
--- a/libpod/runtime.go
+++ b/libpod/runtime.go
@@ -1,13 +1,11 @@
 package libpod
 
 import (
-	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
 	"path/filepath"
 	"sync"
-	"syscall"
 
 	"github.com/BurntSushi/toml"
 	is "github.com/containers/image/storage"
@@ -17,6 +15,7 @@ import (
 	"github.com/containers/libpod/pkg/hooks"
 	sysreg "github.com/containers/libpod/pkg/registries"
 	"github.com/containers/libpod/pkg/rootless"
+	"github.com/containers/libpod/pkg/util"
 	"github.com/containers/storage"
 	"github.com/cri-o/ocicni/pkg/ocicni"
 	"github.com/docker/docker/pkg/namesgenerator"
@@ -215,46 +214,12 @@ var (
 	}
 )
 
-// GetRootlessRuntimeDir returns the runtime directory when running as non root
-func GetRootlessRuntimeDir() (string, error) {
-	runtimeDir := os.Getenv("XDG_RUNTIME_DIR")
-	uid := fmt.Sprintf("%d", rootless.GetRootlessUID())
-	if runtimeDir == "" {
-		tmpDir := filepath.Join("/run", "user", uid)
-		os.MkdirAll(tmpDir, 0700)
-		st, err := os.Stat(tmpDir)
-		if err == nil && int(st.Sys().(*syscall.Stat_t).Uid) == os.Getuid() && st.Mode().Perm() == 0700 {
-			runtimeDir = tmpDir
-		}
-	}
-	if runtimeDir == "" {
-		tmpDir := filepath.Join(os.TempDir(), "user", uid)
-		os.MkdirAll(tmpDir, 0700)
-		st, err := os.Stat(tmpDir)
-		if err == nil && int(st.Sys().(*syscall.Stat_t).Uid) == os.Getuid() && st.Mode().Perm() == 0700 {
-			runtimeDir = tmpDir
-		}
-	}
-	if runtimeDir == "" {
-		home := os.Getenv("HOME")
-		if home == "" {
-			return "", fmt.Errorf("neither XDG_RUNTIME_DIR nor HOME was set non-empty")
-		}
-		resolvedHome, err := filepath.EvalSymlinks(home)
-		if err != nil {
-			return "", errors.Wrapf(err, "cannot resolve %s", home)
-		}
-		runtimeDir = filepath.Join(resolvedHome, "rundir")
-	}
-	return runtimeDir, nil
-}
-
 func getDefaultTmpDir() (string, error) {
 	if !rootless.IsRootless() {
 		return "/var/run/libpod", nil
 	}
 
-	rootlessRuntimeDir, err := GetRootlessRuntimeDir()
+	rootlessRuntimeDir, err := util.GetRootlessRuntimeDir()
 	if err != nil {
 		return "", err
 	}
@@ -269,7 +234,7 @@ func SetXdgRuntimeDir(val string) error {
 	}
 	if val == "" {
 		var err error
-		val, err = GetRootlessRuntimeDir()
+		val, err = util.GetRootlessRuntimeDir()
 		if err != nil {
 			return err
 		}
@@ -309,7 +274,7 @@ func NewRuntime(options ...RuntimeOption) (runtime *Runtime, err error) {
 			foundConfig = false
 		}
 
-		runtimeDir, err := GetRootlessRuntimeDir()
+		runtimeDir, err := util.GetRootlessRuntimeDir()
 		if err != nil {
 			return nil, err
 		}
diff --git a/pkg/util/utils.go b/pkg/util/utils.go
index 28dd015bd..9107eec5c 100644
--- a/pkg/util/utils.go
+++ b/pkg/util/utils.go
@@ -3,10 +3,13 @@ package util
 import (
 	"fmt"
 	"os"
+	"path/filepath"
 	"strconv"
 	"strings"
+	"syscall"
 
 	"github.com/containers/image/types"
+	"github.com/containers/libpod/pkg/rootless"
 	"github.com/containers/storage"
 	"github.com/containers/storage/pkg/idtools"
 	"github.com/opencontainers/image-spec/specs-go/v1"
@@ -210,3 +213,84 @@ func ParseIDMapping(UIDMapSlice, GIDMapSlice []string, subUIDMap, subGIDMap stri
 	}
 	return &options, nil
 }
+
+// GetRootlessRuntimeDir returns the runtime directory when running as non root
+func GetRootlessRuntimeDir() (string, error) {
+	runtimeDir := os.Getenv("XDG_RUNTIME_DIR")
+	uid := fmt.Sprintf("%d", rootless.GetRootlessUID())
+	if runtimeDir == "" {
+		tmpDir := filepath.Join("/run", "user", uid)
+		os.MkdirAll(tmpDir, 0700)
+		st, err := os.Stat(tmpDir)
+		if err == nil && int(st.Sys().(*syscall.Stat_t).Uid) == os.Getuid() && st.Mode().Perm() == 0700 {
+			runtimeDir = tmpDir
+		}
+	}
+	if runtimeDir == "" {
+		tmpDir := filepath.Join(os.TempDir(), "user", uid)
+		os.MkdirAll(tmpDir, 0700)
+		st, err := os.Stat(tmpDir)
+		if err == nil && int(st.Sys().(*syscall.Stat_t).Uid) == os.Getuid() && st.Mode().Perm() == 0700 {
+			runtimeDir = tmpDir
+		}
+	}
+	if runtimeDir == "" {
+		home := os.Getenv("HOME")
+		if home == "" {
+			return "", fmt.Errorf("neither XDG_RUNTIME_DIR nor HOME was set non-empty")
+		}
+		resolvedHome, err := filepath.EvalSymlinks(home)
+		if err != nil {
+			return "", errors.Wrapf(err, "cannot resolve %s", home)
+		}
+		runtimeDir = filepath.Join(resolvedHome, "rundir")
+	}
+	return runtimeDir, nil
+}
+
+// GetRootlessStorageOpts returns the storage ops for containers running as non root
+func GetRootlessStorageOpts() (storage.StoreOptions, error) {
+	var opts storage.StoreOptions
+
+	rootlessRuntime, err := GetRootlessRuntimeDir()
+	if err != nil {
+		return opts, err
+	}
+	opts.RunRoot = filepath.Join(rootlessRuntime, "run")
+
+	dataDir := os.Getenv("XDG_DATA_HOME")
+	if dataDir == "" {
+		home := os.Getenv("HOME")
+		if home == "" {
+			return opts, fmt.Errorf("neither XDG_DATA_HOME nor HOME was set non-empty")
+		}
+		// runc doesn't like symlinks in the rootfs path, and at least
+		// on CoreOS /home is a symlink to /var/home, so resolve any symlink.
+		resolvedHome, err := filepath.EvalSymlinks(home)
+		if err != nil {
+			return opts, errors.Wrapf(err, "cannot resolve %s", home)
+		}
+		dataDir = filepath.Join(resolvedHome, ".local", "share")
+	}
+	opts.GraphRoot = filepath.Join(dataDir, "containers", "storage")
+	opts.GraphDriverName = "vfs"
+	return opts, nil
+}
+
+// GetDefaultStoreOptions returns the storage ops for containers
+func GetDefaultStoreOptions() (storage.StoreOptions, error) {
+	storageOpts := storage.DefaultStoreOptions
+	if rootless.IsRootless() {
+		var err error
+		storageOpts, err = GetRootlessStorageOpts()
+		if err != nil {
+			return storageOpts, err
+		}
+
+		storageConf := filepath.Join(os.Getenv("HOME"), ".config/containers/storage.conf")
+		if _, err := os.Stat(storageConf); err == nil {
+			storage.ReloadConfigurationFile(storageConf, &storageOpts)
+		}
+	}
+	return storageOpts, nil
+}
diff --git a/test/e2e/exec_test.go b/test/e2e/exec_test.go
index cfdc819a6..250e08704 100644
--- a/test/e2e/exec_test.go
+++ b/test/e2e/exec_test.go
@@ -111,6 +111,7 @@ var _ = Describe("Podman exec", func() {
 	})
 
 	It("podman exec with user only in container", func() {
+		podmanTest.RestoreArtifact(fedoraMinimal)
 		testUser := "test123"
 		setup := podmanTest.Podman([]string{"run", "--name", "test1", "-d", fedoraMinimal, "sleep", "60"})
 		setup.WaitWithDefaultTimeout()
diff --git a/test/e2e/run_test.go b/test/e2e/run_test.go
index 1cd2fdf2c..cb436ccca 100644
--- a/test/e2e/run_test.go
+++ b/test/e2e/run_test.go
@@ -577,6 +577,7 @@ USER mail`
 	})
 
 	It("podman run findmnt nothing shared", func() {
+		podmanTest.RestoreArtifact(fedoraMinimal)
 		vol1 := filepath.Join(podmanTest.TempDir, "vol-test1")
 		err := os.MkdirAll(vol1, 0755)
 		Expect(err).To(BeNil())
@@ -592,6 +593,7 @@ USER mail`
 	})
 
 	It("podman run findmnt shared", func() {
+		podmanTest.RestoreArtifact(fedoraMinimal)
 		vol1 := filepath.Join(podmanTest.TempDir, "vol-test1")
 		err := os.MkdirAll(vol1, 0755)
 		Expect(err).To(BeNil())
diff --git a/test/e2e/stats_test.go b/test/e2e/stats_test.go
index 8096f58b2..e456d7114 100644
--- a/test/e2e/stats_test.go
+++ b/test/e2e/stats_test.go
@@ -31,12 +31,6 @@ var _ = Describe("Podman stats", func() {
 		GinkgoWriter.Write([]byte(timedResult))
 	})
 
-	It("podman stats should run with no containers", func() {
-		session := podmanTest.Podman([]string{"stats", "--no-stream"})
-		session.WaitWithDefaultTimeout()
-		Expect(session.ExitCode()).To(Equal(0))
-	})
-
 	It("podman stats with bogus container", func() {
 		session := podmanTest.Podman([]string{"stats", "--no-stream", "123"})
 		session.WaitWithDefaultTimeout()
@@ -53,15 +47,6 @@ var _ = Describe("Podman stats", func() {
 		Expect(session.ExitCode()).To(Equal(0))
 	})
 
-	It("podman stats on a running container no id", func() {
-		session := podmanTest.RunTopContainer("")
-		session.WaitWithDefaultTimeout()
-		Expect(session.ExitCode()).To(Equal(0))
-		session = podmanTest.Podman([]string{"stats", "--no-stream"})
-		session.WaitWithDefaultTimeout()
-		Expect(session.ExitCode()).To(Equal(0))
-	})
-
 	It("podman stats on all containers", func() {
 		session := podmanTest.RunTopContainer("")
 		session.WaitWithDefaultTimeout()
@@ -75,7 +60,7 @@ var _ = Describe("Podman stats", func() {
 		session := podmanTest.RunTopContainer("")
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
-		session = podmanTest.Podman([]string{"stats", "--no-stream", "--format", "\"{{.Container}}\""})
+		session = podmanTest.Podman([]string{"stats", "--all", "--no-stream", "--format", "\"{{.Container}}\""})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 	})
@@ -84,7 +69,7 @@ var _ = Describe("Podman stats", func() {
 		session := podmanTest.RunTopContainer("")
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
-		session = podmanTest.Podman([]string{"stats", "--no-stream", "--format", "json"})
+		session = podmanTest.Podman([]string{"stats", "--all", "--no-stream", "--format", "json"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 		Expect(session.IsJSONOutputValid()).To(BeTrue())