Merge pull request #9423 from Luap99/rootless-cni-no-infra

rootless cni without infra container

24 files changed, 1089 insertions, 1237 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index e09db2a81..074f2f4e0 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -440,7 +440,7 @@ apiv2_test_task:
         time_script: '$SCRIPT_BASE/logcollector.sh time'
 
 compose_test_task:
-    name: "compose test on $DISTRO_NV"
+    name: "compose test on $DISTRO_NV ($PRIV_NAME)"
     alias: compose_test
     only_if: *not_docs
     skip: *tags
@@ -450,6 +450,11 @@ compose_test_task:
     env:
         <<: *stdenvars
         TEST_FLAVOR: compose
+    matrix:
+      - env:
+            PRIV_NAME: root
+      - env:
+            PRIV_NAME: rootless
     clone_script: *noop  # Comes from cache
     gopath_cache: *ro_gopath_cache
     setup_script: *setup
diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh
index 507d22e13..f52e107cc 100755
--- a/contrib/cirrus/runner.sh
+++ b/contrib/cirrus/runner.sh
@@ -288,18 +288,6 @@ dotest() {
         exec_container  # does not return
     fi;
 
-    # shellcheck disable=SC2154
-    if [[ "$PRIV_NAME" == "rootless" ]] && [[ "$UID" -eq 0 ]]; then
-        req_env_vars ROOTLESS_USER
-        msg "Re-executing runner through ssh as user '$ROOTLESS_USER'"
-        msg "************************************************************"
-        set -x
-        exec ssh $ROOTLESS_USER@localhost \
-                -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
-                -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/runner.sh
-        # does not return
-    fi
-
     # containers/automation sets this to 0 for its dbg() function
     # but the e2e integration tests are also sensitive to it.
     unset DEBUG
@@ -340,6 +328,19 @@ msg "************************************************************"
 ((${SETUP_ENVIRONMENT:-0})) || \
     die "Expecting setup_environment.sh to have completed successfully"
 
+# shellcheck disable=SC2154
+if [[ "$PRIV_NAME" == "rootless" ]] && [[ "$UID" -eq 0 ]]; then
+    req_env_vars ROOTLESS_USER
+    msg "Re-executing runner through ssh as user '$ROOTLESS_USER'"
+    msg "************************************************************"
+    set -x
+    exec ssh $ROOTLESS_USER@localhost \
+            -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
+            -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/runner.sh
+    # Does not return!
+fi
+# else: not running rootless, do nothing special
+
 cd "${GOSRC}/"
 
 handler="_run_${TEST_FLAVOR}"
diff --git a/contrib/rootless-cni-infra/Containerfile b/contrib/rootless-cni-infra/Containerfile
deleted file mode 100644
index 4324f39d2..000000000
--- a/contrib/rootless-cni-infra/Containerfile
+++ /dev/null
@@ -1,36 +0,0 @@
-ARG GOLANG_VERSION=1.15
-ARG ALPINE_VERSION=3.12
-ARG CNI_VERSION=v0.8.0
-ARG CNI_PLUGINS_VERSION=v0.8.7
-ARG DNSNAME_VERSION=v1.1.1
-
-FROM golang:${GOLANG_VERSION}-alpine${ALPINE_VERSION} AS golang-base
-RUN apk add --no-cache git
-
-FROM golang-base AS cnitool
-RUN git clone https://github.com/containernetworking/cni /go/src/github.com/containernetworking/cni
-WORKDIR /go/src/github.com/containernetworking/cni
-ARG CNI_VERSION
-RUN git checkout ${CNI_VERSION}
-RUN go build -o /cnitool ./cnitool
-
-FROM golang-base AS dnsname
-RUN git clone https://github.com/containers/dnsname /go/src/github.com/containers/dnsname
-WORKDIR /go/src/github.com/containers/dnsname
-ARG DNSNAME_VERSION
-RUN git checkout ${DNSNAME_VERSION}
-RUN go build -o /dnsname ./plugins/meta/dnsname
-
-FROM alpine:${ALPINE_VERSION}
-RUN apk add --no-cache curl dnsmasq iptables ip6tables iproute2
-ARG TARGETARCH
-ARG CNI_PLUGINS_VERSION
-RUN mkdir -p /opt/cni/bin && \
-    curl -fsSL https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH}-${CNI_PLUGINS_VERSION}.tgz | tar xz -C /opt/cni/bin
-COPY --from=cnitool /cnitool /usr/local/bin
-COPY --from=dnsname /dnsname /opt/cni/bin
-COPY rootless-cni-infra /usr/local/bin
-ENV CNI_PATH=/opt/cni/bin
-CMD ["sleep", "infinity"]
-
-ENV ROOTLESS_CNI_INFRA_VERSION=5
diff --git a/contrib/rootless-cni-infra/README.md b/contrib/rootless-cni-infra/README.md
deleted file mode 100644
index c43b4cf49..000000000
--- a/contrib/rootless-cni-infra/README.md
+++ /dev/null
@@ -1,25 +0,0 @@
-# rootless-cni-infra
-
-Infra container for CNI-in-slirp4netns.
-
-## How it works
-
-When a CNI network is specified for `podman run` in rootless mode, Podman launches the `rootless-cni-infra` container to execute CNI plugins inside slirp4netns.
-
-The infra container is created per user, by executing an equivalent of:
-`podman run -d --name rootless-cni-infra --pid=host --privileged -v $HOME/.config/cni/net.d:/etc/cni/net.d rootless-cni-infra`.
-The infra container is automatically deleted when no CNI network is in use.
-
-Podman then allocates a CNI netns in the infra container, by executing an equivalent of:
-`podman exec rootless-cni-infra rootless-cni-infra alloc $CONTAINER_ID $NETWORK_NAME $POD_NAME`.
-
-The allocated netns is deallocated when the container is being removed, by executing an equivalent of:
-`podman exec rootless-cni-infra rootless-cni-infra dealloc $CONTAINER_ID $NETWORK_NAME`.
-
-The container images live on `quay.io/libpod/rootless-cni-infra`.  The tags have the format `$version-$architecture`.  Please make sure to increase the version number in the Containerfile (i.e., `ROOTLESS_CNI_INFRA_VERSION`) when applying changes to this directory.  After committing the changes, upload the image(s) with the corresponding tag.
-
-## Directory layout
-
-* `/run/rootless-cni-infra/${CONTAINER_ID}/pid`: PID of the `sleep infinity` process that corresponds to the allocated netns
-* `/run/rootless-cni-infra/${CONTAINER_ID}/attached/${NETWORK_NAME}`: CNI result
-* `/run/rootless-cni-infra/${CONTAINER_ID}/attached-args/${NETWORK_NAME}`: CNI args
diff --git a/contrib/rootless-cni-infra/rootless-cni-infra b/contrib/rootless-cni-infra/rootless-cni-infra
deleted file mode 100755
index cceb8d817..000000000
--- a/contrib/rootless-cni-infra/rootless-cni-infra
+++ /dev/null
@@ -1,181 +0,0 @@
-#!/bin/sh
-set -eu
-
-ARG0="$0"
-BASE="/run/rootless-cni-infra"
-
-wait_unshare_net() {
-	pid="$1"
-	# NOTE: busybox shell doesn't support the `for ((i=0; i < $MAX; i++)); do foo; done` statement
-	i=0
-	while :; do
-		if [ "$(readlink /proc/self/ns/net)" != "$(readlink /proc/${pid}/ns/net)" ]; then
-			break
-		fi
-		sleep 0.1
-		if [ $i -ge 10 ]; then
-			echo >&2 "/proc/${pid}/ns/net cannot be unshared"
-			exit 1
-		fi
-		i=$((i + 1))
-	done
-}
-
-# CLI subcommand: "alloc $CONTAINER_ID $NETWORK_NAME $POD_NAME $IP $MAC $CAP_ARGS"
-cmd_entrypoint_alloc() {
-	if [ "$#" -ne 6 ]; then
-		echo >&2 "Usage: $ARG0 alloc CONTAINER_ID NETWORK_NAME POD_NAME IP MAC CAP_ARGS"
-		exit 1
-	fi
-
-	ID="$1"
-	NET="$2"
-	K8S_POD_NAME="$3"
-	IP="$4"
-	MAC="$5"
-	CAP_ARGS="$6"
-
-	dir="${BASE}/${ID}"
-	mkdir -p "${dir}/attached" "${dir}/attached-args"
-
-	pid=""
-	if [ -f "${dir}/pid" ]; then
-		pid=$(cat "${dir}/pid")
-	else
-		unshare -n sleep infinity &
-		pid="$!"
-		wait_unshare_net "${pid}"
-		echo "${pid}" >"${dir}/pid"
-		nsenter -t "${pid}" -n ip link set lo up
-	fi
-	CNI_ARGS="IgnoreUnknown=1;K8S_POD_NAME=${K8S_POD_NAME}"
-	if [ "$IP" ]; then
-		CNI_ARGS="$CNI_ARGS;IP=${IP}"
-	fi
-	if [ "$MAC" ]; then
-		CNI_ARGS="$CNI_ARGS;MAC=${MAC}"
-	fi
-	if [ "$CAP_ARGS" ]; then
-		CAP_ARGS="$CAP_ARGS"
-	fi
-	nwcount=$(find "${dir}/attached" -type f | wc -l)
-	CNI_IFNAME="eth${nwcount}"
-	export CNI_ARGS CNI_IFNAME CAP_ARGS
-	cnitool add "${NET}" "/proc/${pid}/ns/net" >"${dir}/attached/${NET}"
-	echo "${CNI_ARGS}" >"${dir}/attached-args/${NET}"
-
-	# return the result
-	ns="/proc/${pid}/ns/net"
-	echo "{\"ns\":\"${ns}\"}"
-}
-
-# CLI subcommand: "dealloc $CONTAINER_ID $NETWORK_NAME"
-cmd_entrypoint_dealloc() {
-	if [ "$#" -ne 2 ]; then
-		echo >&2 "Usage: $ARG0 dealloc CONTAINER_ID NETWORK_NAME"
-		exit 1
-	fi
-
-	ID=$1
-	NET=$2
-
-	dir="${BASE}/${ID}"
-	if [ ! -f "${dir}/pid" ]; then
-		exit 0
-	fi
-	pid=$(cat "${dir}/pid")
-	if [ -f "${dir}/attached-args/${NET}" ]; then
-		CNI_ARGS=$(cat "${dir}/attached-args/${NET}")
-		export CNI_ARGS
-	fi
-	cnitool del "${NET}" "/proc/${pid}/ns/net"
-	rm -f "${dir}/attached/${NET}" "${dir}/attached-args/${NET}"
-
-	nwcount=$(find "${dir}/attached" -type f | wc -l)
-	if [ "${nwcount}" = 0 ]; then
-		kill -9 "${pid}"
-		rm -rf "${dir}"
-	fi
-
-	# return empty json
-	echo "{}"
-}
-
-# CLI subcommand: "is-idle"
-cmd_entrypoint_is_idle() {
-	if [ ! -d ${BASE} ]; then
-		echo '{"idle": true}'
-	elif [ -z "$(ls -1 ${BASE})" ]; then
-		echo '{"idle": true}'
-	else
-		echo '{"idle": false}'
-	fi
-}
-
-# CLI subcommand: "print-cni-result $CONTAINER_ID $NETWORK_NAME"
-cmd_entrypoint_print_cni_result() {
-	if [ "$#" -ne 2 ]; then
-		echo >&2 "Usage: $ARG0 print-cni-result CONTAINER_ID NETWORK_NAME"
-		exit 1
-	fi
-
-	ID=$1
-	NET=$2
-
-	# the result shall be CNI JSON
-	cat "${BASE}/${ID}/attached/${NET}"
-}
-
-# CLI subcommand: "print-netns-path $CONTAINER_ID"
-cmd_entrypoint_print_netns_path() {
-	if [ "$#" -ne 1 ]; then
-		echo >&2 "Usage: $ARG0 print-netns-path CONTAINER_ID"
-		exit 1
-	fi
-
-	ID=$1
-
-	pid=$(cat "${BASE}/${ID}/pid")
-	path="/proc/${pid}/ns/net"
-
-	# return the result
-	echo "{\"path\":\"${path}\"}"
-}
-
-# CLI subcommand: "help"
-cmd_entrypoint_help() {
-	echo "Usage: ${ARG0} COMMAND"
-	echo
-	echo "Rootless CNI Infra container"
-	echo
-	echo "Commands:"
-	echo "  alloc             Allocate a netns"
-	echo "  dealloc           Deallocate a netns"
-	echo "  is-idle           Print whether the infra container is idle"
-	echo "  print-cni-result  Print CNI result"
-	echo "  print-netns-path  Print netns path"
-	echo "  help              Print help"
-	echo "  version           Print version"
-}
-
-# CLI subcommand: "version"
-cmd_entrypoint_version() {
-	echo "{\"version\": \"${ROOTLESS_CNI_INFRA_VERSION}\"}"
-}
-
-# parse args
-command="${1:-}"
-if [ -z "$command" ]; then
-	echo >&2 "No command was specified. Run \`${ARG0} help\` to see the usage."
-	exit 1
-fi
-
-command_func=$(echo "cmd_entrypoint_${command}" | sed -e "s/-/_/g")
-if ! command -v "${command_func}" >/dev/null 2>&1; then
-	echo >&2 "Unknown command: ${command}. Run \`${ARG0} help\` to see the usage."
-	exit 1
-fi
-
-# start the command func
-shift
-"${command_func}" "$@"
diff --git a/libpod/container_internal.go b/libpod/container_internal.go
index 106e2569b..a53027ab2 100644
--- a/libpod/container_internal.go
+++ b/libpod/container_internal.go
@@ -966,9 +966,7 @@ func (c *Container) completeNetworkSetup() error {
 	if err := c.syncContainer(); err != nil {
 		return err
 	}
-	if rootless.IsRootless() {
-		return c.runtime.setupRootlessNetNS(c)
-	} else if c.config.NetMode.IsSlirp4netns() {
+	if c.config.NetMode.IsSlirp4netns() {
 		return c.runtime.setupSlirp4netns(c)
 	}
 	if err := c.runtime.setupNetNS(c); err != nil {
diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go
index a136fb72d..d167bf188 100644
--- a/libpod/container_internal_linux.go
+++ b/libpod/container_internal_linux.go
@@ -92,11 +92,7 @@ func (c *Container) prepare() error {
 		// Set up network namespace if not already set up
 		noNetNS := c.state.NetNS == nil
 		if c.config.CreateNetNS && noNetNS && !c.config.PostConfigureNetNS {
-			if rootless.IsRootless() && len(c.config.Networks) > 0 {
-				netNS, networkStatus, createNetNSErr = AllocRootlessCNI(context.Background(), c)
-			} else {
-				netNS, networkStatus, createNetNSErr = c.runtime.createNetNS(c)
-			}
+			netNS, networkStatus, createNetNSErr = c.runtime.createNetNS(c)
 			if createNetNSErr != nil {
 				return
 			}
diff --git a/libpod/network/create.go b/libpod/network/create.go
index 1a5aa82fc..4fe9b445f 100644
--- a/libpod/network/create.go
+++ b/libpod/network/create.go
@@ -11,7 +11,6 @@ import (
 	"github.com/containernetworking/cni/pkg/version"
 	"github.com/containers/common/pkg/config"
 	"github.com/containers/podman/v3/pkg/domain/entities"
-	"github.com/containers/podman/v3/pkg/rootless"
 	"github.com/containers/podman/v3/pkg/util"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -223,9 +222,8 @@ func createBridge(name string, options entities.NetworkCreateOptions, runtimeCon
 	plugins = append(plugins, NewPortMapPlugin())
 	plugins = append(plugins, NewFirewallPlugin())
 	plugins = append(plugins, NewTuningPlugin())
-	// if we find the dnsname plugin or are rootless, we add configuration for it
-	// the rootless-cni-infra container has the dnsname plugin always installed
-	if (HasDNSNamePlugin(runtimeConfig.Network.CNIPluginDirs) || rootless.IsRootless()) && !options.DisableDNS {
+	// if we find the dnsname plugin we add configuration for it
+	if HasDNSNamePlugin(runtimeConfig.Network.CNIPluginDirs) && !options.DisableDNS {
 		if options.Internal {
 			logrus.Warnf("dnsname and --internal networks are incompatible.  dnsname plugin not configured for network %s", name)
 		} else {
diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go
index 8bf532f66..157c85431 100644
--- a/libpod/networking_linux.go
+++ b/libpod/networking_linux.go
@@ -3,11 +3,8 @@
 package libpod
 
 import (
-	"bytes"
-	"context"
 	"crypto/rand"
 	"fmt"
-	"io"
 	"io/ioutil"
 	"net"
 	"os"
@@ -27,9 +24,12 @@ import (
 	"github.com/containers/podman/v3/libpod/network"
 	"github.com/containers/podman/v3/pkg/errorhandling"
 	"github.com/containers/podman/v3/pkg/netns"
+	"github.com/containers/podman/v3/pkg/resolvconf"
 	"github.com/containers/podman/v3/pkg/rootless"
-	"github.com/containers/podman/v3/pkg/rootlessport"
+	"github.com/containers/podman/v3/pkg/util"
+	"github.com/containers/storage/pkg/lockfile"
 	"github.com/cri-o/ocicni/pkg/ocicni"
+	"github.com/opencontainers/selinux/go-selinux/label"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
 	"github.com/vishvananda/netlink"
@@ -46,6 +46,9 @@ const (
 
 	// slirp4netnsMTU the default MTU override
 	slirp4netnsMTU = 65520
+
+	// rootlessCNINSName is the file name for the rootless network namespace bind mount
+	rootlessCNINSName = "rootless-cni-ns"
 )
 
 // Get an OCICNI network config
@@ -102,6 +105,361 @@ func (r *Runtime) getPodNetwork(id, name, nsPath string, networks []string, port
 	return ctrNetwork
 }
 
+type rootlessCNI struct {
+	ns   ns.NetNS
+	dir  string
+	lock lockfile.Locker
+}
+
+func (r *rootlessCNI) Do(toRun func() error) error {
+	err := r.ns.Do(func(_ ns.NetNS) error {
+		// before we can run the given function
+		// we have to setup all mounts correctly
+
+		// create a new mount namespace
+		// this should happen inside the netns thread
+		err := unix.Unshare(unix.CLONE_NEWNS)
+		if err != nil {
+			return errors.Wrapf(err, "cannot create a new mount namespace")
+		}
+
+		netNsDir, err := netns.GetNSRunDir()
+		if err != nil {
+			return errors.Wrap(err, "could not get network namespace directory")
+		}
+		newNetNsDir := filepath.Join(r.dir, netNsDir)
+		// mount the netns into the new run to keep them accessible
+		// otherwise cni setup will fail because it cannot access the netns files
+		err = unix.Mount(netNsDir, newNetNsDir, "none", unix.MS_BIND|unix.MS_SHARED|unix.MS_REC, "")
+		if err != nil {
+			return errors.Wrap(err, "failed to mount netns directory for rootless cni")
+		}
+
+		// mount resolv.conf to make use of the host dns
+		err = unix.Mount(filepath.Join(r.dir, "resolv.conf"), "/etc/resolv.conf", "none", unix.MS_BIND, "")
+		if err != nil {
+			return errors.Wrap(err, "failed to mount resolv.conf for rootless cni")
+		}
+
+		// also keep /run/systemd if it exists
+		// many files are symlinked into this dir, for example /dev/log
+		runSystemd := "/run/systemd"
+		_, err = os.Stat(runSystemd)
+		if err == nil {
+			newRunSystemd := filepath.Join(r.dir, runSystemd[1:])
+			err = unix.Mount(runSystemd, newRunSystemd, "none", unix.MS_BIND|unix.MS_REC, "")
+			if err != nil {
+				return errors.Wrap(err, "failed to mount /run/systemd directory for rootless cni")
+			}
+		}
+
+		// cni plugins need access to /var and /run
+		runDir := filepath.Join(r.dir, "run")
+		varDir := filepath.Join(r.dir, "var")
+		// make sure to mount var first
+		err = unix.Mount(varDir, "/var", "none", unix.MS_BIND, "")
+		if err != nil {
+			return errors.Wrap(err, "failed to mount /var for rootless cni")
+		}
+		// recursive mount to keep the netns mount
+		err = unix.Mount(runDir, "/run", "none", unix.MS_BIND|unix.MS_REC, "")
+		if err != nil {
+			return errors.Wrap(err, "failed to mount /run for rootless cni")
+		}
+
+		// run the given function in the correct namespace
+		err = toRun()
+		return err
+	})
+	return err
+}
+
+// cleanup the rootless cni namespace if needed
+// check if we have running containers with the bridge network mode
+func (r *rootlessCNI) cleanup(runtime *Runtime) error {
+	r.lock.Lock()
+	defer r.lock.Unlock()
+	running := func(c *Container) bool {
+		// we cannot use c.state() because it will try to lock the container
+		// using c.state.State directly should be good enough for this use case
+		state := c.state.State
+		return state == define.ContainerStateRunning
+	}
+	ctrs, err := runtime.GetContainersWithoutLock(running)
+	if err != nil {
+		return err
+	}
+	cleanup := true
+	for _, ctr := range ctrs {
+		if ctr.config.NetMode.IsBridge() {
+			cleanup = false
+		}
+	}
+	if cleanup {
+		// make sure the the cni results (cache) dir is empty
+		// libpod instances with another root dir are not covered by the check above
+		// this allows several libpod instances to use the same rootless cni ns
+		contents, err := ioutil.ReadDir(filepath.Join(r.dir, "var/lib/cni/results"))
+		if (err == nil && len(contents) == 0) || os.IsNotExist(err) {
+			logrus.Debug("Cleaning up rootless cni namespace")
+			err = netns.UnmountNS(r.ns)
+			if err != nil {
+				return err
+			}
+			// make the following errors not fatal
+			err = r.ns.Close()
+			if err != nil {
+				logrus.Error(err)
+			}
+			b, err := ioutil.ReadFile(filepath.Join(r.dir, "rootless-cni-slirp4netns.pid"))
+			if err == nil {
+				var i int
+				i, err = strconv.Atoi(string(b))
+				if err == nil {
+					// kill the slirp process so we do not leak it
+					err = syscall.Kill(i, syscall.SIGTERM)
+				}
+			}
+			if err != nil {
+				logrus.Errorf("failed to kill slirp4netns process: %s", err)
+			}
+			err = os.RemoveAll(r.dir)
+			if err != nil {
+				logrus.Error(err)
+			}
+		} else if err != nil && !os.IsNotExist(err) {
+			logrus.Errorf("could not read rootless cni directory, skipping cleanup: %s", err)
+		}
+	}
+	return nil
+}
+
+// getRootlessCNINetNs returns the rootless cni object. If create is set to true
+// the rootless cni namespace will be created if it does not exists already.
+func (r *Runtime) getRootlessCNINetNs(new bool) (*rootlessCNI, error) {
+	var rootlessCNINS *rootlessCNI
+	if rootless.IsRootless() {
+		runDir, err := util.GetRuntimeDir()
+		if err != nil {
+			return nil, err
+		}
+		cniDir := filepath.Join(runDir, "rootless-cni")
+		err = os.MkdirAll(cniDir, 0700)
+		if err != nil {
+			return nil, errors.Wrap(err, "could not create rootless-cni directory")
+		}
+
+		lfile := filepath.Join(cniDir, "rootless-cni.lck")
+		lock, err := lockfile.GetLockfile(lfile)
+		if err != nil {
+			return nil, errors.Wrap(err, "failed to get rootless-cni lockfile")
+		}
+		lock.Lock()
+		defer lock.Unlock()
+
+		nsDir, err := netns.GetNSRunDir()
+		if err != nil {
+			return nil, err
+		}
+		path := filepath.Join(nsDir, rootlessCNINSName)
+		ns, err := ns.GetNS(path)
+		if err != nil {
+			if new {
+				// create a new namespace
+				logrus.Debug("creating rootless cni network namespace")
+				ns, err = netns.NewNSWithName(rootlessCNINSName)
+				if err != nil {
+					return nil, errors.Wrap(err, "error creating rootless cni network namespace")
+				}
+
+				// setup slirp4netns here
+				path := r.config.Engine.NetworkCmdPath
+				if path == "" {
+					var err error
+					path, err = exec.LookPath("slirp4netns")
+					if err != nil {
+						return nil, err
+					}
+				}
+
+				syncR, syncW, err := os.Pipe()
+				if err != nil {
+					return nil, errors.Wrapf(err, "failed to open pipe")
+				}
+				defer errorhandling.CloseQuiet(syncR)
+				defer errorhandling.CloseQuiet(syncW)
+
+				netOptions, err := parseSlirp4netnsNetworkOptions(r, nil)
+				if err != nil {
+					return nil, err
+				}
+				slirpFeatures, err := checkSlirpFlags(path)
+				if err != nil {
+					return nil, errors.Wrapf(err, "error checking slirp4netns binary %s: %q", path, err)
+				}
+				cmdArgs, err := createBasicSlirp4netnsCmdArgs(netOptions, slirpFeatures)
+				if err != nil {
+					return nil, err
+				}
+				// Note we do not use --exit-fd, we kill this process by pid
+				cmdArgs = append(cmdArgs, "-c", "-r", "3")
+				cmdArgs = append(cmdArgs, "--netns-type=path", ns.Path(), "tap0")
+
+				cmd := exec.Command(path, cmdArgs...)
+				logrus.Debugf("slirp4netns command: %s", strings.Join(cmd.Args, " "))
+				cmd.SysProcAttr = &syscall.SysProcAttr{
+					Setpgid: true,
+				}
+
+				// workaround for https://github.com/rootless-containers/slirp4netns/pull/153
+				if !netOptions.noPivotRoot && slirpFeatures.HasEnableSandbox {
+					cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNS
+					cmd.SysProcAttr.Unshareflags = syscall.CLONE_NEWNS
+				}
+
+				// Leak one end of the pipe in slirp4netns
+				cmd.ExtraFiles = append(cmd.ExtraFiles, syncW)
+
+				logPath := filepath.Join(r.config.Engine.TmpDir, "slirp4netns-rootless-cni.log")
+				logFile, err := os.Create(logPath)
+				if err != nil {
+					return nil, errors.Wrapf(err, "failed to open slirp4netns log file %s", logPath)
+				}
+				defer logFile.Close()
+				// Unlink immediately the file so we won't need to worry about cleaning it up later.
+				// It is still accessible through the open fd logFile.
+				if err := os.Remove(logPath); err != nil {
+					return nil, errors.Wrapf(err, "delete file %s", logPath)
+				}
+				cmd.Stdout = logFile
+				cmd.Stderr = logFile
+				if err := cmd.Start(); err != nil {
+					return nil, errors.Wrapf(err, "failed to start slirp4netns process")
+				}
+				// create pid file for the slirp4netns process
+				// this is need to kill the process in the cleanup
+				pid := strconv.Itoa(cmd.Process.Pid)
+				err = ioutil.WriteFile(filepath.Join(cniDir, "rootless-cni-slirp4netns.pid"), []byte(pid), 0700)
+				if err != nil {
+					errors.Wrap(err, "unable to write rootless-cni slirp4netns pid file")
+				}
+
+				defer func() {
+					if err := cmd.Process.Release(); err != nil {
+						logrus.Errorf("unable to release command process: %q", err)
+					}
+				}()
+
+				if err := waitForSync(syncR, cmd, logFile, 1*time.Second); err != nil {
+					return nil, err
+				}
+
+				// build a new resolv.conf file which uses the slirp4netns dns server address
+				resolveIP := slirp4netnsDNS
+				if netOptions.cidr != "" {
+					_, cidr, err := net.ParseCIDR(netOptions.cidr)
+					if err != nil {
+						return nil, errors.Wrap(err, "failed to parse slirp4netns cidr")
+					}
+					// the slirp dns ip is always the third ip in the subnet
+					cidr.IP[len(cidr.IP)-1] = cidr.IP[len(cidr.IP)-1] + 3
+					resolveIP = cidr.IP.String()
+				}
+				conf, err := resolvconf.Get()
+				if err != nil {
+					return nil, err
+				}
+				searchDomains := resolvconf.GetSearchDomains(conf.Content)
+				dnsOptions := resolvconf.GetOptions(conf.Content)
+
+				_, err = resolvconf.Build(filepath.Join(cniDir, "resolv.conf"), []string{resolveIP}, searchDomains, dnsOptions)
+				if err != nil {
+					return nil, errors.Wrap(err, "failed to create rootless cni resolv.conf")
+				}
+
+				// create cni directories to store files
+				// they will be bind mounted to the correct location in a extra mount ns
+				err = os.MkdirAll(filepath.Join(cniDir, "var"), 0700)
+				if err != nil {
+					return nil, errors.Wrap(err, "could not create rootless-cni var directory")
+				}
+				runDir := filepath.Join(cniDir, "run")
+				err = os.MkdirAll(runDir, 0700)
+				if err != nil {
+					return nil, errors.Wrap(err, "could not create rootless-cni run directory")
+				}
+				// relabel the new run directory to the iptables /run label
+				// this is important, otherwise the iptables command will fail
+				err = label.Relabel(runDir, "system_u:object_r:iptables_var_run_t:s0", false)
+				if err != nil {
+					return nil, errors.Wrap(err, "could not create relabel rootless-cni run directory")
+				}
+				// create systemd run directory
+				err = os.MkdirAll(filepath.Join(runDir, "systemd"), 0700)
+				if err != nil {
+					return nil, errors.Wrap(err, "could not create rootless-cni systemd directory")
+				}
+				// create the directory for the netns files at the same location
+				// relative to the rootless-cni location
+				err = os.MkdirAll(filepath.Join(cniDir, nsDir), 0700)
+				if err != nil {
+					return nil, errors.Wrap(err, "could not create rootless-cni netns directory")
+				}
+			} else {
+				// return a error if we could not get the namespace and should no create one
+				return nil, errors.Wrap(err, "error getting rootless cni network namespace")
+			}
+		}
+
+		rootlessCNINS = &rootlessCNI{
+			ns:   ns,
+			dir:  cniDir,
+			lock: lock,
+		}
+	}
+	return rootlessCNINS, nil
+}
+
+// setUpOCICNIPod will set up the cni networks, on error it will also tear down the cni
+// networks. If rootless it will join/create the rootless cni namespace.
+func (r *Runtime) setUpOCICNIPod(podNetwork ocicni.PodNetwork) ([]ocicni.NetResult, error) {
+	rootlessCNINS, err := r.getRootlessCNINetNs(true)
+	if err != nil {
+		return nil, err
+	}
+	var results []ocicni.NetResult
+	setUpPod := func() error {
+		results, err = r.netPlugin.SetUpPod(podNetwork)
+		if err != nil {
+			if err2 := r.netPlugin.TearDownPod(podNetwork); err2 != nil {
+				logrus.Errorf("Error tearing down partially created network namespace for container %s: %v", podNetwork.ID, err2)
+			}
+			return errors.Wrapf(err, "error configuring network namespace for container %s", podNetwork.ID)
+		}
+		return nil
+	}
+	// rootlessCNINS is nil if we are root
+	if rootlessCNINS != nil {
+		// execute the cni setup in the rootless net ns
+		err = rootlessCNINS.Do(setUpPod)
+	} else {
+		err = setUpPod()
+	}
+	return results, err
+}
+
+// getCNIPodName return the pod name (hostname) used by CNI and the dnsname plugin.
+// If we are in the pod network namespace use the pod name otherwise the container name
+func getCNIPodName(c *Container) string {
+	if c.config.NetMode.IsPod() || c.IsInfra() {
+		pod, err := c.runtime.GetPod(c.PodID())
+		if err == nil {
+			return pod.Name()
+		}
+	}
+	return c.Name()
+}
+
 // Create and configure a new network namespace for a container
 func (r *Runtime) configureNetNS(ctr *Container, ctrNS ns.NetNS) ([]*cnitypes.Result, error) {
 	var requestedIP net.IP
@@ -147,17 +505,10 @@ func (r *Runtime) configureNetNS(ctr *Container, ctrNS ns.NetNS) ([]*cnitypes.Re
 		podNetwork.Aliases = aliases
 	}
 
-	results, err := r.netPlugin.SetUpPod(podNetwork)
+	results, err := r.setUpOCICNIPod(podNetwork)
 	if err != nil {
-		return nil, errors.Wrapf(err, "error configuring network namespace for container %s", ctr.ID())
+		return nil, err
 	}
-	defer func() {
-		if err != nil {
-			if err2 := r.netPlugin.TearDownPod(podNetwork); err2 != nil {
-				logrus.Errorf("Error tearing down partially created network namespace for container %s: %v", ctr.ID(), err2)
-			}
-		}
-	}()
 
 	networkStatus := make([]*cnitypes.Result, 0)
 	for idx, r := range results {
@@ -192,52 +543,12 @@ func (r *Runtime) createNetNS(ctr *Container) (n ns.NetNS, q []*cnitypes.Result,
 	logrus.Debugf("Made network namespace at %s for container %s", ctrNS.Path(), ctr.ID())
 
 	networkStatus := []*cnitypes.Result{}
-	if !rootless.IsRootless() && !ctr.config.NetMode.IsSlirp4netns() {
+	if !ctr.config.NetMode.IsSlirp4netns() {
 		networkStatus, err = r.configureNetNS(ctr, ctrNS)
 	}
 	return ctrNS, networkStatus, err
 }
 
-type slirpFeatures struct {
-	HasDisableHostLoopback bool
-	HasMTU                 bool
-	HasEnableSandbox       bool
-	HasEnableSeccomp       bool
-	HasCIDR                bool
-	HasOutboundAddr        bool
-	HasIPv6                bool
-}
-
-type slirp4netnsCmdArg struct {
-	Proto     string `json:"proto,omitempty"`
-	HostAddr  string `json:"host_addr"`
-	HostPort  int32  `json:"host_port"`
-	GuestAddr string `json:"guest_addr"`
-	GuestPort int32  `json:"guest_port"`
-}
-
-type slirp4netnsCmd struct {
-	Execute string            `json:"execute"`
-	Args    slirp4netnsCmdArg `json:"arguments"`
-}
-
-func checkSlirpFlags(path string) (*slirpFeatures, error) {
-	cmd := exec.Command(path, "--help")
-	out, err := cmd.CombinedOutput()
-	if err != nil {
-		return nil, errors.Wrapf(err, "slirp4netns %q", out)
-	}
-	return &slirpFeatures{
-		HasDisableHostLoopback: strings.Contains(string(out), "--disable-host-loopback"),
-		HasMTU:                 strings.Contains(string(out), "--mtu"),
-		HasEnableSandbox:       strings.Contains(string(out), "--enable-sandbox"),
-		HasEnableSeccomp:       strings.Contains(string(out), "--enable-seccomp"),
-		HasCIDR:                strings.Contains(string(out), "--cidr"),
-		HasOutboundAddr:        strings.Contains(string(out), "--outbound-addr"),
-		HasIPv6:                strings.Contains(string(out), "--enable-ipv6"),
-	}, nil
-}
-
 // Configure the network namespace for a rootless container
 func (r *Runtime) setupRootlessNetNS(ctr *Container) error {
 	if ctr.config.NetMode.IsSlirp4netns() {
@@ -247,7 +558,7 @@ func (r *Runtime) setupRootlessNetNS(ctr *Container) error {
 	if err != nil {
 		return err
 	}
-	if len(networks) > 0 {
+	if len(networks) > 0 && len(ctr.config.PortMappings) > 0 {
 		// set up port forwarder for CNI-in-slirp4netns
 		netnsPath := ctr.state.NetNS.Path()
 		// TODO: support slirp4netns port forwarder as well
@@ -256,456 +567,6 @@ func (r *Runtime) setupRootlessNetNS(ctr *Container) error {
 	return nil
 }
 
-// setupSlirp4netns can be called in rootful as well as in rootless
-func (r *Runtime) setupSlirp4netns(ctr *Container) error {
-	path := r.config.Engine.NetworkCmdPath
-	slirpOptions := r.config.Engine.NetworkCmdOptions
-	noPivotRoot := r.config.Engine.NoPivotRoot
-	if path == "" {
-		var err error
-		path, err = exec.LookPath("slirp4netns")
-		if err != nil {
-			logrus.Errorf("could not find slirp4netns, the network namespace won't be configured: %v", err)
-			return nil
-		}
-	}
-
-	syncR, syncW, err := os.Pipe()
-	if err != nil {
-		return errors.Wrapf(err, "failed to open pipe")
-	}
-	defer errorhandling.CloseQuiet(syncR)
-	defer errorhandling.CloseQuiet(syncW)
-
-	havePortMapping := len(ctr.Config().PortMappings) > 0
-	logPath := filepath.Join(ctr.runtime.config.Engine.TmpDir, fmt.Sprintf("slirp4netns-%s.log", ctr.config.ID))
-
-	cidr := ""
-	isSlirpHostForward := false
-	disableHostLoopback := true
-	enableIPv6 := false
-	outboundAddr := ""
-	outboundAddr6 := ""
-	mtu := slirp4netnsMTU
-
-	if ctr.config.NetworkOptions != nil {
-		slirpOptions = append(slirpOptions, ctr.config.NetworkOptions["slirp4netns"]...)
-	}
-
-	for _, o := range slirpOptions {
-		parts := strings.SplitN(o, "=", 2)
-		if len(parts) < 2 {
-			return errors.Errorf("unknown option for slirp4netns: %q", o)
-		}
-		option, value := parts[0], parts[1]
-		switch option {
-		case "cidr":
-			ipv4, _, err := net.ParseCIDR(value)
-			if err != nil || ipv4.To4() == nil {
-				return errors.Errorf("invalid cidr %q", value)
-			}
-			cidr = value
-		case "port_handler":
-			switch value {
-			case "slirp4netns":
-				isSlirpHostForward = true
-			case "rootlesskit":
-				isSlirpHostForward = false
-			default:
-				return errors.Errorf("unknown port_handler for slirp4netns: %q", value)
-			}
-		case "allow_host_loopback":
-			switch value {
-			case "true":
-				disableHostLoopback = false
-			case "false":
-				disableHostLoopback = true
-			default:
-				return errors.Errorf("invalid value of allow_host_loopback for slirp4netns: %q", value)
-			}
-		case "enable_ipv6":
-			switch value {
-			case "true":
-				enableIPv6 = true
-			case "false":
-				enableIPv6 = false
-			default:
-				return errors.Errorf("invalid value of enable_ipv6 for slirp4netns: %q", value)
-			}
-		case "outbound_addr":
-			ipv4 := net.ParseIP(value)
-			if ipv4 == nil || ipv4.To4() == nil {
-				_, err := net.InterfaceByName(value)
-				if err != nil {
-					return errors.Errorf("invalid outbound_addr %q", value)
-				}
-			}
-			outboundAddr = value
-		case "outbound_addr6":
-			ipv6 := net.ParseIP(value)
-			if ipv6 == nil || ipv6.To4() != nil {
-				_, err := net.InterfaceByName(value)
-				if err != nil {
-					return errors.Errorf("invalid outbound_addr6: %q", value)
-				}
-			}
-			outboundAddr6 = value
-		case "mtu":
-			mtu, err = strconv.Atoi(value)
-			if mtu < 68 || err != nil {
-				return errors.Errorf("invalid mtu %q", value)
-			}
-		default:
-			return errors.Errorf("unknown option for slirp4netns: %q", o)
-		}
-	}
-
-	cmdArgs := []string{}
-	slirpFeatures, err := checkSlirpFlags(path)
-	if err != nil {
-		return errors.Wrapf(err, "error checking slirp4netns binary %s: %q", path, err)
-	}
-	if disableHostLoopback && slirpFeatures.HasDisableHostLoopback {
-		cmdArgs = append(cmdArgs, "--disable-host-loopback")
-	}
-	if mtu > -1 && slirpFeatures.HasMTU {
-		cmdArgs = append(cmdArgs, fmt.Sprintf("--mtu=%d", mtu))
-	}
-	if !noPivotRoot && slirpFeatures.HasEnableSandbox {
-		cmdArgs = append(cmdArgs, "--enable-sandbox")
-	}
-	if slirpFeatures.HasEnableSeccomp {
-		cmdArgs = append(cmdArgs, "--enable-seccomp")
-	}
-
-	if cidr != "" {
-		if !slirpFeatures.HasCIDR {
-			return errors.Errorf("cidr not supported")
-		}
-		cmdArgs = append(cmdArgs, fmt.Sprintf("--cidr=%s", cidr))
-	}
-
-	if enableIPv6 {
-		if !slirpFeatures.HasIPv6 {
-			return errors.Errorf("enable_ipv6 not supported")
-		}
-		cmdArgs = append(cmdArgs, "--enable-ipv6")
-	}
-
-	if outboundAddr != "" {
-		if !slirpFeatures.HasOutboundAddr {
-			return errors.Errorf("outbound_addr not supported")
-		}
-		cmdArgs = append(cmdArgs, fmt.Sprintf("--outbound-addr=%s", outboundAddr))
-	}
-
-	if outboundAddr6 != "" {
-		if !slirpFeatures.HasOutboundAddr || !slirpFeatures.HasIPv6 {
-			return errors.Errorf("outbound_addr6 not supported")
-		}
-		if !enableIPv6 {
-			return errors.Errorf("enable_ipv6=true is required for outbound_addr6")
-		}
-		cmdArgs = append(cmdArgs, fmt.Sprintf("--outbound-addr6=%s", outboundAddr6))
-	}
-
-	var apiSocket string
-	if havePortMapping && isSlirpHostForward {
-		apiSocket = filepath.Join(ctr.runtime.config.Engine.TmpDir, fmt.Sprintf("%s.net", ctr.config.ID))
-		cmdArgs = append(cmdArgs, "--api-socket", apiSocket)
-	}
-
-	// the slirp4netns arguments being passed are describes as follows:
-	// from the slirp4netns documentation: https://github.com/rootless-containers/slirp4netns
-	// -c, --configure Brings up the tap interface
-	// -e, --exit-fd=FD specify the FD for terminating slirp4netns
-	// -r, --ready-fd=FD specify the FD to write to when the initialization steps are finished
-	cmdArgs = append(cmdArgs, "-c", "-e", "3", "-r", "4")
-	netnsPath := ""
-	if !ctr.config.PostConfigureNetNS {
-		ctr.rootlessSlirpSyncR, ctr.rootlessSlirpSyncW, err = os.Pipe()
-		if err != nil {
-			return errors.Wrapf(err, "failed to create rootless network sync pipe")
-		}
-		netnsPath = ctr.state.NetNS.Path()
-		cmdArgs = append(cmdArgs, "--netns-type=path", netnsPath, "tap0")
-	} else {
-		defer errorhandling.CloseQuiet(ctr.rootlessSlirpSyncR)
-		defer errorhandling.CloseQuiet(ctr.rootlessSlirpSyncW)
-		netnsPath = fmt.Sprintf("/proc/%d/ns/net", ctr.state.PID)
-		// we don't use --netns-path here (unavailable for slirp4netns < v0.4)
-		cmdArgs = append(cmdArgs, fmt.Sprintf("%d", ctr.state.PID), "tap0")
-	}
-
-	cmd := exec.Command(path, cmdArgs...)
-	logrus.Debugf("slirp4netns command: %s", strings.Join(cmd.Args, " "))
-	cmd.SysProcAttr = &syscall.SysProcAttr{
-		Setpgid: true,
-	}
-
-	// workaround for https://github.com/rootless-containers/slirp4netns/pull/153
-	if !noPivotRoot && slirpFeatures.HasEnableSandbox {
-		cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNS
-		cmd.SysProcAttr.Unshareflags = syscall.CLONE_NEWNS
-	}
-
-	// Leak one end of the pipe in slirp4netns, the other will be sent to conmon
-	cmd.ExtraFiles = append(cmd.ExtraFiles, ctr.rootlessSlirpSyncR, syncW)
-
-	logFile, err := os.Create(logPath)
-	if err != nil {
-		return errors.Wrapf(err, "failed to open slirp4netns log file %s", logPath)
-	}
-	defer logFile.Close()
-	// Unlink immediately the file so we won't need to worry about cleaning it up later.
-	// It is still accessible through the open fd logFile.
-	if err := os.Remove(logPath); err != nil {
-		return errors.Wrapf(err, "delete file %s", logPath)
-	}
-	cmd.Stdout = logFile
-	cmd.Stderr = logFile
-	if err := cmd.Start(); err != nil {
-		return errors.Wrapf(err, "failed to start slirp4netns process")
-	}
-	defer func() {
-		if err := cmd.Process.Release(); err != nil {
-			logrus.Errorf("unable to release command process: %q", err)
-		}
-	}()
-
-	if err := waitForSync(syncR, cmd, logFile, 1*time.Second); err != nil {
-		return err
-	}
-
-	if havePortMapping {
-		if isSlirpHostForward {
-			return r.setupRootlessPortMappingViaSlirp(ctr, cmd, apiSocket)
-		}
-		return r.setupRootlessPortMappingViaRLK(ctr, netnsPath)
-	}
-	return nil
-}
-
-func waitForSync(syncR *os.File, cmd *exec.Cmd, logFile io.ReadSeeker, timeout time.Duration) error {
-	prog := filepath.Base(cmd.Path)
-	if len(cmd.Args) > 0 {
-		prog = cmd.Args[0]
-	}
-	b := make([]byte, 16)
-	for {
-		if err := syncR.SetDeadline(time.Now().Add(timeout)); err != nil {
-			return errors.Wrapf(err, "error setting %s pipe timeout", prog)
-		}
-		// FIXME: return err as soon as proc exits, without waiting for timeout
-		if _, err := syncR.Read(b); err == nil {
-			break
-		} else {
-			if os.IsTimeout(err) {
-				// Check if the process is still running.
-				var status syscall.WaitStatus
-				pid, err := syscall.Wait4(cmd.Process.Pid, &status, syscall.WNOHANG, nil)
-				if err != nil {
-					return errors.Wrapf(err, "failed to read %s process status", prog)
-				}
-				if pid != cmd.Process.Pid {
-					continue
-				}
-				if status.Exited() {
-					// Seek at the beginning of the file and read all its content
-					if _, err := logFile.Seek(0, 0); err != nil {
-						logrus.Errorf("could not seek log file: %q", err)
-					}
-					logContent, err := ioutil.ReadAll(logFile)
-					if err != nil {
-						return errors.Wrapf(err, "%s failed", prog)
-					}
-					return errors.Errorf("%s failed: %q", prog, logContent)
-				}
-				if status.Signaled() {
-					return errors.Errorf("%s killed by signal", prog)
-				}
-				continue
-			}
-			return errors.Wrapf(err, "failed to read from %s sync pipe", prog)
-		}
-	}
-	return nil
-}
-
-func (r *Runtime) setupRootlessPortMappingViaRLK(ctr *Container, netnsPath string) error {
-	syncR, syncW, err := os.Pipe()
-	if err != nil {
-		return errors.Wrapf(err, "failed to open pipe")
-	}
-	defer errorhandling.CloseQuiet(syncR)
-	defer errorhandling.CloseQuiet(syncW)
-
-	logPath := filepath.Join(ctr.runtime.config.Engine.TmpDir, fmt.Sprintf("rootlessport-%s.log", ctr.config.ID))
-	logFile, err := os.Create(logPath)
-	if err != nil {
-		return errors.Wrapf(err, "failed to open rootlessport log file %s", logPath)
-	}
-	defer logFile.Close()
-	// Unlink immediately the file so we won't need to worry about cleaning it up later.
-	// It is still accessible through the open fd logFile.
-	if err := os.Remove(logPath); err != nil {
-		return errors.Wrapf(err, "delete file %s", logPath)
-	}
-
-	if !ctr.config.PostConfigureNetNS {
-		ctr.rootlessPortSyncR, ctr.rootlessPortSyncW, err = os.Pipe()
-		if err != nil {
-			return errors.Wrapf(err, "failed to create rootless port sync pipe")
-		}
-	}
-
-	childIP := slirp4netnsIP
-outer:
-	for _, r := range ctr.state.NetworkStatus {
-		for _, i := range r.IPs {
-			ipv4 := i.Address.IP.To4()
-			if ipv4 != nil {
-				childIP = ipv4.String()
-				break outer
-			}
-		}
-	}
-
-	cfg := rootlessport.Config{
-		Mappings:  ctr.config.PortMappings,
-		NetNSPath: netnsPath,
-		ExitFD:    3,
-		ReadyFD:   4,
-		TmpDir:    ctr.runtime.config.Engine.TmpDir,
-		ChildIP:   childIP,
-	}
-	cfgJSON, err := json.Marshal(cfg)
-	if err != nil {
-		return err
-	}
-	cfgR := bytes.NewReader(cfgJSON)
-	var stdout bytes.Buffer
-	cmd := exec.Command(fmt.Sprintf("/proc/%d/exe", os.Getpid()))
-	cmd.Args = []string{rootlessport.ReexecKey}
-	// Leak one end of the pipe in rootlessport process, the other will be sent to conmon
-
-	if ctr.rootlessPortSyncR != nil {
-		defer errorhandling.CloseQuiet(ctr.rootlessPortSyncR)
-	}
-
-	cmd.ExtraFiles = append(cmd.ExtraFiles, ctr.rootlessPortSyncR, syncW)
-	cmd.Stdin = cfgR
-	// stdout is for human-readable error, stderr is for debug log
-	cmd.Stdout = &stdout
-	cmd.Stderr = io.MultiWriter(logFile, &logrusDebugWriter{"rootlessport: "})
-	cmd.SysProcAttr = &syscall.SysProcAttr{
-		Setpgid: true,
-	}
-	if err := cmd.Start(); err != nil {
-		return errors.Wrapf(err, "failed to start rootlessport process")
-	}
-	defer func() {
-		if err := cmd.Process.Release(); err != nil {
-			logrus.Errorf("unable to release rootlessport process: %q", err)
-		}
-	}()
-	if err := waitForSync(syncR, cmd, logFile, 3*time.Second); err != nil {
-		stdoutStr := stdout.String()
-		if stdoutStr != "" {
-			// err contains full debug log and too verbose, so return stdoutStr
-			logrus.Debug(err)
-			return errors.Errorf("rootlessport " + strings.TrimSuffix(stdoutStr, "\n"))
-		}
-		return err
-	}
-	logrus.Debug("rootlessport is ready")
-	return nil
-}
-
-func (r *Runtime) setupRootlessPortMappingViaSlirp(ctr *Container, cmd *exec.Cmd, apiSocket string) (err error) {
-	const pidWaitTimeout = 60 * time.Second
-	chWait := make(chan error)
-	go func() {
-		interval := 25 * time.Millisecond
-		for i := time.Duration(0); i < pidWaitTimeout; i += interval {
-			// Check if the process is still running.
-			var status syscall.WaitStatus
-			pid, err := syscall.Wait4(cmd.Process.Pid, &status, syscall.WNOHANG, nil)
-			if err != nil {
-				break
-			}
-			if pid != cmd.Process.Pid {
-				continue
-			}
-			if status.Exited() || status.Signaled() {
-				chWait <- fmt.Errorf("slirp4netns exited with status %d", status.ExitStatus())
-			}
-			time.Sleep(interval)
-		}
-	}()
-	defer close(chWait)
-
-	// wait that API socket file appears before trying to use it.
-	if _, err := WaitForFile(apiSocket, chWait, pidWaitTimeout); err != nil {
-		return errors.Wrapf(err, "waiting for slirp4nets to create the api socket file %s", apiSocket)
-	}
-
-	// for each port we want to add we need to open a connection to the slirp4netns control socket
-	// and send the add_hostfwd command.
-	for _, i := range ctr.config.PortMappings {
-		conn, err := net.Dial("unix", apiSocket)
-		if err != nil {
-			return errors.Wrapf(err, "cannot open connection to %s", apiSocket)
-		}
-		defer func() {
-			if err := conn.Close(); err != nil {
-				logrus.Errorf("unable to close connection: %q", err)
-			}
-		}()
-		hostIP := i.HostIP
-		if hostIP == "" {
-			hostIP = "0.0.0.0"
-		}
-		apiCmd := slirp4netnsCmd{
-			Execute: "add_hostfwd",
-			Args: slirp4netnsCmdArg{
-				Proto:     i.Protocol,
-				HostAddr:  hostIP,
-				HostPort:  i.HostPort,
-				GuestPort: i.ContainerPort,
-			},
-		}
-		// create the JSON payload and send it.  Mark the end of request shutting down writes
-		// to the socket, as requested by slirp4netns.
-		data, err := json.Marshal(&apiCmd)
-		if err != nil {
-			return errors.Wrapf(err, "cannot marshal JSON for slirp4netns")
-		}
-		if _, err := conn.Write([]byte(fmt.Sprintf("%s\n", data))); err != nil {
-			return errors.Wrapf(err, "cannot write to control socket %s", apiSocket)
-		}
-		if err := conn.(*net.UnixConn).CloseWrite(); err != nil {
-			return errors.Wrapf(err, "cannot shutdown the socket %s", apiSocket)
-		}
-		buf := make([]byte, 2048)
-		readLength, err := conn.Read(buf)
-		if err != nil {
-			return errors.Wrapf(err, "cannot read from control socket %s", apiSocket)
-		}
-		// if there is no 'error' key in the received JSON data, then the operation was
-		// successful.
-		var y map[string]interface{}
-		if err := json.Unmarshal(buf[0:readLength], &y); err != nil {
-			return errors.Wrapf(err, "error parsing error status from slirp4netns")
-		}
-		if e, found := y["error"]; found {
-			return errors.Errorf("error from slirp4netns while setting up port redirection: %v", e)
-		}
-	}
-	logrus.Debug("slirp4netns port-forwarding setup via add_hostfwd is ready")
-	return nil
-}
-
 // Configure the network namespace using the container process
 func (r *Runtime) setupNetNS(ctr *Container) error {
 	nsProcess := fmt.Sprintf("/proc/%d/ns/net", ctr.state.PID)
@@ -715,8 +576,11 @@ func (r *Runtime) setupNetNS(ctr *Container) error {
 	if _, err := rand.Reader.Read(b); err != nil {
 		return errors.Wrapf(err, "failed to generate random netns name")
 	}
-
-	nsPath := fmt.Sprintf("/run/netns/cni-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
+	nsPath, err := netns.GetNSRunDir()
+	if err != nil {
+		return err
+	}
+	nsPath = filepath.Join(nsPath, fmt.Sprintf("cni-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:]))
 
 	if err := os.MkdirAll(filepath.Dir(nsPath), 0711); err != nil {
 		return err
@@ -774,6 +638,31 @@ func (r *Runtime) closeNetNS(ctr *Container) error {
 	return nil
 }
 
+// Tear down a container's CNI network configuration and joins the
+// rootless net ns as rootless user
+func (r *Runtime) teardownOCICNIPod(podNetwork ocicni.PodNetwork) error {
+	rootlessCNINS, err := r.getRootlessCNINetNs(false)
+	if err != nil {
+		return err
+	}
+	tearDownPod := func() error {
+		err := r.netPlugin.TearDownPod(podNetwork)
+		return errors.Wrapf(err, "error tearing down CNI namespace configuration for container %s", podNetwork.ID)
+	}
+
+	// rootlessCNINS is nil if we are root
+	if rootlessCNINS != nil {
+		// execute the cni setup in the rootless net ns
+		err = rootlessCNINS.Do(tearDownPod)
+		if err == nil {
+			err = rootlessCNINS.cleanup(r)
+		}
+	} else {
+		err = tearDownPod()
+	}
+	return err
+}
+
 // Tear down a container's CNI network configuration, but do not tear down the
 // namespace itself.
 func (r *Runtime) teardownCNI(ctr *Container) error {
@@ -789,8 +678,7 @@ func (r *Runtime) teardownCNI(ctr *Container) error {
 		return err
 	}
 
-	// rootless containers do not use the CNI plugin directly
-	if !rootless.IsRootless() && !ctr.config.NetMode.IsSlirp4netns() && len(networks) > 0 {
+	if !ctr.config.NetMode.IsSlirp4netns() && len(networks) > 0 {
 		var requestedIP net.IP
 		if ctr.requestedIP != nil {
 			requestedIP = ctr.requestedIP
@@ -810,10 +698,8 @@ func (r *Runtime) teardownCNI(ctr *Container) error {
 		}
 
 		podNetwork := r.getPodNetwork(ctr.ID(), ctr.Name(), ctr.state.NetNS.Path(), networks, ctr.config.PortMappings, requestedIP, requestedMAC, ctr.state.NetInterfaceDescriptions)
-
-		if err := r.netPlugin.TearDownPod(podNetwork); err != nil {
-			return errors.Wrapf(err, "error tearing down CNI namespace configuration for container %s", ctr.ID())
-		}
+		err = r.teardownOCICNIPod(podNetwork)
+		return err
 	}
 	return nil
 }
@@ -824,18 +710,6 @@ func (r *Runtime) teardownNetNS(ctr *Container) error {
 		return err
 	}
 
-	networks, _, err := ctr.networks()
-	if err != nil {
-		return err
-	}
-
-	// CNI-in-slirp4netns
-	if rootless.IsRootless() && len(networks) != 0 {
-		if err := DeallocRootlessCNI(context.Background(), ctr); err != nil {
-			return errors.Wrapf(err, "error tearing down CNI-in-slirp4netns for container %s", ctr.ID())
-		}
-	}
-
 	// First unmount the namespace
 	if err := netns.UnmountNS(ctr.state.NetNS); err != nil {
 		return errors.Wrapf(err, "error unmounting network namespace for container %s", ctr.ID())
@@ -1177,7 +1051,7 @@ func (c *Container) NetworkDisconnect(nameOrID, netName string, force bool) erro
 	}
 
 	podConfig := c.runtime.getPodNetwork(c.ID(), c.Name(), c.state.NetNS.Path(), []string{netName}, c.config.PortMappings, nil, nil, c.state.NetInterfaceDescriptions)
-	if err := c.runtime.netPlugin.TearDownPod(podConfig); err != nil {
+	if err := c.runtime.teardownOCICNIPod(podConfig); err != nil {
 		return err
 	}
 
@@ -1241,7 +1115,7 @@ func (c *Container) NetworkConnect(nameOrID, netName string, aliases []string) e
 	podConfig := c.runtime.getPodNetwork(c.ID(), c.Name(), c.state.NetNS.Path(), []string{netName}, c.config.PortMappings, nil, nil, c.state.NetInterfaceDescriptions)
 	podConfig.Aliases = make(map[string][]string, 1)
 	podConfig.Aliases[netName] = aliases
-	results, err := c.runtime.netPlugin.SetUpPod(podConfig)
+	results, err := c.runtime.setUpOCICNIPod(podConfig)
 	if err != nil {
 		return err
 	}
@@ -1288,9 +1162,6 @@ func (c *Container) NetworkConnect(nameOrID, netName string, aliases []string) e
 
 // DisconnectContainerFromNetwork removes a container from its CNI network
 func (r *Runtime) DisconnectContainerFromNetwork(nameOrID, netName string, force bool) error {
-	if rootless.IsRootless() {
-		return errors.New("network connect is not enabled for rootless containers")
-	}
 	ctr, err := r.LookupContainer(nameOrID)
 	if err != nil {
 		return err
@@ -1300,9 +1171,6 @@ func (r *Runtime) DisconnectContainerFromNetwork(nameOrID, netName string, force
 
 // ConnectContainerToNetwork connects a container to a CNI network
 func (r *Runtime) ConnectContainerToNetwork(nameOrID, netName string, aliases []string) error {
-	if rootless.IsRootless() {
-		return errors.New("network disconnect is not enabled for rootless containers")
-	}
 	ctr, err := r.LookupContainer(nameOrID)
 	if err != nil {
 		return err
diff --git a/libpod/networking_slirp4netns.go b/libpod/networking_slirp4netns.go
new file mode 100644
index 000000000..72ab3c919
--- /dev/null
+++ b/libpod/networking_slirp4netns.go
@@ -0,0 +1,538 @@
+// +build linux
+
+package libpod
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"syscall"
+	"time"
+
+	"github.com/containers/podman/v3/pkg/errorhandling"
+	"github.com/containers/podman/v3/pkg/rootlessport"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+type slirpFeatures struct {
+	HasDisableHostLoopback bool
+	HasMTU                 bool
+	HasEnableSandbox       bool
+	HasEnableSeccomp       bool
+	HasCIDR                bool
+	HasOutboundAddr        bool
+	HasIPv6                bool
+}
+
+type slirp4netnsCmdArg struct {
+	Proto     string `json:"proto,omitempty"`
+	HostAddr  string `json:"host_addr"`
+	HostPort  int32  `json:"host_port"`
+	GuestAddr string `json:"guest_addr"`
+	GuestPort int32  `json:"guest_port"`
+}
+
+type slirp4netnsCmd struct {
+	Execute string            `json:"execute"`
+	Args    slirp4netnsCmdArg `json:"arguments"`
+}
+
+type slirp4netnsNetworkOptions struct {
+	cidr                string
+	disableHostLoopback bool
+	enableIPv6          bool
+	isSlirpHostForward  bool
+	noPivotRoot         bool
+	mtu                 int
+	outboundAddr        string
+	outboundAddr6       string
+}
+
+func checkSlirpFlags(path string) (*slirpFeatures, error) {
+	cmd := exec.Command(path, "--help")
+	out, err := cmd.CombinedOutput()
+	if err != nil {
+		return nil, errors.Wrapf(err, "slirp4netns %q", out)
+	}
+	return &slirpFeatures{
+		HasDisableHostLoopback: strings.Contains(string(out), "--disable-host-loopback"),
+		HasMTU:                 strings.Contains(string(out), "--mtu"),
+		HasEnableSandbox:       strings.Contains(string(out), "--enable-sandbox"),
+		HasEnableSeccomp:       strings.Contains(string(out), "--enable-seccomp"),
+		HasCIDR:                strings.Contains(string(out), "--cidr"),
+		HasOutboundAddr:        strings.Contains(string(out), "--outbound-addr"),
+		HasIPv6:                strings.Contains(string(out), "--enable-ipv6"),
+	}, nil
+}
+
+func parseSlirp4netnsNetworkOptions(r *Runtime, extraOptions []string) (*slirp4netnsNetworkOptions, error) {
+	slirpOptions := append(r.config.Engine.NetworkCmdOptions, extraOptions...)
+	slirp4netnsOpts := &slirp4netnsNetworkOptions{
+		// overwrite defaults
+		disableHostLoopback: true,
+		mtu:                 slirp4netnsMTU,
+		noPivotRoot:         r.config.Engine.NoPivotRoot,
+	}
+	for _, o := range slirpOptions {
+		parts := strings.SplitN(o, "=", 2)
+		if len(parts) < 2 {
+			return nil, errors.Errorf("unknown option for slirp4netns: %q", o)
+		}
+		option, value := parts[0], parts[1]
+		switch option {
+		case "cidr":
+			ipv4, _, err := net.ParseCIDR(value)
+			if err != nil || ipv4.To4() == nil {
+				return nil, errors.Errorf("invalid cidr %q", value)
+			}
+			slirp4netnsOpts.cidr = value
+		case "port_handler":
+			switch value {
+			case "slirp4netns":
+				slirp4netnsOpts.isSlirpHostForward = true
+			case "rootlesskit":
+				slirp4netnsOpts.isSlirpHostForward = false
+			default:
+				return nil, errors.Errorf("unknown port_handler for slirp4netns: %q", value)
+			}
+		case "allow_host_loopback":
+			switch value {
+			case "true":
+				slirp4netnsOpts.disableHostLoopback = false
+			case "false":
+				slirp4netnsOpts.disableHostLoopback = true
+			default:
+				return nil, errors.Errorf("invalid value of allow_host_loopback for slirp4netns: %q", value)
+			}
+		case "enable_ipv6":
+			switch value {
+			case "true":
+				slirp4netnsOpts.enableIPv6 = true
+			case "false":
+				slirp4netnsOpts.enableIPv6 = false
+			default:
+				return nil, errors.Errorf("invalid value of enable_ipv6 for slirp4netns: %q", value)
+			}
+		case "outbound_addr":
+			ipv4 := net.ParseIP(value)
+			if ipv4 == nil || ipv4.To4() == nil {
+				_, err := net.InterfaceByName(value)
+				if err != nil {
+					return nil, errors.Errorf("invalid outbound_addr %q", value)
+				}
+			}
+			slirp4netnsOpts.outboundAddr = value
+		case "outbound_addr6":
+			ipv6 := net.ParseIP(value)
+			if ipv6 == nil || ipv6.To4() != nil {
+				_, err := net.InterfaceByName(value)
+				if err != nil {
+					return nil, errors.Errorf("invalid outbound_addr6: %q", value)
+				}
+			}
+			slirp4netnsOpts.outboundAddr6 = value
+		case "mtu":
+			var err error
+			slirp4netnsOpts.mtu, err = strconv.Atoi(value)
+			if slirp4netnsOpts.mtu < 68 || err != nil {
+				return nil, errors.Errorf("invalid mtu %q", value)
+			}
+		default:
+			return nil, errors.Errorf("unknown option for slirp4netns: %q", o)
+		}
+	}
+	return slirp4netnsOpts, nil
+}
+
+func createBasicSlirp4netnsCmdArgs(options *slirp4netnsNetworkOptions, features *slirpFeatures) ([]string, error) {
+	cmdArgs := []string{}
+	if options.disableHostLoopback && features.HasDisableHostLoopback {
+		cmdArgs = append(cmdArgs, "--disable-host-loopback")
+	}
+	if options.mtu > -1 && features.HasMTU {
+		cmdArgs = append(cmdArgs, fmt.Sprintf("--mtu=%d", options.mtu))
+	}
+	if !options.noPivotRoot && features.HasEnableSandbox {
+		cmdArgs = append(cmdArgs, "--enable-sandbox")
+	}
+	if features.HasEnableSeccomp {
+		cmdArgs = append(cmdArgs, "--enable-seccomp")
+	}
+
+	if options.cidr != "" {
+		if !features.HasCIDR {
+			return nil, errors.Errorf("cidr not supported")
+		}
+		cmdArgs = append(cmdArgs, fmt.Sprintf("--cidr=%s", options.cidr))
+	}
+
+	if options.enableIPv6 {
+		if !features.HasIPv6 {
+			return nil, errors.Errorf("enable_ipv6 not supported")
+		}
+		cmdArgs = append(cmdArgs, "--enable-ipv6")
+	}
+
+	if options.outboundAddr != "" {
+		if !features.HasOutboundAddr {
+			return nil, errors.Errorf("outbound_addr not supported")
+		}
+		cmdArgs = append(cmdArgs, fmt.Sprintf("--outbound-addr=%s", options.outboundAddr))
+	}
+
+	if options.outboundAddr6 != "" {
+		if !features.HasOutboundAddr || !features.HasIPv6 {
+			return nil, errors.Errorf("outbound_addr6 not supported")
+		}
+		if !options.enableIPv6 {
+			return nil, errors.Errorf("enable_ipv6=true is required for outbound_addr6")
+		}
+		cmdArgs = append(cmdArgs, fmt.Sprintf("--outbound-addr6=%s", options.outboundAddr6))
+	}
+
+	return cmdArgs, nil
+}
+
+// setupSlirp4netns can be called in rootful as well as in rootless
+func (r *Runtime) setupSlirp4netns(ctr *Container) error {
+	path := r.config.Engine.NetworkCmdPath
+	if path == "" {
+		var err error
+		path, err = exec.LookPath("slirp4netns")
+		if err != nil {
+			logrus.Errorf("could not find slirp4netns, the network namespace won't be configured: %v", err)
+			return nil
+		}
+	}
+
+	syncR, syncW, err := os.Pipe()
+	if err != nil {
+		return errors.Wrapf(err, "failed to open pipe")
+	}
+	defer errorhandling.CloseQuiet(syncR)
+	defer errorhandling.CloseQuiet(syncW)
+
+	havePortMapping := len(ctr.Config().PortMappings) > 0
+	logPath := filepath.Join(ctr.runtime.config.Engine.TmpDir, fmt.Sprintf("slirp4netns-%s.log", ctr.config.ID))
+
+	ctrNetworkSlipOpts := []string{}
+	if ctr.config.NetworkOptions != nil {
+		ctrNetworkSlipOpts = append(ctrNetworkSlipOpts, ctr.config.NetworkOptions["slirp4netns"]...)
+	}
+	netOptions, err := parseSlirp4netnsNetworkOptions(r, ctrNetworkSlipOpts)
+	if err != nil {
+		return err
+	}
+	slirpFeatures, err := checkSlirpFlags(path)
+	if err != nil {
+		return errors.Wrapf(err, "error checking slirp4netns binary %s: %q", path, err)
+	}
+	cmdArgs, err := createBasicSlirp4netnsCmdArgs(netOptions, slirpFeatures)
+	if err != nil {
+		return err
+	}
+
+	// the slirp4netns arguments being passed are describes as follows:
+	// from the slirp4netns documentation: https://github.com/rootless-containers/slirp4netns
+	// -c, --configure Brings up the tap interface
+	// -e, --exit-fd=FD specify the FD for terminating slirp4netns
+	// -r, --ready-fd=FD specify the FD to write to when the initialization steps are finished
+	cmdArgs = append(cmdArgs, "-c", "-e", "3", "-r", "4")
+
+	var apiSocket string
+	if havePortMapping && netOptions.isSlirpHostForward {
+		apiSocket = filepath.Join(ctr.runtime.config.Engine.TmpDir, fmt.Sprintf("%s.net", ctr.config.ID))
+		cmdArgs = append(cmdArgs, "--api-socket", apiSocket)
+	}
+	netnsPath := ""
+	if !ctr.config.PostConfigureNetNS {
+		ctr.rootlessSlirpSyncR, ctr.rootlessSlirpSyncW, err = os.Pipe()
+		if err != nil {
+			return errors.Wrapf(err, "failed to create rootless network sync pipe")
+		}
+		netnsPath = ctr.state.NetNS.Path()
+		cmdArgs = append(cmdArgs, "--netns-type=path", netnsPath, "tap0")
+	} else {
+		defer errorhandling.CloseQuiet(ctr.rootlessSlirpSyncR)
+		defer errorhandling.CloseQuiet(ctr.rootlessSlirpSyncW)
+		netnsPath = fmt.Sprintf("/proc/%d/ns/net", ctr.state.PID)
+		// we don't use --netns-path here (unavailable for slirp4netns < v0.4)
+		cmdArgs = append(cmdArgs, fmt.Sprintf("%d", ctr.state.PID), "tap0")
+	}
+
+	cmd := exec.Command(path, cmdArgs...)
+	logrus.Debugf("slirp4netns command: %s", strings.Join(cmd.Args, " "))
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+
+	// workaround for https://github.com/rootless-containers/slirp4netns/pull/153
+	if !netOptions.noPivotRoot && slirpFeatures.HasEnableSandbox {
+		cmd.SysProcAttr.Cloneflags = syscall.CLONE_NEWNS
+		cmd.SysProcAttr.Unshareflags = syscall.CLONE_NEWNS
+	}
+
+	// Leak one end of the pipe in slirp4netns, the other will be sent to conmon
+	cmd.ExtraFiles = append(cmd.ExtraFiles, ctr.rootlessSlirpSyncR, syncW)
+
+	logFile, err := os.Create(logPath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open slirp4netns log file %s", logPath)
+	}
+	defer logFile.Close()
+	// Unlink immediately the file so we won't need to worry about cleaning it up later.
+	// It is still accessible through the open fd logFile.
+	if err := os.Remove(logPath); err != nil {
+		return errors.Wrapf(err, "delete file %s", logPath)
+	}
+	cmd.Stdout = logFile
+	cmd.Stderr = logFile
+	if err := cmd.Start(); err != nil {
+		return errors.Wrapf(err, "failed to start slirp4netns process")
+	}
+	defer func() {
+		if err := cmd.Process.Release(); err != nil {
+			logrus.Errorf("unable to release command process: %q", err)
+		}
+	}()
+
+	if err := waitForSync(syncR, cmd, logFile, 1*time.Second); err != nil {
+		return err
+	}
+
+	if havePortMapping {
+		if netOptions.isSlirpHostForward {
+			return r.setupRootlessPortMappingViaSlirp(ctr, cmd, apiSocket)
+		}
+		return r.setupRootlessPortMappingViaRLK(ctr, netnsPath)
+	}
+	return nil
+}
+
+func waitForSync(syncR *os.File, cmd *exec.Cmd, logFile io.ReadSeeker, timeout time.Duration) error {
+	prog := filepath.Base(cmd.Path)
+	if len(cmd.Args) > 0 {
+		prog = cmd.Args[0]
+	}
+	b := make([]byte, 16)
+	for {
+		if err := syncR.SetDeadline(time.Now().Add(timeout)); err != nil {
+			return errors.Wrapf(err, "error setting %s pipe timeout", prog)
+		}
+		// FIXME: return err as soon as proc exits, without waiting for timeout
+		if _, err := syncR.Read(b); err == nil {
+			break
+		} else {
+			if os.IsTimeout(err) {
+				// Check if the process is still running.
+				var status syscall.WaitStatus
+				pid, err := syscall.Wait4(cmd.Process.Pid, &status, syscall.WNOHANG, nil)
+				if err != nil {
+					return errors.Wrapf(err, "failed to read %s process status", prog)
+				}
+				if pid != cmd.Process.Pid {
+					continue
+				}
+				if status.Exited() {
+					// Seek at the beginning of the file and read all its content
+					if _, err := logFile.Seek(0, 0); err != nil {
+						logrus.Errorf("could not seek log file: %q", err)
+					}
+					logContent, err := ioutil.ReadAll(logFile)
+					if err != nil {
+						return errors.Wrapf(err, "%s failed", prog)
+					}
+					return errors.Errorf("%s failed: %q", prog, logContent)
+				}
+				if status.Signaled() {
+					return errors.Errorf("%s killed by signal", prog)
+				}
+				continue
+			}
+			return errors.Wrapf(err, "failed to read from %s sync pipe", prog)
+		}
+	}
+	return nil
+}
+
+func (r *Runtime) setupRootlessPortMappingViaRLK(ctr *Container, netnsPath string) error {
+	syncR, syncW, err := os.Pipe()
+	if err != nil {
+		return errors.Wrapf(err, "failed to open pipe")
+	}
+	defer errorhandling.CloseQuiet(syncR)
+	defer errorhandling.CloseQuiet(syncW)
+
+	logPath := filepath.Join(ctr.runtime.config.Engine.TmpDir, fmt.Sprintf("rootlessport-%s.log", ctr.config.ID))
+	logFile, err := os.Create(logPath)
+	if err != nil {
+		return errors.Wrapf(err, "failed to open rootlessport log file %s", logPath)
+	}
+	defer logFile.Close()
+	// Unlink immediately the file so we won't need to worry about cleaning it up later.
+	// It is still accessible through the open fd logFile.
+	if err := os.Remove(logPath); err != nil {
+		return errors.Wrapf(err, "delete file %s", logPath)
+	}
+
+	if !ctr.config.PostConfigureNetNS {
+		ctr.rootlessPortSyncR, ctr.rootlessPortSyncW, err = os.Pipe()
+		if err != nil {
+			return errors.Wrapf(err, "failed to create rootless port sync pipe")
+		}
+	}
+
+	childIP := slirp4netnsIP
+outer:
+	for _, r := range ctr.state.NetworkStatus {
+		for _, i := range r.IPs {
+			ipv4 := i.Address.IP.To4()
+			if ipv4 != nil {
+				childIP = ipv4.String()
+				break outer
+			}
+		}
+	}
+
+	cfg := rootlessport.Config{
+		Mappings:  ctr.config.PortMappings,
+		NetNSPath: netnsPath,
+		ExitFD:    3,
+		ReadyFD:   4,
+		TmpDir:    ctr.runtime.config.Engine.TmpDir,
+		ChildIP:   childIP,
+	}
+	cfgJSON, err := json.Marshal(cfg)
+	if err != nil {
+		return err
+	}
+	cfgR := bytes.NewReader(cfgJSON)
+	var stdout bytes.Buffer
+	cmd := exec.Command(fmt.Sprintf("/proc/%d/exe", os.Getpid()))
+	cmd.Args = []string{rootlessport.ReexecKey}
+	// Leak one end of the pipe in rootlessport process, the other will be sent to conmon
+
+	if ctr.rootlessPortSyncR != nil {
+		defer errorhandling.CloseQuiet(ctr.rootlessPortSyncR)
+	}
+
+	cmd.ExtraFiles = append(cmd.ExtraFiles, ctr.rootlessPortSyncR, syncW)
+	cmd.Stdin = cfgR
+	// stdout is for human-readable error, stderr is for debug log
+	cmd.Stdout = &stdout
+	cmd.Stderr = io.MultiWriter(logFile, &logrusDebugWriter{"rootlessport: "})
+	cmd.SysProcAttr = &syscall.SysProcAttr{
+		Setpgid: true,
+	}
+	if err := cmd.Start(); err != nil {
+		return errors.Wrapf(err, "failed to start rootlessport process")
+	}
+	defer func() {
+		if err := cmd.Process.Release(); err != nil {
+			logrus.Errorf("unable to release rootlessport process: %q", err)
+		}
+	}()
+	if err := waitForSync(syncR, cmd, logFile, 3*time.Second); err != nil {
+		stdoutStr := stdout.String()
+		if stdoutStr != "" {
+			// err contains full debug log and too verbose, so return stdoutStr
+			logrus.Debug(err)
+			return errors.Errorf("rootlessport " + strings.TrimSuffix(stdoutStr, "\n"))
+		}
+		return err
+	}
+	logrus.Debug("rootlessport is ready")
+	return nil
+}
+
+func (r *Runtime) setupRootlessPortMappingViaSlirp(ctr *Container, cmd *exec.Cmd, apiSocket string) (err error) {
+	const pidWaitTimeout = 60 * time.Second
+	chWait := make(chan error)
+	go func() {
+		interval := 25 * time.Millisecond
+		for i := time.Duration(0); i < pidWaitTimeout; i += interval {
+			// Check if the process is still running.
+			var status syscall.WaitStatus
+			pid, err := syscall.Wait4(cmd.Process.Pid, &status, syscall.WNOHANG, nil)
+			if err != nil {
+				break
+			}
+			if pid != cmd.Process.Pid {
+				continue
+			}
+			if status.Exited() || status.Signaled() {
+				chWait <- fmt.Errorf("slirp4netns exited with status %d", status.ExitStatus())
+			}
+			time.Sleep(interval)
+		}
+	}()
+	defer close(chWait)
+
+	// wait that API socket file appears before trying to use it.
+	if _, err := WaitForFile(apiSocket, chWait, pidWaitTimeout); err != nil {
+		return errors.Wrapf(err, "waiting for slirp4nets to create the api socket file %s", apiSocket)
+	}
+
+	// for each port we want to add we need to open a connection to the slirp4netns control socket
+	// and send the add_hostfwd command.
+	for _, i := range ctr.config.PortMappings {
+		conn, err := net.Dial("unix", apiSocket)
+		if err != nil {
+			return errors.Wrapf(err, "cannot open connection to %s", apiSocket)
+		}
+		defer func() {
+			if err := conn.Close(); err != nil {
+				logrus.Errorf("unable to close connection: %q", err)
+			}
+		}()
+		hostIP := i.HostIP
+		if hostIP == "" {
+			hostIP = "0.0.0.0"
+		}
+		apiCmd := slirp4netnsCmd{
+			Execute: "add_hostfwd",
+			Args: slirp4netnsCmdArg{
+				Proto:     i.Protocol,
+				HostAddr:  hostIP,
+				HostPort:  i.HostPort,
+				GuestPort: i.ContainerPort,
+			},
+		}
+		// create the JSON payload and send it.  Mark the end of request shutting down writes
+		// to the socket, as requested by slirp4netns.
+		data, err := json.Marshal(&apiCmd)
+		if err != nil {
+			return errors.Wrapf(err, "cannot marshal JSON for slirp4netns")
+		}
+		if _, err := conn.Write([]byte(fmt.Sprintf("%s\n", data))); err != nil {
+			return errors.Wrapf(err, "cannot write to control socket %s", apiSocket)
+		}
+		if err := conn.(*net.UnixConn).CloseWrite(); err != nil {
+			return errors.Wrapf(err, "cannot shutdown the socket %s", apiSocket)
+		}
+		buf := make([]byte, 2048)
+		readLength, err := conn.Read(buf)
+		if err != nil {
+			return errors.Wrapf(err, "cannot read from control socket %s", apiSocket)
+		}
+		// if there is no 'error' key in the received JSON data, then the operation was
+		// successful.
+		var y map[string]interface{}
+		if err := json.Unmarshal(buf[0:readLength], &y); err != nil {
+			return errors.Wrapf(err, "error parsing error status from slirp4netns")
+		}
+		if e, found := y["error"]; found {
+			return errors.Errorf("error from slirp4netns while setting up port redirection: %v", e)
+		}
+	}
+	logrus.Debug("slirp4netns port-forwarding setup via add_hostfwd is ready")
+	return nil
+}
diff --git a/libpod/rootless_cni_linux.go b/libpod/rootless_cni_linux.go
deleted file mode 100644
index df690e914..000000000
--- a/libpod/rootless_cni_linux.go
+++ /dev/null
@@ -1,372 +0,0 @@
-// +build linux
-
-package libpod
-
-import (
-	"bytes"
-	"context"
-	"io"
-	"path/filepath"
-	"runtime"
-
-	cnitypes "github.com/containernetworking/cni/pkg/types/current"
-	"github.com/containernetworking/plugins/pkg/ns"
-	"github.com/containers/podman/v3/libpod/define"
-	"github.com/containers/podman/v3/libpod/image"
-	"github.com/containers/podman/v3/pkg/env"
-	"github.com/containers/podman/v3/pkg/util"
-	"github.com/containers/storage/pkg/lockfile"
-	"github.com/hashicorp/go-multierror"
-	spec "github.com/opencontainers/runtime-spec/specs-go"
-	"github.com/opencontainers/runtime-tools/generate"
-	"github.com/pkg/errors"
-	"github.com/sirupsen/logrus"
-)
-
-// Built from ../contrib/rootless-cni-infra.
-var rootlessCNIInfraImage = map[string]string{
-	"amd64": "quay.io/libpod/rootless-cni-infra@sha256:adf352454666f7ce9ca3e1098448b5ee18f89c4516471ec99447ec9ece917f36", // 5-amd64
-}
-
-const (
-	rootlessCNIInfraContainerNamespace = "podman-system"
-	rootlessCNIInfraContainerName      = "rootless-cni-infra"
-)
-
-// AllocRootlessCNI allocates a CNI netns inside the rootless CNI infra container.
-// Locks "rootless-cni-infra.lck".
-//
-// When the infra container is not running, it is created.
-//
-// AllocRootlessCNI does not lock c. c should be already locked.
-func AllocRootlessCNI(ctx context.Context, c *Container) (ns.NetNS, []*cnitypes.Result, error) {
-	networks, _, err := c.networks()
-	if err != nil {
-		return nil, nil, err
-	}
-	if len(networks) == 0 {
-		return nil, nil, errors.New("rootless CNI networking requires that the container has joined at least one CNI network")
-	}
-	l, err := getRootlessCNIInfraLock(c.runtime)
-	if err != nil {
-		return nil, nil, err
-	}
-	l.Lock()
-	defer l.Unlock()
-	infra, err := ensureRootlessCNIInfraContainerRunning(ctx, c.runtime)
-	if err != nil {
-		return nil, nil, err
-	}
-	k8sPodName := getCNIPodName(c) // passed to CNI as K8S_POD_NAME
-	ip := ""
-	if c.config.StaticIP != nil {
-		ip = c.config.StaticIP.String()
-	}
-	mac := ""
-	if c.config.StaticMAC != nil {
-		mac = c.config.StaticMAC.String()
-	}
-	aliases, err := c.runtime.state.GetAllNetworkAliases(c)
-	if err != nil {
-		return nil, nil, err
-	}
-	capArgs := ""
-	// add network aliases json encoded as capabilityArgs for cni
-	if len(aliases) > 0 {
-		capabilityArgs := make(map[string]interface{})
-		capabilityArgs["aliases"] = aliases
-		b, err := json.Marshal(capabilityArgs)
-		if err != nil {
-			return nil, nil, err
-		}
-		capArgs = string(b)
-	}
-
-	cniResults := make([]*cnitypes.Result, len(networks))
-	for i, nw := range networks {
-		cniRes, err := rootlessCNIInfraCallAlloc(infra, c.ID(), nw, k8sPodName, ip, mac, capArgs)
-		if err != nil {
-			return nil, nil, err
-		}
-		cniResults[i] = cniRes
-	}
-	nsObj, err := rootlessCNIInfraGetNS(infra, c.ID())
-	if err != nil {
-		return nil, nil, err
-	}
-	logrus.Debugf("rootless CNI: container %q will join %q", c.ID(), nsObj.Path())
-	return nsObj, cniResults, nil
-}
-
-// DeallocRootlessCNI deallocates a CNI netns inside the rootless CNI infra container.
-// Locks "rootless-cni-infra.lck".
-//
-// When the infra container is no longer needed, it is removed.
-//
-// DeallocRootlessCNI does not lock c. c should be already locked.
-func DeallocRootlessCNI(ctx context.Context, c *Container) error {
-	networks, _, err := c.networks()
-	if err != nil {
-		return err
-	}
-	if len(networks) == 0 {
-		return errors.New("rootless CNI networking requires that the container has joined at least one CNI network")
-	}
-	l, err := getRootlessCNIInfraLock(c.runtime)
-	if err != nil {
-		return err
-	}
-	l.Lock()
-	defer l.Unlock()
-	infra, _ := getRootlessCNIInfraContainer(c.runtime)
-	if infra == nil {
-		return nil
-	}
-	var errs *multierror.Error
-	for _, nw := range networks {
-		err := rootlessCNIInfraCallDealloc(infra, c.ID(), nw)
-		if err != nil {
-			errs = multierror.Append(errs, err)
-		}
-	}
-	if isIdle, err := rootlessCNIInfraIsIdle(infra); isIdle || err != nil {
-		if err != nil {
-			logrus.Warn(err)
-		}
-		logrus.Debugf("rootless CNI: removing infra container %q", infra.ID())
-		infra.lock.Lock()
-		defer infra.lock.Unlock()
-		if err := c.runtime.removeContainer(ctx, infra, true, false, true); err != nil {
-			return err
-		}
-		logrus.Debugf("rootless CNI: removed infra container %q", infra.ID())
-	}
-	return errs.ErrorOrNil()
-}
-
-func getRootlessCNIInfraLock(r *Runtime) (lockfile.Locker, error) {
-	fname := filepath.Join(r.config.Engine.TmpDir, "rootless-cni-infra.lck")
-	return lockfile.GetLockfile(fname)
-}
-
-// getCNIPodName return the pod name (hostname) used by CNI and the dnsname plugin.
-// If we are in the pod network namespace use the pod name otherwise the container name
-func getCNIPodName(c *Container) string {
-	if c.config.NetMode.IsPod() || c.IsInfra() {
-		pod, err := c.runtime.GetPod(c.PodID())
-		if err == nil {
-			return pod.Name()
-		}
-	}
-	return c.Name()
-}
-
-func rootlessCNIInfraCallAlloc(infra *Container, id, nw, k8sPodName, ip, mac, capArgs string) (*cnitypes.Result, error) {
-	logrus.Debugf("rootless CNI: alloc %q, %q, %q, %q, %q, %q", id, nw, k8sPodName, ip, mac, capArgs)
-	var err error
-
-	_, err = rootlessCNIInfraExec(infra, "alloc", id, nw, k8sPodName, ip, mac, capArgs)
-	if err != nil {
-		return nil, err
-	}
-	cniResStr, err := rootlessCNIInfraExec(infra, "print-cni-result", id, nw)
-	if err != nil {
-		return nil, err
-	}
-	var cniRes cnitypes.Result
-	if err := json.Unmarshal([]byte(cniResStr), &cniRes); err != nil {
-		return nil, errors.Wrapf(err, "unmarshaling as cnitypes.Result: %q", cniResStr)
-	}
-	return &cniRes, nil
-}
-
-func rootlessCNIInfraCallDealloc(infra *Container, id, nw string) error {
-	logrus.Debugf("rootless CNI: dealloc %q, %q", id, nw)
-	_, err := rootlessCNIInfraExec(infra, "dealloc", id, nw)
-	return err
-}
-
-func rootlessCNIInfraIsIdle(infra *Container) (bool, error) {
-	type isIdle struct {
-		Idle bool `json:"idle"`
-	}
-	resStr, err := rootlessCNIInfraExec(infra, "is-idle")
-	if err != nil {
-		return false, err
-	}
-	var res isIdle
-	if err := json.Unmarshal([]byte(resStr), &res); err != nil {
-		return false, errors.Wrapf(err, "unmarshaling as isIdle: %q", resStr)
-	}
-	return res.Idle, nil
-}
-
-func rootlessCNIInfraGetNS(infra *Container, id string) (ns.NetNS, error) {
-	type printNetnsPath struct {
-		Path string `json:"path"`
-	}
-	resStr, err := rootlessCNIInfraExec(infra, "print-netns-path", id)
-	if err != nil {
-		return nil, err
-	}
-	var res printNetnsPath
-	if err := json.Unmarshal([]byte(resStr), &res); err != nil {
-		return nil, errors.Wrapf(err, "unmarshaling as printNetnsPath: %q", resStr)
-	}
-	nsObj, err := ns.GetNS(res.Path)
-	if err != nil {
-		return nil, err
-	}
-	return nsObj, nil
-}
-
-func getRootlessCNIInfraContainer(r *Runtime) (*Container, error) {
-	containers, err := r.GetContainersWithoutLock(func(c *Container) bool {
-		return c.Namespace() == rootlessCNIInfraContainerNamespace &&
-			c.Name() == rootlessCNIInfraContainerName
-	})
-	if err != nil {
-		return nil, err
-	}
-	if len(containers) == 0 {
-		return nil, nil
-	}
-	return containers[0], nil
-}
-
-func ensureRootlessCNIInfraContainerRunning(ctx context.Context, r *Runtime) (*Container, error) {
-	c, err := getRootlessCNIInfraContainer(r)
-	if err != nil {
-		return nil, err
-	}
-	if c == nil {
-		return startRootlessCNIInfraContainer(ctx, r)
-	}
-	st, err := c.ContainerState()
-	if err != nil {
-		return nil, err
-	}
-	if st.State == define.ContainerStateRunning {
-		logrus.Debugf("rootless CNI: infra container %q is already running", c.ID())
-		return c, nil
-	}
-	logrus.Debugf("rootless CNI: infra container %q is %q, being started", c.ID(), st.State)
-	if err := c.initAndStart(ctx); err != nil {
-		return nil, err
-	}
-	logrus.Debugf("rootless CNI: infra container %q is running", c.ID())
-	return c, nil
-}
-
-func startRootlessCNIInfraContainer(ctx context.Context, r *Runtime) (*Container, error) {
-	imageName, ok := rootlessCNIInfraImage[runtime.GOARCH]
-	if !ok {
-		return nil, errors.Errorf("cannot find rootless-podman-network-sandbox image for %s", runtime.GOARCH)
-	}
-	logrus.Debugf("rootless CNI: ensuring image %q to exist", imageName)
-	newImage, err := r.ImageRuntime().New(ctx, imageName, "", "", nil, nil,
-		image.SigningOptions{}, nil, util.PullImageMissing, nil)
-	if err != nil {
-		return nil, err
-	}
-	logrus.Debugf("rootless CNI: image %q is ready", imageName)
-
-	g, err := generate.New("linux")
-	if err != nil {
-		return nil, err
-	}
-	g.SetupPrivileged(true)
-	// Set --pid=host for ease of propagating "/proc/PID/ns/net" string
-	if err := g.RemoveLinuxNamespace(string(spec.PIDNamespace)); err != nil {
-		return nil, err
-	}
-	g.RemoveMount("/proc")
-	procMount := spec.Mount{
-		Destination: "/proc",
-		Type:        "bind",
-		Source:      "/proc",
-		Options:     []string{"rbind", "nosuid", "noexec", "nodev"},
-	}
-	g.AddMount(procMount)
-	// Mount CNI networks
-	etcCNINetD := spec.Mount{
-		Destination: "/etc/cni/net.d",
-		Type:        "bind",
-		Source:      r.config.Network.NetworkConfigDir,
-		Options:     []string{"ro", "bind"},
-	}
-	g.AddMount(etcCNINetD)
-
-	inspectData, err := newImage.Inspect(ctx)
-	if err != nil {
-		return nil, err
-	}
-	imageEnv, err := env.ParseSlice(inspectData.Config.Env)
-	if err != nil {
-		return nil, err
-	}
-	for k, v := range imageEnv {
-		g.AddProcessEnv(k, v)
-	}
-	if len(inspectData.Config.Cmd) == 0 {
-		return nil, errors.Errorf("rootless CNI infra image %q has no command specified", imageName)
-	}
-	g.SetProcessArgs(inspectData.Config.Cmd)
-
-	var options []CtrCreateOption
-	options = append(options, WithRootFSFromImage(newImage.ID(), imageName, imageName))
-	options = append(options, WithCtrNamespace(rootlessCNIInfraContainerNamespace))
-	options = append(options, WithName(rootlessCNIInfraContainerName))
-	options = append(options, WithPrivileged(true))
-	options = append(options, WithSecLabels([]string{"disable"}))
-	options = append(options, WithRestartPolicy("always"))
-	options = append(options, WithNetNS(nil, false, "slirp4netns", nil))
-	c, err := r.NewContainer(ctx, g.Config, options...)
-	if err != nil {
-		return nil, err
-	}
-	logrus.Debugf("rootless CNI infra container %q is created, now being started", c.ID())
-	if err := c.initAndStart(ctx); err != nil {
-		return nil, err
-	}
-	logrus.Debugf("rootless CNI: infra container %q is running", c.ID())
-
-	return c, nil
-}
-
-func rootlessCNIInfraExec(c *Container, args ...string) (string, error) {
-	cmd := "rootless-cni-infra"
-	var (
-		outB    bytes.Buffer
-		errB    bytes.Buffer
-		streams define.AttachStreams
-		config  ExecConfig
-	)
-	streams.OutputStream = &nopWriteCloser{Writer: &outB}
-	streams.ErrorStream = &nopWriteCloser{Writer: &errB}
-	streams.AttachOutput = true
-	streams.AttachError = true
-	config.Command = append([]string{cmd}, args...)
-	config.Privileged = true
-	logrus.Debugf("rootlessCNIInfraExec: c.ID()=%s, config=%+v, streams=%v, begin",
-		c.ID(), config, streams)
-	code, err := c.Exec(&config, &streams, nil)
-	logrus.Debugf("rootlessCNIInfraExec: c.ID()=%s, config=%+v, streams=%v, end (code=%d, err=%v)",
-		c.ID(), config, streams, code, err)
-	if err != nil {
-		return "", err
-	}
-	if code != 0 {
-		return "", errors.Errorf("command %s %v in container %s failed with status %d, stdout=%q, stderr=%q",
-			cmd, args, c.ID(), code, outB.String(), errB.String())
-	}
-	return outB.String(), nil
-}
-
-type nopWriteCloser struct {
-	io.Writer
-}
-
-func (nwc *nopWriteCloser) Close() error {
-	return nil
-}
diff --git a/libpod/runtime.go b/libpod/runtime.go
index 201482c65..d4bb691ef 100644
--- a/libpod/runtime.go
+++ b/libpod/runtime.go
@@ -436,13 +436,11 @@ func makeRuntime(ctx context.Context, runtime *Runtime) (retErr error) {
 	}
 
 	// Set up the CNI net plugin
-	if !rootless.IsRootless() {
-		netPlugin, err := ocicni.InitCNI(runtime.config.Network.DefaultNetwork, runtime.config.Network.NetworkConfigDir, runtime.config.Network.CNIPluginDirs...)
-		if err != nil {
-			return errors.Wrapf(err, "error configuring CNI network plugin")
-		}
-		runtime.netPlugin = netPlugin
+	netPlugin, err := ocicni.InitCNI(runtime.config.Network.DefaultNetwork, runtime.config.Network.NetworkConfigDir, runtime.config.Network.CNIPluginDirs...)
+	if err != nil {
+		return errors.Wrapf(err, "error configuring CNI network plugin")
 	}
+	runtime.netPlugin = netPlugin
 
 	// We now need to see if the system has restarted
 	// We check for the presence of a file in our tmp directory to verify this
diff --git a/libpod/runtime_pod_infra_linux.go b/libpod/runtime_pod_infra_linux.go
index 0a09e40ea..1ae375ed9 100644
--- a/libpod/runtime_pod_infra_linux.go
+++ b/libpod/runtime_pod_infra_linux.go
@@ -104,7 +104,7 @@ func (r *Runtime) makeInfraContainer(ctx context.Context, p *Pod, imgName, rawIm
 		default:
 			// Since user namespace sharing is not implemented, we only need to check if it's rootless
 			netmode := "bridge"
-			if isRootless || p.config.InfraContainer.Slirp4netns {
+			if p.config.InfraContainer.Slirp4netns {
 				netmode = "slirp4netns"
 				if len(p.config.InfraContainer.NetworkOptions) != 0 {
 					options = append(options, WithNetworkOptions(p.config.InfraContainer.NetworkOptions))
diff --git a/pkg/netns/netns_linux.go b/pkg/netns/netns_linux.go
index 0b7d1782c..ecefb65ff 100644
--- a/pkg/netns/netns_linux.go
+++ b/pkg/netns/netns_linux.go
@@ -35,9 +35,9 @@ import (
 	"golang.org/x/sys/unix"
 )
 
-// get NSRunDir returns the dir of where to create the netNS. When running
+// GetNSRunDir returns the dir of where to create the netNS. When running
 // rootless, it needs to be at a location writable by user.
-func getNSRunDir() (string, error) {
+func GetNSRunDir() (string, error) {
 	if rootless.IsRootless() {
 		rootlessDir, err := util.GetRuntimeDir()
 		if err != nil {
@@ -51,15 +51,21 @@ func getNSRunDir() (string, error) {
 // NewNS creates a new persistent (bind-mounted) network namespace and returns
 // an object representing that namespace, without switching to it.
 func NewNS() (ns.NetNS, error) {
-	nsRunDir, err := getNSRunDir()
+	b := make([]byte, 16)
+	_, err := rand.Reader.Read(b)
 	if err != nil {
-		return nil, err
+		return nil, fmt.Errorf("failed to generate random netns name: %v", err)
 	}
+	nsName := fmt.Sprintf("cni-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
+	return NewNSWithName(nsName)
+}
 
-	b := make([]byte, 16)
-	_, err = rand.Reader.Read(b)
+// NewNSWithName creates a new persistent (bind-mounted) network namespace and returns
+// an object representing that namespace, without switching to it.
+func NewNSWithName(name string) (ns.NetNS, error) {
+	nsRunDir, err := GetNSRunDir()
 	if err != nil {
-		return nil, fmt.Errorf("failed to generate random netns name: %v", err)
+		return nil, err
 	}
 
 	// Create the directory for mounting network namespaces
@@ -93,10 +99,8 @@ func NewNS() (ns.NetNS, error) {
 		}
 	}
 
-	nsName := fmt.Sprintf("cni-%x-%x-%x-%x-%x", b[0:4], b[4:6], b[6:8], b[8:10], b[10:])
-
 	// create an empty file at the mount point
-	nsPath := path.Join(nsRunDir, nsName)
+	nsPath := path.Join(nsRunDir, name)
 	mountPointFd, err := os.Create(nsPath)
 	if err != nil {
 		return nil, err
@@ -177,7 +181,7 @@ func NewNS() (ns.NetNS, error) {
 
 // UnmountNS unmounts the NS held by the netns object
 func UnmountNS(ns ns.NetNS) error {
-	nsRunDir, err := getNSRunDir()
+	nsRunDir, err := GetNSRunDir()
 	if err != nil {
 		return err
 	}
diff --git a/pkg/specgen/generate/namespaces.go b/pkg/specgen/generate/namespaces.go
index b87375a92..845dfdad7 100644
--- a/pkg/specgen/generate/namespaces.go
+++ b/pkg/specgen/generate/namespaces.go
@@ -236,9 +236,6 @@ func namespaceOptions(ctx context.Context, s *specgen.SpecGenerator, rt *libpod.
 	case specgen.Private:
 		fallthrough
 	case specgen.Bridge:
-		if postConfigureNetNS && rootless.IsRootless() {
-			return nil, errors.New("CNI networks not supported with user namespaces")
-		}
 		portMappings, err := createPortMappings(ctx, s, img)
 		if err != nil {
 			return nil, err
diff --git a/pkg/specgen/generate/pod_create.go b/pkg/specgen/generate/pod_create.go
index 5d7bf1930..20151f016 100644
--- a/pkg/specgen/generate/pod_create.go
+++ b/pkg/specgen/generate/pod_create.go
@@ -4,6 +4,7 @@ import (
 	"context"
 
 	"github.com/containers/podman/v3/libpod"
+	"github.com/containers/podman/v3/pkg/rootless"
 	"github.com/containers/podman/v3/pkg/specgen"
 	"github.com/pkg/errors"
 	"github.com/sirupsen/logrus"
@@ -94,8 +95,19 @@ func createPodOptions(p *specgen.PodSpecGenerator, rt *libpod.Runtime) ([]libpod
 	}
 
 	switch p.NetNS.NSMode {
-	case specgen.Bridge, specgen.Default, "":
-		logrus.Debugf("Pod using default network mode")
+	case specgen.Default, "":
+		if p.NoInfra {
+			logrus.Debugf("No networking because the infra container is missing")
+			break
+		}
+		if rootless.IsRootless() {
+			logrus.Debugf("Pod will use slirp4netns")
+			options = append(options, libpod.WithPodSlirp4netns(p.NetworkOptions))
+		} else {
+			logrus.Debugf("Pod using bridge network mode")
+		}
+	case specgen.Bridge:
+		logrus.Debugf("Pod using bridge network mode")
 	case specgen.Host:
 		logrus.Debugf("Pod will use host networking")
 		options = append(options, libpod.WithPodHostNetwork())
diff --git a/test/compose/mount_and_label/tests.sh b/test/compose/mount_and_label/tests.sh
index 07ff089b5..fa929bed6 100644
--- a/test/compose/mount_and_label/tests.sh
+++ b/test/compose/mount_and_label/tests.sh
@@ -1,4 +1,5 @@
 # -*- bash -*-
 
 test_port 5000 = "Podman rulez!"
-podman container inspect -l --format '{{.Config.Labels}}' | grep "the_best"
+podman container inspect -l --format '{{.Config.Labels}}'
+like "$output" "io.podman:the_best" "$testname : Container label is set"
diff --git a/test/compose/test-compose b/test/compose/test-compose
index 9558fbf58..704c71a9f 100755
--- a/test/compose/test-compose
+++ b/test/compose/test-compose
@@ -13,7 +13,8 @@ TEST_ROOTDIR=$(realpath $(dirname $0))
 # Podman executable
 PODMAN_BIN=$(realpath $TEST_ROOTDIR/../../bin)/podman
 
-# Local path to docker socket (we will add the unix:/ prefix when we need it)
+# Local path to docker socket with unix prefix
+# The path will be changed for rootless users
 DOCKER_SOCK=/var/run/docker.sock
 
 # END   stuff you can but probably shouldn't customize
@@ -40,6 +41,13 @@ echo 0 >$failures_file
 ###############################################################################
 # BEGIN infrastructure code - the helper functions used in tests themselves
 
+#################
+#  is_rootless  #  Check if we run as normal user
+#################
+function is_rootless() {
+    [ "$(id -u)" -ne 0 ]
+}
+
 #########
 #  die  #  Exit error with a message to stderr
 #########
@@ -155,7 +163,7 @@ function test_port() {
     local op="$2"                # '=' or '~'
     local expect="$3"            # what to expect from curl output
 
-    local actual=$(curl --retry 5 --retry-connrefused -s http://127.0.0.1:$port/)
+    local actual=$(curl --retry 10 --retry-all-errors -s http://127.0.0.1:$port/)
     local curl_rc=$?
     if [ $curl_rc -ne 0 ]; then
         _show_ok 0 "$testname - curl failed with status $curl_rc"
@@ -179,7 +187,12 @@ function start_service() {
     test -x $PODMAN_BIN || die "Not found: $PODMAN_BIN"
 
     # FIXME: use ${testname} subdir but we can't: 50-char limit in runroot
-    rm -rf            $WORKDIR/{root,runroot,cni}
+    if ! is_rootless; then
+        rm -rf $WORKDIR/{root,runroot,cni}
+    else
+        $PODMAN_BIN unshare rm -rf $WORKDIR/{root,runroot,cni}
+    fi
+    rm -f $DOCKER_SOCK
     mkdir --mode 0755 $WORKDIR/{root,runroot,cni}
     chcon --reference=/var/lib/containers $WORKDIR/root
     cp /etc/cni/net.d/*podman*conflist $WORKDIR/cni/
@@ -190,7 +203,7 @@ function start_service() {
         --cgroup-manager=systemd \
         --cni-config-dir $WORKDIR/cni \
         system service \
-        --time 0 unix:/$DOCKER_SOCK \
+        --time 0 unix://$DOCKER_SOCK \
         &> $WORKDIR/server.log &
     service_pid=$!
 
@@ -211,10 +224,11 @@ function start_service() {
 ############
 function podman() {
     echo "\$ podman $*"           >>$WORKDIR/output.log
-    $PODMAN_BIN \
+    output=$($PODMAN_BIN \
         --root    $WORKDIR/root    \
         --runroot $WORKDIR/runroot \
-        "$@"    >>$WORKDIR/output.log 2>&1
+        "$@")
+    echo -n "$output" >>$WORKDIR/output.log
 }
 
 ###################
@@ -239,6 +253,14 @@ done
 ###############################################################################
 # BEGIN entry handler (subtest invoker)
 
+# When rootless use a socket path accessible by the rootless user
+if is_rootless; then
+    DOCKER_SOCK="$WORKDIR/docker.sock"
+    DOCKER_HOST="unix://$DOCKER_SOCK"
+    # export DOCKER_HOST docker-compose will use it
+    export DOCKER_HOST
+fi
+
 # Identify the tests to run. If called with args, use those as globs.
 tests_to_run=()
 if [ -n "$*" ]; then
@@ -308,7 +330,7 @@ for t in ${tests_to_run[@]}; do
         fi
 
         # Done. Clean up.
-        docker-compose down     &> $logfile
+        docker-compose down     &>> $logfile
         rc=$?
         if [[ $rc -eq 0 ]]; then
             _show_ok 1 "$testname - down"
@@ -322,7 +344,11 @@ for t in ${tests_to_run[@]}; do
     wait $service_pid
 
     # FIXME: otherwise we get EBUSY
-    umount $WORKDIR/root/overlay  &>/dev/null
+    if ! is_rootless; then
+        umount $WORKDIR/root/overlay  &>/dev/null
+    else
+        $PODMAN_BIN unshare umount $WORKDIR/root/overlay  &>/dev/null
+    fi
 
     # FIXME: run 'podman ps'?
 #    rm -rf $WORKDIR/${testname}
@@ -336,9 +362,13 @@ done
 test_count=$(<$testcounter_file)
 failure_count=$(<$failures_file)
 
-#if [ -z "$PODMAN_TESTS_KEEP_WORKDIR" ]; then
-#    rm -rf $WORKDIR
-#fi
+if [ -z "$PODMAN_TESTS_KEEP_WORKDIR" ]; then
+     if ! is_rootless; then
+        rm -rf $WORKDIR
+    else
+        $PODMAN_BIN unshare rm -rf $WORKDIR
+    fi
+fi
 
 echo "1..${test_count}"
 
diff --git a/test/compose/two_networks/Readme.md b/test/compose/two_networks/Readme.md
new file mode 100644
index 000000000..471004f7d
--- /dev/null
+++ b/test/compose/two_networks/Readme.md
@@ -0,0 +1,8 @@
+two networks
+===============
+
+This test checks that we can create containers with more than one network.
+
+Validation
+------------
+* podman container inspect two_networks_con1_1 --format '{{len .NetworkSettings.Networks}}' shows 2
diff --git a/test/compose/two_networks/docker-compose.yml b/test/compose/two_networks/docker-compose.yml
new file mode 100644
index 000000000..686396ccc
--- /dev/null
+++ b/test/compose/two_networks/docker-compose.yml
@@ -0,0 +1,11 @@
+version: '3'
+services:
+  con1:
+    image: alpine
+    command: top
+    networks:
+      - net1
+      - net2
+networks:
+  net1:
+  net2:
diff --git a/test/compose/two_networks/tests.sh b/test/compose/two_networks/tests.sh
new file mode 100644
index 000000000..1cc88aa5f
--- /dev/null
+++ b/test/compose/two_networks/tests.sh
@@ -0,0 +1,7 @@
+# -*- bash -*-
+
+podman container inspect two_networks_con1_1 --format '{{len .NetworkSettings.Networks}}'
+is "$output" "2" "$testname : Container is connected to both networks"
+podman container inspect two_networks_con1_1 --format '{{.NetworkSettings.Networks}}'
+like "$output" "two_networks_net1" "$testname : First network name exists"
+like "$output" "two_networks_net2" "$testname : Second network name exists"
diff --git a/test/e2e/network_connect_disconnect_test.go b/test/e2e/network_connect_disconnect_test.go
index e9a7b421f..6974c7614 100644
--- a/test/e2e/network_connect_disconnect_test.go
+++ b/test/e2e/network_connect_disconnect_test.go
@@ -33,14 +33,12 @@ var _ = Describe("Podman network connect and disconnect", func() {
 	})
 
 	It("bad network name in disconnect should result in error", func() {
-		SkipIfRootless("network connect and disconnect are only rootful")
 		dis := podmanTest.Podman([]string{"network", "disconnect", "foobar", "test"})
 		dis.WaitWithDefaultTimeout()
 		Expect(dis.ExitCode()).ToNot(BeZero())
 	})
 
 	It("bad container name in network disconnect should result in error", func() {
-		SkipIfRootless("network connect and disconnect are only rootful")
 		netName := "aliasTest" + stringid.GenerateNonCryptoID()
 		session := podmanTest.Podman([]string{"network", "create", netName})
 		session.WaitWithDefaultTimeout()
@@ -72,7 +70,6 @@ var _ = Describe("Podman network connect and disconnect", func() {
 	})
 
 	It("podman network disconnect", func() {
-		SkipIfRootless("network connect and disconnect are only rootful")
 		netName := "aliasTest" + stringid.GenerateNonCryptoID()
 		session := podmanTest.Podman([]string{"network", "create", netName})
 		session.WaitWithDefaultTimeout()
@@ -102,14 +99,12 @@ var _ = Describe("Podman network connect and disconnect", func() {
 	})
 
 	It("bad network name in connect should result in error", func() {
-		SkipIfRootless("network connect and disconnect are only rootful")
 		dis := podmanTest.Podman([]string{"network", "connect", "foobar", "test"})
 		dis.WaitWithDefaultTimeout()
 		Expect(dis.ExitCode()).ToNot(BeZero())
 	})
 
 	It("bad container name in network connect should result in error", func() {
-		SkipIfRootless("network connect and disconnect are only rootful")
 		netName := "aliasTest" + stringid.GenerateNonCryptoID()
 		session := podmanTest.Podman([]string{"network", "create", netName})
 		session.WaitWithDefaultTimeout()
@@ -141,7 +136,6 @@ var _ = Describe("Podman network connect and disconnect", func() {
 	})
 
 	It("podman connect on a container that already is connected to the network should error", func() {
-		SkipIfRootless("network connect and disconnect are only rootful")
 		netName := "aliasTest" + stringid.GenerateNonCryptoID()
 		session := podmanTest.Podman([]string{"network", "create", netName})
 		session.WaitWithDefaultTimeout()
@@ -159,7 +153,6 @@ var _ = Describe("Podman network connect and disconnect", func() {
 
 	It("podman network connect", func() {
 		SkipIfRemote("This requires a pending PR to be merged before it will work")
-		SkipIfRootless("network connect and disconnect are only rootful")
 		netName := "aliasTest" + stringid.GenerateNonCryptoID()
 		session := podmanTest.Podman([]string{"network", "create", netName})
 		session.WaitWithDefaultTimeout()
@@ -203,18 +196,23 @@ var _ = Describe("Podman network connect and disconnect", func() {
 	})
 
 	It("podman network connect when not running", func() {
-		SkipIfRootless("network connect and disconnect are only rootful")
-		netName := "aliasTest" + stringid.GenerateNonCryptoID()
-		session := podmanTest.Podman([]string{"network", "create", netName})
+		netName1 := "connect1" + stringid.GenerateNonCryptoID()
+		session := podmanTest.Podman([]string{"network", "create", netName1})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(BeZero())
-		defer podmanTest.removeCNINetwork(netName)
+		defer podmanTest.removeCNINetwork(netName1)
 
-		ctr := podmanTest.Podman([]string{"create", "--name", "test", ALPINE, "top"})
+		netName2 := "connect2" + stringid.GenerateNonCryptoID()
+		session = podmanTest.Podman([]string{"network", "create", netName2})
+		session.WaitWithDefaultTimeout()
+		Expect(session.ExitCode()).To(BeZero())
+		defer podmanTest.removeCNINetwork(netName2)
+
+		ctr := podmanTest.Podman([]string{"create", "--name", "test", "--network", netName1, ALPINE, "top"})
 		ctr.WaitWithDefaultTimeout()
 		Expect(ctr.ExitCode()).To(BeZero())
 
-		dis := podmanTest.Podman([]string{"network", "connect", netName, "test"})
+		dis := podmanTest.Podman([]string{"network", "connect", netName2, "test"})
 		dis.WaitWithDefaultTimeout()
 		Expect(dis.ExitCode()).To(BeZero())
 
@@ -286,7 +284,6 @@ var _ = Describe("Podman network connect and disconnect", func() {
 	})
 
 	It("podman network disconnect when not running", func() {
-		SkipIfRootless("network connect and disconnect are only rootful")
 		netName1 := "aliasTest" + stringid.GenerateNonCryptoID()
 		session := podmanTest.Podman([]string{"network", "create", netName1})
 		session.WaitWithDefaultTimeout()
diff --git a/test/e2e/run_networking_test.go b/test/e2e/run_networking_test.go
index 0e6e636bc..4c66e2823 100644
--- a/test/e2e/run_networking_test.go
+++ b/test/e2e/run_networking_test.go
@@ -641,22 +641,26 @@ var _ = Describe("Podman run networking", func() {
 		Expect(run.OutputToString()).To(ContainSubstring(ipAddr))
 	})
 
-	It("podman rootless fails custom CNI network with --uidmap", func() {
-		SkipIfNotRootless("The configuration works with rootless")
-
+	It("podman cni network works across user ns", func() {
 		netName := stringid.GenerateNonCryptoID()
 		create := podmanTest.Podman([]string{"network", "create", netName})
 		create.WaitWithDefaultTimeout()
 		Expect(create.ExitCode()).To(BeZero())
 		defer podmanTest.removeCNINetwork(netName)
 
-		run := podmanTest.Podman([]string{"run", "--rm", "--net", netName, "--uidmap", "0:1:4096", ALPINE, "true"})
+		name := "nc-server"
+		run := podmanTest.Podman([]string{"run", "-d", "--name", name, "--net", netName, ALPINE, "nc", "-l", "-p", "8080"})
+		run.WaitWithDefaultTimeout()
+		Expect(run.ExitCode()).To(Equal(0))
+
+		run = podmanTest.Podman([]string{"run", "--rm", "--net", netName, "--uidmap", "0:1:4096", ALPINE, "sh", "-c", fmt.Sprintf("echo podman | nc -w 1 %s.dns.podman 8080", name)})
 		run.WaitWithDefaultTimeout()
-		Expect(run.ExitCode()).To(Equal(125))
+		Expect(run.ExitCode()).To(Equal(0))
 
-		remove := podmanTest.Podman([]string{"network", "rm", netName})
-		remove.WaitWithDefaultTimeout()
-		Expect(remove.ExitCode()).To(BeZero())
+		log := podmanTest.Podman([]string{"logs", name})
+		log.WaitWithDefaultTimeout()
+		Expect(log.ExitCode()).To(Equal(0))
+		Expect(log.OutputToString()).To(Equal("podman"))
 	})
 
 	It("podman run with new:pod and static-ip", func() {
@@ -762,7 +766,7 @@ var _ = Describe("Podman run networking", func() {
 		Expect(session.ExitCode()).To(Equal(1))
 		Expect(session.ErrorToString()).To(ContainSubstring("can't resolve 'con1'"))
 
-		session = podmanTest.Podman([]string{"run", "--name", "con4", "--network", net, ALPINE, "nslookup", pod2})
+		session = podmanTest.Podman([]string{"run", "--name", "con4", "--network", net, ALPINE, "nslookup", pod2 + ".dns.podman"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(BeZero())
 	})
diff --git a/test/system/500-networking.bats b/test/system/500-networking.bats
index 4868ad6a0..804dd46b1 100644
--- a/test/system/500-networking.bats
+++ b/test/system/500-networking.bats
@@ -143,13 +143,6 @@ load helpers
 
     run_podman network rm $mynetname
     run_podman 1 network rm $mynetname
-
-    # rootless CNI leaves behind an image pulled by SHA, hence with no tag.
-    # Remove it if present; we can only remove it by ID.
-    run_podman images --format '{{.Id}}' rootless-cni-infra
-    if [ -n "$output" ]; then
-        run_podman rmi $output
-    fi
 }
 
 @test "podman network reload" {