Merge pull request #7460 from AkihiroSuda/allow-rootless-cni

rootless: support `podman network create` (CNI-in-slirp4netns)

21 files changed, 605 insertions, 48 deletions

diff --git a/cmd/podman/networks/create.go b/cmd/podman/networks/create.go
index dabf6f0d2..68a577ae1 100644
--- a/cmd/podman/networks/create.go
+++ b/cmd/podman/networks/create.go
@@ -21,9 +21,6 @@ var (
 		RunE:    networkCreate,
 		Args:    cobra.MaximumNArgs(1),
 		Example: `podman network create podman1`,
-		Annotations: map[string]string{
-			registry.ParentNSRequired: "",
-		},
 	}
 )
 
diff --git a/cmd/podman/networks/inspect.go b/cmd/podman/networks/inspect.go
index f00d6b63c..c5872def7 100644
--- a/cmd/podman/networks/inspect.go
+++ b/cmd/podman/networks/inspect.go
@@ -22,9 +22,6 @@ var (
 		RunE:    networkInspect,
 		Example: `podman network inspect podman`,
 		Args:    cobra.MinimumNArgs(1),
-		Annotations: map[string]string{
-			registry.ParentNSRequired: "",
-		},
 	}
 )
 
diff --git a/cmd/podman/networks/list.go b/cmd/podman/networks/list.go
index 3a2651cbc..b6fb2bb80 100644
--- a/cmd/podman/networks/list.go
+++ b/cmd/podman/networks/list.go
@@ -25,9 +25,6 @@ var (
 		Long:    networklistDescription,
 		RunE:    networkList,
 		Example: `podman network list`,
-		Annotations: map[string]string{
-			registry.ParentNSRequired: "",
-		},
 	}
 )
 
diff --git a/cmd/podman/networks/rm.go b/cmd/podman/networks/rm.go
index dfbb5d081..ac49993b7 100644
--- a/cmd/podman/networks/rm.go
+++ b/cmd/podman/networks/rm.go
@@ -19,9 +19,6 @@ var (
 		RunE:    networkRm,
 		Example: `podman network rm podman`,
 		Args:    cobra.MinimumNArgs(1),
-		Annotations: map[string]string{
-			registry.ParentNSRequired: "",
-		},
 	}
 )
 
diff --git a/contrib/rootless-cni-infra/Containerfile b/contrib/rootless-cni-infra/Containerfile
new file mode 100644
index 000000000..c5d812a6e
--- /dev/null
+++ b/contrib/rootless-cni-infra/Containerfile
@@ -0,0 +1,35 @@
+ARG GOLANG_VERSION=1.15
+ARG ALPINE_VERSION=3.12
+ARG CNI_VERSION=v0.8.0
+ARG CNI_PLUGINS_VERSION=v0.8.7
+# Aug 20, 2020
+ARG DNSNAME_VESION=78b4da7bbfc51c27366da630e1df1c4f2e8b1b5b
+
+FROM golang:${GOLANG_VERSION}-alpine${ALPINE_VERSION} AS golang-base
+RUN apk add --no-cache git
+
+FROM golang-base AS cnitool
+RUN git clone https://github.com/containernetworking/cni /go/src/github.com/containernetworking/cni
+WORKDIR /go/src/github.com/containernetworking/cni
+ARG CNI_VERSION
+RUN git checkout ${CNI_VERSION}
+RUN go build -o /cnitool ./cnitool
+
+FROM golang-base AS dnsname
+RUN git clone https://github.com/containers/dnsname /go/src/github.com/containers/dnsname
+WORKDIR /go/src/github.com/containers/dnsname
+ARG DNSNAME_VERSION
+RUN git checkout ${DNSNAME_VERSION}
+RUN go build -o /dnsname ./plugins/meta/dnsname
+
+FROM alpine:${ALPINE_VERSION}
+RUN apk add --no-cache curl dnsmasq iptables ip6tables iproute2
+ARG TARGETARCH
+ARG CNI_PLUGINS_VERSION
+RUN mkdir -p /opt/cni/bin && \
+    curl -fsSL https://github.com/containernetworking/plugins/releases/download/${CNI_PLUGINS_VERSION}/cni-plugins-linux-${TARGETARCH}-${CNI_PLUGINS_VERSION}.tgz | tar xz -C /opt/cni/bin
+COPY --from=cnitool /cnitool /usr/local/bin
+COPY --from=dnsname /dnsname /opt/cni/bin
+COPY rootless-cni-infra /usr/local/bin
+ENV CNI_PATH=/opt/cni/bin
+CMD ["sleep", "infinity"]
diff --git a/contrib/rootless-cni-infra/README.md b/contrib/rootless-cni-infra/README.md
new file mode 100644
index 000000000..937e057fb
--- /dev/null
+++ b/contrib/rootless-cni-infra/README.md
@@ -0,0 +1,22 @@
+# rootless-cni-infra
+
+Infra container for CNI-in-slirp4netns.
+
+## How it works
+
+When a CNI network is specified for `podman run` in rootless mode, Podman launches the `rootless-cni-infra` container to execute CNI plugins inside slirp4netns.
+
+The infra container is created per user, by executing an equivalent of:
+`podman run -d --name rootless-cni-infra --pid=host --privileged -v $HOME/.config/cni/net.d:/etc/cni/net.d rootless-cni-infra`.
+The infra container is automatically deleted when no CNI network is in use.
+
+Podman then allocates a CNI netns in the infra container, by executing an equivalent of:
+`podman exec rootless-cni-infra rootless-cni-infra alloc $CONTAINER_ID $NETWORK_NAME $POD_NAME`.
+
+The allocated netns is deallocated when the container is being removed, by executing an equivalent of:
+`podman exec rootless-cni-infra rootless-cni-infra dealloc $CONTAINER_ID $NETWORK_NAME`.
+
+## Directory layout
+
+* `/run/rootless-cni-infra/${CONTAINER_ID}/pid`: PID of the `sleep infinity` process that corresponds to the allocated netns
+* `/run/rootless-cni-infra/${CONTAINER_ID}/attached/${NETWORK_NAME}`: CNI result
diff --git a/contrib/rootless-cni-infra/rootless-cni-infra b/contrib/rootless-cni-infra/rootless-cni-infra
new file mode 100755
index 000000000..5a574d2eb
--- /dev/null
+++ b/contrib/rootless-cni-infra/rootless-cni-infra
@@ -0,0 +1,147 @@
+#!/bin/sh
+set -eu
+
+ARG0="$0"
+VERSION="0.1.0"
+BASE="/run/rootless-cni-infra"
+
+# CLI subcommand: "alloc $CONTAINER_ID $NETWORK_NAME $POD_NAME"
+cmd_entrypoint_alloc() {
+	if [ "$#" -ne 3 ]; then
+		echo >&2 "Usage: $ARG0 alloc CONTAINER_ID NETWORK_NAME POD_NAME"
+		exit 1
+	fi
+
+	ID="$1"
+	NET="$2"
+	K8S_POD_NAME="$3"
+
+	dir="${BASE}/${ID}"
+	mkdir -p "${dir}/attached"
+
+	pid=""
+	if [ -f "${dir}/pid" ]; then
+		pid=$(cat "${dir}/pid")
+	else
+		unshare -n sleep infinity &
+		pid="$!"
+		echo "${pid}" >"${dir}/pid"
+		nsenter -t "${pid}" -n ip link set lo up
+	fi
+	CNI_ARGS="IgnoreUnknown=1;K8S_POD_NAME=${K8S_POD_NAME}"
+	nwcount=$(find "${dir}/attached" -type f | wc -l)
+	CNI_IFNAME="eth${nwcount}"
+	export CNI_ARGS CNI_IFNAME
+	cnitool add "${NET}" "/proc/${pid}/ns/net" >"${dir}/attached/${NET}"
+
+	# return the result
+	ns="/proc/${pid}/ns/net"
+	echo "{\"ns\":\"${ns}\"}"
+}
+
+# CLI subcommand: "dealloc $CONTAINER_ID $NETWORK_NAME"
+cmd_entrypoint_dealloc() {
+	if [ "$#" -ne 2 ]; then
+		echo >&2 "Usage: $ARG0 dealloc CONTAINER_ID NETWORK_NAME"
+		exit 1
+	fi
+
+	ID=$1
+	NET=$2
+
+	dir="${BASE}/${ID}"
+	if [ ! -f "${dir}/pid" ]; then
+		exit 0
+	fi
+	pid=$(cat "${dir}/pid")
+	cnitool del "${NET}" "/proc/${pid}/ns/net"
+	rm -f "${dir}/attached/${NET}"
+
+	nwcount=$(find "${dir}/attached" -type f | wc -l)
+	if [ "${nwcount}" = 0 ]; then
+		kill -9 "${pid}"
+		rm -rf "${dir}"
+	fi
+
+	# return empty json
+	echo "{}"
+}
+
+# CLI subcommand: "is-idle"
+cmd_entrypoint_is_idle() {
+	if [ ! -d ${BASE} ]; then
+		echo '{"idle": true}'
+	elif [ -z "$(ls -1 ${BASE})" ]; then
+		echo '{"idle": true}'
+	else
+		echo '{"idle": false}'
+	fi
+}
+
+# CLI subcommand: "print-cni-result $CONTAINER_ID $NETWORK_NAME"
+cmd_entrypoint_print_cni_result() {
+	if [ "$#" -ne 2 ]; then
+		echo >&2 "Usage: $ARG0 print-cni-result CONTAINER_ID NETWORK_NAME"
+		exit 1
+	fi
+
+	ID=$1
+	NET=$2
+
+	# the result shall be CNI JSON
+	cat "${BASE}/${ID}/attached/${NET}"
+}
+
+# CLI subcommand: "print-netns-path $CONTAINER_ID"
+cmd_entrypoint_print_netns_path() {
+	if [ "$#" -ne 1 ]; then
+		echo >&2 "Usage: $ARG0 print-netns-path CONTAINER_ID"
+		exit 1
+	fi
+
+	ID=$1
+
+	pid=$(cat "${BASE}/${ID}/pid")
+	path="/proc/${pid}/ns/net"
+
+	# return the result
+	echo "{\"path\":\"${path}\"}"
+}
+
+# CLI subcommand: "help"
+cmd_entrypoint_help() {
+	echo "Usage: ${ARG0} COMMAND"
+	echo
+	echo "Rootless CNI Infra container"
+	echo
+	echo "Commands:"
+	echo "  alloc             Allocate a netns"
+	echo "  dealloc           Deallocate a netns"
+	echo "  is-idle           Print whether the infra container is idle"
+	echo "  print-cni-result  Print CNI result"
+	echo "  print-netns-path  Print netns path"
+	echo "  help              Print help"
+	echo "  version           Print version"
+}
+
+# CLI subcommand: "version"
+cmd_entrypoint_version() {
+	echo "{\"version\": \"${VERSION}\"}"
+}
+
+# parse args
+command="${1:-}"
+if [ -z "$command" ]; then
+	echo >&2 "No command was specified. Run \`${ARG0} help\` to see the usage."
+	exit 1
+fi
+
+command_func=$(echo "cmd_entrypoint_${command}" | sed -e "s/-/_/g")
+if ! command -v "${command_func}" >/dev/null 2>&1; then
+	echo >&2 "Unknown command: ${command}. Run \`${ARG0} help\` to see the usage."
+	exit 1
+fi
+
+# start the command func
+shift
+"${command_func}" "$@"
diff --git a/libpod/container_internal.go b/libpod/container_internal.go
index c41d81a2b..c3f07a48b 100644
--- a/libpod/container_internal.go
+++ b/libpod/container_internal.go
@@ -957,8 +957,10 @@ func (c *Container) completeNetworkSetup() error {
 	if err := c.syncContainer(); err != nil {
 		return err
 	}
-	if c.config.NetMode.IsSlirp4netns() {
+	if rootless.IsRootless() {
 		return c.runtime.setupRootlessNetNS(c)
+	} else if c.config.NetMode.IsSlirp4netns() {
+		return c.runtime.setupSlirp4netns(c)
 	}
 	if err := c.runtime.setupNetNS(c); err != nil {
 		return err
diff --git a/libpod/container_internal_linux.go b/libpod/container_internal_linux.go
index f789b0069..605b526a4 100644
--- a/libpod/container_internal_linux.go
+++ b/libpod/container_internal_linux.go
@@ -84,7 +84,11 @@ func (c *Container) prepare() error {
 		// Set up network namespace if not already set up
 		noNetNS := c.state.NetNS == nil
 		if c.config.CreateNetNS && noNetNS && !c.config.PostConfigureNetNS {
-			netNS, networkStatus, createNetNSErr = c.runtime.createNetNS(c)
+			if rootless.IsRootless() && len(c.config.Networks) > 0 {
+				netNS, networkStatus, createNetNSErr = AllocRootlessCNI(context.Background(), c)
+			} else {
+				netNS, networkStatus, createNetNSErr = c.runtime.createNetNS(c)
+			}
 			if createNetNSErr != nil {
 				return
 			}
@@ -98,8 +102,12 @@ func (c *Container) prepare() error {
 		}
 
 		// handle rootless network namespace setup
-		if noNetNS && c.config.NetMode.IsSlirp4netns() && !c.config.PostConfigureNetNS {
-			createNetNSErr = c.runtime.setupRootlessNetNS(c)
+		if noNetNS && !c.config.PostConfigureNetNS {
+			if rootless.IsRootless() {
+				createNetNSErr = c.runtime.setupRootlessNetNS(c)
+			} else if c.config.NetMode.IsSlirp4netns() {
+				createNetNSErr = c.runtime.setupSlirp4netns(c)
+			}
 		}
 	}()
 	// Mount storage if not mounted
diff --git a/libpod/container_validate.go b/libpod/container_validate.go
index d657e3549..b78168cd1 100644
--- a/libpod/container_validate.go
+++ b/libpod/container_validate.go
@@ -2,7 +2,6 @@ package libpod
 
 import (
 	"github.com/containers/podman/v2/libpod/define"
-	"github.com/containers/podman/v2/pkg/rootless"
 	spec "github.com/opencontainers/runtime-spec/specs-go"
 	"github.com/pkg/errors"
 )
@@ -68,16 +67,6 @@ func (c *Container) validate() error {
 		}
 	}
 
-	// Rootless has some requirements, compared to networks.
-	if rootless.IsRootless() {
-		if len(c.config.Networks) > 0 {
-			return errors.Wrapf(define.ErrInvalidArg, "cannot join CNI networks if running rootless")
-		}
-
-		// TODO: Should we make sure network mode is set to Slirp if set
-		// at all?
-	}
-
 	// Can only set static IP or MAC is creating a network namespace.
 	if !c.config.CreateNetNS && (c.config.StaticIP != nil || c.config.StaticMAC != nil) {
 		return errors.Wrapf(define.ErrInvalidArg, "cannot set static IP or MAC address if not creating a network namespace")
diff --git a/libpod/networking_linux.go b/libpod/networking_linux.go
index 6f266e5d6..c0508ce39 100644
--- a/libpod/networking_linux.go
+++ b/libpod/networking_linux.go
@@ -4,6 +4,7 @@ package libpod
 
 import (
 	"bytes"
+	"context"
 	"crypto/rand"
 	"fmt"
 	"io"
@@ -208,6 +209,20 @@ func checkSlirpFlags(path string) (*slirpFeatures, error) {
 
 // Configure the network namespace for a rootless container
 func (r *Runtime) setupRootlessNetNS(ctr *Container) error {
+	if ctr.config.NetMode.IsSlirp4netns() {
+		return r.setupSlirp4netns(ctr)
+	}
+	if len(ctr.config.Networks) > 0 {
+		// set up port forwarder for CNI-in-slirp4netns
+		netnsPath := ctr.state.NetNS.Path()
+		// TODO: support slirp4netns port forwarder as well
+		return r.setupRootlessPortMappingViaRLK(ctr, netnsPath)
+	}
+	return nil
+}
+
+// setupSlirp4netns can be called in rootful as well as in rootless
+func (r *Runtime) setupSlirp4netns(ctr *Container) error {
 	path := r.config.Engine.NetworkCmdPath
 
 	if path == "" {
@@ -711,7 +726,7 @@ func (r *Runtime) teardownNetNS(ctr *Container) error {
 
 	logrus.Debugf("Tearing down network namespace at %s for container %s", ctr.state.NetNS.Path(), ctr.ID())
 
-	// rootless containers do not use the CNI plugin
+	// rootless containers do not use the CNI plugin directly
 	if !rootless.IsRootless() && !ctr.config.NetMode.IsSlirp4netns() {
 		var requestedIP net.IP
 		if ctr.requestedIP != nil {
@@ -738,6 +753,13 @@ func (r *Runtime) teardownNetNS(ctr *Container) error {
 		}
 	}
 
+	// CNI-in-slirp4netns
+	if rootless.IsRootless() && len(ctr.config.Networks) != 0 {
+		if err := DeallocRootlessCNI(context.Background(), ctr); err != nil {
+			return errors.Wrapf(err, "error tearing down CNI-in-slirp4netns for container %s", ctr.ID())
+		}
+	}
+
 	// First unmount the namespace
 	if err := netns.UnmountNS(ctr.state.NetNS); err != nil {
 		return errors.Wrapf(err, "error unmounting network namespace for container %s", ctr.ID())
diff --git a/libpod/networking_unsupported.go b/libpod/networking_unsupported.go
index dd72a3fd8..76bb01424 100644
--- a/libpod/networking_unsupported.go
+++ b/libpod/networking_unsupported.go
@@ -8,6 +8,10 @@ func (r *Runtime) setupRootlessNetNS(ctr *Container) error {
 	return define.ErrNotImplemented
 }
 
+func (r *Runtime) setupSlirp4netns(ctr *Container) error {
+	return define.ErrNotImplemented
+}
+
 func (r *Runtime) setupNetNS(ctr *Container) error {
 	return define.ErrNotImplemented
 }
diff --git a/libpod/oci_conmon_linux.go b/libpod/oci_conmon_linux.go
index f66835771..bb138ca14 100644
--- a/libpod/oci_conmon_linux.go
+++ b/libpod/oci_conmon_linux.go
@@ -1086,7 +1086,7 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co
 	cmd.ExtraFiles = append(cmd.ExtraFiles, childSyncPipe, childStartPipe)
 	cmd.ExtraFiles = append(cmd.ExtraFiles, envFiles...)
 
-	if r.reservePorts && !ctr.config.NetMode.IsSlirp4netns() {
+	if r.reservePorts && !rootless.IsRootless() && !ctr.config.NetMode.IsSlirp4netns() {
 		ports, err := bindPorts(ctr.config.PortMappings)
 		if err != nil {
 			return err
@@ -1098,7 +1098,7 @@ func (r *ConmonOCIRuntime) createOCIContainer(ctr *Container, restoreOptions *Co
 		cmd.ExtraFiles = append(cmd.ExtraFiles, ports...)
 	}
 
-	if ctr.config.NetMode.IsSlirp4netns() {
+	if ctr.config.NetMode.IsSlirp4netns() || rootless.IsRootless() {
 		if ctr.config.PostConfigureNetNS {
 			havePortMapping := len(ctr.Config().PortMappings) > 0
 			if havePortMapping {
diff --git a/libpod/rootless_cni_linux.go b/libpod/rootless_cni_linux.go
new file mode 100644
index 000000000..76dbfdcae
--- /dev/null
+++ b/libpod/rootless_cni_linux.go
@@ -0,0 +1,320 @@
+// +build linux
+
+package libpod
+
+import (
+	"bytes"
+	"context"
+	"io"
+	"path/filepath"
+	"runtime"
+
+	cnitypes "github.com/containernetworking/cni/pkg/types/current"
+	"github.com/containernetworking/plugins/pkg/ns"
+	"github.com/containers/podman/v2/libpod/define"
+	"github.com/containers/podman/v2/libpod/image"
+	"github.com/containers/podman/v2/pkg/util"
+	"github.com/containers/storage/pkg/lockfile"
+	"github.com/hashicorp/go-multierror"
+	spec "github.com/opencontainers/runtime-spec/specs-go"
+	"github.com/opencontainers/runtime-tools/generate"
+	"github.com/pkg/errors"
+	"github.com/sirupsen/logrus"
+)
+
+var rootlessCNIInfraImage = map[string]string{
+	// Built from ../contrib/rootless-cni-infra
+	// TODO: move to Podman's official quay
+	"amd64": "ghcr.io/akihirosuda/podman-rootless-cni-infra:gd34868a13-amd64",
+}
+
+const (
+	rootlessCNIInfraContainerNamespace = "podman-system"
+	rootlessCNIInfraContainerName      = "rootless-cni-infra"
+)
+
+// AllocRootlessCNI allocates a CNI netns inside the rootless CNI infra container.
+// Locks "rootless-cni-infra.lck".
+//
+// When the infra container is not running, it is created.
+//
+// AllocRootlessCNI does not lock c. c should be already locked.
+func AllocRootlessCNI(ctx context.Context, c *Container) (ns.NetNS, []*cnitypes.Result, error) {
+	if len(c.config.Networks) == 0 {
+		return nil, nil, errors.New("allocRootlessCNI shall not be called when len(c.config.Networks) == 0")
+	}
+	l, err := getRootlessCNIInfraLock(c.runtime)
+	if err != nil {
+		return nil, nil, err
+	}
+	l.Lock()
+	defer l.Unlock()
+	infra, err := ensureRootlessCNIInfraContainerRunning(ctx, c.runtime)
+	if err != nil {
+		return nil, nil, err
+	}
+	k8sPodName := getPodOrContainerName(c) // passed to CNI as K8S_POD_NAME
+	cniResults := make([]*cnitypes.Result, len(c.config.Networks))
+	for i, nw := range c.config.Networks {
+		cniRes, err := rootlessCNIInfraCallAlloc(infra, c.ID(), nw, k8sPodName)
+		if err != nil {
+			return nil, nil, err
+		}
+		cniResults[i] = cniRes
+	}
+	nsObj, err := rootlessCNIInfraGetNS(infra, c.ID())
+	if err != nil {
+		return nil, nil, err
+	}
+	logrus.Debugf("rootless CNI: container %q will join %q", c.ID(), nsObj.Path())
+	return nsObj, cniResults, nil
+}
+
+// DeallocRootlessCNI deallocates a CNI netns inside the rootless CNI infra container.
+// Locks "rootless-cni-infra.lck".
+//
+// When the infra container is no longer needed, it is removed.
+//
+// DeallocRootlessCNI does not lock c. c should be already locked.
+func DeallocRootlessCNI(ctx context.Context, c *Container) error {
+	if len(c.config.Networks) == 0 {
+		return errors.New("deallocRootlessCNI shall not be called when len(c.config.Networks) == 0")
+	}
+	l, err := getRootlessCNIInfraLock(c.runtime)
+	if err != nil {
+		return err
+	}
+	l.Lock()
+	defer l.Unlock()
+	infra, _ := getRootlessCNIInfraContainer(c.runtime)
+	if infra == nil {
+		return nil
+	}
+	var errs *multierror.Error
+	for _, nw := range c.config.Networks {
+		err := rootlessCNIInfraCallDelloc(infra, c.ID(), nw)
+		if err != nil {
+			errs = multierror.Append(errs, err)
+		}
+	}
+	if isIdle, err := rootlessCNIInfraIsIdle(infra); isIdle || err != nil {
+		if err != nil {
+			logrus.Warn(err)
+		}
+		logrus.Debugf("rootless CNI: removing infra container %q", infra.ID())
+		if err := c.runtime.removeContainer(ctx, infra, true, false, true); err != nil {
+			return err
+		}
+		logrus.Debugf("rootless CNI: removed infra container %q", infra.ID())
+	}
+	return errs.ErrorOrNil()
+}
+
+func getRootlessCNIInfraLock(r *Runtime) (lockfile.Locker, error) {
+	fname := filepath.Join(r.config.Engine.TmpDir, "rootless-cni-infra.lck")
+	return lockfile.GetLockfile(fname)
+}
+
+func getPodOrContainerName(c *Container) string {
+	pod, err := c.runtime.GetPod(c.PodID())
+	if err != nil || pod.config.Name == "" {
+		return c.Name()
+	}
+	return pod.config.Name
+}
+
+func rootlessCNIInfraCallAlloc(infra *Container, id, nw, k8sPodName string) (*cnitypes.Result, error) {
+	logrus.Debugf("rootless CNI: alloc %q, %q, %q", id, nw, k8sPodName)
+	var err error
+
+	_, err = rootlessCNIInfraExec(infra, "alloc", id, nw, k8sPodName)
+	if err != nil {
+		return nil, err
+	}
+	cniResStr, err := rootlessCNIInfraExec(infra, "print-cni-result", id, nw)
+	if err != nil {
+		return nil, err
+	}
+	var cniRes cnitypes.Result
+	if err := json.Unmarshal([]byte(cniResStr), &cniRes); err != nil {
+		return nil, errors.Wrapf(err, "unmarshaling as cnitypes.Result: %q", cniResStr)
+	}
+	return &cniRes, nil
+}
+
+func rootlessCNIInfraCallDelloc(infra *Container, id, nw string) error {
+	logrus.Debugf("rootless CNI: dealloc %q, %q", id, nw)
+	_, err := rootlessCNIInfraExec(infra, "dealloc", id, nw)
+	return err
+}
+
+func rootlessCNIInfraIsIdle(infra *Container) (bool, error) {
+	type isIdle struct {
+		Idle bool `json:"idle"`
+	}
+	resStr, err := rootlessCNIInfraExec(infra, "is-idle")
+	if err != nil {
+		return false, err
+	}
+	var res isIdle
+	if err := json.Unmarshal([]byte(resStr), &res); err != nil {
+		return false, errors.Wrapf(err, "unmarshaling as isIdle: %q", resStr)
+	}
+	return res.Idle, nil
+}
+
+func rootlessCNIInfraGetNS(infra *Container, id string) (ns.NetNS, error) {
+	type printNetnsPath struct {
+		Path string `json:"path"`
+	}
+	resStr, err := rootlessCNIInfraExec(infra, "print-netns-path", id)
+	if err != nil {
+		return nil, err
+	}
+	var res printNetnsPath
+	if err := json.Unmarshal([]byte(resStr), &res); err != nil {
+		return nil, errors.Wrapf(err, "unmarshaling as printNetnsPath: %q", resStr)
+	}
+	nsObj, err := ns.GetNS(res.Path)
+	if err != nil {
+		return nil, err
+	}
+	return nsObj, nil
+}
+
+func getRootlessCNIInfraContainer(r *Runtime) (*Container, error) {
+	containers, err := r.GetContainersWithoutLock(func(c *Container) bool {
+		return c.Namespace() == rootlessCNIInfraContainerNamespace &&
+			c.Name() == rootlessCNIInfraContainerName
+	})
+	if err != nil {
+		return nil, err
+	}
+	if len(containers) == 0 {
+		return nil, nil
+	}
+	return containers[0], nil
+}
+
+func ensureRootlessCNIInfraContainerRunning(ctx context.Context, r *Runtime) (*Container, error) {
+	c, err := getRootlessCNIInfraContainer(r)
+	if err != nil {
+		return nil, err
+	}
+	if c == nil {
+		return startRootlessCNIInfraContainer(ctx, r)
+	}
+	st, err := c.ContainerState()
+	if err != nil {
+		return nil, err
+	}
+	if st.State == define.ContainerStateRunning {
+		logrus.Debugf("rootless CNI: infra container %q is already running", c.ID())
+		return c, nil
+	}
+	logrus.Debugf("rootless CNI: infra container %q is %q, being started", c.ID(), st.State)
+	if err := c.initAndStart(ctx); err != nil {
+		return nil, err
+	}
+	logrus.Debugf("rootless CNI: infra container %q is running", c.ID())
+	return c, nil
+}
+
+func startRootlessCNIInfraContainer(ctx context.Context, r *Runtime) (*Container, error) {
+	imageName, ok := rootlessCNIInfraImage[runtime.GOARCH]
+	if !ok {
+		return nil, errors.Errorf("cannot find rootless-podman-network-sandbox image for %s", runtime.GOARCH)
+	}
+	logrus.Debugf("rootless CNI: ensuring image %q to exist", imageName)
+	newImage, err := r.ImageRuntime().New(ctx, imageName, "", "", nil, nil,
+		image.SigningOptions{}, nil, util.PullImageMissing)
+	if err != nil {
+		return nil, err
+	}
+	logrus.Debugf("rootless CNI: image %q is ready", imageName)
+
+	g, err := generate.New("linux")
+	if err != nil {
+		return nil, err
+	}
+	g.SetupPrivileged(true)
+	// Set --pid=host for ease of propagating "/proc/PID/ns/net" string
+	if err := g.RemoveLinuxNamespace(string(spec.PIDNamespace)); err != nil {
+		return nil, err
+	}
+	g.RemoveMount("/proc")
+	procMount := spec.Mount{
+		Destination: "/proc",
+		Type:        "bind",
+		Source:      "/proc",
+		Options:     []string{"rbind", "nosuid", "noexec", "nodev"},
+	}
+	g.AddMount(procMount)
+	// Mount CNI networks
+	etcCNINetD := spec.Mount{
+		Destination: "/etc/cni/net.d",
+		Type:        "bind",
+		Source:      r.config.Network.NetworkConfigDir,
+		Options:     []string{"ro"},
+	}
+	g.AddMount(etcCNINetD)
+	// FIXME: how to propagate ProcessArgs and Envs from Dockerfile?
+	g.SetProcessArgs([]string{"sleep", "infinity"})
+	g.AddProcessEnv("CNI_PATH", "/opt/cni/bin")
+	var options []CtrCreateOption
+	options = append(options, WithRootFSFromImage(newImage.ID(), imageName, imageName))
+	options = append(options, WithCtrNamespace(rootlessCNIInfraContainerNamespace))
+	options = append(options, WithName(rootlessCNIInfraContainerName))
+	options = append(options, WithPrivileged(true))
+	options = append(options, WithSecLabels([]string{"disable"}))
+	options = append(options, WithRestartPolicy("always"))
+	options = append(options, WithNetNS(nil, false, "slirp4netns", nil))
+	c, err := r.NewContainer(ctx, g.Config, options...)
+	if err != nil {
+		return nil, err
+	}
+	logrus.Debugf("rootless CNI infra container %q is created, now being started", c.ID())
+	if err := c.initAndStart(ctx); err != nil {
+		return nil, err
+	}
+	logrus.Debugf("rootless CNI: infra container %q is running", c.ID())
+
+	return c, nil
+}
+
+func rootlessCNIInfraExec(c *Container, args ...string) (string, error) {
+	cmd := "rootless-cni-infra"
+	var (
+		outB    bytes.Buffer
+		errB    bytes.Buffer
+		streams define.AttachStreams
+		config  ExecConfig
+	)
+	streams.OutputStream = &nopWriteCloser{Writer: &outB}
+	streams.ErrorStream = &nopWriteCloser{Writer: &errB}
+	streams.AttachOutput = true
+	streams.AttachError = true
+	config.Command = append([]string{cmd}, args...)
+	config.Privileged = true
+	logrus.Debugf("rootlessCNIInfraExec: c.ID()=%s, config=%+v, streams=%v, begin",
+		c.ID(), config, streams)
+	code, err := c.Exec(&config, &streams, nil)
+	logrus.Debugf("rootlessCNIInfraExec: c.ID()=%s, config=%+v, streams=%v, end (code=%d, err=%v)",
+		c.ID(), config, streams, code, err)
+	if err != nil {
+		return "", err
+	}
+	if code != 0 {
+		return "", errors.Errorf("command %s %v in container %s failed with status %d, stdout=%q, stderr=%q",
+			cmd, args, c.ID(), code, outB.String(), errB.String())
+	}
+	return outB.String(), nil
+}
+
+type nopWriteCloser struct {
+	io.Writer
+}
+
+func (nwc *nopWriteCloser) Close() error {
+	return nil
+}
diff --git a/libpod/runtime_ctr.go b/libpod/runtime_ctr.go
index 936dce2e9..241448981 100644
--- a/libpod/runtime_ctr.go
+++ b/libpod/runtime_ctr.go
@@ -772,7 +772,11 @@ func (r *Runtime) LookupContainer(idOrName string) (*Container, error) {
 func (r *Runtime) GetContainers(filters ...ContainerFilter) ([]*Container, error) {
 	r.lock.RLock()
 	defer r.lock.RUnlock()
+	return r.GetContainersWithoutLock(filters...)
+}
 
+// GetContainersWithoutLock is same as GetContainers but without lock
+func (r *Runtime) GetContainersWithoutLock(filters ...ContainerFilter) ([]*Container, error) {
 	if !r.valid {
 		return nil, define.ErrRuntimeStopped
 	}
diff --git a/pkg/domain/infra/abi/network.go b/pkg/domain/infra/abi/network.go
index c06714cbb..807e4b272 100644
--- a/pkg/domain/infra/abi/network.go
+++ b/pkg/domain/infra/abi/network.go
@@ -5,6 +5,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
+	"os"
 	"path/filepath"
 	"strings"
 
@@ -216,6 +217,9 @@ func createBridge(r *libpod.Runtime, name string, options entities.NetworkCreate
 	if err != nil {
 		return "", err
 	}
+	if err := os.MkdirAll(network.GetCNIConfDir(runtimeConfig), 0755); err != nil {
+		return "", err
+	}
 	cniPathName := filepath.Join(network.GetCNIConfDir(runtimeConfig), fmt.Sprintf("%s.conflist", name))
 	err = ioutil.WriteFile(cniPathName, b, 0644)
 	return cniPathName, err
diff --git a/pkg/network/files.go b/pkg/network/files.go
index 38ce38b97..a2090491f 100644
--- a/pkg/network/files.go
+++ b/pkg/network/files.go
@@ -14,11 +14,16 @@ import (
 	"github.com/pkg/errors"
 )
 
-func GetCNIConfDir(config *config.Config) string {
-	if len(config.Network.NetworkConfigDir) < 1 {
-		return CNIConfigDir
+func GetCNIConfDir(configArg *config.Config) string {
+	if len(configArg.Network.NetworkConfigDir) < 1 {
+		dc, err := config.DefaultConfig()
+		if err != nil {
+			// Fallback to hard-coded dir
+			return CNIConfigDir
+		}
+		return dc.Network.NetworkConfigDir
 	}
-	return config.Network.NetworkConfigDir
+	return configArg.Network.NetworkConfigDir
 }
 
 // LoadCNIConfsFromDir loads all the CNI configurations from a dir
diff --git a/rootless.md b/rootless.md
index 196ed52c3..22b03e340 100644
--- a/rootless.md
+++ b/rootless.md
@@ -28,9 +28,6 @@ can easily fail
 * Can not use overlayfs driver, but does support fuse-overlayfs
   * Ubuntu supports non root overlay, but no other Linux distros do.
 * Only other supported driver is VFS.
-* No CNI Support
-  * CNI wants to modify IPTables, plus other network manipulation that requires CAP_SYS_ADMIN.
-  * There is potential we could probably do some sort of denylisting of the relevant plugins, and add a new plugin for rootless networking - slirp4netns as one example and there may be others
 * Cannot use ping out of the box.
   * [(Can be fixed by setting sysctl on host)](https://github.com/containers/podman/blob/master/troubleshooting.md#6-rootless-containers-cannot-ping-hosts)
 * Requires new shadow-utils (not found in older (RHEL7/Centos7 distros) Should be fixed in RHEL7.7 release)
diff --git a/test/e2e/common_test.go b/test/e2e/common_test.go
index ed55484e3..b6bbae15b 100644
--- a/test/e2e/common_test.go
+++ b/test/e2e/common_test.go
@@ -245,6 +245,12 @@ func PodmanTestCreateUtil(tempDir string, remote bool) *PodmanTestIntegration {
 	}
 	os.Setenv("DISABLE_HC_SYSTEMD", "true")
 	CNIConfigDir := "/etc/cni/net.d"
+	if rootless.IsRootless() {
+		CNIConfigDir = filepath.Join(os.Getenv("HOME"), ".config/cni/net.d")
+	}
+	if err := os.MkdirAll(CNIConfigDir, 0755); err != nil {
+		panic(err)
+	}
 
 	storageFs := STORAGE_FS
 	if rootless.IsRootless() {
diff --git a/test/e2e/network_create_test.go b/test/e2e/network_create_test.go
index f97e6c1f1..13d515d8e 100644
--- a/test/e2e/network_create_test.go
+++ b/test/e2e/network_create_test.go
@@ -74,7 +74,6 @@ var _ = Describe("Podman network create", func() {
 	)
 
 	BeforeEach(func() {
-		SkipIfRootless()
 		tempdir, err = CreateTempDirInTempDir()
 		if err != nil {
 			os.Exit(1)
@@ -180,6 +179,7 @@ var _ = Describe("Podman network create", func() {
 
 	It("podman network create with name and IPv6 subnet", func() {
 		SkipIfRemote()
+		SkipIfRootless()
 		var (
 			results []network.NcList
 		)
diff --git a/test/e2e/network_test.go b/test/e2e/network_test.go
index 55f12f16a..c35b82fc1 100644
--- a/test/e2e/network_test.go
+++ b/test/e2e/network_test.go
@@ -7,6 +7,7 @@ import (
 	"path/filepath"
 	"strings"
 
+	"github.com/containers/podman/v2/pkg/rootless"
 	. "github.com/containers/podman/v2/test/utils"
 	"github.com/containers/storage/pkg/stringid"
 	. "github.com/onsi/ginkgo"
@@ -32,7 +33,6 @@ var _ = Describe("Podman network", func() {
 	)
 
 	BeforeEach(func() {
-		SkipIfRootless()
 		tempdir, err = CreateTempDirInTempDir()
 		if err != nil {
 			os.Exit(1)
@@ -74,13 +74,12 @@ var _ = Describe("Podman network", func() {
       }
     ]
 }`
-		cniPath = "/etc/cni/net.d"
 	)
 
 	It("podman network list", func() {
 		// Setup, use uuid to prevent conflict with other tests
 		uuid := stringid.GenerateNonCryptoID()
-		secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid))
+		secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid))
 		writeConf([]byte(secondConf), secondPath)
 		defer removeConf(secondPath)
 
@@ -93,7 +92,7 @@ var _ = Describe("Podman network", func() {
 	It("podman network list -q", func() {
 		// Setup, use uuid to prevent conflict with other tests
 		uuid := stringid.GenerateNonCryptoID()
-		secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid))
+		secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid))
 		writeConf([]byte(secondConf), secondPath)
 		defer removeConf(secondPath)
 
@@ -106,7 +105,7 @@ var _ = Describe("Podman network", func() {
 	It("podman network list --filter success", func() {
 		// Setup, use uuid to prevent conflict with other tests
 		uuid := stringid.GenerateNonCryptoID()
-		secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid))
+		secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid))
 		writeConf([]byte(secondConf), secondPath)
 		defer removeConf(secondPath)
 
@@ -119,7 +118,7 @@ var _ = Describe("Podman network", func() {
 	It("podman network list --filter failure", func() {
 		// Setup, use uuid to prevent conflict with other tests
 		uuid := stringid.GenerateNonCryptoID()
-		secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid))
+		secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid))
 		writeConf([]byte(secondConf), secondPath)
 		defer removeConf(secondPath)
 
@@ -138,7 +137,7 @@ var _ = Describe("Podman network", func() {
 	It("podman network rm", func() {
 		// Setup, use uuid to prevent conflict with other tests
 		uuid := stringid.GenerateNonCryptoID()
-		secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid))
+		secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid))
 		writeConf([]byte(secondConf), secondPath)
 		defer removeConf(secondPath)
 
@@ -166,11 +165,16 @@ var _ = Describe("Podman network", func() {
 	It("podman network inspect", func() {
 		// Setup, use uuid to prevent conflict with other tests
 		uuid := stringid.GenerateNonCryptoID()
-		secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid))
+		secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid))
 		writeConf([]byte(secondConf), secondPath)
 		defer removeConf(secondPath)
 
-		session := podmanTest.Podman([]string{"network", "inspect", "podman-integrationtest", "podman"})
+		expectedNetworks := []string{"podman-integrationtest"}
+		if !rootless.IsRootless() {
+			// rootful image contains "podman/cni/87-podman-bridge.conflist" for "podman" network
+			expectedNetworks = append(expectedNetworks, "podman")
+		}
+		session := podmanTest.Podman(append([]string{"network", "inspect"}, expectedNetworks...))
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))
 		Expect(session.IsJSONOutputValid()).To(BeTrue())
@@ -179,7 +183,7 @@ var _ = Describe("Podman network", func() {
 	It("podman network inspect", func() {
 		// Setup, use uuid to prevent conflict with other tests
 		uuid := stringid.GenerateNonCryptoID()
-		secondPath := filepath.Join(cniPath, fmt.Sprintf("%s.conflist", uuid))
+		secondPath := filepath.Join(podmanTest.CNIConfigDir, fmt.Sprintf("%s.conflist", uuid))
 		writeConf([]byte(secondConf), secondPath)
 		defer removeConf(secondPath)