Initial checkin from CRI-O repo

Signed-off-by: Matthew Heon <matthew.heon@gmail.com>

1 files changed, 493 insertions, 0 deletions

diff --git a/test/helpers.bash b/test/helpers.bash
new file mode 100644
index 000000000..22955d333
--- /dev/null
+++ b/test/helpers.bash
@@ -0,0 +1,493 @@
+#!/bin/bash
+
+# Root directory of integration tests.
+INTEGRATION_ROOT=$(dirname "$(readlink -f "$BASH_SOURCE")")
+
+# Test data path.
+TESTDATA="${INTEGRATION_ROOT}/testdata"
+
+# Root directory of the repository.
+CRIO_ROOT=${CRIO_ROOT:-$(cd "$INTEGRATION_ROOT/../.."; pwd -P)}
+
+# Path of the crio binary.
+CRIO_BINARY=${CRIO_BINARY:-${CRIO_ROOT}/cri-o/bin/crio}
+# Path of the crictl binary.
+CRICTL_PATH=$(command -v crictl || true)
+CRICTL_BINARY=${CRICTL_PATH:-/usr/bin/crictl}
+# Path to kpod binary.
+KPOD_BINARY=${KPOD_BINARY:-${CRIO_ROOT}/cri-o/bin/kpod}
+# Path of the conmon binary.
+CONMON_BINARY=${CONMON_BINARY:-${CRIO_ROOT}/cri-o/bin/conmon}
+# Path of the pause binary.
+PAUSE_BINARY=${PAUSE_BINARY:-${CRIO_ROOT}/cri-o/bin/pause}
+# Path of the default seccomp profile.
+SECCOMP_PROFILE=${SECCOMP_PROFILE:-${CRIO_ROOT}/cri-o/seccomp.json}
+# Name of the default apparmor profile.
+APPARMOR_PROFILE=${APPARMOR_PROFILE:-crio-default}
+# Runtime
+RUNTIME=${RUNTIME:-runc}
+RUNTIME_PATH=$(command -v $RUNTIME || true)
+RUNTIME_BINARY=${RUNTIME_PATH:-/usr/local/sbin/runc}
+# Path of the apparmor_parser binary.
+APPARMOR_PARSER_BINARY=${APPARMOR_PARSER_BINARY:-/sbin/apparmor_parser}
+# Path of the apparmor profile for test.
+APPARMOR_TEST_PROFILE_PATH=${APPARMOR_TEST_PROFILE_PATH:-${TESTDATA}/apparmor_test_deny_write}
+# Path of the apparmor profile for unloading crio-default.
+FAKE_CRIO_DEFAULT_PROFILE_PATH=${FAKE_CRIO_DEFAULT_PROFILE_PATH:-${TESTDATA}/fake_crio_default}
+# Name of the apparmor profile for test.
+APPARMOR_TEST_PROFILE_NAME=${APPARMOR_TEST_PROFILE_NAME:-apparmor-test-deny-write}
+# Path of boot config.
+BOOT_CONFIG_FILE_PATH=${BOOT_CONFIG_FILE_PATH:-/boot/config-`uname -r`}
+# Path of apparmor parameters file.
+APPARMOR_PARAMETERS_FILE_PATH=${APPARMOR_PARAMETERS_FILE_PATH:-/sys/module/apparmor/parameters/enabled}
+# Path of the bin2img binary.
+BIN2IMG_BINARY=${BIN2IMG_BINARY:-${CRIO_ROOT}/cri-o/test/bin2img/bin2img}
+# Path of the copyimg binary.
+COPYIMG_BINARY=${COPYIMG_BINARY:-${CRIO_ROOT}/cri-o/test/copyimg/copyimg}
+# Path of tests artifacts.
+ARTIFACTS_PATH=${ARTIFACTS_PATH:-${CRIO_ROOT}/cri-o/.artifacts}
+# Path of the checkseccomp binary.
+CHECKSECCOMP_BINARY=${CHECKSECCOMP_BINARY:-${CRIO_ROOT}/cri-o/test/checkseccomp/checkseccomp}
+# XXX: This is hardcoded inside cri-o at the moment.
+DEFAULT_LOG_PATH=/var/log/crio/pods
+# Cgroup manager to be used
+CGROUP_MANAGER=${CGROUP_MANAGER:-cgroupfs}
+# Image volumes handling
+IMAGE_VOLUMES=${IMAGE_VOLUMES:-mkdir}
+# Container pids limit
+PIDS_LIMIT=${PIDS_LIMIT:-1024}
+# Log size max limit
+LOG_SIZE_MAX_LIMIT=${LOG_SIZE_MAX_LIMIT:--1}
+
+TESTDIR=$(mktemp -d)
+
+# kpod pull needs a configuration file for shortname pulls
+export REGISTRIES_CONFIG_PATH="$INTEGRATION_ROOT/registries.conf"
+
+# Setup default hooks dir
+HOOKSDIR=$TESTDIR/hooks
+mkdir ${HOOKSDIR}
+HOOKS_OPTS="--hooks-dir-path=$HOOKSDIR"
+
+# Setup default secrets mounts
+MOUNT_PATH="$TESTDIR/secrets"
+mkdir ${MOUNT_PATH}
+MOUNT_FILE="${MOUNT_PATH}/test.txt"
+touch ${MOUNT_FILE}
+echo "Testing secrets mounts!" > ${MOUNT_FILE}
+
+DEFAULT_MOUNTS_OPTS="--default-mounts=${MOUNT_PATH}:/container/path1"
+
+# We may need to set some default storage options.
+case "$(stat -f -c %T ${TESTDIR})" in
+    aufs)
+        # None of device mapper, overlay, or aufs can be used dependably over aufs, and of course btrfs and zfs can't,
+        # and we have to explicitly specify the "vfs" driver in order to use it, so do that now.
+        STORAGE_OPTIONS=${STORAGE_OPTIONS:---storage-driver vfs}
+        ;;
+esac
+
+if [ -e /usr/sbin/selinuxenabled ] && /usr/sbin/selinuxenabled; then
+    . /etc/selinux/config
+    filelabel=$(awk -F'"' '/^file.*=.*/ {print $2}' /etc/selinux/${SELINUXTYPE}/contexts/lxc_contexts)
+    chcon -R ${filelabel} $TESTDIR
+fi
+CRIO_SOCKET="$TESTDIR/crio.sock"
+CRIO_CONFIG="$TESTDIR/crio.conf"
+CRIO_CNI_CONFIG="$TESTDIR/cni/net.d/"
+CRIO_CNI_PLUGIN=${CRIO_CNI_PLUGIN:-/opt/cni/bin/}
+POD_CIDR="10.88.0.0/16"
+POD_CIDR_MASK="10.88.*.*"
+
+KPOD_OPTIONS="--root $TESTDIR/crio $STORAGE_OPTIONS --runroot $TESTDIR/crio-run --runtime ${RUNTIME_BINARY}"
+
+cp "$CONMON_BINARY" "$TESTDIR/conmon"
+
+PATH=$PATH:$TESTDIR
+
+# Make sure we have a copy of the redis:alpine image.
+if ! [ -d "$ARTIFACTS_PATH"/redis-image ]; then
+    mkdir -p "$ARTIFACTS_PATH"/redis-image
+    if ! "$COPYIMG_BINARY" --import-from=docker://redis:alpine --export-to=dir:"$ARTIFACTS_PATH"/redis-image --signature-policy="$INTEGRATION_ROOT"/policy.json ; then
+        echo "Error pulling docker://redis"
+        rm -fr "$ARTIFACTS_PATH"/redis-image
+        exit 1
+    fi
+fi
+
+# TODO: remove the code below for pulling redis:alpine using a canonical reference once
+#       https://github.com/kubernetes-incubator/cri-o/issues/531 is complete and we can
+#       pull the image using a tagged reference and then subsequently find the image without
+#       having to explicitly record the canonical reference as one of the image's names
+if ! [ -d "$ARTIFACTS_PATH"/redis-image-digest ]; then
+    mkdir -p "$ARTIFACTS_PATH"/redis-image-digest
+    if ! "$COPYIMG_BINARY" --import-from=docker://redis@sha256:03789f402b2ecfb98184bf128d180f398f81c63364948ff1454583b02442f73b --export-to=dir:"$ARTIFACTS_PATH"/redis-image-digest --signature-policy="$INTEGRATION_ROOT"/policy.json ; then
+        echo "Error pulling docker://redis@sha256:03789f402b2ecfb98184bf128d180f398f81c63364948ff1454583b02442f73b"
+        rm -fr "$ARTIFACTS_PATH"/redis-image-digest
+        exit 1
+    fi
+fi
+
+# Make sure we have a copy of the runcom/stderr-test image.
+if ! [ -d "$ARTIFACTS_PATH"/stderr-test ]; then
+    mkdir -p "$ARTIFACTS_PATH"/stderr-test
+    if ! "$COPYIMG_BINARY" --import-from=docker://runcom/stderr-test:latest --export-to=dir:"$ARTIFACTS_PATH"/stderr-test --signature-policy="$INTEGRATION_ROOT"/policy.json ; then
+        echo "Error pulling docker://stderr-test"
+        rm -fr "$ARTIFACTS_PATH"/stderr-test
+        exit 1
+    fi
+fi
+
+# Make sure we have a copy of the busybox:latest image.
+if ! [ -d "$ARTIFACTS_PATH"/busybox-image ]; then
+    mkdir -p "$ARTIFACTS_PATH"/busybox-image
+    if ! "$COPYIMG_BINARY" --import-from=docker://busybox --export-to=dir:"$ARTIFACTS_PATH"/busybox-image --signature-policy="$INTEGRATION_ROOT"/policy.json ; then
+        echo "Error pulling docker://busybox"
+        rm -fr "$ARTIFACTS_PATH"/busybox-image
+        exit 1
+    fi
+fi
+
+# Make sure we have a copy of the mrunalp/oom:latest image.
+if ! [ -d "$ARTIFACTS_PATH"/oom-image ]; then
+    mkdir -p "$ARTIFACTS_PATH"/oom-image
+    if ! "$COPYIMG_BINARY" --import-from=docker://mrunalp/oom --export-to=dir:"$ARTIFACTS_PATH"/oom-image --signature-policy="$INTEGRATION_ROOT"/policy.json ; then
+        echo "Error pulling docker://mrunalp/oom"
+        rm -fr "$ARTIFACTS_PATH"/oom-image
+        exit 1
+    fi
+fi
+
+# Make sure we have a copy of the mrunalp/image-volume-test:latest image.
+if ! [ -d "$ARTIFACTS_PATH"/image-volume-test-image ]; then
+    mkdir -p "$ARTIFACTS_PATH"/image-volume-test-image
+    if ! "$COPYIMG_BINARY" --import-from=docker://mrunalp/image-volume-test --export-to=dir:"$ARTIFACTS_PATH"/image-volume-test-image --signature-policy="$INTEGRATION_ROOT"/policy.json ; then
+        echo "Error pulling docker://mrunalp/image-volume-test-image"
+        rm -fr "$ARTIFACTS_PATH"/image-volume-test-image
+        exit 1
+    fi
+fi
+# Run crio using the binary specified by $CRIO_BINARY.
+# This must ONLY be run on engines created with `start_crio`.
+function crio() {
+	"$CRIO_BINARY" --listen "$CRIO_SOCKET" "$@"
+}
+
+# DEPRECATED
+OCIC_BINARY=${OCIC_BINARY:-${CRIO_ROOT}/cri-o/bin/crioctl}
+# Run crioctl using the binary specified by $OCIC_BINARY.
+function crioctl() {
+	"$OCIC_BINARY" --connect "$CRIO_SOCKET" "$@"
+}
+
+# Run crictl using the binary specified by $CRICTL_BINARY.
+function crictl() {
+	"$CRICTL_BINARY" -r "$CRIO_SOCKET" -i "$CRIO_SOCKET" "$@"
+}
+
+# Communicate with Docker on the host machine.
+# Should rarely use this.
+function docker_host() {
+	command docker "$@"
+}
+
+# Retry a command $1 times until it succeeds. Wait $2 seconds between retries.
+function retry() {
+	local attempts=$1
+	shift
+	local delay=$1
+	shift
+	local i
+
+	for ((i=0; i < attempts; i++)); do
+		run "$@"
+		if [[ "$status" -eq 0 ]] ; then
+			return 0
+		fi
+		sleep $delay
+	done
+
+	echo "Command \"$@\" failed $attempts times. Output: $output"
+	false
+}
+
+# Waits until the given crio becomes reachable.
+function wait_until_reachable() {
+	retry 15 1 crictl status
+}
+
+# Start crio.
+function start_crio() {
+	if [[ -n "$1" ]]; then
+		seccomp="$1"
+	else
+		seccomp="$SECCOMP_PROFILE"
+	fi
+
+	if [[ -n "$2" ]]; then
+		apparmor="$2"
+	else
+		apparmor="$APPARMOR_PROFILE"
+	fi
+
+	# Don't forget: bin2img, copyimg, and crio have their own default drivers, so if you override any, you probably need to override them all
+	if ! [ "$3" = "--no-pause-image" ] ; then
+		"$BIN2IMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --source-binary "$PAUSE_BINARY"
+	fi
+	"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=docker.io/library/redis:alpine --import-from=dir:"$ARTIFACTS_PATH"/redis-image --signature-policy="$INTEGRATION_ROOT"/policy.json
+# TODO: remove the code below for copying redis:alpine in using a canonical reference once
+#       https://github.com/kubernetes-incubator/cri-o/issues/531 is complete and we can
+#       copy the image using a tagged reference and then subsequently find the image without
+#       having to explicitly record the canonical reference as one of the image's names
+	"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=docker.io/library/redis@sha256:03789f402b2ecfb98184bf128d180f398f81c63364948ff1454583b02442f73b --import-from=dir:"$ARTIFACTS_PATH"/redis-image-digest --signature-policy="$INTEGRATION_ROOT"/policy.json
+	"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=mrunalp/oom --import-from=dir:"$ARTIFACTS_PATH"/oom-image --signature-policy="$INTEGRATION_ROOT"/policy.json
+	"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=docker.io/library/mrunalp/image-volume-test --import-from=dir:"$ARTIFACTS_PATH"/image-volume-test-image --signature-policy="$INTEGRATION_ROOT"/policy.json
+	"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=docker.io/library/busybox:latest --import-from=dir:"$ARTIFACTS_PATH"/busybox-image --signature-policy="$INTEGRATION_ROOT"/policy.json
+	"$COPYIMG_BINARY" --root "$TESTDIR/crio" $STORAGE_OPTIONS --runroot "$TESTDIR/crio-run" --image-name=docker.io/library/runcom/stderr-test:latest --import-from=dir:"$ARTIFACTS_PATH"/stderr-test --signature-policy="$INTEGRATION_ROOT"/policy.json
+	"$CRIO_BINARY" ${DEFAULT_MOUNTS_OPTS} ${HOOKS_OPTS} --conmon "$CONMON_BINARY" --listen "$CRIO_SOCKET" --cgroup-manager "$CGROUP_MANAGER" --registry "docker.io" --runtime "$RUNTIME_BINARY" --root "$TESTDIR/crio" --runroot "$TESTDIR/crio-run" $STORAGE_OPTIONS --seccomp-profile "$seccomp" --apparmor-profile "$apparmor" --cni-config-dir "$CRIO_CNI_CONFIG" --cni-plugin-dir "$CRIO_CNI_PLUGIN" --signature-policy "$INTEGRATION_ROOT"/policy.json --image-volumes "$IMAGE_VOLUMES" --pids-limit "$PIDS_LIMIT" --log-size-max "$LOG_SIZE_MAX_LIMIT" --config /dev/null config >$CRIO_CONFIG
+
+	# Prepare the CNI configuration files, we're running with non host networking by default
+	if [[ -n "$4" ]]; then
+		netfunc="$4"
+	else
+		netfunc="prepare_network_conf"
+	fi
+	${netfunc} $POD_CIDR
+
+	"$CRIO_BINARY" --log-level debug --config "$CRIO_CONFIG" & CRIO_PID=$!
+	wait_until_reachable
+
+	run crictl inspecti redis:alpine
+	if [ "$status" -ne 0 ] ; then
+		crictl pull redis:alpine
+	fi
+	REDIS_IMAGEID=$(crictl inspecti redis:alpine | head -1 | sed -e "s/ID: //g")
+	run crictl inspecti redis@sha256:03789f402b2ecfb98184bf128d180f398f81c63364948ff1454583b02442f73b
+	if [ "$status" -ne 0 ] ; then
+		crictl pull redis@sha256:03789f402b2ecfb98184bf128d180f398f81c63364948ff1454583b02442f73b
+	fi
+	REDIS_IMAGEID_DIGESTED=$(crictl inspecti redis@sha256:03789f402b2ecfb98184bf128d180f398f81c63364948ff1454583b02442f73b | head -1 | sed -e "s/ID: //g")
+	run crictl inspecti mrunalp/oom
+	if [ "$status" -ne 0 ] ; then
+		  crictl pull mrunalp/oom
+	fi
+	OOM_IMAGEID=$(crictl inspecti mrunalp/oom | head -1 | sed -e "s/ID: //g")
+	run crioctl image status --id=runcom/stderr-test
+	if [ "$status" -ne 0 ] ; then
+		crictl pull runcom/stderr-test:latest
+	fi
+	STDERR_IMAGEID=$(crictl inspecti runcom/stderr-test | head -1 | sed -e "s/ID: //g")
+	run crictl inspecti busybox
+	if [ "$status" -ne 0 ] ; then
+		crictl pull busybox:latest
+	fi
+	BUSYBOX_IMAGEID=$(crictl inspecti busybox | head -1 | sed -e "s/ID: //g")
+	run crictl inspecti mrunalp/image-volume-test
+	if [ "$status" -ne 0 ] ; then
+		  crictl pull mrunalp/image-volume-test:latest
+	fi
+	VOLUME_IMAGEID=$(crictl inspecti mrunalp/image-volume-test | head -1 | sed -e "s/ID: //g")
+}
+
+function cleanup_ctrs() {
+	run crictl ps --quiet
+	if [ "$status" -eq 0 ]; then
+		if [ "$output" != "" ]; then
+			printf '%s\n' "$output" | while IFS= read -r line
+			do
+			   crictl stop "$line"
+			   crictl rm "$line"
+			done
+		fi
+	fi
+	rm -f /run/hookscheck
+}
+
+function cleanup_images() {
+	run crictl images --quiet
+	if [ "$status" -eq 0 ]; then
+		if [ "$output" != "" ]; then
+			printf '%s\n' "$output" | while IFS= read -r line
+			do
+			   crictl rmi "$line"
+			done
+		fi
+	fi
+}
+
+function cleanup_pods() {
+	run crictl sandboxes --quiet
+	if [ "$status" -eq 0 ]; then
+		if [ "$output" != "" ]; then
+			printf '%s\n' "$output" | while IFS= read -r line
+			do
+			   crictl stops "$line"
+			   crictl rms "$line"
+			done
+		fi
+	fi
+}
+
+# Stop crio.
+function stop_crio() {
+	if [ "$CRIO_PID" != "" ]; then
+		kill "$CRIO_PID" >/dev/null 2>&1
+		wait "$CRIO_PID"
+		rm -f "$CRIO_CONFIG"
+	fi
+
+	cleanup_network_conf
+}
+
+function restart_crio() {
+	if [ "$CRIO_PID" != "" ]; then
+		kill "$CRIO_PID" >/dev/null 2>&1
+		wait "$CRIO_PID"
+		start_crio
+	else
+		echo "you must start crio first"
+		exit 1
+	fi
+}
+
+function cleanup_test() {
+	rm -rf "$TESTDIR"
+}
+
+
+function load_apparmor_profile() {
+	"$APPARMOR_PARSER_BINARY" -r "$1"
+}
+
+function remove_apparmor_profile() {
+	"$APPARMOR_PARSER_BINARY" -R "$1"
+}
+
+function is_seccomp_enabled() {
+	if ! "$CHECKSECCOMP_BINARY" ; then
+		echo 0
+		return
+	fi
+	echo 1
+}
+
+function is_apparmor_enabled() {
+	if [[ -f "$APPARMOR_PARAMETERS_FILE_PATH" ]]; then
+		out=$(cat "$APPARMOR_PARAMETERS_FILE_PATH")
+		if [[ "$out" =~ "Y" ]]; then
+			echo 1
+			return
+		fi
+	fi
+	echo 0
+}
+
+function prepare_network_conf() {
+	mkdir -p $CRIO_CNI_CONFIG
+	cat >$CRIO_CNI_CONFIG/10-crio.conf <<-EOF
+{
+    "cniVersion": "0.2.0",
+    "name": "crionet",
+    "type": "bridge",
+    "bridge": "cni0",
+    "isGateway": true,
+    "ipMasq": true,
+    "ipam": {
+        "type": "host-local",
+        "subnet": "$1",
+        "routes": [
+            { "dst": "0.0.0.0/0"  }
+        ]
+    }
+}
+EOF
+
+	cat >$CRIO_CNI_CONFIG/99-loopback.conf <<-EOF
+{
+    "cniVersion": "0.2.0",
+    "type": "loopback"
+}
+EOF
+
+	echo 0
+}
+
+function prepare_plugin_test_args_network_conf() {
+	mkdir -p $CRIO_CNI_CONFIG
+	cat >$CRIO_CNI_CONFIG/10-plugin-test-args.conf <<-EOF
+{
+    "cniVersion": "0.2.0",
+    "name": "crionet_test_args",
+    "type": "bridge-custom",
+    "bridge": "cni0",
+    "isGateway": true,
+    "ipMasq": true,
+    "ipam": {
+        "type": "host-local",
+        "subnet": "$1",
+        "routes": [
+            { "dst": "0.0.0.0/0"  }
+        ]
+    }
+}
+EOF
+
+	echo 0
+}
+
+function check_pod_cidr() {
+	run crioctl ctr execsync --id $1 ip addr show dev eth0 scope global 2>&1
+	echo "$output"
+	[ "$status" -eq 0  ]
+	[[ "$output" =~ $POD_CIDR_MASK  ]]
+}
+
+function parse_pod_ip() {
+	for arg
+	do
+		cidr=`echo "$arg" | grep $POD_CIDR_MASK`
+		if [ "$cidr" == "$arg" ]
+		then
+			echo `echo "$arg" | sed "s/\/[0-9][0-9]//"`
+		fi
+	done
+}
+
+function get_host_ip() {
+	gateway_dev=`ip -o route show default 0.0.0.0/0 | sed 's/.*dev \([^[:space:]]*\).*/\1/'`
+	[ "$gateway_dev" ]
+	host_ip=`ip -o -4 addr show dev $gateway_dev scope global | sed 's/.*inet \([0-9.]*\).*/\1/'`
+}
+
+function ping_pod() {
+	inet=`crioctl ctr execsync --id $1 ip addr show dev eth0 scope global 2>&1 | grep inet`
+
+	IFS=" "
+	ip=`parse_pod_ip $inet`
+
+	ping -W 1 -c 5 $ip
+
+	echo $?
+}
+
+function ping_pod_from_pod() {
+	inet=`crioctl ctr execsync --id $1 ip addr show dev eth0 scope global 2>&1 | grep inet`
+
+	IFS=" "
+	ip=`parse_pod_ip $inet`
+
+	run crioctl ctr execsync --id $2 ping -W 1 -c 2 $ip
+	echo "$output"
+	[ "$status" -eq 0   ]
+}
+
+
+function cleanup_network_conf() {
+	rm -rf $CRIO_CNI_CONFIG
+
+	echo 0
+}
+
+function temp_sandbox_conf() {
+	sed -e s/\"namespace\":.*/\"namespace\":\ \"$1\",/g "$TESTDATA"/sandbox_config.json > $TESTDIR/sandbox_config_$1.json
+}