Cirrus: Support special-case modes of testing

Previously libpod CI was fairly straight-forward, run unit and
integration tests in a standard set of 3 VMs.  Off on the side was a
single special case of running tests as an ordinary user.  There is a
desire to stop using the PAPR system to support testing inside of a
container.

Since having two special cases potentially invites more
down the road, make provisions to handle them more gracefully.  This
commit introduces an environment variable:  ``$SPECIALMODE``.  It's
value has the following meanings within the CI scripts:

Mode 'none': Nothing special, business as usual (default)
Mode 'rootless': Rootless testing
Mode 'in_podman': Build container, run integration tests in it.

This will make adding additional special-cases later easier, as well as
extending the special cases in a Matrix across multiple OS's.

Signed-off-by: Chris Evich <cevich@redhat.com>

13 files changed, 134 insertions, 213 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 7bc00dbb5..3516c7d61 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -67,9 +67,9 @@ env:
     RHEL_BASE_IMAGE: "rhel-guest-image-7-6-210-x86-64-qcow2-1548099756"
 
     ####
-    #### Default to NOT running in rootless-testing mode
+    #### Default to NOT operating in any special-case testing mode
     ####
-    ROOTLESS_USER: ""
+    SPECIALMODE: "none"  # don't do anything special
 
     ####
     #### Credentials and other secret-sauces, decrypted at runtime when authorized.
@@ -257,8 +257,8 @@ testing_task:
         master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
 
 
-# This task executes tests as a regular user on a system
-rootless_testing_task:
+# This task executes tests under unique environments/conditions
+special_testing_task:
 
     depends_on:
         - "gating"
@@ -274,15 +274,14 @@ rootless_testing_task:
         image_name: "${FEDORA_CACHE_IMAGE_NAME}"
 
     env:
-        ROOTLESS_USER: "olympiclongjumpingwithjesus"
+        matrix:
+            SPECIALMODE: 'rootless'  # See docs
+            SPECIALMODE: 'in_podman'  # See docs
 
     timeout_in: 120m
 
     setup_environment_script: '$SCRIPT_BASE/setup_environment.sh'
-    rootless_test_script: >-
-        ssh $ROOTLESS_USER@localhost
-        -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no
-        $CIRRUS_WORKING_DIR/$SCRIPT_BASE/rootless_test.sh
+    integration_test_script: '$SCRIPT_BASE/integration_test.sh'
 
     on_failure:
         master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
diff --git a/.papr.yml b/.papr.yml
deleted file mode 100644
index ed20c6039..000000000
--- a/.papr.yml
+++ /dev/null
@@ -1,97 +0,0 @@
-branches:
-  - master
-  - auto
-  - try
-
-host:
-  distro: fedora/28/atomic
-  specs:
-     ram: 8192
-     cpus: 4
-required: true
-timeout: 90m
-
-tests:
-  - rpm-ostree usroverlay && rpm -Uvh https://kojipkgs.fedoraproject.org//packages/podman/0.10.1/1.gite4a1553.fc28/x86_64/podman-0.10.1-1.gite4a1553.fc28.x86_64.rpm
-  - CONTAINER_RUNTIME="podman" sh .papr_prepare.sh
-
-artifacts:
-  - build.log
-
-context: "FAH28 - Containerized (Podman in Podman)"
-
----
-
-host:
-  distro: centos/7/atomic/smoketested
-  specs:
-     ram: 8192
-     cpus: 4
-extra-repos:
-    - name: epel
-      metalink: https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch
-      gpgcheck: 0
-    - name: cri-o
-      baseurl: https://cbs.centos.org/repos/virt7-container-common-candidate/$basearch/os
-      gpgcheck: 0
-
-required: true
-
-timeout: 90m
-
-tests:
-  - CONTAINER_RUNTIME="docker" sh .papr_prepare.sh
-
-artifacts:
-  - build.log
-
-context: "CAH 7-smoketested - Containerized (Podman in Docker)"
-
-#---
-#
-#host:
-#  distro: centos/7/cloud
-#  specs:
-#     ram: 8192
-#     cpus: 4
-#extra-repos:
-#    - name: epel
-#      metalink: https://mirrors.fedoraproject.org/metalink?repo=epel-7&arch=$basearch
-#      gpgcheck: 0
-#    - name: cri-o
-#      baseurl: https://cbs.centos.org/repos/virt7-container-common-candidate/$basearch/os
-#      gpgcheck: 0
-#
-#packages:
-#    - btrfs-progs-devel
-#    - glib2-devel
-#    - glibc-devel
-#    - glibc-static
-#    - git
-#    - go-md2man
-#    - gpgme-devel
-#    - libassuan-devel
-#    - libgpg-error-devel
-#    - libseccomp-devel
-#    - libselinux-devel
-#    - ostree-devel
-#    - pkgconfig
-#    - make
-#    - nc
-#    - go-compilers-golang-compiler
-#    - podman
-#
-#required: true
-#
-#timeout: 90m
-#
-#tests:
-#    - sed 's/^expand-check.*/expand-check=0/g' -i /etc/selinux/semanage.conf
-#    - sh .papr.sh -b -i -t
-#
-#artifacts:
-#  - build.log
-#
-#context: "CentOS 7 Cloud"
-#
-#---
diff --git a/.papr_prepare.sh b/.papr_prepare.sh
deleted file mode 100644
index b93f7b91f..000000000
--- a/.papr_prepare.sh
+++ /dev/null
@@ -1,22 +0,0 @@
-#!/bin/bash
-set -xeuo pipefail
-
-DIST=${DIST:=Fedora}
-CONTAINER_RUNTIME=${CONTAINER_RUNTIME:=docker}
-IMAGE=fedorapodmanbuild
-if [[ ${DIST} != "Fedora" ]]; then
-    IMAGE=centospodmanbuild
-fi
-
-# Since CRIU 3.11 has been pushed to Fedora 28 the checkpoint/restore
-# test cases are actually run. As CRIU uses iptables to lock and unlock
-# the network during checkpoint and restore it needs the following two
-# modules loaded.
-modprobe ip6table_nat || :
-modprobe iptable_nat || :
-
-# Build the test image
-${CONTAINER_RUNTIME} build -t ${IMAGE} -f Dockerfile.${DIST} . 2>build.log
-
-# Run the tests
-${CONTAINER_RUNTIME} run --rm --privileged --net=host -v $PWD:/go/src/github.com/containers/libpod:Z --workdir /go/src/github.com/containers/libpod -e CGROUP_MANAGER=cgroupfs -e STORAGE_OPTIONS="--storage-driver=vfs" -e CRIO_ROOT="/go/src/github.com/containers/libpod" -e PODMAN_BINARY="/usr/bin/podman" -e CONMON_BINARY="/usr/libexec/podman/conmon" -e DIST=$DIST -e CONTAINER_RUNTIME=$CONTAINER_RUNTIME $IMAGE sh ./.papr.sh -b -i -t
diff --git a/Dockerfile.CentOS b/Dockerfile.centos
index 605dc9df4..605dc9df4 100644
--- a/Dockerfile.CentOS
+++ b/Dockerfile.centos
diff --git a/Dockerfile.Fedora b/Dockerfile.fedora
index e38e2e056..e38e2e056 100644
--- a/Dockerfile.Fedora
+++ b/Dockerfile.fedora
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md
index 0dabf5df6..ea358d2d7 100644
--- a/contrib/cirrus/README.md
+++ b/contrib/cirrus/README.md
@@ -63,26 +63,26 @@ task (pass or fail) is set based on the exit status of the last script to execut
    Total execution time is capped at 2-hours (includes all the above)
    but this script normally completes in less than an hour.
 
-### ``rootless_testing`` Task
+### ``special_testing`` Task
+
+This task exercises podman under specialized environments or conditions.
+The specific differences from the ``testing`` task depend upon the
+contents of the ``$SPECIALMODE`` environment variable.
+
+| Value     | Meaning                                                               |
+| rootless  | Setup a regular user to build/run integration tests.                  |
+| in_podman | Setup a container image, build/run integration tests inside container |
 
 ***N/B: Steps below are performed by automation***
 
 1. After `gating` passes, spin up one VM per
-   `matrix: image_name` item. Once accessible, ``ssh``
-   into each VM as the `root` user.
+   `matrix: image_name` item.
+
+2. ``setup_environment.sh``: Mostly the same as
+   in ``testing`` task, then specialized depending on ``$SPECIALMODE``.
+
+3. Which tests and how they execute depends on ``$SPECIALMODE``.
 
-2. ``setup_environment.sh``: Configure root's `.bash_profile`
-   the same as for other tasks.  However, also add a regular
-   user account, chown all the source code to them.  Set up
-   fresh ssh pub/priv. keys for the root user, adding the
-   public part to the user's `authorized_keys` file.
-
-3. As root, call ssh to connect to localhost as the user,
-   and run the ``rootless_test.sh`` script from the source
-   tree.  This is needed so the user has a clean process tree
-   and environment - i.e. without `sudo`, `su`, `runuser`,
-   etc. in the mix.  From here, all testing as the user may
-   be performed.
 
 ### ``optional_testing`` Task
 
diff --git a/.papr.sh b/contrib/cirrus/container_test.sh
index c5aada904..e6c1a3a47 100644
--- a/.papr.sh
+++ b/contrib/cirrus/container_test.sh
@@ -1,7 +1,7 @@
 #!/bin/bash
 set -xeuo pipefail
 
-export GOPATH=/go
+export GOPATH=/var/tmp/go
 export PATH=$HOME/gopath/bin:$PATH:$GOPATH/bin
 export GOSRC=$GOPATH/src/github.com/containers/libpod
 
@@ -125,7 +125,7 @@ fi
 # Run integration tests
 if [ $integrationtest -eq 1 ]; then
     make TAGS="${TAGS}" test-binaries
-    make varlink_generate GOPATH=/go
-    make ginkgo GOPATH=/go $INTEGRATION_TEST_ENVS
-    make ginkgo-remote GOPATH=/go $INTEGRATION_TEST_ENVS
+    make varlink_generate
+    make ginkgo $INTEGRATION_TEST_ENVS
+    make ginkgo-remote $INTEGRATION_TEST_ENVS
 fi
diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh
index 58c8af289..8a2507f38 100755
--- a/contrib/cirrus/integration_test.sh
+++ b/contrib/cirrus/integration_test.sh
@@ -5,33 +5,64 @@ source $(dirname $0)/lib.sh
 
 req_env_var "
 GOSRC $GOSRC
+SCRIPT_BASE $SCRIPT_BASE
 OS_RELEASE_ID $OS_RELEASE_ID
 OS_RELEASE_VER $OS_RELEASE_VER
+CONTAINER_RUNTIME $CONTAINER_RUNTIME
 "
 
-record_timestamp "integration test start"
+exit_handler() {
+    set +ex
+    record_timestamp "integration test end"
+}
+trap exit_handler EXIT
 
-clean_env
+record_timestamp "integration test start"
 
-set -x
 cd "$GOSRC"
-case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
-    ubuntu-18)
-        make install PREFIX=/usr ETCDIR=/etc
-        make test-binaries
-        SKIP_USERNS=1 make localintegration
-        ;;
-    fedora-29) ;&  # Continue to the next item
-    fedora-28) ;&
-    centos-7) ;&
-    rhel-7)
-        make install PREFIX=/usr ETCDIR=/etc
-        make podman-remote
-        install bin/podman-remote /usr/bin
-        make test-binaries
-        make localintegration
-        ;;
-    *) bad_os_id_ver ;;
-esac
-
-record_timestamp "integration test end"
+
+if [[ "$SPECIALMODE" == "in_podman" ]]
+then
+    set -x
+    ${CONTAINER_RUNTIME} run --rm --privileged --net=host \
+        -v $GOSRC:$GOSRC:Z \
+        --workdir $GOSRC \
+        -e "CGROUP_MANAGER=cgroupfs" \
+        -e "STORAGE_OPTIONS=--storage-driver=vfs" \
+        -e "CRIO_ROOT=$GOSRC" \
+        -e "PODMAN_BINARY=/usr/bin/podman" \
+        -e "CONMON_BINARY=/usr/libexec/podman/conmon" \
+        -e "DIST=$OS_RELEASE_ID" \
+        -e "CONTAINER_RUNTIME=$CONTAINER_RUNTIME" \
+        ${OS_RELEASE_ID}podmanbuild bash $GOSRC/$SCRIPT_BASE/container_test.sh -b -i -t
+
+    exit $?
+elif [[ "$SPECIALMODE" == "rootless" ]]
+then
+    req_env_var "ROOTLESS_USER $ROOTLESS_USER"
+    set -x
+    ssh $ROOTLESS_USER@localhost \
+                -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \
+                $GOSRC/$SCRIPT_BASE/rootless_test.sh
+    exit $?
+else
+    set -x
+    make
+    make install PREFIX=/usr ETCDIR=/etc
+    make test-binaries
+    clean_env
+
+    case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
+        ubuntu-18) ;;
+        fedora-29) ;&  # Continue to the next item
+        fedora-28) ;&
+        centos-7) ;&
+        rhel-7)
+            make podman-remote
+            install bin/podman-remote /usr/bin
+            ;;
+        *) bad_os_id_ver ;;
+    esac
+    make localintegration
+    exit $?
+fi
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index e941610e2..6c45b2c5d 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -18,6 +18,8 @@ CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-DEADBEEF}  # a human
 CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-HEAD}
 CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-FETCH_HEAD}
 TIMESTAMPS_FILEPATH="${TIMESTAMPS_FILEPATH:-/var/tmp/timestamps}"
+SPECIALMODE="${SPECIALMODE:-none}"
+export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
 
 if ! [[ "$PATH" =~ "/usr/local/bin" ]]
 then
@@ -81,6 +83,7 @@ CIRRUS_USER_COLLABORATOR $CIRRUS_USER_COLLABORATOR
 CIRRUS_USER_PERMISSION $CIRRUS_USER_PERMISSION
 CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR
 CIRRUS_HTTP_CACHE_HOST $CIRRUS_HTTP_CACHE_HOST
+SPECIALMODE $SPECIALMODE
 $(go env)
 PACKER_BUILDS $PACKER_BUILDS
     " | while read NAME VALUE
@@ -127,15 +130,6 @@ bad_os_id_ver() {
     exit 42
 }
 
-run_rootless() {
-    if [[ -z "$ROOTLESS_USER" ]]
-    then
-        return 1
-    else
-        return 0
-    fi
-}
-
 stub() {
     echo "STUB: Pretending to do $1"
 }
@@ -179,6 +173,13 @@ setup_rootless() {
         return 0
     fi
 
+    # Only do this once
+    cd $GOSRC
+    make install.catatonit
+    go get github.com/onsi/ginkgo/ginkgo
+    go get github.com/onsi/gomega/...
+    dnf -y update runc
+
     # Guarantee independence from specific values
     ROOTLESS_UID=$[RANDOM+1000]
     ROOTLESS_GID=$[RANDOM+1000]
diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh
index d0e2ceb95..88b38f45b 100755
--- a/contrib/cirrus/rootless_test.sh
+++ b/contrib/cirrus/rootless_test.sh
@@ -12,9 +12,9 @@ OS_RELEASE_ID $OS_RELEASE_ID
 OS_RELEASE_VER $OS_RELEASE_VER
 "
 
-if ! run_rootless
+if [[ "$UID" == "0" ]]
 then
-    echo "Error: Expected rootless env. vars not set or empty"
+    echo "Error: Expected to be running as a regular user"
     exit 1
 fi
 
@@ -24,16 +24,9 @@ echo "Hello, my name is $USER and I live in $PWD can I be your friend?"
 record_timestamp "rootless test start"
 
 cd "$GOSRC"
-case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
-    ubuntu-18) ;&  # Continue to the next item
-    fedora-29) ;&
-    fedora-28)
-        make
-        make varlink_generate
-        make test-binaries
-        make ginkgo
-        ;;
-    *) bad_os_id_ver ;;
-esac
+make
+make varlink_generate
+make test-binaries
+make ginkgo
 
 record_timestamp "rootless test end"
diff --git a/contrib/cirrus/setup_container_environment.sh b/contrib/cirrus/setup_container_environment.sh
new file mode 100755
index 000000000..23df4fe8b
--- /dev/null
+++ b/contrib/cirrus/setup_container_environment.sh
@@ -0,0 +1,23 @@
+#!/bin/bash
+set -e
+
+source $(dirname $0)/lib.sh
+
+req_env_var "
+GOSRC $GOSRC
+OS_RELEASE_ID $OS_RELEASE_ID
+CONTAINER_RUNTIME $CONTAINER_RUNTIME
+"
+
+DIST=$OS_RELEASE_ID
+IMAGE=${DIST}podmanbuild
+
+# Since CRIU 3.11 has been pushed to Fedora 28 the checkpoint/restore
+# test cases are actually run. As CRIU uses iptables to lock and unlock
+# the network during checkpoint and restore it needs the following two
+# modules loaded.
+modprobe ip6table_nat || :
+modprobe iptable_nat || :
+
+# Build the test image
+${CONTAINER_RUNTIME} build -t ${IMAGE} -f Dockerfile.${DIST} .
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 96d0e1b55..55706954e 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -43,7 +43,6 @@ then
         "export OS_RELEASE_ID=\"$(os_release_id)\"" \
         "export OS_RELEASE_VER=\"$(os_release_ver)\"" \
         "export OS_REL_VER=\"$(os_release_id)-$(os_release_ver)\"" \
-        "export ROOTLESS_USER=$ROOTLESS_USER" \
         "export BUILT_IMAGE_SUFFIX=\"-$CIRRUS_REPO_NAME-${CIRRUS_CHANGE_IN_REPO:0:8}\"" \
         "export GOPATH=\"/var/tmp/go\"" \
         'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \
@@ -75,14 +74,17 @@ then
     # Reload to incorporate any changes from above
     source "$SCRIPT_BASE/lib.sh"
 
-    if run_rootless
-    then
-        setup_rootless
-        make install.catatonit
-        go get github.com/onsi/ginkgo/ginkgo
-        go get github.com/onsi/gomega/...
-        dnf -y update runc
-    fi
+    case "$SPECIALMODE" in
+        rootless)
+            X=$(echo "export ROOTLESS_USER='some${RANDOM}dude'" | \
+                tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X"
+            setup_rootless
+            ;;
+        in_podman)  # Assumed to be Fedora
+            dnf install -y podman buildah
+            $SCRIPT_BASE/setup_container_environment.sh
+            ;;
+    esac
 fi
 
 show_env_vars
diff --git a/contrib/cirrus/unit_test.sh b/contrib/cirrus/unit_test.sh
index fd9e82509..4ace19d10 100755
--- a/contrib/cirrus/unit_test.sh
+++ b/contrib/cirrus/unit_test.sh
@@ -15,17 +15,8 @@ clean_env
 
 set -x
 cd "$GOSRC"
-case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in
-    ubuntu-18) ;&  # Continue to the next item
-    fedora-29) ;&
-    fedora-28) ;&
-    centos-7) ;&
-    rhel-7)
-        make install.tools
-        make localunit
-        make
-        ;;
-    *) bad_os_id_ver ;;
-esac
+make install.tools
+make localunit
+make
 
 record_timestamp "unit test end"