summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChris Evich <cevich@redhat.com>2019-03-13 15:13:17 -0400
committerChris Evich <cevich@redhat.com>2019-05-21 08:44:01 -0400
commit51a95e6ef1b4616f308d0a7a5507c2ece74a9f0f (patch)
treefc90cfd09731d180663cb47c38cf09de402d96ac
parent191a08ae43fcb2b846e6118073f852e4eb875d5d (diff)
downloadpodman-51a95e6ef1b4616f308d0a7a5507c2ece74a9f0f.tar.gz
podman-51a95e6ef1b4616f308d0a7a5507c2ece74a9f0f.tar.bz2
podman-51a95e6ef1b4616f308d0a7a5507c2ece74a9f0f.zip
Cirrus: Run tests on test-built cache-images
Also, add jq and catatonit installs to images. Signed-off-by: Chris Evich <cevich@redhat.com>
-rw-r--r--.cirrus.yml37
-rw-r--r--contrib/cirrus/README.md51
-rw-r--r--contrib/cirrus/lib.sh52
-rw-r--r--contrib/cirrus/packer/centos_setup.sh11
-rw-r--r--contrib/cirrus/packer/fedora_setup.sh7
-rw-r--r--contrib/cirrus/packer/rhel_setup.sh14
-rw-r--r--contrib/cirrus/packer/ubuntu_setup.sh4
-rwxr-xr-xcontrib/cirrus/resetup_and_run_rootless.sh32
-rwxr-xr-xcontrib/cirrus/rootless_test.sh12
-rwxr-xr-xcontrib/cirrus/setup_and_run_rootless.sh26
-rwxr-xr-xcontrib/cirrus/setup_environment.sh1
11 files changed, 148 insertions, 99 deletions
diff --git a/.cirrus.yml b/.cirrus.yml
index 6035284c8..08875a354 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -26,6 +26,7 @@ env:
# Command to prefix every output line with a timestamp
# (can't do inline awk script, Cirrus-CI or YAML mangles quoting)
TIMESTAMP: "awk --file ${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/timestamp.awk"
+ BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}"
####
#### Cache-image names to test with
@@ -89,22 +90,6 @@ env:
# Name where this repositories cloud resources are located
GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
- # Space separated list of environment variables to unset before testing
- UNSET_ENV_VARS: >-
- GCP_PROJECT_ID GCE_SSH_USERNAME SERVICE_ACCOUNT RHSM_COMMAND BUILT_IMAGE_SUFFIX
- IRCID RHEL_BASE_IMAGE FAH_BASE_IMAGE FEDORA_BASE_IMAGE CENTOS_BASE_IMAGE
- UBUNTU_BASE_IMAGE PACKER_VER PACKER_BUILDS RUNC_COMMIT CRIU_COMMIT
- CRIO_COMMIT CNI_COMMIT FEDORA_CNI_COMMIT PACKER_BASE SCRIPT_BASE
- CIRRUS_SHELL CIRRUS_WORKING_DIR ENVLIB BUILT_IMAGE_SUFFIX CIRRUS_CI
- CI_NODE_INDEX CI_NODE_TOTAL CIRRUS_BASE_BRANCH CIRRUS_BASE_SHA
- CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO CIRRUS_CLONE_DEPTH
- CIRRUS_COMMIT_MESSAGE CIRRUS_CHANGE_MESSAGE CIRRUS_REPO_CLONE_HOST
- CIRRUS_DEFAULT_BRANCH CIRRUS_PR CIRRUS_TAG CIRRUS_OS CIRRUS_TASK_NAME
- CIRRUS_TASK_ID CIRRUS_REPO_NAME CIRRUS_REPO_OWNER CIRRUS_REPO_FULL_NAME
- CIRRUS_REPO_CLONE_URL CIRRUS_SHELL CIRRUS_USER_COLLABORATOR CIRRUS_USER_PERMISSION
- CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST PACKER_BUILDS BUILT_IMAGE_SUFFIX
- XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID ROOTLESS_USER
-
# Default VM to use unless set or modified by task
gce_instance:
@@ -401,10 +386,6 @@ test_build_cache_images_task:
# VMs created by packer are not cleaned up by cirrus, must allow task to complete
auto_cancellation: $CI != "true"
- env:
- # Unique to this specific run of _all_ tasks
- BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}"
-
gce_instance:
image_project: "libpod-218412"
zone: "us-central1-a"
@@ -421,7 +402,7 @@ test_build_cache_images_task:
# Test building of new cache-images for future PR testing, in this PR.
-test_built_images_task:
+verify_test_built_images_task:
only_if: >-
$CIRRUS_BRANCH != 'master' &&
@@ -433,13 +414,6 @@ test_built_images_task:
- "gating"
- "test_build_cache_images"
- # VMs created by packer are not cleaned up by cirrus, must allow task to complete
- auto_cancellation: $CI != "true"
-
- env:
- # Unique to this specific run of _all_ tasks
- BUILT_IMAGE_SUFFIX: "-${CIRRUS_REPO_NAME}-${CIRRUS_BUILD_ID}"
-
gce_instance:
matrix:
# Images are generated separately, from build_images_task (below)
@@ -455,11 +429,7 @@ test_built_images_task:
environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
unit_test_script: '$SCRIPT_BASE/unit_test.sh |& ${TIMESTAMP}'
integration_test_script: '$SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP}'
- rootless_testing_script:
- - export ROOTLESS_USER=yarrpilferingpirate
- - export ROOTLESS_UID=234567
- - export ROOTLESS_GID=234567
- - cd $GOSRC && $SCRIPT_BASE/setup_and_run_rootless.sh |& ${TIMESTAMP}
+ rootless_testing_script: '$SCRIPT_BASE/resetup_and_run_rootless.sh |& ${TIMESTAMP}'
# Build new cache-images for future PR testing, but only after a PR merge.
@@ -524,6 +494,7 @@ success_task:
- "special_testing"
- "optional_testing"
- "test_build_cache_images"
+ - "verify_test_built_images"
- "build_cache_images"
env:
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md
index ce3ea7777..1d62f71ce 100644
--- a/contrib/cirrus/README.md
+++ b/contrib/cirrus/README.md
@@ -124,29 +124,54 @@ in the PR description, add the magic string: ``***CIRRUS: TEST IMAGES***``
This is a retained as a secondary method for tracking/auditing
creation of VM images, should it ever be needed.
-***Manual Steps:*** In order to utilize built images, their names must be upated
-in ``.cirrus.yml``. For example, if the image ``blah-1234`` was produced above:
+### ``verify_test_built_images`` Task
+
+Only runs following successful ``test_build_cache_images_task`` task. Uses
+images following the standard naming format; ***however, only runs a limited
+sub-set of automated tests***. Validating newly built images fully, requires
+updating ``.cirrus.yml``.
+
+***Manual Steps:*** Assuming `verify_test_built_images` passes, then
+you'll find the new image names displayed at the end of the
+`test_build_cache_images_task` in the `build_vm_images` output.
+For example:
+
+```
+...cut...
+==> Builds finished. The artifacts of successful builds are:
+--> rhel-7: A disk image was created: rhel-7-libpod-5699523102900224
+--> rhel-7:
+--> ubuntu-18: A disk image was created: ubuntu-18-libpod-5699523102900224
+--> ubuntu-18:
+--> fedora-29: A disk image was created: fedora-29-libpod-5699523102900224
+--> fedora-29:
+--> fedora-28: A disk image was created: fedora-28-libpod-5699523102900224
+```
+
+An updated (or new) pull-request with this change, will utilize
+the new cache-images:
+
```yaml
env:
+ ...cut...
####
#### Cache-image names to test with
###
- BLAH_CACHE_IMAGE_NAME: "blah-1234"
+ FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5699523102900224"
+ PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5699523102900224"
+ UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5699523102900224"
+ PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-5699523102900224"
+ ...cut...
```
-A new pull-request with that change, will run tasks utilizing that image.
-
-
-### ``test_built_images`` Task
-
-Only runs following successful ``test_build_cache_images_task`` task. Uses
-images following the standard naming format, with execution of
-the 'gate', 'testing' and 'rootless_testing' scripts. Validating the images
-suitability for wide-spread use.
+Take care to also update the PR description if any 'magic' phrases were used
+(they affect which tests run). In other words, you'll likely want to
+remove the ``***CIRRUS: TEST IMAGES***`` string - otherwise Cirrus-CI will
+simply build new and test again.
-### ``cache_images`` Task
+### ``build_cache_images`` Task *(Deprecated)*
Exactly the same as ``test_build_cache_images_task`` task, but only runs on
the master branch. Requires a magic string to be in the `HEAD`
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 565646c12..fb1685104 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -19,9 +19,13 @@ CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-HEAD}
CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-FETCH_HEAD}
SPECIALMODE="${SPECIALMODE:-none}"
export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
-ROOTLESS_USER="${ROOTLESS_USER:-}"
-ROOTLESS_UID="${ROOTLESS_UID:-}"
-ROOTLESS_GID="${ROOTLESS_GID:-}"
+
+if [[ "$USER" == "root" ]]
+then
+ ROOTLESS_USER="${ROOTLESS_USER:-}"
+else
+ ROOTLESS_USER="${ROOTLESS_USER:-$USER}"
+fi
if ! [[ "$PATH" =~ "/usr/local/bin" ]]
then
@@ -35,6 +39,23 @@ then
source "$HOME/$ENVLIB"
fi
+# Space separated list of environment variables to unset before testing
+UNSET_ENV_VARS='
+ GCP_PROJECT_ID GCE_SSH_USERNAME SERVICE_ACCOUNT RHSM_COMMAND BUILT_IMAGE_SUFFIX
+ IRCID RHEL_BASE_IMAGE FAH_BASE_IMAGE FEDORA_BASE_IMAGE CENTOS_BASE_IMAGE
+ UBUNTU_BASE_IMAGE PACKER_VER PACKER_BUILDS RUNC_COMMIT CRIU_COMMIT
+ CRIO_COMMIT CNI_COMMIT FEDORA_CNI_COMMIT PACKER_BASE SCRIPT_BASE
+ CIRRUS_SHELL CIRRUS_WORKING_DIR ENVLIB CIRRUS_CI
+ CI_NODE_INDEX CI_NODE_TOTAL CIRRUS_BASE_BRANCH CIRRUS_BASE_SHA
+ CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO CIRRUS_CLONE_DEPTH
+ CIRRUS_COMMIT_MESSAGE CIRRUS_CHANGE_MESSAGE CIRRUS_REPO_CLONE_HOST
+ CIRRUS_DEFAULT_BRANCH CIRRUS_PR CIRRUS_TAG CIRRUS_OS CIRRUS_TASK_NAME
+ CIRRUS_TASK_ID CIRRUS_REPO_NAME CIRRUS_REPO_OWNER CIRRUS_REPO_FULL_NAME
+ CIRRUS_REPO_CLONE_URL CIRRUS_SHELL CIRRUS_USER_COLLABORATOR CIRRUS_USER_PERMISSION
+ CIRRUS_WORKING_DIR CIRRUS_HTTP_CACHE_HOST PACKER_BUILDS
+ XDG_DATA_DIRS XDG_RUNTIME_DIR XDG_SESSION_ID ROOTLESS_USER
+'
+
# Pass in a list of one or more envariable names; exit non-zero with
# helpful error message if any value is empty
req_env_var() {
@@ -126,12 +147,14 @@ os_release_id() {
eval "$(egrep -m 1 '^ID=' /etc/os-release | tr -d \' | tr -d \")"
echo "$ID"
}
+export OS_RELEASE_ID="$(os_release_id)"
# Return a GCE image-name compatible string representation of distribution major version
os_release_ver() {
eval "$(egrep -m 1 '^VERSION_ID=' /etc/os-release | tr -d \' | tr -d \")"
echo "$VERSION_ID" | cut -d '.' -f 1
}
+export OS_RELEASE_VER="$(os_release_VER)"
bad_os_id_ver() {
echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $ARGS"
@@ -158,6 +181,10 @@ ircmsg() {
setup_rootless() {
req_env_var ROOTLESS_USER GOSRC ENVLIB
+ make install.catatonit
+ go get github.com/onsi/ginkgo/ginkgo
+ go get github.com/onsi/gomega/...
+
if passwd --status $ROOTLESS_USER
then
echo "Updating $ROOTLESS_USER user permissions on possibly changed libpod code"
@@ -180,7 +207,8 @@ setup_rootless() {
chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOSRC"
echo "creating ssh keypair for $USER"
- ssh-keygen -P "" -f $HOME/.ssh/id_rsa
+ [[ -r "$HOME/.ssh/id_rsa" ]] || \
+ ssh-keygen -P "" -f "$HOME/.ssh/id_rsa"
echo "Allowing ssh key for $ROOTLESS_USER"
(umask 077 && mkdir "/home/$ROOTLESS_USER/.ssh")
@@ -199,11 +227,14 @@ setup_rootless() {
install -o $ROOTLESS_USER -g $ROOTLESS_USER -m 0700 \
"$HOME/$ENVLIB" "/home/$ROOTLESS_USER/$ENVLIB"
+ # Allow the tests to run
+ echo "export ROOTLESS_USER=$ROOTLESS_USER" >> "/home/$ROOTLESS_USER/$ENVLIB"
+
echo "Configuring user's go environment variables"
su --login --command 'go env' $ROOTLESS_USER | \
while read envline
do
- X=$(echo "export $envline" | tee -a "/home/$ROOTLESS_USER/.bash_profile") && echo "$X"
+ X=$(echo "export $envline" >> "/home/$ROOTLESS_USER/$ENVLIB")
done
}
@@ -242,6 +273,10 @@ install_cni_plugins() {
}
install_runc_from_git(){
+ req_env_var "
+ GOPATH $GOPATH
+ OS_RELEASE_ID $OS_RELEASE_ID
+ "
wd=$(pwd)
DEST="$GOPATH/src/github.com/opencontainers/runc"
rm -rf "$DEST"
@@ -249,7 +284,12 @@ install_runc_from_git(){
cd "$DEST"
ooe.sh git fetch origin --tags
ooe.sh git checkout -q "$RUNC_COMMIT"
- ooe.sh make static BUILDTAGS="seccomp apparmor selinux"
+ if [[ "${OS_RELEASE_ID}" == "ubuntu" ]]
+ then
+ ooe.sh make static BUILDTAGS="seccomp apparmor"
+ else
+ ooe.sh make BUILDTAGS="seccomp selinux"
+ fi
sudo install -m 755 runc /usr/bin/runc
cd $wd
}
diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh
index b86ba1a1c..f9728f2f7 100644
--- a/contrib/cirrus/packer/centos_setup.sh
+++ b/contrib/cirrus/packer/centos_setup.sh
@@ -59,7 +59,6 @@ ooe.sh sudo yum -y install \
python3-dateutil \
python3-psutil \
python3-pytoml \
- runc \
skopeo-containers \
unzip \
vim \
@@ -68,14 +67,16 @@ ooe.sh sudo yum -y install \
install_scl_git
-install_cni_plugins
-
-install_buildah
-
install_conmon
+install_runc
+
install_criu
+install_cni_plugins
+
+install_buildah
+
install_packer_copied_files
sudo /tmp/libpod/hack/install_catatonit.sh
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index fef74179d..d9229c668 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -60,7 +60,6 @@ ooe.sh sudo dnf install -y \
python3-dateutil \
python3-psutil \
python3-pytoml \
- runc \
skopeo-containers \
slirp4netns \
unzip \
@@ -70,13 +69,15 @@ ooe.sh sudo dnf install -y \
install_varlink
+install_conmon
+
+install_runc
+
CNI_COMMIT=$FEDORA_CNI_COMMIT
install_cni_plugins
install_buildah
-install_conmon
-
install_packer_copied_files
sudo /tmp/libpod/hack/install_catatonit.sh
diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh
index b6240be80..dee35a102 100644
--- a/contrib/cirrus/packer/rhel_setup.sh
+++ b/contrib/cirrus/packer/rhel_setup.sh
@@ -22,6 +22,9 @@ ooe.sh sudo subscription-manager repos \
--enable=rhel-7-server-extras-rpms \
--enable=rhel-server-rhscl-7-rpms
+sudo sed -r -i -e 's/^notify_only=.+/notify_only=0/g' \
+ /etc/yum/pluginconf.d/search-disabled-repos.conf
+
ooe.sh sudo yum -y update
ooe.sh sudo yum -y install \
@@ -66,7 +69,6 @@ ooe.sh sudo yum -y install \
python34-psutil \
python34-pytoml \
python34-PyYAML \
- runc \
skopeo-containers \
unzip \
vim \
@@ -75,14 +77,16 @@ ooe.sh sudo yum -y install \
install_scl_git
-install_cni_plugins
-
-install_buildah
-
install_conmon
+install_runc
+
install_criu
+install_cni_plugins
+
+install_buildah
+
install_packer_copied_files
sudo /tmp/libpod/hack/install_catatonit.sh
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index 93dbd0fa0..d02b71e12 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -90,10 +90,10 @@ ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/*
ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub
ooe.sh sudo update-grub
-install_runc
-
install_conmon
+install_runc
+
install_criu
install_cni_plugins
diff --git a/contrib/cirrus/resetup_and_run_rootless.sh b/contrib/cirrus/resetup_and_run_rootless.sh
new file mode 100755
index 000000000..1c387fdf5
--- /dev/null
+++ b/contrib/cirrus/resetup_and_run_rootless.sh
@@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -e
+
+# N/B: This script is only intended to be used for the special-case of
+# setting up and executing the rootless tests AFTER normal tests complete
+# while testing a freshly built image.
+
+source $(dirname $0)/lib.sh
+
+# must be after source lib.sh b/c it loads $ENVLIB
+export ROOTLESS_USER="pilferingpirate$RANDOM"
+
+req_env_var "
+CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR
+GOSRC $GOSRC
+SCRIPT_BASE $SCRIPT_BASE
+ROOTLESS_USER $ROOTLESS_USER
+"
+
+if ! run_rootless
+then
+ die 86 "Error: Expected rootless env. var not set or empty"
+fi
+
+cd $GOSRC
+make clean
+setup_rootless
+
+ssh $ROOTLESS_USER@localhost \
+ -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \
+ $CIRRUS_WORKING_DIR/$SCRIPT_BASE/rootless_test.sh
diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh
index 13386bb09..d4cc6965e 100755
--- a/contrib/cirrus/rootless_test.sh
+++ b/contrib/cirrus/rootless_test.sh
@@ -1,12 +1,10 @@
#!/bin/bash
-set -ex
+set -e
-source "$HOME/.bash_profile"
-source $GOSRC/$SCRIPT_BASE/lib.sh
-cd $GOSRC
+source $(dirname $0)/lib.sh
-req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER
+req_env_var GOSRC ROOTLESS_USER
if [[ "$UID" == "0" ]]
then
@@ -17,6 +15,10 @@ fi
echo "."
echo "Hello, my name is $USER and I live in $PWD can I be your friend?"
+show_env_vars
+
+export PODMAN_VARLINK_ADDRESS=unix:/tmp/podman-$(id -u)
+
cd "$GOSRC"
make
make varlink_generate
diff --git a/contrib/cirrus/setup_and_run_rootless.sh b/contrib/cirrus/setup_and_run_rootless.sh
deleted file mode 100755
index 7ca34a6ac..000000000
--- a/contrib/cirrus/setup_and_run_rootless.sh
+++ /dev/null
@@ -1,26 +0,0 @@
-#!/bin/bash
-
-set -ex
-
-source $(dirname $0)/lib.sh
-
-req_env_var "
-CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR
-GOSRC $GOSRC
-SCRIPT_BASE $SCRIPT_BASE
-ROOTLESS_USER $ROOTLESS_USER
-ROOTLESS_UID $ROOTLESS_UID
-ROOTLESS_GID $ROOTLESS_GID
-"
-
-if run_rootless
-then
- die 86 "Error: Expected rootless env. vars not set or empty"
-fi
-
-cd $GOSRC
-setup_rootless
-
-ssh $ROOTLESS_USER@localhost \
- -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \
- $CIRRUS_WORKING_DIR/$SCRIPT_BASE/rootless_test.sh
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 1a89a5b71..b3319a3e1 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -37,7 +37,6 @@ then
"export OS_RELEASE_VER=\"$(os_release_ver)\"" \
"export OS_REL_VER=\"$(os_release_id)-$(os_release_ver)\"" \
"export TEST_REMOTE_CLIENT=\"$TEST_REMOTE_CLIENT\"" \
- "export BUILT_IMAGE_SUFFIX=\"-$CIRRUS_REPO_NAME-${CIRRUS_CHANGE_IN_REPO:0:8}\"" \
"export GOPATH=\"/var/tmp/go\"" \
'export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"' \
'export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"'