summaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorChris Evich <cevich@redhat.com>2020-05-12 12:23:35 -0400
committerChris Evich <cevich@redhat.com>2020-06-09 14:47:23 -0400
commit455a94d1f932cd32dc07d9bf62131423610f8e2b (patch)
tree0387e09597aa4354cda3d2390fad254313112e1e
parent2869cce1d53aedfe2500f3d921fb901dd5994690 (diff)
downloadpodman-455a94d1f932cd32dc07d9bf62131423610f8e2b.tar.gz
podman-455a94d1f932cd32dc07d9bf62131423610f8e2b.tar.bz2
podman-455a94d1f932cd32dc07d9bf62131423610f8e2b.zip
Cirrus: Include packages for containers/conmon CI
This allows the containers/conmon repository to share the same VM images produced by containers/libpod. Included are several packages which are downloaded only since they might otherwise interfere with testing for some repos. This allows stable versions to be at the ready at testing runtime, avoiding any version updates surprising developers. Also, re-enable running the VM-image check test which was not working due to a logic problem in Cirrus-CI configuration. Update the neglected tests so that they pass on all distros. Signed-off-by: Chris Evich <cevich@redhat.com>
-rw-r--r--.cirrus.yml21
-rwxr-xr-xcontrib/cirrus/check_image.sh42
-rw-r--r--contrib/cirrus/lib.sh13
-rw-r--r--contrib/cirrus/packer/fedora_packaging.sh85
-rw-r--r--contrib/cirrus/packer/fedora_setup.sh8
-rw-r--r--contrib/cirrus/packer/ubuntu_packaging.sh31
-rwxr-xr-xcontrib/cirrus/setup_environment.sh8
7 files changed, 129 insertions, 79 deletions
diff --git a/.cirrus.yml b/.cirrus.yml
index 58c7cd871..0f3971dd2 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -39,7 +39,7 @@ env:
UBUNTU_NAME: "ubuntu-20"
PRIOR_UBUNTU_NAME: "ubuntu-19"
- _BUILT_IMAGE_SUFFIX: "libpod-6268069335007232" # From the packer output of 'build_vm_images_script'
+ _BUILT_IMAGE_SUFFIX: "libpod-6508632441356288"
FEDORA_CACHE_IMAGE_NAME: "${FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
PRIOR_FEDORA_CACHE_IMAGE_NAME: "${PRIOR_FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
UBUNTU_CACHE_IMAGE_NAME: "${UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}"
@@ -72,10 +72,6 @@ env:
GCE_SSH_USERNAME: cirrus-ci
# Name where this repositories cloud resources are located
GCP_PROJECT_ID: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
- RELEASE_GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46]
- RELEASE_GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1]
- RELEASE_GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
-
# Default VM to use unless set or modified by task
@@ -601,6 +597,7 @@ special_testing_bindings_task:
env:
SPECIALMODE: 'bindings' # See docs
+ ADD_SECOND_PARTITION: 'true' # More root fs space is required
timeout_in: 40m
@@ -701,20 +698,16 @@ verify_test_built_images_task:
env:
TEST_REMOTE_CLIENT: 'false'
matrix:
- # Required env. var. by check_image_script
PACKER_BUILDER_NAME: "${FEDORA_NAME}"
PACKER_BUILDER_NAME: "${PRIOR_FEDORA_NAME}"
+ PACKER_BUILDER_NAME: "${UBUNTU_NAME}"
PACKER_BUILDER_NAME: "${PRIOR_UBUNTU_NAME}"
- # Multiple test failures on ${UBUNTU_CACHE_IMAGE_NAME}
- # PACKER_BUILDER_NAME: "${UBUNTU_NAME}"
networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
installed_packages_script: '$SCRIPT_BASE/logcollector.sh packages'
environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
- # Verify expectations once per image
- check_image_script: >-
- [[ "$TEST_REMOTE_CLIENT" == "false" ]] || \
- $SCRIPT_BASE/check_image.sh |& ${TIMESTAMP}
+ # Verify expectations of built images
+ check_image_script: '$SCRIPT_BASE/check_image.sh |& ${TIMESTAMP}'
# Note: A truncated form of normal testing. It only needs to confirm new images
# "probably" work. A full round of testing will happen again after $*_CACHE_IMAGE_NAME
# are updated in this or another PR (w/o '***CIRRUS: TEST IMAGES***').
@@ -732,6 +725,10 @@ docs_task:
depends_on:
- "gating"
+ env:
+ RELEASE_GCPJSON: ENCRYPTED[789d8f7e9a5972ce350fd8e60f1032ccbf4a35c3938b604774b711aad280e12c21faf10e25af1e0ba33597ffb9e39e46]
+ RELEASE_GCPNAME: ENCRYPTED[417d50488a4bd197bcc925ba6574de5823b97e68db1a17e3a5fde4bcf26576987345e75f8d9ea1c15a156b4612c072a1]
+ RELEASE_GCPROJECT: ENCRYPTED[7c80e728e046b1c76147afd156a32c1c57d4a1ac1eab93b7e68e718c61ca8564fc61fef815952b8ae0a64e7034b8fe4f]
script:
- "$SCRIPT_BASE/build_swagger.sh |& ${TIMESTAMP}"
diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh
index 5423f67d6..0d33e55bf 100755
--- a/contrib/cirrus/check_image.sh
+++ b/contrib/cirrus/check_image.sh
@@ -6,7 +6,7 @@ source $(dirname $0)/lib.sh
EVIL_UNITS="$($CIRRUS_WORKING_DIR/$PACKER_BASE/systemd_banish.sh --list)"
-req_env_var PACKER_BUILDER_NAME TEST_REMOTE_CLIENT EVIL_UNITS OS_RELEASE_ID
+req_env_var PACKER_BUILDER_NAME TEST_REMOTE_CLIENT EVIL_UNITS OS_RELEASE_ID CG_FS_TYPE
NFAILS=0
echo "Validating VM image"
@@ -22,7 +22,8 @@ item_test 'Minimum available memory' $MEM_FREE -ge $MIN_MEM_MB || let "NFAILS+=1
# We're testing a custom-built podman; make sure there isn't a distro-provided
# binary anywhere; that could potentially taint our results.
-item_test "remove_packaged_podman_files() did it's job" -z "$(type -P podman)" || let "NFAILS+=1"
+remove_packaged_podman_files
+item_test "remove_packaged_podman_files() does it's job" -z "$(type -P podman)" || let "NFAILS+=1"
# Integration Tests require varlink in Fedora
item_test "The varlink executable is present" -x "$(type -P varlink)" || let "NFAILS+=1"
@@ -39,8 +40,10 @@ for REQ_UNIT in google-accounts-daemon.service \
google-shutdown-scripts.service \
google-startup-scripts.service
do
- item_test "required $REQ_UNIT enabled" \
- "$(systemctl list-unit-files --no-legend $REQ_UNIT)" = "$REQ_UNIT enabled" || let "NFAILS+=1"
+ # enabled/disabled appears at the end of the line, on some Ubuntu's it appears twice
+ service_status=$(systemctl list-unit-files --no-legend $REQ_UNIT | tac -s ' ' | head -1)
+ item_test "required $REQ_UNIT status is enabled" \
+ "$service_status" = "enabled" || let "NFAILS+=1"
done
for evil_unit in $EVIL_UNITS
@@ -50,19 +53,28 @@ do
item_test "No $evil_unit unit is present or active:" "$unit_status" -ne "0" || let "NFAILS+=1"
done
-if [[ "$OS_RELEASE_ID" == "ubuntu" ]] && [[ -x "/usr/lib/cri-o-runc/sbin/runc" ]]
-then
- SAMESAME=$(diff --brief /usr/lib/cri-o-runc/sbin/runc /usr/bin/runc &> /dev/null; echo $?)
- item_test "On ubuntu /usr/bin/runc is /usr/lib/cri-o-runc/sbin/runc" "$SAMESAME" -eq "0" || let "NFAILS+=1"
-fi
-
-if [[ "$OS_RELEASE_ID" == "ubuntu" ]]
-then
- item_test "On ubuntu, no periodic apt crap is enabled" -z "$(egrep $PERIODIC_APT_RE /etc/apt/apt.conf.d/*)"
-fi
-
echo "Checking items specific to ${PACKER_BUILDER_NAME}${BUILT_IMAGE_SUFFIX}"
case "$PACKER_BUILDER_NAME" in
+ ubuntu*)
+ item_test "On ubuntu, no periodic apt crap is enabled" -z "$(egrep $PERIODIC_APT_RE /etc/apt/apt.conf.d/*)"
+ ;;
+ fedora*)
+ # Only runc -OR- crun should be installed, never both
+ case "$CG_FS_TYPE" in
+ tmpfs)
+ HAS=runc
+ HAS_NOT=crun
+ ;;
+ cgroup2fs)
+ HAS=crun
+ HAS_NOT=runc
+ ;;
+ esac
+ HAS_RC=$(rpm -qV $HAS &> /dev/null; echo $?)
+ HAS_NOT_RC=$(rpm -qV $HAS_NOT &> /dev/null; echo $?)
+ item_test "With a cgroups-fs type $CG_FS_TYPE, the $HAS package is installed" $HAS_RC -eq 0
+ item_test "With a cgroups-fs type $CG_FS_TYPE, the $HAS_NOT package is not installed" $HAS_NOT_RC -ne 0
+ ;;
xfedora*)
echo "Kernel Command-line: $(cat /proc/cmdline)"
item_test \
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index cc5a3ffa7..66e8060cf 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -39,6 +39,8 @@ PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer}
# Important filepaths
SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}"
AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv"
+# Downloaded, but not installed packages.
+PACKAGE_DOWNLOAD_DIR=/var/cache/download
# Log remote-client system test varlink output here
export VARLINK_LOG=/var/tmp/varlink.log
@@ -422,7 +424,7 @@ remove_packaged_podman_files() {
then
LISTING_CMD="$SUDO dpkg-query -L podman"
else
- LISTING_CMD='$SUDO rpm -ql podman'
+ LISTING_CMD="$SUDO rpm -ql podman"
fi
# yum/dnf/dpkg may list system directories, only remove files
@@ -437,6 +439,14 @@ remove_packaged_podman_files() {
sync && echo 3 > /proc/sys/vm/drop_caches
}
+# The version of CRI-O and Kubernetes must always match
+get_kubernetes_version(){
+ # TODO: Look up the kube RPM/DEB version installed, or in $PACKAGE_DOWNLOAD_DIR
+ # and retrieve the major-minor version directly.
+ local KUBERNETES_VERSION="1.15"
+ echo "$KUBERNETES_VERSION"
+}
+
canonicalize_image_names() {
req_env_var IMGNAMES
echo "Adding all current base images to \$IMGNAMES for timestamp update"
@@ -479,6 +489,7 @@ _finalize() {
fi
echo "Re-initializing so next boot does 'first-boot' setup again."
cd /
+ $SUDO rm -rf $GOPATH/src # Actual source will be cloned at runtime
$SUDO rm -rf /var/lib/cloud/instanc*
$SUDO rm -rf /root/.ssh/*
$SUDO rm -rf /etc/ssh/*key*
diff --git a/contrib/cirrus/packer/fedora_packaging.sh b/contrib/cirrus/packer/fedora_packaging.sh
index e80d48bc8..aecaaef93 100644
--- a/contrib/cirrus/packer/fedora_packaging.sh
+++ b/contrib/cirrus/packer/fedora_packaging.sh
@@ -11,6 +11,8 @@ echo "Updating/Installing repos and packages for $OS_REL_VER"
source $GOSRC/$SCRIPT_BASE/lib.sh
+req_env_var GOSRC SCRIPT_BASE BIGTO INSTALL_AUTOMATION_VERSION FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE
+
# Pre-req. to install automation tooing
$LILTO $SUDO dnf install -y git
@@ -35,7 +37,7 @@ fi
$BIGTO ooe.sh $SUDO dnf update -y
-REMOVE_PACKAGES=()
+REMOVE_PACKAGES=(runc)
INSTALL_PACKAGES=(\
autoconf
automake
@@ -50,8 +52,11 @@ INSTALL_PACKAGES=(\
containernetworking-plugins
containers-common
criu
+ crun
+ curl
device-mapper-devel
dnsmasq
+ e2fsprogs-devel
emacs-nox
file
findutils
@@ -60,16 +65,26 @@ INSTALL_PACKAGES=(\
gcc
git
glib2-devel
+ glibc-devel
glibc-static
gnupg
go-md2man
golang
+ gpgme
gpgme-devel
+ grubby
+ hostname
iproute
iptables
jq
+ krb5-workstation
+ libassuan
libassuan-devel
+ libblkid-devel
libcap-devel
+ libffi-devel
+ libgpg-error-devel
+ libguestfs-tools
libmsi1
libnet
libnet-devel
@@ -79,56 +94,60 @@ INSTALL_PACKAGES=(\
libselinux-devel
libtool
libvarlink-util
+ libxml2-devel
+ libxslt-devel
lsof
make
+ mlocate
msitools
+ nfs-utils
nmap-ncat
+ openssl
+ openssl-devel
ostree-devel
pandoc
+ pkgconfig
podman
+ policycoreutils
procps-ng
protobuf
protobuf-c
protobuf-c-devel
protobuf-devel
- python
+ python2
+ python3-PyYAML
python3-dateutil
python3-psutil
python3-pytoml
+ python3-libsemanage
+ python3-libselinux
+ python3-libvirt
+ redhat-rpm-config
+ rpcbind
rsync
+ sed
selinux-policy-devel
skopeo
skopeo-containers
slirp4netns
+ socat
+ tar
unzip
vim
wget
which
xz
zip
+ zlib-devel
+)
+DOWNLOAD_PACKAGES=(\
+ "cri-o-$(get_kubernetes_version)*"
+ cri-tools
+ "kubernetes-$(get_kubernetes_version)*"
+ runc
+ oci-umount
+ parallel
)
-
-case "$OS_RELEASE_VER" in
- 30)
- INSTALL_PACKAGES+=(\
- atomic-registries
- golang-github-cpuguy83-go-md2man
- python2-future
- runc
- )
- REMOVE_PACKAGES+=(crun)
- ;;
- 31)
- INSTALL_PACKAGES+=(crun)
- REMOVE_PACKAGES+=(runc)
- ;;
- 32)
- INSTALL_PACKAGES+=(crun)
- REMOVE_PACKAGES+=(runc)
- ;;
- *)
- bad_os_id_ver ;;
-esac
echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'"
$BIGTO ooe.sh $SUDO dnf install -y ${INSTALL_PACKAGES[@]}
@@ -136,6 +155,18 @@ $BIGTO ooe.sh $SUDO dnf install -y ${INSTALL_PACKAGES[@]}
[[ ${#REMOVE_PACKAGES[@]} -eq 0 ]] || \
$LILTO ooe.sh $SUDO dnf erase -y ${REMOVE_PACKAGES[@]}
-export GOPATH="$(mktemp -d)"
-trap "$SUDO rm -rf $GOPATH" EXIT
-ooe.sh $SUDO $GOSRC/hack/install_catatonit.sh
+if [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then
+ echo "Downloading packages for optional installation at runtime, as needed."
+ # Required for cri-o
+ ooe.sh $SUDO dnf -y module enable cri-o:$(get_kubernetes_version)
+ $SUDO mkdir -p "$PACKAGE_DOWNLOAD_DIR"
+ cd "$PACKAGE_DOWNLOAD_DIR"
+ $LILTO ooe.sh $SUDO dnf download -y --resolve ${DOWNLOAD_PACKAGES[@]}
+ ls -la "$PACKAGE_DOWNLOAD_DIR/"
+fi
+
+echo "Installing runtime tooling"
+# Save some runtime by having these already available
+cd $GOSRC
+$SUDO make install.tools
+$SUDO $GOSRC/hack/install_catatonit.sh
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index 3830b3bc4..25b568e8a 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -12,11 +12,11 @@ req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NA
workaround_bfq_bug
-# Do not enable update-stesting on the previous Fedora release
-if [[ "$FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then
- DISABLE_UPDATES_TESTING=0
-else
+# Do not enable updates-testing on the previous Fedora release
+if [[ "$PRIOR_FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then
DISABLE_UPDATES_TESTING=1
+else
+ DISABLE_UPDATES_TESTING=0
fi
bash $PACKER_BASE/fedora_packaging.sh
diff --git a/contrib/cirrus/packer/ubuntu_packaging.sh b/contrib/cirrus/packer/ubuntu_packaging.sh
index fd0280230..09f9aab9f 100644
--- a/contrib/cirrus/packer/ubuntu_packaging.sh
+++ b/contrib/cirrus/packer/ubuntu_packaging.sh
@@ -11,6 +11,8 @@ echo "Updating/Installing repos and packages for $OS_REL_VER"
source $GOSRC/$SCRIPT_BASE/lib.sh
+req_env_var GOSRC SCRIPT_BASE BIGTO SUDOAPTGET INSTALL_AUTOMATION_VERSION
+
echo "Updating/configuring package repositories."
$BIGTO $SUDOAPTGET update
@@ -99,6 +101,7 @@ INSTALL_PACKAGES=(\
protobuf-c-compiler
protobuf-compiler
python-protobuf
+ python2
python3-dateutil
python3-pip
python3-psutil
@@ -118,6 +121,11 @@ INSTALL_PACKAGES=(\
zip
zlib1g-dev
)
+DOWNLOAD_PACKAGES=(\
+ cri-o-$(get_kubernetes_version)
+ cri-tools
+ parallel
+)
# These aren't resolvable on Ubuntu 20
if [[ "$OS_RELEASE_VER" -le 19 ]]; then
@@ -137,16 +145,15 @@ echo "Installing general testing and system dependencies"
$LILTO ooe.sh $SUDOAPTGET update
$BIGTO ooe.sh $SUDOAPTGET install ${INSTALL_PACKAGES[@]}
-export GOPATH="$(mktemp -d)"
-trap "$SUDO rm -rf $GOPATH" EXIT
-echo "Installing cataonit and libseccomp.sudo"
-cd $GOSRC
-ooe.sh $SUDO hack/install_catatonit.sh
-ooe.sh $SUDO make install.libseccomp.sudo
-
-CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc"
-if $SUDO dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH"
-then
- echo "Linking $CRIO_RUNC_PATH to /usr/bin/runc for ease of testing."
- $SUDO ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc"
+if [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then
+ echo "Downloading packages for optional installation at runtime, as needed."
+ $SUDO ln -s /var/cache/apt/archives "$PACKAGE_DOWNLOAD_DIR"
+ $LILTO ooe.sh $SUDOAPTGET install --download-only ${DOWNLOAD_PACKAGES[@]}
+ ls -la "$PACKAGE_DOWNLOAD_DIR/"
fi
+
+echo "Installing runtime tooling"
+cd $GOSRC
+$SUDO hack/install_catatonit.sh
+$SUDO make install.libseccomp.sudo
+$SUDO make install.tools
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 25b7ff941..31d63677f 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -78,14 +78,6 @@ case "$CG_FS_TYPE" in
warn "Forcing testing with crun instead of runc"
X=$(echo "export OCI_RUNTIME=/usr/bin/crun" | \
tee -a /etc/environment) && eval "$X" && echo "$X"
-
- if [[ "$OS_RELEASE_ID" == "fedora" ]]; then
- warn "Upgrading to the latest crun"
- # Normally not something to do for stable testing
- # but crun is new, and late-breaking fixes may be required
- # on short notice
- dnf update -y crun containers-common
- fi
;;
*)
die 110 "Unsure how to handle cgroup filesystem type '$CG_FS_TYPE'"