diff options
Diffstat (limited to 'contrib/cirrus')
| -rw-r--r-- | contrib/cirrus/cron-fail_addrs.csv | 1 | ||||
| -rw-r--r-- | contrib/cirrus/lib.sh | 73 | ||||
| -rwxr-xr-x | contrib/cirrus/logcollector.sh | 9 | ||||
| -rwxr-xr-x | contrib/cirrus/logformatter | 3 | ||||
| -rwxr-xr-x | contrib/cirrus/runner.sh | 19 | ||||
| -rwxr-xr-x | contrib/cirrus/setup_environment.sh | 75 |
6 files changed, 110 insertions, 70 deletions
diff --git a/contrib/cirrus/cron-fail_addrs.csv b/contrib/cirrus/cron-fail_addrs.csv new file mode 100644 index 000000000..c25fc1226 --- /dev/null +++ b/contrib/cirrus/cron-fail_addrs.csv @@ -0,0 +1 @@ +rh.container.bot@gmail.com diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index e5124d8e4..04e8a3c1c 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -6,32 +6,31 @@ # BEGIN Global export of all variables set -a -if [[ "$CI" == "true" ]]; then - # Due to differences across platforms and runtime execution environments, - # handling of the (otherwise) default shell setup is non-uniform. Rather - # than attempt to workaround differences, simply force-load/set required - # items every time this library is utilized. - source /etc/profile - source /etc/environment - USER="$(whoami)" - HOME="$(getent passwd $USER | cut -d : -f 6)" - # Some platforms set and make this read-only - [[ -n "$UID" ]] || \ - UID=$(getent passwd $USER | cut -d : -f 3) - GID=$(getent passwd $USER | cut -d : -f 4) -fi +# Due to differences across platforms and runtime execution environments, +# handling of the (otherwise) default shell setup is non-uniform. Rather +# than attempt to workaround differences, simply force-load/set required +# items every time this library is utilized. +_waserrexit=0 +if [[ "$SHELLOPTS" =~ errexit ]]; then _waserrexit=1; fi +set +e # Assumed in F33 for setting global vars +source /etc/profile +source /etc/environment +if [[ -r "/etc/ci_environment" ]]; then source /etc/ci_environment; fi +USER="$(whoami)" +HOME="$(getent passwd $USER | cut -d : -f 6)" +# Some platforms set and make this read-only +[[ -n "$UID" ]] || \ + UID=$(getent passwd $USER | cut -d : -f 3) +if ((_waserrexit)); then set -e; fi # During VM Image build, the 'containers/automation' installation -# was performed. The final step of that installation sets the -# installation location in $AUTOMATION_LIB_PATH in /etc/environment -# or in the default shell profile. +# was performed. The final step of installation sets the library +# location $AUTOMATION_LIB_PATH in /etc/environment or in the +# default shell profile depending on distribution. # shellcheck disable=SC2154 if [[ -n "$AUTOMATION_LIB_PATH" ]]; then - for libname in defaults anchors console_output utils; do - # There's no way shellcheck can process this location - # shellcheck disable=SC1090 - source $AUTOMATION_LIB_PATH/${libname}.sh - done + # shellcheck source=/usr/share/automation/lib/common_lib.sh + source $AUTOMATION_LIB_PATH/common_lib.sh else ( echo "WARNING: It does not appear that containers/automation was installed." @@ -88,8 +87,10 @@ CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and uniq # The starting place for linting and code validation EPOCH_TEST_COMMIT="$CIRRUS_BASE_SHA" -# Regex of env. vars. to explicitly pass when executing tests -# inside a container or as a rootless user +# Regex defining all CI-releated env. vars. necessary for all possible +# testing operations on all platforms and versions. This is necessary +# to avoid needlessly passing through global/system values across +# contexts, such as host->container or root->rootless user PASSTHROUGH_ENV_RE='(^CI.*)|(^CIRRUS)|(^DISTRO_NV)|(^GOPATH)|(^GOCACHE)|(^GOSRC)|(^SCRIPT_BASE)|(CGROUP_MANAGER)|(OCI_RUNTIME)|(^TEST.*)|(^PODBIN_NAME)|(^PRIV_NAME)|(^ALT_NAME)|(^ROOTLESS_USER)|(SKIP_USERNS)|(.*_NAME)|(.*_FQIN)' # Unsafe env. vars for display SECRET_ENV_RE='(ACCOUNT)|(GC[EP]..+)|(SSH)|(PASSWORD)|(TOKEN)' @@ -107,10 +108,8 @@ lilto() { err_retry 8 1000 "" "$@"; } # just over 4 minutes max bigto() { err_retry 7 5670 "" "$@"; } # 12 minutes max # Print shell-escaped variable=value pairs, one per line, based on -# variable name matching a regex. This is intended to support -# passthrough of CI variables from host -> container or from root -> user. -# For all other vars. we rely on tooling to load this library from inside -# the container or as rootless user to pickup the remainder. +# variable name matching a regex. This is intended to catch +# variables being passed down from higher layers, like Cirrus-CI. passthrough_envars(){ local xchars local envname @@ -176,22 +175,7 @@ setup_rootless() { echo "${ROOTLESS_USER}:$[rootless_uid * 100]:65536" | \ tee -a /etc/subuid >> /etc/subgid - # Env. vars set by Cirrus and setup_environment.sh must be explicitly - # transferred to the test-user. - msg "Configuring rootless user's environment variables:" - - ( - echo "# Added by ${BASH_SOURCE[0]} ${FUNCNAME[0]}()" - echo "export SETUP_ENVIRONMENT=1" - ) >> "/home/$ROOTLESS_USER/.bashrc" - - while read -r env_var_val; do - echo "export $env_var_val" >> "/home/$ROOTLESS_USER/.bashrc" - done <<<"$(passthrough_envars)" - chown $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.bashrc" - cat "/home/$ROOTLESS_USER/.bashrc" | indent 2 - - msg "Ensure the systems ssh process is up and running within 5 minutes" + msg "Ensure the ssh daemon is up and running within 5 minutes" systemctl start sshd lilto ssh $ROOTLESS_USER@localhost \ -o UserKnownHostsFile=/dev/null \ @@ -208,6 +192,7 @@ install_test_configs() { # as the default). This config prevents allocation of network address space used # by default in google cloud. https://cloud.google.com/vpc/docs/vpc#ip-ranges install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist /etc/cni/net.d/ + install -v -D -m 644 ./test/registries.conf /etc/containers/ } diff --git a/contrib/cirrus/logcollector.sh b/contrib/cirrus/logcollector.sh index 9dcc252c0..7bf651b36 100755 --- a/contrib/cirrus/logcollector.sh +++ b/contrib/cirrus/logcollector.sh @@ -45,8 +45,10 @@ case $1 in containernetworking-plugins \ containers-common \ criu \ + crun \ golang \ podman \ + runc \ skopeo \ slirp4netns \ ) @@ -56,9 +58,7 @@ case $1 in PKG_LST_CMD='rpm -q --qf=%{N}-%{V}-%{R}-%{ARCH}\n' PKG_NAMES+=(\ container-selinux \ - crun \ libseccomp \ - runc \ ) ;; ubuntu*) @@ -76,5 +76,10 @@ case $1 in # Any not-present packages will be listed as such $PKG_LST_CMD "${PKG_NAMES[@]}" | sort -u ;; + time) + # Assumed to be empty/undefined outside of Cirrus-CI (.cirrus.yml) + # shellcheck disable=SC2154 + if [[ -r "$STATS_LOGFILE" ]]; then cat "$STATS_LOGFILE"; fi + ;; *) die "Warning, $(basename $0) doesn't know how to handle the parameter '$1'" esac diff --git a/contrib/cirrus/logformatter b/contrib/cirrus/logformatter index 0cbd1f34b..bcafbc473 100755 --- a/contrib/cirrus/logformatter +++ b/contrib/cirrus/logformatter @@ -56,6 +56,7 @@ a.codelink:hover { background: #000; color: #999; } /* The timing tests at bottom: remove underline, it's too cluttery. */ a.timing { text-decoration: none; } +.timing:hover { background: #FF9; } /* highlight row for easy reading */ /* BATS styles */ .bats-passed { color: #393; } @@ -292,7 +293,7 @@ END_HTML $spaces = 1 if $spaces < 1; $spaces++ if $time < 10; my $spacing = ' ' x $spaces; - $line = qq{<a class="timing" href="#t--$id">$name</a>$spacing$time}; + $line = qq{<span class="timing"><a href="#t--$id">$name</a>$spacing$time</span>}; } else { $in_timing = 0; diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh index b97a696d9..bf2b1a52b 100755 --- a/contrib/cirrus/runner.sh +++ b/contrib/cirrus/runner.sh @@ -40,7 +40,7 @@ function _run_automation() { req_env_vars CI DEST_BRANCH IMAGE_SUFFIX TEST_FLAVOR TEST_ENVIRON \ PODBIN_NAME PRIV_NAME DISTRO_NV CONTAINER USER HOME \ - UID GID AUTOMATION_LIB_PATH SCRIPT_BASE OS_RELEASE_ID \ + UID AUTOMATION_LIB_PATH SCRIPT_BASE OS_RELEASE_ID \ OS_RELEASE_VER CG_FS_TYPE bigto ooe.sh dnf install -y ShellCheck # small/quick addition $SCRIPT_BASE/shellcheck.sh @@ -50,6 +50,7 @@ function _run_validate() { # Confirm compile via prior task + cache bin/podman --version bin/podman-remote --version + make validate # Some items require a build } @@ -63,6 +64,12 @@ function _run_unit() { } function _run_apiv2() { + # TODO Remove once VM's with dependency + if [[ "$OS_RELEASE_ID" == "fedora" ]]; then + dnf install -y python3-docker + else + apt-get -qq -y install python3-docker + fi make localapiv2 |& logformatter } @@ -253,6 +260,16 @@ dotest() { |& logformatter } +# Nearly every task in .cirrus.yml makes use of this shell script +# wrapped by /usr/bin/time to collect runtime statistics. Because the +# --output option is used to log stats to a file, every child-process +# inherits an open FD3 pointing at the log. However, some testing +# operations depend on making use of FD3, and so it must be explicitly +# closed here (and for all further child-processes). +# STATS_LOGFILE assumed empty/undefined outside of Cirrus-CI (.cirrus.yml) +# shellcheck disable=SC2154 +exec 3<&- + msg "************************************************************" # Required to be defined by caller # shellcheck disable=SC2154 diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 3135a5e65..da175cc05 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -1,5 +1,12 @@ #!/usr/bin/env bash +# This script is intended to be executed early by automation before +# performing other substantial operations. It relies heavily on +# desired setup information being passed in environment variables +# from Cirrus-CI and/or other orchestration tooling. To that end, +# VM's must always be considered single-purpose, single-use, +# disposable entities. i.e. One setup, one test, then always discarded. + set -e # shellcheck source=./contrib/cirrus/lib.sh @@ -29,6 +36,17 @@ do fi done +# Ensure that all lower-level contexts and child-processes have +# ready access to higher level orchestration (e.g Cirrus-CI) +# variables. +echo -e "\n# Begin single-use VM global variables (${BASH_SOURCE[0]})" \ + > "/etc/ci_environment" +( + while read -r env_var_val; do + echo "$env_var_val" + done <<<"$(passthrough_envars)" +) >> "/etc/ci_environment" + # This is a possible manual maintenance gaff, check to be sure everything matches. # shellcheck disable=SC2154 [[ "$DISTRO_NV" == "$OS_REL_VER" ]] || \ @@ -50,9 +68,9 @@ case "$CG_FS_TYPE" in if ((CONTAINER==0)); then warn "Forcing testing with runc instead of crun" if [[ "$OS_RELEASE_ID" == "ubuntu" ]]; then - echo "export OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" >> /etc/environment + echo "OCI_RUNTIME=/usr/lib/cri-o-runc/sbin/runc" >> /etc/ci_environment else - echo "export OCI_RUNTIME=runc" >> /etc/environment + echo "OCI_RUNTIME=runc" >> /etc/ci_environment fi fi ;; @@ -61,21 +79,32 @@ case "$CG_FS_TYPE" in # This is necessary since we've built/installed from source, # which uses runc as the default. warn "Forcing testing with crun instead of runc" - echo "export OCI_RUNTIME=crun" >> /etc/environment + echo "OCI_RUNTIME=crun" >> /etc/ci_environment fi ;; *) die_unknown CG_FS_TYPE esac +if ((CONTAINER==0)); then # Not yet running inside a container + # Discovered reemergence of BFQ scheduler bug in kernel 5.8.12-200 + # which causes a kernel panic when system is under heavy I/O load. + # Previously discovered in F32beta and confirmed fixed. It's been + # observed in F31 kernels as well. Deploy workaround for all VMs + # to ensure a more stable I/O scheduler (elevator). + echo "mq-deadline" > /sys/block/sda/queue/scheduler + warn "I/O scheduler: $(cat /sys/block/sda/queue/scheduler)" +fi + # Which distribution are we testing on. case "$OS_RELEASE_ID" in ubuntu*) ;; fedora*) - if ((CONTAINER==0)); then # Not yet running inside a container + if ((CONTAINER==0)); then msg "Configuring / Expanding host storage." # VM is setup to allow flexibility in testing alternate storage. # For general use, simply make use of all available space. - ooe.sh bash "$SCRIPT_BASE/add_second_partition.sh" + bash "$SCRIPT_BASE/add_second_partition.sh" + $SCRIPT_BASE/logcollector.sh df # All SELinux distros need this for systemd-in-a-container msg "Enabling container_manage_cgroup" @@ -91,15 +120,13 @@ case "$TEST_ENVIRON" in host) if [[ "$OS_RELEASE_ID" == "fedora" ]]; then # The e2e tests wrongly guess `--cgroup-manager cgroupfs` - msg "Forcing CGROUP_MANAGER=systemd" - _cgm="export CGROUP_MANAGER=systemd" - echo "$_cgm" >> /etc/environment - source /etc/environment + warn "Forcing CGROUP_MANAGER=systemd" + echo "CGROUP_MANAGER=systemd" >> /etc/ci_environment fi ;; container) if ((CONTAINER==0)); then # not yet inside a container - msg "Force loading iptables modules" + warn "Force loading iptables modules" # Since CRIU 3.11, uses iptables to lock and unlock # the network during checkpoint and restore. Needs # the following two modules loaded on the host. @@ -107,10 +134,8 @@ case "$TEST_ENVIRON" in modprobe iptable_nat || : else # The e2e tests wrongly guess `--cgroup-manager systemd` - msg "Forcing CGROUP_MANAGER=cgroupfs" - _cgm="export CGROUP_MANAGER=cgroupfs" - echo "$_cgm" >> /etc/environment - source /etc/environment + warn "Forcing CGROUP_MANAGER=cgroupfs" + echo "CGROUP_MANAGER=cgroupfs" >> /etc/ci_environment fi ;; *) die_unknown TEST_ENVIRON @@ -123,15 +148,14 @@ case "$PRIV_NAME" in if [[ "$TEST_ENVIRON" == "container" ]] && ((container)); then # There's no practical way to detect userns w/in a container # affected/related tests are sensitive to this variable. - _suns='export SKIP_USERNS=1' - echo "$_suns" >> /etc/environment - source /etc/environment + warn "Disabling usernamespace integration testing" + echo "SKIP_USERNS=1" >> /etc/ci_environment fi ;; rootless) - _ru="export ROOTLESS_USER='${ROOTLESS_USER:-some${RANDOM}dude}'" - echo "$_ru" >> /etc/environment - source /etc/environment + # Needs to exist for setup_rootless() + ROOTLESS_USER="${ROOTLESS_USER:-some${RANDOM}dude}" + echo "ROOTLESS_USER=$ROOTLESS_USER" >> /etc/ci_environment setup_rootless ;; *) die_unknown PRIV_NAME @@ -178,11 +202,18 @@ case "$TEST_FLAVOR" in remove_packaged_podman_files make install PREFIX=/usr ETCDIR=/etc fi + + install_test_configs ;; vendor) make clean ;; release) ;; *) die_unknown TEST_FLAVOR esac -# Must be the very last command. Establishes successful setup. -echo 'export SETUP_ENVIRONMENT=1' >> /etc/environment +# Must be the very last command. Prevents setup from running twice. +echo 'SETUP_ENVIRONMENT=1' >> /etc/ci_environment +echo -e "\n# End of global variable definitions" \ + >> /etc/ci_environment + +msg "Global CI Environment vars.:" +grep -Ev '^#' /etc/ci_environment | sort | indent |