Merge pull request #6784 from cevich/new_ci_cd

Cirrus: Make efficient and performant

1 files changed, 133 insertions, 246 deletions

diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 0dbb57ab3..b57aee1af 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -3,243 +3,161 @@
 # Library of common, shared utility functions.  This file is intended
 # to be sourced by other scripts, not called directly.
 
-# Global details persist here
-source /etc/environment  # not always loaded under all circumstances
-
-# Automation environment doesn't automatically load for Ubuntu 18
-if [[ -r '/usr/share/automation/environment' ]]; then
-    source '/usr/share/automation/environment'
+# BEGIN Global export of all variables
+set -a
+
+# Due to differences across platforms and runtime execution environments,
+# handling of the (otherwise) default shell setup is non-uniform.  Rather
+# than attempt to workaround differences, simply force-load/set required
+# items every time this library is utilized.
+source /etc/profile
+source /etc/environment
+USER="$(whoami)"
+HOME="$(getent passwd $USER | cut -d : -f 6)"
+# Some platforms set and make this read-only
+[[ -n "$UID" ]] || \
+    UID=$(getent passwd $USER | cut -d : -f 3)
+GID=$(getent passwd $USER | cut -d : -f 4)
+
+# During VM Image build, the 'containers/automation' installation
+# was performed.  The final step of that installation sets the
+# installation location in $AUTOMATION_LIB_PATH in /etc/environment
+# or in the default shell profile.
+# shellcheck disable=SC2154
+if [[ -n "$AUTOMATION_LIB_PATH" ]]; then
+    for libname in defaults anchors console_output utils; do
+        # There's no way shellcheck can process this location
+        # shellcheck disable=SC1090
+        source $AUTOMATION_LIB_PATH/${libname}.sh
+    done
+else
+    (
+    echo "WARNING: It does not appear that containers/automation was installed."
+    echo "         Functionality of most of this library will be negatively impacted"
+    echo "         This ${BASH_SOURCE[0]} was loaded by ${BASH_SOURCE[1]}"
+    ) > /dev/stderr
 fi
 
-# Under some contexts these values are not set, make sure they are.
-export USER="$(whoami)"
-export HOME="$(getent passwd $USER | cut -d : -f 6)"
-[[ -n "$UID" ]] || export UID=$(getent passwd $USER | cut -d : -f 3)
-export GID=$(getent passwd $USER | cut -d : -f 4)
+OS_RELEASE_ID="$(source /etc/os-release; echo $ID)"
+# GCE image-name compatible string representation of distribution _major_ version
+OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)"
+# Combined to ease soe usage
+OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
 
 # Essential default paths, many are overridden when executing under Cirrus-CI
-export GOPATH="${GOPATH:-/var/tmp/go}"
+GOPATH="${GOPATH:-/var/tmp/go}"
 if type -P go &> /dev/null
 then
-    # required for go 1.12+
-    export GOCACHE="${GOCACHE:-$HOME/.cache/go-build}"
+    # Cirrus-CI caches $GOPATH contents
+    export GOCACHE="${GOCACHE:-$GOPATH/cache/go-build}"
     # called processes like `make` and other tools need these vars.
     eval "export $(go env)"
 
     # Ensure compiled tooling is reachable
-    export PATH="$PATH:$GOPATH/bin"
+    PATH="$PATH:$GOPATH/bin:$HOME/.local/bin"
 fi
-CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/podman}"
-export GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}"
-export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"
-export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$(realpath $(dirname ${BASH_SOURCE[0]})/../../)}"
+GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}"
+PATH="$HOME/bin:/usr/local/bin:$PATH"
+LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+
 # Saves typing / in case location ever moves
 SCRIPT_BASE=${SCRIPT_BASE:-./contrib/cirrus}
-# Important filepaths
-SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}"
+
 # Downloaded, but not installed packages.
 PACKAGE_DOWNLOAD_DIR=/var/cache/download
 
 # Log remote-client system test varlink output here
-export VARLINK_LOG=/var/tmp/varlink.log
-
-cd $GOSRC
-if type -P git &> /dev/null && [[ -d "$GOSRC/.git" ]]
-then
-    CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)}
-else # pick something unique and obviously not from Cirrus
-    CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-unknown_$(date +%s)}
-fi
+PODMAN_SERVER_LOG=$CIRRUS_WORKING_DIR/varlink.log
 
 # Defaults when not running under CI
 export CI="${CI:-false}"
 CIRRUS_CI="${CIRRUS_CI:-false}"
 DEST_BRANCH="${DEST_BRANCH:-master}"
 CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}"
-CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod}
+CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-podman}
 CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)}  # difficult to reliably discover
 CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)}  # must be short and unique
 
-OS_RELEASE_ID="$(source /etc/os-release; echo $ID)"
-# GCE image-name compatible string representation of distribution _major_ version
-OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)"
-# Combined to ease soe usage
-OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
+# Needed for linting and code validation
+EPOCH_TEST_COMMIT=${CIRRUS_BASE_SHA:-$CIRRUS_LAST_GREEN_CHANGE}
 
-# IN_PODMAN container image
-IN_PODMAN_IMAGE="quay.io/libpod/${OS_RELEASE_ID}_podman:$_BUILT_IMAGE_SUFFIX"
-# Image for uploading releases
-UPLDREL_IMAGE="quay.io/libpod/upldrel:master"
-
-# This is needed under some environments/contexts
-SUDO=''
-[[ "$UID" -eq 0 ]] || \
-    SUDO='sudo -E'
-
-# Avoid getting stuck waiting for user input
-export DEBIAN_FRONTEND="noninteractive"
-SUDOAPTGET="$SUDO apt-get -qq --yes"
-SUDOAPTADD="$SUDO add-apt-repository --yes"
-# Regex that finds enabled periodic apt configuration items
-PERIODIC_APT_RE='^(APT::Periodic::.+")1"\;'
-# Short-cuts for retrying/timeout calls
-LILTO="timeout_attempt_delay_command 120s 5 30s"
-BIGTO="timeout_attempt_delay_command 300s 5 60s"
-
-# Safe env. vars. to transfer from root -> $ROOTLESS_USER  (go env handled separately)
-ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(REMOTE.*)'
+# Regex of env. vars. to explicitly pass when executing tests
+# inside a container or as a rootless user
+PASSTHROUGH_ENV_RE='(^CI.*)|(^CIRRUS)|(^DISTRO_NV)|(^GOPATH)|(^GOCACHE)|(^GOSRC)|(^SCRIPT_BASE)|(CGROUP_MANAGER)|(OCI_RUNTIME)|(^TEST.*)|(^PODBIN_NAME)|(^PRIV_NAME)|(^ALT_NAME)|(^ROOTLESS_USER)|(SKIP_USERNS)|(.*_NAME)|(.*_FQIN)'
 # Unsafe env. vars for display
-SECRET_ENV_RE='(ACCOUNT)|(GC[EP]..+)|(SSH)'
+SECRET_ENV_RE='(ACCOUNT)|(GC[EP]..+)|(SSH)|(PASSWORD)|(TOKEN)'
 
-SPECIALMODE="${SPECIALMODE:-none}"
-RCLI="${RCLI:-false}"
-export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
-
-# When running as root, this may be empty or not, as a user, it MUST be set.
-if [[ "$USER" == "root" ]]
-then
-    ROOTLESS_USER="${ROOTLESS_USER:-}"
-else
-    ROOTLESS_USER="${ROOTLESS_USER:-$USER}"
-fi
 # Type of filesystem used for cgroups
 CG_FS_TYPE="$(stat -f -c %T /sys/fs/cgroup)"
 
-# Pass in a list of one or more envariable names; exit non-zero with
-# helpful error message if any value is empty
-req_env_var() {
-    # Provide context. If invoked from function use its name; else script name
-    local caller=${FUNCNAME[1]}
-    if [[ -n "$caller" ]]; then
-        # Indicate that it's a function name
-        caller="$caller()"
-    else
-        # Not called from a function: use script name
-        caller=$(basename $0)
-    fi
-
-    # Usage check
-    [[ -n "$1" ]] || die 1 "FATAL: req_env_var: invoked without arguments"
-
-    # Each input arg is an envariable name, e.g. HOME PATH etc. Expand each.
-    # If any is empty, bail out and explain why.
-    for i; do
-        if [[ -z "${!i}" ]]; then
-            die 9 "FATAL: $caller requires \$$i to be non-empty"
-        fi
+# Set to 1 in all podman container images
+CONTAINER="${CONTAINER:-0}"
+
+# END Global export of all variables
+set +a
+
+lilto() { err_retry 8 1000 "" "$@"; }  # just over 4 minutes max
+bigto() { err_retry 7 5670 "" "$@"; }  # 12 minutes max
+
+# Print shell-escaped variable=value pairs, one per line, based on
+# variable name matching a regex.  This is intended to support
+# passthrough of CI variables from host -> container or from root -> user.
+# For all other vars. we rely on tooling to load this library from inside
+# the container or as rootless user to pickup the remainder.
+passthrough_envars(){
+    local xchars
+    local envname
+    local envval
+    # Avoid values containing entirely punctuation|control|whitespace
+    xchars='[:punct:][:cntrl:][:space:]'
+    warn "Will pass env. vars. matching the following regex:
+    $PASSTHROUGH_ENV_RE"
+    for envname in $(awk 'BEGIN{for(v in ENVIRON) print v}' | \
+                         grep -Ev "SETUP_ENVIRONMENT" | \
+                         grep -Ev "$SECRET_ENV_RE" | \
+                         grep -E "$PASSTHROUGH_ENV_RE"); do
+
+            envval="${!envname}"
+            [[ -n $(tr -d "$xchars" <<<"$envval") ]] || continue
+
+            # Properly escape values to prevent injection
+            printf -- "$envname=%q\n" "$envval"
     done
 }
 
-item_test() {
-    ITEM="$1"
-    shift
-    TEST_ARGS="$@"
-    req_env_var ITEM TEST_ARGS
-
-    if ERR=$(test "$@" 2>&1)
-    then
-        echo "ok $ITEM"
-        return 0
-    else
-        RET=$?
-        echo -n "not ok $ITEM: $TEST_ARGS"
-        if [[ -z "$ERR" ]]
-        then
-            echo ""
-        else  # test command itself failed
-            echo -n ":"  # space follows :'s in $ERR
-            echo "$ERR" | cut -d : -f 4-  # omit filename, line number, and command
-        fi
-        return $RET
-    fi
-}
-
-show_env_vars() {
-    echo "Showing selection of environment variable definitions:"
-    _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \
-        egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \
-        sort -u)
-    for _env_var_name in $_ENV_VAR_NAMES
-    do
-        # Supports older BASH versions
-        printf "    ${_env_var_name}=%q\n" "$(printenv $_env_var_name)"
-    done
-}
-
-die() {
-    echo "************************************************"
-    echo ">>>>> ${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}"
-    echo "************************************************"
-    exit ${1:-1}
-}
-
-warn() {
-    echo ">>>>> ${1:-WARNING (but no message given!) in ${FUNCNAME[1]}()}" > /dev/stderr
-}
-
-bad_os_id_ver() {
-    echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)"
-    exit 42
-}
-
-stub() {
-    echo "STUB: Pretending to do $1"
-}
-
-timeout_attempt_delay_command() {
-    TIMEOUT=$1
-    ATTEMPTS=$2
-    DELAY=$3
-    shift 3
-    STDOUTERR=$(mktemp -p '' $(basename $0)_XXXXX)
-    req_env_var ATTEMPTS DELAY
-    echo "Retrying $ATTEMPTS times with a $DELAY delay, and $TIMEOUT timeout for command: $@"
-    for (( COUNT=1 ; COUNT <= $ATTEMPTS ; COUNT++ ))
-    do
-        echo "##### (attempt #$COUNT)" &>> "$STDOUTERR"
-        if timeout --foreground $TIMEOUT "$@" &>> "$STDOUTERR"
-        then
-            echo "##### (success after #$COUNT attempts)" &>> "$STDOUTERR"
-            break
-        else
-            echo "##### (failed with exit: $?)" &>> "$STDOUTERR"
-            sleep $DELAY
-        fi
-    done
-    cat "$STDOUTERR"
-    rm -f "$STDOUTERR"
-    if (( COUNT > $ATTEMPTS ))
-    then
-        echo "##### (exceeded $ATTEMPTS attempts)"
-        exit 125
-    fi
-}
-
 setup_rootless() {
-    req_env_var ROOTLESS_USER GOPATH GOSRC SECRET_ENV_RE ROOTLESS_ENV_RE
+    req_env_vars ROOTLESS_USER GOPATH GOSRC SECRET_ENV_RE
+
+    local rootless_uid
+    local rootless_gid
+    local env_var_val
 
-    # Only do this once
+    # Only do this once; established by setup_environment.sh
+    # shellcheck disable=SC2154
     if passwd --status $ROOTLESS_USER
     then
-        echo "Updating $ROOTLESS_USER user permissions on possibly changed libpod code"
+        msg "Updating $ROOTLESS_USER user permissions on possibly changed libpod code"
         chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOPATH" "$GOSRC"
         return 0
     fi
-
-    cd $GOSRC
+    msg "************************************************************"
+    msg "Setting up rootless user '$ROOTLESS_USER'"
+    msg "************************************************************"
+    cd $GOSRC || exit 1
     # Guarantee independence from specific values
-    ROOTLESS_UID=$[RANDOM+1000]
-    ROOTLESS_GID=$[RANDOM+1000]
-    echo "creating $ROOTLESS_UID:$ROOTLESS_GID $ROOTLESS_USER user"
-    groupadd -g $ROOTLESS_GID $ROOTLESS_USER
-    useradd -g $ROOTLESS_GID -u $ROOTLESS_UID --no-user-group --create-home $ROOTLESS_USER
+    rootless_uid=$[RANDOM+1000]
+    rootless_gid=$[RANDOM+1000]
+    msg "creating $rootless_uid:$rootless_gid $ROOTLESS_USER user"
+    groupadd -g $rootless_gid $ROOTLESS_USER
+    useradd -g $rootless_gid -u $rootless_uid --no-user-group --create-home $ROOTLESS_USER
     chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOPATH" "$GOSRC"
 
-    echo "creating ssh key pair for $USER"
+    msg "creating ssh key pair for $USER"
     [[ -r "$HOME/.ssh/id_rsa" ]] || \
         ssh-keygen -P "" -f "$HOME/.ssh/id_rsa"
 
-    echo "Allowing ssh key for $ROOTLESS_USER"
+    msg "Allowing ssh key for $ROOTLESS_USER"
     (umask 077 && mkdir "/home/$ROOTLESS_USER/.ssh")
     chown -R $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.ssh"
     install -o $ROOTLESS_USER -g $ROOTLESS_USER -m 0600 \
@@ -247,49 +165,38 @@ setup_rootless() {
     # Makes debugging easier
     cat /root/.ssh/authorized_keys >> "/home/$ROOTLESS_USER/.ssh/authorized_keys"
 
-    echo "Configuring subuid and subgid"
+    msg "Configuring subuid and subgid"
     grep -q "${ROOTLESS_USER}" /etc/subuid || \
-        echo "${ROOTLESS_USER}:$[ROOTLESS_UID * 100]:65536" | \
+        echo "${ROOTLESS_USER}:$[rootless_uid * 100]:65536" | \
             tee -a /etc/subuid >> /etc/subgid
 
     # Env. vars set by Cirrus and setup_environment.sh must be explicitly
     # transferred to the test-user.
-    echo "Configuring rootless user's environment variables:"
-    echo "# Added by $GOSRC/$SCRIPT_PATH/lib.sh setup_rootless()"
-    _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \
-        egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \
-        egrep "$ROOTLESS_ENV_RE" | \
-        sort -u)
-    for _env_var_name in $_ENV_VAR_NAMES
-    do
-        # Works with older versions of bash
-        printf "${_env_var_name}=%q\n" "$(printenv $_env_var_name)" >> "/home/$ROOTLESS_USER/.bashrc"
-    done
+    msg "Configuring rootless user's environment variables:"
+
+    (
+        echo "# Added by ${BASH_SOURCE[0]} ${FUNCNAME[0]}()"
+        echo "export SETUP_ENVIRONMENT=1"
+    ) >> "/home/$ROOTLESS_USER/.bashrc"
 
-    echo "Ensure the systems ssh process is up and running within 5 minutes"
+    while read -r env_var_val; do
+        echo "export $env_var_val" >> "/home/$ROOTLESS_USER/.bashrc"
+    done <<<"$(passthrough_envars)"
+    chown $ROOTLESS_USER:$ROOTLESS_USER "/home/$ROOTLESS_USER/.bashrc"
+    cat "/home/$ROOTLESS_USER/.bashrc" | indent 2
+
+    msg "Ensure the systems ssh process is up and running within 5 minutes"
     systemctl start sshd
-    NOW=$(date +%s)
-    TIMEOUT=$(date --date '+5 minutes' +%s)
-    while [[ "$(date +%s)" -lt "$TIMEOUT" ]]
-    do
-        if timeout --foreground -k 1s 1s \
-            ssh $ROOTLESS_USER@localhost \
-            -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \
-            true
-        then
-            break
-        else
-            sleep 2s
-        fi
-    done
-    [[ "$(date +%s)" -lt "$TIMEOUT" ]] || \
-        die 11 "Timeout exceeded waiting for localhost ssh capability"
+    lilto ssh $ROOTLESS_USER@localhost \
+           -o UserKnownHostsFile=/dev/null \
+           -o StrictHostKeyChecking=no \
+           -o CheckHostIP=no true
 }
 
 install_test_configs() {
     echo "Installing cni config, policy and registry config"
-    req_env_var GOSRC SCRIPT_BASE
-    cd $GOSRC
+    req_env_vars GOSRC SCRIPT_BASE
+    cd $GOSRC || exit 1
     install -v -D -m 644 ./cni/87-podman-bridge.conflist /etc/cni/net.d/
     # This config must always sort last in the list of networks (podman picks first one
     # as the default).  This config prevents allocation of network address space used
@@ -306,7 +213,7 @@ install_test_configs() {
 # can only run the compiled source version.
 remove_packaged_podman_files() {
     echo "Removing packaged podman files to prevent conflicts with source build and testing."
-    req_env_var OS_RELEASE_ID
+    req_env_vars OS_RELEASE_ID
 
     # If any binaries are resident they could cause unexpected pollution
     for unit in io.podman.service io.podman.socket
@@ -324,9 +231,9 @@ remove_packaged_podman_files() {
 
     if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]]
     then
-        LISTING_CMD="$SUDO dpkg-query -L podman"
+        LISTING_CMD="dpkg-query -L podman"
     else
-        LISTING_CMD="$SUDO rpm -ql podman"
+        LISTING_CMD="rpm -ql podman"
     fi
 
     # yum/dnf/dpkg may list system directories, only remove files
@@ -334,29 +241,9 @@ remove_packaged_podman_files() {
     do
         # Sub-directories may contain unrelated/valuable stuff
         if [[ -d "$fullpath" ]]; then continue; fi
-        ooe.sh $SUDO rm -vf "$fullpath"
+        ooe.sh rm -vf "$fullpath"
     done
 
     # Be super extra sure and careful vs performant and completely safe
-    sync && echo 3 > /proc/sys/vm/drop_caches
-}
-
-# The version of CRI-O and Kubernetes must always match
-get_kubernetes_version(){
-    # TODO: Look up the kube RPM/DEB version installed, or in $PACKAGE_DOWNLOAD_DIR
-    #       and retrieve the major-minor version directly.
-    local KUBERNETES_VERSION="1.15"
-    echo "$KUBERNETES_VERSION"
-}
-
-canonicalize_image_names() {
-    req_env_var IMGNAMES
-    echo "Adding all current base images to \$IMGNAMES for timestamp update"
-    export IMGNAMES="\
-$IMGNAMES
-$UBUNTU_BASE_IMAGE
-$PRIOR_UBUNTU_BASE_IMAGE
-$FEDORA_BASE_IMAGE
-$PRIOR_FEDORA_BASE_IMAGE
-"
+    sync && echo 3 > /proc/sys/vm/drop_caches || true
 }