summaryrefslogtreecommitdiff
path: root/contrib/cirrus/lib.sh
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/cirrus/lib.sh')
-rw-r--r--contrib/cirrus/lib.sh266
1 files changed, 114 insertions, 152 deletions
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index a285b133b..334202aa9 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -3,35 +3,92 @@
# Library of common, shared utility functions. This file is intended
# to be sourced by other scripts, not called directly.
+# Global details persist here
+source /etc/environment # not always loaded under all circumstances
+
# Under some contexts these values are not set, make sure they are.
-export USER="$(whoami)"
-export HOME="$(getent passwd $USER | cut -d : -f 6)"
-
-# These are normally set by cirrus, but can't be for VMs setup by hack/get_ci_vm.sh
-# Pick some reasonable defaults
-ENVLIB=${ENVLIB:-.bash_profile}
-CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-/var/tmp/go/src/github.com/containers/libpod}"
-GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}"
+USER="$(whoami)"
+HOME="$(getent passwd $USER | cut -d : -f 6)"
+[[ -n "$UID" ]] || UID=$(getent passwd $USER | cut -d : -f 3)
+GID=$(getent passwd $USER | cut -d : -f 4)
+
+# Essential default paths, many are overriden when executing under Cirrus-CI
+export GOPATH="${GOPATH:-/var/tmp/go}"
+if type -P go &> /dev/null
+then
+ # required for go 1.12+
+ export GOCACHE="${GOCACHE:-$HOME/.cache/go-build}"
+ eval "$(go env)"
+ # required by make and other tools
+ export $(go env | cut -d '=' -f 1)
+
+ # Ensure compiled tooling is reachable
+ export PATH="$PATH:$GOPATH/bin"
+fi
+CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/libpod}"
+export GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}"
+export PATH="$HOME/bin:$GOPATH/bin:/usr/local/bin:$PATH"
+export LD_LIBRARY_PATH="/usr/local/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+TIMESTAMPS_FILEPATH="${TIMESTAMPS_FILEPATH:-/var/tmp/timestamps}"
+SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}"
+# Saves typing / in case location ever moves
SCRIPT_BASE=${SCRIPT_BASE:-./contrib/cirrus}
PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer}
-CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-DEADBEEF} # a human
-CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-HEAD}
-CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-FETCH_HEAD}
-SPECIALMODE="${SPECIALMODE:-none}"
-export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
-if ! [[ "$PATH" =~ "/usr/local/bin" ]]
+cd $GOSRC
+if type -P git &> /dev/null
then
- export PATH="$PATH:/usr/local/bin"
+ CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)}
+else # pick something unique and obviously not from Cirrus
+ CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-no_git_$(date +%s)}
fi
-# In ci/testing environment, ensure variables are always loaded
-if [[ -r "$HOME/$ENVLIB" ]] && [[ -n "$CI" ]]
+# Defaults when not running under CI
+export CI="${CI:-false}"
+CIRRUS_CI="${CIRRUS_CI:-false}"
+CONTINUOUS_INTEGRATION="${CONTINUOUS_INTEGRATION:-false}"
+CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod}
+CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover
+CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique
+# Vars. for image-building
+PACKER_VER="1.3.5"
+# CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
+
+# Base-images rarely change, define them here so they're out of the way.
+PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28}"
+# Google-maintained base-image names
+UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20181203a"
+# Manually produced base-image names (see $SCRIPT_BASE/README.md)
+FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245"
+# FEDORA_BASE_IMAGE: "fedora-cloud-base-30-1-2-1556821664"
+PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-28-1-1-1544474897"
+# PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245"
+BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}"
+
+# Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly)
+ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)'
+# Unsafe env. vars for display
+SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)'
+
+SPECIALMODE="${SPECIALMODE:-none}"
+TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}"
+export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
+
+# When running as root, this may be empty or not, as a user, it MUST be set.
+if [[ "$USER" == "root" ]]
then
- # Make sure this is always loaded
- source "$HOME/$ENVLIB"
+ ROOTLESS_USER="${ROOTLESS_USER:-}"
+else
+ ROOTLESS_USER="${ROOTLESS_USER:-$USER}"
fi
+# GCE image-name compatible string representation of distribution name
+OS_RELEASE_ID="$(source /etc/os-release; echo $ID)"
+# GCE image-name compatible string representation of distribution _major_ version
+OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)"
+# Combined to ease soe usage
+OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
+
# Pass in a list of one or more envariable names; exit non-zero with
# helpful error message if any value is empty
req_env_var() {
@@ -57,81 +114,30 @@ req_env_var() {
done
}
-# Some env. vars may contain secrets. Display values for known "safe"
-# and useful variables.
-# ref: https://cirrus-ci.org/guide/writing-tasks/#environment-variables
show_env_vars() {
- # This is almost always multi-line, print it separately
- echo "export CIRRUS_CHANGE_MESSAGE=$CIRRUS_CHANGE_MESSAGE"
- echo "
-BUILDTAGS $BUILDTAGS
-BUILT_IMAGE_SUFFIX $BUILT_IMAGE_SUFFIX
-ROOTLESS_USER $ROOTLESS_USER
-CI $CI
-CIRRUS_CI $CIRRUS_CI
-CI_NODE_INDEX $CI_NODE_INDEX
-CI_NODE_TOTAL $CI_NODE_TOTAL
-CONTINUOUS_INTEGRATION $CONTINUOUS_INTEGRATION
-CIRRUS_BASE_BRANCH $CIRRUS_BASE_BRANCH
-CIRRUS_BASE_SHA $CIRRUS_BASE_SHA
-CIRRUS_BRANCH $CIRRUS_BRANCH
-CIRRUS_BUILD_ID $CIRRUS_BUILD_ID
-CIRRUS_CHANGE_IN_REPO $CIRRUS_CHANGE_IN_REPO
-CIRRUS_CLONE_DEPTH $CIRRUS_CLONE_DEPTH
-CIRRUS_DEFAULT_BRANCH $CIRRUS_DEFAULT_BRANCH
-CIRRUS_PR $CIRRUS_PR
-CIRRUS_TAG $CIRRUS_TAG
-CIRRUS_OS $CIRRUS_OS
-OS $OS
-CIRRUS_TASK_NAME $CIRRUS_TASK_NAME
-CIRRUS_TASK_ID $CIRRUS_TASK_ID
-CIRRUS_REPO_NAME $CIRRUS_REPO_NAME
-CIRRUS_REPO_OWNER $CIRRUS_REPO_OWNER
-CIRRUS_REPO_FULL_NAME $CIRRUS_REPO_FULL_NAME
-CIRRUS_REPO_CLONE_URL $CIRRUS_REPO_CLONE_URL
-CIRRUS_SHELL $CIRRUS_SHELL
-CIRRUS_USER_COLLABORATOR $CIRRUS_USER_COLLABORATOR
-CIRRUS_USER_PERMISSION $CIRRUS_USER_PERMISSION
-CIRRUS_WORKING_DIR $CIRRUS_WORKING_DIR
-CIRRUS_HTTP_CACHE_HOST $CIRRUS_HTTP_CACHE_HOST
-SPECIALMODE $SPECIALMODE
-$(go env)
-PACKER_BUILDS $PACKER_BUILDS
- " | while read NAME VALUE
+ echo "Showing selection of environment variable definitions:"
+ _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \
+ egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \
+ sort -u)
+ for _env_var_name in $_ENV_VAR_NAMES
do
- [[ -z "$NAME" ]] || echo "export $NAME=\"$VALUE\""
+ # Supports older BASH versions
+ printf " ${_env_var_name}=%q\n" "$(printenv $_env_var_name)"
done
echo ""
echo "##### $(go version) #####"
echo ""
}
-# Unset environment variables not needed for testing purposes
-clean_env() {
- req_env_var UNSET_ENV_VARS
- echo "Unsetting $(echo $UNSET_ENV_VARS | wc -w) environment variables"
- unset -v UNSET_ENV_VARS $UNSET_ENV_VARS || true # don't fail on read-only
-}
-
die() {
- echo "${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}"
+ echo "************************************************"
+ echo ">>>>> ${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}"
+ echo "************************************************"
exit ${1:-1}
}
-# Return a GCE image-name compatible string representation of distribution name
-os_release_id() {
- eval "$(egrep -m 1 '^ID=' /etc/os-release | tr -d \' | tr -d \")"
- echo "$ID"
-}
-
-# Return a GCE image-name compatible string representation of distribution major version
-os_release_ver() {
- eval "$(egrep -m 1 '^VERSION_ID=' /etc/os-release | tr -d \' | tr -d \")"
- echo "$VERSION_ID" | cut -d '.' -f 1
-}
-
bad_os_id_ver() {
- echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $ARGS"
+ echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)"
exit 42
}
@@ -140,8 +146,8 @@ stub() {
}
ircmsg() {
- req_env_var CIRRUS_TASK_ID
- [[ -n "$*" ]] || die 9 "ircmsg() invoked without args"
+ req_env_var CIRRUS_TASK_ID IRCID
+ [[ -n "$*" ]] || die 9 "ircmsg() invoked without message text argument"
# Sometimes setup_environment.sh didn't run
SCRIPT="$(dirname $0)/podbot.py"
NICK="podbot_$CIRRUS_TASK_ID"
@@ -153,8 +159,9 @@ ircmsg() {
}
setup_rootless() {
- req_env_var ROOTLESS_USER GOSRC ENVLIB
+ req_env_var ROOTLESS_USER GOSRC
+ # Only do this once
if passwd --status $ROOTLESS_USER
then
echo "Updating $ROOTLESS_USER user permissions on possibly changed libpod code"
@@ -162,12 +169,7 @@ setup_rootless() {
return 0
fi
- # Only do this once
cd $GOSRC
- make install.catatonit
- go get github.com/onsi/ginkgo/ginkgo
- go get github.com/onsi/gomega/...
-
# Guarantee independence from specific values
ROOTLESS_UID=$[RANDOM+1000]
ROOTLESS_GID=$[RANDOM+1000]
@@ -177,7 +179,8 @@ setup_rootless() {
chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOSRC"
echo "creating ssh keypair for $USER"
- ssh-keygen -P "" -f $HOME/.ssh/id_rsa
+ [[ -r "$HOME/.ssh/id_rsa" ]] || \
+ ssh-keygen -P "" -f "$HOME/.ssh/id_rsa"
echo "Allowing ssh key for $ROOTLESS_USER"
(umask 077 && mkdir "/home/$ROOTLESS_USER/.ssh")
@@ -192,16 +195,19 @@ setup_rootless() {
echo "${ROOTLESS_USER}:$[ROOTLESS_UID * 100]:65536" | \
tee -a /etc/subuid >> /etc/subgid
- echo "Copying $HOME/$ENVLIB"
- install -o $ROOTLESS_USER -g $ROOTLESS_USER -m 0700 \
- "$HOME/$ENVLIB" "/home/$ROOTLESS_USER/$ENVLIB"
-
- echo "Configuring user's go environment variables"
- su --login --command 'go env' $ROOTLESS_USER | \
- while read envline
- do
- X=$(echo "export $envline" | tee -a "/home/$ROOTLESS_USER/$ENVLIB") && echo "$X"
- done
+ # Env. vars set by Cirrus and setup_environment.sh must be explicitly
+ # transfered to the test-user.
+ echo "Configuring rootless user's environment variables:"
+ echo "# Added by $GOSRC/$SCRIPT_PATH/lib.sh setup_rootless()"
+ _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \
+ egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \
+ egrep "$ROOTLESS_ENV_RE" | \
+ sort -u)
+ for _env_var_name in $_ENV_VAR_NAMES
+ do
+ # Works with older versions of bash
+ printf "${_env_var_name}=%q\n" "$(printenv $_env_var_name)" >> "/home/$ROOTLESS_USER/.bashrc"
+ done
}
# Helper/wrapper script to only show stderr/stdout on non-zero exit
@@ -239,6 +245,7 @@ install_cni_plugins() {
}
install_runc_from_git(){
+ req_env_var GOPATH OS_RELEASE_ID RUNC_COMMIT
wd=$(pwd)
DEST="$GOPATH/src/github.com/opencontainers/runc"
rm -rf "$DEST"
@@ -246,13 +253,17 @@ install_runc_from_git(){
cd "$DEST"
ooe.sh git fetch origin --tags
ooe.sh git checkout -q "$RUNC_COMMIT"
- ooe.sh make static BUILDTAGS="seccomp apparmor selinux"
+ if [[ "${OS_RELEASE_ID}" == "ubuntu" ]]
+ then
+ ooe.sh make static BUILDTAGS="seccomp apparmor"
+ else
+ ooe.sh make BUILDTAGS="seccomp selinux"
+ fi
sudo install -m 755 runc /usr/bin/runc
cd $wd
}
install_runc(){
- OS_RELEASE_ID=$(os_release_id)
echo "Installing RunC from commit $RUNC_COMMIT"
echo "Platform is $OS_RELEASE_ID"
req_env_var GOPATH RUNC_COMMIT OS_RELEASE_ID
@@ -298,8 +309,6 @@ install_conmon(){
}
install_criu(){
- OS_RELEASE_ID=$(os_release_id)
- OS_RELEASE_VER=$(os_release_ver)
echo "Installing CRIU"
echo "Installing CRIU from commit $CRIU_COMMIT"
echo "Platform is $OS_RELEASE_ID"
@@ -309,21 +318,6 @@ install_criu(){
ooe.sh sudo -E add-apt-repository -y ppa:criu/ppa
ooe.sh sudo -E apt-get -qq -y update
ooe.sh sudo -E apt-get -qq -y install criu
- elif [[ ( "$OS_RELEASE_ID" =~ "centos" || "$OS_RELEASE_ID" =~ "rhel" ) && "$OS_RELEASE_VER" =~ "7"* ]]; then
- echo "Configuring Repositories for latest CRIU"
- ooe.sh sudo tee /etc/yum.repos.d/adrian-criu-el7.repo <<EOF
-[adrian-criu-el7]
-name=Copr repo for criu-el7 owned by adrian
-baseurl=https://copr-be.cloud.fedoraproject.org/results/adrian/criu-el7/epel-7-$basearch/
-type=rpm-md
-skip_if_unavailable=True
-gpgcheck=1
-gpgkey=https://copr-be.cloud.fedoraproject.org/results/adrian/criu-el7/pubkey.gpg
-repo_gpgcheck=0
-enabled=1
-enabled_metadata=1
-EOF
- ooe.sh sudo yum -y install criu
elif [[ "$OS_RELEASE_ID" =~ "fedora" ]]; then
echo "Using CRIU from distribution"
else
@@ -338,16 +332,6 @@ EOF
fi
}
-install_packer_copied_files(){
- # Install cni config, policy and registry config
- sudo install -D -m 755 /tmp/libpod/cni/87-podman-bridge.conflist \
- /etc/cni/net.d/87-podman-bridge.conflist
- sudo install -D -m 755 /tmp/libpod/test/policy.json \
- /etc/containers/policy.json
- sudo install -D -m 755 /tmp/libpod/test/redhat_sigstore.yaml \
- /etc/containers/registries.d/registry.access.redhat.com.yaml
-}
-
install_varlink() {
echo "Installing varlink from the cheese-factory"
ooe.sh sudo -H pip3 install varlink
@@ -376,7 +360,7 @@ rh_finalize(){
fi
echo "Resetting to fresh-state for usage as cloud-image."
PKG=$(type -P dnf || type -P yum || echo "")
- [[ -z "$PKG" ]] || sudo $PKG clean all # not on atomic
+ sudo $PKG clean all
sudo rm -rf /var/cache/{yum,dnf}
sudo rm -f /etc/udev/rules.d/*-persistent-*.rules
sudo touch /.unconfigured # force firstboot to run
@@ -389,25 +373,3 @@ ubuntu_finalize(){
sudo rm -rf /var/cache/apt
_finalize
}
-
-rhel_exit_handler() {
- set +ex
- req_env_var GOPATH RHSMCMD
- cd /
- sudo rm -rf "$RHSMCMD"
- sudo rm -rf "$GOPATH"
- sudo subscription-manager remove --all
- sudo subscription-manager unregister
- sudo subscription-manager clean
-}
-
-rhsm_enable() {
- req_env_var RHSM_COMMAND
- export GOPATH="$(mktemp -d)"
- export RHSMCMD="$(mktemp)"
- trap "rhel_exit_handler" EXIT
- # Avoid logging sensitive details
- echo "$RHSM_COMMAND" > "$RHSMCMD"
- ooe.sh sudo bash "$RHSMCMD"
- sudo rm -rf "$RHSMCMD"
-}
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2024-10-04 08:26:19 +0000