diff options
Diffstat (limited to 'contrib/cirrus')
| -rw-r--r-- | contrib/cirrus/README.md | 29 | ||||
| -rwxr-xr-x | contrib/cirrus/build_vm_images.sh | 2 | ||||
| -rwxr-xr-x | contrib/cirrus/integration_test.sh | 11 | ||||
| -rw-r--r-- | contrib/cirrus/lib.sh | 77 | ||||
| -rwxr-xr-x | contrib/cirrus/lib.sh.t | 17 | ||||
| -rw-r--r-- | contrib/cirrus/packer/Makefile | 15 | ||||
| -rw-r--r-- | contrib/cirrus/packer/centos_setup.sh | 84 | ||||
| -rw-r--r-- | contrib/cirrus/packer/fah_base-setup.sh | 45 | ||||
| -rw-r--r-- | contrib/cirrus/packer/fah_setup.sh | 21 | ||||
| -rw-r--r-- | contrib/cirrus/packer/fedora_setup.sh | 2 | ||||
| -rw-r--r-- | contrib/cirrus/packer/libpod_base_images.yml | 53 | ||||
| -rw-r--r-- | contrib/cirrus/packer/libpod_images.yml | 18 | ||||
| -rw-r--r-- | contrib/cirrus/packer/rhel_base-setup.sh | 78 | ||||
| -rw-r--r-- | contrib/cirrus/packer/rhel_setup.sh | 96 | ||||
| -rwxr-xr-x | contrib/cirrus/rootless_test.sh | 7 | ||||
| -rwxr-xr-x | contrib/cirrus/setup_environment.sh | 32 | ||||
| -rwxr-xr-x | contrib/cirrus/system_test.sh | 9 |
17 files changed, 81 insertions, 515 deletions
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index 1d62f71ce..69d8653fe 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -136,11 +136,10 @@ you'll find the new image names displayed at the end of the `test_build_cache_images_task` in the `build_vm_images` output. For example: + ``` ...cut... ==> Builds finished. The artifacts of successful builds are: ---> rhel-7: A disk image was created: rhel-7-libpod-5699523102900224 ---> rhel-7: --> ubuntu-18: A disk image was created: ubuntu-18-libpod-5699523102900224 --> ubuntu-18: --> fedora-29: A disk image was created: fedora-29-libpod-5699523102900224 @@ -148,8 +147,8 @@ For example: --> fedora-28: A disk image was created: fedora-28-libpod-5699523102900224 ``` -An updated (or new) pull-request with this change, will utilize -the new cache-images: +Now edit `.cirrus.yml`, updating the `*_IMAGE_NAME` lines to reflect the +images from above: ```yaml @@ -161,14 +160,14 @@ env: FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5699523102900224" PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5699523102900224" UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5699523102900224" - PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-5699523102900224" ...cut... ``` -Take care to also update the PR description if any 'magic' phrases were used -(they affect which tests run). In other words, you'll likely want to -remove the ``***CIRRUS: TEST IMAGES***`` string - otherwise Cirrus-CI will -simply build new and test again. +***NOTE:*** If re-using the same PR with new images in `.cirrus.yml`, +take care to also *update the PR description* to remove +the magic ``***CIRRUS: TEST IMAGES***`` string. Keeping it and +`--force` pushing would needlessly cause Cirrus-CI to build +and test images again. ### ``build_cache_images`` Task *(Deprecated)* @@ -216,15 +215,6 @@ the ``cache_images`` Task) some input parameters are required: or [end-user credentials](https://cloud.google.com/docs/authentication/end-user#creating_your_client_credentials) -* ``RHEL_IMAGE_FILE`` and ``RHEL_CSUM_FILE`` complete paths - to a `rhel-server-ec2-*.raw.xz` and it's cooresponding - checksum file. These must be supplied manually because - they're not available directly via URL like other images. - -* ``RHSM_COMMAND`` contains the complete string needed to register - the VM for installing package dependencies. The VM will be de-registered - upon completion. - * Optionally, CSV's may be specified to ``PACKER_BUILDS`` to limit the base-images produced. For example, ``PACKER_BUILDS=fedora,image-builder-image``. @@ -270,9 +260,6 @@ When ready, change to the ``packer`` sub-directory, and build the images: $ cd libpod/contrib/cirrus/packer $ make libpod_base_images GCP_PROJECT_ID=<VALUE> \ GOOGLE_APPLICATION_CREDENTIALS=<VALUE> \ - RHEL_IMAGE_FILE=<VALUE> \ - RHEL_CSUM_FILE=<VALUE> \ - RHSM_COMMAND=<VALUE> \ PACKER_BUILDS=<OPTIONAL> ``` diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index 773f8e486..f4567ce74 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -3,7 +3,7 @@ set -e source $(dirname $0)/lib.sh -ENV_VARS='CNI_COMMIT CRIO_COMMIT RUNC_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX CENTOS_BASE_IMAGE UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE FAH_BASE_IMAGE RHEL_BASE_IMAGE RHSM_COMMAND SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE' +ENV_VARS='CNI_COMMIT CRIO_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE' req_env_var $ENV_VARS # Must also be made available through make, into packer process export $ENV_VARS diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index e65b3b8ec..e7f582b42 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -25,16 +25,19 @@ then elif [[ "$SPECIALMODE" == "rootless" ]] then req_env_var ROOTLESS_USER - set -x - ssh $ROOTLESS_USER@localhost \ + + if [[ "$USER" == "$ROOTLESS_USER" ]] + then + $GOSRC/$SCRIPT_BASE/rootless_test.sh + else + ssh $ROOTLESS_USER@localhost \ -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \ $GOSRC/$SCRIPT_BASE/rootless_test.sh - exit $? + fi else make make install PREFIX=/usr ETCDIR=/etc make test-binaries - make install.tools if [[ "$TEST_REMOTE_CLIENT" == "true" ]] then make remoteintegration diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 7886d230b..d0a2e1579 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -3,6 +3,9 @@ # Library of common, shared utility functions. This file is intended # to be sourced by other scripts, not called directly. +# Global details persist here +source /etc/environment # not always loaded under all circumstances + # Under some contexts these values are not set, make sure they are. USER="$(whoami)" HOME="$(getent passwd $USER | cut -d : -f 6)" @@ -18,6 +21,9 @@ then eval "$(go env)" # required by make and other tools export $(go env | cut -d '=' -f 1) + + # Ensure compiled tooling is reachable + export PATH="$PATH:$GOPATH/bin" fi CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/libpod}" export GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}" @@ -47,29 +53,25 @@ CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and uniq # Vars. for image-building PACKER_VER="1.3.5" # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) -PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28,rhel-7,centos-7}" # Base-images rarely change, define them here so they're out of the way. +PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28}" # Google-maintained base-image names UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20181203a" -CENTOS_BASE_IMAGE="centos-7-v20181113" # Manually produced base-image names (see $SCRIPT_BASE/README.md) FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245" PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-28-1-1-1544474897" -FAH_BASE_IMAGE="fedora-atomichost-29-20181025-1-1541787861" -# RHEL image must be imported, native image bills for subscription. -RHEL_BASE_IMAGE="rhel-guest-image-7-6-210-x86-64-qcow2-1548099756" BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" -RHSM_COMMAND="${RHSM_COMMAND:-/bin/true}" # Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly) ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)' # Unsafe env. vars for display -SECRET_ENV_RE='(IRCID)|(RHSM)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' +SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' SPECIALMODE="${SPECIALMODE:-none}" TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}" export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} + # When running as root, this may be empty or not, as a user, it MUST be set. if [[ "$USER" == "root" ]] then @@ -79,9 +81,9 @@ else fi # GCE image-name compatible string representation of distribution name -OS_RELEASE_ID="$(egrep -m 1 '^ID=' /etc/os-release | cut -d = -f 2 | tr -d \' | tr -d \")" -# GCE image-name compatible string representation of distribution major version -OS_RELEASE_VER="$(egrep -m 1 '^VERSION_ID=' /etc/os-release | cut -d = -f 2 | tr -d \' | tr -d \" | cut -d '.' -f 1)" +OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" +# GCE image-name compatible string representation of distribution _major_ version +OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)" # Combined to ease soe usage OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" @@ -118,8 +120,7 @@ show_env_vars() { for _env_var_name in $_ENV_VAR_NAMES do # Supports older BASH versions - _value="$(printenv $_env_var_name)" - printf " ${_env_var_name}=%q\n" "${_value}" + printf " ${_env_var_name}=%q\n" "$(printenv $_env_var_name)" done echo "" echo "##### $(go version) #####" @@ -127,12 +128,14 @@ show_env_vars() { } die() { - echo "${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}" + echo "************************************************" + echo ">>>>> ${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}" + echo "************************************************" exit ${1:-1} } bad_os_id_ver() { - echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $ARGS" + echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)" exit 42 } @@ -141,7 +144,7 @@ stub() { } ircmsg() { - req_env_var CIRRUS_TASK_ID MSG + req_env_var CIRRUS_TASK_ID IRCID [[ -n "$*" ]] || die 9 "ircmsg() invoked without message text argument" # Sometimes setup_environment.sh didn't run SCRIPT="$(dirname $0)/podbot.py" @@ -193,14 +196,15 @@ setup_rootless() { # Env. vars set by Cirrus and setup_environment.sh must be explicitly # transfered to the test-user. echo "Configuring rootless user's environment variables:" + echo "# Added by $GOSRC/$SCRIPT_PATH/lib.sh setup_rootless()" _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \ egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \ egrep "$ROOTLESS_ENV_RE" | \ sort -u) for _env_var_name in $_ENV_VAR_NAMES do - _value="$(printenv $_env_var_name)" - printf "${_env_var_name}=%q" "${_value}" | tee -a "/home/$ROOTLESS_USER/.bashrc" + # Works with older versions of bash + printf "${_env_var_name}=%q\n" "$(printenv $_env_var_name)" >> "/home/$ROOTLESS_USER/.bashrc" done } @@ -312,21 +316,6 @@ install_criu(){ ooe.sh sudo -E add-apt-repository -y ppa:criu/ppa ooe.sh sudo -E apt-get -qq -y update ooe.sh sudo -E apt-get -qq -y install criu - elif [[ ( "$OS_RELEASE_ID" =~ "centos" || "$OS_RELEASE_ID" =~ "rhel" ) && "$OS_RELEASE_VER" =~ "7"* ]]; then - echo "Configuring Repositories for latest CRIU" - ooe.sh sudo tee /etc/yum.repos.d/adrian-criu-el7.repo <<EOF -[adrian-criu-el7] -name=Copr repo for criu-el7 owned by adrian -baseurl=https://copr-be.cloud.fedoraproject.org/results/adrian/criu-el7/epel-7-$basearch/ -type=rpm-md -skip_if_unavailable=True -gpgcheck=1 -gpgkey=https://copr-be.cloud.fedoraproject.org/results/adrian/criu-el7/pubkey.gpg -repo_gpgcheck=0 -enabled=1 -enabled_metadata=1 -EOF - ooe.sh sudo yum -y install criu elif [[ "$OS_RELEASE_ID" =~ "fedora" ]]; then echo "Using CRIU from distribution" else @@ -369,7 +358,7 @@ rh_finalize(){ fi echo "Resetting to fresh-state for usage as cloud-image." PKG=$(type -P dnf || type -P yum || echo "") - [[ -z "$PKG" ]] || sudo $PKG clean all # not on atomic + sudo $PKG clean all sudo rm -rf /var/cache/{yum,dnf} sudo rm -f /etc/udev/rules.d/*-persistent-*.rules sudo touch /.unconfigured # force firstboot to run @@ -382,25 +371,3 @@ ubuntu_finalize(){ sudo rm -rf /var/cache/apt _finalize } - -rhel_exit_handler() { - set +ex - req_env_var GOPATH RHSMCMD - cd / - sudo rm -rf "$RHSMCMD" - sudo rm -rf "$GOPATH" - sudo subscription-manager remove --all - sudo subscription-manager unregister - sudo subscription-manager clean -} - -rhsm_enable() { - req_env_var RHSM_COMMAND - export GOPATH="$(mktemp -d)" - export RHSMCMD="$(mktemp)" - trap "rhel_exit_handler" EXIT - # Avoid logging sensitive details - echo "$RHSM_COMMAND" > "$RHSMCMD" - ooe.sh sudo bash "$RHSMCMD" - sudo rm -rf "$RHSMCMD" -} diff --git a/contrib/cirrus/lib.sh.t b/contrib/cirrus/lib.sh.t index ce51f8ad2..1f05b3bb5 100755 --- a/contrib/cirrus/lib.sh.t +++ b/contrib/cirrus/lib.sh.t @@ -10,12 +10,13 @@ rc=0 function check_result { testnum=$(expr $testnum + 1) - if [ "$1" = "$2" ]; then - echo "ok $testnum $3 = $1" + MSG=$(echo "$1" | tr -d '*>\012'|sed -e 's/^ \+//') + if [ "$MSG" = "$2" ]; then + echo "ok $testnum $3 = $MSG" else echo "not ok $testnum $3" echo "# expected: $2" - echo "# actual: $1" + echo "# actual: $MSG" rc=1 fi } @@ -64,11 +65,17 @@ test_rev '' 1 'FATAL: req_env_var: invoked without arguments' unset FOO BAR test_rev FOO 9 'FATAL: test_rev() requires $FOO to be non-empty' test_rev BAR 9 'FATAL: test_rev() requires $BAR to be non-empty' - -# OK if desired envariable is unset +# OK if desired envariable was unset FOO=1 test_rev FOO 0 '' +# OK if multiple vars are non-empty +FOO="stuff" +BAR="things" +ENV_VARS="FOO BAR" +test_rev "$ENV_VARS" 0 '' +unset BAR + # ...but error if any single desired one is unset test_rev "FOO BAR" 9 'FATAL: test_rev() requires $BAR to be non-empty' diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile index 91a1dfeef..0ae0deca6 100644 --- a/contrib/cirrus/packer/Makefile +++ b/contrib/cirrus/packer/Makefile @@ -49,6 +49,7 @@ ifndef PACKER_BUILDS $(error PACKER_BUILDS is undefined, expected builder-names CSV) endif ./packer build -only=${PACKER_BUILDS} \ + -force \ -var GOSRC=$(GOSRC) \ -var PACKER_BASE=$(PACKER_BASE) \ -var SCRIPT_BASE=$(SCRIPT_BASE) \ @@ -78,16 +79,8 @@ endif ifndef GOOGLE_APPLICATION_CREDENTIALS $(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json) endif -ifndef RHEL_IMAGE_FILE - $(error RHEL_IMAGE_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz file) -endif -ifndef RHEL_CSUM_FILE - $(error RHEL_CSUM_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz.SHA256SUM file) -endif -ifndef RHSM_COMMAND - $(error RHSM_COMMAND is undefined, expected string required for temporarily registering VM) -endif PACKER_CACHE_DIR=/tmp ./packer build \ + -force \ -var TIMESTAMP=$(TIMESTAMP) \ -var TTYDEV=$(TTYDEV) \ -var GCP_PROJECT_ID=$(GCP_PROJECT_ID) \ @@ -95,10 +88,6 @@ endif -var GOSRC=$(GOSRC) \ -var PACKER_BASE=$(PACKER_BASE) \ -var SCRIPT_BASE=$(SCRIPT_BASE) \ - -var RHEL_BASE_IMAGE_NAME=$(shell basename $(RHEL_IMAGE_FILE) | tr -d '[[:space:]]' | sed -r -e 's/\.${ARCH}\.raw\.xz//' | tr '[[:upper:]]' '[[:lower:]]' | tr '[[:punct:]]' '-') \ - -var RHEL_IMAGE_FILE=$(RHEL_IMAGE_FILE) \ - -var RHEL_CSUM_FILE=$(RHEL_CSUM_FILE) \ - -var 'RHSM_COMMAND=$(RHSM_COMMAND)' \ -var POST_MERGE_BUCKET_SUFFIX=$(POST_MERGE_BUCKET_SUFFIX) \ -only $(PACKER_BUILDS) \ libpod_base_images.json diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh deleted file mode 100644 index f63625789..000000000 --- a/contrib/cirrus/packer/centos_setup.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# This script is called by packer on the subject CentOS VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source /tmp/libpod/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RUNC_COMMIT - -install_ooe - -export GOPATH="$(mktemp -d)" -trap "sudo rm -rf $GOPATH" EXIT - -ooe.sh sudo yum -y update - -ooe.sh sudo yum -y install centos-release-scl epel-release - -ooe.sh sudo yum -y install \ - PyYAML \ - atomic-registries \ - bats \ - btrfs-progs-devel \ - bzip2 \ - device-mapper-devel \ - emacs-nox \ - findutils \ - glib2-devel \ - glibc-static \ - gnupg \ - golang \ - golang-github-cpuguy83-go-md2man \ - golang-github-cpuguy83-go-md2man \ - gpgme-devel \ - iptables \ - jq \ - libassuan-devel \ - libcap-devel \ - libnet \ - libnet-devel \ - libnl3-devel \ - libseccomp-devel \ - libselinux-devel \ - lsof \ - make \ - nmap-ncat \ - ostree-devel \ - protobuf \ - protobuf-c \ - protobuf-c-devel \ - protobuf-compiler \ - protobuf-devel \ - protobuf-python \ - python \ - python2-future \ - python3-dateutil \ - python3-psutil \ - python3-pytoml \ - skopeo-containers \ - unzip \ - vim \ - which \ - xz - -install_scl_git - -install_conmon - -install_runc - -install_criu - -install_cni_plugins - -install_buildah - -sudo /tmp/libpod/hack/install_catatonit.sh - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fah_base-setup.sh b/contrib/cirrus/packer/fah_base-setup.sh deleted file mode 100644 index 606c4f336..000000000 --- a/contrib/cirrus/packer/fah_base-setup.sh +++ /dev/null @@ -1,45 +0,0 @@ - -# N/B: This script is not intended to be run by humans. It is used to configure the -# FAH base image for importing, so that it will boot in GCE. - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -install_ooe - -if [[ "$1" == "pre" ]] -then - echo "Upgrading Atomic Host" - setenforce 0 - ooe.sh atomic host upgrade - - echo "Configuring Repositories" - ooe.sh sudo tee /etc/yum.repos.d/ngompa-gce-oslogin.repo <<EOF -[ngompa-gce-oslogin] -name=Copr repo for gce-oslogin owned by ngompa -baseurl=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/fedora-\$releasever-\$basearch/ -type=rpm-md -skip_if_unavailable=True -gpgcheck=1 -gpgkey=https://copr-be.cloud.fedoraproject.org/results/ngompa/gce-oslogin/pubkey.gpg -repo_gpgcheck=0 -enabled=1 -enabled_metadata=1 -EOF - echo "Installing necessary packages and google services" - # Google services are enabled by default, upon install. - ooe.sh rpm-ostree install rng-tools google-compute-engine google-compute-engine-oslogin - echo "Rebooting..." - systemctl reboot # Required for upgrade + package installs to be active -elif [[ "$1" == "post" ]] -then - echo "Enabling necessary services" - systemctl enable rngd # Must reboot before enabling - rh_finalize - echo "SUCCESS!" -else - echo "Expected to be called with 'pre' or 'post'" - exit 6 -fi diff --git a/contrib/cirrus/packer/fah_setup.sh b/contrib/cirrus/packer/fah_setup.sh deleted file mode 100644 index 18c4db0af..000000000 --- a/contrib/cirrus/packer/fah_setup.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash - -# This script is called by packer on the subject fah VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source /tmp/libpod/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE - -install_ooe - -ooe.sh sudo atomic host upgrade - -ooe.sh sudo rpm-ostree uninstall cloud-init - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 41d1785f9..0e9867a89 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -8,7 +8,7 @@ set -e # Load in library (copied by packer, before this script was run) source /tmp/libpod/$SCRIPT_BASE/lib.sh -req_env_var SCRIPT_BASE FEDORA_CNI_COMMIT CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RUNC_COMMIT +req_env_var SCRIPT_BASE FEDORA_CNI_COMMIT CNI_COMMIT CRIO_COMMIT CRIU_COMMIT install_ooe diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml index bf568b40e..2caa28a04 100644 --- a/contrib/cirrus/packer/libpod_base_images.yml +++ b/contrib/cirrus/packer/libpod_base_images.yml @@ -11,12 +11,6 @@ variables: TIMESTAMP: # Required for output from qemu builders TTYDEV: - # RHEL images require click-through agreements to obtain (required) - RHEL_BASE_IMAGE_NAME: - RHEL_IMAGE_FILE: - RHEL_CSUM_FILE: - # RHEL requires a subscription to install/update packages - RHSM_COMMAND: # Latest Fedora release FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/29/Cloud/x86_64/images/Fedora-Cloud-Base-29-1.2.x86_64.qcow2" @@ -26,9 +20,6 @@ variables: PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-Base-28-1.1.x86_64.qcow2" PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/28/Cloud/x86_64/images/Fedora-Cloud-28-1.1-x86_64-CHECKSUM" PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-28-1-1' # Name to use in GCE - FAH_IMAGE_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1.x86_64.qcow2" - FAH_CSUM_URL: "https://dl.fedoraproject.org/pub/alt/atomic/stable/Fedora-Atomic-29-20181025.1/AtomicHost/x86_64/images/Fedora-AtomicHost-29-20181025.1-x86_64-CHECKSUM" - FAH_BASE_IMAGE_NAME: 'fedora-atomichost-29-20181025-1' # Name to use in GCE # The name of the image in GCE used for packer build libpod_images.yml IBI_BASE_NAME: 'image-builder-image' @@ -45,28 +36,9 @@ variables: sensitive-variables: - 'GOOGLE_APPLICATION_CREDENTIALS' - 'GCP_PROJECT_ID' - - 'RHSM_COMMAND' # What images to produce in which cloud builders: - - name: '{{user `IBI_BASE_NAME`}}' - type: 'googlecompute' - image_name: '{{user `IBI_BASE_NAME`}}-{{user `TIMESTAMP`}}' - image_family: '{{user `IBI_BASE_NAME`}}' - source_image_project_id: 'centos-cloud' - source_image_family: 'centos-7' - project_id: '{{user `GCP_PROJECT_ID`}}' - account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' - communicator: 'ssh' - ssh_username: 'centos' - ssh_pty: 'true' - # The only supported zone in Cirrus-CI, as of addition of this comment - zone: 'us-central1-a' - # Enable nested virtualization in case it's ever needed - image_licenses: - - 'https://www.googleapis.com/compute/v1/projects/vm-options/global/licenses/enable-vmx' - min_cpu_platform: "Intel Broadwell" # nested-virt requirement - - &nested_virt name: 'fedora' type: 'qemu' @@ -109,18 +81,6 @@ builders: iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}' iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}' - - <<: *nested_virt - name: 'fah' - iso_url: '{{user `FAH_IMAGE_URL`}}' - iso_checksum_url: '{{user `FAH_CSUM_URL`}}' - disk_size: 10240 - - - <<: *nested_virt - name: 'rhel' - iso_url: 'file://{{user `RHEL_IMAGE_FILE`}}' - iso_checksum_url: 'file://{{user `RHEL_CSUM_FILE`}}' - disk_size: 10240 - provisioners: - type: 'shell' inline: @@ -150,7 +110,6 @@ provisioners: - 'GOSRC=/tmp/libpod' - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' - 'PACKER_BASE={{user `PACKER_BASE`}}' - - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}' - <<: *shell_script inline: ['{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'] @@ -161,7 +120,7 @@ provisioners: post-processors: - - type: "compress" - only: ['fedora', 'prior-fedora', 'fah', 'rhel'] + only: ['fedora', 'prior-fedora'] output: '/tmp/{{build_name}}/disk.raw.tar.gz' format: '.tar.gz' compression_level: 9 @@ -180,14 +139,4 @@ post-processors: image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}' image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}' - - <<: *gcp_import - only: ['fah'] - image_name: "{{user `FAH_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" - image_description: 'Based on {{user `FAH_IMAGE_URL`}}' - image_family: '{{user `FAH_BASE_IMAGE_NAME`}}' - - <<: *gcp_import - only: ['rhel'] - image_name: "{{user `RHEL_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" - image_description: 'Based on {{user `RHEL_IMAGE_FILE`}}' - image_family: '{{user `RHEL_BASE_IMAGE_NAME`}}' - type: 'manifest' diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml index 5c42fdae1..722d912c2 100644 --- a/contrib/cirrus/packer/libpod_images.yml +++ b/contrib/cirrus/packer/libpod_images.yml @@ -3,12 +3,9 @@ # All of these are required variables: # Names of GCE Base images to start from, in .cirrus.yml - RHEL_BASE_IMAGE: '{{env `RHEL_BASE_IMAGE`}}' - CENTOS_BASE_IMAGE: '{{env `CENTOS_BASE_IMAGE`}}' UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}' FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}' PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}' - FAH_BASE_IMAGE: '{{env `FAH_BASE_IMAGE`}}' # libpod dependencies to build and install into images FEDORA_CNI_COMMIT: "{{env `FEDORA_CNI_COMMIT`}}" @@ -25,7 +22,6 @@ variables: # Protected credentials, decrypted by Cirrus at runtime GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}' GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}' - RHSM_COMMAND: '{{env `RHSM_COMMAND`}}' SERVICE_ACCOUNT: '{{env `SERVICE_ACCOUNT`}}' GOOGLE_APPLICATION_CREDENTIALS: '{{env `GOOGLE_APPLICATION_CREDENTIALS`}}' @@ -37,7 +33,6 @@ variables: sensitive-variables: - 'GCE_SSH_USERNAME' - 'GCP_PROJECT_ID' - - 'RHSM_COMMAND' - 'SERVICE_ACCOUNT' # What images to produce in which cloud @@ -60,14 +55,6 @@ builders: # v----- is a YAML alias, allows partial re-use of the anchor object - <<: *gce_hosted_image - name: 'rhel-7' - source_image: '{{user `RHEL_BASE_IMAGE`}}' - - - <<: *gce_hosted_image - name: 'centos-7' - source_image: '{{user `CENTOS_BASE_IMAGE`}}' - - - <<: *gce_hosted_image name: 'fedora-29' source_image: '{{user `FEDORA_BASE_IMAGE`}}' @@ -75,10 +62,6 @@ builders: name: 'fedora-28' source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}' - - <<: *gce_hosted_image - name: 'fah-29' - source_image: '{{user `FAH_BASE_IMAGE`}}' - # The brains of the operation, making actual modifications to the base-image. provisioners: - type: 'file' @@ -95,7 +78,6 @@ provisioners: - 'CRIU_COMMIT={{user `CRIU_COMMIT`}}' - 'RUNC_COMMIT={{user `RUNC_COMMIT`}}' - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' - - 'RHSM_COMMAND={{user `RHSM_COMMAND`}}' post-processors: - - type: 'manifest' # writes packer-manifest.json diff --git a/contrib/cirrus/packer/rhel_base-setup.sh b/contrib/cirrus/packer/rhel_base-setup.sh deleted file mode 100644 index 8d5892d7d..000000000 --- a/contrib/cirrus/packer/rhel_base-setup.sh +++ /dev/null @@ -1,78 +0,0 @@ -#!/bin/bash - -# N/B: This script is not intended to be run by humans. It is used to configure the -# rhel base image for importing, so that it will boot in GCE - -set -e - -[[ "$1" == "post" ]] || exit 0 # pre stage is not needed - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var RHSM_COMMAND - -install_ooe - -rhsm_enable - -echo "Setting up repos" -# Frequently needed -ooe.sh sudo yum -y install https://dl.fedoraproject.org/pub/epel/epel-release-latest-7.noarch.rpm - -# Required for google to manage ssh keys -ooe.sh sudo tee /etc/yum.repos.d/google-cloud-sdk.repo << EOM -[google-cloud-compute] -name=google-cloud-compute -baseurl=https://packages.cloud.google.com/yum/repos/google-cloud-compute-el7-x86_64 -enabled=1 -gpgcheck=1 -repo_gpgcheck=1 -gpgkey=https://packages.cloud.google.com/yum/doc/yum-key.gpg - https://packages.cloud.google.com/yum/doc/rpm-package-key.gpg -EOM - -echo "Updating all packages" -ooe.sh sudo yum -y update - -echo "Installing/removing packages" -ooe.sh sudo yum -y install rng-tools google-compute-engine google-compute-engine-oslogin - -echo "Enabling critical services" -ooe.sh sudo systemctl enable \ - rngd \ - google-accounts-daemon \ - google-clock-skew-daemon \ - google-instance-setup \ - google-network-daemon \ - google-shutdown-scripts \ - google-startup-scripts - -rhel_exit_handler # release subscription! - -echo "Configuring boot" -cat << "EOF" | sudo tee /etc/default/grub -GRUB_TIMEOUT=0 -GRUB_DISTRIBUTOR="$(sed 's, release .*$,,g' /etc/system-release)" -GRUB_DEFAULT=saved -GRUB_DISABLE_SUBMENU=true -GRUB_TERMINAL="serial console" -GRUB_SERIAL_COMMAND="serial --speed=38400" -GRUB_CMDLINE_LINUX="crashkernel=auto console=ttyS0,38400n8" -GRUB_DISABLE_RECOVERY="true" -EOF -sudo grub2-mkconfig -o /boot/grub2/grub.cfg - -echo "Configuring networking" -ooe.sh sudo nmcli connection modify 'System eth0' 802-3-ethernet.mtu 1460 -ooe.sh sudo nmcli connection modify 'System eth0' connection.autoconnect yes -ooe.sh sudo nmcli connection modify 'System eth0' connection.autoconnect-priority -ooe.sh sudo nmcli connection modify 'System eth0' ipv4.method auto -ooe.sh sudo nmcli connection modify 'System eth0' ipv4.dhcp-send-hostname yes -ooe.sh sudo nmcli connection modify 'System eth0' ipv4.dhcp-timeout 0 -ooe.sh sudo nmcli connection modify 'System eth0' ipv4.never-default no -ooe.sh /usr/bin/google_instance_setup - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/rhel_setup.sh b/contrib/cirrus/packer/rhel_setup.sh deleted file mode 100644 index 577c8721a..000000000 --- a/contrib/cirrus/packer/rhel_setup.sh +++ /dev/null @@ -1,96 +0,0 @@ -#!/bin/bash - -# This script is called by packer on the subject CentOS VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source /tmp/libpod/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RHSM_COMMAND RUNC_COMMIT - -install_ooe - -rhsm_enable - -ooe.sh sudo yum -y erase "rh-amazon-rhui-client*" -ooe.sh sudo subscription-manager repos "--disable=*" -ooe.sh sudo subscription-manager repos \ - --enable=rhel-7-server-rpms \ - --enable=rhel-7-server-optional-rpms \ - --enable=rhel-7-server-extras-rpms \ - --enable=rhel-server-rhscl-7-rpms - -sudo sed -r -i -e 's/^notify_only=.+/notify_only=0/g' \ - /etc/yum/pluginconf.d/search-disabled-repos.conf - -ooe.sh sudo yum -y update - -ooe.sh sudo yum -y install \ - PyYAML \ - atomic-registries \ - bats \ - btrfs-progs-devel \ - bzip2 \ - device-mapper-devel \ - emacs-nox \ - findutils \ - glib2-devel \ - glibc-static \ - gnupg \ - golang \ - golang-github-cpuguy83-go-md2man \ - golang-github-cpuguy83-go-md2man \ - gpgme-devel \ - iptables \ - jq \ - libassuan-devel \ - libcap-devel \ - libnet \ - libnet-devel \ - libnl3-devel \ - libseccomp-devel \ - libselinux-devel \ - lsof \ - make \ - nmap-ncat \ - ostree-devel \ - protobuf \ - protobuf-c \ - protobuf-c-devel \ - protobuf-compiler \ - protobuf-devel \ - protobuf-python \ - python \ - python2-future \ - python2-pyyaml \ - python34-dateutil \ - python34-psutil \ - python34-pytoml \ - python34-PyYAML \ - skopeo-containers \ - unzip \ - vim \ - which \ - xz - -install_scl_git - -install_conmon - -install_runc - -install_criu - -install_cni_plugins - -install_buildah - -sudo /tmp/libpod/hack/install_catatonit.sh - -rhel_exit_handler # release subscription! - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/rootless_test.sh b/contrib/cirrus/rootless_test.sh index 5a509c101..3b668034b 100755 --- a/contrib/cirrus/rootless_test.sh +++ b/contrib/cirrus/rootless_test.sh @@ -4,20 +4,23 @@ set -e source $(dirname $0)/lib.sh -req_env_var GOSRC ROOTLESS_USER - if [[ "$UID" == "0" ]] then echo "Error: Expected to be running as a regular user" exit 1 fi +# Ensure environment setup correctly +req_env_var GOSRC ROOTLESS_USER + echo "." echo "Hello, my name is $USER and I live in $PWD can I be your friend?" +echo "." export PODMAN_VARLINK_ADDRESS=unix:/tmp/podman-$(id -u) show_env_vars +set -x cd "$GOSRC" make make varlink_generate diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 5fc90d61d..447f4f310 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -12,15 +12,11 @@ exithandler() { RET=$? set +e show_env_vars - record_timestamp "env. setup end" echo "$(basename $0) exit status: $RET" [[ "$RET" -eq "0" ]] && date +%s >> "SETUP_MARKER_FILEPATH" } trap exithandler EXIT -# Must be bash, always bash -[[ "$SHELL" =~ "bash" ]] || chsh -s /bin/bash - # Verify basic dependencies for depbin in go rsync unzip sha256sum curl make python3 git do @@ -39,8 +35,10 @@ case "${OS_REL_VER}" in ubuntu-18) ;; fedora-29) ;; fedora-28) ;; - centos-7) ;; - rhel-7) ;; + centos-7) # Current VM is an image-builder-image no local podman/testing + echo "No further setup required for VM image building" + exit 0 + ;; *) bad_os_id_ver ;; esac @@ -59,19 +57,27 @@ sudo install -D -m 755 $GOSRC/test/registries.conf \ # cri-o if installed will mess with testing in non-obvious ways rm -f /etc/cni/net.d/*cri* -go get github.com/onsi/ginkgo/ginkgo -go get github.com/onsi/gomega/... +make install.tools case "$SPECIALMODE" in + none) ;; # Do the normal thing rootless) - X=$(echo "export ROOTLESS_USER='some${RANDOM}dude'" | \ - tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X" - X=$(echo "export SPECIALMODE='$SPECIALMODE'"| \ - tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X" - setup_rootless + # Only do this once, even if ROOTLESS_USER (somehow) changes + if ! grep -q 'ROOTLESS_USER' /etc/environment + then + X=$(echo "export ROOTLESS_USER='${ROOTLESS_USER:-some${RANDOM}dude}'" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + X=$(echo "export SPECIALMODE='${SPECIALMODE}'" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + X=$(echo "export TEST_REMOTE_CLIENT='${TEST_REMOTE_CLIENT}'" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + setup_rootless + fi ;; in_podman) # Assumed to be Fedora dnf install -y podman buildah $SCRIPT_BASE/setup_container_environment.sh ;; + *) + die 111 "Unsupported \$SPECIAL_MODE: $SPECIALMODE" esac diff --git a/contrib/cirrus/system_test.sh b/contrib/cirrus/system_test.sh index 0da170d6c..a2cc1af05 100755 --- a/contrib/cirrus/system_test.sh +++ b/contrib/cirrus/system_test.sh @@ -8,12 +8,9 @@ req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER set -x cd "$GOSRC" -case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in - ubuntu-18) ;& # Continue to the next item - fedora-28) ;& - fedora-29) ;& - centos-7) ;& - rhel-7) +case "${OS_RELEASE_ID}" in + ubuntu) ;& # Continue to the next item + fedora) make install.tools make make test-binaries |