From 95141f88d4e6f994c2972a63be19c82ef759ea5b Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Tue, 30 Apr 2019 13:09:32 -0400 Subject: Cirrus: fixups based on review feedback Also remove disused distros (RHEL/CentOS/FAH) and fix get_ci_vm script Signed-off-by: Chris Evich --- contrib/cirrus/README.md | 29 +++------ contrib/cirrus/build_vm_images.sh | 2 +- contrib/cirrus/integration_test.sh | 11 ++-- contrib/cirrus/lib.sh | 77 +++++++--------------- contrib/cirrus/lib.sh.t | 17 +++-- contrib/cirrus/packer/Makefile | 15 +---- contrib/cirrus/packer/centos_setup.sh | 84 ------------------------ contrib/cirrus/packer/fah_base-setup.sh | 45 ------------- contrib/cirrus/packer/fah_setup.sh | 21 ------ contrib/cirrus/packer/fedora_setup.sh | 2 +- contrib/cirrus/packer/libpod_base_images.yml | 53 +-------------- contrib/cirrus/packer/libpod_images.yml | 18 ------ contrib/cirrus/packer/rhel_base-setup.sh | 78 ---------------------- contrib/cirrus/packer/rhel_setup.sh | 96 ---------------------------- contrib/cirrus/rootless_test.sh | 7 +- contrib/cirrus/setup_environment.sh | 32 ++++++---- contrib/cirrus/system_test.sh | 9 +-- 17 files changed, 81 insertions(+), 515 deletions(-) delete mode 100644 contrib/cirrus/packer/centos_setup.sh delete mode 100644 contrib/cirrus/packer/fah_base-setup.sh delete mode 100644 contrib/cirrus/packer/fah_setup.sh delete mode 100644 contrib/cirrus/packer/rhel_base-setup.sh delete mode 100644 contrib/cirrus/packer/rhel_setup.sh (limited to 'contrib/cirrus') diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index 1d62f71ce..69d8653fe 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -136,11 +136,10 @@ you'll find the new image names displayed at the end of the `test_build_cache_images_task` in the `build_vm_images` output. For example: + ``` ...cut... ==> Builds finished. The artifacts of successful builds are: ---> rhel-7: A disk image was created: rhel-7-libpod-5699523102900224 ---> rhel-7: --> ubuntu-18: A disk image was created: ubuntu-18-libpod-5699523102900224 --> ubuntu-18: --> fedora-29: A disk image was created: fedora-29-libpod-5699523102900224 @@ -148,8 +147,8 @@ For example: --> fedora-28: A disk image was created: fedora-28-libpod-5699523102900224 ``` -An updated (or new) pull-request with this change, will utilize -the new cache-images: +Now edit `.cirrus.yml`, updating the `*_IMAGE_NAME` lines to reflect the +images from above: ```yaml @@ -161,14 +160,14 @@ env: FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5699523102900224" PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-5699523102900224" UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5699523102900224" - PRIOR_RHEL_CACHE_IMAGE_NAME: "rhel-7-libpod-5699523102900224" ...cut... ``` -Take care to also update the PR description if any 'magic' phrases were used -(they affect which tests run). In other words, you'll likely want to -remove the ``***CIRRUS: TEST IMAGES***`` string - otherwise Cirrus-CI will -simply build new and test again. +***NOTE:*** If re-using the same PR with new images in `.cirrus.yml`, +take care to also *update the PR description* to remove +the magic ``***CIRRUS: TEST IMAGES***`` string. Keeping it and +`--force` pushing would needlessly cause Cirrus-CI to build +and test images again. ### ``build_cache_images`` Task *(Deprecated)* @@ -216,15 +215,6 @@ the ``cache_images`` Task) some input parameters are required: or [end-user credentials](https://cloud.google.com/docs/authentication/end-user#creating_your_client_credentials) -* ``RHEL_IMAGE_FILE`` and ``RHEL_CSUM_FILE`` complete paths - to a `rhel-server-ec2-*.raw.xz` and it's cooresponding - checksum file. These must be supplied manually because - they're not available directly via URL like other images. - -* ``RHSM_COMMAND`` contains the complete string needed to register - the VM for installing package dependencies. The VM will be de-registered - upon completion. - * Optionally, CSV's may be specified to ``PACKER_BUILDS`` to limit the base-images produced. For example, ``PACKER_BUILDS=fedora,image-builder-image``. @@ -270,9 +260,6 @@ When ready, change to the ``packer`` sub-directory, and build the images: $ cd libpod/contrib/cirrus/packer $ make libpod_base_images GCP_PROJECT_ID= \ GOOGLE_APPLICATION_CREDENTIALS= \ - RHEL_IMAGE_FILE= \ - RHEL_CSUM_FILE= \ - RHSM_COMMAND= \ PACKER_BUILDS= ``` diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index 773f8e486..f4567ce74 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -3,7 +3,7 @@ set -e source $(dirname $0)/lib.sh -ENV_VARS='CNI_COMMIT CRIO_COMMIT RUNC_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX CENTOS_BASE_IMAGE UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE FAH_BASE_IMAGE RHEL_BASE_IMAGE RHSM_COMMAND SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE' +ENV_VARS='CNI_COMMIT CRIO_COMMIT PACKER_BUILDS BUILT_IMAGE_SUFFIX UBUNTU_BASE_IMAGE FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE' req_env_var $ENV_VARS # Must also be made available through make, into packer process export $ENV_VARS diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index e65b3b8ec..e7f582b42 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -25,16 +25,19 @@ then elif [[ "$SPECIALMODE" == "rootless" ]] then req_env_var ROOTLESS_USER - set -x - ssh $ROOTLESS_USER@localhost \ + + if [[ "$USER" == "$ROOTLESS_USER" ]] + then + $GOSRC/$SCRIPT_BASE/rootless_test.sh + else + ssh $ROOTLESS_USER@localhost \ -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no -o CheckHostIP=no \ $GOSRC/$SCRIPT_BASE/rootless_test.sh - exit $? + fi else make make install PREFIX=/usr ETCDIR=/etc make test-binaries - make install.tools if [[ "$TEST_REMOTE_CLIENT" == "true" ]] then make remoteintegration diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 7886d230b..d0a2e1579 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -3,6 +3,9 @@ # Library of common, shared utility functions. This file is intended # to be sourced by other scripts, not called directly. +# Global details persist here +source /etc/environment # not always loaded under all circumstances + # Under some contexts these values are not set, make sure they are. USER="$(whoami)" HOME="$(getent passwd $USER | cut -d : -f 6)" @@ -18,6 +21,9 @@ then eval "$(go env)" # required by make and other tools export $(go env | cut -d '=' -f 1) + + # Ensure compiled tooling is reachable + export PATH="$PATH:$GOPATH/bin" fi CIRRUS_WORKING_DIR="${CIRRUS_WORKING_DIR:-$GOPATH/src/github.com/containers/libpod}" export GOSRC="${GOSRC:-$CIRRUS_WORKING_DIR}" @@ -47,29 +53,25 @@ CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and uniq # Vars. for image-building PACKER_VER="1.3.5" # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) -PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28,rhel-7,centos-7}" # Base-images rarely change, define them here so they're out of the way. +PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28}" # Google-maintained base-image names UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20181203a" -CENTOS_BASE_IMAGE="centos-7-v20181113" # Manually produced base-image names (see $SCRIPT_BASE/README.md) FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245" PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-28-1-1-1544474897" -FAH_BASE_IMAGE="fedora-atomichost-29-20181025-1-1541787861" -# RHEL image must be imported, native image bills for subscription. -RHEL_BASE_IMAGE="rhel-guest-image-7-6-210-x86-64-qcow2-1548099756" BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" -RHSM_COMMAND="${RHSM_COMMAND:-/bin/true}" # Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly) ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)' # Unsafe env. vars for display -SECRET_ENV_RE='(IRCID)|(RHSM)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' +SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(^GC[EP]..+)|(SSH)' SPECIALMODE="${SPECIALMODE:-none}" TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}" export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} + # When running as root, this may be empty or not, as a user, it MUST be set. if [[ "$USER" == "root" ]] then @@ -79,9 +81,9 @@ else fi # GCE image-name compatible string representation of distribution name -OS_RELEASE_ID="$(egrep -m 1 '^ID=' /etc/os-release | cut -d = -f 2 | tr -d \' | tr -d \")" -# GCE image-name compatible string representation of distribution major version -OS_RELEASE_VER="$(egrep -m 1 '^VERSION_ID=' /etc/os-release | cut -d = -f 2 | tr -d \' | tr -d \" | cut -d '.' -f 1)" +OS_RELEASE_ID="$(source /etc/os-release; echo $ID)" +# GCE image-name compatible string representation of distribution _major_ version +OS_RELEASE_VER="$(source /etc/os-release; echo $VERSION_ID | cut -d '.' -f 1)" # Combined to ease soe usage OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}" @@ -118,8 +120,7 @@ show_env_vars() { for _env_var_name in $_ENV_VAR_NAMES do # Supports older BASH versions - _value="$(printenv $_env_var_name)" - printf " ${_env_var_name}=%q\n" "${_value}" + printf " ${_env_var_name}=%q\n" "$(printenv $_env_var_name)" done echo "" echo "##### $(go version) #####" @@ -127,12 +128,14 @@ show_env_vars() { } die() { - echo "${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}" + echo "************************************************" + echo ">>>>> ${2:-FATAL ERROR (but no message given!) in ${FUNCNAME[1]}()}" + echo "************************************************" exit ${1:-1} } bad_os_id_ver() { - echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $ARGS" + echo "Unknown/Unsupported distro. $OS_RELEASE_ID and/or version $OS_RELEASE_VER for $(basename $0)" exit 42 } @@ -141,7 +144,7 @@ stub() { } ircmsg() { - req_env_var CIRRUS_TASK_ID MSG + req_env_var CIRRUS_TASK_ID IRCID [[ -n "$*" ]] || die 9 "ircmsg() invoked without message text argument" # Sometimes setup_environment.sh didn't run SCRIPT="$(dirname $0)/podbot.py" @@ -193,14 +196,15 @@ setup_rootless() { # Env. vars set by Cirrus and setup_environment.sh must be explicitly # transfered to the test-user. echo "Configuring rootless user's environment variables:" + echo "# Added by $GOSRC/$SCRIPT_PATH/lib.sh setup_rootless()" _ENV_VAR_NAMES=$(awk 'BEGIN{for(v in ENVIRON) print v}' | \ egrep -v "(^PATH$)|(^BASH_FUNC)|(^[[:punct:][:space:]]+)|$SECRET_ENV_RE" | \ egrep "$ROOTLESS_ENV_RE" | \ sort -u) for _env_var_name in $_ENV_VAR_NAMES do - _value="$(printenv $_env_var_name)" - printf "${_env_var_name}=%q" "${_value}" | tee -a "/home/$ROOTLESS_USER/.bashrc" + # Works with older versions of bash + printf "${_env_var_name}=%q\n" "$(printenv $_env_var_name)" >> "/home/$ROOTLESS_USER/.bashrc" done } @@ -312,21 +316,6 @@ install_criu(){ ooe.sh sudo -E add-apt-repository -y ppa:criu/ppa ooe.sh sudo -E apt-get -qq -y update ooe.sh sudo -E apt-get -qq -y install criu - elif [[ ( "$OS_RELEASE_ID" =~ "centos" || "$OS_RELEASE_ID" =~ "rhel" ) && "$OS_RELEASE_VER" =~ "7"* ]]; then - echo "Configuring Repositories for latest CRIU" - ooe.sh sudo tee /etc/yum.repos.d/adrian-criu-el7.repo < "$RHSMCMD" - ooe.sh sudo bash "$RHSMCMD" - sudo rm -rf "$RHSMCMD" -} diff --git a/contrib/cirrus/lib.sh.t b/contrib/cirrus/lib.sh.t index ce51f8ad2..1f05b3bb5 100755 --- a/contrib/cirrus/lib.sh.t +++ b/contrib/cirrus/lib.sh.t @@ -10,12 +10,13 @@ rc=0 function check_result { testnum=$(expr $testnum + 1) - if [ "$1" = "$2" ]; then - echo "ok $testnum $3 = $1" + MSG=$(echo "$1" | tr -d '*>\012'|sed -e 's/^ \+//') + if [ "$MSG" = "$2" ]; then + echo "ok $testnum $3 = $MSG" else echo "not ok $testnum $3" echo "# expected: $2" - echo "# actual: $1" + echo "# actual: $MSG" rc=1 fi } @@ -64,11 +65,17 @@ test_rev '' 1 'FATAL: req_env_var: invoked without arguments' unset FOO BAR test_rev FOO 9 'FATAL: test_rev() requires $FOO to be non-empty' test_rev BAR 9 'FATAL: test_rev() requires $BAR to be non-empty' - -# OK if desired envariable is unset +# OK if desired envariable was unset FOO=1 test_rev FOO 0 '' +# OK if multiple vars are non-empty +FOO="stuff" +BAR="things" +ENV_VARS="FOO BAR" +test_rev "$ENV_VARS" 0 '' +unset BAR + # ...but error if any single desired one is unset test_rev "FOO BAR" 9 'FATAL: test_rev() requires $BAR to be non-empty' diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile index 91a1dfeef..0ae0deca6 100644 --- a/contrib/cirrus/packer/Makefile +++ b/contrib/cirrus/packer/Makefile @@ -49,6 +49,7 @@ ifndef PACKER_BUILDS $(error PACKER_BUILDS is undefined, expected builder-names CSV) endif ./packer build -only=${PACKER_BUILDS} \ + -force \ -var GOSRC=$(GOSRC) \ -var PACKER_BASE=$(PACKER_BASE) \ -var SCRIPT_BASE=$(SCRIPT_BASE) \ @@ -77,17 +78,9 @@ ifndef GCP_PROJECT_ID endif ifndef GOOGLE_APPLICATION_CREDENTIALS $(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json) -endif -ifndef RHEL_IMAGE_FILE - $(error RHEL_IMAGE_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz file) -endif -ifndef RHEL_CSUM_FILE - $(error RHEL_CSUM_FILE is undefined, expected full path to a rhel-server-ec2-*.raw.xz.SHA256SUM file) -endif -ifndef RHSM_COMMAND - $(error RHSM_COMMAND is undefined, expected string required for temporarily registering VM) endif PACKER_CACHE_DIR=/tmp ./packer build \ + -force \ -var TIMESTAMP=$(TIMESTAMP) \ -var TTYDEV=$(TTYDEV) \ -var GCP_PROJECT_ID=$(GCP_PROJECT_ID) \ @@ -95,10 +88,6 @@ endif -var GOSRC=$(GOSRC) \ -var PACKER_BASE=$(PACKER_BASE) \ -var SCRIPT_BASE=$(SCRIPT_BASE) \ - -var RHEL_BASE_IMAGE_NAME=$(shell basename $(RHEL_IMAGE_FILE) | tr -d '[[:space:]]' | sed -r -e 's/\.${ARCH}\.raw\.xz//' | tr '[[:upper:]]' '[[:lower:]]' | tr '[[:punct:]]' '-') \ - -var RHEL_IMAGE_FILE=$(RHEL_IMAGE_FILE) \ - -var RHEL_CSUM_FILE=$(RHEL_CSUM_FILE) \ - -var 'RHSM_COMMAND=$(RHSM_COMMAND)' \ -var POST_MERGE_BUCKET_SUFFIX=$(POST_MERGE_BUCKET_SUFFIX) \ -only $(PACKER_BUILDS) \ libpod_base_images.json diff --git a/contrib/cirrus/packer/centos_setup.sh b/contrib/cirrus/packer/centos_setup.sh deleted file mode 100644 index f63625789..000000000 --- a/contrib/cirrus/packer/centos_setup.sh +++ /dev/null @@ -1,84 +0,0 @@ -#!/bin/bash - -# This script is called by packer on the subject CentOS VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source /tmp/libpod/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE CNI_COMMIT CRIO_COMMIT CRIU_COMMIT RUNC_COMMIT - -install_ooe - -export GOPATH="$(mktemp -d)" -trap "sudo rm -rf $GOPATH" EXIT - -ooe.sh sudo yum -y update - -ooe.sh sudo yum -y install centos-release-scl epel-release - -ooe.sh sudo yum -y install \ - PyYAML \ - atomic-registries \ - bats \ - btrfs-progs-devel \ - bzip2 \ - device-mapper-devel \ - emacs-nox \ - findutils \ - glib2-devel \ - glibc-static \ - gnupg \ - golang \ - golang-github-cpuguy83-go-md2man \ - golang-github-cpuguy83-go-md2man \ - gpgme-devel \ - iptables \ - jq \ - libassuan-devel \ - libcap-devel \ - libnet \ - libnet-devel \ - libnl3-devel \ - libseccomp-devel \ - libselinux-devel \ - lsof \ - make \ - nmap-ncat \ - ostree-devel \ - protobuf \ - protobuf-c \ - protobuf-c-devel \ - protobuf-compiler \ - protobuf-devel \ - protobuf-python \ - python \ - python2-future \ - python3-dateutil \ - python3-psutil \ - python3-pytoml \ - skopeo-containers \ - unzip \ - vim \ - which \ - xz - -install_scl_git - -install_conmon - -install_runc - -install_criu - -install_cni_plugins - -install_buildah - -sudo /tmp/libpod/hack/install_catatonit.sh - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fah_base-setup.sh b/contrib/cirrus/packer/fah_base-setup.sh deleted file mode 100644 index 606c4f336..000000000 --- a/contrib/cirrus/packer/fah_base-setup.sh +++ /dev/null @@ -1,45 +0,0 @@ - -# N/B: This script is not intended to be run by humans. It is used to configure the -# FAH base image for importing, so that it will boot in GCE. - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -install_ooe - -if [[ "$1" == "pre" ]] -then - echo "Upgrading Atomic Host" - setenforce 0 - ooe.sh atomic host upgrade - - echo "Configuring Repositories" - ooe.sh sudo tee /etc/yum.repos.d/ngompa-gce-oslogin.repo <> "SETUP_MARKER_FILEPATH" } trap exithandler EXIT -# Must be bash, always bash -[[ "$SHELL" =~ "bash" ]] || chsh -s /bin/bash - # Verify basic dependencies for depbin in go rsync unzip sha256sum curl make python3 git do @@ -39,8 +35,10 @@ case "${OS_REL_VER}" in ubuntu-18) ;; fedora-29) ;; fedora-28) ;; - centos-7) ;; - rhel-7) ;; + centos-7) # Current VM is an image-builder-image no local podman/testing + echo "No further setup required for VM image building" + exit 0 + ;; *) bad_os_id_ver ;; esac @@ -59,19 +57,27 @@ sudo install -D -m 755 $GOSRC/test/registries.conf \ # cri-o if installed will mess with testing in non-obvious ways rm -f /etc/cni/net.d/*cri* -go get github.com/onsi/ginkgo/ginkgo -go get github.com/onsi/gomega/... +make install.tools case "$SPECIALMODE" in + none) ;; # Do the normal thing rootless) - X=$(echo "export ROOTLESS_USER='some${RANDOM}dude'" | \ - tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X" - X=$(echo "export SPECIALMODE='$SPECIALMODE'"| \ - tee -a "$HOME/$ENVLIB") && eval "$X" && echo "$X" - setup_rootless + # Only do this once, even if ROOTLESS_USER (somehow) changes + if ! grep -q 'ROOTLESS_USER' /etc/environment + then + X=$(echo "export ROOTLESS_USER='${ROOTLESS_USER:-some${RANDOM}dude}'" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + X=$(echo "export SPECIALMODE='${SPECIALMODE}'" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + X=$(echo "export TEST_REMOTE_CLIENT='${TEST_REMOTE_CLIENT}'" | \ + tee -a /etc/environment) && eval "$X" && echo "$X" + setup_rootless + fi ;; in_podman) # Assumed to be Fedora dnf install -y podman buildah $SCRIPT_BASE/setup_container_environment.sh ;; + *) + die 111 "Unsupported \$SPECIAL_MODE: $SPECIALMODE" esac diff --git a/contrib/cirrus/system_test.sh b/contrib/cirrus/system_test.sh index 0da170d6c..a2cc1af05 100755 --- a/contrib/cirrus/system_test.sh +++ b/contrib/cirrus/system_test.sh @@ -8,12 +8,9 @@ req_env_var GOSRC OS_RELEASE_ID OS_RELEASE_VER set -x cd "$GOSRC" -case "${OS_RELEASE_ID}-${OS_RELEASE_VER}" in - ubuntu-18) ;& # Continue to the next item - fedora-28) ;& - fedora-29) ;& - centos-7) ;& - rhel-7) +case "${OS_RELEASE_ID}" in + ubuntu) ;& # Continue to the next item + fedora) make install.tools make make test-binaries -- cgit v1.2.3-54-g00ecf