From 7cbdd48f78c7a47ade3e1324346734c20fb57769 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Mon, 2 Mar 2020 09:40:27 -0500 Subject: Cirrus: Use opensuse open build Ubuntu packages This is necessary as the projectatomic PPA is no longer maintained. Signed-off-by: Chris Evich --- contrib/cirrus/packer/ubuntu_setup.sh | 183 +++++++++++++++++++--------------- 1 file changed, 100 insertions(+), 83 deletions(-) (limited to 'contrib/cirrus/packer/ubuntu_setup.sh') diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index 7c39a76f8..1299fd464 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -15,9 +15,6 @@ install_ooe export GOPATH="$(mktemp -d)" trap "sudo rm -rf $GOPATH" EXIT -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - # Stop disruption upon boot ASAP after booting echo "Disabling all packaging activity on boot" # Don't let sed process sed's temporary files @@ -32,98 +29,109 @@ $BIGTO $SUDOAPTGET update echo "Upgrading all packages" $BIGTO $SUDOAPTGET upgrade -echo "Adding PPAs" +echo "Adding third-party repositories and PPAs" $LILTO $SUDOAPTGET install software-properties-common -$LILTO $SUDOAPTADD ppa:projectatomic/ppa $LILTO $SUDOAPTADD ppa:criu/ppa if [[ "$OS_RELEASE_VER" -eq "18" ]] then $LILTO $SUDOAPTADD ppa:longsleep/golang-backports fi -$LILTO $SUDOAPTGET update - -echo "Installing general testing and system dependencies" -$BIGTO $SUDOAPTGET install \ - apparmor \ - aufs-tools \ - autoconf \ - automake \ - bash-completion \ - bats \ - bison \ - btrfs-tools \ - build-essential \ - containernetworking-plugins \ - containers-common \ - cri-o-runc \ - criu \ - curl \ - conmon \ - dnsmasq \ - e2fslibs-dev \ - emacs-nox \ - file \ - gawk \ - gcc \ - gettext \ - go-md2man \ - golang \ - iproute2 \ - iptables \ - jq \ - libaio-dev \ - libapparmor-dev \ - libcap-dev \ - libdevmapper-dev \ - libdevmapper1.02.1 \ - libfuse-dev \ - libfuse2 \ - libglib2.0-dev \ - libgpgme11-dev \ - liblzma-dev \ - libnet1 \ - libnet1-dev \ - libnl-3-dev \ - libvarlink \ - libprotobuf-c-dev \ - libprotobuf-dev \ - libseccomp-dev \ - libseccomp2 \ - libsystemd-dev \ - libtool \ - libudev-dev \ - lsof \ - make \ - netcat \ - pkg-config \ - podman \ - protobuf-c-compiler \ - protobuf-compiler \ - python-future \ - python-minimal \ - python-protobuf \ - python3-dateutil \ - python3-pip \ - python3-psutil \ - python3-pytoml \ - python3-setuptools \ - skopeo \ - slirp4netns \ - socat \ - unzip \ - vim \ - xz-utils \ +echo "Configuring/Instaling deps from Open build server" +VERSION_ID=$(source /etc/os-release; echo $VERSION_ID) +echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /" \ + > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list +curl -L -o /tmp/Release.key "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key" +apt-key add - < /tmp/Release.key + +INSTALL_PACKAGES=(\ + apparmor + aufs-tools + autoconf + automake + bash-completion + bison + btrfs-tools + build-essential + containernetworking-plugins + containers-common + cri-o-runc + criu + curl + conmon + dnsmasq + e2fslibs-dev + emacs-nox + file + gawk + gcc + gettext + go-md2man + golang + iproute2 + iptables + jq + libaio-dev + libapparmor-dev + libcap-dev + libdevmapper-dev + libdevmapper1.02.1 + libfuse-dev + libfuse2 + libglib2.0-dev + libgpgme11-dev + liblzma-dev + libnet1 + libnet1-dev + libnl-3-dev + libvarlink + libprotobuf-c-dev + libprotobuf-dev + libseccomp-dev + libseccomp2 + libsystemd-dev + libtool + libudev-dev + lsof + make + netcat + pkg-config + podman + protobuf-c-compiler + protobuf-compiler + python-future + python-minimal + python-protobuf + python3-dateutil + python3-pip + python3-psutil + python3-pytoml + python3-setuptools + skopeo + slirp4netns + socat + unzip + vim + xz-utils zip +) if [[ "$OS_RELEASE_VER" -ge "19" ]] then - echo "Installing Ubuntu > 18 packages" - $LILTO $SUDOAPTGET install fuse3 libfuse3-dev libbtrfs-dev -fi + INSTALL_PACKAGES+=(\ + bats + fuse3 + libbtrfs-dev + libfuse3-dev + ) +else + echo "Downloading version of bats with fix for a \$IFS related bug in 'run' command" + cd /tmp + BATS_URL='http://launchpadlibrarian.net/438140887/bats_1.1.0+git104-g1c83a1b-1_all.deb' + curl -L -O "$BATS_URL" + cd - + INSTALL_PACKAGES+=(/tmp/$(basename $BATS_URL)) -if [[ "$OS_RELEASE_VER" -eq "18" ]] -then echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting." SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g' ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* @@ -131,9 +139,18 @@ then ooe.sh sudo update-grub fi +echo "Installing general testing and system dependencies" +# Necessary to update cache of newly added repos +$LILTO $SUDOAPTGET update +$BIGTO $SUDOAPTGET install ${INSTALL_PACKAGES[@]} + +echo "Installing cataonit and libseccomp.sudo" ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo +# Ensure there are no disruptive periodic services enabled by default in image +systemd_banish + ubuntu_finalize echo "SUCCESS!" -- cgit v1.2.3-54-g00ecf From e810dd135ebe1bc732d411a2bcab574e06c41bb4 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Mon, 2 Mar 2020 10:32:21 -0500 Subject: Cirrus: Include packages for buildah CI Also, move some setup steps at VM image build time to save runtime. Signed-off-by: Chris Evich --- contrib/cirrus/packer/fedora_setup.sh | 15 ++++++++++++--- contrib/cirrus/packer/ubuntu_setup.sh | 35 ++++++++++++++++++++++++++++------- contrib/cirrus/setup_environment.sh | 19 ++++--------------- 3 files changed, 44 insertions(+), 25 deletions(-) (limited to 'contrib/cirrus/packer/ubuntu_setup.sh') diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 20014e5f3..cc7f1c34f 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -26,7 +26,6 @@ else warn "NOT enabling updates-testing repository for image based on $PRIOR_FEDORA_BASE_IMAGE" fi -echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'" REMOVE_PACKAGES=() INSTALL_PACKAGES=(\ autoconf @@ -74,6 +73,7 @@ INSTALL_PACKAGES=(\ make msitools nmap-ncat + ostree-devel pandoc podman procps-ng @@ -86,11 +86,15 @@ INSTALL_PACKAGES=(\ python3-dateutil python3-psutil python3-pytoml + rsync + runc selinux-policy-devel skopeo + skopeo-containers slirp4netns unzip vim + wget which xz zip @@ -112,16 +116,21 @@ case "$OS_RELEASE_VER" in *) bad_os_id_ver ;; esac + +echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'" $BIGTO ooe.sh sudo dnf install -y ${INSTALL_PACKAGES[@]} [[ "${#REMOVE_PACKAGES[@]}" -eq "0" ]] || \ $LILTO ooe.sh sudo dnf erase -y ${REMOVE_PACKAGES[@]} -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish +echo "Enabling cgroup management from containers" +ooe.sh sudo setsebool container_manage_cgroup true ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh +# Ensure there are no disruptive periodic services enabled by default in image +systemd_banish + rh_finalize echo "SUCCESS!" diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index 1299fd464..a270faf45 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -40,9 +40,9 @@ fi echo "Configuring/Instaling deps from Open build server" VERSION_ID=$(source /etc/os-release; echo $VERSION_ID) echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /" \ - > /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list -curl -L -o /tmp/Release.key "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key" -apt-key add - < /tmp/Release.key + | ooe.sh sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list +ooe.sh curl -L -o /tmp/Release.key "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key" +ooe.sh sudo apt-key add - < /tmp/Release.key INSTALL_PACKAGES=(\ apparmor @@ -51,14 +51,15 @@ INSTALL_PACKAGES=(\ automake bash-completion bison - btrfs-tools build-essential + bzip2 + conmon containernetworking-plugins containers-common + coreutils cri-o-runc criu curl - conmon dnsmasq e2fslibs-dev emacs-nox @@ -66,6 +67,7 @@ INSTALL_PACKAGES=(\ gawk gcc gettext + git go-md2man golang iproute2 @@ -84,17 +86,19 @@ INSTALL_PACKAGES=(\ libnet1 libnet1-dev libnl-3-dev - libvarlink libprotobuf-c-dev libprotobuf-dev libseccomp-dev libseccomp2 + libselinux-dev libsystemd-dev libtool libudev-dev + libvarlink lsof make netcat + openssl pkg-config podman protobuf-c-compiler @@ -107,19 +111,26 @@ INSTALL_PACKAGES=(\ python3-psutil python3-pytoml python3-setuptools + rsync + runc + scons skopeo slirp4netns socat unzip vim + wget xz-utils + yum-utils zip + zlib1g-dev ) if [[ "$OS_RELEASE_VER" -ge "19" ]] then INSTALL_PACKAGES+=(\ bats + btrfs-progs fuse3 libbtrfs-dev libfuse3-dev @@ -130,7 +141,10 @@ else BATS_URL='http://launchpadlibrarian.net/438140887/bats_1.1.0+git104-g1c83a1b-1_all.deb' curl -L -O "$BATS_URL" cd - - INSTALL_PACKAGES+=(/tmp/$(basename $BATS_URL)) + INSTALL_PACKAGES+=(\ + /tmp/$(basename $BATS_URL) + btrfs-tools + ) echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting." SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g' @@ -151,6 +165,13 @@ ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo # Ensure there are no disruptive periodic services enabled by default in image systemd_banish +CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc" +if sudo dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH" +then + echo "Linking $CRIO_RUNC_PATH to /usr/bin/runc for ease of testing." + sudo ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc" +fi + ubuntu_finalize echo "SUCCESS!" diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 9b4a56acd..eceb80b00 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -39,23 +39,8 @@ done cd "${GOSRC}/" case "${OS_RELEASE_ID}" in ubuntu) - CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc" - if dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH" - then - echo "Linking $CRIO_RUNC_PATH to /usr/bin/runc for ease of testing." - ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc" - fi ;; fedora) - # This is temporary and should be removed once conmon is in stable - # and the images can be rebuilt properly. - if [[ "$OS_RELEASE_VER" -eq "30" ]]; then - dnf -y install https://kojipkgs.fedoraproject.org//packages/conmon/2.0.13/1.fc30/x86_64/conmon-2.0.13-1.fc30.x86_64.rpm - else - dnf -y install https://kojipkgs.fedoraproject.org//packages/conmon/2.0.13/1.fc31/x86_64/conmon-2.0.13-1.fc31.x86_64.rpm - fi - # End of temporary patch - # All SELinux distros need this for systemd-in-a-container setsebool container_manage_cgroup true if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then @@ -67,6 +52,10 @@ case "${OS_RELEASE_ID}" in echo "mq-deadline" > /sys/block/sda/queue/scheduler cat /sys/block/sda/queue/scheduler + if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then + bash "$SCRIPT_BASE/add_second_partition.sh" + fi + warn "Forcing systemd cgroup manager" X=$(echo "export CGROUP_MANAGER=systemd" | \ tee -a /etc/environment) && eval "$X" && echo "$X" -- cgit v1.2.3-54-g00ecf From 6ac04366ee64b9938dfb4801a528986fc8cb9956 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Mon, 9 Mar 2020 16:53:25 -0400 Subject: Cirrus: Enable future installing buildah packages Many of the packages required for CI in buildah overlap with libpod. When building new VM images, attempt to source a package list from the buildah repository. If found, also install the listed packages on the VM. Signed-off-by: Chris Evich --- contrib/cirrus/lib.sh | 21 +++++++++++++++++++++ contrib/cirrus/packer/fedora_setup.sh | 2 ++ contrib/cirrus/packer/ubuntu_setup.sh | 2 ++ 3 files changed, 25 insertions(+) (limited to 'contrib/cirrus/packer/ubuntu_setup.sh') diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 3b76e49f3..2031432b9 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -34,6 +34,7 @@ PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer} # Important filepaths SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}" AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv" +BUILDAH_PACKAGES_FILEPATH="./contrib/cirrus/packages.sh" # in buildah repo. # Log remote-client system test varlink output here export VARLINK_LOG=/var/tmp/varlink.log @@ -448,6 +449,26 @@ systemd_banish() { $GOSRC/$PACKER_BASE/systemd_banish.sh } +install_buildah_packages() { + git clone https://github.com/containers/buildah.git /tmp/buildah + if [[ -r "$BUILDAH_PACKAGES_FILEPATH" ]]; then + source "$BUILDAH_PACKAGES_FILEPATH" + req_env_var UBUNTU_BUILDAH_PACKAGES FEDORA_BUILDAH_PACKAGES OS_RELEASE_ID + case "$OS_RELEASE_ID" in + fedora) + $BIGTO ooe.sh sudo dnf install -y ${FEDORA_BUILDAH_PACKAGES[@]} + ;; + ubuntu) + $LILTO $SUDOAPTGET update + $BIGTO $SUDOAPTGET install ${UBUNTU_BUILDAH_PACKAGES[@]} + ;; + *) bad_os_id_ver ;; + esac + else + warn "Could not find $BUILDAH_PACKAGES_FILEPATH in buildah repository root." + fi +} + _finalize() { set +e # Don't fail at the very end if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]] diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index cc7f1c34f..81a46b13f 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -120,6 +120,8 @@ esac echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'" $BIGTO ooe.sh sudo dnf install -y ${INSTALL_PACKAGES[@]} +install_buildah_packages + [[ "${#REMOVE_PACKAGES[@]}" -eq "0" ]] || \ $LILTO ooe.sh sudo dnf erase -y ${REMOVE_PACKAGES[@]} diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index a270faf45..46e7a620f 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -158,6 +158,8 @@ echo "Installing general testing and system dependencies" $LILTO $SUDOAPTGET update $BIGTO $SUDOAPTGET install ${INSTALL_PACKAGES[@]} +install_buildah_packages + echo "Installing cataonit and libseccomp.sudo" ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo -- cgit v1.2.3-54-g00ecf