Merge pull request #5368 from cevich/opensuse_ubuntu_packages

Opensuse openbuild ubuntu + buildah packages

4 files changed, 164 insertions, 103 deletions

diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 5895d84f4..2031432b9 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -34,6 +34,7 @@ PACKER_BASE=${PACKER_BASE:-./contrib/cirrus/packer}
 # Important filepaths
 SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_complete}"
 AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv"
+BUILDAH_PACKAGES_FILEPATH="./contrib/cirrus/packages.sh"  # in buildah repo.
 
 # Log remote-client system test varlink output here
 export VARLINK_LOG=/var/tmp/varlink.log
@@ -61,8 +62,8 @@ PACKER_VER="1.4.2"
 # Base-images rarely change, define them here so they're out of the way.
 export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-31,fedora-30}"
 # Manually produced base-image names (see $SCRIPT_BASE/README.md)
-export UBUNTU_BASE_IMAGE="ubuntu-1904-disco-v20190724"
-export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a"
+export UBUNTU_BASE_IMAGE="ubuntu-1910-eoan-v20200211"
+export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20200218"
 # Manually produced base-image names (see $SCRIPT_BASE/README.md)
 export FEDORA_BASE_IMAGE="fedora-cloud-base-31-1-9-1578586410"
 export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1578586410"
@@ -448,6 +449,26 @@ systemd_banish() {
     $GOSRC/$PACKER_BASE/systemd_banish.sh
 }
 
+install_buildah_packages() {
+    git clone https://github.com/containers/buildah.git /tmp/buildah
+    if [[ -r "$BUILDAH_PACKAGES_FILEPATH" ]]; then
+        source "$BUILDAH_PACKAGES_FILEPATH"
+        req_env_var UBUNTU_BUILDAH_PACKAGES FEDORA_BUILDAH_PACKAGES OS_RELEASE_ID
+        case "$OS_RELEASE_ID" in
+            fedora)
+                $BIGTO ooe.sh sudo dnf install -y ${FEDORA_BUILDAH_PACKAGES[@]}
+                ;;
+            ubuntu)
+                $LILTO $SUDOAPTGET update
+                $BIGTO $SUDOAPTGET install ${UBUNTU_BUILDAH_PACKAGES[@]}
+                ;;
+            *) bad_os_id_ver ;;
+        esac
+    else
+        warn "Could not find $BUILDAH_PACKAGES_FILEPATH in buildah repository root."
+    fi
+}
+
 _finalize() {
     set +e  # Don't fail at the very end
     if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]]
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index 20014e5f3..81a46b13f 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -26,7 +26,6 @@ else
     warn "NOT enabling updates-testing repository for image based on $PRIOR_FEDORA_BASE_IMAGE"
 fi
 
-echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'"
 REMOVE_PACKAGES=()
 INSTALL_PACKAGES=(\
     autoconf
@@ -74,6 +73,7 @@ INSTALL_PACKAGES=(\
     make
     msitools
     nmap-ncat
+    ostree-devel
     pandoc
     podman
     procps-ng
@@ -86,11 +86,15 @@ INSTALL_PACKAGES=(\
     python3-dateutil
     python3-psutil
     python3-pytoml
+    rsync
+    runc
     selinux-policy-devel
     skopeo
+    skopeo-containers
     slirp4netns
     unzip
     vim
+    wget
     which
     xz
     zip
@@ -112,16 +116,23 @@ case "$OS_RELEASE_VER" in
     *)
         bad_os_id_ver ;;
 esac
+
+echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'"
 $BIGTO ooe.sh sudo dnf install -y ${INSTALL_PACKAGES[@]}
 
+install_buildah_packages
+
 [[ "${#REMOVE_PACKAGES[@]}" -eq "0" ]] || \
     $LILTO ooe.sh sudo dnf erase -y ${REMOVE_PACKAGES[@]}
 
-# Ensure there are no disruptive periodic services enabled by default in image
-systemd_banish
+echo "Enabling cgroup management from containers"
+ooe.sh sudo setsebool container_manage_cgroup true
 
 ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh
 
+# Ensure there are no disruptive periodic services enabled by default in image
+systemd_banish
+
 rh_finalize
 
 echo "SUCCESS!"
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index 7c39a76f8..46e7a620f 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -15,9 +15,6 @@ install_ooe
 export GOPATH="$(mktemp -d)"
 trap "sudo rm -rf $GOPATH" EXIT
 
-# Ensure there are no disruptive periodic services enabled by default in image
-systemd_banish
-
 # Stop disruption upon boot ASAP after booting
 echo "Disabling all packaging activity on boot"
 # Don't let sed process sed's temporary files
@@ -32,98 +29,123 @@ $BIGTO $SUDOAPTGET update
 echo "Upgrading all packages"
 $BIGTO $SUDOAPTGET upgrade
 
-echo "Adding PPAs"
+echo "Adding third-party repositories and PPAs"
 $LILTO $SUDOAPTGET install software-properties-common
-$LILTO $SUDOAPTADD ppa:projectatomic/ppa
 $LILTO $SUDOAPTADD ppa:criu/ppa
 if [[ "$OS_RELEASE_VER" -eq "18" ]]
 then
     $LILTO $SUDOAPTADD ppa:longsleep/golang-backports
 fi
 
-$LILTO $SUDOAPTGET update
-
-echo "Installing general testing and system dependencies"
-$BIGTO $SUDOAPTGET install \
-    apparmor \
-    aufs-tools \
-    autoconf \
-    automake \
-    bash-completion \
-    bats \
-    bison \
-    btrfs-tools \
-    build-essential \
-    containernetworking-plugins \
-    containers-common \
-    cri-o-runc \
-    criu \
-    curl \
-    conmon \
-    dnsmasq \
-    e2fslibs-dev \
-    emacs-nox \
-    file \
-    gawk \
-    gcc \
-    gettext \
-    go-md2man \
-    golang \
-    iproute2 \
-    iptables \
-    jq \
-    libaio-dev \
-    libapparmor-dev \
-    libcap-dev \
-    libdevmapper-dev \
-    libdevmapper1.02.1 \
-    libfuse-dev \
-    libfuse2 \
-    libglib2.0-dev \
-    libgpgme11-dev \
-    liblzma-dev \
-    libnet1 \
-    libnet1-dev \
-    libnl-3-dev \
-    libvarlink \
-    libprotobuf-c-dev \
-    libprotobuf-dev \
-    libseccomp-dev \
-    libseccomp2 \
-    libsystemd-dev \
-    libtool \
-    libudev-dev \
-    lsof \
-    make \
-    netcat \
-    pkg-config \
-    podman \
-    protobuf-c-compiler \
-    protobuf-compiler \
-    python-future \
-    python-minimal \
-    python-protobuf \
-    python3-dateutil \
-    python3-pip \
-    python3-psutil \
-    python3-pytoml \
-    python3-setuptools \
-    skopeo \
-    slirp4netns \
-    socat \
-    unzip \
-    vim \
-    xz-utils \
+echo "Configuring/Instaling deps from Open build server"
+VERSION_ID=$(source /etc/os-release; echo $VERSION_ID)
+echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /" \
+    | ooe.sh sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
+ooe.sh curl -L -o /tmp/Release.key "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key"
+ooe.sh sudo apt-key add - < /tmp/Release.key
+
+INSTALL_PACKAGES=(\
+    apparmor
+    aufs-tools
+    autoconf
+    automake
+    bash-completion
+    bison
+    build-essential
+    bzip2
+    conmon
+    containernetworking-plugins
+    containers-common
+    coreutils
+    cri-o-runc
+    criu
+    curl
+    dnsmasq
+    e2fslibs-dev
+    emacs-nox
+    file
+    gawk
+    gcc
+    gettext
+    git
+    go-md2man
+    golang
+    iproute2
+    iptables
+    jq
+    libaio-dev
+    libapparmor-dev
+    libcap-dev
+    libdevmapper-dev
+    libdevmapper1.02.1
+    libfuse-dev
+    libfuse2
+    libglib2.0-dev
+    libgpgme11-dev
+    liblzma-dev
+    libnet1
+    libnet1-dev
+    libnl-3-dev
+    libprotobuf-c-dev
+    libprotobuf-dev
+    libseccomp-dev
+    libseccomp2
+    libselinux-dev
+    libsystemd-dev
+    libtool
+    libudev-dev
+    libvarlink
+    lsof
+    make
+    netcat
+    openssl
+    pkg-config
+    podman
+    protobuf-c-compiler
+    protobuf-compiler
+    python-future
+    python-minimal
+    python-protobuf
+    python3-dateutil
+    python3-pip
+    python3-psutil
+    python3-pytoml
+    python3-setuptools
+    rsync
+    runc
+    scons
+    skopeo
+    slirp4netns
+    socat
+    unzip
+    vim
+    wget
+    xz-utils
+    yum-utils
     zip
+    zlib1g-dev
+)
 
 if [[ "$OS_RELEASE_VER" -ge "19" ]]
 then
-    echo "Installing Ubuntu > 18 packages"
-    $LILTO $SUDOAPTGET install fuse3 libfuse3-dev libbtrfs-dev
-fi
+    INSTALL_PACKAGES+=(\
+        bats
+        btrfs-progs
+        fuse3
+        libbtrfs-dev
+        libfuse3-dev
+    )
+else
+    echo "Downloading version of bats with fix for a \$IFS related bug in 'run' command"
+    cd /tmp
+    BATS_URL='http://launchpadlibrarian.net/438140887/bats_1.1.0+git104-g1c83a1b-1_all.deb'
+    curl -L -O "$BATS_URL"
+    cd -
+    INSTALL_PACKAGES+=(\
+        /tmp/$(basename $BATS_URL)
+        btrfs-tools
+    )
 
-if [[ "$OS_RELEASE_VER" -eq "18" ]]
-then
     echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting."
     SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g'
     ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/*
@@ -131,9 +153,27 @@ then
     ooe.sh sudo update-grub
 fi
 
+echo "Installing general testing and system dependencies"
+# Necessary to update cache of newly added repos
+$LILTO $SUDOAPTGET update
+$BIGTO $SUDOAPTGET install ${INSTALL_PACKAGES[@]}
+
+install_buildah_packages
+
+echo "Installing cataonit and libseccomp.sudo"
 ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh
 ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo
 
+# Ensure there are no disruptive periodic services enabled by default in image
+systemd_banish
+
+CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc"
+if sudo dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH"
+then
+    echo "Linking $CRIO_RUNC_PATH to /usr/bin/runc for ease of testing."
+    sudo ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc"
+fi
+
 ubuntu_finalize
 
 echo "SUCCESS!"
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 9b4a56acd..eceb80b00 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -39,23 +39,8 @@ done
 cd "${GOSRC}/"
 case "${OS_RELEASE_ID}" in
     ubuntu)
-        CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc"
-        if dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH"
-        then
-            echo "Linking $CRIO_RUNC_PATH to /usr/bin/runc for ease of testing."
-            ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc"
-        fi
         ;;
     fedora)
-	# This is temporary and should be removed once conmon is in stable
-	# and the images can be rebuilt properly.
-	if [[ "$OS_RELEASE_VER" -eq "30" ]]; then
-		dnf -y install https://kojipkgs.fedoraproject.org//packages/conmon/2.0.13/1.fc30/x86_64/conmon-2.0.13-1.fc30.x86_64.rpm
-	else
-		dnf -y install https://kojipkgs.fedoraproject.org//packages/conmon/2.0.13/1.fc31/x86_64/conmon-2.0.13-1.fc31.x86_64.rpm
-	fi
-	# End of temporary patch
-
         # All SELinux distros need this for systemd-in-a-container
         setsebool container_manage_cgroup true
         if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
@@ -67,6 +52,10 @@ case "${OS_RELEASE_ID}" in
         echo "mq-deadline" > /sys/block/sda/queue/scheduler
         cat /sys/block/sda/queue/scheduler
 
+        if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
+            bash "$SCRIPT_BASE/add_second_partition.sh"
+        fi
+
         warn "Forcing systemd cgroup manager"
         X=$(echo "export CGROUP_MANAGER=systemd" | \
             tee -a /etc/environment) && eval "$X" && echo "$X"