Merge pull request #5853 from cevich/unify_in_podman_build

Unify in_podman container packaging & VM packaging

12 files changed, 464 insertions, 578 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 0122b6b55..5898fa160 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -39,7 +39,7 @@ env:
     UBUNTU_NAME: "ubuntu-19"
     PRIOR_UBUNTU_NAME: "ubuntu-18"
 
-    _BUILT_IMAGE_SUFFIX: "libpod-6301182083727360"
+    _BUILT_IMAGE_SUFFIX: "libpod-6220812239765504"
     FEDORA_CACHE_IMAGE_NAME: "${FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "${PRIOR_FEDORA_NAME}-${_BUILT_IMAGE_SUFFIX}"
     UBUNTU_CACHE_IMAGE_NAME: "${UBUNTU_NAME}-${_BUILT_IMAGE_SUFFIX}"
@@ -156,6 +156,32 @@ gating_task:
         failed_branch_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_branch_failure.sh'
 
 
+# Ensure these container images can build
+container_image_build_task:
+    alias: 'container_image_build'
+    depends_on:
+        - "gating"
+
+    # Only run for PRs, quay.io will automatically build after bramch-push
+    only_if: $CIRRUS_BRANCH != $DEST_BRANCH
+
+    matrix:
+        - name: "build in_podman image ${FEDORA_NAME} "
+          container:
+              dockerfile: Dockerfile
+        - name: "build in_podman image ${UBUNTU_NAME}"
+          container:
+              dockerfile: Dockerfile.ubuntu
+        - name: "build gate image $DEST_BRANCH branch"
+          container:
+              dockerfile: contrib/gate/Dockerfile
+
+    container:
+        dockerfile: Dockerfile
+
+    script: make install.remote
+
+
 # This task checks to make sure that we can still build an rpm from the
 # source code using contrib/rpm/podman.spec.in
 rpmbuild_task:
@@ -389,6 +415,7 @@ testing_task:
         - "varlink_api"
         - "build_each_commit"
         - "build_without_cgo"
+        - "container_image_build"
 
     # Only test build cache-images, if that's what's requested
     only_if: >-
@@ -679,6 +706,7 @@ test_build_cache_images_task:
 
     depends_on:
         - "gating"
+        - 'container_image_build'
 
     # VMs created by packer are not cleaned up by cirrus, must allow task to complete
     auto_cancellation: $CI != "true"
@@ -780,6 +808,7 @@ success_task:
         - "varlink_api"
         - "build_each_commit"
         - "build_without_cgo"
+        - "container_image_build"
         - "meta"
         - "image_prune"
         - "testing"
diff --git a/Dockerfile b/Dockerfile
index f85c47937..623747295 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,117 +1,26 @@
-FROM golang:1.12
-
-RUN apt-get update && apt-get install -y \
-    apparmor \
-    autoconf \
-    automake \
-    bison \
-    build-essential \
-    curl \
-    e2fslibs-dev \
-    file \
-    gawk \
-    gettext \
-    go-md2man \
-    iptables \
-    pkg-config \
-    libaio-dev \
-    libcap-dev \
-    libfuse-dev \
-    libnet-dev \
-    libnl-3-dev \
-    libprotobuf-dev \
-    libprotobuf-c-dev \
-    libseccomp2 \
-    libseccomp-dev \
-    libtool \
-    libudev-dev \
-    protobuf-c-compiler \
-    protobuf-compiler \
-    libglib2.0-dev \
-    libapparmor-dev \
-    btrfs-tools \
-    libdevmapper1.02.1 \
-    libdevmapper-dev \
-    libgpgme11-dev \
-    liblzma-dev \
-    netcat \
-    socat \
-    lsof \
-    xz-utils \
-    unzip \
-    python3-yaml \
-    --no-install-recommends \
-    && apt-get clean
-
-# Install runc
-ENV RUNC_COMMIT 029124da7af7360afa781a0234d1b083550f797c
-RUN set -x \
-	&& export GOPATH="$(mktemp -d)" \
-	&& git clone https://github.com/opencontainers/runc.git "$GOPATH/src/github.com/opencontainers/runc" \
-	&& cd "$GOPATH/src/github.com/opencontainers/runc" \
-	&& git fetch origin --tags \
-	&& git checkout --detach -q "$RUNC_COMMIT" \
-	&& make static BUILDTAGS="seccomp selinux" \
-	&& cp runc /usr/bin/runc \
-	&& rm -rf "$GOPATH"
-
-# Install conmon
-ENV CONMON_COMMIT 65fe0226d85b69fc9e527e376795c9791199153d
-RUN set -x \
-	&& export GOPATH="$(mktemp -d)" \
-	&& git clone https://github.com/containers/conmon.git "$GOPATH/src/github.com/containers/conmon.git" \
-	&& cd "$GOPATH/src/github.com/containers/conmon.git" \
-	&& git fetch origin --tags \
-	&& git checkout --detach -q "$CONMON_COMMIT" \
-	&& make \
-	&& install -D -m 755 bin/conmon /usr/libexec/podman/conmon \
-	&& rm -rf "$GOPATH"
-
-# Install CNI plugins
-ENV CNI_COMMIT 485be65581341430f9106a194a98f0f2412245fb
-RUN set -x \
-       && export GOPATH="$(mktemp -d)" GOCACHE="$(mktemp -d)" \
-       && git clone https://github.com/containernetworking/plugins.git "$GOPATH/src/github.com/containernetworking/plugins" \
-       && cd "$GOPATH/src/github.com/containernetworking/plugins" \
-       && git checkout --detach -q "$CNI_COMMIT" \
-       && ./build_linux.sh \
-       && mkdir -p /usr/libexec/cni \
-       && cp bin/* /usr/libexec/cni \
-       && rm -rf "$GOPATH"
-
-# Install ginkgo
-RUN set -x \
-       && export GOPATH=/go \
-       && go get -u github.com/onsi/ginkgo/ginkgo \
-       && install -D -m 755 "$GOPATH"/bin/ginkgo /usr/bin/
-
-# Install gomega
-RUN set -x \
-       && export GOPATH=/go \
-       && go get github.com/onsi/gomega/...
-
-# Install latest stable criu version
-RUN set -x \
-      && cd /tmp \
-      && git clone https://github.com/checkpoint-restore/criu.git \
-      && cd criu \
-      && make \
-      && install -D -m 755  criu/criu /usr/sbin/ \
-      && rm -rf /tmp/criu
-
-# Install cni config
-#RUN make install.cni
-RUN mkdir -p /etc/cni/net.d/
-COPY cni/87-podman-bridge.conflist /etc/cni/net.d/87-podman-bridge.conflist
-
-# Make sure we have some policy for pulling images
-RUN mkdir -p /etc/containers && curl https://raw.githubusercontent.com/projectatomic/registries/master/registries.fedora -o /etc/containers/registries.conf
-
-COPY test/policy.json /etc/containers/policy.json
-COPY test/redhat_sigstore.yaml /etc/containers/registries.d/registry.access.redhat.com.yaml
-
-ADD . /go/src/github.com/containers/libpod
-
-RUN set -x && cd /go/src/github.com/containers/libpod
-
-WORKDIR /go/src/github.com/containers/libpod
+FROM registry.fedoraproject.org/fedora:latest
+
+# This container image is utilized by the containers CI automation system
+# for building and testing libpod inside a container environment.
+# It is assumed that the source to be tested will overwrite $GOSRC (below)
+# at runtime.
+ENV GOPATH=/var/tmp/go
+ENV GOSRC=$GOPATH/src/github.com/containers/libpod
+ENV SCRIPT_BASE=./contrib/cirrus
+ENV PACKER_BASE=$SCRIPT_BASE/packer
+
+# Only add minimal tooling necessary to complete setup.
+ADD /$SCRIPT_BASE $GOSRC/$SCRIPT_BASE
+ADD /hack/install_catatonit.sh $GOSRC/hack/
+ADD /cni/*.conflist $GOSRC/cni/
+ADD /test/*.json $GOSRC/test/
+ADD /test/*.conf $GOSRC/test/
+WORKDIR $GOSRC
+
+# Re-use repositories and package setup as in VMs under CI
+RUN bash $PACKER_BASE/fedora_packaging.sh && \
+    dnf clean all && \
+    rm -rf /var/cache/dnf
+
+# Mirror steps taken under CI
+RUN bash -c 'source $GOSRC/$SCRIPT_BASE/lib.sh && install_test_configs'
diff --git a/Dockerfile.centos b/Dockerfile.centos
deleted file mode 100644
index f5a2b891c..000000000
--- a/Dockerfile.centos
+++ /dev/null
@@ -1,77 +0,0 @@
-FROM registry.centos.org/centos/centos:7
-
-RUN yum -y install btrfs-progs-devel \
-              atomic-registries \
-              autoconf \
-              automake \
-              bzip2 \
-              device-mapper-devel \
-              findutils \
-              file \
-              git \
-              glibc-static \
-              glib2-devel \
-              gnupg \
-              golang \
-              golang-github-cpuguy83-go-md2man \
-              gpgme-devel \
-              libassuan-devel \
-              libseccomp-devel \
-              libselinux-devel \
-              libtool \
-              containers-common \
-              runc \
-              make \
-              lsof \
-              which\
-              golang-github-cpuguy83-go-md2man \
-              nmap-ncat \
-              xz \
-              iptables && yum clean all
-
-# Install CNI plugins
-ENV CNI_COMMIT 485be65581341430f9106a194a98f0f2412245fb
-RUN set -x \
-       && export GOPATH="$(mktemp -d)" GOCACHE="$(mktemp -d)" \
-       && git clone https://github.com/containernetworking/plugins.git "$GOPATH/src/github.com/containernetworking/plugins" \
-       && cd "$GOPATH/src/github.com/containernetworking/plugins" \
-       && git checkout --detach -q "$CNI_COMMIT" \
-       && ./build_linux.sh \
-       && mkdir -p /usr/libexec/cni \
-       && cp bin/* /usr/libexec/cni \
-       && rm -rf "$GOPATH"
-
-# Install ginkgo
-RUN set -x \
-       && export GOPATH=/go \
-       && go get -u github.com/onsi/ginkgo/ginkgo \
-       && install -D -m 755 "$GOPATH"/bin/ginkgo /usr/bin/
-
-# Install gomega
-RUN set -x \
-       && export GOPATH=/go \
-       && go get github.com/onsi/gomega/...
-
-# Install conmon
-ENV CONMON_COMMIT 6f3572558b97bc60dd8f8c7f0807748e6ce2c440
-RUN set -x \
-	&& export GOPATH="$(mktemp -d)" \
-	&& git clone https://github.com/containers/conmon.git "$GOPATH/src/github.com/containers/conmon.git" \
-	&& cd "$GOPATH/src/github.com/containers/conmon.git" \
-	&& git fetch origin --tags \
-	&& git checkout --detach -q "$CONMON_COMMIT" \
-	&& make \
-	&& install -D -m 755 bin/conmon /usr/libexec/podman/conmon \
-	&& rm -rf "$GOPATH"
-
-# Install cni config
-#RUN make install.cni
-RUN mkdir -p /etc/cni/net.d/
-COPY cni/87-podman-bridge.conflist /etc/cni/net.d/87-podman-bridge.conflist
-
-# Make sure we have some policy for pulling images
-RUN mkdir -p /etc/containers
-COPY test/policy.json /etc/containers/policy.json
-COPY test/redhat_sigstore.yaml /etc/containers/registries.d/registry.access.redhat.com.yaml
-
-WORKDIR /go/src/github.com/containers/libpod
diff --git a/Dockerfile.fedora b/Dockerfile.fedora
deleted file mode 100644
index 45b2c3670..000000000
--- a/Dockerfile.fedora
+++ /dev/null
@@ -1,73 +0,0 @@
-FROM registry.fedoraproject.org/fedora:30
-
-RUN dnf -y install btrfs-progs-devel \
-              atomic-registries \
-              autoconf \
-              automake \
-              bzip2 \
-              device-mapper-devel \
-              file \
-              findutils \
-              git \
-              glib2-devel \
-              glibc-static \
-              gnupg \
-              golang \
-              golang-github-cpuguy83-go-md2man \
-              gpgme-devel \
-              libassuan-devel \
-              libseccomp-devel \
-              libselinux-devel \
-              libtool \
-              containers-common \
-              runc \
-              make \
-              lsof \
-              which\
-              golang-github-cpuguy83-go-md2man \
-              procps-ng \
-              nmap-ncat \
-              xz \
-              slirp4netns \
-              container-selinux \
-              containernetworking-plugins \
-              iproute \
-              iptables && dnf clean all
-
-# Install ginkgo
-RUN set -x \
-       && export GOPATH=/go GOCACHE="$(mktemp -d)" \
-       && go get -u github.com/onsi/ginkgo/ginkgo \
-       && install -D -m 755 "$GOPATH"/bin/ginkgo /usr/bin/
-
-# Install gomega
-RUN set -x \
-       && export GOPATH=/go GOCACHE="$(mktemp -d)" \
-       && go get github.com/onsi/gomega/...
-
-# Install conmon
-ENV CONMON_COMMIT 6f3572558b97bc60dd8f8c7f0807748e6ce2c440
-RUN set -x \
-	&& export GOPATH="$(mktemp -d)" GOCACHE="$(mktemp -d)" \
-	&& git clone https://github.com/containers/conmon.git "$GOPATH/src/github.com/containers/conmon.git" \
-	&& cd "$GOPATH/src/github.com/containers/conmon.git" \
-	&& git fetch origin --tags \
-	&& git checkout --detach -q "$CONMON_COMMIT" \
-	&& make \
-	&& install -D -m 755 bin/conmon /usr/libexec/podman/conmon \
-	&& rm -rf "$GOPATH"
-
-# Install cni config
-#RUN make install.cni
-RUN mkdir -p /etc/cni/net.d/
-COPY cni/87-podman-bridge.conflist /etc/cni/net.d/87-podman-bridge.conflist
-
-# Make sure we have some policy for pulling images
-RUN mkdir -p /etc/containers
-COPY test/policy.json /etc/containers/policy.json
-COPY test/redhat_sigstore.yaml /etc/containers/registries.d/registry.access.redhat.com.yaml
-
-# Install varlink stuff
-RUN pip3 install varlink
-
-WORKDIR /go/src/github.com/containers/libpod
diff --git a/Dockerfile.ubuntu b/Dockerfile.ubuntu
new file mode 100644
index 000000000..3a8f837b9
--- /dev/null
+++ b/Dockerfile.ubuntu
@@ -0,0 +1,29 @@
+# Must resemble $UBUNTU_BASE_IMAGE in ./contrib/cirrus/lib.sh
+FROM ubuntu:latest
+
+# This container image is intended for building and testing libpod
+# from inside a container environment.  It is assumed that the source
+# to be tested will overwrite $GOSRC (below) at runtime.
+ENV GOPATH=/var/tmp/go
+ENV GOSRC=$GOPATH/src/github.com/containers/libpod
+ENV SCRIPT_BASE=./contrib/cirrus
+ENV PACKER_BASE=$SCRIPT_BASE/packer
+
+RUN export DEBIAN_FRONTEND="noninteractive" && \
+    apt-get -qq update --yes && \
+    apt-get -qq upgrade --yes && \
+    apt-get -qq install curl git && \
+    apt-get -qq autoremove --yes && \
+    rm -rf /var/cache/apt
+
+# Only add minimal tooling necessary to complete setup.
+ADD / $GOSRC
+WORKDIR $GOSRC
+
+# Re-use repositories and package setup as in VMs under CI
+RUN bash $PACKER_BASE/ubuntu_packaging.sh && \
+    apt-get -qq autoremove --yes && \
+    rm -rf /var/cache/apt
+
+# Mirror steps taken under CI
+RUN bash -c 'source $GOSRC/$SCRIPT_BASE/lib.sh && install_test_configs'
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 04f14eeb3..dd4f66f56 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -6,6 +6,11 @@
 # Global details persist here
 source /etc/environment  # not always loaded under all circumstances
 
+# Automation environment doesn't automatically load for Ubuntu 18
+if [[ -r '/usr/share/automation/environment' ]]; then
+    source '/usr/share/automation/environment'
+fi
+
 # Under some contexts these values are not set, make sure they are.
 export USER="$(whoami)"
 export HOME="$(getent passwd $USER | cut -d : -f 6)"
@@ -72,10 +77,15 @@ IN_PODMAN_IMAGE="quay.io/libpod/in_podman:$DEST_BRANCH"
 # Image for uploading releases
 UPLDREL_IMAGE="quay.io/libpod/upldrel:master"
 
+# This is needed under some environments/contexts
+SUDO=''
+[[ "$UID" -eq 0 ]] || \
+    SUDO='sudo -E'
+
 # Avoid getting stuck waiting for user input
 export DEBIAN_FRONTEND="noninteractive"
-SUDOAPTGET="ooe.sh sudo -E apt-get -qq --yes"
-SUDOAPTADD="ooe.sh sudo -E add-apt-repository --yes"
+SUDOAPTGET="$SUDO apt-get -qq --yes"
+SUDOAPTADD="$SUDO add-apt-repository --yes"
 # Regex that finds enabled periodic apt configuration items
 PERIODIC_APT_RE='^(APT::Periodic::.+")1"\;'
 # Short-cuts for retrying/timeout calls
@@ -109,6 +119,9 @@ OS_REL_VER="${OS_RELEASE_ID}-${OS_RELEASE_VER}"
 # Type of filesystem used for cgroups
 CG_FS_TYPE="$(stat -f -c %T /sys/fs/cgroup)"
 
+# When building images, the version of automation tooling to install
+INSTALL_AUTOMATION_VERSION=1.1.3
+
 # Installed into cache-images, supports overrides
 # by user-data in case of breakage or for debugging.
 CUSTOM_CLOUD_CONFIG_DEFAULTS="$GOSRC/$PACKER_BASE/cloud-init/$OS_RELEASE_ID/cloud.cfg.d"
@@ -354,25 +367,18 @@ setup_rootless() {
         die 11 "Timeout exceeded waiting for localhost ssh capability"
 }
 
-# Helper/wrapper script to only show stderr/stdout on non-zero exit
-install_ooe() {
-    req_env_var SCRIPT_BASE
-    echo "Installing script to mask stdout/stderr unless non-zero exit."
-    sudo install -D -m 755 "$GOSRC/$SCRIPT_BASE/ooe.sh" /usr/local/bin/ooe.sh
-}
-
 # Grab a newer version of git from software collections
 # https://www.softwarecollections.org/en/
 # and use it with a wrapper
 install_scl_git() {
     echo "Installing SoftwareCollections updated 'git' version."
-    ooe.sh sudo yum -y install rh-git29
-    cat << "EOF" | sudo tee /usr/bin/git
+    ooe.sh $SUDO yum -y install rh-git29
+    cat << "EOF" | $SUDO tee /usr/bin/git
 #!/bin/bash
 
 scl enable rh-git29 -- git $@
 EOF
-    sudo chmod 755 /usr/bin/git
+    $SUDO chmod 755 /usr/bin/git
 }
 
 install_test_configs() {
@@ -414,9 +420,9 @@ remove_packaged_podman_files() {
 
     if [[ "$OS_RELEASE_ID" =~ "ubuntu" ]]
     then
-        LISTING_CMD="sudo -E dpkg-query -L podman"
+        LISTING_CMD="$SUDO dpkg-query -L podman"
     else
-        LISTING_CMD='sudo rpm -ql podman'
+        LISTING_CMD='$SUDO rpm -ql podman'
     fi
 
     # yum/dnf/dpkg may list system directories, only remove files
@@ -424,7 +430,7 @@ remove_packaged_podman_files() {
     do
         # Sub-directories may contain unrelated/valuable stuff
         if [[ -d "$fullpath" ]]; then continue; fi
-        ooe.sh sudo rm -vf "$fullpath"
+        ooe.sh $SUDO rm -vf "$fullpath"
     done
 
     # Be super extra sure and careful vs performant and completely safe
@@ -447,43 +453,60 @@ systemd_banish() {
     $GOSRC/$PACKER_BASE/systemd_banish.sh
 }
 
+# This can be removed when the kernel bug fix is included in Fedora
+workaround_bfq_bug() {
+    if [[ "$OS_RELEASE_ID" == "fedora" ]] && [[ $OS_RELEASE_VER -le 32 ]]; then
+        warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539"
+        warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447"
+        echo "mq-deadline" | sudo tee /sys/block/sda/queue/scheduler > /dev/null
+        echo -n "IO Scheduler set to: "
+        $SUDO cat /sys/block/sda/queue/scheduler
+    fi
+}
+
+# Warning: DO NOT USE.
+# This is called by other functions as the very last step during the VM Image build
+# process.  It's purpose is to "reset" the image, so all the first-boot operations
+# happen at test runtime (like generating new ssh host keys, resizing partitions, etc.)
 _finalize() {
     set +e  # Don't fail at the very end
     if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]]
     then
         echo "Installing custom cloud-init defaults"
-        sudo cp -v "$CUSTOM_CLOUD_CONFIG_DEFAULTS"/* /etc/cloud/cloud.cfg.d/
+        $SUDO cp -v "$CUSTOM_CLOUD_CONFIG_DEFAULTS"/* /etc/cloud/cloud.cfg.d/
     else
         echo "Could not find any files in $CUSTOM_CLOUD_CONFIG_DEFAULTS"
     fi
     echo "Re-initializing so next boot does 'first-boot' setup again."
     cd /
-    sudo rm -rf /var/lib/cloud/instanc*
-    sudo rm -rf /root/.ssh/*
-    sudo rm -rf /etc/ssh/*key*
-    sudo rm -rf /etc/ssh/moduli
-    sudo rm -rf /home/*
-    sudo rm -rf /tmp/*
-    sudo rm -rf /tmp/.??*
-    sudo sync
-    sudo fstrim -av
+    $SUDO rm -rf /var/lib/cloud/instanc*
+    $SUDO rm -rf /root/.ssh/*
+    $SUDO rm -rf /etc/ssh/*key*
+    $SUDO rm -rf /etc/ssh/moduli
+    $SUDO rm -rf /home/*
+    $SUDO rm -rf /tmp/*
+    $SUDO rm -rf /tmp/.??*
+    $SUDO sync
+    $SUDO fstrim -av
 }
 
+# Called during VM Image setup, not intended for general use.
 rh_finalize() {
     set +e  # Don't fail at the very end
     echo "Resetting to fresh-state for usage as cloud-image."
     PKG=$(type -P dnf || type -P yum || echo "")
-    sudo $PKG clean all
-    sudo rm -rf /var/cache/{yum,dnf}
-    sudo rm -f /etc/udev/rules.d/*-persistent-*.rules
-    sudo touch /.unconfigured  # force firstboot to run
+    $SUDO $PKG clean all
+    $SUDO rm -rf /var/cache/{yum,dnf}
+    $SUDO rm -f /etc/udev/rules.d/*-persistent-*.rules
+    $SUDO touch /.unconfigured  # force firstboot to run
     _finalize
 }
 
+# Called during VM Image setup, not intended for general use.
 ubuntu_finalize() {
     set +e  # Don't fail at the very end
     echo "Resetting to fresh-state for usage as cloud-image."
     $LILTO $SUDOAPTGET autoremove
-    sudo rm -rf /var/cache/apt
+    $SUDO rm -rf /var/cache/apt
     _finalize
 }
diff --git a/contrib/cirrus/packer/fedora_packaging.sh b/contrib/cirrus/packer/fedora_packaging.sh
new file mode 100644
index 000000000..e80d48bc8
--- /dev/null
+++ b/contrib/cirrus/packer/fedora_packaging.sh
@@ -0,0 +1,141 @@
+#!/bin/bash
+
+# This script is called from fedora_setup.sh and various Dockerfiles.
+# It's not intended to be used outside of those contexts.  It assumes the lib.sh
+# library has already been sourced, and that all "ground-up" package-related activity
+# needs to be done, including repository setup and initial update.
+
+set -e
+
+echo "Updating/Installing repos and packages for $OS_REL_VER"
+
+source $GOSRC/$SCRIPT_BASE/lib.sh
+
+# Pre-req. to install automation tooing
+$LILTO $SUDO dnf install -y git
+
+# Install common automation tooling (i.e. ooe.sh)
+curl --silent --show-error --location \
+     --url "https://raw.githubusercontent.com/containers/automation/master/bin/install_automation.sh" | \
+     $SUDO env INSTALL_PREFIX=/usr/share /bin/bash -s - "$INSTALL_AUTOMATION_VERSION"
+# Reload installed environment right now (happens automatically in a new process)
+source /usr/share/automation/environment
+
+# Set this to 1 to NOT enable updates-testing repository
+DISABLE_UPDATES_TESTING=${DISABLE_UPDATES_TESTING:0}
+
+# Do not enable update-stesting on the previous Fedora release
+if ((DISABLE_UPDATES_TESTING!=0)); then
+    warn "Enabling updates-testing repository for image based on $FEDORA_BASE_IMAGE"
+    $LILTO $SUDO ooe.sh dnf install -y 'dnf-command(config-manager)'
+    $LILTO $SUDO ooe.sh dnf config-manager --set-enabled updates-testing
+else
+    warn "NOT enabling updates-testing repository for image based on $PRIOR_FEDORA_BASE_IMAGE"
+fi
+
+$BIGTO ooe.sh $SUDO dnf update -y
+
+REMOVE_PACKAGES=()
+INSTALL_PACKAGES=(\
+    autoconf
+    automake
+    bash-completion
+    bats
+    bridge-utils
+    btrfs-progs-devel
+    buildah
+    bzip2
+    conmon
+    container-selinux
+    containernetworking-plugins
+    containers-common
+    criu
+    device-mapper-devel
+    dnsmasq
+    emacs-nox
+    file
+    findutils
+    fuse3
+    fuse3-devel
+    gcc
+    git
+    glib2-devel
+    glibc-static
+    gnupg
+    go-md2man
+    golang
+    gpgme-devel
+    iproute
+    iptables
+    jq
+    libassuan-devel
+    libcap-devel
+    libmsi1
+    libnet
+    libnet-devel
+    libnl3-devel
+    libseccomp
+    libseccomp-devel
+    libselinux-devel
+    libtool
+    libvarlink-util
+    lsof
+    make
+    msitools
+    nmap-ncat
+    ostree-devel
+    pandoc
+    podman
+    procps-ng
+    protobuf
+    protobuf-c
+    protobuf-c-devel
+    protobuf-devel
+    python
+    python3-dateutil
+    python3-psutil
+    python3-pytoml
+    rsync
+    selinux-policy-devel
+    skopeo
+    skopeo-containers
+    slirp4netns
+    unzip
+    vim
+    wget
+    which
+    xz
+    zip
+)
+
+case "$OS_RELEASE_VER" in
+    30)
+        INSTALL_PACKAGES+=(\
+            atomic-registries
+            golang-github-cpuguy83-go-md2man
+            python2-future
+            runc
+        )
+        REMOVE_PACKAGES+=(crun)
+        ;;
+    31)
+        INSTALL_PACKAGES+=(crun)
+        REMOVE_PACKAGES+=(runc)
+        ;;
+    32)
+        INSTALL_PACKAGES+=(crun)
+        REMOVE_PACKAGES+=(runc)
+        ;;
+    *)
+        bad_os_id_ver ;;
+esac
+
+echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'"
+$BIGTO ooe.sh $SUDO dnf install -y ${INSTALL_PACKAGES[@]}
+
+[[ ${#REMOVE_PACKAGES[@]} -eq 0 ]] || \
+    $LILTO ooe.sh $SUDO dnf erase -y ${REMOVE_PACKAGES[@]}
+
+export GOPATH="$(mktemp -d)"
+trap "$SUDO rm -rf $GOPATH" EXIT
+ooe.sh $SUDO $GOSRC/hack/install_catatonit.sh
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index fcef7360b..3830b3bc4 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -6,139 +6,26 @@
 set -e
 
 # Load in library (copied by packer, before this script was run)
-source /tmp/libpod/$SCRIPT_BASE/lib.sh
+source $GOSRC/$SCRIPT_BASE/lib.sh
 
-req_env_var SCRIPT_BASE PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER
+req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER
 
-install_ooe
-
-if [[ $OS_RELEASE_VER -le 31 ]]; then
-    warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539"
-    warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447"
-    echo "mq-deadline" | sudo tee /sys/block/sda/queue/scheduler > /dev/null
-    sudo cat /sys/block/sda/queue/scheduler
-fi
-
-export GOPATH="$(mktemp -d)"
-trap "sudo rm -rf $GOPATH" EXIT
-
-$BIGTO ooe.sh sudo dnf update -y
+workaround_bfq_bug
 
 # Do not enable update-stesting on the previous Fedora release
 if [[ "$FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then
-    warn "Enabling updates-testing repository for image based on $FEDORA_BASE_IMAGE"
-    $LILTO ooe.sh sudo dnf install -y 'dnf-command(config-manager)'
-    $LILTO ooe.sh sudo dnf config-manager --set-enabled updates-testing
+    DISABLE_UPDATES_TESTING=0
 else
-    warn "NOT enabling updates-testing repository for image based on $PRIOR_FEDORA_BASE_IMAGE"
+    DISABLE_UPDATES_TESTING=1
 fi
 
-REMOVE_PACKAGES=()
-INSTALL_PACKAGES=(\
-    autoconf
-    automake
-    bash-completion
-    bats
-    bridge-utils
-    btrfs-progs-devel
-    buildah
-    bzip2
-    conmon
-    container-selinux
-    containernetworking-plugins
-    containers-common
-    criu
-    device-mapper-devel
-    dnsmasq
-    emacs-nox
-    file
-    findutils
-    fuse3
-    fuse3-devel
-    gcc
-    git
-    glib2-devel
-    glibc-static
-    gnupg
-    go-md2man
-    golang
-    gpgme-devel
-    iproute
-    iptables
-    jq
-    libassuan-devel
-    libcap-devel
-    libmsi1
-    libnet
-    libnet-devel
-    libnl3-devel
-    libseccomp
-    libseccomp-devel
-    libselinux-devel
-    libtool
-    libvarlink-util
-    lsof
-    make
-    msitools
-    nmap-ncat
-    ostree-devel
-    pandoc
-    podman
-    procps-ng
-    protobuf
-    protobuf-c
-    protobuf-c-devel
-    protobuf-devel
-    python
-    python3-dateutil
-    python3-psutil
-    python3-pytoml
-    rsync
-    selinux-policy-devel
-    skopeo
-    skopeo-containers
-    slirp4netns
-    unzip
-    vim
-    wget
-    which
-    xz
-    zip
-)
-
-case "$OS_RELEASE_VER" in
-    30)
-        INSTALL_PACKAGES+=(\
-            atomic-registries
-            golang-github-cpuguy83-go-md2man
-            python2-future
-            runc
-        )
-        REMOVE_PACKAGES+=(crun)
-        ;;
-    31)
-        INSTALL_PACKAGES+=(crun)
-        REMOVE_PACKAGES+=(runc)
-        ;;
-    32)
-        INSTALL_PACKAGES+=(crun)
-        REMOVE_PACKAGES+=(runc)
-        ;;
-    *)
-        bad_os_id_ver ;;
-esac
-
-echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'"
-$BIGTO ooe.sh sudo dnf install -y ${INSTALL_PACKAGES[@]}
-
-[[ "${#REMOVE_PACKAGES[@]}" -eq "0" ]] || \
-    $LILTO ooe.sh sudo dnf erase -y ${REMOVE_PACKAGES[@]}
+bash $PACKER_BASE/fedora_packaging.sh
+# Load installed environment right now (happens automatically in a new process)
+source /usr/share/automation/environment
 
 echo "Enabling cgroup management from containers"
 ooe.sh sudo setsebool container_manage_cgroup true
 
-ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh
-
 # Ensure there are no disruptive periodic services enabled by default in image
 systemd_banish
 
diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml
index c23439201..e33ad775e 100644
--- a/contrib/cirrus/packer/libpod_images.yml
+++ b/contrib/cirrus/packer/libpod_images.yml
@@ -71,6 +71,7 @@ provisioners:
       environment_vars:
           - 'PACKER_BUILDER_NAME={{build_name}}'
           - 'GOSRC=/tmp/libpod'
+          - 'PACKER_BASE={{user `PACKER_BASE`}}'
           - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
 
 post-processors:
diff --git a/contrib/cirrus/packer/ubuntu_packaging.sh b/contrib/cirrus/packer/ubuntu_packaging.sh
new file mode 100644
index 000000000..b57bc95e9
--- /dev/null
+++ b/contrib/cirrus/packer/ubuntu_packaging.sh
@@ -0,0 +1,168 @@
+#!/bin/bash
+
+# This script is called from ubuntu_setup.sh and various Dockerfiles.
+# It's not intended to be used outside of those contexts.  It assumes the lib.sh
+# library has already been sourced, and that all "ground-up" package-related activity
+# needs to be done, including repository setup and initial update.
+
+set -e
+
+echo "Updating/Installing repos and packages for $OS_REL_VER"
+
+source $GOSRC/$SCRIPT_BASE/lib.sh
+
+echo "Updating/configuring package repositories."
+$BIGTO $SUDOAPTGET update
+
+echo "Installing deps to add third-party repositories and automation tooling"
+$LILTO $SUDOAPTGET install software-properties-common git curl
+
+# Install common automation tooling (i.e. ooe.sh)
+curl --silent --show-error --location \
+     --url "https://raw.githubusercontent.com/containers/automation/master/bin/install_automation.sh" | \
+     $SUDO env INSTALL_PREFIX=/usr/share /bin/bash -s - "$INSTALL_AUTOMATION_VERSION"
+# Reload installed environment right now (happens automatically in a new process)
+source /usr/share/automation/environment
+
+$LILTO ooe.sh $SUDOAPTADD ppa:criu/ppa
+
+# Install newer version of golang
+if [[ "$OS_RELEASE_VER" -eq "18" ]]
+then
+    $LILTO ooe.sh $SUDOAPTADD ppa:longsleep/golang-backports
+fi
+
+echo "Configuring/Instaling deps from Open build server"
+VERSION_ID=$(source /etc/os-release; echo $VERSION_ID)
+echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /" \
+    | ooe.sh $SUDO tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
+ooe.sh curl -L -o /tmp/Release.key "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key"
+ooe.sh $SUDO apt-key add - < /tmp/Release.key
+
+INSTALL_PACKAGES=(\
+    apparmor
+    aufs-tools
+    autoconf
+    automake
+    bash-completion
+    bison
+    build-essential
+    buildah
+    bzip2
+    conmon
+    containernetworking-plugins
+    containers-common
+    coreutils
+    cri-o-runc
+    criu
+    curl
+    dnsmasq
+    e2fslibs-dev
+    emacs-nox
+    file
+    gawk
+    gcc
+    gettext
+    git
+    go-md2man
+    golang
+    iproute2
+    iptables
+    jq
+    libaio-dev
+    libapparmor-dev
+    libcap-dev
+    libdevmapper-dev
+    libdevmapper1.02.1
+    libfuse-dev
+    libfuse2
+    libglib2.0-dev
+    libgpgme11-dev
+    liblzma-dev
+    libnet1
+    libnet1-dev
+    libnl-3-dev
+    libprotobuf-c-dev
+    libprotobuf-dev
+    libseccomp-dev
+    libseccomp2
+    libselinux-dev
+    libsystemd-dev
+    libtool
+    libudev-dev
+    libvarlink
+    lsof
+    make
+    netcat
+    openssl
+    pkg-config
+    podman
+    protobuf-c-compiler
+    protobuf-compiler
+    python-future
+    python-minimal
+    python-protobuf
+    python3-dateutil
+    python3-pip
+    python3-psutil
+    python3-pytoml
+    python3-setuptools
+    rsync
+    runc
+    scons
+    skopeo
+    slirp4netns
+    socat
+    sudo
+    unzip
+    vim
+    wget
+    xz-utils
+    yum-utils
+    zip
+    zlib1g-dev
+)
+
+if [[ $OS_RELEASE_VER -ge 19 ]]
+then
+    INSTALL_PACKAGES+=(\
+        bats
+        btrfs-progs
+        fuse3
+        libbtrfs-dev
+        libfuse3-dev
+    )
+else
+    echo "Downloading version of bats with fix for a \$IFS related bug in 'run' command"
+    cd /tmp
+    BATS_URL='http://launchpadlibrarian.net/438140887/bats_1.1.0+git104-g1c83a1b-1_all.deb'
+    curl -L -O "$BATS_URL"
+    cd -
+    INSTALL_PACKAGES+=(\
+        /tmp/$(basename $BATS_URL)
+        btrfs-tools
+    )
+fi
+
+# Do this at the last possible moment to avoid dpkg lock conflicts
+echo "Upgrading all packages"
+$BIGTO ooe.sh $SUDOAPTGET upgrade
+
+echo "Installing general testing and system dependencies"
+# Necessary to update cache of newly added repos
+$LILTO ooe.sh $SUDOAPTGET update
+$BIGTO ooe.sh $SUDOAPTGET install ${INSTALL_PACKAGES[@]}
+
+export GOPATH="$(mktemp -d)"
+trap "$SUDO rm -rf $GOPATH" EXIT
+echo "Installing cataonit and libseccomp.sudo"
+cd $GOSRC
+ooe.sh $SUDO hack/install_catatonit.sh
+ooe.sh $SUDO make install.libseccomp.sudo
+
+CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc"
+if $SUDO dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH"
+then
+    echo "Linking $CRIO_RUNC_PATH to /usr/bin/runc for ease of testing."
+    $SUDO ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc"
+fi
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index 4b6e99358..2febbd265 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -8,164 +8,21 @@ set -e
 # Load in library (copied by packer, before this script was run)
 source $GOSRC/$SCRIPT_BASE/lib.sh
 
-req_env_var SCRIPT_BASE
+req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC UBUNTU_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER
 
-install_ooe
-
-export GOPATH="$(mktemp -d)"
-trap "sudo rm -rf $GOPATH" EXIT
+# Ensure there are no disruptive periodic services enabled by default in image
+systemd_banish
 
 # Stop disruption upon boot ASAP after booting
 echo "Disabling all packaging activity on boot"
-# Don't let sed process sed's temporary files
-_FILEPATHS=$(sudo ls -1 /etc/apt/apt.conf.d)
-for filename in $_FILEPATHS; do \
+for filename in $(sudo ls -1 /etc/apt/apt.conf.d); do \
     echo "Checking/Patching $filename"
     sudo sed -i -r -e "s/$PERIODIC_APT_RE/"'\10"\;/' "/etc/apt/apt.conf.d/$filename"; done
 
-echo "Updating/configuring package repositories."
-$BIGTO $SUDOAPTGET update
-
-echo "Upgrading all packages"
-$BIGTO $SUDOAPTGET upgrade
-
-echo "Adding third-party repositories and PPAs"
-$LILTO $SUDOAPTGET install software-properties-common
-$LILTO $SUDOAPTADD ppa:criu/ppa
-if [[ "$OS_RELEASE_VER" -eq "18" ]]
-then
-    $LILTO $SUDOAPTADD ppa:longsleep/golang-backports
-fi
-
-echo "Configuring/Instaling deps from Open build server"
-VERSION_ID=$(source /etc/os-release; echo $VERSION_ID)
-echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /" \
-    | ooe.sh sudo tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list
-ooe.sh curl -L -o /tmp/Release.key "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key"
-ooe.sh sudo apt-key add - < /tmp/Release.key
-
-INSTALL_PACKAGES=(\
-    apparmor
-    aufs-tools
-    autoconf
-    automake
-    bash-completion
-    bison
-    build-essential
-    buildah
-    bzip2
-    conmon
-    containernetworking-plugins
-    containers-common
-    coreutils
-    cri-o-runc
-    criu
-    curl
-    dnsmasq
-    e2fslibs-dev
-    emacs-nox
-    file
-    gawk
-    gcc
-    gettext
-    git
-    go-md2man
-    golang
-    iproute2
-    iptables
-    jq
-    libaio-dev
-    libapparmor-dev
-    libcap-dev
-    libdevmapper-dev
-    libdevmapper1.02.1
-    libfuse-dev
-    libfuse2
-    libglib2.0-dev
-    libgpgme11-dev
-    liblzma-dev
-    libnet1
-    libnet1-dev
-    libnl-3-dev
-    libprotobuf-c-dev
-    libprotobuf-dev
-    libseccomp-dev
-    libseccomp2
-    libselinux-dev
-    libsystemd-dev
-    libtool
-    libudev-dev
-    libvarlink
-    lsof
-    make
-    netcat
-    openssl
-    pkg-config
-    podman
-    protobuf-c-compiler
-    protobuf-compiler
-    python-future
-    python-minimal
-    python-protobuf
-    python3-dateutil
-    python3-pip
-    python3-psutil
-    python3-pytoml
-    python3-setuptools
-    rsync
-    runc
-    scons
-    skopeo
-    slirp4netns
-    socat
-    unzip
-    vim
-    wget
-    xz-utils
-    yum-utils
-    zip
-    zlib1g-dev
-)
-
-if [[ "$OS_RELEASE_VER" -ge "19" ]]
-then
-    INSTALL_PACKAGES+=(\
-        bats
-        btrfs-progs
-        fuse3
-        libbtrfs-dev
-        libfuse3-dev
-    )
-else
-    echo "Downloading version of bats with fix for a \$IFS related bug in 'run' command"
-    cd /tmp
-    BATS_URL='http://launchpadlibrarian.net/438140887/bats_1.1.0+git104-g1c83a1b-1_all.deb'
-    curl -L -O "$BATS_URL"
-    cd -
-    INSTALL_PACKAGES+=(\
-        /tmp/$(basename $BATS_URL)
-        btrfs-tools
-    )
-fi
-
-echo "Installing general testing and system dependencies"
-# Necessary to update cache of newly added repos
-$LILTO $SUDOAPTGET update
-$BIGTO $SUDOAPTGET install ${INSTALL_PACKAGES[@]}
-
-echo "Installing cataonit and libseccomp.sudo"
-ooe.sh sudo /tmp/libpod/hack/install_catatonit.sh
-ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo
-
-# Ensure there are no disruptive periodic services enabled by default in image
-systemd_banish
+bash $PACKER_BASE/ubuntu_packaging.sh
 
-CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc"
-if sudo dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH"
-then
-    echo "Linking $CRIO_RUNC_PATH to /usr/bin/runc for ease of testing."
-    sudo ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc"
-fi
+# Load installed environment right now (happens automatically in a new process)
+source /usr/share/automation/environment
 
 echo "Making Ubuntu kernel to enable cgroup swap accounting as it is not the default."
 SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g'
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 57c9ec52a..6bec9625e 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -43,16 +43,8 @@ case "${OS_RELEASE_ID}" in
     fedora)
         # All SELinux distros need this for systemd-in-a-container
         setsebool container_manage_cgroup true
-        if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
-            bash "$SCRIPT_BASE/add_second_partition.sh"
-        fi
 
-        if [[ $OS_RELEASE_VER -le 31 ]]; then
-            warn "Switching io scheduler to 'deadline' to avoid RHBZ 1767539"
-            warn "aka https://bugzilla.kernel.org/show_bug.cgi?id=205447"
-            echo "mq-deadline" > /sys/block/sda/queue/scheduler
-            cat /sys/block/sda/queue/scheduler
-        fi
+        workaround_bfq_bug
 
         if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
             bash "$SCRIPT_BASE/add_second_partition.sh"