Merge pull request #13376 from cevich/f36_update

Update to F36 CI VM Images + Testing netavark/aardvark-dns

6 files changed, 94 insertions, 162 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index cf97f4467..958c6b0c7 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -7,10 +7,6 @@ env:
     ####
     # Name of the ultimate destination branch for this CI run, PR or post-merge.
     DEST_BRANCH: "main"
-    # Netavark branch to use when TEST_ENVIRON=host-netavark
-    NETAVARK_BRANCH: "main"
-    # Aardvark branch to use
-    AARDVARK_BRANCH: "main"
     # Overrides default location (/tmp/cirrus) for repo clone
     GOPATH: &gopath "/var/tmp/go"
     GOBIN: "${GOPATH}/bin"
@@ -24,21 +20,16 @@ env:
     # Runner statistics log file path/name
     STATS_LOGFILE_SFX: 'runner_stats.log'
     STATS_LOGFILE: '$GOSRC/${CIRRUS_TASK_NAME}-${STATS_LOGFILE_SFX}'
-    # Netavark/aardvark location/options when TEST_ENVIRON=host-netavark
-    NETAVARK_URL: "https://api.cirrus-ci.com/v1/artifact/github/containers/netavark/success/binary.zip?branch=${NETAVARK_BRANCH}"
-    NETAVARK_DEBUG: 0  # set non-zero to use the debug-mode binary
-    AARDVARK_URL: "https://api.cirrus-ci.com/v1/artifact/github/containers/aardvark-dns/success/binary.zip?branch=${AARDVARK_BRANCH}"
-    AARDVARK_DEBUG: 0  # set non-zero to use the debug-mode binary
 
     ####
     #### Cache-image names to test with (double-quotes around names are critical)
     ####
-    FEDORA_NAME: "fedora-35"
-    PRIOR_FEDORA_NAME: "fedora-34"
+    FEDORA_NAME: "fedora-36"
+    PRIOR_FEDORA_NAME: "fedora-35"
     UBUNTU_NAME: "ubuntu-2110"
 
     # Google-cloud VM Images
-    IMAGE_SUFFIX: "c4831699639992320"
+    IMAGE_SUFFIX: "c4955393725038592"
     FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}"
     UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}"
@@ -53,7 +44,7 @@ env:
     #### N/B: Required ALL of these are set for every single task.
     ####
     TEST_FLAVOR:             # int, sys, ext_svc, validate, automation, etc.
-    TEST_ENVIRON: host       # 'host', 'host-netavark', or 'container'
+    TEST_ENVIRON: host       # 'host', or 'container'
     PODBIN_NAME: podman      # 'podman' or 'remote'
     PRIV_NAME: root          # 'root' or 'rootless'
     DISTRO_NV:               # any {PRIOR_,}{FEDORA,UBUNTU}_NAME value
@@ -161,11 +152,11 @@ build_task:
               CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
               # ID for re-use of build output
               _BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
-        #- env: &priorfedora_envvars
-        #      DISTRO_NV: ${PRIOR_FEDORA_NAME}
-        #      VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
-        #      CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
-        #      _BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
+        - env: &priorfedora_envvars
+              DISTRO_NV: ${PRIOR_FEDORA_NAME}
+              VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
+              CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
+              _BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
         - env: &ubuntu_envvars
               DISTRO_NV: ${UBUNTU_NAME}
               VM_IMAGE_NAME: ${UBUNTU_CACHE_IMAGE_NAME}
@@ -394,7 +385,7 @@ unit_test_task:
         - validate
     matrix:
         - env: *stdenvars
-        #- env: *priorfedora_envvars
+        - env: *priorfedora_envvars
         - env: *ubuntu_envvars
         # Special-case: Rootless on latest Fedora (standard) VM
         - name: "Rootless unit on $DISTRO_NV"
@@ -515,11 +506,11 @@ container_integration_test_task:
               _BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
               VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
               CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
-        #- env:
-        #      DISTRO_NV: ${PRIOR_FEDORA_NAME}
-        #      _BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
-        #      VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
-        #      CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
+        - env:
+              DISTRO_NV: ${PRIOR_FEDORA_NAME}
+              _BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
+              VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME}
+              CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN}
     gce_instance: *standardvm
     timeout_in: 90m
     env:
@@ -553,41 +544,6 @@ rootless_integration_test_task:
     always: *int_logs_artifacts
 
 
-# Run various scenarios using upstream netavark/aardvark-dns binaries
-netavark_task:
-    name: "Netavark $TEST_FLAVOR $PODBIN_NAME $PRIV_NAME"
-    alias: netavark
-    only_if: *not_build
-    skip: *branches_and_tags
-    depends_on:
-        - unit_test
-    gce_instance: *standardvm
-    matrix:
-        - env: &nenv
-            DISTRO_NV: ${FEDORA_NAME}
-            _BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID}
-            VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME}
-            CTR_FQIN: ${FEDORA_CONTAINER_FQIN}
-            TEST_FLAVOR: int
-            TEST_ENVIRON: host-netavark
-        - env:
-            <<: *nenv
-            TEST_FLAVOR: int
-            PRIV_NAME: rootless
-        - env:
-            <<: *nenv
-            TEST_FLAVOR: sys
-        - env:
-            <<: *nenv
-            TEST_FLAVOR: sys
-            PRIV_NAME: rootless
-    clone_script: *noop  # Comes from cache
-    gopath_cache: *ro_gopath_cache
-    setup_script: *setup
-    main_script: *main
-    always: *int_logs_artifacts
-
-
 # Always run subsequent to integration tests.  While parallelism is lost
 # with runtime, debugging system-test failures can be more challenging
 # for some golang developers.  Otherwise the following tasks run across
@@ -841,7 +797,6 @@ success_task:
         - remote_integration_test
         - container_integration_test
         - rootless_integration_test
-        - netavark
         - local_system_test
         - remote_system_test
         - rootless_system_test
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 09a255e6f..0f02c166f 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -97,7 +97,7 @@ EPOCH_TEST_COMMIT="$CIRRUS_BASE_SHA"
 # testing operations on all platforms and versions.  This is necessary
 # to avoid needlessly passing through global/system values across
 # contexts, such as host->container or root->rootless user
-PASSTHROUGH_ENV_RE='(^CI.*)|(^CIRRUS)|(^DISTRO_NV)|(^GOPATH)|(^GOCACHE)|(^GOSRC)|(^SCRIPT_BASE)|(CGROUP_MANAGER)|(OCI_RUNTIME)|(^TEST.*)|(^PODBIN_NAME)|(^PRIV_NAME)|(^ALT_NAME)|(^ROOTLESS_USER)|(SKIP_USERNS)|(.*_NAME)|(.*_FQIN)'
+PASSTHROUGH_ENV_RE='(^CI.*)|(^CIRRUS)|(^DISTRO_NV)|(^GOPATH)|(^GOCACHE)|(^GOSRC)|(^SCRIPT_BASE)|(CGROUP_MANAGER)|(OCI_RUNTIME)|(^TEST.*)|(^PODBIN_NAME)|(^PRIV_NAME)|(^ALT_NAME)|(^ROOTLESS_USER)|(SKIP_USERNS)|(.*_NAME)|(.*_FQIN)|(NETWORK_BACKEND)'
 # Unsafe env. vars for display
 SECRET_ENV_RE='(ACCOUNT)|(GC[EP]..+)|(SSH)|(PASSWORD)|(TOKEN)'
 
@@ -169,10 +169,6 @@ setup_rootless() {
     groupadd -g $rootless_gid $ROOTLESS_USER
     useradd -g $rootless_gid -u $rootless_uid --no-user-group --create-home $ROOTLESS_USER
 
-    # We also set up rootless user for image-scp tests (running as root)
-    if [[ $PRIV_NAME = "rootless" ]]; then
-        chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOPATH" "$GOSRC"
-    fi
     echo "$ROOTLESS_USER ALL=(root) NOPASSWD: ALL" > /etc/sudoers.d/ci-rootless
 
     mkdir -p "$HOME/.ssh" "/home/$ROOTLESS_USER/.ssh"
@@ -216,20 +212,39 @@ setup_rootless() {
 install_test_configs() {
     msg "Installing ./test/registries.conf system-wide."
     install -v -D -m 644 ./test/registries.conf /etc/containers/
-    if [[ "$TEST_ENVIRON" =~ netavark ]]; then
-        # belt-and-suspenders: any pre-existing CNI config. will spoil
-        # default use tof netavark (when both are installed).
-        rm -rf /etc/cni/net.d/*
-    else
-        echo "Installing cni config, policy and registry config"
-        req_env_vars GOSRC SCRIPT_BASE
-        cd $GOSRC || exit 1
-        install -v -D -m 644 ./cni/87-podman-bridge.conflist /etc/cni/net.d/
-        # This config must always sort last in the list of networks (podman picks first one
-        # as the default).  This config prevents allocation of network address space used
-        # by default in google cloud.  https://cloud.google.com/vpc/docs/vpc#ip-ranges
-        install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist /etc/cni/net.d/
-    fi
+}
+
+use_cni() {
+    msg "Unsetting NETWORK_BACKEND for all subsequent environments."
+    echo "export -n NETWORK_BACKEND" >> /etc/ci_environment
+    echo "unset NETWORK_BACKEND" >> /etc/ci_environment
+    export -n NETWORK_BACKEND
+    unset NETWORK_BACKEND
+    msg "Installing default CNI configuration"
+    cd $GOSRC || exit 1
+    rm -rvf /etc/cni/net.d
+    mkdir -p /etc/cni/net.d
+    install -v -D -m 644 ./cni/87-podman-bridge.conflist \
+        /etc/cni/net.d/
+    # This config must always sort last in the list of networks (podman picks
+    # first one as the default).  This config prevents allocation of network
+    # address space used by default in google cloud.
+    # https://cloud.google.com/vpc/docs/vpc#ip-ranges
+    install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist \
+        /etc/cni/net.d/
+}
+
+use_netavark() {
+    msg "Forcing NETWORK_BACKEND=netavark for all subsequent environments."
+    echo "NETWORK_BACKEND=netavark" >> /etc/ci_environment
+    export NETWORK_BACKEND=netavark  # needed for install_test_configs()
+    msg "Removing any/all CNI configuration"
+    rm -rvf /etc/cni/net.d/*
+
+    # TODO: Remove this when netavark/aardvark-dns development slows down
+    warn "Updating netavark/aardvark-dns to avoid frequent VM image rebuilds"
+    # N/B: This is coming from updates-testing repo in F36
+    lilto dnf update -y netavark aardvark-dns
 }
 
 # Remove all files provided by the distro version of podman.
diff --git a/contrib/cirrus/logcollector.sh b/contrib/cirrus/logcollector.sh
index 0cfbf7135..d712713b4 100755
--- a/contrib/cirrus/logcollector.sh
+++ b/contrib/cirrus/logcollector.sh
@@ -40,32 +40,34 @@ case $1 in
     packages)
         # These names are common to Fedora and Ubuntu
         PKG_NAMES=(\
-                    conmon \
-                    containernetworking-plugins \
-                    containers-common \
-                    criu \
-                    crun \
-                    golang \
-                    podman \
-                    runc \
-                    skopeo \
-                    slirp4netns \
+                    conmon
+                    containernetworking-plugins
+                    containers-common
+                    criu
+                    crun
+                    golang
+                    podman
+                    runc
+                    skopeo
+                    slirp4netns
         )
         case $OS_RELEASE_ID in
             fedora)
                 cat /etc/fedora-release
                 PKG_LST_CMD='rpm -q --qf=%{N}-%{V}-%{R}-%{ARCH}\n'
                 PKG_NAMES+=(\
-                    container-selinux \
-                    libseccomp \
+                    aardvark
+                    container-selinux
+                    libseccomp
+                    netavark
                 )
                 ;;
             ubuntu)
                 cat /etc/issue
                 PKG_LST_CMD='dpkg-query --show --showformat=${Package}-${Version}-${Architecture}\n'
                 PKG_NAMES+=(\
-                    cri-o-runc \
-                    libseccomp2 \
+                    cri-o-runc
+                    libseccomp2
                 )
                 ;;
             *) bad_os_id_ver ;;
@@ -74,19 +76,6 @@ case $1 in
         echo "Cgroups: " $(stat -f -c %T /sys/fs/cgroup)
         # Any not-present packages will be listed as such
         $PKG_LST_CMD "${PKG_NAMES[@]}" | sort -u
-
-        # TODO: Remove this once netavark/aardvark-dns packages are used
-        if [[ "$TEST_ENVIRON" =~ netavark ]]; then
-            _npath=/usr/local/libexec/podman/
-            for name in netavark aardvark-dns; do
-                echo "$name binary details:"
-                if [[ -r "$_npath/${name}.info" ]]; then
-                    cat "$_npath/${name}.info"
-                else
-                    echo "WARNING: $_npath/${name}.info not found."
-                fi
-            done
-        fi
         ;;
     time)
         # Assumed to be empty/undefined outside of Cirrus-CI (.cirrus.yml)
diff --git a/contrib/cirrus/runner.sh b/contrib/cirrus/runner.sh
index 5d6ba9921..101270703 100755
--- a/contrib/cirrus/runner.sh
+++ b/contrib/cirrus/runner.sh
@@ -12,7 +12,7 @@ set -eo pipefail
 # most notably:
 #
 #    PODBIN_NAME  : "podman" (i.e. local) or "remote"
-#    TEST_ENVIRON : 'host', 'host-netavark', or 'container'; desired environment in which to run
+#    TEST_ENVIRON : 'host', or 'container'; desired environment in which to run
 #    CONTAINER    : 1 if *currently* running inside a container, 0 if host
 #
 
@@ -449,6 +449,13 @@ if [[ "$PRIV_NAME" == "rootless" ]] && [[ "$UID" -eq 0 ]]; then
     # https://github.com/containers/podman/issues/10857
     rm -rf /var/lib/cni
 
+    # This must be done at the last second, otherwise `make` calls
+    # in setup_environment (as root) will balk about ownership.
+    msg "Recursively chowning \$GOPATH and \$GOSRC to $ROOTLESS_USER"
+    if [[ $PRIV_NAME = "rootless" ]]; then
+        chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOPATH" "$GOSRC"
+    fi
+
     req_env_vars ROOTLESS_USER
     msg "Re-executing runner through ssh as user '$ROOTLESS_USER'"
     msg "************************************************************"
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 742289733..e3eb46783 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -38,13 +38,12 @@ done
 
 cp hack/podman-registry /bin
 
-# Make sure cni network plugins directory exists
-mkdir -p /etc/cni/net.d
-
 # Some test operations & checks require a git "identity"
 _gc='git config --file /root/.gitconfig'
 $_gc user.email "TMcTestFace@example.com"
 $_gc user.name "Testy McTestface"
+# Bypass git safety/security checks when operating in a throwaway environment
+git config --system --add safe.directory $GOSRC
 
 # Ensure that all lower-level contexts and child-processes have
 # ready access to higher level orchestration (e.g Cirrus-CI)
@@ -84,13 +83,6 @@ case "$CG_FS_TYPE" in
             else
                 echo "OCI_RUNTIME=runc" >> /etc/ci_environment
             fi
-
-            # As a general policy CGv1 + runc should coincide with the "older"
-            # VM Images in CI.  Verify this is the case.
-            if [[ -n "$VM_IMAGE_NAME" ]] && [[ ! "$VM_IMAGE_NAME" =~ prior ]]
-            then
-                die "Most recent distro. version should never run with CGv1"
-            fi
         fi
         ;;
     cgroup2fs)
@@ -99,13 +91,6 @@ case "$CG_FS_TYPE" in
             # which uses runc as the default.
             warn "Forcing testing with crun instead of runc"
             echo "OCI_RUNTIME=crun" >> /etc/ci_environment
-
-            # As a general policy CGv2 + crun should coincide with the "newer"
-            # VM Images in CI.  Verify this is the case.
-            if [[ -n "$VM_IMAGE_NAME" ]] && [[ "$VM_IMAGE_NAME" =~ prior ]]
-            then
-                die "Least recent distro. version should never run with CGv2"
-            fi
         fi
         ;;
     *) die_unknown CG_FS_TYPE
@@ -130,6 +115,19 @@ case "$OS_RELEASE_ID" in
             msg "Enabling container_manage_cgroup"
             setsebool container_manage_cgroup true
         fi
+
+        # For release 36 and later, netavark/aardvark is the default
+        # networking stack for podman.  All previous releases only have
+        # CNI networking available.  Upgrading from one to the other is
+        # not supported at this time.  Support execution of the upgrade
+        # tests in F36 and later, by disabling Netavark and enabling CNI.
+        if [[ "$OS_RELEASE_VER" -ge 36 ]] && \
+           [[ "$TEST_FLAVOR" != "upgrade_test" ]];
+        then
+            use_netavark
+        else # Fedora < 36, or upgrade testing.
+            use_cni
+        fi
         ;;
     *) die_unknown OS_RELEASE_ID
 esac
@@ -137,7 +135,7 @@ esac
 # Required to be defined by caller: The environment where primary testing happens
 # shellcheck disable=SC2154
 case "$TEST_ENVIRON" in
-    host*)
+    host)
         # The e2e tests wrongly guess `--cgroup-manager` option
         # shellcheck disable=SC2154
         if [[ "$CG_FS_TYPE" == "cgroup2fs" ]] || [[ "$PRIV_NAME" == "root" ]]
@@ -148,43 +146,6 @@ case "$TEST_ENVIRON" in
             warn "Forcing CGROUP_MANAGER=cgroupfs"
             echo "CGROUP_MANAGER=cgroupfs" >> /etc/ci_environment
         fi
-        # TODO: For the foreseeable future, need to support running tests
-        # with and without the latest netavark/aardvark.  Once they're more
-        # stable and widely supported in Fedora, they can be pre-installed
-        # from its RPM at VM image build-time.
-        if [[ "$TEST_ENVIRON" =~ netavark ]]; then
-            for info in "netavark $NETAVARK_BRANCH $NETAVARK_URL $NETAVARK_DEBUG" \
-                        "aardvark-dns $AARDVARK_BRANCH $AARDVARK_URL $AARDVARK_DEBUG"; do
-
-                read _name _branch _url _debug <<<"$info"
-                req_env_vars _name _branch _url _debug
-                msg "Downloading latest $_name from upstream branch '$_branch'"
-                # Use identifiable archive filename in of a get_ci_env.sh environment
-                curl --fail --location -o /tmp/$_name.zip "$_url"
-
-                # Needs to be in a specific location
-                # ref: https://github.com/containers/common/blob/main/pkg/config/config_linux.go#L39
-                _pdir=/usr/local/libexec/podman
-                mkdir -p $_pdir
-                cd $_pdir
-                msg "$PWD"
-                unzip /tmp/$_name.zip
-                if ((_debug)); then
-                    warn "Using debug $_name binary"
-                    mv $_name.debug $_name
-                else
-                    rm $_name.debug
-                fi
-                chmod 0755 $_pdir/$_name
-                cd -
-            done
-
-            restorecon -F -v $_nvdir
-            # This is critical, it signals to all tests that netavark
-            # use is expected.
-            msg "Forcing NETWORK_BACKEND=netavark in all subsequent environments."
-            echo "NETWORK_BACKEND=netavark" >> /etc/ci_environment
-        fi
         ;;
     container)
         if ((CONTAINER==0)); then  # not yet inside a container
@@ -345,6 +306,9 @@ case "$TEST_FLAVOR" in
         # Guarantee the docker daemon can't be started, even by accident
         rm -vf $(type -P dockerd)
 
+        msg "Recursively chowning source to $ROOTLESS_USER"
+        chown -R $ROOTLESS_USER:$ROOTLESS_USER "$GOPATH" "$GOSRC"
+
         msg "Obtaining necessary gitlab-runner testing bits"
         slug="gitlab.com/gitlab-org/gitlab-runner"
         helper_fqin="registry.gitlab.com/gitlab-org/gitlab-runner/gitlab-runner-helper:x86_64-latest-pwsh"
diff --git a/test/e2e/run_networking_test.go b/test/e2e/run_networking_test.go
index 49c387227..baec444e4 100644
--- a/test/e2e/run_networking_test.go
+++ b/test/e2e/run_networking_test.go
@@ -1133,6 +1133,8 @@ EXPOSE 2004-2005/tcp`, ALPINE)
 	})
 
 	It("podman run with ipam none driver", func() {
+		// Test fails, issue #13931
+		SkipIfNetavark(podmanTest)
 		net := "ipam" + stringid.GenerateNonCryptoID()
 		session := podmanTest.Podman([]string{"network", "create", "--ipam-driver=none", net})
 		session.WaitWithDefaultTimeout()