Cirrus: Add support for testing F30

Remove disused `build_cache_images` task, and
update relevant dockerfiles for F30.

Fix problem of cloud-init failing to expand root-device on boot
(/var/lib/cloud/instance left in improper state).

Fix problem of cloud-init racing with google-network-daemon.service on
boot (looking for cloudconfig metadata too early).  Causing
root-device to _sometimes_ fail to expand.

Fix problem of hack/get_ci_vm.sh argument passing.

Signed-off-by: Chris Evich <cevich@redhat.com>

12 files changed, 70 insertions, 112 deletions

diff --git a/.cirrus.yml b/.cirrus.yml
index 848dc2b6d..91caf7fe0 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -31,9 +31,9 @@ env:
     ####
     #### Cache-image names to test with
     ###
-    FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-4844850202017792"
-    PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-4844850202017792"
-    UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-4844850202017792"
+    FEDORA_CACHE_IMAGE_NAME: "fedora-30-libpod-5156500369047552"
+    PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5156500369047552"
+    UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5156500369047552"
 
     ####
     #### Variables for composing new cache-images (used in PR testing) from
@@ -211,8 +211,9 @@ build_each_commit_task:
 
     setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
     build_each_commit_script:
+        - 'source $SCRIPT_BASE/lib.sh'
         - 'git fetch --depth $CIRRUS_CLONE_DEPTH origin $CIRRUS_BASE_BRANCH |& ${TIMESTAMP}'
-        - 'env GOPATH=/var/tmp/go/ make build-all-new-commits GIT_BASE_BRANCH=origin/$CIRRUS_BASE_BRANCH |& ${TIMESTAMP}'
+        - 'make build-all-new-commits GIT_BASE_BRANCH=origin/$CIRRUS_BASE_BRANCH |& ${TIMESTAMP}'
 
     on_failure:
         failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh'
@@ -267,7 +268,7 @@ testing_task:
         matrix:
             # Images are generated separately, from build_images_task (below)
             image_name: "${FEDORA_CACHE_IMAGE_NAME}"
-            #image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
+            image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}"
             image_name: "${UBUNTU_CACHE_IMAGE_NAME}"
 
     timeout_in: 120m
@@ -383,6 +384,12 @@ test_build_cache_images_task:
 
     environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
     build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}'
+    df_script: '${DFCMD}'
+    journalctl_b_script: 'journalctl -b'
+
+    on_failure:
+        failed_df_script: '${DFCMD}'
+        failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"'
 
 
 # Test building of new cache-images for future PR testing, in this PR.
@@ -401,8 +408,8 @@ verify_test_built_images_task:
     gce_instance:
         matrix:
             # Images are generated separately, from build_images_task (below)
-            #image_name: "fedora-28${BUILT_IMAGE_SUFFIX}"
             image_name: "fedora-29${BUILT_IMAGE_SUFFIX}"
+            image_name: "fedora-30${BUILT_IMAGE_SUFFIX}"
             image_name: "ubuntu-18${BUILT_IMAGE_SUFFIX}"
 
     env:
@@ -428,56 +435,6 @@ verify_test_built_images_task:
         failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"'
 
 
-# Build new cache-images for future PR testing, but only after a PR merge.
-# The cache-images save install/setup time needed test every PR.  The 'active' images
-# are selected by the 'image_name' items tasks above.  Currently this requires
-# manually updating the names, but this could be automated (see comment below).
-build_cache_images_task:
-    # Only produce new cache-images after a PR merge, and if a magic string
-    # is present in the most recent ___commit-message___.
-    only_if: >-
-        $CIRRUS_BRANCH == 'master' &&
-        $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*REBUILD\s*IMAGES\s*\*\*\*.*'
-
-    # Require tests to pass first.
-    depends_on:
-        - "gating"
-        - "testing"
-        - "rootless_testing"
-
-    # VMs created by packer are not cleaned up by cirrus
-    auto_cancellation: $CI != "true"
-
-    gce_instance:
-        image_project: "libpod-218412"
-        zone: "us-central1-a"  # Required by Cirrus for the time being
-        cpu: 4
-        memory: "4Gb"
-        disk: 200
-        image_name: "${IMAGE_BUILDER_CACHE_IMAGE_NAME}"
-        # Additional permissions for building GCE images, within a GCE VM
-        scopes:
-            - compute
-            - devstorage.full_control
-
-    environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
-    build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}'
-
-    # TODO,Continuous Delivery: Automatically open a libpod PR after using 'sed' to replace
-    #                           the image_names with the new (just build) images.  That will
-    #                           cause a new round of testing to happen (via the PR) using
-    #                           the new images.  When all is good, the PR may be manually
-    #                           merged so all PR testing uses the new images.  The script
-    #                           names (below) describe their purpose in this workflow.
-    # deploy_images_script:
-    #   - clone_podman_release_branch.sh
-    #   - modify_cirrus_yaml_image_names.sh
-    #   - commit_and_create_upstream_pr.sh
-
-    on_failure:
-        failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh |& ${TIMESTAMP}'
-
-
 # Post message to IRC if everything passed
 success_task:
 
@@ -494,7 +451,6 @@ success_task:
         - "special_testing_in_podman"
         - "test_build_cache_images"
         - "verify_test_built_images"
-        - "build_cache_images"
 
     env:
         CIRRUS_WORKING_DIR: "/usr/src/libpod"
diff --git a/Dockerfile.fedora b/Dockerfile.fedora
index c34d4bb16..5cbc6e1fe 100644
--- a/Dockerfile.fedora
+++ b/Dockerfile.fedora
@@ -1,9 +1,8 @@
-FROM registry.fedoraproject.org/fedora:29
+FROM registry.fedoraproject.org/fedora:30
 
 RUN dnf -y install btrfs-progs-devel \
               atomic-registries \
               bzip2 \
-              conmon \
               device-mapper-devel \
               findutils \
               git \
diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md
index 94494a558..5ff4f290f 100644
--- a/contrib/cirrus/README.md
+++ b/contrib/cirrus/README.md
@@ -96,10 +96,18 @@ images following the standard naming format; ***however, only runs a limited
 sub-set of automated tests***.  Validating newly built images fully, requires
 updating ``.cirrus.yml``.
 
-***Manual Steps:***  Assuming `verify_test_built_images` passes, then
+***N/B: Steps below are performed by automation***
+
+1. Using the just build VM images, launch VMs and wait for them to boot.
+
+2. Execute the `setup_environment.sh` as in the `testing` task.
+
+2. Execute the `integration_test.sh` as in the `testing` task.
+
+
+***Manual Steps:***  Assuming the automated steps pass, then
 you'll find the new image names displayed at the end of the
-`test_build_cache_images_task` in the `build_vm_images` output.
-For example:
+`test_build_cache_images`.  For example:
 
 
 ```
@@ -135,18 +143,6 @@ the magic ``***CIRRUS: TEST IMAGES***`` string.  Keeping it and
 and test images again.
 
 
-### ``build_cache_images`` Task  *(Deprecated)*
-
-Exactly the same as ``test_build_cache_images_task`` task, but only runs on
-the master branch.  Requires a magic string to be in the `HEAD`
-commit message: ``***CIRRUS: BUILD IMAGES***``
-
-When successful, the manifest file along with all VM disks, are moved
-into a dedicated google storage bucket, separate from the one used by
-`test_build_cache_images_task`.  These may be used to create new cache-images for
-PR testing by manually importing them as described above.
-
-
 ### Base-images
 
 Base-images are VM disk-images specially prepared for executing as GCE VMs.
@@ -158,10 +154,9 @@ as the standard 'cloud-init' services.
    with services pre-installed, for many platforms. For example,
    RHEL, CentOS, and Ubuntu.
 
-*  Google does ***not*** provide any images for Fedora or Fedora Atomic
-   Host (as of 11/2018), nor do they provide a base-image prepared to
-   run packer for creating other images in the ``build_vm_images`` Task
-   (above).
+*  Google does ***not*** provide any images for Fedora (as of 5/2019), nor do
+   they provide a base-image prepared to run packer for creating other images
+   in the ``test_build_vm_images`` Task (above).
 
 *  Base images do not need to be produced often, but doing so completely
    manually would be time-consuming and error-prone.  Therefor a special
diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh
index f9ba010cd..b163834d5 100755
--- a/contrib/cirrus/integration_test.sh
+++ b/contrib/cirrus/integration_test.sh
@@ -28,8 +28,6 @@ then
         -e "DIST=$OS_RELEASE_ID" \
         -e "CONTAINER_RUNTIME=$CONTAINER_RUNTIME" \
         $IN_PODMAN_IMAGE bash $GOSRC/$SCRIPT_BASE/container_test.sh -b -i -t
-
-    exit $?
 elif [[ "$SPECIALMODE" == "rootless" ]]
 then
     req_env_var ROOTLESS_USER
@@ -52,5 +50,4 @@ else
     else
         make local${TESTSUITE}
     fi
-    exit $?
 fi
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index 97901cfc7..98685faa9 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -55,15 +55,15 @@ PACKER_VER="1.3.5"
 # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
 
 # Base-images rarely change, define them here so they're out of the way.
-PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28}"
+PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-30,fedora-29}"
 # Google-maintained base-image names
 UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20181203a"
 # Manually produced base-image names (see $SCRIPT_BASE/README.md)
-FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245"
-# FEDORA_BASE_IMAGE: "fedora-cloud-base-30-1-2-1556821664"
-PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-28-1-1-1544474897"
-# PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245"
+FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1559164849"
+PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849"
 BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}"
+# IN_PODMAN container image
+IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest"
 
 # Safe env. vars. to transfer from root -> $ROOTLESS_USER  (go env handled separetly)
 ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(TEST_REMOTE.*)'
@@ -74,9 +74,6 @@ SPECIALMODE="${SPECIALMODE:-none}"
 TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}"
 export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
 
-# IN_PODMAN container image
-IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest"
-
 # When running as root, this may be empty or not, as a user, it MUST be set.
 if [[ "$USER" == "root" ]]
 then
@@ -347,7 +344,7 @@ _finalize(){
     set +e  # make errors non-fatal
     echo "Removing leftover giblets from cloud-init"
     cd /
-    sudo rm -rf /var/lib/cloud/instance?
+    sudo rm -rf /var/lib/cloud/instanc*
     sudo rm -rf /root/.ssh/*
     sudo rm -rf /home/*
     sudo rm -rf /tmp/*
diff --git a/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service b/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service
new file mode 100644
index 000000000..4d2197d87
--- /dev/null
+++ b/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service
@@ -0,0 +1,20 @@
+[Unit]
+Description=Initial cloud-init job (metadata service crawler)
+DefaultDependencies=no
+Wants=cloud-init-local.service
+After=cloud-init-local.service
+Wants=google-network-daemon.service
+After=google-network-daemon.service
+Before=systemd-user-sessions.service
+
+[Service]
+Type=oneshot
+ExecStart=/usr/bin/cloud-init init
+RemainAfterExit=yes
+TimeoutSec=0
+
+# Output needs to appear in instance console output
+StandardOutput=journal+console
+
+[Install]
+WantedBy=cloud-init.target
diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh
index 2e6d3eceb..a425b2b57 100644
--- a/contrib/cirrus/packer/fedora_base-setup.sh
+++ b/contrib/cirrus/packer/fedora_base-setup.sh
@@ -16,11 +16,17 @@ echo "Updating packages"
 ooe.sh dnf -y update
 
 echo "Installing necessary packages and google services"
-ooe.sh dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin
+ooe.sh dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin ethtool
 
 echo "Enabling services"
 ooe.sh systemctl enable rngd
 
+# There is a race that can happen on boot between the GCE services configuring
+# the VM, and cloud-init trying to do similar activities.  Use a customized
+# unit file to make sure cloud-init starts after the google-compute-* services.
+echo "Setting cloud-init service to start after google-network-daemon.service"
+cp -v $GOSRC/$PACKER_BASE/cloud-init/fedora/cloud-init.service /etc/systemd/system/
+
 rh_finalize
 
 echo "SUCCESS!"
diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml
index 560cb321c..e519d2fba 100644
--- a/contrib/cirrus/packer/libpod_base_images.yml
+++ b/contrib/cirrus/packer/libpod_base_images.yml
@@ -78,7 +78,7 @@ builders:
       ssh_username: 'root'
 
     - <<: *nested_virt
-      name: 'prior-fedora'
+      name: 'prior_fedora'
       iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}'
       iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}'
 
@@ -121,7 +121,7 @@ provisioners:
 
 post-processors:
     - - type: "compress"
-        only: ['fedora', 'prior-fedora']
+        only: ['fedora', 'prior_fedora']
         output: '/tmp/{{build_name}}/disk.raw.tar.gz'
         format: '.tar.gz'
         compression_level: 9
@@ -136,7 +136,7 @@ post-processors:
         image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}'
         image_family: '{{user `FEDORA_BASE_IMAGE_NAME`}}'
       - <<: *gcp_import
-        only: ['prior-fedora']
+        only: ['prior_fedora']
         image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
         image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}'
         image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}'
diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml
index 34d4db7fb..c25da25ac 100644
--- a/contrib/cirrus/packer/libpod_images.yml
+++ b/contrib/cirrus/packer/libpod_images.yml
@@ -55,11 +55,11 @@ builders:
 
     #      v----- is a YAML alias, allows partial re-use of the anchor object
     - <<: *gce_hosted_image
-      name: 'fedora-29'
+      name: 'fedora-30'
       source_image: '{{user `FEDORA_BASE_IMAGE`}}'
 
     - <<: *gce_hosted_image
-      name: 'fedora-28'
+      name: 'fedora-29'
       source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}'
 
 # The brains of the operation, making actual modifications to the base-image.
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index 13bce506a..8fdcf5897 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -33,20 +33,8 @@ done
 #       (see docs).
 case "${OS_REL_VER}" in
     ubuntu-18) ;;
-    fedora-29)
-        # Occasionally, and seemingly only on F29 the root disk fails to expand
-        # upon boot.  When this happens, any number of failures could occur if
-        # space runs out.  Until there is time to investigate the actual cause,
-        # workaround this problem by detecting it and acting accordingly.
-        REMAINING=$(df /dev/sda1 | tail -1 | awk '{print $4}')
-        if [[ "$REMAINING" -lt "100000000" ]]  # .cirrus.yml specifies 200gig
-        then
-            echo "Fixing failure to expand root filesystem"
-            growpart /dev/sda 1  # device guaranteed by cloud provider
-            resize2fs /dev/sda1  # growpart & resuze guaranteed by base-image
-        fi
-        ;;
-    fedora-28) ;;
+    fedora-30) ;;
+    fedora-29) ;;
     centos-7)  # Current VM is an image-builder-image no local podman/testing
         echo "No further setup required for VM image building"
         exit 0
diff --git a/hack/get_ci_vm.sh b/hack/get_ci_vm.sh
index 12dd211f4..90e3aea8e 100755
--- a/hack/get_ci_vm.sh
+++ b/hack/get_ci_vm.sh
@@ -168,7 +168,7 @@ parse_args(){
     if echo "$IMAGE_NAME" | grep -q "image-builder-image"
     then
         echo -e "Creating an image-builder VM, I hope you know what you're doing.\n"
-        IBI_ARGS="--scopes=compute-rw,storage-rw,userinfo-email \"--min-cpu-platform=Intel Haswell\""
+        IBI_ARGS="--scopes=compute-rw,storage-rw,userinfo-email"
         SSHUSER="centos"
     else
         unset IBI_ARGS
diff --git a/test/install/Dockerfile.Fedora b/test/install/Dockerfile.Fedora
index 74cee771d..3e7b6e238 100644
--- a/test/install/Dockerfile.Fedora
+++ b/test/install/Dockerfile.Fedora
@@ -1,3 +1,3 @@
-FROM registry.fedoraproject.org/fedora:29
+FROM registry.fedoraproject.org/fedora:30
 
 RUN dnf install -y rpms/x86_64/*