diff options
| -rw-r--r-- | .cirrus.yml | 70 | ||||
| -rw-r--r-- | Dockerfile.fedora | 3 | ||||
| -rw-r--r-- | contrib/cirrus/README.md | 33 | ||||
| -rwxr-xr-x | contrib/cirrus/integration_test.sh | 3 | ||||
| -rw-r--r-- | contrib/cirrus/lib.sh | 15 | ||||
| -rw-r--r-- | contrib/cirrus/packer/cloud-init/fedora/cloud-init.service | 20 | ||||
| -rw-r--r-- | contrib/cirrus/packer/fedora_base-setup.sh | 8 | ||||
| -rw-r--r-- | contrib/cirrus/packer/libpod_base_images.yml | 6 | ||||
| -rw-r--r-- | contrib/cirrus/packer/libpod_images.yml | 4 | ||||
| -rwxr-xr-x | contrib/cirrus/setup_environment.sh | 16 | ||||
| -rwxr-xr-x | hack/get_ci_vm.sh | 2 | ||||
| -rw-r--r-- | test/install/Dockerfile.Fedora | 2 |
12 files changed, 70 insertions, 112 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 848dc2b6d..91caf7fe0 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -31,9 +31,9 @@ env: #### #### Cache-image names to test with ### - FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-4844850202017792" - PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-28-libpod-4844850202017792" - UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-4844850202017792" + FEDORA_CACHE_IMAGE_NAME: "fedora-30-libpod-5156500369047552" + PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-29-libpod-5156500369047552" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-libpod-5156500369047552" #### #### Variables for composing new cache-images (used in PR testing) from @@ -211,8 +211,9 @@ build_each_commit_task: setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' build_each_commit_script: + - 'source $SCRIPT_BASE/lib.sh' - 'git fetch --depth $CIRRUS_CLONE_DEPTH origin $CIRRUS_BASE_BRANCH |& ${TIMESTAMP}' - - 'env GOPATH=/var/tmp/go/ make build-all-new-commits GIT_BASE_BRANCH=origin/$CIRRUS_BASE_BRANCH |& ${TIMESTAMP}' + - 'make build-all-new-commits GIT_BASE_BRANCH=origin/$CIRRUS_BASE_BRANCH |& ${TIMESTAMP}' on_failure: failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh' @@ -267,7 +268,7 @@ testing_task: matrix: # Images are generated separately, from build_images_task (below) image_name: "${FEDORA_CACHE_IMAGE_NAME}" - #image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" + image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" image_name: "${UBUNTU_CACHE_IMAGE_NAME}" timeout_in: 120m @@ -383,6 +384,12 @@ test_build_cache_images_task: environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}' + df_script: '${DFCMD}' + journalctl_b_script: 'journalctl -b' + + on_failure: + failed_df_script: '${DFCMD}' + failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' # Test building of new cache-images for future PR testing, in this PR. @@ -401,8 +408,8 @@ verify_test_built_images_task: gce_instance: matrix: # Images are generated separately, from build_images_task (below) - #image_name: "fedora-28${BUILT_IMAGE_SUFFIX}" image_name: "fedora-29${BUILT_IMAGE_SUFFIX}" + image_name: "fedora-30${BUILT_IMAGE_SUFFIX}" image_name: "ubuntu-18${BUILT_IMAGE_SUFFIX}" env: @@ -428,56 +435,6 @@ verify_test_built_images_task: failed_journalctl_b_script: 'journalctl -b || echo "Uh oh, journalctl -b failed"' -# Build new cache-images for future PR testing, but only after a PR merge. -# The cache-images save install/setup time needed test every PR. The 'active' images -# are selected by the 'image_name' items tasks above. Currently this requires -# manually updating the names, but this could be automated (see comment below). -build_cache_images_task: - # Only produce new cache-images after a PR merge, and if a magic string - # is present in the most recent ___commit-message___. - only_if: >- - $CIRRUS_BRANCH == 'master' && - $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*REBUILD\s*IMAGES\s*\*\*\*.*' - - # Require tests to pass first. - depends_on: - - "gating" - - "testing" - - "rootless_testing" - - # VMs created by packer are not cleaned up by cirrus - auto_cancellation: $CI != "true" - - gce_instance: - image_project: "libpod-218412" - zone: "us-central1-a" # Required by Cirrus for the time being - cpu: 4 - memory: "4Gb" - disk: 200 - image_name: "${IMAGE_BUILDER_CACHE_IMAGE_NAME}" - # Additional permissions for building GCE images, within a GCE VM - scopes: - - compute - - devstorage.full_control - - environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' - build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}' - - # TODO,Continuous Delivery: Automatically open a libpod PR after using 'sed' to replace - # the image_names with the new (just build) images. That will - # cause a new round of testing to happen (via the PR) using - # the new images. When all is good, the PR may be manually - # merged so all PR testing uses the new images. The script - # names (below) describe their purpose in this workflow. - # deploy_images_script: - # - clone_podman_release_branch.sh - # - modify_cirrus_yaml_image_names.sh - # - commit_and_create_upstream_pr.sh - - on_failure: - failed_master_script: '$CIRRUS_WORKING_DIR/$SCRIPT_BASE/notice_master_failure.sh |& ${TIMESTAMP}' - - # Post message to IRC if everything passed success_task: @@ -494,7 +451,6 @@ success_task: - "special_testing_in_podman" - "test_build_cache_images" - "verify_test_built_images" - - "build_cache_images" env: CIRRUS_WORKING_DIR: "/usr/src/libpod" diff --git a/Dockerfile.fedora b/Dockerfile.fedora index c34d4bb16..5cbc6e1fe 100644 --- a/Dockerfile.fedora +++ b/Dockerfile.fedora @@ -1,9 +1,8 @@ -FROM registry.fedoraproject.org/fedora:29 +FROM registry.fedoraproject.org/fedora:30 RUN dnf -y install btrfs-progs-devel \ atomic-registries \ bzip2 \ - conmon \ device-mapper-devel \ findutils \ git \ diff --git a/contrib/cirrus/README.md b/contrib/cirrus/README.md index 94494a558..5ff4f290f 100644 --- a/contrib/cirrus/README.md +++ b/contrib/cirrus/README.md @@ -96,10 +96,18 @@ images following the standard naming format; ***however, only runs a limited sub-set of automated tests***. Validating newly built images fully, requires updating ``.cirrus.yml``. -***Manual Steps:*** Assuming `verify_test_built_images` passes, then +***N/B: Steps below are performed by automation*** + +1. Using the just build VM images, launch VMs and wait for them to boot. + +2. Execute the `setup_environment.sh` as in the `testing` task. + +2. Execute the `integration_test.sh` as in the `testing` task. + + +***Manual Steps:*** Assuming the automated steps pass, then you'll find the new image names displayed at the end of the -`test_build_cache_images_task` in the `build_vm_images` output. -For example: +`test_build_cache_images`. For example: ``` @@ -135,18 +143,6 @@ the magic ``***CIRRUS: TEST IMAGES***`` string. Keeping it and and test images again. -### ``build_cache_images`` Task *(Deprecated)* - -Exactly the same as ``test_build_cache_images_task`` task, but only runs on -the master branch. Requires a magic string to be in the `HEAD` -commit message: ``***CIRRUS: BUILD IMAGES***`` - -When successful, the manifest file along with all VM disks, are moved -into a dedicated google storage bucket, separate from the one used by -`test_build_cache_images_task`. These may be used to create new cache-images for -PR testing by manually importing them as described above. - - ### Base-images Base-images are VM disk-images specially prepared for executing as GCE VMs. @@ -158,10 +154,9 @@ as the standard 'cloud-init' services. with services pre-installed, for many platforms. For example, RHEL, CentOS, and Ubuntu. -* Google does ***not*** provide any images for Fedora or Fedora Atomic - Host (as of 11/2018), nor do they provide a base-image prepared to - run packer for creating other images in the ``build_vm_images`` Task - (above). +* Google does ***not*** provide any images for Fedora (as of 5/2019), nor do + they provide a base-image prepared to run packer for creating other images + in the ``test_build_vm_images`` Task (above). * Base images do not need to be produced often, but doing so completely manually would be time-consuming and error-prone. Therefor a special diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh index f9ba010cd..b163834d5 100755 --- a/contrib/cirrus/integration_test.sh +++ b/contrib/cirrus/integration_test.sh @@ -28,8 +28,6 @@ then -e "DIST=$OS_RELEASE_ID" \ -e "CONTAINER_RUNTIME=$CONTAINER_RUNTIME" \ $IN_PODMAN_IMAGE bash $GOSRC/$SCRIPT_BASE/container_test.sh -b -i -t - - exit $? elif [[ "$SPECIALMODE" == "rootless" ]] then req_env_var ROOTLESS_USER @@ -52,5 +50,4 @@ else else make local${TESTSUITE} fi - exit $? fi diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 97901cfc7..98685faa9 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -55,15 +55,15 @@ PACKER_VER="1.3.5" # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) # Base-images rarely change, define them here so they're out of the way. -PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-29,fedora-28}" +PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-30,fedora-29}" # Google-maintained base-image names UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20181203a" # Manually produced base-image names (see $SCRIPT_BASE/README.md) -FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245" -# FEDORA_BASE_IMAGE: "fedora-cloud-base-30-1-2-1556821664" -PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-28-1-1-1544474897" -# PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1541789245" +FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1559164849" +PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849" BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" +# IN_PODMAN container image +IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest" # Safe env. vars. to transfer from root -> $ROOTLESS_USER (go env handled separetly) ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.*FILEPATH)|(SOURCE.*)|(DEPEND.*)|(.+_DEPS_.+)|(OS_REL.*)|(.+_ENV_RE)|(TRAVIS)|(CI.+)|(TEST_REMOTE.*)' @@ -74,9 +74,6 @@ SPECIALMODE="${SPECIALMODE:-none}" TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}" export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} -# IN_PODMAN container image -IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest" - # When running as root, this may be empty or not, as a user, it MUST be set. if [[ "$USER" == "root" ]] then @@ -347,7 +344,7 @@ _finalize(){ set +e # make errors non-fatal echo "Removing leftover giblets from cloud-init" cd / - sudo rm -rf /var/lib/cloud/instance? + sudo rm -rf /var/lib/cloud/instanc* sudo rm -rf /root/.ssh/* sudo rm -rf /home/* sudo rm -rf /tmp/* diff --git a/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service b/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service new file mode 100644 index 000000000..4d2197d87 --- /dev/null +++ b/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service @@ -0,0 +1,20 @@ +[Unit] +Description=Initial cloud-init job (metadata service crawler) +DefaultDependencies=no +Wants=cloud-init-local.service +After=cloud-init-local.service +Wants=google-network-daemon.service +After=google-network-daemon.service +Before=systemd-user-sessions.service + +[Service] +Type=oneshot +ExecStart=/usr/bin/cloud-init init +RemainAfterExit=yes +TimeoutSec=0 + +# Output needs to appear in instance console output +StandardOutput=journal+console + +[Install] +WantedBy=cloud-init.target diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh index 2e6d3eceb..a425b2b57 100644 --- a/contrib/cirrus/packer/fedora_base-setup.sh +++ b/contrib/cirrus/packer/fedora_base-setup.sh @@ -16,11 +16,17 @@ echo "Updating packages" ooe.sh dnf -y update echo "Installing necessary packages and google services" -ooe.sh dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin +ooe.sh dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin ethtool echo "Enabling services" ooe.sh systemctl enable rngd +# There is a race that can happen on boot between the GCE services configuring +# the VM, and cloud-init trying to do similar activities. Use a customized +# unit file to make sure cloud-init starts after the google-compute-* services. +echo "Setting cloud-init service to start after google-network-daemon.service" +cp -v $GOSRC/$PACKER_BASE/cloud-init/fedora/cloud-init.service /etc/systemd/system/ + rh_finalize echo "SUCCESS!" diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml index 560cb321c..e519d2fba 100644 --- a/contrib/cirrus/packer/libpod_base_images.yml +++ b/contrib/cirrus/packer/libpod_base_images.yml @@ -78,7 +78,7 @@ builders: ssh_username: 'root' - <<: *nested_virt - name: 'prior-fedora' + name: 'prior_fedora' iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}' iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}' @@ -121,7 +121,7 @@ provisioners: post-processors: - - type: "compress" - only: ['fedora', 'prior-fedora'] + only: ['fedora', 'prior_fedora'] output: '/tmp/{{build_name}}/disk.raw.tar.gz' format: '.tar.gz' compression_level: 9 @@ -136,7 +136,7 @@ post-processors: image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}' image_family: '{{user `FEDORA_BASE_IMAGE_NAME`}}' - <<: *gcp_import - only: ['prior-fedora'] + only: ['prior_fedora'] image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}' image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}' diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml index 34d4db7fb..c25da25ac 100644 --- a/contrib/cirrus/packer/libpod_images.yml +++ b/contrib/cirrus/packer/libpod_images.yml @@ -55,11 +55,11 @@ builders: # v----- is a YAML alias, allows partial re-use of the anchor object - <<: *gce_hosted_image - name: 'fedora-29' + name: 'fedora-30' source_image: '{{user `FEDORA_BASE_IMAGE`}}' - <<: *gce_hosted_image - name: 'fedora-28' + name: 'fedora-29' source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}' # The brains of the operation, making actual modifications to the base-image. diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 13bce506a..8fdcf5897 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -33,20 +33,8 @@ done # (see docs). case "${OS_REL_VER}" in ubuntu-18) ;; - fedora-29) - # Occasionally, and seemingly only on F29 the root disk fails to expand - # upon boot. When this happens, any number of failures could occur if - # space runs out. Until there is time to investigate the actual cause, - # workaround this problem by detecting it and acting accordingly. - REMAINING=$(df /dev/sda1 | tail -1 | awk '{print $4}') - if [[ "$REMAINING" -lt "100000000" ]] # .cirrus.yml specifies 200gig - then - echo "Fixing failure to expand root filesystem" - growpart /dev/sda 1 # device guaranteed by cloud provider - resize2fs /dev/sda1 # growpart & resuze guaranteed by base-image - fi - ;; - fedora-28) ;; + fedora-30) ;; + fedora-29) ;; centos-7) # Current VM is an image-builder-image no local podman/testing echo "No further setup required for VM image building" exit 0 diff --git a/hack/get_ci_vm.sh b/hack/get_ci_vm.sh index 12dd211f4..90e3aea8e 100755 --- a/hack/get_ci_vm.sh +++ b/hack/get_ci_vm.sh @@ -168,7 +168,7 @@ parse_args(){ if echo "$IMAGE_NAME" | grep -q "image-builder-image" then echo -e "Creating an image-builder VM, I hope you know what you're doing.\n" - IBI_ARGS="--scopes=compute-rw,storage-rw,userinfo-email \"--min-cpu-platform=Intel Haswell\"" + IBI_ARGS="--scopes=compute-rw,storage-rw,userinfo-email" SSHUSER="centos" else unset IBI_ARGS diff --git a/test/install/Dockerfile.Fedora b/test/install/Dockerfile.Fedora index 74cee771d..3e7b6e238 100644 --- a/test/install/Dockerfile.Fedora +++ b/test/install/Dockerfile.Fedora @@ -1,3 +1,3 @@ -FROM registry.fedoraproject.org/fedora:29 +FROM registry.fedoraproject.org/fedora:30 RUN dnf install -y rpms/x86_64/* |