diff options
| -rw-r--r-- | .cirrus.yml | 25 | ||||
| -rwxr-xr-x | contrib/cirrus/build_vm_images.sh | 2 | ||||
| -rwxr-xr-x | contrib/cirrus/check_image.sh | 2 | ||||
| -rw-r--r-- | contrib/cirrus/lib.sh | 43 | ||||
| -rw-r--r-- | contrib/cirrus/packer/Makefile | 32 | ||||
| -rw-r--r-- | contrib/cirrus/packer/fedora_base-setup.sh | 2 | ||||
| -rw-r--r-- | contrib/cirrus/packer/fedora_setup.sh | 8 | ||||
| -rw-r--r-- | contrib/cirrus/packer/libpod_base_images.yml | 51 | ||||
| -rw-r--r-- | contrib/cirrus/packer/libpod_images.yml | 26 | ||||
| l--------- | contrib/cirrus/packer/prior-fedora_base-setup.sh (renamed from contrib/cirrus/packer/prior_fedora_base-setup.sh) | 0 | ||||
| -rwxr-xr-x | contrib/cirrus/packer/systemd_banish.sh | 28 | ||||
| -rw-r--r-- | contrib/cirrus/packer/ubuntu_setup.sh | 39 | ||||
| -rwxr-xr-x | contrib/cirrus/setup_environment.sh | 13 |
13 files changed, 172 insertions, 99 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 77d5ba702..95092a900 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -30,11 +30,12 @@ env: #### #### Cache-image names to test with (double-quotes around names are critical) ### - _BUILT_IMAGE_SUFFIX: "libpod-5751722641719296" + _BUILT_IMAGE_SUFFIX: "libpod-5664838702858240" FEDORA_CACHE_IMAGE_NAME: "fedora-30-${_BUILT_IMAGE_SUFFIX}" PRIOR_FEDORA_CACHE_IMAGE_NAME: "fedora-29-${_BUILT_IMAGE_SUFFIX}" SPECIAL_FEDORA_CACHE_IMAGE_NAME: "xfedora-30-${_BUILT_IMAGE_SUFFIX}" - UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-${_BUILT_IMAGE_SUFFIX}" + UBUNTU_CACHE_IMAGE_NAME: "ubuntu-19-${_BUILT_IMAGE_SUFFIX}" + PRIOR_UBUNTU_CACHE_IMAGE_NAME: "ubuntu-18-${_BUILT_IMAGE_SUFFIX}" #### #### Variables for composing new cache-images (used in PR testing) from @@ -269,6 +270,7 @@ meta_task: ${PRIOR_FEDORA_CACHE_IMAGE_NAME} ${SPECIAL_FEDORA_CACHE_IMAGE_NAME} ${UBUNTU_CACHE_IMAGE_NAME} + ${PRIOR_UBUNTU_CACHE_IMAGE_NAME} ${IMAGE_BUILDER_CACHE_IMAGE_NAME} BUILDID: "${CIRRUS_BUILD_ID}" REPOREF: "${CIRRUS_CHANGE_IN_REPO}" @@ -325,7 +327,9 @@ testing_task: # Images are generated separately, from build_images_task (below) image_name: "${FEDORA_CACHE_IMAGE_NAME}" image_name: "${PRIOR_FEDORA_CACHE_IMAGE_NAME}" - image_name: "${UBUNTU_CACHE_IMAGE_NAME}" + # Multiple test failures on Ubuntu 19 - Fixes TBD in future PR + # TODO: image_name: "${UBUNTU_CACHE_IMAGE_NAME}" + image_name: "${PRIOR_UBUNTU_CACHE_IMAGE_NAME}" timeout_in: 120m @@ -530,6 +534,8 @@ test_building_snap_task: depends_on: - "gating" + only_if: $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' + container: image: yakshaveinc/snapcraft:core18 snapcraft_script: @@ -542,8 +548,7 @@ test_build_cache_images_task: only_if: >- $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' && - $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' + $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' depends_on: - "gating" @@ -563,10 +568,7 @@ test_build_cache_images_task: - devstorage.full_control networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' - setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' build_vm_images_script: '$SCRIPT_BASE/build_vm_images.sh |& ${TIMESTAMP}' - df_script: '${DFCMD}' - journalctl_b_script: 'journalctl -b' on_failure: failed_df_script: '${DFCMD}' @@ -578,8 +580,7 @@ verify_test_built_images_task: only_if: >- $CIRRUS_BRANCH != $DEST_BRANCH && - $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' && - $CIRRUS_CHANGE_MESSAGE !=~ '.*\*\*\*\s*CIRRUS:\s*SYSTEM\s*TEST\s*\*\*\*.*' + $CIRRUS_CHANGE_MESSAGE =~ '.*\*\*\*\s*CIRRUS:\s*TEST\s*IMAGES\s*\*\*\*.*' depends_on: @@ -601,6 +602,7 @@ verify_test_built_images_task: PACKER_BUILDER_NAME: "fedora-30" PACKER_BUILDER_NAME: "xfedora-30" PACKER_BUILDER_NAME: "ubuntu-18" + # TODO support $UBUNTU_CACHE_IMAGE_NAME: PACKER_BUILDER_NAME: "ubuntu-19" networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh' environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}' @@ -616,7 +618,7 @@ verify_test_built_images_task: $SCRIPT_BASE/integration_test.sh |& ${TIMESTAMP} build_release_script: >- [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \ - '$SCRIPT_BASE/build_release.sh |& ${TIMESTAMP}' + $SCRIPT_BASE/build_release.sh |& ${TIMESTAMP} system_test_script: >- [[ "$PACKER_BUILDER_NAME" == "xfedora-30" ]] || \ $SCRIPT_BASE/system_test.sh |& ${TIMESTAMP} @@ -624,6 +626,7 @@ verify_test_built_images_task: always: <<: *standardlogs + # Post message to IRC if everything passed PR testing success_task: diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index 6230610cb..543f83a14 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -3,7 +3,7 @@ set -e source $(dirname $0)/lib.sh -BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE' +BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE PRIOR_UBUNTU_BASE_IMAGE' ENV_VARS="PACKER_BUILDS BUILT_IMAGE_SUFFIX $BASE_IMAGE_VARS SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO" req_env_var $ENV_VARS # Must also be made available through make, into packer process diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh index ad9a12f49..39f49d0a1 100755 --- a/contrib/cirrus/check_image.sh +++ b/contrib/cirrus/check_image.sh @@ -4,6 +4,8 @@ set -eo pipefail source $(dirname $0)/lib.sh +EVIL_UNITS="$($CIRRUS_WORKING_DIR/$PACKER_BASE/systemd_banish.sh --list)" + req_env_var PACKER_BUILDER_NAME TEST_REMOTE_CLIENT EVIL_UNITS OS_RELEASE_ID NFAILS=0 diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index f26eec87f..f81a8d501 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -36,7 +36,7 @@ SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_c AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv" cd $GOSRC -if type -P git &> /dev/null +if type -P git &> /dev/null && [[ -d "$GOSRC/.git" ]] then CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)} else # pick something unique and obviously not from Cirrus @@ -52,16 +52,17 @@ CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod} CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique # Vars. for image-building -PACKER_VER="1.3.5" +PACKER_VER="1.4.2" # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) # Base-images rarely change, define them here so they're out of the way. -export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-30,xfedora-30,fedora-29}" +export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-30,xfedora-30,fedora-29}" # Google-maintained base-image names -export UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a" +export UBUNTU_BASE_IMAGE="ubuntu-1904-disco-v20190724" +export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a" # Manually produced base-image names (see $SCRIPT_BASE/README.md) -export FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1559164849" -export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849" +export FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1565360543" +export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1565360543" export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}" # IN_PODMAN container image IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest" @@ -81,9 +82,6 @@ ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(. # Unsafe env. vars for display SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(GC[EP]..+)|(SSH)' -# Names of systemd units which should never be running -EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean" - SPECIALMODE="${SPECIALMODE:-none}" TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}" export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman} @@ -322,7 +320,7 @@ EOF sudo chmod 755 /usr/bin/git } -install_test_configs(){ +install_test_configs() { echo "Installing cni config, policy and registry config" req_env_var GOSRC SCRIPT_BASE cd $GOSRC @@ -342,7 +340,7 @@ install_test_configs(){ # of pulling in necessary prerequisites packages as the set can change over time. # For general CI testing however, calling this function makes sure the system # can only run the compiled source version. -remove_packaged_podman_files(){ +remove_packaged_podman_files() { echo "Removing packaged podman files to prevent conflicts with source build and testing." req_env_var OS_RELEASE_ID @@ -379,24 +377,11 @@ remove_packaged_podman_files(){ sync && echo 3 > /proc/sys/vm/drop_caches } -systemd_banish(){ - echo "Disabling periodic services that could destabilize testing (ignoring errors):" - set +e # Not all of these exist on every platform - for unit in $EVIL_UNITS - do - echo "Banishing $unit (ignoring errors)" - ( - sudo systemctl stop $unit - sudo systemctl disable $unit - sudo systemctl disable $unit.timer - sudo systemctl mask $unit - sudo systemctl mask $unit.timer - ) &> /dev/null - done - set -e +systemd_banish() { + $GOSRC/$PACKER_BASE/systemd_banish.sh } -_finalize(){ +_finalize() { set +e # Don't fail at the very end if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]] then @@ -419,7 +404,7 @@ _finalize(){ sudo fstrim -av } -rh_finalize(){ +rh_finalize() { set +e # Don't fail at the very end echo "Resetting to fresh-state for usage as cloud-image." PKG=$(type -P dnf || type -P yum || echo "") @@ -430,7 +415,7 @@ rh_finalize(){ _finalize } -ubuntu_finalize(){ +ubuntu_finalize() { set +e # Don't fail at the very end echo "Resetting to fresh-state for usage as cloud-image." $LILTO $SUDOAPTGET autoremove diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile index d03d22abe..947a2a1e9 100644 --- a/contrib/cirrus/packer/Makefile +++ b/contrib/cirrus/packer/Makefile @@ -3,7 +3,7 @@ # builder name(s) from applicable YAML file, # e.g for names see libpod_images.yml -PACKER_VER ?= 1.3.5 +PACKER_VER ?= 1.4.2 GOARCH=$(shell go env GOARCH) ARCH=$(uname -m) PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip @@ -15,6 +15,9 @@ PACKER_BASE ?= contrib/cirrus/packer SCRIPT_BASE ?= contrib/cirrus POST_MERGE_BUCKET_SUFFIX ?= +UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$UBUNTU_BASE_IMAGE") +PRIOR_UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$PRIOR_UBUNTU_BASE_IMAGE") + # For debugging nested-virt, use #TTYDEV := $(shell tty) TTYDEV := /dev/null @@ -22,6 +25,14 @@ TTYDEV := /dev/null .PHONY: all all: libpod_images +# Utility target for checking required parameters +.PHONY: guard-% +guard-%: + @if [[ -z "$($*)" ]]; then \ + echo "Missing or empty required make variable '$*'."; \ + exit 1; \ + fi; + %.json: %.yml @python3 -c 'import json,yaml; json.dump( yaml.load(open("$<").read()), open("$@","w"), indent=2);' @@ -44,10 +55,7 @@ test: libpod_base_images.json libpod_images.json packer @echo "All good" .PHONY: libpod_images -libpod_images: libpod_images.json packer -ifndef PACKER_BUILDS - $(error PACKER_BUILDS is undefined, expected builder-names CSV) -endif +libpod_images: guard-PACKER_BUILDS libpod_images.json packer ./packer build -only=${PACKER_BUILDS} \ -force \ -var GOSRC=$(GOSRC) \ @@ -72,16 +80,7 @@ cidata.iso: user-data meta-data # This is intended to be run by a human, with admin access to the libpod GCE project. .PHONY: libpod_base_images -libpod_base_images: libpod_base_images.json cidata.iso cidata.ssh packer -ifndef GCP_PROJECT_ID - $(error GCP_PROJECT_ID is undefined, expected complete GCP project ID string e.g. foobar-12345) -endif -ifndef GOOGLE_APPLICATION_CREDENTIALS - $(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json) -endif -ifndef PACKER_BUILDS - $(error PACKER_BUILDS is undefined, expected builder-names CSV) -endif +libpod_base_images: guard-GCP_PROJECT_ID guard-GOOGLE_APPLICATION_CREDENTIALS libpod_base_images.json cidata.iso cidata.ssh packer PACKER_CACHE_DIR=/tmp ./packer build \ -force \ -var TIMESTAMP=$(TIMESTAMP) \ @@ -91,5 +90,6 @@ endif -var GOSRC=$(GOSRC) \ -var PACKER_BASE=$(PACKER_BASE) \ -var SCRIPT_BASE=$(SCRIPT_BASE) \ - -only $(PACKER_BUILDS) \ + -var UBUNTU_BASE_IMAGE=$(UBUNTU_BASE_IMAGE) \ + -var PRIOR_UBUNTU_BASE_IMAGE=$(PRIOR_UBUNTU_BASE_IMAGE) \ libpod_base_images.json diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh index 788a54c34..29c23117f 100644 --- a/contrib/cirrus/packer/fedora_base-setup.sh +++ b/contrib/cirrus/packer/fedora_base-setup.sh @@ -8,8 +8,6 @@ set -e # Load in library (copied by packer, before this script was run) source $GOSRC/$SCRIPT_BASE/lib.sh -[[ "$1" == "post" ]] || exit 0 # nothing to do - install_ooe echo "Updating packages" diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 0e1a82cc0..6709eecaf 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -24,6 +24,8 @@ ooe.sh sudo dnf config-manager --set-enabled updates-testing echo "Installing general build/test dependencies" ooe.sh sudo dnf install -y \ atomic-registries \ + autoconf \ + automake \ bats \ bridge-utils \ btrfs-progs-devel \ @@ -35,10 +37,14 @@ ooe.sh sudo dnf install -y \ device-mapper-devel \ emacs-nox \ findutils \ + fuse3 \ + fuse3-devel \ + gcc \ git \ glib2-devel \ glibc-static \ gnupg \ + go-md2man \ golang \ golang-github-cpuguy83-go-md2man \ gpgme-devel \ @@ -81,6 +87,7 @@ ooe.sh sudo dnf install -y \ xz \ zip + # Ensure there are no disruptive periodic services enabled by default in image systemd_banish @@ -91,6 +98,7 @@ case "$PACKER_BUILDER_NAME" in xfedora*) echo "Configuring CGroups v2 enabled on next boot" sudo grubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=1" + sudo dnf install -y crun ;& # continue to next matching item *) echo "Finalizing $PACKER_BUILDER_NAME VM image" diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml index e519d2fba..bcca440ae 100644 --- a/contrib/cirrus/packer/libpod_base_images.yml +++ b/contrib/cirrus/packer/libpod_base_images.yml @@ -12,6 +12,10 @@ variables: # Required for output from qemu builders TTYDEV: + # Ubuntu releases are mearly copied to this project for control purposes + UBUNTU_BASE_IMAGE: + PRIOR_UBUNTU_BASE_IMAGE: + # Latest Fedora release FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2" FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM" @@ -78,50 +82,67 @@ builders: ssh_username: 'root' - <<: *nested_virt - name: 'prior_fedora' + name: 'prior-fedora' iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}' iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}' + - &imgcopy + name: 'ubuntu' + type: 'googlecompute' + image_name: '{{user `UBUNTU_BASE_IMAGE`}}' + image_family: '{{build_name}}-base' + source_image: '{{user `UBUNTU_BASE_IMAGE`}}' + source_image_project_id: 'ubuntu-os-cloud' + project_id: '{{user `GCP_PROJECT_ID`}}' + account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' + startup_script_file: "systemd_banish.sh" + zone: 'us-central1-a' + disk_size: 20 + communicator: 'none' + + - <<: *imgcopy + name: 'prior-ubuntu' + image_name: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' + source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' + provisioners: - type: 'shell' + only: ['fedora', 'prior-fedora'] inline: - 'mkdir -p /tmp/libpod/{{user `SCRIPT_BASE`}}' - 'mkdir -p /tmp/libpod/{{user `PACKER_BASE`}}' - type: 'file' + only: ['fedora', 'prior-fedora'] source: '{{user `GOSRC`}}/.cirrus.yml' destination: '/tmp/libpod/.cirrus.yml' - type: 'file' + only: ['fedora', 'prior-fedora'] source: '{{user `GOSRC`}}/{{user `SCRIPT_BASE`}}/' destination: '/tmp/libpod/{{user `SCRIPT_BASE`}}/' - type: 'file' + only: ['fedora', 'prior-fedora'] source: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/' destination: '/tmp/libpod/{{user `PACKER_BASE`}}/' - &shell_script + only: ['fedora', 'prior-fedora'] type: 'shell' inline: - - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh' - - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh pre' - expect_disconnect: true # Allow this to reboot the VM + - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/*.sh' + - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh' + expect_disconnect: true # Allow this to reboot the VM if needed environment_vars: - 'TIMESTAMP={{user `TIMESTAMP`}}' - 'GOSRC=/tmp/libpod' - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' - 'PACKER_BASE={{user `PACKER_BASE`}}' - - <<: *shell_script - inline: ['{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'] - expect_disconnect: false - pause_before: '10s' - inline: - - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh post' - post-processors: - - type: "compress" - only: ['fedora', 'prior_fedora'] + only: ['fedora', 'prior-fedora'] output: '/tmp/{{build_name}}/disk.raw.tar.gz' format: '.tar.gz' compression_level: 9 @@ -134,10 +155,10 @@ post-processors: gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}.tar.gz' image_name: "{{user `FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}' - image_family: '{{user `FEDORA_BASE_IMAGE_NAME`}}' + image_family: '{{build_name}}-base' - <<: *gcp_import - only: ['prior_fedora'] + only: ['prior-fedora'] image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}' - image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}' + image_family: '{{build_name}}-base' - type: 'manifest' diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml index cae5d4138..01a65d867 100644 --- a/contrib/cirrus/packer/libpod_images.yml +++ b/contrib/cirrus/packer/libpod_images.yml @@ -2,16 +2,17 @@ # All of these are required variables: - # Names of GCE Base images to start from, in .cirrus.yml - UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}' - FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}' - PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}' - BUILT_IMAGE_SUFFIX: '{{env `BUILT_IMAGE_SUFFIX`}}' GOSRC: '{{env `GOSRC`}}' PACKER_BASE: '{{env `PACKER_BASE`}}' SCRIPT_BASE: '{{env `SCRIPT_BASE`}}' + # Base-image names are required. Using image family-names breaks parallelism + UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}' + PRIOR_UBUNTU_BASE_IMAGE: '{{env `PRIOR_UBUNTU_BASE_IMAGE`}}' + FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}' + PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}' + # Protected credentials, decrypted by Cirrus at runtime GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}' GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}' @@ -28,11 +29,12 @@ sensitive-variables: builders: # v----- is a YAML anchor, allows referencing this object by name (below) - &gce_hosted_image - name: 'ubuntu-18' + name: 'ubuntu-19' type: 'googlecompute' image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}' - image_family: '{{build_name}}-libpod' - source_image: '{{user `UBUNTU_BASE_IMAGE`}}' + image_family: '{{build_name}}-cache' + source_image: '{{user `UBUNTU_BASE_IMAGE`}}' # precedence over family + source_image_family: 'ubuntu-base' # for ref. only disk_size: 20 # REQUIRED: Runtime allocation > this value project_id: '{{user `GCP_PROJECT_ID`}}' service_account_email: '{{user `SERVICE_ACCOUNT`}}' @@ -44,16 +46,24 @@ builders: # v----- is a YAML alias, allows partial re-use of the anchor object - <<: *gce_hosted_image + name: 'ubuntu-18' + source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' + source_image_family: 'prior-ubuntu-base' + + - <<: *gce_hosted_image name: 'fedora-30' source_image: '{{user `FEDORA_BASE_IMAGE`}}' + source_image_family: 'fedora-base' - <<: *gce_hosted_image name: 'xfedora-30' source_image: '{{user `FEDORA_BASE_IMAGE`}}' + source_image_family: 'fedora-base' - <<: *gce_hosted_image name: 'fedora-29' source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}' + source_image_family: 'prior-fedora-base' # The brains of the operation, making actual modifications to the base-image. provisioners: diff --git a/contrib/cirrus/packer/prior_fedora_base-setup.sh b/contrib/cirrus/packer/prior-fedora_base-setup.sh index 998a5d9fd..998a5d9fd 120000 --- a/contrib/cirrus/packer/prior_fedora_base-setup.sh +++ b/contrib/cirrus/packer/prior-fedora_base-setup.sh diff --git a/contrib/cirrus/packer/systemd_banish.sh b/contrib/cirrus/packer/systemd_banish.sh new file mode 100755 index 000000000..396cf906c --- /dev/null +++ b/contrib/cirrus/packer/systemd_banish.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set +e # Not all of these exist on every platform + +# This is intended to be executed on VMs as a startup script on initial-boot. +# Alternativly, it may be executed with the '--list' option to return the list +# of systemd units defined for disablement (useful for testing). + +EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean" + +if [[ "$1" == "--list" ]] +then + echo "$EVIL_UNITS" + exit 0 +fi + +echo "Disabling periodic services that could destabilize testing:" +for unit in $EVIL_UNITS +do + echo "Banishing $unit (ignoring errors)" + ( + sudo systemctl stop $unit + sudo systemctl disable $unit + sudo systemctl disable $unit.timer + sudo systemctl mask $unit + sudo systemctl mask $unit.timer + ) &> /dev/null +done diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index 00d92570f..da7d457a5 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -20,18 +20,25 @@ systemd_banish echo "Updating/configuring package repositories." $LILTO $SUDOAPTGET update + +echo "Upgrading all packages" +$BIGTO $SUDOAPTGET upgrade + +echo "Adding PPAs" $LILTO $SUDOAPTGET install software-properties-common -$LILTO $SUDOAPTADD ppa:longsleep/golang-backports $LILTO $SUDOAPTADD ppa:projectatomic/ppa $LILTO $SUDOAPTADD ppa:criu/ppa +if [[ "$OS_RELEASE_VER" -eq "18" ]] +then + $LILTO $SUDOAPTADD ppa:longsleep/golang-backports +fi -echo "Upgrading all packages" $LILTO $SUDOAPTGET update -$BIGTO $SUDOAPTGET upgrade echo "Installing general testing and system dependencies" $BIGTO $SUDOAPTGET install \ apparmor \ + aufs-tools \ autoconf \ automake \ bats \ @@ -46,6 +53,7 @@ $BIGTO $SUDOAPTGET install \ e2fslibs-dev \ emacs-nox \ gawk \ + gcc \ gettext \ go-md2man \ golang \ @@ -58,6 +66,7 @@ $BIGTO $SUDOAPTGET install \ libdevmapper-dev \ libdevmapper1.02.1 \ libfuse-dev \ + libfuse2 \ libglib2.0-dev \ libgpgme11-dev \ liblzma-dev \ @@ -66,7 +75,7 @@ $BIGTO $SUDOAPTGET install \ libnl-3-dev \ libostree-dev \ libvarlink \ - libprotobuf-c0-dev \ + libprotobuf-c-dev \ libprotobuf-dev \ libseccomp-dev \ libseccomp2 \ @@ -74,6 +83,7 @@ $BIGTO $SUDOAPTGET install \ libtool \ libudev-dev \ lsof \ + make \ netcat \ pkg-config \ podman \ @@ -87,19 +97,28 @@ $BIGTO $SUDOAPTGET install \ python3-psutil \ python3-pytoml \ python3-setuptools \ - slirp4netns \ skopeo \ + slirp4netns \ socat \ unzip \ vim \ xz-utils \ zip -echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting." -SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g' -ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* -ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub -ooe.sh sudo update-grub +if [[ "$OS_RELEASE_VER" -ge "19" ]] +then + echo "Installing Ubuntu > 18 packages" + $LILTO $SUDOAPTGET install fuse3 libfuse3-dev libbtrfs-dev +fi + +if [[ "$OS_RELEASE_VER" -eq "18" ]] +then + echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting." + SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g' + ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* + ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub + ooe.sh sudo update-grub +fi sudo /tmp/libpod/hack/install_catatonit.sh ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index c1a01d6ab..df510deef 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -33,8 +33,8 @@ done # contrib/cirrus/packer/*_setup.sh to be incorporated into VM cache-images # (see docs). cd "${GOSRC}/" -case "${OS_REL_VER}" in - ubuntu-18) +case "${OS_RELEASE_ID}" in + ubuntu) CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc" if dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH" then @@ -42,14 +42,13 @@ case "${OS_REL_VER}" in ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc" fi ;; - fedora-30) ;& # continue to next item - fedora-29) - # All SELinux distros need this for systemd-in-a-container - setsebool container_manage_cgroup true + fedora) + # All SELinux distros need this for systemd-in-a-container + setsebool container_manage_cgroup true if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then bash "$SCRIPT_BASE/add_second_partition.sh"; fi ;; - centos-7) # Current VM is an image-builder-image no local podman/testing + centos) # Current VM is an image-builder-image no local podman/testing echo "No further setup required for VM image building" exit 0 ;; |