diff options
Diffstat (limited to 'contrib')
| -rwxr-xr-x | contrib/cirrus/build_vm_images.sh | 2 | ||||
| -rw-r--r-- | contrib/cirrus/lib.sh | 7 | ||||
| -rw-r--r-- | contrib/cirrus/packer/Makefile | 32 | ||||
| -rw-r--r-- | contrib/cirrus/packer/fedora_setup.sh | 8 | ||||
| -rw-r--r-- | contrib/cirrus/packer/libpod_base_images.yml | 37 | ||||
| -rw-r--r-- | contrib/cirrus/packer/libpod_images.yml | 21 | ||||
| l--------- | contrib/cirrus/packer/prior-fedora_base-setup.sh (renamed from contrib/cirrus/packer/prior_fedora_base-setup.sh) | 0 | ||||
| -rw-r--r-- | contrib/cirrus/packer/ubuntu_setup.sh | 39 | ||||
| -rwxr-xr-x | contrib/cirrus/setup_environment.sh | 13 |
9 files changed, 114 insertions, 45 deletions
diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh index 6230610cb..543f83a14 100755 --- a/contrib/cirrus/build_vm_images.sh +++ b/contrib/cirrus/build_vm_images.sh @@ -3,7 +3,7 @@ set -e source $(dirname $0)/lib.sh -BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE' +BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE PRIOR_UBUNTU_BASE_IMAGE' ENV_VARS="PACKER_BUILDS BUILT_IMAGE_SUFFIX $BASE_IMAGE_VARS SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO" req_env_var $ENV_VARS # Must also be made available through make, into packer process diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index f26eec87f..ec2cc683b 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -52,13 +52,14 @@ CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod} CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique # Vars. for image-building -PACKER_VER="1.3.5" +PACKER_VER="1.4.2" # CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json) # Base-images rarely change, define them here so they're out of the way. -export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-30,xfedora-30,fedora-29}" +export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-30,xfedora-30,fedora-29}" # Google-maintained base-image names -export UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a" +export UBUNTU_BASE_IMAGE="ubuntu-1904-disco-v20190724" +export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a" # Manually produced base-image names (see $SCRIPT_BASE/README.md) export FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1559164849" export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849" diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile index d03d22abe..947a2a1e9 100644 --- a/contrib/cirrus/packer/Makefile +++ b/contrib/cirrus/packer/Makefile @@ -3,7 +3,7 @@ # builder name(s) from applicable YAML file, # e.g for names see libpod_images.yml -PACKER_VER ?= 1.3.5 +PACKER_VER ?= 1.4.2 GOARCH=$(shell go env GOARCH) ARCH=$(uname -m) PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip @@ -15,6 +15,9 @@ PACKER_BASE ?= contrib/cirrus/packer SCRIPT_BASE ?= contrib/cirrus POST_MERGE_BUCKET_SUFFIX ?= +UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$UBUNTU_BASE_IMAGE") +PRIOR_UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$PRIOR_UBUNTU_BASE_IMAGE") + # For debugging nested-virt, use #TTYDEV := $(shell tty) TTYDEV := /dev/null @@ -22,6 +25,14 @@ TTYDEV := /dev/null .PHONY: all all: libpod_images +# Utility target for checking required parameters +.PHONY: guard-% +guard-%: + @if [[ -z "$($*)" ]]; then \ + echo "Missing or empty required make variable '$*'."; \ + exit 1; \ + fi; + %.json: %.yml @python3 -c 'import json,yaml; json.dump( yaml.load(open("$<").read()), open("$@","w"), indent=2);' @@ -44,10 +55,7 @@ test: libpod_base_images.json libpod_images.json packer @echo "All good" .PHONY: libpod_images -libpod_images: libpod_images.json packer -ifndef PACKER_BUILDS - $(error PACKER_BUILDS is undefined, expected builder-names CSV) -endif +libpod_images: guard-PACKER_BUILDS libpod_images.json packer ./packer build -only=${PACKER_BUILDS} \ -force \ -var GOSRC=$(GOSRC) \ @@ -72,16 +80,7 @@ cidata.iso: user-data meta-data # This is intended to be run by a human, with admin access to the libpod GCE project. .PHONY: libpod_base_images -libpod_base_images: libpod_base_images.json cidata.iso cidata.ssh packer -ifndef GCP_PROJECT_ID - $(error GCP_PROJECT_ID is undefined, expected complete GCP project ID string e.g. foobar-12345) -endif -ifndef GOOGLE_APPLICATION_CREDENTIALS - $(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json) -endif -ifndef PACKER_BUILDS - $(error PACKER_BUILDS is undefined, expected builder-names CSV) -endif +libpod_base_images: guard-GCP_PROJECT_ID guard-GOOGLE_APPLICATION_CREDENTIALS libpod_base_images.json cidata.iso cidata.ssh packer PACKER_CACHE_DIR=/tmp ./packer build \ -force \ -var TIMESTAMP=$(TIMESTAMP) \ @@ -91,5 +90,6 @@ endif -var GOSRC=$(GOSRC) \ -var PACKER_BASE=$(PACKER_BASE) \ -var SCRIPT_BASE=$(SCRIPT_BASE) \ - -only $(PACKER_BUILDS) \ + -var UBUNTU_BASE_IMAGE=$(UBUNTU_BASE_IMAGE) \ + -var PRIOR_UBUNTU_BASE_IMAGE=$(PRIOR_UBUNTU_BASE_IMAGE) \ libpod_base_images.json diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh index 0e1a82cc0..6709eecaf 100644 --- a/contrib/cirrus/packer/fedora_setup.sh +++ b/contrib/cirrus/packer/fedora_setup.sh @@ -24,6 +24,8 @@ ooe.sh sudo dnf config-manager --set-enabled updates-testing echo "Installing general build/test dependencies" ooe.sh sudo dnf install -y \ atomic-registries \ + autoconf \ + automake \ bats \ bridge-utils \ btrfs-progs-devel \ @@ -35,10 +37,14 @@ ooe.sh sudo dnf install -y \ device-mapper-devel \ emacs-nox \ findutils \ + fuse3 \ + fuse3-devel \ + gcc \ git \ glib2-devel \ glibc-static \ gnupg \ + go-md2man \ golang \ golang-github-cpuguy83-go-md2man \ gpgme-devel \ @@ -81,6 +87,7 @@ ooe.sh sudo dnf install -y \ xz \ zip + # Ensure there are no disruptive periodic services enabled by default in image systemd_banish @@ -91,6 +98,7 @@ case "$PACKER_BUILDER_NAME" in xfedora*) echo "Configuring CGroups v2 enabled on next boot" sudo grubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=1" + sudo dnf install -y crun ;& # continue to next matching item *) echo "Finalizing $PACKER_BUILDER_NAME VM image" diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml index e519d2fba..722822837 100644 --- a/contrib/cirrus/packer/libpod_base_images.yml +++ b/contrib/cirrus/packer/libpod_base_images.yml @@ -12,6 +12,10 @@ variables: # Required for output from qemu builders TTYDEV: + # Ubuntu releases are mearly copied to this project for control purposes + UBUNTU_BASE_IMAGE: + PRIOR_UBUNTU_BASE_IMAGE: + # Latest Fedora release FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2" FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM" @@ -78,29 +82,52 @@ builders: ssh_username: 'root' - <<: *nested_virt - name: 'prior_fedora' + name: 'prior-fedora' iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}' iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}' + - &imgcopy + name: 'ubuntu' + type: 'googlecompute' + image_name: '{{user `UBUNTU_BASE_IMAGE`}}' + image_family: '{{build_name}}-base' + source_image: '{{user `UBUNTU_BASE_IMAGE`}}' + source_image_project_id: 'ubuntu-os-cloud' + project_id: '{{user `GCP_PROJECT_ID`}}' + account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' + zone: 'us-central1-a' + disk_size: 20 + communicator: 'none' + + - <<: *imgcopy + name: 'prior-ubuntu' + image_name: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' + source_image: '{{user `UBUNTU_BASE_IMAGE`}}' + provisioners: - type: 'shell' + only: ['fedora', 'prior-fedora'] inline: - 'mkdir -p /tmp/libpod/{{user `SCRIPT_BASE`}}' - 'mkdir -p /tmp/libpod/{{user `PACKER_BASE`}}' - type: 'file' + only: ['fedora', 'prior-fedora'] source: '{{user `GOSRC`}}/.cirrus.yml' destination: '/tmp/libpod/.cirrus.yml' - type: 'file' + only: ['fedora', 'prior-fedora'] source: '{{user `GOSRC`}}/{{user `SCRIPT_BASE`}}/' destination: '/tmp/libpod/{{user `SCRIPT_BASE`}}/' - type: 'file' + only: ['fedora', 'prior-fedora'] source: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/' destination: '/tmp/libpod/{{user `PACKER_BASE`}}/' - &shell_script + only: ['fedora', 'prior-fedora'] type: 'shell' inline: - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh' @@ -121,7 +148,7 @@ provisioners: post-processors: - - type: "compress" - only: ['fedora', 'prior_fedora'] + only: ['fedora', 'prior-fedora'] output: '/tmp/{{build_name}}/disk.raw.tar.gz' format: '.tar.gz' compression_level: 9 @@ -134,10 +161,10 @@ post-processors: gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}.tar.gz' image_name: "{{user `FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}' - image_family: '{{user `FEDORA_BASE_IMAGE_NAME`}}' + image_family: '{{build_name}}-base' - <<: *gcp_import - only: ['prior_fedora'] + only: ['prior-fedora'] image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}' - image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}' + image_family: '{{build_name}}-base' - type: 'manifest' diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml index cae5d4138..42ccba186 100644 --- a/contrib/cirrus/packer/libpod_images.yml +++ b/contrib/cirrus/packer/libpod_images.yml @@ -12,6 +12,12 @@ variables: PACKER_BASE: '{{env `PACKER_BASE`}}' SCRIPT_BASE: '{{env `SCRIPT_BASE`}}' + # Base-image names are required. Using image family-names breaks parallelism + UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}' + PRIOR_UBUNTU_BASE_IMAGE: '{{env `PRIOR_UBUNTU_BASE_IMAGE`}}' + FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}' + PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}' + # Protected credentials, decrypted by Cirrus at runtime GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}' GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}' @@ -28,11 +34,12 @@ sensitive-variables: builders: # v----- is a YAML anchor, allows referencing this object by name (below) - &gce_hosted_image - name: 'ubuntu-18' + name: 'ubuntu-19' type: 'googlecompute' image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}' - image_family: '{{build_name}}-libpod' - source_image: '{{user `UBUNTU_BASE_IMAGE`}}' + image_family: '{{build_name}}-cache' + source_image: '{{user `UBUNTU_BASE_IMAGE`}}' # precedence over family + source_image_family: 'ubuntu-base' # for ref. only disk_size: 20 # REQUIRED: Runtime allocation > this value project_id: '{{user `GCP_PROJECT_ID`}}' service_account_email: '{{user `SERVICE_ACCOUNT`}}' @@ -44,16 +51,24 @@ builders: # v----- is a YAML alias, allows partial re-use of the anchor object - <<: *gce_hosted_image + name: 'ubuntu-18' + source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' + source_image_family: 'prior-ubuntu-base' + + - <<: *gce_hosted_image name: 'fedora-30' source_image: '{{user `FEDORA_BASE_IMAGE`}}' + source_image_family: 'fedora-base' - <<: *gce_hosted_image name: 'xfedora-30' source_image: '{{user `FEDORA_BASE_IMAGE`}}' + source_image_family: 'fedora-base' - <<: *gce_hosted_image name: 'fedora-29' source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}' + source_image_family: 'prior-fedora-base' # The brains of the operation, making actual modifications to the base-image. provisioners: diff --git a/contrib/cirrus/packer/prior_fedora_base-setup.sh b/contrib/cirrus/packer/prior-fedora_base-setup.sh index 998a5d9fd..998a5d9fd 120000 --- a/contrib/cirrus/packer/prior_fedora_base-setup.sh +++ b/contrib/cirrus/packer/prior-fedora_base-setup.sh diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh index 00d92570f..da7d457a5 100644 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ b/contrib/cirrus/packer/ubuntu_setup.sh @@ -20,18 +20,25 @@ systemd_banish echo "Updating/configuring package repositories." $LILTO $SUDOAPTGET update + +echo "Upgrading all packages" +$BIGTO $SUDOAPTGET upgrade + +echo "Adding PPAs" $LILTO $SUDOAPTGET install software-properties-common -$LILTO $SUDOAPTADD ppa:longsleep/golang-backports $LILTO $SUDOAPTADD ppa:projectatomic/ppa $LILTO $SUDOAPTADD ppa:criu/ppa +if [[ "$OS_RELEASE_VER" -eq "18" ]] +then + $LILTO $SUDOAPTADD ppa:longsleep/golang-backports +fi -echo "Upgrading all packages" $LILTO $SUDOAPTGET update -$BIGTO $SUDOAPTGET upgrade echo "Installing general testing and system dependencies" $BIGTO $SUDOAPTGET install \ apparmor \ + aufs-tools \ autoconf \ automake \ bats \ @@ -46,6 +53,7 @@ $BIGTO $SUDOAPTGET install \ e2fslibs-dev \ emacs-nox \ gawk \ + gcc \ gettext \ go-md2man \ golang \ @@ -58,6 +66,7 @@ $BIGTO $SUDOAPTGET install \ libdevmapper-dev \ libdevmapper1.02.1 \ libfuse-dev \ + libfuse2 \ libglib2.0-dev \ libgpgme11-dev \ liblzma-dev \ @@ -66,7 +75,7 @@ $BIGTO $SUDOAPTGET install \ libnl-3-dev \ libostree-dev \ libvarlink \ - libprotobuf-c0-dev \ + libprotobuf-c-dev \ libprotobuf-dev \ libseccomp-dev \ libseccomp2 \ @@ -74,6 +83,7 @@ $BIGTO $SUDOAPTGET install \ libtool \ libudev-dev \ lsof \ + make \ netcat \ pkg-config \ podman \ @@ -87,19 +97,28 @@ $BIGTO $SUDOAPTGET install \ python3-psutil \ python3-pytoml \ python3-setuptools \ - slirp4netns \ skopeo \ + slirp4netns \ socat \ unzip \ vim \ xz-utils \ zip -echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting." -SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g' -ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* -ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub -ooe.sh sudo update-grub +if [[ "$OS_RELEASE_VER" -ge "19" ]] +then + echo "Installing Ubuntu > 18 packages" + $LILTO $SUDOAPTGET install fuse3 libfuse3-dev libbtrfs-dev +fi + +if [[ "$OS_RELEASE_VER" -eq "18" ]] +then + echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting." + SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g' + ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* + ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub + ooe.sh sudo update-grub +fi sudo /tmp/libpod/hack/install_catatonit.sh ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index c1a01d6ab..df510deef 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -33,8 +33,8 @@ done # contrib/cirrus/packer/*_setup.sh to be incorporated into VM cache-images # (see docs). cd "${GOSRC}/" -case "${OS_REL_VER}" in - ubuntu-18) +case "${OS_RELEASE_ID}" in + ubuntu) CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc" if dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH" then @@ -42,14 +42,13 @@ case "${OS_REL_VER}" in ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc" fi ;; - fedora-30) ;& # continue to next item - fedora-29) - # All SELinux distros need this for systemd-in-a-container - setsebool container_manage_cgroup true + fedora) + # All SELinux distros need this for systemd-in-a-container + setsebool container_manage_cgroup true if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then bash "$SCRIPT_BASE/add_second_partition.sh"; fi ;; - centos-7) # Current VM is an image-builder-image no local podman/testing + centos) # Current VM is an image-builder-image no local podman/testing echo "No further setup required for VM image building" exit 0 ;; |