diff options
Diffstat (limited to 'contrib/cirrus/packer')
20 files changed, 0 insertions, 1148 deletions
diff --git a/contrib/cirrus/packer/.gitignore b/contrib/cirrus/packer/.gitignore deleted file mode 100644 index 8f7bdeaf7..000000000 --- a/contrib/cirrus/packer/.gitignore +++ /dev/null @@ -1,7 +0,0 @@ -*json -packer -packer*zip -packer_cache -cidata* -meta-data -user-data diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile deleted file mode 100644 index c5a8e4cac..000000000 --- a/contrib/cirrus/packer/Makefile +++ /dev/null @@ -1,94 +0,0 @@ -PACKER_VER ?= 1.4.3 -GOARCH=$(shell go env GOARCH) -ARCH=$(uname -m) -PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip - -# Only needed for libpod_base_images target -TIMESTAMP := $(shell date +%s) -GOPATH ?= /var/tmp/go -GOSRC ?= $(GOPATH)/src/github.com/containers/libpod -PACKER_BASE ?= contrib/cirrus/packer -SCRIPT_BASE ?= contrib/cirrus -POST_MERGE_BUCKET_SUFFIX ?= - -UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$UBUNTU_BASE_IMAGE") -PRIOR_UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$PRIOR_UBUNTU_BASE_IMAGE") - -# For debugging nested-virt, use -#TTYDEV := $(shell tty) -TTYDEV := /dev/null - -.PHONY: all -all: libpod_images - -# Utility target for checking required parameters -.PHONY: guard-% -guard-%: - @if [[ -z "$($*)" ]]; then \ - echo "Missing or empty required make variable '$*'."; \ - exit 1; \ - fi; - -%.json: %.yml - @python3 -c 'import json,yaml; json.dump( yaml.safe_load(open("$<").read()), open("$@","w"), indent=2);' - -${PACKER_DIST_FILENAME}: - @curl -L --silent --show-error \ - -O https://releases.hashicorp.com/packer/${PACKER_VER}/${PACKER_DIST_FILENAME} - -packer: ${PACKER_DIST_FILENAME} - @curl -L --silent --show-error \ - https://releases.hashicorp.com/packer/${PACKER_VER}/packer_${PACKER_VER}_SHA256SUMS \ - | grep linux_${GOARCH} > /tmp/packer_sha256sums - @sha256sum --check /tmp/packer_sha256sums - @unzip -o ${PACKER_DIST_FILENAME} - @touch --reference=Makefile ${PACKER_DIST_FILENAME} - -.PHONY: test -test: libpod_base_images.json libpod_images.json packer - ./packer inspect libpod_base_images.json > /dev/null - ./packer inspect libpod_images.json > /dev/null - @echo "All good" - -.PHONY: libpod_images -libpod_images: guard-PACKER_BUILDS libpod_images.json packer - ./packer build \ - -force \ - $(shell test -z "${PACKER_BUILDS}" || echo "-only=${PACKER_BUILDS}") \ - -var GOPATH=$(GOPATH) \ - -var GOSRC=$(GOSRC) \ - -var PACKER_BASE=$(PACKER_BASE) \ - -var SCRIPT_BASE=$(SCRIPT_BASE) \ - libpod_images.json - -cidata.ssh: - ssh-keygen -f $@ -P "" -q - -cidata.ssh.pub: cidata.ssh - touch $@ - -meta-data: - echo "local-hostname: localhost.localdomain" > $@ - -user-data: cidata.ssh.pub - bash make-user-data.sh - -cidata.iso: user-data meta-data - genisoimage -output cidata.iso -volid cidata -input-charset utf-8 -joliet -rock user-data meta-data - -# This is intended to be run by a human, with admin access to the libpod GCE project. -.PHONY: libpod_base_images -libpod_base_images: guard-GCP_PROJECT_ID guard-GOOGLE_APPLICATION_CREDENTIALS libpod_base_images.json cidata.iso cidata.ssh packer - PACKER_CACHE_DIR=/tmp ./packer build \ - $(shell test -z "${PACKER_BUILDS}" || echo "-only=${PACKER_BUILDS}") \ - -force \ - -var TIMESTAMP=$(TIMESTAMP) \ - -var TTYDEV=$(TTYDEV) \ - -var GCP_PROJECT_ID=$(GCP_PROJECT_ID) \ - -var GOOGLE_APPLICATION_CREDENTIALS=$(GOOGLE_APPLICATION_CREDENTIALS) \ - -var GOSRC=$(GOSRC) \ - -var PACKER_BASE=$(PACKER_BASE) \ - -var SCRIPT_BASE=$(SCRIPT_BASE) \ - -var UBUNTU_BASE_IMAGE=$(UBUNTU_BASE_IMAGE) \ - -var PRIOR_UBUNTU_BASE_IMAGE=$(PRIOR_UBUNTU_BASE_IMAGE) \ - libpod_base_images.json diff --git a/contrib/cirrus/packer/README.how-to-update-cirrus-vms b/contrib/cirrus/packer/README.how-to-update-cirrus-vms deleted file mode 100644 index ac2902ffb..000000000 --- a/contrib/cirrus/packer/README.how-to-update-cirrus-vms +++ /dev/null @@ -1,89 +0,0 @@ -This document briefly describes how to update VMs on Cirrus. - -Examples of when you need to do this: - - - to update crun, conmon, or some other package(s) - - to add and/or remove an OS (eg drop f31, add f33) - - to change system config (eg containers.conf or other /etc files) - - to change kernel command-line (boot time) options - -This is a TWO-STEP process: you need to submit a PR with a magic [CI:IMG] -description string, wait for it to finish, grab a magic string from the -results, then resubmit without [CI:IMG]. - -Procedure, Part One of Two: - - 1) Create a working branch: - - $ git co -b my_branch_name - - 2) Make your changes. Typically, zero or more of the following files: - - .cirrus.yml - contrib/cirrus/packer/*_packaging.sh - - I said zero because sometimes you just want to update VMs - with the latest in dnf or ubuntu repos. That doesn't require - changing anything here, simply running new dnf/apt installs. - - 3) Commit your changes. Be sure to include the magic [CI:IMG] string: - - $ git commit -asm'[CI:IMG] this is my commit message' - - 4) Submit your PR: - - $ gh pr create --fill --web - - - -------------------------- INTERMISSION -------------------------- - ...in which we wait for CI to turn green. In particular, although - we only really need 'test_build_cache_images' (45 minutes or so) - to get the required magic number strings, please be a decent - human being and wait for 'verify_test_built_images' (another hour) - so we can all have confidence in our process. Thank you. - -------------------------- INTERMISSION -------------------------- - - -Procedure, Part Two of Two: - - 1) When 'test_build_cache_images' completes, click it, then click - 'View more details on Cirrus CI', then expand the 'Run build_vm_image' - accordion. This gives you a garishly colorful display of lines. - Each color is a different VM. - - 2) Verify that each VM has the packages you require. (The garish log - doesn't actually list this for all packages, so you may need to - look in the 'verify_test_built_images' log for each individual - VM. Click the 'package_versions' accordion.) - - 3) At the bottom of this log you will see a block like: - - Builds finished. The artifacts of successful builds are: - ubuntu-19: A disk image was created: ubuntu-19-podman-6439450735542272 - fedora-31: A disk image was created: fedora-31-podman-6439450735542272 - ..... - - The long numbers at the end should (MUST!) be all identical. - - 4) Edit .cirrus.yml locally. Find '_BUILT_IMAGE_SUFFIX' near the - top. Copy that long number ("6439450735542272", above) and paste - it here, replacing the previous long number. - - 5) Wait for CI to turn green. I know you might have skipped that, - because 'test_build_cache_images' finishes long before 'verify', - and maybe you're in a hurry, but come on. Be responsible. - - 6) Edit the PR description in github: remove '[CI:IMG]' from the - title. Again, *in github*, in the web UI, use the 'Edit' button - at top right next to the PR title. Remove the '[CI:IMG]' string - from the PR title, press Save. If you forget to do this, the - VM-building steps will run again (taking a long time) but it - will be a waste of time. - - 7) Update your PR: - - $ git add .cirrus.yml (to get the new magic IMAGE_SUFFIX string) - $ git commit --amend (remove [CI:IMG] for consistency with 6) - $ git push --force - -You can probably take it from here. diff --git a/contrib/cirrus/packer/README.md b/contrib/cirrus/packer/README.md deleted file mode 100644 index 9a07ed960..000000000 --- a/contrib/cirrus/packer/README.md +++ /dev/null @@ -1,3 +0,0 @@ -These are definitions and scripts consumed by packer to produce the -various distribution images used for CI testing. For more details -see the [Cirrus CI documentation](../README.md) diff --git a/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service b/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service deleted file mode 100644 index 4d2197d87..000000000 --- a/contrib/cirrus/packer/cloud-init/fedora/cloud-init.service +++ /dev/null @@ -1,20 +0,0 @@ -[Unit] -Description=Initial cloud-init job (metadata service crawler) -DefaultDependencies=no -Wants=cloud-init-local.service -After=cloud-init-local.service -Wants=google-network-daemon.service -After=google-network-daemon.service -Before=systemd-user-sessions.service - -[Service] -Type=oneshot -ExecStart=/usr/bin/cloud-init init -RemainAfterExit=yes -TimeoutSec=0 - -# Output needs to appear in instance console output -StandardOutput=journal+console - -[Install] -WantedBy=cloud-init.target diff --git a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg b/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg deleted file mode 100644 index 672d1907b..000000000 --- a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/40_enable_root.cfg +++ /dev/null @@ -1 +0,0 @@ -disable_root: 0 diff --git a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/50_custom_disk_setup.cfg b/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/50_custom_disk_setup.cfg deleted file mode 100644 index c0fdf0e23..000000000 --- a/contrib/cirrus/packer/cloud-init/fedora/cloud.cfg.d/50_custom_disk_setup.cfg +++ /dev/null @@ -1,4 +0,0 @@ -#cloud-config -growpart: - mode: false -resize_rootfs: false diff --git a/contrib/cirrus/packer/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg b/contrib/cirrus/packer/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg deleted file mode 100644 index 672d1907b..000000000 --- a/contrib/cirrus/packer/cloud-init/ubuntu/cloud.cfg.d/40_enable_root.cfg +++ /dev/null @@ -1 +0,0 @@ -disable_root: 0 diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh deleted file mode 100644 index bf29a1aec..000000000 --- a/contrib/cirrus/packer/fedora_base-setup.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# N/B: This script is not intended to be run by humans. It is used to configure the -# fedora base image for importing, so that it will boot in GCE - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -echo "Updating packages" -dnf -y update - -echo "Installing necessary packages and google services" -dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin ethtool - -echo "Enabling services" -systemctl enable rngd - -# There is a race that can happen on boot between the GCE services configuring -# the VM, and cloud-init trying to do similar activities. Use a customized -# unit file to make sure cloud-init starts after the google-compute-* services. -echo "Setting cloud-init service to start after google-network-daemon.service" -cp -v $GOSRC/$PACKER_BASE/cloud-init/fedora/cloud-init.service /etc/systemd/system/ - -# ref: https://cloud.google.com/compute/docs/startupscript -# The mechanism used by Cirrus-CI to execute tasks on the system is through an -# "agent" process launched as a GCP startup-script (from the metadata service). -# This agent is responsible for cloning the repository and executing all task -# scripts and other operations. Therefor, on SELinux-enforcing systems, the -# service must be labeled properly to ensure it's child processes can -# run with the proper contexts. -METADATA_SERVICE_CTX=unconfined_u:unconfined_r:unconfined_t:s0 -METADATA_SERVICE_PATH=systemd/system/google-startup-scripts.service -sed -r -e \ - "s/Type=oneshot/Type=oneshot\nSELinuxContext=$METADATA_SERVICE_CTX/" \ - /lib/$METADATA_SERVICE_PATH > /etc/$METADATA_SERVICE_PATH - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/fedora_packaging.sh b/contrib/cirrus/packer/fedora_packaging.sh deleted file mode 100644 index fcf9eb93f..000000000 --- a/contrib/cirrus/packer/fedora_packaging.sh +++ /dev/null @@ -1,194 +0,0 @@ -#!/usr/bin/env bash - -# This script is called from fedora_setup.sh and various Dockerfiles. -# It's not intended to be used outside of those contexts. It assumes the lib.sh -# library has already been sourced, and that all "ground-up" package-related activity -# needs to be done, including repository setup and initial update. - -set -e - -echo "Updating/Installing repos and packages for $OS_REL_VER" - -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var GOSRC SCRIPT_BASE BIGTO INSTALL_AUTOMATION_VERSION FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE - -# Pre-req. to install automation tooing -$LILTO $SUDO dnf install -y git - -# Install common automation tooling (i.e. ooe.sh) -curl --silent --show-error --location \ - --url "https://raw.githubusercontent.com/containers/automation/master/bin/install_automation.sh" | \ - $SUDO env INSTALL_PREFIX=/usr/share /bin/bash -s - "$INSTALL_AUTOMATION_VERSION" -# Reload installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -# Set this to 1 to NOT enable updates-testing repository -DISABLE_UPDATES_TESTING=${DISABLE_UPDATES_TESTING:0} - -# Do not enable updates-testing on the previous Fedora release -if ((DISABLE_UPDATES_TESTING!=0)); then - warn "Enabling updates-testing repository for image based on $FEDORA_BASE_IMAGE" - $LILTO $SUDO ooe.sh dnf install -y 'dnf-command(config-manager)' - $LILTO $SUDO ooe.sh dnf config-manager --set-enabled updates-testing -else - warn "NOT enabling updates-testing repository for image based on $PRIOR_FEDORA_BASE_IMAGE" -fi - -$BIGTO ooe.sh $SUDO dnf update -y - -# Fedora, as of 31, uses cgroups v2 by default. runc does not support -# cgroups v2, only crun does. (As of 2020-07-30 runc support is -# forthcoming but not even close to ready yet). To ensure a reliable -# runtime environment, force-remove runc if it is present. -# However, because a few other repos. which use these images still need -# it, ensure the runc package is cached in $PACKAGE_DOWNLOAD_DIR so -# it may be swap it in when required. -REMOVE_PACKAGES=(runc) - -INSTALL_PACKAGES=(\ - autoconf - automake - bash-completion - bats - bridge-utils - btrfs-progs-devel - buildah - bzip2 - conmon - container-selinux - containernetworking-plugins - containers-common - criu - crun - curl - device-mapper-devel - dnsmasq - e2fsprogs-devel - emacs-nox - file - findutils - fuse3 - fuse3-devel - gcc - git - glib2-devel - glibc-devel - glibc-static - gnupg - go-md2man - golang - gpgme - gpgme-devel - grubby - hostname - httpd-tools - iproute - iptables - jq - krb5-workstation - libassuan - libassuan-devel - libblkid-devel - libcap-devel - libffi-devel - libgpg-error-devel - libguestfs-tools - libmsi1 - libnet - libnet-devel - libnl3-devel - libseccomp - libseccomp-devel - libselinux-devel - libtool - libvarlink-util - libxml2-devel - libxslt-devel - lsof - make - mlocate - msitools - nfs-utils - nmap-ncat - openssl - openssl-devel - ostree-devel - pandoc - pkgconfig - podman - policycoreutils - procps-ng - protobuf - protobuf-c - protobuf-c-devel - protobuf-devel - python2 - python3-PyYAML - python3-dateutil - python3-libselinux - python3-libsemanage - python3-libvirt - python3-psutil - python3-pytoml - python3-requests - redhat-rpm-config - rpcbind - rsync - sed - selinux-policy-devel - skopeo - skopeo-containers - slirp4netns - socat - tar - unzip - vim - wget - which - xz - zip - zlib-devel -) -DOWNLOAD_PACKAGES=(\ - "cri-o-$(get_kubernetes_version)*" - cri-tools - "kubernetes-$(get_kubernetes_version)*" - runc - oci-umount - parallel -) - -echo "Installing general build/test dependencies for Fedora '$OS_RELEASE_VER'" -$BIGTO ooe.sh $SUDO dnf install -y ${INSTALL_PACKAGES[@]} - -# AD-HOC CODE FOR SPECIAL-CASE SITUATIONS! -# On 2020-07-23 we needed this code to upgrade crun on f31, a build -# that is not yet in stable. Since CI:IMG PRs are a two-step process, -# the key part is that we UN-COMMENT-THIS-OUT during the first step, -# then re-comment it on the second (once we have the built images). -# That way this will be dead code in future CI:IMG PRs but will -# serve as an example for anyone in a similar future situation. -# $BIGTO ooe.sh $SUDO dnf --enablerepo=updates-testing -y upgrade crun - -[[ ${#REMOVE_PACKAGES[@]} -eq 0 ]] || \ - $LILTO ooe.sh $SUDO dnf erase -y "${REMOVE_PACKAGES[@]}" - -if [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then - echo "Downloading packages for optional installation at runtime, as needed." - # Required for cri-o - ooe.sh $SUDO dnf -y module enable cri-o:$(get_kubernetes_version) - $SUDO mkdir -p "$PACKAGE_DOWNLOAD_DIR" - cd "$PACKAGE_DOWNLOAD_DIR" - $LILTO ooe.sh $SUDO dnf download -y --resolve "${DOWNLOAD_PACKAGES[@]}" -fi - -echo "Installing runtime tooling" -# Save some runtime by having these already available -cd $GOSRC -# Required since initially go was not installed -source $GOSRC/$SCRIPT_BASE/lib.sh -echo "Go environment has been setup:" -go env -$SUDO make install.tools -$SUDO $GOSRC/hack/install_catatonit.sh diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh deleted file mode 100644 index 16ae87d8a..000000000 --- a/contrib/cirrus/packer/fedora_setup.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# This script is called by packer on the subject fedora VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER - -workaround_bfq_bug - -# Do not enable updates-testing on the previous Fedora release -if [[ "$PRIOR_FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then - DISABLE_UPDATES_TESTING=1 -else - DISABLE_UPDATES_TESTING=0 -fi - -bash $PACKER_BASE/fedora_packaging.sh -# Load installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -echo "Enabling cgroup management from containers" -ooe.sh sudo setsebool container_manage_cgroup true - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/image-builder-image_base-setup.sh b/contrib/cirrus/packer/image-builder-image_base-setup.sh deleted file mode 100644 index 26fbe2903..000000000 --- a/contrib/cirrus/packer/image-builder-image_base-setup.sh +++ /dev/null @@ -1,71 +0,0 @@ -#!/usr/bin/env bash - -# This script is called by packer on a vanilla CentOS VM, to setup the image -# used for building images FROM base images. It's not intended to be used -# outside of this context. - -set -e - -[[ "$1" == "post" ]] || exit 0 # pre stage not needed - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var TIMESTAMP GOSRC SCRIPT_BASE PACKER_BASE - -install_ooe - -echo "Updating packages" -ooe.sh sudo yum -y update - -echo "Configuring repositories" -ooe.sh sudo yum -y install centos-release-scl epel-release - -echo "Installing packages" -ooe.sh sudo yum -y install \ - genisoimage \ - golang \ - google-cloud-sdk \ - libvirt \ - libvirt-admin \ - libvirt-client \ - libvirt-daemon \ - make \ - python36 \ - python36-PyYAML \ - qemu-img \ - qemu-kvm \ - qemu-kvm-tools \ - qemu-user \ - rsync \ - rng-tools \ - unzip \ - util-linux \ - vim - -sudo systemctl enable rngd - -sudo ln -s /usr/libexec/qemu-kvm /usr/bin/ - -sudo tee /etc/modprobe.d/kvm-nested.conf <<EOF -options kvm-intel nested=1 -options kvm-intel enable_shadow_vmcs=1 -options kvm-intel enable_apicv=1 -options kvm-intel ept=1 -EOF - -echo "Installing packer" -sudo mkdir -p /root/$(basename $PACKER_BASE) -sudo cp $GOSRC/$PACKER_BASE/*packer* /root/$(basename $PACKER_BASE) -sudo mkdir -p /root/$(basename $SCRIPT_BASE) -sudo cp $GOSRC/$SCRIPT_BASE/*.sh /root/$(basename $SCRIPT_BASE) - -install_scl_git - -echo "Cleaning up" -cd / -rm -rf $GOSRC - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml deleted file mode 100644 index f53bfafc5..000000000 --- a/contrib/cirrus/packer/libpod_base_images.yml +++ /dev/null @@ -1,164 +0,0 @@ ---- - -variables: - # Complete local path to this repository (Required) - GOSRC: - # Relative path to this (packer) subdirectory (Required) - PACKER_BASE: - # Relative path to cirrus scripts subdirectory (Required) - SCRIPT_BASE: - # Unique ID for naming new base-images (required) - TIMESTAMP: - # Required for output from qemu builders - TTYDEV: - - # Ubuntu releases are merely copied to this project for control purposes - UBUNTU_BASE_IMAGE: - PRIOR_UBUNTU_BASE_IMAGE: - - # Latest Fedora release - FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/32/Cloud/x86_64/images/Fedora-Cloud-Base-32-1.6.x86_64.qcow2" - FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/32/Cloud/x86_64/images/Fedora-Cloud-32-1.6-x86_64-CHECKSUM" - FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-32-1-6' - - # Prior Fedora release - PRIOR_FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-Base-31-1.9.x86_64.qcow2" - PRIOR_FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/31/Cloud/x86_64/images/Fedora-Cloud-31-1.9-x86_64-CHECKSUM" - PRIOR_FEDORA_BASE_IMAGE_NAME: 'fedora-cloud-base-31-1-9' - - # The name of the image in GCE used for packer build libpod_images.yml - IBI_BASE_NAME: 'image-builder-image' - CIDATA_ISO: 'cidata.iso' # produced by Makefile - - # Path to json file (required, likely ~/.config/gcloud/legacy_credentials/*/adc.json) - GOOGLE_APPLICATION_CREDENTIALS: - # The complete project ID (required, not the short name) - GCP_PROJECT_ID: - # Pre-existing storage bucket w/ lifecycle-enabled - XFERBUCKET: "packer-import" # pre-created, globally unique, lifecycle-enabled - -# Don't leak sensitive values in error messages / output -sensitive-variables: - - 'GOOGLE_APPLICATION_CREDENTIALS' - - 'GCP_PROJECT_ID' - -# What images to produce in which cloud -builders: - - &nested_virt - name: 'fedora' - type: 'qemu' - accelerator: "kvm" - iso_url: '{{user `FEDORA_IMAGE_URL`}}' - disk_image: true - format: "raw" - disk_size: 5120 - iso_checksum_url: '{{user `FEDORA_CSUM_URL`}}' - iso_checksum_type: "sha256" - output_directory: '/tmp/{{build_name}}' - vm_name: "disk.raw" # actually qcow2, name required for post-processing - boot_wait: '5s' - shutdown_command: 'shutdown -h now' - headless: true - qemu_binary: "/usr/libexec/qemu-kvm" - qemuargs: # List-of-list format required to override packer-generated args - - - "-m" - - "1024" - - - "-cpu" - - "host" - - - "-device" - - "virtio-rng-pci" - - - "-chardev" - - "tty,id=pts,path={{user `TTYDEV`}}" - - - "-device" - - "isa-serial,chardev=pts" - - - "-cdrom" - - "{{user `CIDATA_ISO`}}" - - - "-netdev" - - "user,id=net0,hostfwd=tcp::{{ .SSHHostPort }}-:22" - - - "-device" - - "virtio-net,netdev=net0" - communicator: 'ssh' - ssh_private_key_file: 'cidata.ssh' - ssh_username: 'root' - - - <<: *nested_virt - name: 'prior-fedora' - iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}' - iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}' - - - &imgcopy - name: 'ubuntu' - type: 'googlecompute' - image_name: '{{user `UBUNTU_BASE_IMAGE`}}' - image_family: '{{build_name}}-base' - source_image: '{{user `UBUNTU_BASE_IMAGE`}}' - source_image_project_id: 'ubuntu-os-cloud' - project_id: '{{user `GCP_PROJECT_ID`}}' - account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' - startup_script_file: "systemd_banish.sh" - zone: 'us-central1-a' - disk_size: 20 - communicator: 'none' - - - <<: *imgcopy - name: 'prior-ubuntu' - image_name: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' - source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' - -provisioners: - - type: 'shell' - only: ['fedora', 'prior-fedora'] - inline: - - 'mkdir -p /tmp/libpod/{{user `SCRIPT_BASE`}}' - - 'mkdir -p /tmp/libpod/{{user `PACKER_BASE`}}' - - - type: 'file' - only: ['fedora', 'prior-fedora'] - source: '{{user `GOSRC`}}/.cirrus.yml' - destination: '/tmp/libpod/.cirrus.yml' - - - type: 'file' - only: ['fedora', 'prior-fedora'] - source: '{{user `GOSRC`}}/{{user `SCRIPT_BASE`}}/' - destination: '/tmp/libpod/{{user `SCRIPT_BASE`}}/' - - - type: 'file' - only: ['fedora', 'prior-fedora'] - source: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/' - destination: '/tmp/libpod/{{user `PACKER_BASE`}}/' - - - &shell_script - only: ['fedora', 'prior-fedora'] - type: 'shell' - inline: - - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/*.sh' - - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh' - expect_disconnect: true # Allow this to reboot the VM if needed - environment_vars: - - 'TIMESTAMP={{user `TIMESTAMP`}}' - - 'GOSRC=/tmp/libpod' - - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' - - 'PACKER_BASE={{user `PACKER_BASE`}}' - -post-processors: - - - type: "compress" - only: ['fedora', 'prior-fedora'] - output: '/tmp/{{build_name}}/disk.raw.tar.gz' - format: '.tar.gz' - compression_level: 9 - - &gcp_import - only: ['fedora'] - type: "googlecompute-import" - project_id: '{{user `GCP_PROJECT_ID`}}' - account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}' - bucket: '{{user `XFERBUCKET`}}' - gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}.tar.gz' - image_name: "{{user `FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" - image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}' - image_family: '{{build_name}}-base' - - <<: *gcp_import - only: ['prior-fedora'] - image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}" - image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}' - image_family: '{{build_name}}-base' - - type: 'manifest' diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml deleted file mode 100644 index 38f5a8250..000000000 --- a/contrib/cirrus/packer/libpod_images.yml +++ /dev/null @@ -1,86 +0,0 @@ ---- - -# All of these are required -variables: - BUILT_IMAGE_SUFFIX: '{{env `BUILT_IMAGE_SUFFIX`}}' - GOPATH: '{{env `GOPATH`}}' - GOSRC: '{{env `GOSRC`}}' - PACKER_BASE: '{{env `PACKER_BASE`}}' - SCRIPT_BASE: '{{env `SCRIPT_BASE`}}' - - # Base-image names are required. Using image family-names breaks parallelism - UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}' - PRIOR_UBUNTU_BASE_IMAGE: '{{env `PRIOR_UBUNTU_BASE_IMAGE`}}' - FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}' - PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}' - - # Protected credentials, decrypted by Cirrus at runtime - GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}' - GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}' - SERVICE_ACCOUNT: '{{env `SERVICE_ACCOUNT`}}' - GOOGLE_APPLICATION_CREDENTIALS: '{{env `GOOGLE_APPLICATION_CREDENTIALS`}}' - -# Don't leak sensitive values in error messages / output -sensitive-variables: - - 'GCE_SSH_USERNAME' - - 'GCP_PROJECT_ID' - - 'SERVICE_ACCOUNT' - -# What images to produce in which cloud -builders: - # v----- is a YAML anchor, allows referencing this object by name (below) - - &gce_hosted_image - name: 'ubuntu-20' - type: 'googlecompute' - image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}' - image_family: '{{build_name}}-cache' - source_image: '{{user `UBUNTU_BASE_IMAGE`}}' # precedence over family - source_image_family: 'ubuntu-base' # for ref. only - disk_size: 20 # REQUIRED: Runtime allocation > this value - project_id: '{{user `GCP_PROJECT_ID`}}' - service_account_email: '{{user `SERVICE_ACCOUNT`}}' - communicator: 'ssh' - ssh_username: '{{user `GCE_SSH_USERNAME`}}' - ssh_pty: 'true' - # The only supported zone in Cirrus-CI, as of addition of this comment - zone: 'us-central1-a' - - # v----- is a YAML alias, allows partial re-use of the anchor object - - <<: *gce_hosted_image - name: 'ubuntu-19' - source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}' - source_image_family: 'prior-ubuntu-base' - - - <<: *gce_hosted_image - name: 'fedora-32' - source_image: '{{user `FEDORA_BASE_IMAGE`}}' - source_image_family: 'fedora-base' - - - <<: *gce_hosted_image - name: 'fedora-31' - source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}' - source_image_family: 'prior-fedora-base' - -# The brains of the operation, making actual modifications to the base-image. -provisioners: - - type: 'shell' - inline: - - 'set -ex' - # The 'file' provisioner item (below) will create the final component - - 'mkdir -vp $(dirname {{user `GOSRC`}})' - - - type: 'file' - source: '{{user `GOSRC`}}' - destination: '{{user `GOSRC`}}' - - - type: 'shell' - script: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{split build_name "-" 0}}_setup.sh' - environment_vars: - - 'PACKER_BUILDER_NAME={{build_name}}' - - 'GOPATH={{user `GOPATH`}}' - - 'GOSRC={{user `GOSRC`}}' - - 'PACKER_BASE={{user `PACKER_BASE`}}' - - 'SCRIPT_BASE={{user `SCRIPT_BASE`}}' - -post-processors: - - type: 'manifest' # writes packer-manifest.json diff --git a/contrib/cirrus/packer/make-user-data.sh b/contrib/cirrus/packer/make-user-data.sh deleted file mode 100644 index 676a50f5c..000000000 --- a/contrib/cirrus/packer/make-user-data.sh +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env bash - -# This script is utilized by Makefile, it's not intended to be run by humans - -cat <<EOF > user-data -#cloud-config -timezone: US/Eastern -growpart: - mode: auto -disable_root: false -ssh_pwauth: True -ssh_import_id: [root] -ssh_authorized_keys: - - $(cat cidata.ssh.pub) -users: - - name: root - primary-group: root - homedir: /root - system: true -EOF diff --git a/contrib/cirrus/packer/prior-fedora_base-setup.sh b/contrib/cirrus/packer/prior-fedora_base-setup.sh deleted file mode 100644 index bf29a1aec..000000000 --- a/contrib/cirrus/packer/prior-fedora_base-setup.sh +++ /dev/null @@ -1,44 +0,0 @@ -#!/usr/bin/env bash - -# N/B: This script is not intended to be run by humans. It is used to configure the -# fedora base image for importing, so that it will boot in GCE - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -echo "Updating packages" -dnf -y update - -echo "Installing necessary packages and google services" -dnf -y install rng-tools google-compute-engine-tools google-compute-engine-oslogin ethtool - -echo "Enabling services" -systemctl enable rngd - -# There is a race that can happen on boot between the GCE services configuring -# the VM, and cloud-init trying to do similar activities. Use a customized -# unit file to make sure cloud-init starts after the google-compute-* services. -echo "Setting cloud-init service to start after google-network-daemon.service" -cp -v $GOSRC/$PACKER_BASE/cloud-init/fedora/cloud-init.service /etc/systemd/system/ - -# ref: https://cloud.google.com/compute/docs/startupscript -# The mechanism used by Cirrus-CI to execute tasks on the system is through an -# "agent" process launched as a GCP startup-script (from the metadata service). -# This agent is responsible for cloning the repository and executing all task -# scripts and other operations. Therefor, on SELinux-enforcing systems, the -# service must be labeled properly to ensure it's child processes can -# run with the proper contexts. -METADATA_SERVICE_CTX=unconfined_u:unconfined_r:unconfined_t:s0 -METADATA_SERVICE_PATH=systemd/system/google-startup-scripts.service -sed -r -e \ - "s/Type=oneshot/Type=oneshot\nSELinuxContext=$METADATA_SERVICE_CTX/" \ - /lib/$METADATA_SERVICE_PATH > /etc/$METADATA_SERVICE_PATH - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -rh_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/systemd_banish.sh b/contrib/cirrus/packer/systemd_banish.sh deleted file mode 100755 index 2219f2a4f..000000000 --- a/contrib/cirrus/packer/systemd_banish.sh +++ /dev/null @@ -1,28 +0,0 @@ -#!/usr/bin/env bash - -set +e # Not all of these exist on every platform - -# This is intended to be executed on VMs as a startup script on initial-boot. -# Alternatively, it may be executed with the '--list' option to return the list -# of systemd units defined for disablement (useful for testing). - -EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean" - -if [[ "$1" == "--list" ]] -then - echo "$EVIL_UNITS" - exit 0 -fi - -echo "Disabling periodic services that could destabilize testing:" -for unit in $EVIL_UNITS -do - echo "Banishing $unit (ignoring errors)" - ( - sudo systemctl stop $unit - sudo systemctl disable $unit - sudo systemctl disable $unit.timer - sudo systemctl mask $unit - sudo systemctl mask $unit.timer - ) &> /dev/null -done diff --git a/contrib/cirrus/packer/ubuntu_packaging.sh b/contrib/cirrus/packer/ubuntu_packaging.sh deleted file mode 100644 index c478028b5..000000000 --- a/contrib/cirrus/packer/ubuntu_packaging.sh +++ /dev/null @@ -1,175 +0,0 @@ -#!/usr/bin/env bash - -# This script is called from ubuntu_setup.sh and various Dockerfiles. -# It's not intended to be used outside of those contexts. It assumes the lib.sh -# library has already been sourced, and that all "ground-up" package-related activity -# needs to be done, including repository setup and initial update. - -set -e - -echo "Updating/Installing repos and packages for $OS_REL_VER" - -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var GOSRC SCRIPT_BASE BIGTO SUDOAPTGET INSTALL_AUTOMATION_VERSION - -echo "Updating/configuring package repositories." -$BIGTO $SUDOAPTGET update - -echo "Installing deps to add third-party repositories and automation tooling" -$LILTO $SUDOAPTGET install software-properties-common git curl - -# Install common automation tooling (i.e. ooe.sh) -curl --silent --show-error --location \ - --url "https://raw.githubusercontent.com/containers/automation/master/bin/install_automation.sh" | \ - $SUDO env INSTALL_PREFIX=/usr/share /bin/bash -s - "$INSTALL_AUTOMATION_VERSION" -# Reload installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -$LILTO ooe.sh $SUDOAPTADD ppa:criu/ppa - -echo "Configuring/Instaling deps from Open build server" -VERSION_ID=$(source /etc/os-release; echo $VERSION_ID) -echo "deb http://download.opensuse.org/repositories/devel:/kubic:/libcontainers:/stable/xUbuntu_$VERSION_ID/ /" \ - | ooe.sh $SUDO tee /etc/apt/sources.list.d/devel:kubic:libcontainers:stable.list -ooe.sh curl -L -o /tmp/Release.key "https://download.opensuse.org/repositories/devel:kubic:libcontainers:stable/xUbuntu_${VERSION_ID}/Release.key" -ooe.sh $SUDO apt-key add - < /tmp/Release.key - -INSTALL_PACKAGES=(\ - apache2-utils - apparmor - aufs-tools - autoconf - automake - bash-completion - bats - bison - btrfs-progs - build-essential - buildah - bzip2 - conmon - containernetworking-plugins - containers-common - coreutils - cri-o-runc - criu - curl - dnsmasq - e2fslibs-dev - emacs-nox - file - fuse3 - gawk - gcc - gettext - git - go-md2man - golang-1.14 - iproute2 - iptables - jq - libaio-dev - libapparmor-dev - libbtrfs-dev - libcap-dev - libdevmapper-dev - libdevmapper1.02.1 - libfuse-dev - libfuse2 - libfuse3-dev - libglib2.0-dev - libgpgme11-dev - liblzma-dev - libnet1 - libnet1-dev - libnl-3-dev - libprotobuf-c-dev - libprotobuf-dev - libseccomp-dev - libseccomp2 - libselinux-dev - libsystemd-dev - libtool - libudev-dev - libvarlink - lsof - make - netcat - openssl - pkg-config - podman - protobuf-c-compiler - protobuf-compiler - python-dateutil - python-protobuf - python2 - python3-dateutil - python3-pip - python3-psutil - python3-pytoml - python3-requests - python3-setuptools - rsync - runc - scons - skopeo - slirp4netns - socat - sudo - unzip - vim - wget - xz-utils - zip - zlib1g-dev -) -DOWNLOAD_PACKAGES=(\ - cri-o-$(get_kubernetes_version) - cri-tools - parallel -) - -# These aren't resolvable on Ubuntu 20 -if [[ "$OS_RELEASE_VER" -le 19 ]]; then - INSTALL_PACKAGES+=(\ - python-future - python-minimal - yum-utils - ) -else - INSTALL_PACKAGES+=(\ - python-is-python3 - ) -fi - -# Do this at the last possible moment to avoid dpkg lock conflicts -echo "Upgrading all packages" -$BIGTO ooe.sh $SUDOAPTGET upgrade - -echo "Installing general testing and system dependencies" -# Necessary to update cache of newly added repos -$LILTO ooe.sh $SUDOAPTGET update -$BIGTO ooe.sh $SUDOAPTGET install "${INSTALL_PACKAGES[@]}" - -if [[ ${#DOWNLOAD_PACKAGES[@]} -gt 0 ]]; then - echo "Downloading packages for optional installation at runtime, as needed." - $SUDO ln -s /var/cache/apt/archives "$PACKAGE_DOWNLOAD_DIR" - $LILTO ooe.sh $SUDOAPTGET install --download-only "${DOWNLOAD_PACKAGES[@]}" -fi - -echo "Configuring Go environment" -# There are multiple (otherwise conflicting) versions of golang available -# on Ubuntu. Being primarily localized by env. vars and defaults, dropping -# a symlink is the appropriate way to "install" a specific version system-wide. -$SUDO ln -sf /usr/lib/go-1.14/bin/go /usr/bin/go -# Initially go was not installed -cd $GOSRC -source $SCRIPT_BASE/lib.sh -echo "Go environment has been setup:" -go env - -echo "Building/Installing runtime tooling" -$SUDO hack/install_catatonit.sh -$SUDO make install.libseccomp.sudo -$SUDO make install.tools GO_BUILD='go build' # -mod=vendor breaks this diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh deleted file mode 100644 index d650e6c76..000000000 --- a/contrib/cirrus/packer/ubuntu_setup.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash - -# This script is called by packer on the subject Ubuntu VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC UBUNTU_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -# Stop disruption upon boot ASAP after booting -echo "Disabling all packaging activity on boot" -for filename in $(sudo ls -1 /etc/apt/apt.conf.d); do \ - echo "Checking/Patching $filename" - sudo sed -i -r -e "s/$PERIODIC_APT_RE/"'\10"\;/' "/etc/apt/apt.conf.d/$filename"; done - -bash $PACKER_BASE/ubuntu_packaging.sh - -# Load installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -echo "Making Ubuntu kernel to enable cgroup swap accounting as it is not the default." -SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g' -ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/* -ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub -ooe.sh sudo update-grub - -ubuntu_finalize - -echo "SUCCESS!" diff --git a/contrib/cirrus/packer/xfedora_setup.sh b/contrib/cirrus/packer/xfedora_setup.sh deleted file mode 100644 index 16ae87d8a..000000000 --- a/contrib/cirrus/packer/xfedora_setup.sh +++ /dev/null @@ -1,34 +0,0 @@ -#!/usr/bin/env bash - -# This script is called by packer on the subject fedora VM, to setup the podman -# build/test environment. It's not intended to be used outside of this context. - -set -e - -# Load in library (copied by packer, before this script was run) -source $GOSRC/$SCRIPT_BASE/lib.sh - -req_env_var SCRIPT_BASE PACKER_BASE INSTALL_AUTOMATION_VERSION PACKER_BUILDER_NAME GOSRC FEDORA_BASE_IMAGE OS_RELEASE_ID OS_RELEASE_VER - -workaround_bfq_bug - -# Do not enable updates-testing on the previous Fedora release -if [[ "$PRIOR_FEDORA_BASE_IMAGE" =~ "${OS_RELEASE_ID}-cloud-base-${OS_RELEASE_VER}" ]]; then - DISABLE_UPDATES_TESTING=1 -else - DISABLE_UPDATES_TESTING=0 -fi - -bash $PACKER_BASE/fedora_packaging.sh -# Load installed environment right now (happens automatically in a new process) -source /usr/share/automation/environment - -echo "Enabling cgroup management from containers" -ooe.sh sudo setsebool container_manage_cgroup true - -# Ensure there are no disruptive periodic services enabled by default in image -systemd_banish - -rh_finalize - -echo "SUCCESS!" |