summaryrefslogtreecommitdiff
path: root/contrib/cirrus
diff options
context:
space:
mode:
Diffstat (limited to 'contrib/cirrus')
-rwxr-xr-xcontrib/cirrus/build_release.sh24
-rwxr-xr-xcontrib/cirrus/build_vm_images.sh2
-rwxr-xr-xcontrib/cirrus/check_image.sh2
-rwxr-xr-xcontrib/cirrus/integration_test.sh5
-rw-r--r--contrib/cirrus/lib.sh43
-rw-r--r--contrib/cirrus/packer/Makefile32
-rw-r--r--contrib/cirrus/packer/fedora_base-setup.sh2
-rw-r--r--contrib/cirrus/packer/fedora_setup.sh11
-rw-r--r--contrib/cirrus/packer/libpod_base_images.yml51
-rw-r--r--contrib/cirrus/packer/libpod_images.yml26
l---------contrib/cirrus/packer/prior-fedora_base-setup.sh (renamed from contrib/cirrus/packer/prior_fedora_base-setup.sh)0
-rwxr-xr-xcontrib/cirrus/packer/systemd_banish.sh28
-rw-r--r--contrib/cirrus/packer/ubuntu_setup.sh39
-rwxr-xr-xcontrib/cirrus/podbot.py15
-rwxr-xr-xcontrib/cirrus/setup_environment.sh13
-rwxr-xr-xcontrib/cirrus/success.sh18
-rwxr-xr-xcontrib/cirrus/upload_release_archive.sh62
17 files changed, 254 insertions, 119 deletions
diff --git a/contrib/cirrus/build_release.sh b/contrib/cirrus/build_release.sh
index 287643f47..07db88f81 100755
--- a/contrib/cirrus/build_release.sh
+++ b/contrib/cirrus/build_release.sh
@@ -1,5 +1,7 @@
#!/bin/bash
+set -e
+
source $(dirname $0)/lib.sh
req_env_var TEST_REMOTE_CLIENT OS_RELEASE_ID GOSRC
@@ -13,18 +15,20 @@ fi
if [[ -n "$CROSS_PLATFORM" ]]
then
+ # Will fail if $CROSS_PLATFORM is unsupported cross-compile $GOOS value
+ make podman-remote-${CROSS_PLATFORM}-release
+
echo "Compiling podman-remote release archive for ${CROSS_PLATFORM}"
- case "$CROSS_PLATFORM" in
- linux) ;&
- windows) ;&
- darwin)
- make podman-remote-${CROSS_PLATFORM}-release
- ;;
- *)
- die 1 "Unknown/unsupported cross-compile platform '$CROSS_PLATFORM'"
- ;;
- esac
+ if [[ "$CROSS_PLATFORM" == "windows" ]]
+ then
+ # TODO: Remove next line, part of VM images next time they're built.
+ dnf install -y libmsi1 msitools pandoc
+ make podman.msi
+ fi
else
echo "Compiling release archive for $OS_RELEASE_ID"
make podman-release
fi
+
+echo "Preserving build details for later use."
+mv -v release.txt actual_release.txt # Another 'make' during testing could overwrite it
diff --git a/contrib/cirrus/build_vm_images.sh b/contrib/cirrus/build_vm_images.sh
index 6230610cb..543f83a14 100755
--- a/contrib/cirrus/build_vm_images.sh
+++ b/contrib/cirrus/build_vm_images.sh
@@ -3,7 +3,7 @@
set -e
source $(dirname $0)/lib.sh
-BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE'
+BASE_IMAGE_VARS='FEDORA_BASE_IMAGE PRIOR_FEDORA_BASE_IMAGE UBUNTU_BASE_IMAGE PRIOR_UBUNTU_BASE_IMAGE'
ENV_VARS="PACKER_BUILDS BUILT_IMAGE_SUFFIX $BASE_IMAGE_VARS SERVICE_ACCOUNT GCE_SSH_USERNAME GCP_PROJECT_ID PACKER_VER SCRIPT_BASE PACKER_BASE CIRRUS_BUILD_ID CIRRUS_CHANGE_IN_REPO"
req_env_var $ENV_VARS
# Must also be made available through make, into packer process
diff --git a/contrib/cirrus/check_image.sh b/contrib/cirrus/check_image.sh
index ad9a12f49..39f49d0a1 100755
--- a/contrib/cirrus/check_image.sh
+++ b/contrib/cirrus/check_image.sh
@@ -4,6 +4,8 @@ set -eo pipefail
source $(dirname $0)/lib.sh
+EVIL_UNITS="$($CIRRUS_WORKING_DIR/$PACKER_BASE/systemd_banish.sh --list)"
+
req_env_var PACKER_BUILDER_NAME TEST_REMOTE_CLIENT EVIL_UNITS OS_RELEASE_ID
NFAILS=0
diff --git a/contrib/cirrus/integration_test.sh b/contrib/cirrus/integration_test.sh
index 00c3b0ec3..110066ea7 100755
--- a/contrib/cirrus/integration_test.sh
+++ b/contrib/cirrus/integration_test.sh
@@ -37,11 +37,8 @@ case "$SPECIALMODE" in
-o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE}
;;
cgroupv2)
- # FIXME: use the package once all the fixes are in a release
- # yum install -y crun
setenforce 0
- yum builddep -y crun
- (git clone --depth=1 https://github.com/containers/crun && cd crun && ./autogen.sh && ./configure --prefix=/usr && make -j4 && make install)
+ dnf install -y crun
export OCI_RUNTIME=/usr/bin/crun
make
make install PREFIX=/usr ETCDIR=/etc
diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh
index f26eec87f..f81a8d501 100644
--- a/contrib/cirrus/lib.sh
+++ b/contrib/cirrus/lib.sh
@@ -36,7 +36,7 @@ SETUP_MARKER_FILEPATH="${SETUP_MARKER_FILEPATH:-/var/tmp/.setup_environment_sh_c
AUTHOR_NICKS_FILEPATH="${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/git_authors_to_irc_nicks.csv"
cd $GOSRC
-if type -P git &> /dev/null
+if type -P git &> /dev/null && [[ -d "$GOSRC/.git" ]]
then
CIRRUS_CHANGE_IN_REPO=${CIRRUS_CHANGE_IN_REPO:-$(git show-ref --hash=8 HEAD || date +%s)}
else # pick something unique and obviously not from Cirrus
@@ -52,16 +52,17 @@ CIRRUS_REPO_NAME=${CIRRUS_REPO_NAME:-libpod}
CIRRUS_BASE_SHA=${CIRRUS_BASE_SHA:-unknown$(date +%s)} # difficult to reliably discover
CIRRUS_BUILD_ID=${CIRRUS_BUILD_ID:-$RANDOM$(date +%s)} # must be short and unique
# Vars. for image-building
-PACKER_VER="1.3.5"
+PACKER_VER="1.4.2"
# CSV of cache-image names to build (see $PACKER_BASE/libpod_images.json)
# Base-images rarely change, define them here so they're out of the way.
-export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,fedora-30,xfedora-30,fedora-29}"
+export PACKER_BUILDS="${PACKER_BUILDS:-ubuntu-18,ubuntu-19,fedora-30,xfedora-30,fedora-29}"
# Google-maintained base-image names
-export UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a"
+export UBUNTU_BASE_IMAGE="ubuntu-1904-disco-v20190724"
+export PRIOR_UBUNTU_BASE_IMAGE="ubuntu-1804-bionic-v20190722a"
# Manually produced base-image names (see $SCRIPT_BASE/README.md)
-export FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1559164849"
-export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1559164849"
+export FEDORA_BASE_IMAGE="fedora-cloud-base-30-1-2-1565360543"
+export PRIOR_FEDORA_BASE_IMAGE="fedora-cloud-base-29-1-2-1565360543"
export BUILT_IMAGE_SUFFIX="${BUILT_IMAGE_SUFFIX:--$CIRRUS_REPO_NAME-${CIRRUS_BUILD_ID}}"
# IN_PODMAN container image
IN_PODMAN_IMAGE="quay.io/libpod/in_podman:latest"
@@ -81,9 +82,6 @@ ROOTLESS_ENV_RE='(CIRRUS_.+)|(ROOTLESS_.+)|(.+_IMAGE.*)|(.+_BASE)|(.*DIRPATH)|(.
# Unsafe env. vars for display
SECRET_ENV_RE='(IRCID)|(ACCOUNT)|(GC[EP]..+)|(SSH)'
-# Names of systemd units which should never be running
-EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean"
-
SPECIALMODE="${SPECIALMODE:-none}"
TEST_REMOTE_CLIENT="${TEST_REMOTE_CLIENT:-false}"
export CONTAINER_RUNTIME=${CONTAINER_RUNTIME:-podman}
@@ -322,7 +320,7 @@ EOF
sudo chmod 755 /usr/bin/git
}
-install_test_configs(){
+install_test_configs() {
echo "Installing cni config, policy and registry config"
req_env_var GOSRC SCRIPT_BASE
cd $GOSRC
@@ -342,7 +340,7 @@ install_test_configs(){
# of pulling in necessary prerequisites packages as the set can change over time.
# For general CI testing however, calling this function makes sure the system
# can only run the compiled source version.
-remove_packaged_podman_files(){
+remove_packaged_podman_files() {
echo "Removing packaged podman files to prevent conflicts with source build and testing."
req_env_var OS_RELEASE_ID
@@ -379,24 +377,11 @@ remove_packaged_podman_files(){
sync && echo 3 > /proc/sys/vm/drop_caches
}
-systemd_banish(){
- echo "Disabling periodic services that could destabilize testing (ignoring errors):"
- set +e # Not all of these exist on every platform
- for unit in $EVIL_UNITS
- do
- echo "Banishing $unit (ignoring errors)"
- (
- sudo systemctl stop $unit
- sudo systemctl disable $unit
- sudo systemctl disable $unit.timer
- sudo systemctl mask $unit
- sudo systemctl mask $unit.timer
- ) &> /dev/null
- done
- set -e
+systemd_banish() {
+ $GOSRC/$PACKER_BASE/systemd_banish.sh
}
-_finalize(){
+_finalize() {
set +e # Don't fail at the very end
if [[ -d "$CUSTOM_CLOUD_CONFIG_DEFAULTS" ]]
then
@@ -419,7 +404,7 @@ _finalize(){
sudo fstrim -av
}
-rh_finalize(){
+rh_finalize() {
set +e # Don't fail at the very end
echo "Resetting to fresh-state for usage as cloud-image."
PKG=$(type -P dnf || type -P yum || echo "")
@@ -430,7 +415,7 @@ rh_finalize(){
_finalize
}
-ubuntu_finalize(){
+ubuntu_finalize() {
set +e # Don't fail at the very end
echo "Resetting to fresh-state for usage as cloud-image."
$LILTO $SUDOAPTGET autoremove
diff --git a/contrib/cirrus/packer/Makefile b/contrib/cirrus/packer/Makefile
index d03d22abe..947a2a1e9 100644
--- a/contrib/cirrus/packer/Makefile
+++ b/contrib/cirrus/packer/Makefile
@@ -3,7 +3,7 @@
# builder name(s) from applicable YAML file,
# e.g for names see libpod_images.yml
-PACKER_VER ?= 1.3.5
+PACKER_VER ?= 1.4.2
GOARCH=$(shell go env GOARCH)
ARCH=$(uname -m)
PACKER_DIST_FILENAME := packer_${PACKER_VER}_linux_${GOARCH}.zip
@@ -15,6 +15,9 @@ PACKER_BASE ?= contrib/cirrus/packer
SCRIPT_BASE ?= contrib/cirrus
POST_MERGE_BUCKET_SUFFIX ?=
+UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$UBUNTU_BASE_IMAGE")
+PRIOR_UBUNTU_BASE_IMAGE = $(shell source ../lib.sh && echo "$$PRIOR_UBUNTU_BASE_IMAGE")
+
# For debugging nested-virt, use
#TTYDEV := $(shell tty)
TTYDEV := /dev/null
@@ -22,6 +25,14 @@ TTYDEV := /dev/null
.PHONY: all
all: libpod_images
+# Utility target for checking required parameters
+.PHONY: guard-%
+guard-%:
+ @if [[ -z "$($*)" ]]; then \
+ echo "Missing or empty required make variable '$*'."; \
+ exit 1; \
+ fi;
+
%.json: %.yml
@python3 -c 'import json,yaml; json.dump( yaml.load(open("$<").read()), open("$@","w"), indent=2);'
@@ -44,10 +55,7 @@ test: libpod_base_images.json libpod_images.json packer
@echo "All good"
.PHONY: libpod_images
-libpod_images: libpod_images.json packer
-ifndef PACKER_BUILDS
- $(error PACKER_BUILDS is undefined, expected builder-names CSV)
-endif
+libpod_images: guard-PACKER_BUILDS libpod_images.json packer
./packer build -only=${PACKER_BUILDS} \
-force \
-var GOSRC=$(GOSRC) \
@@ -72,16 +80,7 @@ cidata.iso: user-data meta-data
# This is intended to be run by a human, with admin access to the libpod GCE project.
.PHONY: libpod_base_images
-libpod_base_images: libpod_base_images.json cidata.iso cidata.ssh packer
-ifndef GCP_PROJECT_ID
- $(error GCP_PROJECT_ID is undefined, expected complete GCP project ID string e.g. foobar-12345)
-endif
-ifndef GOOGLE_APPLICATION_CREDENTIALS
- $(error GOOGLE_APPLICATION_CREDENTIALS is undefined, expected absolute path to JSON file, like $HOME/.config/gcloud/legacy_credentials/*/adc.json)
-endif
-ifndef PACKER_BUILDS
- $(error PACKER_BUILDS is undefined, expected builder-names CSV)
-endif
+libpod_base_images: guard-GCP_PROJECT_ID guard-GOOGLE_APPLICATION_CREDENTIALS libpod_base_images.json cidata.iso cidata.ssh packer
PACKER_CACHE_DIR=/tmp ./packer build \
-force \
-var TIMESTAMP=$(TIMESTAMP) \
@@ -91,5 +90,6 @@ endif
-var GOSRC=$(GOSRC) \
-var PACKER_BASE=$(PACKER_BASE) \
-var SCRIPT_BASE=$(SCRIPT_BASE) \
- -only $(PACKER_BUILDS) \
+ -var UBUNTU_BASE_IMAGE=$(UBUNTU_BASE_IMAGE) \
+ -var PRIOR_UBUNTU_BASE_IMAGE=$(PRIOR_UBUNTU_BASE_IMAGE) \
libpod_base_images.json
diff --git a/contrib/cirrus/packer/fedora_base-setup.sh b/contrib/cirrus/packer/fedora_base-setup.sh
index 788a54c34..29c23117f 100644
--- a/contrib/cirrus/packer/fedora_base-setup.sh
+++ b/contrib/cirrus/packer/fedora_base-setup.sh
@@ -8,8 +8,6 @@ set -e
# Load in library (copied by packer, before this script was run)
source $GOSRC/$SCRIPT_BASE/lib.sh
-[[ "$1" == "post" ]] || exit 0 # nothing to do
-
install_ooe
echo "Updating packages"
diff --git a/contrib/cirrus/packer/fedora_setup.sh b/contrib/cirrus/packer/fedora_setup.sh
index 0e1a82cc0..f84645a04 100644
--- a/contrib/cirrus/packer/fedora_setup.sh
+++ b/contrib/cirrus/packer/fedora_setup.sh
@@ -24,6 +24,8 @@ ooe.sh sudo dnf config-manager --set-enabled updates-testing
echo "Installing general build/test dependencies"
ooe.sh sudo dnf install -y \
atomic-registries \
+ autoconf \
+ automake \
bats \
bridge-utils \
btrfs-progs-devel \
@@ -35,10 +37,14 @@ ooe.sh sudo dnf install -y \
device-mapper-devel \
emacs-nox \
findutils \
+ fuse3 \
+ fuse3-devel \
+ gcc \
git \
glib2-devel \
glibc-static \
gnupg \
+ go-md2man \
golang \
golang-github-cpuguy83-go-md2man \
gpgme-devel \
@@ -47,6 +53,7 @@ ooe.sh sudo dnf install -y \
jq \
libassuan-devel \
libcap-devel \
+ libmsi1 \
libnet \
libnet-devel \
libnl3-devel \
@@ -56,9 +63,11 @@ ooe.sh sudo dnf install -y \
libvarlink-util \
lsof \
make \
+ msitools \
nmap-ncat \
ostree \
ostree-devel \
+ pandoc \
podman \
procps-ng \
protobuf \
@@ -81,6 +90,7 @@ ooe.sh sudo dnf install -y \
xz \
zip
+
# Ensure there are no disruptive periodic services enabled by default in image
systemd_banish
@@ -91,6 +101,7 @@ case "$PACKER_BUILDER_NAME" in
xfedora*)
echo "Configuring CGroups v2 enabled on next boot"
sudo grubby --update-kernel=ALL --args="systemd.unified_cgroup_hierarchy=1"
+ sudo dnf install -y crun
;& # continue to next matching item
*)
echo "Finalizing $PACKER_BUILDER_NAME VM image"
diff --git a/contrib/cirrus/packer/libpod_base_images.yml b/contrib/cirrus/packer/libpod_base_images.yml
index e519d2fba..bcca440ae 100644
--- a/contrib/cirrus/packer/libpod_base_images.yml
+++ b/contrib/cirrus/packer/libpod_base_images.yml
@@ -12,6 +12,10 @@ variables:
# Required for output from qemu builders
TTYDEV:
+ # Ubuntu releases are mearly copied to this project for control purposes
+ UBUNTU_BASE_IMAGE:
+ PRIOR_UBUNTU_BASE_IMAGE:
+
# Latest Fedora release
FEDORA_IMAGE_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-Base-30-1.2.x86_64.qcow2"
FEDORA_CSUM_URL: "https://dl.fedoraproject.org/pub/fedora/linux/releases/30/Cloud/x86_64/images/Fedora-Cloud-30-1.2-x86_64-CHECKSUM"
@@ -78,50 +82,67 @@ builders:
ssh_username: 'root'
- <<: *nested_virt
- name: 'prior_fedora'
+ name: 'prior-fedora'
iso_url: '{{user `PRIOR_FEDORA_IMAGE_URL`}}'
iso_checksum_url: '{{user `PRIOR_FEDORA_CSUM_URL`}}'
+ - &imgcopy
+ name: 'ubuntu'
+ type: 'googlecompute'
+ image_name: '{{user `UBUNTU_BASE_IMAGE`}}'
+ image_family: '{{build_name}}-base'
+ source_image: '{{user `UBUNTU_BASE_IMAGE`}}'
+ source_image_project_id: 'ubuntu-os-cloud'
+ project_id: '{{user `GCP_PROJECT_ID`}}'
+ account_file: '{{user `GOOGLE_APPLICATION_CREDENTIALS`}}'
+ startup_script_file: "systemd_banish.sh"
+ zone: 'us-central1-a'
+ disk_size: 20
+ communicator: 'none'
+
+ - <<: *imgcopy
+ name: 'prior-ubuntu'
+ image_name: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}'
+ source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}'
+
provisioners:
- type: 'shell'
+ only: ['fedora', 'prior-fedora']
inline:
- 'mkdir -p /tmp/libpod/{{user `SCRIPT_BASE`}}'
- 'mkdir -p /tmp/libpod/{{user `PACKER_BASE`}}'
- type: 'file'
+ only: ['fedora', 'prior-fedora']
source: '{{user `GOSRC`}}/.cirrus.yml'
destination: '/tmp/libpod/.cirrus.yml'
- type: 'file'
+ only: ['fedora', 'prior-fedora']
source: '{{user `GOSRC`}}/{{user `SCRIPT_BASE`}}/'
destination: '/tmp/libpod/{{user `SCRIPT_BASE`}}/'
- type: 'file'
+ only: ['fedora', 'prior-fedora']
source: '{{user `GOSRC`}}/{{user `PACKER_BASE`}}/'
destination: '/tmp/libpod/{{user `PACKER_BASE`}}/'
- &shell_script
+ only: ['fedora', 'prior-fedora']
type: 'shell'
inline:
- - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'
- - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh pre'
- expect_disconnect: true # Allow this to reboot the VM
+ - 'chmod +x /tmp/libpod/{{user `PACKER_BASE`}}/*.sh'
+ - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh'
+ expect_disconnect: true # Allow this to reboot the VM if needed
environment_vars:
- 'TIMESTAMP={{user `TIMESTAMP`}}'
- 'GOSRC=/tmp/libpod'
- 'SCRIPT_BASE={{user `SCRIPT_BASE`}}'
- 'PACKER_BASE={{user `PACKER_BASE`}}'
- - <<: *shell_script
- inline: ['{{user `GOSRC`}}/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh']
- expect_disconnect: false
- pause_before: '10s'
- inline:
- - '/tmp/libpod/{{user `PACKER_BASE`}}/{{build_name}}_base-setup.sh post'
-
post-processors:
- - type: "compress"
- only: ['fedora', 'prior_fedora']
+ only: ['fedora', 'prior-fedora']
output: '/tmp/{{build_name}}/disk.raw.tar.gz'
format: '.tar.gz'
compression_level: 9
@@ -134,10 +155,10 @@ post-processors:
gcs_object_name: '{{build_name}}-{{user `TIMESTAMP`}}.tar.gz'
image_name: "{{user `FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
image_description: 'Based on {{user `FEDORA_IMAGE_URL`}}'
- image_family: '{{user `FEDORA_BASE_IMAGE_NAME`}}'
+ image_family: '{{build_name}}-base'
- <<: *gcp_import
- only: ['prior_fedora']
+ only: ['prior-fedora']
image_name: "{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}-{{user `TIMESTAMP`}}"
image_description: 'Based on {{user `PRIOR_FEDORA_IMAGE_URL`}}'
- image_family: '{{user `PRIOR_FEDORA_BASE_IMAGE_NAME`}}'
+ image_family: '{{build_name}}-base'
- type: 'manifest'
diff --git a/contrib/cirrus/packer/libpod_images.yml b/contrib/cirrus/packer/libpod_images.yml
index cae5d4138..01a65d867 100644
--- a/contrib/cirrus/packer/libpod_images.yml
+++ b/contrib/cirrus/packer/libpod_images.yml
@@ -2,16 +2,17 @@
# All of these are required
variables:
- # Names of GCE Base images to start from, in .cirrus.yml
- UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}'
- FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}'
- PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}'
-
BUILT_IMAGE_SUFFIX: '{{env `BUILT_IMAGE_SUFFIX`}}'
GOSRC: '{{env `GOSRC`}}'
PACKER_BASE: '{{env `PACKER_BASE`}}'
SCRIPT_BASE: '{{env `SCRIPT_BASE`}}'
+ # Base-image names are required. Using image family-names breaks parallelism
+ UBUNTU_BASE_IMAGE: '{{env `UBUNTU_BASE_IMAGE`}}'
+ PRIOR_UBUNTU_BASE_IMAGE: '{{env `PRIOR_UBUNTU_BASE_IMAGE`}}'
+ FEDORA_BASE_IMAGE: '{{env `FEDORA_BASE_IMAGE`}}'
+ PRIOR_FEDORA_BASE_IMAGE: '{{env `PRIOR_FEDORA_BASE_IMAGE`}}'
+
# Protected credentials, decrypted by Cirrus at runtime
GCE_SSH_USERNAME: '{{env `GCE_SSH_USERNAME`}}'
GCP_PROJECT_ID: '{{env `GCP_PROJECT_ID`}}'
@@ -28,11 +29,12 @@ sensitive-variables:
builders:
# v----- is a YAML anchor, allows referencing this object by name (below)
- &gce_hosted_image
- name: 'ubuntu-18'
+ name: 'ubuntu-19'
type: 'googlecompute'
image_name: '{{build_name}}{{user `BUILT_IMAGE_SUFFIX`}}'
- image_family: '{{build_name}}-libpod'
- source_image: '{{user `UBUNTU_BASE_IMAGE`}}'
+ image_family: '{{build_name}}-cache'
+ source_image: '{{user `UBUNTU_BASE_IMAGE`}}' # precedence over family
+ source_image_family: 'ubuntu-base' # for ref. only
disk_size: 20 # REQUIRED: Runtime allocation > this value
project_id: '{{user `GCP_PROJECT_ID`}}'
service_account_email: '{{user `SERVICE_ACCOUNT`}}'
@@ -44,16 +46,24 @@ builders:
# v----- is a YAML alias, allows partial re-use of the anchor object
- <<: *gce_hosted_image
+ name: 'ubuntu-18'
+ source_image: '{{user `PRIOR_UBUNTU_BASE_IMAGE`}}'
+ source_image_family: 'prior-ubuntu-base'
+
+ - <<: *gce_hosted_image
name: 'fedora-30'
source_image: '{{user `FEDORA_BASE_IMAGE`}}'
+ source_image_family: 'fedora-base'
- <<: *gce_hosted_image
name: 'xfedora-30'
source_image: '{{user `FEDORA_BASE_IMAGE`}}'
+ source_image_family: 'fedora-base'
- <<: *gce_hosted_image
name: 'fedora-29'
source_image: '{{user `PRIOR_FEDORA_BASE_IMAGE`}}'
+ source_image_family: 'prior-fedora-base'
# The brains of the operation, making actual modifications to the base-image.
provisioners:
diff --git a/contrib/cirrus/packer/prior_fedora_base-setup.sh b/contrib/cirrus/packer/prior-fedora_base-setup.sh
index 998a5d9fd..998a5d9fd 120000
--- a/contrib/cirrus/packer/prior_fedora_base-setup.sh
+++ b/contrib/cirrus/packer/prior-fedora_base-setup.sh
diff --git a/contrib/cirrus/packer/systemd_banish.sh b/contrib/cirrus/packer/systemd_banish.sh
new file mode 100755
index 000000000..396cf906c
--- /dev/null
+++ b/contrib/cirrus/packer/systemd_banish.sh
@@ -0,0 +1,28 @@
+#!/bin/bash
+
+set +e # Not all of these exist on every platform
+
+# This is intended to be executed on VMs as a startup script on initial-boot.
+# Alternativly, it may be executed with the '--list' option to return the list
+# of systemd units defined for disablement (useful for testing).
+
+EVIL_UNITS="cron crond atd apt-daily-upgrade apt-daily fstrim motd-news systemd-tmpfiles-clean"
+
+if [[ "$1" == "--list" ]]
+then
+ echo "$EVIL_UNITS"
+ exit 0
+fi
+
+echo "Disabling periodic services that could destabilize testing:"
+for unit in $EVIL_UNITS
+do
+ echo "Banishing $unit (ignoring errors)"
+ (
+ sudo systemctl stop $unit
+ sudo systemctl disable $unit
+ sudo systemctl disable $unit.timer
+ sudo systemctl mask $unit
+ sudo systemctl mask $unit.timer
+ ) &> /dev/null
+done
diff --git a/contrib/cirrus/packer/ubuntu_setup.sh b/contrib/cirrus/packer/ubuntu_setup.sh
index 00d92570f..da7d457a5 100644
--- a/contrib/cirrus/packer/ubuntu_setup.sh
+++ b/contrib/cirrus/packer/ubuntu_setup.sh
@@ -20,18 +20,25 @@ systemd_banish
echo "Updating/configuring package repositories."
$LILTO $SUDOAPTGET update
+
+echo "Upgrading all packages"
+$BIGTO $SUDOAPTGET upgrade
+
+echo "Adding PPAs"
$LILTO $SUDOAPTGET install software-properties-common
-$LILTO $SUDOAPTADD ppa:longsleep/golang-backports
$LILTO $SUDOAPTADD ppa:projectatomic/ppa
$LILTO $SUDOAPTADD ppa:criu/ppa
+if [[ "$OS_RELEASE_VER" -eq "18" ]]
+then
+ $LILTO $SUDOAPTADD ppa:longsleep/golang-backports
+fi
-echo "Upgrading all packages"
$LILTO $SUDOAPTGET update
-$BIGTO $SUDOAPTGET upgrade
echo "Installing general testing and system dependencies"
$BIGTO $SUDOAPTGET install \
apparmor \
+ aufs-tools \
autoconf \
automake \
bats \
@@ -46,6 +53,7 @@ $BIGTO $SUDOAPTGET install \
e2fslibs-dev \
emacs-nox \
gawk \
+ gcc \
gettext \
go-md2man \
golang \
@@ -58,6 +66,7 @@ $BIGTO $SUDOAPTGET install \
libdevmapper-dev \
libdevmapper1.02.1 \
libfuse-dev \
+ libfuse2 \
libglib2.0-dev \
libgpgme11-dev \
liblzma-dev \
@@ -66,7 +75,7 @@ $BIGTO $SUDOAPTGET install \
libnl-3-dev \
libostree-dev \
libvarlink \
- libprotobuf-c0-dev \
+ libprotobuf-c-dev \
libprotobuf-dev \
libseccomp-dev \
libseccomp2 \
@@ -74,6 +83,7 @@ $BIGTO $SUDOAPTGET install \
libtool \
libudev-dev \
lsof \
+ make \
netcat \
pkg-config \
podman \
@@ -87,19 +97,28 @@ $BIGTO $SUDOAPTGET install \
python3-psutil \
python3-pytoml \
python3-setuptools \
- slirp4netns \
skopeo \
+ slirp4netns \
socat \
unzip \
vim \
xz-utils \
zip
-echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting."
-SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g'
-ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/*
-ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub
-ooe.sh sudo update-grub
+if [[ "$OS_RELEASE_VER" -ge "19" ]]
+then
+ echo "Installing Ubuntu > 18 packages"
+ $LILTO $SUDOAPTGET install fuse3 libfuse3-dev libbtrfs-dev
+fi
+
+if [[ "$OS_RELEASE_VER" -eq "18" ]]
+then
+ echo "Forced Ubuntu 18 kernel to enable cgroup swap accounting."
+ SEDCMD='s/^GRUB_CMDLINE_LINUX="(.*)"/GRUB_CMDLINE_LINUX="\1 cgroup_enable=memory swapaccount=1"/g'
+ ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub.d/*
+ ooe.sh sudo sed -re "$SEDCMD" -i /etc/default/grub
+ ooe.sh sudo update-grub
+fi
sudo /tmp/libpod/hack/install_catatonit.sh
ooe.sh sudo make -C /tmp/libpod install.libseccomp.sudo
diff --git a/contrib/cirrus/podbot.py b/contrib/cirrus/podbot.py
index 1be41a8ed..9ca4915a7 100755
--- a/contrib/cirrus/podbot.py
+++ b/contrib/cirrus/podbot.py
@@ -12,7 +12,7 @@ import sys
class IRC:
- response_timeout = 10 # seconds
+ response_timeout = 30 # seconds
irc = socket.socket()
def __init__(self, server, nickname, channel):
@@ -90,9 +90,16 @@ class IRC:
if len(sys.argv) < 3:
print("Error: Must pass desired nick and message as parameters")
else:
- irc = IRC("irc.freenode.net", sys.argv[1], "#podman")
- err = irc.connect(*os.environ.get('IRCID', 'Big Bug').split(" ", 2))
- if not err:
+ for try_again in (True,False):
+ irc = IRC("irc.freenode.net", sys.argv[1], "#podman")
+ err = irc.connect(*os.environ.get('IRCID', 'Big Bug').split(" ", 2))
+ if err and try_again:
+ print("Trying again in 5 seconds...")
+ time.sleep(5)
+ continue
+ elif err:
+ break
irc.message(" ".join(sys.argv[2:]))
time.sleep(5.0) # avoid join/quit spam
irc.quit()
+ break
diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh
index c1a01d6ab..df510deef 100755
--- a/contrib/cirrus/setup_environment.sh
+++ b/contrib/cirrus/setup_environment.sh
@@ -33,8 +33,8 @@ done
# contrib/cirrus/packer/*_setup.sh to be incorporated into VM cache-images
# (see docs).
cd "${GOSRC}/"
-case "${OS_REL_VER}" in
- ubuntu-18)
+case "${OS_RELEASE_ID}" in
+ ubuntu)
CRIO_RUNC_PATH="/usr/lib/cri-o-runc/sbin/runc"
if dpkg -L cri-o-runc | grep -m 1 -q "$CRIO_RUNC_PATH"
then
@@ -42,14 +42,13 @@ case "${OS_REL_VER}" in
ln -f "$CRIO_RUNC_PATH" "/usr/bin/runc"
fi
;;
- fedora-30) ;& # continue to next item
- fedora-29)
- # All SELinux distros need this for systemd-in-a-container
- setsebool container_manage_cgroup true
+ fedora)
+ # All SELinux distros need this for systemd-in-a-container
+ setsebool container_manage_cgroup true
if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then
bash "$SCRIPT_BASE/add_second_partition.sh"; fi
;;
- centos-7) # Current VM is an image-builder-image no local podman/testing
+ centos) # Current VM is an image-builder-image no local podman/testing
echo "No further setup required for VM image building"
exit 0
;;
diff --git a/contrib/cirrus/success.sh b/contrib/cirrus/success.sh
index 30d375d95..3b171757f 100755
--- a/contrib/cirrus/success.sh
+++ b/contrib/cirrus/success.sh
@@ -4,7 +4,7 @@ set -e
source $(dirname $0)/lib.sh
-req_env_var CIRRUS_BRANCH CIRRUS_BUILD_ID CIRRUS_REPO_FULL_NAME CIRRUS_BASE_SHA CIRRUS_CHANGE_IN_REPO
+req_env_var CIRRUS_BRANCH CIRRUS_REPO_FULL_NAME CIRRUS_BASE_SHA CIRRUS_CHANGE_IN_REPO CIRRUS_CHANGE_MESSAGE
cd $CIRRUS_WORKING_DIR
@@ -18,18 +18,21 @@ then
then
SHARANGE="${CIRRUS_BASE_SHA}..${CIRRUS_CHANGE_IN_REPO}"
EXCLUDE_RE='merge-robot'
+ EMAILCSET='[:alnum:]-+_@.'
AUTHOR_NICKS=$(egrep -v '(^[[:space:]]*$)|(^[[:space:]]*#)' "$AUTHOR_NICKS_FILEPATH" | sort -u)
# Depending on branch-state, it's possible SHARANGE could be _WAY_ too big
MAX_NICKS=10
# newline separated
GITLOG="git log --format='%ae'"
- COMMIT_AUTHORS=$($GITLOGt $SHARANGE || $GITLOG -1 HEAD | \
- sort -u | \
+ COMMIT_AUTHORS=$($GITLOG $SHARANGE || $GITLOG -1 HEAD | \
+ tr --delete --complement "$EMAILCSET[:space:]" | \
egrep -v "$EXCLUDE_RE" | \
+ sort -u | \
tail -$MAX_NICKS)
for c_email in $COMMIT_AUTHORS
do
+ c_email=$(echo "$c_email" | tr --delete --complement "$EMAILCSET")
echo -e "\tExamining $c_email"
NICK=$(echo "$AUTHOR_NICKS" | grep -m 1 "$c_email" | \
awk --field-separator ',' '{print $2}' | tr -d '[[:blank:]]')
@@ -40,8 +43,13 @@ then
echo -e "\t\tNot found in $(basename $AUTHOR_NICKS_FILEPATH), using e-mail username."
NICK=$(echo "$c_email" | cut -d '@' -f 1)
fi
- echo -e "\tUsing nick $NICK"
- NICKS="${NICKS:+$NICKS, }$NICK"
+ if ! echo "$NICKS" | grep -q "$NICK"
+ then
+ echo -e "\tUsing nick $NICK"
+ NICKS="${NICKS:+$NICKS, }$NICK"
+ else
+ echo -e "\tNot re-adding duplicate nick $NICK"
+ fi
done
fi
diff --git a/contrib/cirrus/upload_release_archive.sh b/contrib/cirrus/upload_release_archive.sh
index 942255821..25107f0ef 100755
--- a/contrib/cirrus/upload_release_archive.sh
+++ b/contrib/cirrus/upload_release_archive.sh
@@ -9,6 +9,7 @@ req_env_var CI UPLDREL_IMAGE CIRRUS_BUILD_ID GOSRC RELEASE_GCPJSON RELEASE_GCPNA
[[ "$CI" == "true" ]] || \
die 56 "$0 must be run under Cirrus-CI to function"
+# We store "releases" for each PR, mostly to validate the process is functional
unset PR_OR_BRANCH BUCKET
if [[ -n "$CIRRUS_PR" ]]
then
@@ -22,31 +23,76 @@ else
die 1 "Expecting either \$CIRRUS_PR or \$CIRRUS_BRANCH to be non-empty."
fi
-# Functional local podman required for uploading a release
+echo "Parsing actual_release.txt contents: $(< actual_release.txt)"
cd $GOSRC
+RELEASETXT=$(<actual_release.txt) # see build_release.sh
+[[ -n "$RELEASETXT" ]] || \
+ die 3 "Could not obtain metadata from actual_release.txt"
+RELEASE_INFO=$(echo "$RELEASETXT" | grep -m 1 'X-RELEASE-INFO:' | sed -r -e 's/X-RELEASE-INFO:\s*(.+)/\1/')
+if [[ "$?" -ne "0" ]] || [[ -z "$RELEASE_INFO" ]]
+then
+ die 4 "Metadata is empty or invalid: '$RELEASETXT'"
+fi
+# Format specified in Makefile
+# e.g. libpod v1.3.1-166-g60df124e fedora 29 amd64
+# or libpod-remote v1.3.1-166-g60df124e windows - amd64
+FIELDS="RELEASE_BASENAME RELEASE_VERSION RELEASE_DIST RELEASE_DIST_VER RELEASE_ARCH"
+read $FIELDS <<< $RELEASE_INFO
+req_env_var $FIELDS
+
+# Functional local podman required for uploading
+echo "Verifying a local, functional podman, building one if necessary."
[[ -n "$(type -P podman)" ]] || \
- make install || \
+ make install PREFIX=/usr || \
die 57 "$0 requires working podman binary on path to function"
TMPF=$(mktemp -p '' $(basename $0)_XXXX.json)
trap "rm -f $TMPF" EXIT
set +x
echo "$RELEASE_GCPJSON" > "$TMPF"
+[[ "$OS_RELEASE_ID" == "ubuntu" ]] || \
+ chcon -t container_file_t "$TMPF"
unset RELEASE_GCPJSON
cd $GOSRC
-for filename in $(ls -1 *.tar.gz *.zip)
+for filename in $(ls -1 *.tar.gz *.zip *.msi)
do
- echo "Running podman ... $UPLDREL_IMAGE $filename"
+ unset EXT
+ EXT=$(echo "$filename" | sed -r -e 's/.+\.(.+$)/\1/g')
+ if [[ -z "$EXT" ]] || [[ "$EXT" == "$filename" ]]
+ then
+ echo "Warning: Not processing $filename (invalid extension '$EXT')"
+ continue
+ fi
+
+ [[ "$OS_RELEASE_ID" == "ubuntu" ]] || \
+ chcon -t container_file_t "$filename"
+ # Form the generic "latest" file for this branch or pr
+ TO_PREFIX="${RELEASE_BASENAME}-latest-${PR_OR_BRANCH}-${RELEASE_DIST}"
+ # Form the fully-versioned filename for historical sake
+ ALSO_PREFIX="${RELEASE_BASENAME}-${RELEASE_VERSION}-${PR_OR_BRANCH}-${RELEASE_DIST}"
+ TO_SUFFIX="${RELEASE_ARCH}.${EXT}"
+ if [[ "$RELEASE_DIST" == "windows" ]] || [[ "$RELEASE_DIST" == "darwin" ]]
+ then
+ TO_FILENAME="${TO_PREFIX}-${TO_SUFFIX}"
+ ALSO_FILENAME="${ALSO_PREFIX}-${TO_SUFFIX}"
+ else
+ TO_FILENAME="${TO_PREFIX}-${RELEASE_DIST_VER}-${TO_SUFFIX}"
+ ALSO_FILENAME="${ALSO_PREFIX}-${TO_SUFFIX}"
+ fi
+
+ echo "Running podman ... $UPLDREL_IMAGE for $filename -> $TO_FILENAME"
+ echo "Warning: upload failures are completely ignored, avoiding any needless holdup of PRs."
podman run -i --rm \
-e "GCPNAME=$RELEASE_GCPNAME" \
-e "GCPPROJECT=$RELEASE_GCPROJECT" \
-e "GCPJSON_FILEPATH=$TMPF" \
- -e "REL_ARC_FILEPATH=/tmp/$filename" \
+ -e "FROM_FILEPATH=/tmp/$filename" \
+ -e "TO_FILENAME=$TO_FILENAME" \
+ -e "ALSO_FILENAME=$ALSO_FILENAME" \
-e "PR_OR_BRANCH=$PR_OR_BRANCH" \
-e "BUCKET=$BUCKET" \
- --security-opt label=disable \
-v "$TMPF:$TMPF:ro" \
- -v "$GOSRC/$filename:/tmp/$filename:ro" \
- $UPLDREL_IMAGE
+ -v "$(realpath $GOSRC/$filename):/tmp/$filename:ro" \
+ $UPLDREL_IMAGE || true
done
generated by cgit v1.2.3-54-g00ecf (git 2.39.0) at 2024-10-06 05:23:52 +0000