diff options
Diffstat (limited to 'contrib')
| -rw-r--r-- | contrib/cirrus/99-do-not-use-google-subnets.conflist | 21 | ||||
| -rw-r--r-- | contrib/cirrus/lib.sh | 16 | ||||
| -rwxr-xr-x | contrib/cirrus/setup_environment.sh | 11 | ||||
| -rwxr-xr-x | contrib/imgprune/entrypoint.sh | 63 |
4 files changed, 87 insertions, 24 deletions
diff --git a/contrib/cirrus/99-do-not-use-google-subnets.conflist b/contrib/cirrus/99-do-not-use-google-subnets.conflist new file mode 100644 index 000000000..e9ab638ed --- /dev/null +++ b/contrib/cirrus/99-do-not-use-google-subnets.conflist @@ -0,0 +1,21 @@ +{ + "cniVersion": "0.4.0", + "name": "do-not-use-google-subnets", + "plugins": [ + { + "type": "bridge", + "name": "do-not-use-google-subnets", + "bridge": "do-not-use-google-subnets", + "ipam": { + "type": "host-local", + "ranges": [ + [ + { + "subnet": "10.128.0.0/9" + } + ] + ] + } + } + ] +} diff --git a/contrib/cirrus/lib.sh b/contrib/cirrus/lib.sh index 555f3e717..f66e63140 100644 --- a/contrib/cirrus/lib.sh +++ b/contrib/cirrus/lib.sh @@ -323,13 +323,15 @@ EOF install_test_configs(){ echo "Installing cni config, policy and registry config" - req_env_var GOSRC - sudo install -D -m 755 $GOSRC/cni/87-podman-bridge.conflist \ - /etc/cni/net.d/87-podman-bridge.conflist - sudo install -D -m 755 $GOSRC/test/policy.json \ - /etc/containers/policy.json - sudo install -D -m 755 $GOSRC/test/registries.conf \ - /etc/containers/registries.conf + req_env_var GOSRC SCRIPT_BASE + cd $GOSRC + install -v -D -m 644 ./cni/87-podman-bridge.conflist /etc/cni/net.d/ + # This config must always sort last in the list of networks (podman picks first one + # as the default). This config prevents allocation of network address space used + # by default in google cloud. https://cloud.google.com/vpc/docs/vpc#ip-ranges + install -v -D -m 644 $SCRIPT_BASE/99-do-not-use-google-subnets.conflist /etc/cni/net.d/ + install -v -D -m 644 ./test/policy.json /etc/containers/ + install -v -D -m 644 ./test/registries.conf /etc/containers/ } # Remove all files (except conmon, for now) provided by the distro version of podman. diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index 323a05489..7c7659169 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -44,11 +44,15 @@ case "${OS_REL_VER}" in ;; fedora-30) ;& # continue to next item fedora-29) + # All SELinux distros need this for systemd-in-a-container + setsebool container_manage_cgroup true if [[ "$ADD_SECOND_PARTITION" == "true" ]]; then bash "$SCRIPT_BASE/add_second_partition.sh"; fi ;; centos-7) # Current VM is an image-builder-image no local podman/testing - echo "No further setup required for VM image building" + echo "No further setup required for VM image building" + # All SELinux distros need this for systemd-in-a-container + setsebool container_manage_cgroup true exit 0 ;; *) bad_os_id_ver ;; @@ -57,8 +61,7 @@ esac # Reload to incorporate any changes from above source "$SCRIPT_BASE/lib.sh" -install_test_configs - +# Must execute before possible setup_rootless() make install.tools case "$SPECIALMODE" in @@ -92,3 +95,5 @@ case "$SPECIALMODE" in *) die 111 "Unsupported \$SPECIALMODE: $SPECIALMODE" esac + +install_test_configs diff --git a/contrib/imgprune/entrypoint.sh b/contrib/imgprune/entrypoint.sh index a4b77523b..829e9938e 100755 --- a/contrib/imgprune/entrypoint.sh +++ b/contrib/imgprune/entrypoint.sh @@ -6,27 +6,49 @@ source /usr/local/bin/lib_entrypoint.sh req_env_var GCPJSON GCPNAME GCPPROJECT IMGNAMES +BASE_IMAGES="" +# When executing under Cirrus-CI, have access to current source +if [[ "$CI" == "true" ]] && [[ -r "$CIRRUS_WORKING_DIR/$SCRIPT_BASE" ]] +then + # Avoid importing anything that might conflict + eval "$(egrep -sh '^export .+BASE_IMAGE=' < $CIRRUS_WORKING_DIR/$SCRIPT_BASE/lib.sh)" + BASE_IMAGES="$UBUNTU_BASE_IMAGE $PRIOR_UBUNTU_BASE_IMAGE $FEDORA_BASE_IMAGE $PRIOR_FEDORA_BASE_IMAGE" +else + # metadata labeling may have broken for some reason in the future + echo "Warning: Running outside of Cirrus-CI, very minor-risk of base-image deletion." +fi + gcloud_init # For safety's sake + limit nr background processes -PRUNE_LIMIT=10 +PRUNE_LIMIT=5 THEFUTURE=$(date --date='+1 hour' +%s) -TOO_OLD='90 days ago' +TOO_OLD='30 days ago' THRESHOLD=$(date --date="$TOO_OLD" +%s) # Format Ref: https://cloud.google.com/sdk/gcloud/reference/topic/formats FORMAT='value[quote](name,selfLink,creationTimestamp,labels)' PROJRE="/v1/projects/$GCPPROJECT/global/" -BASE_IMAGE_RE='cloud-base' -RECENTLY=$(date --date='30 days ago' --iso-8601=date) -EXCLUDE="$IMGNAMES $IMAGE_BUILDER_CACHE_IMAGE_NAME" # whitespace separated values +RECENTLY=$(date --date='3 days ago' --iso-8601=date) # Filter Ref: https://cloud.google.com/sdk/gcloud/reference/topic/filters -FILTER="selfLink~$PROJRE AND creationTimestamp<$RECENTLY AND NOT name=($EXCLUDE)" +FILTER="selfLink~$PROJRE AND creationTimestamp<$RECENTLY AND NOT name=($IMGNAMES $BASE_IMAGES)" TODELETE=$(mktemp -p '' todelete.XXXXXX) +IMGCOUNT=$(mktemp -p '' imgcount.XXXXXX) + +# Search-loop runs in a sub-process, must store count in file +echo "0" > "$IMGCOUNT" +count_image() { + local count + count=$(<"$IMGCOUNT") + let 'count+=1' + echo "$count" > "$IMGCOUNT" +} -echo "Searching images for pruning candidates older than $TOO_OLD ($THRESHOLD):" +echo "Using filter: $FILTER" +echo "Searching images for pruning candidates older than $TOO_OLD ($(date --date="$TOO_OLD" --iso-8601=date)):" $GCLOUD compute images list --format="$FORMAT" --filter="$FILTER" | \ while read name selfLink creationTimestamp labels do + count_image created_ymd=$(date --date=$creationTimestamp --iso-8601=date) last_used=$(egrep --only-matching --max-count=1 'last-used=[[:digit:]]+' <<< $labels || true) markmsgpfx="Marking $name (created $created_ymd) for deletion" @@ -52,16 +74,29 @@ $GCLOUD compute images list --format="$FORMAT" --filter="$FILTER" | \ echo "$name" >> $TODELETE continue fi - - echo "NOT $markmsgpfx: last used on $last_used_ymd)" done -echo "Pruning up to $PRUNE_LIMIT images that were marked for deletion:" -for image_name in $(tail -$PRUNE_LIMIT $TODELETE | sort --random-sort) +COUNT=$(<"$IMGCOUNT") +echo "########################################################################" +echo "Deleting up to $PRUNE_LIMIT images marked ($(wc -l < $TODELETE)) of all searched ($COUNT):" + +# Require a minimum number of images to exist +NEED="$[$PRUNE_LIMIT*2]" +if [[ "$COUNT" -lt "$NEED" ]] +then + die 0 Safety-net Insufficient images \($COUNT\) to process deletions \($NEED\) + exit 0 +fi + +for image_name in $(sort --random-sort $TODELETE | tail -$PRUNE_LIMIT) do - # This can take quite some time (minutes), run in parallel disconnected from terminal - echo "TODO: Would have: $GCLOUD compute images delete $image_name &" - sleep "$[1+RANDOM/1000]s" & # Simlate background operation + if echo "$IMGNAMES $BASE_IMAGES" | grep -q "$image_name" + then + # double-verify in-use images were filtered out in search loop above + die 8 FATAL ATTEMPT TO DELETE IN-USE IMAGE \'$image_name\' - THIS SHOULD NEVER HAPPEN + fi + echo "Deleting $image_name in parallel..." + $GCLOUD compute images delete $image_name & done wait || true # Nothing to delete: No background jobs |