From abf4f77bc76b456f6f4c2ecbfc4db4dcb2523ee2 Mon Sep 17 00:00:00 2001 From: Chris Evich Date: Mon, 25 Apr 2022 14:45:40 -0400 Subject: Cirrus: Simplify use of cache in automation With the increasingly complex `.cirrus.yml` task relationships, build cache wasn't always working as intended. Recently, non-build tasks were observed assuming authority over `gopath_cache`. Ref.: https://github.com/containers/podman/pull/13998#issuecomment-1108834538 Address this by an overall simplification using artifacts instead of cache. Using artifacts allows establishing concrete authorship/authority over cached repo. content. In this way, dependent tasks may simply consume the artifact with `curl` instead of relying on complex caching algorithms. Also/Minor: Add YAML checking to the pre-commit configuration. Signed-off-by: Chris Evich --- contrib/cirrus/pr-should-include-tests | 27 ++++++++++++++------------- contrib/cirrus/setup_environment.sh | 24 ++++++++++++++++++------ 2 files changed, 32 insertions(+), 19 deletions(-) (limited to 'contrib/cirrus') diff --git a/contrib/cirrus/pr-should-include-tests b/contrib/cirrus/pr-should-include-tests index 0d39047a6..57ca39d9b 100755 --- a/contrib/cirrus/pr-should-include-tests +++ b/contrib/cirrus/pr-should-include-tests @@ -30,19 +30,20 @@ fi # Nothing changed under test subdirectory. # # This is OK if the only files being touched are "safe" ones. -filtered_changes=$(git diff --name-only $base $head | - fgrep -vx .cirrus.yml | - fgrep -vx .gitignore | - fgrep -vx Makefile | - fgrep -vx go.mod | - fgrep -vx go.sum | - egrep -v '^[^/]+\.md$' | - egrep -v '^.github' | - egrep -v '^contrib/' | - egrep -v '^docs/' | - egrep -v '^hack/' | - egrep -v '^nix/' | - egrep -v '^vendor/' | +filtered_changes=$(git diff --name-only $base $head | + fgrep -vx .cirrus.yml | + fgrep -vx .pre-commit-config.yaml | + fgrep -vx .gitignore | + fgrep -vx Makefile | + fgrep -vx go.mod | + fgrep -vx go.sum | + egrep -v '^[^/]+\.md$' | + egrep -v '^.github' | + egrep -v '^contrib/' | + egrep -v '^docs/' | + egrep -v '^hack/' | + egrep -v '^nix/' | + egrep -v '^vendor/' | egrep -v '^version/') if [[ -z "$filtered_changes" ]]; then exit 0 diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index e3eb46783..34a276f10 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -217,6 +217,7 @@ case "$TEST_FLAVOR" in validate) dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm # For some reason, this is also needed for validation + make install.tools make .install.pre-commit ;; automation) ;; @@ -226,10 +227,12 @@ case "$TEST_FLAVOR" in if [[ "$ALT_NAME" =~ RPM ]]; then bigto dnf install -y glibc-minimal-langpack go-rpm-macros rpkg rpm-build shadow-utils-subid-devel fi + make install.tools ;; docker-py) remove_packaged_podman_files - make && make install PREFIX=/usr ETCDIR=/etc + make install.tools + make install PREFIX=/usr ETCDIR=/etc msg "Installing previously downloaded/cached packages" dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm @@ -239,13 +242,17 @@ case "$TEST_FLAVOR" in pip install --requirement $GOSRC/test/python/requirements.txt ;; build) make clean ;; - unit) ;; + unit) + make install.tools + ;; compose_v2) + make install.tools dnf -y remove docker-compose curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose ;& # Continue with next item apiv2) + make install.tools msg "Installing previously downloaded/cached packages" dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm virtualenv .venv/requests @@ -254,6 +261,7 @@ case "$TEST_FLAVOR" in pip install --requirement $GOSRC/test/apiv2/python/requirements.txt ;& # continue with next item compose) + make install.tools rpm -ivh $PACKAGE_DOWNLOAD_DIR/podman-docker* ;& # continue with next item int) ;& @@ -262,6 +270,7 @@ case "$TEST_FLAVOR" in bud) ;& bindings) ;& endpoint) + make install.tools # Use existing host bits when testing is to happen inside a container # since this script will run again in that environment. # shellcheck disable=SC2154 @@ -270,11 +279,11 @@ case "$TEST_FLAVOR" in die "Refusing to config. host-test in container"; fi remove_packaged_podman_files - make && make install PREFIX=/usr ETCDIR=/etc + make install PREFIX=/usr ETCDIR=/etc elif [[ "$TEST_ENVIRON" == "container" ]]; then if ((CONTAINER)); then remove_packaged_podman_files - make && make install PREFIX=/usr ETCDIR=/etc + make install PREFIX=/usr ETCDIR=/etc fi else die "Invalid value for \$TEST_ENVIRON=$TEST_ENVIRON" @@ -291,7 +300,7 @@ case "$TEST_FLAVOR" in # Ref: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27270#note_499585550 remove_packaged_podman_files - make && make install PREFIX=/usr ETCDIR=/etc + make install PREFIX=/usr ETCDIR=/etc msg "Installing docker and containerd" # N/B: Tests check/expect `docker info` output, and this `!= podman info` @@ -324,7 +333,10 @@ case "$TEST_FLAVOR" in docker.io/gitlab/gitlab-runner-helper:x86_64-latest-pwsh ;; swagger) ;& # use next item - consistency) make clean ;; + consistency) + make clean + make install.tools + ;; release) ;; *) die_unknown TEST_FLAVOR esac -- cgit v1.2.3-54-g00ecf