diff options
| -rw-r--r-- | .cirrus.yml | 209 | ||||
| -rw-r--r-- | .pre-commit-config.yaml | 1 | ||||
| -rwxr-xr-x | contrib/cirrus/pr-should-include-tests | 27 | ||||
| -rwxr-xr-x | contrib/cirrus/setup_environment.sh | 24 |
4 files changed, 129 insertions, 132 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 958c6b0c7..64b3afa06 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -51,6 +51,11 @@ env: VM_IMAGE_NAME: # One of the "Google-cloud VM Images" (above) CTR_FQIN: # One of the "Container FQIN's" (above) + # Curl-command prefix for downloading task artifacts, simply add the + # the url-encoded task name, artifact name, and path as a suffix. + ARTCURL: >- + curl --fail --location -O + --url https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID} # Default timeout for each task timeout_in: 60m @@ -96,7 +101,6 @@ ext_svc_check_task: _gc='git config --file /root/.gitconfig' $_gc user.email "TMcTestFace@example.com" $_gc user.name "Testy McTestface" - make install.tools setup_script: &setup '$GOSRC/$SCRIPT_BASE/setup_environment.sh' main_script: &main '/usr/bin/time --verbose --output="$STATS_LOGFILE" $GOSRC/$SCRIPT_BASE/runner.sh' @@ -123,14 +127,12 @@ automation_task: always: *runner_stats -# N/B: This task is critical. It builds all binaries and release archives -# for the project, using all primary OS platforms and versions. Assuming -# the builds are successful, a cache is stored of the entire `$GOPATH` -# contents. For all subsequent tasks, the _BUILD_CACHE_HANDLE value -# is used as a key to reuse this cache, saving both time and money. -# The only exceptions are tasks which only run inside a container, they -# will not have access the cache and therefore must rely on cloning the -# repository. +# N/B: This task is critical. It builds all binaries for all supported +# OS platforms and versions. On success, the contents of the repository +# are preserved as an artifact. This saves most subsequent tasks about +# 3 minutes of otherwise duplicative effort. It also ensures that the +# exact same binaries used throughout CI testing, are available for +# future consumption|inspection by the final 'artifacts' task. build_task: alias: 'build' name: 'Build for $DISTRO_NV' @@ -151,37 +153,27 @@ build_task: VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} CTR_FQIN: ${FEDORA_CONTAINER_FQIN} # ID for re-use of build output - _BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID} - env: &priorfedora_envvars DISTRO_NV: ${PRIOR_FEDORA_NAME} VM_IMAGE_NAME: ${PRIOR_FEDORA_CACHE_IMAGE_NAME} CTR_FQIN: ${PRIOR_FEDORA_CONTAINER_FQIN} - _BUILD_CACHE_HANDLE: ${PRIOR_FEDORA_NAME}-build-${CIRRUS_BUILD_ID} - env: &ubuntu_envvars DISTRO_NV: ${UBUNTU_NAME} VM_IMAGE_NAME: ${UBUNTU_CACHE_IMAGE_NAME} CTR_FQIN: ${UBUNTU_CONTAINER_FQIN} - _BUILD_CACHE_HANDLE: ${UBUNTU_NAME}-build-${CIRRUS_BUILD_ID} env: TEST_FLAVOR: build - # Ref: https://cirrus-ci.org/guide/writing-tasks/#cache-instruction - gopath_cache: &gopath_cache - folder: *gopath # Required hard-coded path, no variables. - fingerprint_script: echo "$_BUILD_CACHE_HANDLE" - # Cheat: Clone here when cache is empty, guaranteeing consistency. - populate_script: *full_clone - # A normal clone would invalidate useful cache - clone_script: &noop mkdir -p $CIRRUS_WORKING_DIR + clone_script: *full_clone setup_script: *setup main_script: *main - always: &binary_artifacts - <<: *runner_stats - gosrc_artifacts: - path: ./* # Grab everything in top-level $GOSRC - type: application/octet-stream - binary_artifacts: - path: ./bin/* - type: application/octet-stream + # Cirrus-CI is very slow uploading one file at time, and the repo contains + # thousands of files. Speed this up by archiving into tarball first. + repo_prep_script: &repo_prep >- + tar cjf /tmp/repo.tbz -C $GOSRC . && mv /tmp/repo.tbz $GOSRC/ + repo_artifacts: &repo_artifacts + path: ./repo.tbz + type: application/octet-stream + always: *runner_stats # Confirm the result of building on at least one platform appears sane. @@ -208,10 +200,12 @@ validate_task: env: <<: *stdenvars TEST_FLAVOR: validate - gopath_cache: &ro_gopath_cache - <<: *gopath_cache - reupload_on_changes: false - clone_script: *noop + # N/B: This script depends on ${DISTRO_NV} being defined for the task. + clone_script: &get_gosrc | + cd /tmp + echo "$ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tbz" + time $ARTCURL/Build%20for%20${DISTRO_NV}/repo/repo.tbz + time tar xjf /tmp/repo.tbz -C $GOSRC setup_script: *setup main_script: *main always: *runner_stats @@ -231,8 +225,7 @@ bindings_task: env: <<: *stdenvars TEST_FLAVOR: bindings - gopath_cache: *ro_gopath_cache - clone_script: *noop # Comes from cache + clone_script: *get_gosrc setup_script: *setup main_script: *main always: &logs_artifacts @@ -260,19 +253,18 @@ swagger_task: env: <<: *stdenvars TEST_FLAVOR: swagger - # TODO: Due to podman 3.0 activity (including new images), avoid - # disturbing the status-quo just to incorporate this one new - # container image. Uncomment line below when CI activities normalize. - #CTR_FQIN: 'quay.io/libpod/gcsupld:${IMAGE_SUFFIX}' - CTR_FQIN: 'quay.io/libpod/gcsupld:c4813063494828032' + CTR_FQIN: 'quay.io/libpod/gcsupld:${IMAGE_SUFFIX}' GCPJSON: ENCRYPTED[927dc01e755eaddb4242b0845cf86c9098d1e3dffac38c70aefb1487fd8b4fe6dd6ae627b3bffafaba70e2c63172664e] GCPNAME: ENCRYPTED[c145e9c16b6fb88d476944a454bf4c1ccc84bb4ecaca73bdd28bdacef0dfa7959ebc8171a27b2e4064d66093b2cdba49] GCPPROJECT: 'libpod-218412' - gopath_cache: *ro_gopath_cache - clone_script: *noop # Comes from cache + clone_script: *get_gosrc setup_script: *setup main_script: *main - always: *binary_artifacts + always: + <<: *runner_stats + swagger_artifacts: + path: ./swagger.yaml + type: text/plain # Check that all included go modules from other sources match @@ -291,7 +283,7 @@ consistency_task: TEST_FLAVOR: consistency TEST_ENVIRON: container CTR_FQIN: ${FEDORA_CONTAINER_FQIN} - clone_script: *full_clone # build-cache not available to container tasks + clone_script: *get_gosrc setup_script: *setup main_script: *main always: *runner_stats @@ -322,11 +314,14 @@ alt_build_task: ALT_NAME: 'Test build RPM' - env: ALT_NAME: 'Alt Arch. Cross' - gopath_cache: *ro_gopath_cache - clone_script: *noop # Comes from cache + # This task cannot make use of the shared repo.tbz artifact. + clone_script: *full_clone setup_script: *setup main_script: *main - always: *binary_artifacts + # Produce a new repo.tbz artifact for consumption by 'artifacts' task. + repo_prep_script: *repo_prep + repo_artifacts: *repo_artifacts + always: *runner_stats # Confirm building the remote client, natively on a Mac OS-X VM. @@ -348,10 +343,14 @@ osx_alt_build_task: - brew install go-md2man - go version build_amd64_script: - - make podman-remote-release-darwin_amd64.zip + - make podman-remote-release-darwin_amd64.zip GOARCH=amd64 build_arm64_script: - make podman-remote-release-darwin_arm64.zip GOARCH=arm64 - always: *binary_artifacts + # This task cannot make use of the shared repo.tbz artifact and must + # produce a new repo.tbz artifact for consumption by 'artifacts' task. + repo_prep_script: *repo_prep + repo_artifacts: *repo_artifacts + always: *runner_stats # Verify podman is compatible with the docker python-module. @@ -367,8 +366,7 @@ docker-py_test_task: <<: *stdenvars TEST_FLAVOR: docker-py TEST_ENVIRON: container - gopath_cache: *ro_gopath_cache - clone_script: *noop # Comes from cache + clone_script: *get_gosrc setup_script: *setup main_script: *main always: *runner_stats @@ -395,8 +393,7 @@ unit_test_task: gce_instance: *standardvm env: TEST_FLAVOR: unit - clone_script: *noop # Comes from cache - gopath_cache: *ro_gopath_cache + clone_script: *get_gosrc setup_script: *setup main_script: *main always: *logs_artifacts @@ -416,8 +413,7 @@ apiv2_test_task: env: <<: *stdenvars TEST_FLAVOR: apiv2 - clone_script: *noop # Comes from cache - gopath_cache: *ro_gopath_cache + clone_script: *get_gosrc setup_script: *setup main_script: *main always: *logs_artifacts @@ -431,8 +427,6 @@ compose_test_task: depends_on: - validate gce_instance: *standardvm - env: - <<: *stdenvars matrix: - env: TEST_FLAVOR: compose @@ -446,8 +440,9 @@ compose_test_task: - env: TEST_FLAVOR: compose_v2 PRIV_NAME: rootless - clone_script: *noop # Comes from cache - gopath_cache: *ro_gopath_cache + env: + <<: *stdenvars + clone_script: *get_gosrc setup_script: *setup main_script: *main always: *logs_artifacts @@ -469,8 +464,7 @@ local_integration_test_task: &local_integration_test_task timeout_in: 90m env: TEST_FLAVOR: int - clone_script: *noop # Comes from cache - gopath_cache: *ro_gopath_cache + clone_script: *get_gosrc setup_script: *setup main_script: *main always: &int_logs_artifacts @@ -516,8 +510,7 @@ container_integration_test_task: env: TEST_FLAVOR: int TEST_ENVIRON: container - clone_script: *noop # Comes from cache - gopath_cache: *ro_gopath_cache + clone_script: *get_gosrc setup_script: *setup main_script: *main always: *int_logs_artifacts @@ -537,8 +530,7 @@ rootless_integration_test_task: env: TEST_FLAVOR: int PRIV_NAME: rootless - clone_script: *noop # Comes from cache - gopath_cache: *ro_gopath_cache + clone_script: *get_gosrc setup_script: *setup main_script: *main always: *int_logs_artifacts @@ -559,8 +551,7 @@ local_system_test_task: &local_system_test_task gce_instance: *standardvm env: TEST_FLAVOR: sys - clone_script: *noop # Comes from cache - gopath_cache: *ro_gopath_cache + clone_script: *get_gosrc setup_script: *setup main_script: *main always: *logs_artifacts @@ -619,8 +610,7 @@ buildah_bud_test_task: PODBIN_NAME: remote gce_instance: *standardvm timeout_in: 45m - clone_script: *noop - gopath_cache: *ro_gopath_cache + clone_script: *get_gosrc setup_script: *setup main_script: *main always: *int_logs_artifacts @@ -638,8 +628,7 @@ rootless_system_test_task: env: TEST_FLAVOR: sys PRIV_NAME: rootless - clone_script: *noop # Comes from cache - gopath_cache: *ro_gopath_cache + clone_script: *get_gosrc setup_script: *setup main_script: *main always: *logs_artifacts @@ -661,8 +650,7 @@ rootless_gitlab_test_task: <<: *ubuntu_envvars TEST_FLAVOR: 'gitlab' PRIV_NAME: rootless - clone_script: *noop # Comes from cache - gopath_cache: *ro_gopath_cache + clone_script: *get_gosrc setup_script: *setup main_script: *main always: @@ -694,8 +682,7 @@ upgrade_test_task: VM_IMAGE_NAME: ${FEDORA_CACHE_IMAGE_NAME} # ID for re-use of build output _BUILD_CACHE_HANDLE: ${FEDORA_NAME}-build-${CIRRUS_BUILD_ID} - clone_script: *noop - gopath_cache: *ro_gopath_cache + clone_script: *get_gosrc setup_script: *setup main_script: *main always: *logs_artifacts @@ -715,11 +702,6 @@ image_build_task: &image-build image_name: build-push-${IMAGE_SUFFIX} # More muscle required for parallel multi-arch build type: "n2-standard-4" - env: - PODMAN_USERNAME: ENCRYPTED[b9f0f2550029dd2196e086d9dd6c2d1fec7e328630b15990d9bb610f9fcccb5baab8b64a8c3e72b0c1d0f5917cf65aa1] - PODMAN_PASSWORD: ENCRYPTED[e3444f6072853f0c8db7f964ead5e2204116af485469fa0de367f26b9316b460fd842a9882f552b9e9a83bbaf650d8b4] - CONTAINERS_USERNAME: ENCRYPTED[54a372d5f22f424173c114c6fb25c3214956cad323d5b285c7393a71041884ce96471d0ff733774e5dab9fa5a3c8795c] - CONTAINERS_PASSWORD: ENCRYPTED[4ecc3fb534935095a99fb1f2e320ac6bc87f3e7e186746e41cbcc4b5f5379a014b9fc8cc90e1f3d5abdbaf31580a4ab9] matrix: - env: CTXDIR: contrib/podmanimage/upstream @@ -729,6 +711,12 @@ image_build_task: &image-build CTXDIR: contrib/podmanimage/stable - env: CTXDIR: contrib/hello + env: + PODMAN_USERNAME: ENCRYPTED[b9f0f2550029dd2196e086d9dd6c2d1fec7e328630b15990d9bb610f9fcccb5baab8b64a8c3e72b0c1d0f5917cf65aa1] + PODMAN_PASSWORD: ENCRYPTED[e3444f6072853f0c8db7f964ead5e2204116af485469fa0de367f26b9316b460fd842a9882f552b9e9a83bbaf650d8b4] + CONTAINERS_USERNAME: ENCRYPTED[54a372d5f22f424173c114c6fb25c3214956cad323d5b285c7393a71041884ce96471d0ff733774e5dab9fa5a3c8795c] + CONTAINERS_PASSWORD: ENCRYPTED[4ecc3fb534935095a99fb1f2e320ac6bc87f3e7e186746e41cbcc4b5f5379a014b9fc8cc90e1f3d5abdbaf31580a4ab9] + clone_script: &noop mkdir -p $CIRRUS_WORKING_DIR script: - set -a; source /etc/automation_environment; set +a - main.sh $CIRRUS_REPO_CLONE_URL $CTXDIR @@ -828,42 +816,41 @@ artifacts_task: env: CTR_FQIN: ${FEDORA_CONTAINER_FQIN} TEST_ENVIRON: container - CURL: "curl --fail --location -O https://api.cirrus-ci.com/v1/artifact/build/${CIRRUS_BUILD_ID}" # In order to keep the download URL and Cirrus-CI artifact.zip contents # simple, nothing should exist in $CIRRUS_WORKING_DIR except for artifacts. clone_script: *noop - script: - # Assume the latest Fedora release build is most useful - - $CURL/Build%20for%20$FEDORA_NAME/binary/bin/podman - - $CURL/Build%20for%20$FEDORA_NAME/binary/bin/podman-remote - - $CURL/Build%20for%20$FEDORA_NAME/binary/bin/rootlessport - - chmod +x podman* rootlessport - # Architecture in filename & can't use wildcards in a URL + fedora_binaries_script: + - mkdir -p /tmp/fed + - cd /tmp/fed + - $ARTCURL/Build%20for%20${FEDORA_NAME}/repo/repo.tbz + - tar xjf repo.tbz + - cp ./bin/* $CIRRUS_WORKING_DIR/ + alt_binaries_script: - mkdir -p /tmp/alt - cd /tmp/alt - - $CURL/Alt%20Arch.%20Cross/gosrc.zip - - unzip gosrc.zip - - cd $CIRRUS_WORKING_DIR - - mv /tmp/alt/*.tar.gz ./ - # Windows MSI filename has version number + - $ARTCURL/Alt%20Arch.%20Cross/repo/repo.tbz + - tar xjf repo.tbz + - mv ./*.tar.gz $CIRRUS_WORKING_DIR/ + win_binaries_script: - mkdir -p /tmp/win - cd /tmp/win - - $CURL/Windows%20Cross/gosrc.zip - - unzip gosrc.zip - - cd $CIRRUS_WORKING_DIR - - mv /tmp/win/podman-remote*.zip /tmp/win/*.msi ./ - # OSX - - $CURL/OSX%20Cross/gosrc/podman-remote-release-darwin_amd64.zip - - $CURL/OSX%20Cross/gosrc/podman-remote-release-darwin_arm64.zip - # Always show contents to assist in debugging + - $ARTCURL/Windows%20Cross/repo/repo.tbz + - tar xjf repo.tbz + - mv ./podman-remote*.zip ./*.msi $CIRRUS_WORKING_DIR/ + osx_binaries_script: + - mkdir -p /tmp/osx + - cd /tmp/osx + - $ARTCURL/OSX%20Cross/repo/repo.tbz + - tar xjf repo.tbz + - mv ./podman-remote-release-darwin_*.zip $CIRRUS_WORKING_DIR/ always: - contents_script: ls -1 $CIRRUS_WORKING_DIR - # Produce downloadable files and an automatic zip-file accessible - # by a consistent URL, based on contents of $CIRRUS_WORKING_DIR - # Ref: https://cirrus-ci.org/guide/writing-tasks/#latest-build-artifacts - binary_artifacts: - path: ./* - type: application/octet-stream + contents_script: ls -la $CIRRUS_WORKING_DIR + # Produce downloadable files and an automatic zip-file accessible + # by a consistent URL, based on contents of $CIRRUS_WORKING_DIR + # Ref: https://cirrus-ci.org/guide/writing-tasks/#latest-build-artifacts + binary_artifacts: + path: ./* + type: application/octet-stream # When a new tag is pushed, confirm that the code and commits @@ -878,11 +865,9 @@ release_task: env: <<: *stdenvars TEST_FLAVOR: release - gopath_cache: *ro_gopath_cache - clone_script: *noop # Comes from cache + clone_script: *get_gosrc setup_script: *setup main_script: *main - always: *binary_artifacts # When preparing to release a new version, this task may be manually @@ -905,8 +890,6 @@ release_test_task: env: <<: *stdenvars TEST_FLAVOR: release - gopath_cache: *ro_gopath_cache - clone_script: *noop # Comes from cache + clone_script: *get_gosrc setup_script: *setup main_script: *main - always: *binary_artifacts diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3416a93ee..ca023d18a 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -17,3 +17,4 @@ repos: - id: check-byte-order-marker - id: check-executables-have-shebangs - id: check-merge-conflict + - id: check-yaml diff --git a/contrib/cirrus/pr-should-include-tests b/contrib/cirrus/pr-should-include-tests index 0d39047a6..57ca39d9b 100755 --- a/contrib/cirrus/pr-should-include-tests +++ b/contrib/cirrus/pr-should-include-tests @@ -30,19 +30,20 @@ fi # Nothing changed under test subdirectory. # # This is OK if the only files being touched are "safe" ones. -filtered_changes=$(git diff --name-only $base $head | - fgrep -vx .cirrus.yml | - fgrep -vx .gitignore | - fgrep -vx Makefile | - fgrep -vx go.mod | - fgrep -vx go.sum | - egrep -v '^[^/]+\.md$' | - egrep -v '^.github' | - egrep -v '^contrib/' | - egrep -v '^docs/' | - egrep -v '^hack/' | - egrep -v '^nix/' | - egrep -v '^vendor/' | +filtered_changes=$(git diff --name-only $base $head | + fgrep -vx .cirrus.yml | + fgrep -vx .pre-commit-config.yaml | + fgrep -vx .gitignore | + fgrep -vx Makefile | + fgrep -vx go.mod | + fgrep -vx go.sum | + egrep -v '^[^/]+\.md$' | + egrep -v '^.github' | + egrep -v '^contrib/' | + egrep -v '^docs/' | + egrep -v '^hack/' | + egrep -v '^nix/' | + egrep -v '^vendor/' | egrep -v '^version/') if [[ -z "$filtered_changes" ]]; then exit 0 diff --git a/contrib/cirrus/setup_environment.sh b/contrib/cirrus/setup_environment.sh index e3eb46783..34a276f10 100755 --- a/contrib/cirrus/setup_environment.sh +++ b/contrib/cirrus/setup_environment.sh @@ -217,6 +217,7 @@ case "$TEST_FLAVOR" in validate) dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm # For some reason, this is also needed for validation + make install.tools make .install.pre-commit ;; automation) ;; @@ -226,10 +227,12 @@ case "$TEST_FLAVOR" in if [[ "$ALT_NAME" =~ RPM ]]; then bigto dnf install -y glibc-minimal-langpack go-rpm-macros rpkg rpm-build shadow-utils-subid-devel fi + make install.tools ;; docker-py) remove_packaged_podman_files - make && make install PREFIX=/usr ETCDIR=/etc + make install.tools + make install PREFIX=/usr ETCDIR=/etc msg "Installing previously downloaded/cached packages" dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm @@ -239,13 +242,17 @@ case "$TEST_FLAVOR" in pip install --requirement $GOSRC/test/python/requirements.txt ;; build) make clean ;; - unit) ;; + unit) + make install.tools + ;; compose_v2) + make install.tools dnf -y remove docker-compose curl -SL https://github.com/docker/compose/releases/download/v2.2.3/docker-compose-linux-x86_64 -o /usr/local/bin/docker-compose chmod +x /usr/local/bin/docker-compose ;& # Continue with next item apiv2) + make install.tools msg "Installing previously downloaded/cached packages" dnf install -y $PACKAGE_DOWNLOAD_DIR/python3*.rpm virtualenv .venv/requests @@ -254,6 +261,7 @@ case "$TEST_FLAVOR" in pip install --requirement $GOSRC/test/apiv2/python/requirements.txt ;& # continue with next item compose) + make install.tools rpm -ivh $PACKAGE_DOWNLOAD_DIR/podman-docker* ;& # continue with next item int) ;& @@ -262,6 +270,7 @@ case "$TEST_FLAVOR" in bud) ;& bindings) ;& endpoint) + make install.tools # Use existing host bits when testing is to happen inside a container # since this script will run again in that environment. # shellcheck disable=SC2154 @@ -270,11 +279,11 @@ case "$TEST_FLAVOR" in die "Refusing to config. host-test in container"; fi remove_packaged_podman_files - make && make install PREFIX=/usr ETCDIR=/etc + make install PREFIX=/usr ETCDIR=/etc elif [[ "$TEST_ENVIRON" == "container" ]]; then if ((CONTAINER)); then remove_packaged_podman_files - make && make install PREFIX=/usr ETCDIR=/etc + make install PREFIX=/usr ETCDIR=/etc fi else die "Invalid value for \$TEST_ENVIRON=$TEST_ENVIRON" @@ -291,7 +300,7 @@ case "$TEST_FLAVOR" in # Ref: https://gitlab.com/gitlab-org/gitlab-runner/-/issues/27270#note_499585550 remove_packaged_podman_files - make && make install PREFIX=/usr ETCDIR=/etc + make install PREFIX=/usr ETCDIR=/etc msg "Installing docker and containerd" # N/B: Tests check/expect `docker info` output, and this `!= podman info` @@ -324,7 +333,10 @@ case "$TEST_FLAVOR" in docker.io/gitlab/gitlab-runner-helper:x86_64-latest-pwsh ;; swagger) ;& # use next item - consistency) make clean ;; + consistency) + make clean + make install.tools + ;; release) ;; *) die_unknown TEST_FLAVOR esac |